# curl.ly robots.txt - SEO Optimized
# Last updated: 2026-03-27

# Default rules for all bots
User-agent: *
Allow: /
Allow: /features
Allow: /features/
Allow: /pricing
Allow: /about
Allow: /contact
Allow: /use-cases
Allow: /resources
Allow: /privacy
Allow: /terms
Allow: /cookies
Allow: /data-rights

# Protected areas - disallow indexing
Disallow: /app/
Disallow: /admin/
Disallow: /r/
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /accept-invite
Disallow: /auth-callback
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /api/
Disallow: /private/
Disallow: /temp/

# Googlebot - Full access to allowed pages
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Social media bots - allow for previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# Sitemaps - All sitemaps for comprehensive indexing
Sitemap: https://curl.ly/sitemap.xml
Sitemap: https://ldxlhzrfreebrospyvnw.supabase.co/functions/v1/sitemap

# Host directive
Host: https://curl.ly