User-agent: Googlebot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

User-agent: Bingbot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

User-agent: Twitterbot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

User-agent: facebookexternalhit
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# OpenAI - ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# xAI - Grok
User-agent: xAIBot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# Anthropic - Claude
User-agent: ClaudeBot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# GitHub Copilot (by Microsoft)
User-agent: GitHub-Copilot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# Google Gemini
User-agent: Google-Extended
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# Common AI content crawlers
User-agent: CCBot
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

# Fallback rule for all others
User-agent: *
Allow: /
Disallow: /terms-of-use
Disallow: /privacy-policy

Sitemap: https://leadshuttle.com/sitemap.xml

# Crawl-delay to be respectful to servers
Crawl-delay: 1