# Rank Pilot AI SEO — robots.txt
# Public marketing pages are crawlable; authenticated app routes are blocked.

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

# AI / LLM crawlers — explicitly allowed for GEO/AEO visibility
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: *
Allow: /

# Block admin/system folders (optional but recommended)
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/

# Allow Google to crawl JS, CSS, images
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.svg$
Allow: /*.webp$

# Block authenticated / private app routes
Disallow: /dashboard
Disallow: /admin
Disallow: /account
Disallow: /billing
Disallow: /website/
Disallow: /websites/
Disallow: /autopilot
Disallow: /reports
Disallow: /analytics
Disallow: /competitors
Disallow: /backlink-builder
Disallow: /rank-tracker
Disallow: /serp-checker
Disallow: /seo-check
Disallow: /seo-plan-creator
Disallow: /self-seo
Disallow: /site-audit
Disallow: /speed-test
Disallow: /blog-writer
Disallow: /ai-tools
Disallow: /directory-listings
Disallow: /ga-tracking
Disallow: /connectivity
Disallow: /queue-management
Disallow: /managed-services
Disallow: /agency
Disallow: /setup-guide
Disallow: /onpage-editor
Disallow: /challenge-dashboard
Disallow: /challenge-setup
Disallow: /challenge-certificate
Disallow: /daily-task
Disallow: /accept-invite
Disallow: /checkout-return
Disallow: /managed-services-return
Disallow: /gsc-callback
Disallow: /reset-password
Disallow: /unsubscribe
Disallow: /welcome
Disallow: /download-app

Sitemap: https://www.rankpilotaiseo.co.uk/sitemap.xml