# www.robotstxt.org

User-agent: *
# Account and editor pages send X-Robots-Tag: noindex. Allow crawling so search engines can read it.
# Allow all crawlers
Allow: /

# Keep legacy honeypot URLs out of search results.
Disallow: /lists/hp-

# Disallow specific private routes
Disallow: /auth/
Disallow: /auth
Disallow: /api/
Disallow: /api

# ...except the OG image endpoint. Every page advertises one of these in its
# og:image meta, so crawlers find them whatever robots.txt says — and Search
# Console was filing them as PAGES under "Crawled - currently not indexed".
#
# Blocking is the wrong tool here: a disallowed URL can still be indexed from
# links alone, and blocking guarantees the crawler never reads the
# `X-Robots-Tag: noindex, noimageindex` the endpoint now returns. Allowing the
# fetch is what lets that directive be seen and the URLs drop out cleanly.
# Longest-match wins in robots.txt, so this beats the two Disallow lines above.
Allow: /api/og

# Allow important public pages
Allow: /explore
Allow: /trending
Allow: /recommended
Allow: /leaderboard
Allow: /groups
Allow: /about
Allow: /helpfaqpage
Allow: /contactpage
Allow: /privacypolicy
Allow: /termsandconditions
Allow: /cookiepolicy
Allow: /lists/
Allow: /profile/
Allow: /categories/
Allow: /location/
Allow: /shared/
Allow: /duels/
Allow: /duel/

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# SEO tool crawlers — allowed so we can audit our own site
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

# Block aggressive/low-value bots
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ============================================================
# Block AI Training Crawlers & Scrapers
# ============================================================

# OpenAI
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

# Google AI (Gemini training, not search)
User-agent: Google-Extended
Disallow: /

# Anthropic / Claude
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Common Crawl (used by many AI trainers)
User-agent: CCBot
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /

# Amazon / Alexa AI
User-agent: Amazonbot
Disallow: /

# ByteDance / TikTok AI
User-agent: Bytespider
Disallow: /

# Diffbot (data extraction)
User-agent: Diffbot
Disallow: /

# Apple AI training (not regular Applebot)
User-agent: Applebot-Extended
Disallow: /

# You.com AI
User-agent: YouBot
Disallow: /

# Omgili (data mining)
User-agent: Omgilibot
Disallow: /
User-agent: omgili
Disallow: /

# Facebook AI training
User-agent: FacebookBot
Disallow: /

# Firecrawl and generic scraping tools
User-agent: FirecrawlAgent
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: ISSCyberRiskCrawler
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: Newsbreak
Disallow: /

User-agent: ICC-Crawler
Disallow: /

# Sitemap location — points at the LIVE serverless sitemap (api/sitemap.ts,
# wired via vercel.json rewrite /sitemap.xml -> /api/sitemap). It queries
# Supabase fresh on each crawl (1h CDN cache), so new lists/profiles/duels
# are discoverable immediately instead of waiting on a static rebuild. Uses
# www host to match Vercel's apex → www redirect (www is canonical).
Sitemap: https://www.topfived.com/sitemap.xml