User-agent: *
Allow: /
Allow: /_next/static/
Allow: /sitemap.xml
Allow: /sitemap/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Allow: /api/v1/openapi.json
Disallow: /*?_rsc=*
Disallow: /api/*
Disallow: /api/
Disallow: /dashboard
Disallow: /yt-thumb/
Disallow: /browse/*?page=*
Disallow: /gone
# Content-Signal: search=yes, ai-input=yes, ai-train=no

# Search, AI Citation & SEO Audit Crawlers
User-agent: Googlebot
User-agent: GoogleOther
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Applebot
User-agent: AhrefsBot
User-agent: SemrushBot
Allow: /
Allow: /_next/static/
Allow: /sitemap.xml
Allow: /sitemap/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/v1/openapi.json
Disallow: /*?_rsc=*
Disallow: /api/*
Disallow: /api/
Disallow: /dashboard
Disallow: /gone

# Aggressive Bulk Scrapers
User-agent: Bytespider
User-agent: MJ12bot
User-agent: DotBot
User-agent: ShapBot
User-agent: KeenableBot
Disallow: /

# Agentmap: https://sayitvid.com/.well-known/ai-catalog.json
Sitemap: https://sayitvid.com/sitemap.xml