#
# FULL ACCESS ROBOTS.TXT FILE
# This file grants ALL known major crawlers, including AI bots, full access (Allow: /)
# to the entire site.
#

# --- Standard Search Engine Crawlers ---
User-agent: Googlebot
Allow: /
Allow: /*?startPlanning=
Disallow: /*?

User-agent: Bingbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

# --- Social Media/Link Preview Bots ---
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# --- Blocked High-Volume Scrapers ---
User-agent: AhrefsBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Allow: /
Disallow: /api/
Crawl-delay: 600

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# --- Major LLM Training & Generative AI Bots ---
User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: MistralAI-User
Allow: /

# --- General Wildcard Bot ---
User-agent: *
Allow: /
Disallow: /*?

# Directives for search engines to find the sitemap
Sitemap: https://meetinthemiddle.site/sitemap.xml