# GigXchange -- robots.txt
# Block internal tools, admin, and non-public content from search engines
# Last updated: 2026-05-29

User-agent: *

# Admin & internal tools
Disallow: /administrator
Disallow: /admin
Disallow: /sales/
Disallow: /strategy/
Disallow: /marketing/
Disallow: /archive/
Disallow: /audit/
Disallow: /screenshots/
Disallow: /sql/
Disallow: /supabase/
Disallow: /flyer-templates/
Disallow: /instagram/
Disallow: /node_modules/
Disallow: /Company/
Disallow: /Finance/
Disallow: /GDPR/
Disallow: /compliance/
Disallow: /data/
Disallow: /huggingface/
Disallow: /reel-output/
Disallow: /reels/
Disallow: /brand-assets-captured/
Disallow: /linkedin-screenshots/
Disallow: /jmb-screenshots-check/
Disallow: /gumtree-screenshots/
Disallow: /test-screenshots-fresh/

# Test files
Disallow: /test-

# Worker SSR template files -- never serve directly to users or crawlers.
# Worker injects real data and serves at /profile/<slug>, /event/<slug>.
# The raw underscore-prefixed files are placeholders only.
Disallow: /_profile.html
Disallow: /_event.html
Disallow: /_open-mic.html
Disallow: /_open-mics-city.html
Disallow: /_open-mic-finder.html
Disallow: /_member-spotlight.html
Disallow: /_member-spotlight-hub.html

# App pages (require auth, not useful to index)
Disallow: /app
Disallow: /dashboard
Disallow: /bookings
Disallow: /messages
Disallow: /payments
Disallow: /explore
Disallow: /events
Disallow: /settings
Disallow: /setup
Disallow: /login

# /profile is public (SSR meta via Cloudflare Worker) -- do NOT disallow
Disallow: /profile/test-

Sitemap: https://gigxchange.app/sitemap.xml

# AI crawlers -- allow indexing for GEO (Generative Engine Optimisation)
# Structured manifest at /llms.txt (also /.well-known/llms.txt via 301)
# Machine-readable data at /data/*.json and /.well-known/*.json
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Amazonbot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: cohere-ai
User-agent: Diffbot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Allow: /data/rates-snapshot.json
Allow: /data/scene-scores.json

# Blocked: SEO-analytics scrapers (sell data to competitors,
# consume bandwidth, drive zero traffic to us). Honour-based --
# real abuse protection lives in Cloudflare bot-fight mode.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: LinkpadBot
Disallow: /

# OpenAI search crawler (separate section — different crawl behaviour from GPTBot)
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/
Allow: /data/rates-snapshot.json
Allow: /data/scene-scores.json