# Allow all crawlers including AI/LLM bots
User-agent: *
Allow: /

# AI/LLM-specific crawlers (explicitly welcome)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Sitemap location
Sitemap: https://www.egyptariffs.com/sitemap.xml

# LLM-friendly resources
Allow: /llms.txt
Allow: /for-llms
Allow: /feed.xml

# Optimize crawling
Crawl-delay: 1

# Important broker pages for SEO
Allow: /brokers
Allow: /brokers/city/*
Allow: /brokers/specialty/*
Allow: /brokers/*

# Important pages
Allow: /faq
Allow: /about
Allow: /contact
Allow: /tariff/*
Allow: /governorates/*
Allow: /categories/*
Allow: /questions/*

# Prevent duplicate content
Disallow: /api/
Disallow: /_next/
Disallow: /*?*page=
Disallow: /*?*sort=