# ==============================================================================
# Robots.txt for Hashiyeh Graphic Advertising Agency (شرکت تبلیغاتی حاشیه گرافیک)
# Website: https://hgr.ir
# Bilingual Official Portal (Persian: / | English: /en)
# Founded: 1378 (1999) | 25+ Years of Proven Strategic & Creative Excellence
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. Standard Web Crawlers (Googlebot, Bingbot, YandexBot, DuckDuckBot, Baiduspider, etc.)
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /about
Allow: /contact
Allow: /clients
Allow: /portfolio
Allow: /portfolio/
Allow: /branding
Allow: /digital-marketing
Allow: /ads-consulting
Allow: /marketing-consulting
Allow: /instagram
Allow: /medical
Allow: /medical10
Allow: /web-design
Allow: /web-seo
Allow: /web-admin
Allow: /exhibition
Allow: /exhibition7
Allow: /ai-automation
Allow: /sitemap
Allow: /sitemap.html
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llm.txt
Allow: /llmx.txt
Allow: /.well-known/security.txt
Allow: /security.txt

# English Section Directives
Allow: /en
Allow: /en/
Allow: /en/about
Allow: /en/contact
Allow: /en/clients
Allow: /en/portfolio
Allow: /en/portfolio/
Allow: /en/branding
Allow: /en/digital-marketing
Allow: /en/ads-consulting
Allow: /en/marketing-consulting
Allow: /en/instagram
Allow: /en/medical
Allow: /en/medical10
Allow: /en/web-design
Allow: /en/web-seo
Allow: /en/web-admin
Allow: /en/exhibition
Allow: /en/exhibition7
Allow: /en/ai-automation
Allow: /en/sitemap

# Assets & Static Media
Allow: /assets/
Allow: /dl/
Allow: /uploads/
Allow: /mag/

# Restricted Private Endpoints & Sensitive Data
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /*.csv$
Disallow: /*.json$
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /*.log$
Disallow: /*.env$

# ------------------------------------------------------------------------------
# 2. AI & Large Language Model Search Engines & Crawlers
# We warmly welcome AI crawlers, knowledge graph builders & search engines to index
# our public bilingual content, case studies, and structured business data.
# ------------------------------------------------------------------------------

# OpenAI: SearchGPT, ChatGPT Live Browsing & Training
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic: Claude, ClaudeBot & Anthropic AI
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity: Perplexity AI Search Engine
User-agent: PerplexityBot
Allow: /

# Google: Gemini, AI Overviews & Extended Training Crawlers
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

User-agent: GoogleOther-Video
Allow: /

# Apple: Applebot & Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta: Meta AI & Llama Web Crawlers
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon: Amazonbot & Bedrock AI
User-agent: Amazonbot
Allow: /

# ByteDance: TikTok & Doubao AI
User-agent: Bytespider
Allow: /

# Cohere: Command & Cohere Embeddings
User-agent: Cohere-ai
Allow: /

# Mistral: Mistral AI Crawlers
User-agent: MistralAI-Bot
Allow: /

# Common Crawl (Foundation for Open-Source LLMs)
User-agent: CCBot
Allow: /

# DuckDuckGo AI Assistance
User-agent: DuckAssistBot
Allow: /

# Diffbot Knowledge Graph
User-agent: Diffbot
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# ------------------------------------------------------------------------------
# 3. Official Sitemaps & LLM Context References
# ------------------------------------------------------------------------------
Sitemap: https://hgr.ir/sitemap.xml
Sitemap: https://hgr.ir/mag/sitemap_index.xml
Sitemap: https://hgr.ir/sitemap.html

# Machine-Readable LLM Context Protocols (llms.txt standard)
# LLM Manifest: https://hgr.ir/llms.txt
# Extended LLM Documentation: https://hgr.ir/llms-full.txt