# robots.txt — SahraTec (sahratec.com)
# Last updated: June 2026
# Optimized for: Search engines + AI crawlers (2026 best practices)

# ============================================
# GOOGLE — Full access to product & content
# ============================================
User-agent: Googlebot
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /wishlist/
Disallow: /search?
Disallow: /order/
Disallow: /profile/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

# ============================================
# BING
# ============================================
User-agent: Bingbot
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /wishlist/
Disallow: /search?
Disallow: /order/
Disallow: /profile/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

# ============================================
# AI CRAWLERS — Allow for AI search discovery
# (ChatGPT, Perplexity, Claude, Gemini, Meta)
# ============================================
User-agent: GPTBot
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: ChatGPT-User
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: PerplexityBot
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: ClaudeBot
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Anthropic-AI
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Google-Extended
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: meta-externalagent
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /order/
Disallow: /admin/
Content-Signal: ai-train=no, search=yes, ai-input=yes

# ============================================
# AGGRESSIVE SCRAPERS — Block
# ============================================
User-agent: AhrefsBot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: SemrushBot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: MJ12bot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: DotBot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: BLEXBot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

# ============================================
# ALL OTHER BOTS — Default rules
# ============================================
User-agent: *
Allow: /ar/
Allow: /categories/
Allow: /
Disallow: /account/
Disallow: /checkout/
Disallow: /cart/
Disallow: /wishlist/
Disallow: /search?
Disallow: /order/
Disallow: /profile/
Disallow: /admin/

# ============================================
# SITEMAP
# ============================================
Sitemap: https://sahratec.com/sitemap.xml
Content-Signal: ai-train=no, search=yes, ai-input=yes