# Google AdSense crawler
User-agent: Mediapartners-Google
Allow: /

# All other crawlers
User-agent: *
Disallow: /kv-admin/
Disallow: /user_login/
Disallow: /logout/
Disallow: /password_reset/*
Disallow: /reset/*
Disallow: /index.php/onlinemusicservices/
Disallow: /*adminhtml*
Disallow: /search
Disallow: /cart/
Disallow: /payment/
Disallow: /donate/
Disallow: /activate/
Disallow: /reset-password/
Disallow: /change-password/
Disallow: /verify-email/
Disallow: /validate-otp/
Disallow: /dashboard/
Disallow: /author-details/

# Explicitly allow important files & assets
Allow: /ads.txt
Allow: /static/
Allow: /media/
Allow: /sitemap.xml
Allow: /sitemap-news.xml

# =========================
# AI Crawlers (relevant for India traffic)
# =========================

# ChatGPT (via OpenAI training + Bing)
User-agent: GPTBot
Allow: /

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /

# Perplexity AI (growing usage)
User-agent: PerplexityBot
Allow: /

# Claude
User-agent: ClaudeBot
Allow: /

# =========================
# Search Engines
# =========================

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 10

# Bing (used by ChatGPT browsing)
User-agent: Bingbot
Allow: /
Crawl-delay: 10

# Yandex
User-agent: YandexBot
Allow: /
Crawl-delay: 10

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 10

# Host directive for preferred domain
Host: https://www.newsmart.in

Sitemap: https://www.newsmart.in/sitemap.xml
Sitemap: https://www.newsmart.in/sitemap-news.xml