# robots.txt — managramasethu.in

User-agent: *
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /account
Disallow: /checkout
Disallow: /checkouts/

# Internal search (thin content)
Disallow: /search

# Sorting & filter duplicate URLs
Disallow: /*?*sort_by=*
Disallow: /*?*filter=*

# Shopify preview URLs
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*

# Shopify system endpoints
Disallow: /sf_private_access_tokens
Disallow: /.well-known/shopify/monorail
Disallow: /recommendations/products
Disallow: /*/recommendations/products

Sitemap: https://managramasethu.in/sitemap.xml 


# Google AdsBot (must be explicitly defined)
User-agent: AdsBot-Google
Disallow: /checkout
Disallow: /checkouts/
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /account
Disallow: /sf_private_access_tokens
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*


# Block very old / low-value crawler
User-agent: Nutch
Disallow: /


# AI / LLM Crawlers (Allowed for public content understanding)
# =============================================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /


# Throttle aggressive SEO tools
# ==============================
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10