# robots.txt for https://asiavirtualsolutions.com
# Optimized for WordPress + WooCommerce + SEO + AI Discovery & Attribution

# XML Sitemaps
Sitemap: https://asiavirtualsolutions.com/post-sitemap.xml
Sitemap: https://asiavirtualsolutions.com/page-sitemap.xml
Sitemap: https://asiavirtualsolutions.com/product-sitemap.xml
Sitemap: https://asiavirtualsolutions.com/category-sitemap.xml
Sitemap: https://asiavirtualsolutions.com/product_cat-sitemap.xml
Sitemap: https://asiavirtualsolutions.com/video-sitemap.xml
Sitemap: https://asiavirtualsolutions.com/local-sitemap.xml

# AI Language Model Guidance
LLMS: https://asiavirtualsolutions.com/llms.txt
LLMS-Full: https://asiavirtualsolutions.com/llms-full.txt
LLMS-JSON: https://asiavirtualsolutions.com/llms.json
LLMS-YAML: https://asiavirtualsolutions.com/llms.yaml

########################################
# Search Engines (standard SEO)
########################################

User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10

# Other common search crawlers (examples, optional to expand)
User-agent: DuckDuckBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Baiduspider
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

########################################
# Preferred AI / LLM Crawlers (explicitly allowed)
########################################
# These are the bots most directly tied to AI search/assistant products.

# OpenAI / ChatGPT (training + search)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

# Google AI (Gemini + AI Overviews / extensions)
User-agent: Google-Extended
Allow: /

# Perplexity AI (AI search engine)
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# CCBot – used by several AI/search systems
User-agent: CCBot
Allow: /

########################################
# Blocked SEO / Scraper / Undesired Bots
########################################
# Classic aggressive SEO crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Examples of non-essential big-tech bots you may not need
# (adjust based on your own policy & logs)
User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

########################################
# General Rules for All Other Bots
########################################

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# WooCommerce & low-value / action URLs (UX + crawl budget)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /add-to-cart=
Disallow: /trackback/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php

# Avoid thin/duplicate feeds, comments & search results
Disallow: /feed/
Disallow: /comments/
Disallow: /*/feed/
Disallow: /*/comments/
Disallow: /?s=
Disallow: /?orderby=