# Robots.txt for ChallengeRate B2B Marketplace
# Optimized for Indian B2B commerce and wholesale trade

User-agent: *
Allow: /

# Allow important pages for SEO
Allow: /product/
Allow: /category/
Allow: /search
Allow: /posts/
Allow: /manufacturer/

# Block admin and internal pages
Disallow: /admin
Disallow: /api/
Disallow: /next/
Disallow: /_next/
Disallow: /checkout
Disallow: /account
Disallow: /order/
Disallow: /cart
Disallow: /wishlist

# Allow auth pages for better user discovery
Allow: /login
Allow: /register
Allow: /forgot-password
Allow: /reset-password

# Block development and utility endpoints
Disallow: /seed
Disallow: /preview
Disallow: /package

# Allow search engines to crawl media files
Allow: /media/
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$

# Specific bot instructions

# Google Bot - Allow everything for better indexing
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /account
Disallow: /checkout

# Bing Bot - Allow everything for Microsoft search
User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /account
Disallow: /checkout

# Commercial bots restrictions
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Allow social media bots for better sharing
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Crawl delay to prevent server overload
Crawl-delay: 1

# Sitemap references
Sitemap: https://www.challengerate.com/sitemap.xml
Sitemap: https://www.challengerate.com/main-sitemap.xml
Sitemap: https://www.challengerate.com/pages-sitemap.xml
Sitemap: https://www.challengerate.com/posts-sitemap.xml
Sitemap: https://www.challengerate.com/products-sitemap.xml

# Additional information
# Host: www.challengerate.com

# Clean URLs
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term
Clean-param: gclid&fbclid&msclkid
Clean-param: _ga&_gid&_gl

# ChallengeRate.com - India's Premier B2B Wholesale Marketplace
# For support: Contact ChallengeRate Team
# Last updated: 2025-09-14