# ProGrowth Services - Robots.txt
# This file tells search engine crawlers which pages or files the crawler can or can't request from your site.

User-agent: *
Allow: /

# Allow all bots to access CSS, JS, and images
Allow: /assets/
Allow: /images/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$

# Disallow admin pages - prevent indexing
Disallow: /admin
Disallow: /admin/*

# Disallow sensitive areas
Disallow: /api/
Disallow: /.env
Disallow: /node_modules/
Disallow: /src/
Disallow: /scripts/
Disallow: /*.json$

# Disallow sitemap files from being indexed as content
Disallow: /sitemap*.xml

# Legacy URLs - Redirected, discourage indexing
Disallow: /locations/

# Common bot behavior
Crawl-delay: 1

# Sitemap locations
Sitemap: https://www.progrowth.services/sitemap-index.xml
Sitemap: https://www.progrowth.services/sitemap.xml
Sitemap: https://www.progrowth.services/video-sitemap.xml
Sitemap: https://www.progrowth.services/image-sitemap.xml

# Google-specific directives
User-agent: Googlebot
Allow: /

# Bing-specific directives
User-agent: Bingbot
Allow: /

# AI Bots - Allow access for LLM training and knowledge indexing
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bard
Allow: /

User-agent: Anthropic-AI
Allow: /

# Allow SEO audit tools (for legitimate SEO analysis)
User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

# Block aggressive crawlers that don't respect crawl-delay
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /