# ===== ROBOTS.TXT FOR CONCRETESHIELDCOATINGSINC.COM =====
# Last updated: December 2024

# ===== DEFAULT RULES (ALL BOTS) =====
User-agent: *
Allow: /

# Block admin and system directories
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /cache/
Disallow: /logs/

# Block sensitive files
Disallow: /*.bak$
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.zip$

# Block query strings (duplicate content prevention)
Disallow: /*?*

# Block common CMS/system paths (if applicable)
Disallow: /admin/
Disallow: /includes/
Disallow: /config/

# ===== GOOGLE BOTS =====
# Meta's link-preview crawler must be able to fetch ad URLs with UTM
# parameters (the * group above blocks /*?*).
User-agent: facebookexternalhit
Allow: /

User-agent: Googlebot
Allow: /
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$

User-agent: Googlebot-Image
Allow: /images/
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$

# ===== BING BOT =====
User-agent: Bingbot
Allow: /

# ===== BLOCK KNOWN BAD BOTS =====
# These bots are known for spam or malicious activity

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

# ===== SITEMAPS =====
Sitemap: https://www.concreteshieldcoatingsinc.com/sitemap.xml

# ===== HOST DIRECTIVE =====
Host: https://www.concreteshieldcoatingsinc.com