# ============================================================
# Robots.txt — dannyreed.biz
# Managed by Pro AI SEO | Updated: 2026-04-21
# ============================================================

# ── ALLOW ALL LEGITIMATE CRAWLERS ──────────────────────────
User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /wp-cron.php
Disallow: /?s=
Disallow: /search/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# ── ALLOW WP ADMIN AJAX (needed for front-end) ─────────────
Allow: /wp-admin/admin-ajax.php

# ── BLOCK AGGRESSIVE/SCRAPER BOTS ──────────────────────────
User-agent: WPScan
Disallow: /

User-agent: sqlmap
Disallow: /

User-agent: nikto
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

# ── MAJOR SEARCH ENGINES ────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/

User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/

# ── SOCIAL CRAWLERS ─────────────────────────────────────────
User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

# ── SITEMAP ─────────────────────────────────────────────────
Sitemap: https://dannyreed.biz/wp-sitemap.xml