# robots.txt for https://www.lsbrx.com

User-agent: *

# -----------------------------
# Block system & sensitive areas
# -----------------------------
Disallow: /admin/
Disallow: /Admin/
Disallow: /bin/
Disallow: /obj/
Disallow: /App_Data/
Disallow: /App_Code/
Disallow: /error/
Disallow: /invoice/
Disallow: /cgi-bin/
Disallow: /tmp/

# -----------------------------
# Block APIs (crawl waste)
# -----------------------------
Disallow: /api/
Disallow: /Course/Get
Disallow: /Home/Get

# -----------------------------
# Block unnecessary parameters
# -----------------------------
Disallow: /*?sessionid=
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?searchQuery=

# -----------------------------
# Allow core sections
# -----------------------------
Allow: /courses/
Allow: /blog/
Allow: /podcast/
Allow: /images/

# -----------------------------
# Sitemap
# -----------------------------
Sitemap: https://www.lsbrx.com/sitemap.xml


# -----------------------------
# Googlebot
# -----------------------------
User-agent: Googlebot
Disallow: /admin/
Disallow: /api/


# -----------------------------
# Bingbot
# -----------------------------
User-agent: Bingbot
Disallow: /admin/
Disallow: /api/
Crawl-delay: 3


# -----------------------------
# Block aggressive crawlers
# -----------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /