# https://learnpunjabi.net/ - robots.txt
# Every lesson, guide and story is open to well-behaved crawlers.
#
# The only exclusions are the signed-in surfaces: learner profiles, their
# posts, and the account pages. Those also carry a noindex meta tag and their
# API refuses anonymous callers, so this file is a courtesy rather than the
# control that keeps them out of search results.
#
# NOTE ON GROUPS: a crawler obeys ONLY the most specific User-agent group that
# matches it, and ignores the rest — so a bare "User-agent: Googlebot /
# Allow: /" group would quietly re-open everything below. Every group that is
# allowed in therefore repeats the same Disallow list.

User-agent: *
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /admin.html
Disallow: /api/

# Be explicit for major search engines
User-agent: Googlebot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /admin.html
Disallow: /api/

User-agent: Googlebot-Image
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /admin.html
Disallow: /api/

User-agent: DuckDuckBot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /admin.html
Disallow: /api/

User-agent: Applebot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /admin.html
Disallow: /api/

# AI crawlers - allowed over the learning content, never over learner profiles.
# (Set Disallow: / on any of these to block it outright.)
User-agent: GPTBot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /u/
Disallow: /u.html
Disallow: /profile.html
Disallow: /account.html
Disallow: /api/

# Block aggressive backlink scrapers (bandwidth, not SEO)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

Sitemap: https://learnpunjabi.net/sitemap.xml