# Robots.txt for gemstonewow.com
#
# Account, admin and form-handling routes are kept out of every crawl. The
# sign-in and password pages are NOT disallowed: they answer with noindex, and a
# crawler has to be allowed to fetch a page before it can read that and drop it.
#
# /auth/ and /*/index are a different matter. CodeIgniter serves a controller at
# its own name as well as at its pretty route, so /auth/login answers alongside
# /login and /faq/index alongside /faq. Both aliases return 200 and the canonical
# echoes whichever path was asked for, which makes every one of them a duplicate
# that nominates itself. Nothing links to them, so they are simply kept out of
# the crawl. The real fix is for the canonical to name the page's own address
# instead of the requested one; until then, these two rules carry it.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /application/
Disallow: /system/
Disallow: /vendor/
Disallow: /docker/
Disallow: /install
Disallow: /uploads/temp/
Disallow: /user/
Disallow: /logout
Disallow: /lang/
Disallow: /auth/
Disallow: /news/comment/
Disallow: /confirm-register/
Disallow: /*/index$

# Search and AI crawlers we actively want, held to the same boundaries. These
# have to repeat the rules above: a named group replaces the wildcard group
# outright, so an agent listed here with nothing but "Allow: /" would be free to
# walk straight into the account and admin areas.
User-agent: Googlebot
User-agent: Bingbot
User-agent: YandexBot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
Allow: /
Disallow: /admin/
Disallow: /application/
Disallow: /system/
Disallow: /vendor/
Disallow: /docker/
Disallow: /install
Disallow: /uploads/temp/
Disallow: /user/
Disallow: /logout
Disallow: /lang/
Disallow: /auth/
Disallow: /news/comment/
Disallow: /confirm-register/
Disallow: /*/index$

# Sitemap, AI & IndexNow
Host: https://gemstonewow.com
Sitemap: https://gemstonewow.com/sitemap.xml
# IndexNow key: https://gemstonewow.com/7b976ac82c7346c9ab62b46bbad1d789.txt