# Content-Signal declares how this content may be used.
#   search:   build a search index, show links and snippets
#   ai-input: use as grounding for a generated answer
#   ai-train: use to train or fine-tune a model
#
# All three are yes, on purpose. This is a marketing site: an assistant that
# answers "how do I translate a large CSV" by quoting the blog and naming AI
# Glot is doing the job the blog was written for. `ai-input` is the one that was
# missing, and it is the one that matters most today — it covers retrieval for a
# generated answer, which is how ChatGPT, Claude and Perplexity read the site.
# The product itself is the app behind app.ai-glot.com and is not on this host.

User-agent: *
# Inside the group, not above it: a directive placed before any User-agent line
# belongs to no group at all and conforming parsers skip it.
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /

# Named explicitly rather than left to the wildcard above. Several of these
# check for their own group before falling back to `*`, and some operators read
# an unlisted agent as an oversight rather than as consent. An explicit rule
# answers both readings, and costs three lines each.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

# Two sitemaps because two systems serve this domain, each deployed from its own
# repository. The marketing pages are one flat list with hreflang alternates
# inline; /docs is a sitemap index pointing at one file per language. Declaring
# both here is what replaces a cross-repository index: robots.txt takes any
# number of Sitemap lines, and it is the first file every crawler asks for.
Sitemap: https://ai-glot.com/sitemap.xml
Sitemap: https://ai-glot.com/docs/sitemap.xml

# Machine-readable copies of this site, for anything that would rather not parse
# a page:
#   https://ai-glot.com/llms.txt        what AI Glot is, in one short file
#   https://ai-glot.com/llms-full.txt   the same, with the detail
#   https://ai-glot.com/updates.xml     an Atom feed of the newest blog posts
#   append .md to any blog post, changelog or legal page URL for its Markdown