# ──────────────────────────────────────────────────────────────────────
#  theaustinlocalgroup.com
#  The Austin Local Group · Mike Vilece · Compass Real Estate · Austin, TX
#
#  This site is intentionally open to AI agents and LLM crawlers.
#  Machine-readable entry points:
#    /llms.txt     — site summary + key pages (https://llmstxt.org)
#    /AGENTS.md    — what an agent can do for a visitor here
#    /sitemap.xml  — every public HTML page
#
#  Markdown: request any listed page with `Accept: text/markdown`, or
#  fetch the .md file directly (/index.md, /about.md, /contact.md,
#  /service-areas.md, /relocation-guide.md).
# ──────────────────────────────────────────────────────────────────────

# ── AI crawler policy ──────────────────────────────────────────────
#   This file used to carry "Content-Signal:" lines declaring ai-train=no.
#   They were removed: that directive is not part of the robots.txt
#   standard, so SEO validators report the entire file as invalid syntax
#   when they see it. No crawler acted on those lines, so nothing about
#   how this site is crawled has changed.
#
#   The training-data collectors below (GPTBot, Google-Extended, CCBot)
#   are currently allowed. To actually opt out of AI training, flip their
#   `Allow: /` to `Disallow: /`. That does NOT affect whether ChatGPT,
#   Claude, Perplexity or Google AI can cite this site in answers. Citing
#   is handled by the live-retrieval agents (ChatGPT-User, OAI-SearchBot,
#   Claude-User, Claude-SearchBot, Perplexity-User).

User-agent: *
Allow: /

# ── Microsoft / Bing ──────────────────────────────────────────────────
User-agent: bingbot
Allow: /

# ── OpenAI / ChatGPT ──────────────────────────────────────────────────
# GPTBot collects training data.
User-agent: GPTBot
Allow: /

# ChatGPT-User fetches pages live when a user asks about them.
User-agent: ChatGPT-User
Allow: /

# OAI-SearchBot builds the ChatGPT search index.
User-agent: OAI-SearchBot
Allow: /

# ── Anthropic / Claude ────────────────────────────────────────────────
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Claude-User fetches pages live on behalf of a Claude user.
User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ── Google AI (Gemini / Vertex grounding) ─────────────────────────────
# Google-Extended is a permission token, not a crawler. Allowing it opts this
# site into Gemini grounding AND Google model training — Google does not let
# those be separated. Disallow it if the training opt-out matters more.
User-agent: Google-Extended
Allow: /

# ── Perplexity ────────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ── Common Crawl ──────────────────────────────────────────────────────
# CCBot's archive is a common source of LLM training corpora.
User-agent: CCBot
Allow: /

# ── Other AI crawlers ─────────────────────────────────────────────────
User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

Sitemap: https://theaustinlocalgroup.com/sitemap.xml