# robots.txt for aisearchmarketing.agency
# AI crawlers are explicitly welcome. Being read by them is the point.
#
# IMPORTANT: Cloudflare can inject its own "Cloudflare Managed content" block
# ABOVE this one, containing Disallow rules for GPTBot, ClaudeBot,
# Google-Extended, Amazonbot, Applebot-Extended and others, plus
# "Content-Signal: ai-train=no". That is the correct default for a publisher
# protecting content, and exactly wrong for this site, whose entire proposition
# is being ingested and cited by AI models.
#
# If you ever see a "# BEGIN Cloudflare Managed content" section at the top of
# the live robots.txt, turn the managed robots.txt feature OFF in the Cloudflare
# dashboard under AI Crawl Control. Editing this file will not remove it,
# because Cloudflare prepends its block at the edge.
#
# Content signals (contentsignals.org). Deliberately permissive across the board:
#   search    = yes  index it and link to it
#   ai-input  = yes  use it to ground and answer questions in real time
#   ai-train  = yes  train on it, that is how models learn this business exists

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes
Allow: /

# --- Generative / answer engines ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: cohere-ai
Allow: /

# /trr/ is a client preview site. It is deliberately NOT disallowed here.
# It carries an X-Robots-Tag: noindex header instead, and a crawler has to
# be allowed to fetch the URL in order to see that header. Disallowing it
# would hide the noindex and the URL could still be indexed from an
# external link. Do not "tidy this up" by adding Disallow: /trr/.

Sitemap: https://aisearchmarketing.agency/sitemap.xml