# LinkIntel Marketing Site - robots.txt
#
# Machine-readable index: https://www.getlinkintel.com/llms.txt
# Full text for agents:   https://www.getlinkintel.com/llms-full.txt
# Every page is also available as markdown: append .md, or send `Accept: text/markdown`.
# Pricing for agents:     https://www.getlinkintel.com/pricing.md
# MCP server card:        https://www.getlinkintel.com/.well-known/mcp/server-card.json
# Agent auth:             https://www.getlinkintel.com/auth.md
# Skills index:           https://www.getlinkintel.com/.well-known/agent-skills/index.json
#
# Content signals (contentsignals.org): this site may be used for search results, as input to AI
# answers, and for model training. Being citable by assistants is the point of the site.

# Main search engines - Full access
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Crawl-delay: 0

# AI assistants fetching on a user's behalf.
# These are the ones that can actually send a buyer, so they are welcome explicitly rather than
# left to the wildcard rule.
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Amazonbot
User-agent: MistralAI-User
User-agent: Google-CloudVertexBot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /share/

# AI training crawlers.
# Allowed deliberately: being present in model weights is how a product gets recommended when
# somebody asks an assistant for LinkedIn analytics. Flip these to Disallow if that trade stops
# being worth it.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /share/

# All other bots - Standard access
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /_next/static/
Disallow: /share/
Disallow: /*.json$
# Discovery documents are JSON on purpose; the longer rule wins over the one above.
Allow: /.well-known/
Allow: /openapi.json
Allow: /x-algorithm.json

# Block aggressive SEO/marketing crawlers
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: SerpstatBot
User-agent: MegaIndex
User-agent: spbot
User-agent: PetalBot
User-agent: Cliqzbot
Disallow: /

# Block content scrapers
User-agent: ia_archiver
User-agent: ScoutJet
User-agent: Sogou
User-agent: spider
User-agent: Yeti
User-agent: grapeshot
User-agent: HTTrack
User-agent: WebCopier
Disallow: /

# Agent discovery manifest (ARD)
Agentmap: https://www.getlinkintel.com/.well-known/ai-catalog.json

# Sitemaps
Sitemap: https://www.getlinkintel.com/sitemap.xml

# Host preference (helps with canonicalization)
Host: https://www.getlinkintel.com