# forms-legal.com robots.txt
# Static file (replaces src/app/robots.ts) — eliminates route-handler crash class.

# Content signals (https://contentsignals.org/): how the content of this site
# may be used once accessed. search = building a search index and showing
# links and short excerpts; ai-input = using the content to ground or answer a
# query in real time; ai-train = training or fine-tuning models.
# A Content-Signal line applies to the user-agent group it sits in, so every
# group below repeats it (a named group replaces the * group for that bot).

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# AI-friendly public endpoint (explicitly Allow'd before Disallow: /api/).
# /api/citation is left to user-triggered agents (group at the end): it returns
# a citation string, nothing to index, and crawlers re-requesting old citation
# URLs spent ~37,000 requests on it in one week of October 2026.
Allow: /api/search-docs
Disallow: /api/
Disallow: /my-documents
Disallow: /search
Disallow: /sign/
Disallow: /seo-dashboard
Disallow: /en/seo-dashboard
Disallow: /es/seo-dashboard
Disallow: /pt/seo-dashboard
Disallow: /fr/seo-dashboard
Disallow: /de/seo-dashboard
# Next.js router payloads (?_rsc=...): a few hundred bytes of route data with no
# page text. Crawlers that fetch them index nothing; the clean URL has the HTML.
Disallow: /*?_rsc=
Disallow: /*&_rsc=
# Generated share images (opengraph-image / twitter-image routes): a PNG per
# page that only link-preview fetchers need. Search and AI crawlers were
# re-fetching them as if they were pages.
Disallow: /opengraph-image$
Disallow: /twitter-image$
Disallow: /*/opengraph-image$
Disallow: /*/twitter-image$
Disallow: /opengraph-image?
Disallow: /twitter-image?
Disallow: /*/opengraph-image?
Disallow: /*/twitter-image?
# Document editor pages (<document URL>/edit, also with ?jurisdiction=...): the
# fill-in form for a document, sent with noindex, nofollow and no-store. The page
# to index is the document URL without /edit. "$" and "?" keep the rule to the
# editor itself: a bare "/*/edit" would also match /es/editorial-guidelines.
Disallow: /*/edit$
Disallow: /*/edit?

# Sitemap discovery
Sitemap: https://forms-legal.com/sitemap-index.xml

# Google search. Same rules as *, except the editor (/edit) stays open: Google
# does not crawl it, and if it ever does, it reads the noindex on the page
# instead of keeping a blocked URL in its index. A named group replaces the *
# group for that bot, so every other rule is repeated here.
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/search-docs
Disallow: /api/
Disallow: /my-documents
Disallow: /search
Disallow: /sign/
Disallow: /seo-dashboard
Disallow: /en/seo-dashboard
Disallow: /es/seo-dashboard
Disallow: /pt/seo-dashboard
Disallow: /fr/seo-dashboard
Disallow: /de/seo-dashboard
Disallow: /*?_rsc=
Disallow: /*&_rsc=
Disallow: /opengraph-image$
Disallow: /twitter-image$
Disallow: /*/opengraph-image$
Disallow: /*/twitter-image$
Disallow: /opengraph-image?
Disallow: /twitter-image?
Disallow: /*/opengraph-image?
Disallow: /*/twitter-image?

# Bing search (also the index behind Copilot). Same as Googlebot: every rule of
# *, editor left open so Bing reads its noindex and drops the /edit URLs it
# still holds. To be reviewed against Bing's crawl logs in December 2026.
# Bing's other crawlers (msnbot, adidxbot) are not named and follow *, so
# they get the /edit rule; that is intended.
User-agent: Bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/search-docs
Disallow: /api/
Disallow: /my-documents
Disallow: /search
Disallow: /sign/
Disallow: /seo-dashboard
Disallow: /en/seo-dashboard
Disallow: /es/seo-dashboard
Disallow: /pt/seo-dashboard
Disallow: /fr/seo-dashboard
Disallow: /de/seo-dashboard
Disallow: /*?_rsc=
Disallow: /*&_rsc=
Disallow: /opengraph-image$
Disallow: /twitter-image$
Disallow: /*/opengraph-image$
Disallow: /*/twitter-image$
Disallow: /opengraph-image?
Disallow: /twitter-image?
Disallow: /*/opengraph-image?
Disallow: /*/twitter-image?

# OpenAI: search indexing + training
User-agent: OAI-SearchBot
User-agent: GPTBot
# Anthropic
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: anthropic-ai
# Perplexity
User-agent: PerplexityBot
# Google AI training (Gemini)
User-agent: Google-Extended
# Apple: Applebot-Extended (Apple Intelligence) and Applebot (Siri/Spotlight).
# Applebot must be named: without its own line it follows the Googlebot group
# (support.apple.com/en-us/119829) and would skip the /edit rule.
User-agent: Applebot-Extended
User-agent: Applebot
# Meta
User-agent: FacebookBot
User-agent: meta-externalagent
# Other AI engines
User-agent: YouBot
User-agent: Bytespider
User-agent: cohere-ai
# Common Crawl (LLM training data path — every major model pulls from monthly CC snapshots)
User-agent: CCBot
# Other crawlers named explicitly, so they can get rules of their own later
User-agent: Amazonbot
User-agent: YandexBot
User-agent: KakaoBot
# SEO link indexes (Moz, Majestic)
User-agent: DotBot
User-agent: MJ12bot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/search-docs
Disallow: /api/
Disallow: /my-documents
Disallow: /search
Disallow: /sign/
Disallow: /seo-dashboard
Disallow: /en/seo-dashboard
Disallow: /es/seo-dashboard
Disallow: /pt/seo-dashboard
Disallow: /fr/seo-dashboard
Disallow: /de/seo-dashboard
# Same as above: a named group replaces the * group for these bots, so it must
# repeat every rule.
Disallow: /*?_rsc=
Disallow: /*&_rsc=
Disallow: /opengraph-image$
Disallow: /twitter-image$
Disallow: /*/opengraph-image$
Disallow: /*/twitter-image$
Disallow: /opengraph-image?
Disallow: /twitter-image?
Disallow: /*/opengraph-image?
Disallow: /*/twitter-image?
Disallow: /*/edit$
Disallow: /*/edit?

# Agents that fetch a page because a person asked them to (not crawlers): they
# may also call the citation API to cite the page they were asked about.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/search-docs
Allow: /api/citation
Disallow: /api/
Disallow: /my-documents
Disallow: /search
Disallow: /sign/
Disallow: /seo-dashboard
Disallow: /en/seo-dashboard
Disallow: /es/seo-dashboard
Disallow: /pt/seo-dashboard
Disallow: /fr/seo-dashboard
Disallow: /de/seo-dashboard
Disallow: /*?_rsc=
Disallow: /*&_rsc=
Disallow: /opengraph-image$
Disallow: /twitter-image$
Disallow: /*/opengraph-image$
Disallow: /*/twitter-image$
Disallow: /opengraph-image?
Disallow: /twitter-image?
Disallow: /*/opengraph-image?
Disallow: /*/twitter-image?

# Link-preview fetchers: they build the card from the share image, so they keep
# access to it (Twitterbot and LinkedInBot obey robots.txt). Every other rule
# is the same as above.
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: facebookexternalhit
User-agent: Slackbot
User-agent: Discordbot
User-agent: TelegramBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/search-docs
Disallow: /api/
Disallow: /my-documents
Disallow: /search
Disallow: /sign/
Disallow: /seo-dashboard
Disallow: /en/seo-dashboard
Disallow: /es/seo-dashboard
Disallow: /pt/seo-dashboard
Disallow: /fr/seo-dashboard
Disallow: /de/seo-dashboard
Disallow: /*?_rsc=
Disallow: /*&_rsc=

Host: forms-legal.com