# AI training crawlers and bulk collectors: not permitted. The catalog pages
# are published for people and for search and answer engines that cite their
# source. Bulk collection of the organised catalog, for model training or for
# an AI index, is licensed, not free: see
# https://nsnresolve.com/guides/data-licensing or contact mcp@nsnresolve.com.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Bytespider
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Meta-ExternalAgent
User-agent: meta-webindexer
User-agent: Meta-WebIndexer
User-agent: Amazonbot
User-agent: Diffbot
User-agent: cohere-ai
User-agent: omgili
Disallow: /

# Amazon search discovery remains welcome; bulk catalog indexing is licensed.
# Amzn-User, Amazon's user-triggered fetcher, follows the default allow group.
User-agent: Amzn-SearchBot
Allow: /
Disallow: /nsn
Disallow: /pn
Disallow: /cage
Disallow: /fsg
Disallow: /fsc
Disallow: /og

# Parallel's bulk search index requires licensed catalog access.
# Shap-User requests remain covered by the default allow group.
# Payment options: https://nsnresolve.com/machine-payments.md
User-agent: ShapBot
Allow: /
Disallow: /nsn$
Disallow: /nsn/
Disallow: /pn$
Disallow: /pn/
Disallow: /cage$
Disallow: /cage/
Disallow: /fsg$
Disallow: /fsg/
Disallow: /fsc$
Disallow: /fsc/
Disallow: /og$
Disallow: /og/

# SEO index crawlers are welcome at a civil pace. These three state that they
# obey Crawl-delay; ten seconds is about 8,600 pages a day each.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
Crawl-delay: 10
Allow: /

# Search and answer engines that link back are welcome (Googlebot, Bingbot,
# OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot).
User-agent: *
Allow: /

# Content Signals (contentsignals.org): search and answer engines may index
# and quote; model training is not permitted.
Content-Signal: search=yes, ai-input=yes, ai-train=no

Sitemap: https://nsnresolve.com/sitemap.xml