ai-crawler-robots 1.0.0__tar.gz

ai_crawler_robots-1.0.0/LICENSE

@@ -0,0 +1,28 @@
+MIT License
+
+Copyright (c) 2026 Pathwren
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+The crawler table shipped in data.json is dedicated to the public domain under
+CC0-1.0. Robots tokens, user-agent strings and documentation URLs are taken
+from each operator's own published documentation; the categories and the prose
+are the index's own.

ai_crawler_robots-1.0.0/PKG-INFO

@@ -0,0 +1,186 @@
+Metadata-Version: 2.4
+Name: ai-crawler-robots
+Version: 1.0.0
+Summary: Lint a robots.txt and audit it against 56 AI crawlers: which ones it really blocks, which rules do nothing, and which fetchers robots.txt cannot stop at all. RFC 9309, zero dependencies, no network.
+Author: Pathwren
+License: MIT
+Project-URL: Homepage, https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/
+Project-URL: Documentation, https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/
+Project-URL: Data source, https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/data.json
+Project-URL: Companion package, https://pypi.org/project/ai-crawler-index/
+Keywords: robots.txt,robots,rfc9309,lint,linter,crawler,ai,gptbot,claudebot,seo,crawl,user-agent,offline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# ai-crawler-robots
+
+**A robots.txt that looks right and blocks nothing is the normal failure.**
+This package reads the file you already have and tells you which of the
+56 AI crawlers it actually stops.
+
+```python
+from ai_crawler_robots import lint, audit, is_allowed
+
+lint(open("robots.txt").read())
+# [{'level': 'error', 'line': 3, 'code': 'ua-is-not-a-token',
+#   'message': 'User-agent takes a product token, not a full user-agent string. '
+#              'This group matches no crawler.',
+#   'found': 'Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)',
+#   'fix': 'User-agent: GPTBot'}]
+
+is_allowed(text, "GPTBot", "/blog/post")
+# {'allowed': False, 'rule': 'Disallow: /blog', 'line': 8, 'group': 'GPTBot', ...}
+
+audit(text)["blocked"]        # ['anthropic-claudebot', 'openai-gptbot', ...]
+audit(text)["unenforceable"]  # crawlers that will not obey it whatever it says
+```
+
+## Install
+
+```sh
+pip install ai-crawler-robots
+```
+
+Python >= 3.8. **Zero dependencies, standard library only.** No network at
+import: 56 crawler records and 8 ready-made stances ship
+inside the wheel (68.1 KB), snapshot taken 2026-09-01.
+
+**Nothing in this package fetches a URL.** There is no argument anywhere that
+takes one. `refresh()` is the only call that touches the network, you have to
+name it, and it fetches exactly one document: the same
+[data.json](https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/data.json) that ships in the wheel.
+
+## The faults it finds
+
+The most common robots.txt failure is not a syntax error — it is a file that a
+person reads as correct and a parser reads as empty. 23 checks, of
+which these are the ones that cost real traffic:
+
+| code | what it means |
+| --- | --- |
+| `user_agent_is_a_ua_string` | `User-agent:` carrying a whole browser-style string. RFC 9309 matches a **product token**; that group matches nothing. |
+| `unknown-token` | a token no crawler in the table sends — usually a typo, `ChatGPT-user` for `ChatGPT-User`, or a name somebody invented |
+| `duplicate_group` | a second group for the same token; only the first one applies, the rest is decoration |
+| `noindex_in_robots` | `Noindex:` in robots.txt. Google stopped honouring it on 1 September 2019 and it never worked anywhere else. |
+| `unenforceable` | the rule is correct and the operator states robots.txt does not apply to that fetcher. You need an edge rule, and this says so. |
+| `crawl_delay` | `Crawl-delay:` is not in RFC 9309 and Google, OpenAI and Anthropic all ignore it |
+| `path_without_slash` | `Disallow: admin` does not match `/admin` |
+| `html_response` / `empty` / `bom` | the file never parsed at all |
+
+Every finding carries a line number, what was found, and a `fix` string you can
+paste.
+
+## Command line
+
+```sh
+ai-crawler-robots lint robots.txt                       # the faults, worst first
+ai-crawler-robots audit robots.txt                      # every AI crawler: blocked / allowed / unenforceable
+ai-crawler-robots check robots.txt --agent GPTBot --path /blog/
+ai-crawler-robots diff old.txt new.txt                  # what changed by effect, not by text
+ai-crawler-robots policy block-ai-training              # print a ready-made stance
+ai-crawler-robots policy block-ai-training --merge robots.txt
+ai-crawler-robots explain GPTBot                        # who it is, what blocking costs
+```
+
+`lint` exits `1` when it finds an error, `0` otherwise, so it works as a CI
+gate:
+
+```sh
+ai-crawler-robots lint robots.txt --strict || exit 1
+```
+
+## `audit` — the question people actually have
+
+```sh
+$ ai-crawler-robots audit robots.txt
+robots.txt: 56 crawlers, 12 blocked, 39 allowed, 5 unenforceable
+
+BLOCKED (12)
+  openai-gptbot            GPTBot              OpenAI       ai-training
+  anthropic-claudebot      ClaudeBot           Anthropic    ai-training
+  ...
+ALLOWED, AND YOU PROBABLY MEANT TO BLOCK (7)
+  google-extended          Google-Extended     Google       ai-training
+  ...
+UNENFORCEABLE BY robots.txt (5)
+  These fetch on behalf of a user in a live session. Their operators state
+  robots.txt does not govern them. Blocking needs an edge rule.
+```
+
+The third group is the one that surprises people. A user-triggered fetcher is
+not a crawler, most operators say so in their own documentation, and a
+`Disallow` aimed at one is a line that will never fire.
+
+## `diff` — did that edit do anything?
+
+```sh
+$ ai-crawler-robots diff robots.old robots.new
+2 crawler(s) changed verdict for /
+  google-extended     allowed -> blocked
+  meta-externalagent  allowed -> blocked
+44 unchanged. Formatting-only changes are not reported.
+```
+
+Compares by **effect**. Reordering groups, changing whitespace and rewriting
+comments produce an empty diff, because none of them change what a crawler does.
+
+## Ready-made stances
+
+8 of them, the same bodies published at the index:
+
+- `allow-all` — Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold
+- `block-ai-training` — Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers
+- `block-all-ai` — Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed
+- `block-datasets` — Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot
+- `allow-ai-search-only` — Be findable and citable in assistants without contributing to training corpora
+- `block-seo-tools` — Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic
+- `block-disputed` — The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead
+- `maximum-ai-visibility` — Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines
+
+`--merge` adds a stance to an existing file **without touching the groups
+already in it**, so your Googlebot rules survive.
+
+## Where the data comes from
+
+The [AI Crawler Index](https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/) — an independent, non-commercial public
+reference. Every crawler record links to the operator's own documentation, and
+`explain` prints that link. Presence in the table means the operator documents
+that crawler; it is not an opinion about whether you should allow it.
+
+`refresh()` fetches today's table and caches it six hours, which is how often
+the index behind it is rebuilt:
+
+```python
+from ai_crawler_robots import refresh
+refresh()   # {'crawlers': 56, 'generated_at': '...'}
+```
+
+## Companion packages
+
+Same table, different questions:
+
+- [`ai-crawler-index`](https://pypi.org/project/ai-crawler-index/) — what does
+  this user-agent claim to be?
+- [`ai-crawler-verify`](https://pypi.org/project/ai-crawler-verify/) — is that
+  claim true? Checks the address against the operator's published ranges.
+- [`ai-crawler-logs`](https://pypi.org/project/ai-crawler-logs/) — who was
+  actually in your access log, and what to paste to act on it.
+
+## Licence
+
+Code MIT. Bundled data CC0-1.0. Independent and non-commercial; not affiliated
+with, endorsed by or speaking for any crawler operator named in the table.

ai_crawler_robots-1.0.0/README.md

@@ -0,0 +1,158 @@
+# ai-crawler-robots
+
+**A robots.txt that looks right and blocks nothing is the normal failure.**
+This package reads the file you already have and tells you which of the
+56 AI crawlers it actually stops.
+
+```python
+from ai_crawler_robots import lint, audit, is_allowed
+
+lint(open("robots.txt").read())
+# [{'level': 'error', 'line': 3, 'code': 'ua-is-not-a-token',
+#   'message': 'User-agent takes a product token, not a full user-agent string. '
+#              'This group matches no crawler.',
+#   'found': 'Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)',
+#   'fix': 'User-agent: GPTBot'}]
+
+is_allowed(text, "GPTBot", "/blog/post")
+# {'allowed': False, 'rule': 'Disallow: /blog', 'line': 8, 'group': 'GPTBot', ...}
+
+audit(text)["blocked"]        # ['anthropic-claudebot', 'openai-gptbot', ...]
+audit(text)["unenforceable"]  # crawlers that will not obey it whatever it says
+```
+
+## Install
+
+```sh
+pip install ai-crawler-robots
+```
+
+Python >= 3.8. **Zero dependencies, standard library only.** No network at
+import: 56 crawler records and 8 ready-made stances ship
+inside the wheel (68.1 KB), snapshot taken 2026-09-01.
+
+**Nothing in this package fetches a URL.** There is no argument anywhere that
+takes one. `refresh()` is the only call that touches the network, you have to
+name it, and it fetches exactly one document: the same
+[data.json](https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/data.json) that ships in the wheel.
+
+## The faults it finds
+
+The most common robots.txt failure is not a syntax error — it is a file that a
+person reads as correct and a parser reads as empty. 23 checks, of
+which these are the ones that cost real traffic:
+
+| code | what it means |
+| --- | --- |
+| `user_agent_is_a_ua_string` | `User-agent:` carrying a whole browser-style string. RFC 9309 matches a **product token**; that group matches nothing. |
+| `unknown-token` | a token no crawler in the table sends — usually a typo, `ChatGPT-user` for `ChatGPT-User`, or a name somebody invented |
+| `duplicate_group` | a second group for the same token; only the first one applies, the rest is decoration |
+| `noindex_in_robots` | `Noindex:` in robots.txt. Google stopped honouring it on 1 September 2019 and it never worked anywhere else. |
+| `unenforceable` | the rule is correct and the operator states robots.txt does not apply to that fetcher. You need an edge rule, and this says so. |
+| `crawl_delay` | `Crawl-delay:` is not in RFC 9309 and Google, OpenAI and Anthropic all ignore it |
+| `path_without_slash` | `Disallow: admin` does not match `/admin` |
+| `html_response` / `empty` / `bom` | the file never parsed at all |
+
+Every finding carries a line number, what was found, and a `fix` string you can
+paste.
+
+## Command line
+
+```sh
+ai-crawler-robots lint robots.txt                       # the faults, worst first
+ai-crawler-robots audit robots.txt                      # every AI crawler: blocked / allowed / unenforceable
+ai-crawler-robots check robots.txt --agent GPTBot --path /blog/
+ai-crawler-robots diff old.txt new.txt                  # what changed by effect, not by text
+ai-crawler-robots policy block-ai-training              # print a ready-made stance
+ai-crawler-robots policy block-ai-training --merge robots.txt
+ai-crawler-robots explain GPTBot                        # who it is, what blocking costs
+```
+
+`lint` exits `1` when it finds an error, `0` otherwise, so it works as a CI
+gate:
+
+```sh
+ai-crawler-robots lint robots.txt --strict || exit 1
+```
+
+## `audit` — the question people actually have
+
+```sh
+$ ai-crawler-robots audit robots.txt
+robots.txt: 56 crawlers, 12 blocked, 39 allowed, 5 unenforceable
+
+BLOCKED (12)
+  openai-gptbot            GPTBot              OpenAI       ai-training
+  anthropic-claudebot      ClaudeBot           Anthropic    ai-training
+  ...
+ALLOWED, AND YOU PROBABLY MEANT TO BLOCK (7)
+  google-extended          Google-Extended     Google       ai-training
+  ...
+UNENFORCEABLE BY robots.txt (5)
+  These fetch on behalf of a user in a live session. Their operators state
+  robots.txt does not govern them. Blocking needs an edge rule.
+```
+
+The third group is the one that surprises people. A user-triggered fetcher is
+not a crawler, most operators say so in their own documentation, and a
+`Disallow` aimed at one is a line that will never fire.
+
+## `diff` — did that edit do anything?
+
+```sh
+$ ai-crawler-robots diff robots.old robots.new
+2 crawler(s) changed verdict for /
+  google-extended     allowed -> blocked
+  meta-externalagent  allowed -> blocked
+44 unchanged. Formatting-only changes are not reported.
+```
+
+Compares by **effect**. Reordering groups, changing whitespace and rewriting
+comments produce an empty diff, because none of them change what a crawler does.
+
+## Ready-made stances
+
+8 of them, the same bodies published at the index:
+
+- `allow-all` — Every crawler on this index is named and allowed. Use when you want maximum reach into search and assistants and have nothing to withhold
+- `block-ai-training` — Refuse the crawlers that feed model training. Keep the ones that put you in ChatGPT, Claude, Perplexity and Gemini answers
+- `block-all-ai` — Training, AI search, user-triggered fetches and corpus builders, all refused. Classic search engines still allowed
+- `block-datasets` — Refuse the crawlers whose output is a dataset other people train on: Common Crawl, AI2, Webz.io, Diffbot, ImagesiftBot
+- `allow-ai-search-only` — Be findable and citable in assistants without contributing to training corpora
+- `block-seo-tools` — Ahrefs, Semrush and friends. No user-facing consequence, and often the largest single slice of your bot traffic
+- `block-disputed` — The ones repeatedly reported as ignoring robots.txt. Included for completeness — expect to enforce this at the edge instead
+- `maximum-ai-visibility` — Allow every AI crawler and every search engine; refuse only SEO scrapers. For sites whose goal is to be found and cited by machines
+
+`--merge` adds a stance to an existing file **without touching the groups
+already in it**, so your Googlebot rules survive.
+
+## Where the data comes from
+
+The [AI Crawler Index](https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/) — an independent, non-commercial public
+reference. Every crawler record links to the operator's own documentation, and
+`explain` prints that link. Presence in the table means the operator documents
+that crawler; it is not an opinion about whether you should allow it.
+
+`refresh()` fetches today's table and caches it six hours, which is how often
+the index behind it is rebuilt:
+
+```python
+from ai_crawler_robots import refresh
+refresh()   # {'crawlers': 56, 'generated_at': '...'}
+```
+
+## Companion packages
+
+Same table, different questions:
+
+- [`ai-crawler-index`](https://pypi.org/project/ai-crawler-index/) — what does
+  this user-agent claim to be?
+- [`ai-crawler-verify`](https://pypi.org/project/ai-crawler-verify/) — is that
+  claim true? Checks the address against the operator's published ranges.
+- [`ai-crawler-logs`](https://pypi.org/project/ai-crawler-logs/) — who was
+  actually in your access log, and what to paste to act on it.
+
+## Licence
+
+Code MIT. Bundled data CC0-1.0. Independent and non-commercial; not affiliated
+with, endorsed by or speaking for any crawler operator named in the table.

ai_crawler_robots-1.0.0/pyproject.toml

@@ -0,0 +1,60 @@
+# GENERATED by playbooks/package_lane.py from lane.json. Do not edit:
+# every field below has a single source of truth in that file.
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ai-crawler-robots"
+version = "1.0.0"
+description = "Lint a robots.txt and audit it against 56 AI crawlers: which ones it really blocks, which rules do nothing, and which fetchers robots.txt cannot stop at all. RFC 9309, zero dependencies, no network."
+readme = "README.md"
+requires-python = ">=3.8"
+license = { text = "MIT" }
+authors = [{ name = "Pathwren" }]
+keywords = [
+    "robots.txt",
+    "robots",
+    "rfc9309",
+    "lint",
+    "linter",
+    "crawler",
+    "ai",
+    "gptbot",
+    "claudebot",
+    "seo",
+    "crawl",
+    "user-agent",
+    "offline",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Text Processing :: Linguistic",
+    "Typing :: Typed",
+]
+dependencies = []
+
+[project.scripts]
+ai-crawler-robots = "ai_crawler_robots:_main"
+
+[project.urls]
+"Homepage" = "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/"
+"Documentation" = "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/"
+"Data source" = "https://www.pathwren.workers.dev/c/pypi-registry/ai-crawler-robots/data.json"
+"Companion package" = "https://pypi.org/project/ai-crawler-index/"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+packages = ["ai_crawler_robots"]
+
+[tool.setuptools.package-data]
+ai_crawler_robots = ["data.json", "py.typed"]

ai_crawler_robots-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+