django-bots 1.2.0__tar.gz → 1.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_bots-1.2.0/src/django_bots.egg-info → django_bots-1.2.2}/PKG-INFO +1 -1
- {django_bots-1.2.0 → django_bots-1.2.2}/pyproject.toml +1 -1
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/ai.py +13 -6
- django_bots-1.2.2/src/django_bots/data/VERSIONS.json +3 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/data/ai_robots.json +42 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/middleware.py +7 -8
- {django_bots-1.2.0 → django_bots-1.2.2/src/django_bots.egg-info}/PKG-INFO +1 -1
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_ai.py +15 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_middleware.py +4 -0
- django_bots-1.2.0/src/django_bots/data/VERSIONS.json +0 -3
- {django_bots-1.2.0 → django_bots-1.2.2}/LICENSE +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/README.md +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/setup.cfg +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/__init__.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/apps.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/checks.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/conf.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/crawlers.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/data/LICENSE-ai-robots-txt +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/data/ai_categories.json +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/py.typed +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templates/django_bots/robots.txt +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templatetags/__init__.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templatetags/bots.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templatetags/user_agents.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/useragent.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/utils.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/views.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/SOURCES.txt +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/dependency_links.txt +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/requires.txt +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/top_level.txt +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_checks.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_conf.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_crawlers.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_scripts.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_templatetags.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_useragent.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_useragent_compat.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_utils.py +0 -0
- {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_views.py +0 -0
|
@@ -4,7 +4,7 @@ requires = [ "setuptools>=77" ]
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "django-bots"
|
|
7
|
-
version = "1.2.
|
|
7
|
+
version = "1.2.2"
|
|
8
8
|
description = "User-agent parsing, crawler detection and AI bot blocking for Django."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
keywords = [ "ai", "bots", "crawlers", "django", "robots.txt", "user-agents" ]
|
|
@@ -28,6 +28,7 @@ __all__ = [
|
|
|
28
28
|
"categorize",
|
|
29
29
|
"is_ai_bot",
|
|
30
30
|
"match_ai_bot",
|
|
31
|
+
"match_ai_bots",
|
|
31
32
|
"robots_rules",
|
|
32
33
|
]
|
|
33
34
|
|
|
@@ -127,7 +128,7 @@ URL_OR_EMAIL = re.compile(
|
|
|
127
128
|
@lru_cache(maxsize=1)
|
|
128
129
|
def _get_matcher(
|
|
129
130
|
cache_size: int, allow: tuple[str, ...], extra: tuple[str, ...]
|
|
130
|
-
) -> Callable[[str], str
|
|
131
|
+
) -> Callable[[str], tuple[str, ...]]:
|
|
131
132
|
"""Build the matcher, rebuilt whenever one of the settings changes."""
|
|
132
133
|
# Upstream lists some names in two spellings, so the first one is reported.
|
|
133
134
|
names = ai_bot_names()
|
|
@@ -143,17 +144,17 @@ def _get_matcher(
|
|
|
143
144
|
# An empty alternation matches everything, so fall back to a never-matching regex.
|
|
144
145
|
regex = re.compile(rf"\b(?:{alternation})\b" if canonical else "(?!)", re.I)
|
|
145
146
|
|
|
146
|
-
def match(ua_string: str) -> str
|
|
147
|
-
found = regex.
|
|
148
|
-
return canonical[
|
|
147
|
+
def match(ua_string: str) -> tuple[str, ...]:
|
|
148
|
+
found = regex.finditer(URL_OR_EMAIL.sub(" ", ua_string))
|
|
149
|
+
return tuple(dict.fromkeys(canonical[m.group().lower()] for m in found))
|
|
149
150
|
|
|
150
151
|
if cache_size > 0:
|
|
151
152
|
return lru_cache(maxsize=cache_size)(match)
|
|
152
153
|
return match
|
|
153
154
|
|
|
154
155
|
|
|
155
|
-
def
|
|
156
|
-
"""Return
|
|
156
|
+
def match_ai_bots(ua_string: str) -> tuple[str, ...]:
|
|
157
|
+
"""Return every AI bot name the user-agent string matches, in the order they appear."""
|
|
157
158
|
matcher = _get_matcher(
|
|
158
159
|
bots_settings.UA_CACHE_SIZE,
|
|
159
160
|
tuple(bots_settings.AI_ALLOW),
|
|
@@ -162,6 +163,12 @@ def match_ai_bot(ua_string: str) -> str | None:
|
|
|
162
163
|
return matcher(ua_string[:UA_MAX_LENGTH])
|
|
163
164
|
|
|
164
165
|
|
|
166
|
+
def match_ai_bot(ua_string: str) -> str | None:
|
|
167
|
+
"""Return the name of the first AI bot the user-agent string matches, or ``None``."""
|
|
168
|
+
names = match_ai_bots(ua_string)
|
|
169
|
+
return names[0] if names else None
|
|
170
|
+
|
|
171
|
+
|
|
165
172
|
def is_ai_bot(ua_string: str) -> bool:
|
|
166
173
|
"""Whether the user-agent string matches an AI bot."""
|
|
167
174
|
return match_ai_bot(ua_string) is not None
|
|
@@ -6,6 +6,13 @@
|
|
|
6
6
|
"frequency": "Unclear at this time.",
|
|
7
7
|
"description": "AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search results. More info can be found at https://knownagents.com/agents/addsearchbot"
|
|
8
8
|
},
|
|
9
|
+
"AgentDataBot": {
|
|
10
|
+
"operator": "Unclear at this time.",
|
|
11
|
+
"respect": "Unclear at this time.",
|
|
12
|
+
"function": "AI Data Providers",
|
|
13
|
+
"frequency": "Unclear at this time.",
|
|
14
|
+
"description": "AgentDataBot visits public web pages to identify technology stacks, company details, and team members. The data builds AgentData's company-intelligence database for AI ag… More info can be found at https://knownagents.com/agents/agentdatabot"
|
|
15
|
+
},
|
|
9
16
|
"AgentTimes": {
|
|
10
17
|
"operator": "[The Agent Times](https://theagenttimes.com/about)",
|
|
11
18
|
"respect": "Unclear at this time.",
|
|
@@ -181,6 +188,13 @@
|
|
|
181
188
|
"frequency": "Unclear at this time.",
|
|
182
189
|
"description": "bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi… More info can be found at https://knownagents.com/agents/bigsur-ai"
|
|
183
190
|
},
|
|
191
|
+
"BixelBot": {
|
|
192
|
+
"operator": "Unclear at this time.",
|
|
193
|
+
"respect": "Unclear at this time.",
|
|
194
|
+
"function": "AI Data Providers",
|
|
195
|
+
"frequency": "Unclear at this time.",
|
|
196
|
+
"description": "BixelBot collects public company and market signals for Bixel's structured company-data platform, which supplies verified business information to AI agents and developers… More info can be found at https://knownagents.com/agents/bixelbot"
|
|
197
|
+
},
|
|
184
198
|
"Bravebot": {
|
|
185
199
|
"operator": "https://safe.search.brave.com/help/brave-search-crawler",
|
|
186
200
|
"respect": "Yes",
|
|
@@ -293,6 +307,13 @@
|
|
|
293
307
|
"frequency": "Unclear at this time.",
|
|
294
308
|
"description": "AutoRAG is an all-in-one AI search solution."
|
|
295
309
|
},
|
|
310
|
+
"CloudflareBrowserRenderingCrawler": {
|
|
311
|
+
"operator": "Cloudflare that returns rendered website content for research, monitoring, and AI data workflows",
|
|
312
|
+
"respect": "Unclear at this time.",
|
|
313
|
+
"function": "AI Data Providers",
|
|
314
|
+
"frequency": "Unclear at this time.",
|
|
315
|
+
"description": "CloudflareBrowserRenderingCrawler is a web crawler operated by Cloudflare that returns rendered website content for research, monitoring, and AI data workflows. More info can be found at https://knownagents.com/agents/cloudflarebrowserrenderingcrawler"
|
|
316
|
+
},
|
|
296
317
|
"CloudVertexBot": {
|
|
297
318
|
"operator": "Unclear at this time.",
|
|
298
319
|
"respect": "Unclear at this time.",
|
|
@@ -651,6 +672,20 @@
|
|
|
651
672
|
"frequency": "Unclear at this time.",
|
|
652
673
|
"description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://knownagents.com/agents/kangaroo-bot"
|
|
653
674
|
},
|
|
675
|
+
"KeenableBot": {
|
|
676
|
+
"operator": "[Keenable](https://keenable.ai/)",
|
|
677
|
+
"respect": "Unclear at this time.",
|
|
678
|
+
"function": "AI Search Crawlers",
|
|
679
|
+
"frequency": "Unclear at this time.",
|
|
680
|
+
"description": "KeenableBot is a web crawler operated by Keenable for indexing web content for AI search."
|
|
681
|
+
},
|
|
682
|
+
"Kimi-Agent": {
|
|
683
|
+
"operator": "Moonshot AI that browses websites while carrying out user-directed research and other tasks",
|
|
684
|
+
"respect": "Unclear at this time.",
|
|
685
|
+
"function": "AI Agents",
|
|
686
|
+
"frequency": "Unclear at this time.",
|
|
687
|
+
"description": "Kimi-Agent is a browser-enabled AI agent operated by Moonshot AI that browses websites while carrying out user-directed research and other tasks. More info can be found at https://knownagents.com/agents/kimi-agent"
|
|
688
|
+
},
|
|
654
689
|
"Kimi-SearchBot": {
|
|
655
690
|
"operator": "[Moonshot AI](https://www.moonshot.ai)",
|
|
656
691
|
"respect": "[Yes](https://www.kimi.ai/policies/kimi-crawlers)",
|
|
@@ -966,6 +1001,13 @@
|
|
|
966
1001
|
"function": "AI research crawler",
|
|
967
1002
|
"respect": "Unclear at this time."
|
|
968
1003
|
},
|
|
1004
|
+
"qodercli": {
|
|
1005
|
+
"operator": "Qoder that helps developers build, debug, and modify software from the terminal",
|
|
1006
|
+
"respect": "Unclear at this time.",
|
|
1007
|
+
"function": "AI Coding Agents",
|
|
1008
|
+
"frequency": "Unclear at this time.",
|
|
1009
|
+
"description": "qodercli is a command-line AI coding agent operated by Qoder that helps developers build, debug, and modify software from the terminal. More info can be found at https://knownagents.com/agents/qodercli"
|
|
1010
|
+
},
|
|
969
1011
|
"QualifiedBot": {
|
|
970
1012
|
"operator": "[Qualified](https://www.qualified.com)",
|
|
971
1013
|
"respect": "Unclear at this time.",
|
|
@@ -17,7 +17,7 @@ from django.utils.cache import patch_cache_control
|
|
|
17
17
|
from django.utils.functional import SimpleLazyObject
|
|
18
18
|
from django.utils.module_loading import import_string
|
|
19
19
|
|
|
20
|
-
from django_bots.ai import ai_bot_category,
|
|
20
|
+
from django_bots.ai import ai_bot_category, match_ai_bots
|
|
21
21
|
from django_bots.conf import bots_settings
|
|
22
22
|
from django_bots.utils import get_user_agent
|
|
23
23
|
|
|
@@ -59,8 +59,9 @@ class UserAgentMiddleware:
|
|
|
59
59
|
class AIBotBlockMiddleware:
|
|
60
60
|
"""Answer requests from AI bots with ``BOTS_AI_BLOCK_STATUS`` or ``BOTS_AI_BLOCK_VIEW``.
|
|
61
61
|
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
A request is blocked when any AI bot name it matches is in
|
|
63
|
+
``BOTS_AI_BLOCK_CATEGORIES``. Only the raw user-agent string is checked, so the
|
|
64
|
+
user agent is never fully parsed.
|
|
64
65
|
Paths in ``BOTS_AI_BLOCK_EXEMPT_PATHS`` are never blocked. They're compared with
|
|
65
66
|
``request.path_info``, without the script prefix.
|
|
66
67
|
"""
|
|
@@ -115,11 +116,9 @@ class AIBotBlockMiddleware:
|
|
|
115
116
|
def blocks(request: HttpRequest) -> bool:
|
|
116
117
|
if request.path_info in bots_settings.AI_BLOCK_EXEMPT_PATHS:
|
|
117
118
|
return False
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
and ai_bot_category(name) in bots_settings.AI_BLOCK_CATEGORIES
|
|
122
|
-
)
|
|
119
|
+
names = match_ai_bots(request.headers.get("user-agent", ""))
|
|
120
|
+
blocked = bots_settings.AI_BLOCK_CATEGORIES
|
|
121
|
+
return any(ai_bot_category(name) in blocked for name in names)
|
|
123
122
|
|
|
124
123
|
@staticmethod
|
|
125
124
|
def forbidden() -> HttpResponse:
|
|
@@ -16,6 +16,7 @@ from django_bots.ai import (
|
|
|
16
16
|
categorize,
|
|
17
17
|
is_ai_bot,
|
|
18
18
|
match_ai_bot,
|
|
19
|
+
match_ai_bots,
|
|
19
20
|
robots_rules,
|
|
20
21
|
)
|
|
21
22
|
from django_bots.conf import UA_MAX_LENGTH
|
|
@@ -381,3 +382,17 @@ def test_extra_names_are_training():
|
|
|
381
382
|
)
|
|
382
383
|
def test_user_agent_category(ua_string, expected):
|
|
383
384
|
assert parse(ua_string).ai_bot_category == expected
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
@pytest.mark.parametrize(
|
|
388
|
+
("ua_string", "expected"),
|
|
389
|
+
[
|
|
390
|
+
("Operator/1.0 ClaudeBot/1.0", ("Operator", "ClaudeBot")),
|
|
391
|
+
("GPTBot/1.2 gptbot GPTBot", ("GPTBot",)),
|
|
392
|
+
(GPTBOT, ("GPTBot",)),
|
|
393
|
+
(WINDOWS_CHROME, ()),
|
|
394
|
+
],
|
|
395
|
+
)
|
|
396
|
+
def test_match_all_names(ua_string, expected):
|
|
397
|
+
assert match_ai_bots(ua_string) == expected
|
|
398
|
+
assert match_ai_bot(ua_string) == (expected[0] if expected else None)
|
|
@@ -166,6 +166,10 @@ CHATGPT_USER = (
|
|
|
166
166
|
(["training"], CHATGPT_USER, 200),
|
|
167
167
|
(["training"], GPTBOT, 403),
|
|
168
168
|
([], GPTBOT, 200),
|
|
169
|
+
(None, "Operator/1.0 ClaudeBot/1.0", 403),
|
|
170
|
+
(None, f"Code {GPTBOT}", 403),
|
|
171
|
+
(None, f"{CURSOR} GPTBot/1.2", 403),
|
|
172
|
+
(["training"], f"Operator/1.0 {CHATGPT_USER}", 200),
|
|
169
173
|
],
|
|
170
174
|
)
|
|
171
175
|
def test_block_categories(rf, settings, categories, user_agent, status):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|