django-bots 1.2.0__tar.gz → 1.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {django_bots-1.2.0/src/django_bots.egg-info → django_bots-1.2.2}/PKG-INFO +1 -1
  2. {django_bots-1.2.0 → django_bots-1.2.2}/pyproject.toml +1 -1
  3. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/ai.py +13 -6
  4. django_bots-1.2.2/src/django_bots/data/VERSIONS.json +3 -0
  5. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/data/ai_robots.json +42 -0
  6. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/middleware.py +7 -8
  7. {django_bots-1.2.0 → django_bots-1.2.2/src/django_bots.egg-info}/PKG-INFO +1 -1
  8. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_ai.py +15 -0
  9. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_middleware.py +4 -0
  10. django_bots-1.2.0/src/django_bots/data/VERSIONS.json +0 -3
  11. {django_bots-1.2.0 → django_bots-1.2.2}/LICENSE +0 -0
  12. {django_bots-1.2.0 → django_bots-1.2.2}/README.md +0 -0
  13. {django_bots-1.2.0 → django_bots-1.2.2}/setup.cfg +0 -0
  14. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/__init__.py +0 -0
  15. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/apps.py +0 -0
  16. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/checks.py +0 -0
  17. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/conf.py +0 -0
  18. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/crawlers.py +0 -0
  19. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/data/LICENSE-ai-robots-txt +0 -0
  20. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/data/ai_categories.json +0 -0
  21. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/py.typed +0 -0
  22. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templates/django_bots/robots.txt +0 -0
  23. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templatetags/__init__.py +0 -0
  24. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templatetags/bots.py +0 -0
  25. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/templatetags/user_agents.py +0 -0
  26. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/useragent.py +0 -0
  27. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/utils.py +0 -0
  28. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots/views.py +0 -0
  29. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/SOURCES.txt +0 -0
  30. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/dependency_links.txt +0 -0
  31. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/requires.txt +0 -0
  32. {django_bots-1.2.0 → django_bots-1.2.2}/src/django_bots.egg-info/top_level.txt +0 -0
  33. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_checks.py +0 -0
  34. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_conf.py +0 -0
  35. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_crawlers.py +0 -0
  36. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_scripts.py +0 -0
  37. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_templatetags.py +0 -0
  38. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_useragent.py +0 -0
  39. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_useragent_compat.py +0 -0
  40. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_utils.py +0 -0
  41. {django_bots-1.2.0 → django_bots-1.2.2}/tests/test_views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-bots
3
- Version: 1.2.0
3
+ Version: 1.2.2
4
4
  Summary: User-agent parsing, crawler detection and AI bot blocking for Django.
5
5
  Author-email: M4p4 <homejr@protonmail.com>
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ requires = [ "setuptools>=77" ]
4
4
 
5
5
  [project]
6
6
  name = "django-bots"
7
- version = "1.2.0"
7
+ version = "1.2.2"
8
8
  description = "User-agent parsing, crawler detection and AI bot blocking for Django."
9
9
  readme = "README.md"
10
10
  keywords = [ "ai", "bots", "crawlers", "django", "robots.txt", "user-agents" ]
@@ -28,6 +28,7 @@ __all__ = [
28
28
  "categorize",
29
29
  "is_ai_bot",
30
30
  "match_ai_bot",
31
+ "match_ai_bots",
31
32
  "robots_rules",
32
33
  ]
33
34
 
@@ -127,7 +128,7 @@ URL_OR_EMAIL = re.compile(
127
128
  @lru_cache(maxsize=1)
128
129
  def _get_matcher(
129
130
  cache_size: int, allow: tuple[str, ...], extra: tuple[str, ...]
130
- ) -> Callable[[str], str | None]:
131
+ ) -> Callable[[str], tuple[str, ...]]:
131
132
  """Build the matcher, rebuilt whenever one of the settings changes."""
132
133
  # Upstream lists some names in two spellings, so the first one is reported.
133
134
  names = ai_bot_names()
@@ -143,17 +144,17 @@ def _get_matcher(
143
144
  # An empty alternation matches everything, so fall back to a never-matching regex.
144
145
  regex = re.compile(rf"\b(?:{alternation})\b" if canonical else "(?!)", re.I)
145
146
 
146
- def match(ua_string: str) -> str | None:
147
- found = regex.search(URL_OR_EMAIL.sub(" ", ua_string))
148
- return canonical[found.group().lower()] if found else None
147
+ def match(ua_string: str) -> tuple[str, ...]:
148
+ found = regex.finditer(URL_OR_EMAIL.sub(" ", ua_string))
149
+ return tuple(dict.fromkeys(canonical[m.group().lower()] for m in found))
149
150
 
150
151
  if cache_size > 0:
151
152
  return lru_cache(maxsize=cache_size)(match)
152
153
  return match
153
154
 
154
155
 
155
- def match_ai_bot(ua_string: str) -> str | None:
156
- """Return the name of the AI bot the user-agent string matches, or ``None``."""
156
+ def match_ai_bots(ua_string: str) -> tuple[str, ...]:
157
+ """Return every AI bot name the user-agent string matches, in the order they appear."""
157
158
  matcher = _get_matcher(
158
159
  bots_settings.UA_CACHE_SIZE,
159
160
  tuple(bots_settings.AI_ALLOW),
@@ -162,6 +163,12 @@ def match_ai_bot(ua_string: str) -> str | None:
162
163
  return matcher(ua_string[:UA_MAX_LENGTH])
163
164
 
164
165
 
166
+ def match_ai_bot(ua_string: str) -> str | None:
167
+ """Return the name of the first AI bot the user-agent string matches, or ``None``."""
168
+ names = match_ai_bots(ua_string)
169
+ return names[0] if names else None
170
+
171
+
165
172
  def is_ai_bot(ua_string: str) -> bool:
166
173
  """Whether the user-agent string matches an AI bot."""
167
174
  return match_ai_bot(ua_string) is not None
@@ -0,0 +1,3 @@
1
+ {
2
+ "ai_robots_txt": "v2.0"
3
+ }
@@ -6,6 +6,13 @@
6
6
  "frequency": "Unclear at this time.",
7
7
  "description": "AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search results. More info can be found at https://knownagents.com/agents/addsearchbot"
8
8
  },
9
+ "AgentDataBot": {
10
+ "operator": "Unclear at this time.",
11
+ "respect": "Unclear at this time.",
12
+ "function": "AI Data Providers",
13
+ "frequency": "Unclear at this time.",
14
+ "description": "AgentDataBot visits public web pages to identify technology stacks, company details, and team members. The data builds AgentData's company-intelligence database for AI ag… More info can be found at https://knownagents.com/agents/agentdatabot"
15
+ },
9
16
  "AgentTimes": {
10
17
  "operator": "[The Agent Times](https://theagenttimes.com/about)",
11
18
  "respect": "Unclear at this time.",
@@ -181,6 +188,13 @@
181
188
  "frequency": "Unclear at this time.",
182
189
  "description": "bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi… More info can be found at https://knownagents.com/agents/bigsur-ai"
183
190
  },
191
+ "BixelBot": {
192
+ "operator": "Unclear at this time.",
193
+ "respect": "Unclear at this time.",
194
+ "function": "AI Data Providers",
195
+ "frequency": "Unclear at this time.",
196
+ "description": "BixelBot collects public company and market signals for Bixel's structured company-data platform, which supplies verified business information to AI agents and developers… More info can be found at https://knownagents.com/agents/bixelbot"
197
+ },
184
198
  "Bravebot": {
185
199
  "operator": "https://safe.search.brave.com/help/brave-search-crawler",
186
200
  "respect": "Yes",
@@ -293,6 +307,13 @@
293
307
  "frequency": "Unclear at this time.",
294
308
  "description": "AutoRAG is an all-in-one AI search solution."
295
309
  },
310
+ "CloudflareBrowserRenderingCrawler": {
311
+ "operator": "Cloudflare that returns rendered website content for research, monitoring, and AI data workflows",
312
+ "respect": "Unclear at this time.",
313
+ "function": "AI Data Providers",
314
+ "frequency": "Unclear at this time.",
315
+ "description": "CloudflareBrowserRenderingCrawler is a web crawler operated by Cloudflare that returns rendered website content for research, monitoring, and AI data workflows. More info can be found at https://knownagents.com/agents/cloudflarebrowserrenderingcrawler"
316
+ },
296
317
  "CloudVertexBot": {
297
318
  "operator": "Unclear at this time.",
298
319
  "respect": "Unclear at this time.",
@@ -651,6 +672,20 @@
651
672
  "frequency": "Unclear at this time.",
652
673
  "description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://knownagents.com/agents/kangaroo-bot"
653
674
  },
675
+ "KeenableBot": {
676
+ "operator": "[Keenable](https://keenable.ai/)",
677
+ "respect": "Unclear at this time.",
678
+ "function": "AI Search Crawlers",
679
+ "frequency": "Unclear at this time.",
680
+ "description": "KeenableBot is a web crawler operated by Keenable for indexing web content for AI search."
681
+ },
682
+ "Kimi-Agent": {
683
+ "operator": "Moonshot AI that browses websites while carrying out user-directed research and other tasks",
684
+ "respect": "Unclear at this time.",
685
+ "function": "AI Agents",
686
+ "frequency": "Unclear at this time.",
687
+ "description": "Kimi-Agent is a browser-enabled AI agent operated by Moonshot AI that browses websites while carrying out user-directed research and other tasks. More info can be found at https://knownagents.com/agents/kimi-agent"
688
+ },
654
689
  "Kimi-SearchBot": {
655
690
  "operator": "[Moonshot AI](https://www.moonshot.ai)",
656
691
  "respect": "[Yes](https://www.kimi.ai/policies/kimi-crawlers)",
@@ -966,6 +1001,13 @@
966
1001
  "function": "AI research crawler",
967
1002
  "respect": "Unclear at this time."
968
1003
  },
1004
+ "qodercli": {
1005
+ "operator": "Qoder that helps developers build, debug, and modify software from the terminal",
1006
+ "respect": "Unclear at this time.",
1007
+ "function": "AI Coding Agents",
1008
+ "frequency": "Unclear at this time.",
1009
+ "description": "qodercli is a command-line AI coding agent operated by Qoder that helps developers build, debug, and modify software from the terminal. More info can be found at https://knownagents.com/agents/qodercli"
1010
+ },
969
1011
  "QualifiedBot": {
970
1012
  "operator": "[Qualified](https://www.qualified.com)",
971
1013
  "respect": "Unclear at this time.",
@@ -17,7 +17,7 @@ from django.utils.cache import patch_cache_control
17
17
  from django.utils.functional import SimpleLazyObject
18
18
  from django.utils.module_loading import import_string
19
19
 
20
- from django_bots.ai import ai_bot_category, match_ai_bot
20
+ from django_bots.ai import ai_bot_category, match_ai_bots
21
21
  from django_bots.conf import bots_settings
22
22
  from django_bots.utils import get_user_agent
23
23
 
@@ -59,8 +59,9 @@ class UserAgentMiddleware:
59
59
  class AIBotBlockMiddleware:
60
60
  """Answer requests from AI bots with ``BOTS_AI_BLOCK_STATUS`` or ``BOTS_AI_BLOCK_VIEW``.
61
61
 
62
- Only bots in ``BOTS_AI_BLOCK_CATEGORIES`` are blocked. Only the raw user-agent
63
- string is checked, so the user agent is never fully parsed.
62
+ A request is blocked when any AI bot name it matches is in
63
+ ``BOTS_AI_BLOCK_CATEGORIES``. Only the raw user-agent string is checked, so the
64
+ user agent is never fully parsed.
64
65
  Paths in ``BOTS_AI_BLOCK_EXEMPT_PATHS`` are never blocked. They're compared with
65
66
  ``request.path_info``, without the script prefix.
66
67
  """
@@ -115,11 +116,9 @@ class AIBotBlockMiddleware:
115
116
  def blocks(request: HttpRequest) -> bool:
116
117
  if request.path_info in bots_settings.AI_BLOCK_EXEMPT_PATHS:
117
118
  return False
118
- name = match_ai_bot(request.headers.get("user-agent", ""))
119
- return (
120
- name is not None
121
- and ai_bot_category(name) in bots_settings.AI_BLOCK_CATEGORIES
122
- )
119
+ names = match_ai_bots(request.headers.get("user-agent", ""))
120
+ blocked = bots_settings.AI_BLOCK_CATEGORIES
121
+ return any(ai_bot_category(name) in blocked for name in names)
123
122
 
124
123
  @staticmethod
125
124
  def forbidden() -> HttpResponse:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-bots
3
- Version: 1.2.0
3
+ Version: 1.2.2
4
4
  Summary: User-agent parsing, crawler detection and AI bot blocking for Django.
5
5
  Author-email: M4p4 <homejr@protonmail.com>
6
6
  License-Expression: MIT
@@ -16,6 +16,7 @@ from django_bots.ai import (
16
16
  categorize,
17
17
  is_ai_bot,
18
18
  match_ai_bot,
19
+ match_ai_bots,
19
20
  robots_rules,
20
21
  )
21
22
  from django_bots.conf import UA_MAX_LENGTH
@@ -381,3 +382,17 @@ def test_extra_names_are_training():
381
382
  )
382
383
  def test_user_agent_category(ua_string, expected):
383
384
  assert parse(ua_string).ai_bot_category == expected
385
+
386
+
387
+ @pytest.mark.parametrize(
388
+ ("ua_string", "expected"),
389
+ [
390
+ ("Operator/1.0 ClaudeBot/1.0", ("Operator", "ClaudeBot")),
391
+ ("GPTBot/1.2 gptbot GPTBot", ("GPTBot",)),
392
+ (GPTBOT, ("GPTBot",)),
393
+ (WINDOWS_CHROME, ()),
394
+ ],
395
+ )
396
+ def test_match_all_names(ua_string, expected):
397
+ assert match_ai_bots(ua_string) == expected
398
+ assert match_ai_bot(ua_string) == (expected[0] if expected else None)
@@ -166,6 +166,10 @@ CHATGPT_USER = (
166
166
  (["training"], CHATGPT_USER, 200),
167
167
  (["training"], GPTBOT, 403),
168
168
  ([], GPTBOT, 200),
169
+ (None, "Operator/1.0 ClaudeBot/1.0", 403),
170
+ (None, f"Code {GPTBOT}", 403),
171
+ (None, f"{CURSOR} GPTBot/1.2", 403),
172
+ (["training"], f"Operator/1.0 {CHATGPT_USER}", 200),
169
173
  ],
170
174
  )
171
175
  def test_block_categories(rf, settings, categories, user_agent, status):
@@ -1,3 +0,0 @@
1
- {
2
- "ai_robots_txt": "v1.52"
3
- }
File without changes
File without changes
File without changes