django-bots 1.0.0__py3-none-any.whl

django_bots/__init__.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import json
+from importlib.metadata import version
+from importlib.resources import files
+
+__version__ = version("django-bots")
+
+DATA_VERSIONS: dict[str, str] = {
+    **json.loads(files(__name__).joinpath("data", "VERSIONS.json").read_text("utf-8")),
+    "crawler_user_agents": version("crawler-user-agents"),
+    "ua_parser": version("ua-parser"),
+    "uap_core": version("ua-parser-builtins"),
+}
+"""Versions of the bundled and installed data sources."""

django_bots/ai.py

@@ -0,0 +1,94 @@
+"""Detect AI bots by user-agent string.
+
+The bot list is vendored from `ai.robots.txt
+<https://github.com/ai-robots-txt/ai.robots.txt>`_, copyright (c) 2024
+ai.robots.txt, released under the MIT license.
+
+Each name matches case-insensitively at word boundaries in the user-agent string,
+after URLs and email addresses are removed from it. Results are cached per string in
+an LRU of ``BOTS_UA_CACHE_SIZE`` entries.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Callable
+from functools import lru_cache
+from importlib.resources import files
+from typing import Any
+
+from django_bots.conf import bots_settings
+
+__all__ = ["ai_bot_names", "ai_bots", "is_ai_bot", "match_ai_bot", "robots_rules"]
+
+
+def _read_data(name: str) -> Any:
+    return json.loads(files("django_bots").joinpath("data", name).read_text("utf-8"))
+
+
+@lru_cache(maxsize=1)
+def ai_bots() -> dict[str, dict[str, Any]]:
+    """Return the vendored ai.robots.txt entries, keyed by bot name."""
+    bots: dict[str, dict[str, Any]] = _read_data("ai_robots.json")
+    return bots
+
+
+def ai_bot_names() -> list[str]:
+    """Return the AI bot names in use, after ``BOTS_AI_ALLOW`` and ``BOTS_AI_EXTRA``."""
+    allow = {name.lower() for name in bots_settings.AI_ALLOW}
+    names = [*ai_bots(), *bots_settings.AI_EXTRA]
+    return list(dict.fromkeys(name for name in names if name.lower() not in allow))
+
+
+# Contact links name the operator, not the bot: "openai.com" would match "OpenAI".
+URL_OR_EMAIL = re.compile(r"(?:https?://|www\.)[^\s;)]+|[\w.+-]+@[\w.-]+")
+
+
+@lru_cache(maxsize=1)
+def _get_matcher(
+    cache_size: int, allow: tuple[str, ...], extra: tuple[str, ...]
+) -> Callable[[str], str | None]:
+    """Build the matcher, rebuilt whenever one of the settings changes."""
+    # Upstream lists some names in two spellings, so the first one is reported.
+    canonical: dict[str, str] = {}
+    for name in ai_bot_names():
+        canonical.setdefault(name.lower(), name)
+    # Longest first, so a name wins over a shorter name it starts with.
+    alternation = "|".join(map(re.escape, sorted(canonical, key=len, reverse=True)))
+    # An empty alternation matches everything, so fall back to a never-matching regex.
+    regex = re.compile(rf"\b(?:{alternation})\b" if canonical else "(?!)", re.I)
+
+    def match(ua_string: str) -> str | None:
+        found = regex.search(URL_OR_EMAIL.sub(" ", ua_string))
+        return canonical[found.group().lower()] if found else None
+
+    if cache_size > 0:
+        return lru_cache(maxsize=cache_size)(match)
+    return match
+
+
+def match_ai_bot(ua_string: str) -> str | None:
+    """Return the name of the AI bot the user-agent string matches, or ``None``."""
+    matcher = _get_matcher(
+        bots_settings.UA_CACHE_SIZE,
+        tuple(bots_settings.AI_ALLOW),
+        tuple(bots_settings.AI_EXTRA),
+    )
+    return matcher(ua_string)
+
+
+def is_ai_bot(ua_string: str) -> bool:
+    """Whether the user-agent string matches an AI bot."""
+    return match_ai_bot(ua_string) is not None
+
+
+def robots_rules() -> str:
+    """Return a robots.txt group that disallows every AI bot in ``ai_bot_names()``.
+
+    Returns an empty string when no names are left.
+    """
+    names = ai_bot_names()
+    if not names:
+        return ""
+    return "\n".join([*(f"User-agent: {name}" for name in names), "Disallow: /"])

django_bots/apps.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from django.apps import AppConfig
+from django.core.checks import Tags, register
+
+from django_bots.checks import (
+    check_ai_block_view,
+    check_django_user_agents_installed,
+    check_user_agents_cache,
+)
+
+
+class DjangoBotsConfig(AppConfig):
+    name = "django_bots"
+    verbose_name = "Bots"
+
+    def ready(self) -> None:
+        register(check_user_agents_cache, Tags.compatibility)
+        register(check_django_user_agents_installed, Tags.compatibility)
+        register(check_ai_block_view)

django_bots/checks.py

@@ -0,0 +1,70 @@
+"""System checks for django-bots settings and settings left over from django-user-agents."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from django.apps import AppConfig, apps
+from django.conf import settings
+from django.core.checks import CheckMessage, Error, Warning
+from django.utils.module_loading import import_string
+
+from django_bots.conf import bots_settings
+
+__all__ = [
+    "check_ai_block_view",
+    "check_django_user_agents_installed",
+    "check_user_agents_cache",
+]
+
+
+def check_user_agents_cache(
+    app_configs: Sequence[AppConfig] | None, **kwargs: Any
+) -> list[CheckMessage]:
+    if not hasattr(settings, "USER_AGENTS_CACHE"):
+        return []
+    return [
+        Warning(
+            "USER_AGENTS_CACHE is set, but django-bots doesn't use Django's cache framework.",
+            hint="Remove the setting. Parsed user agents are cached in process, sized by BOTS_UA_CACHE_SIZE.",
+            id="django_bots.W001",
+        )
+    ]
+
+
+def check_django_user_agents_installed(
+    app_configs: Sequence[AppConfig] | None, **kwargs: Any
+) -> list[CheckMessage]:
+    if not apps.is_installed("django_user_agents"):
+        return []
+    return [
+        Error(
+            "django_user_agents and django_bots are both in INSTALLED_APPS.",
+            hint="Remove django_user_agents. Both apps provide the user_agents template library.",
+            id="django_bots.E001",
+        )
+    ]
+
+
+def check_ai_block_view(
+    app_configs: Sequence[AppConfig] | None, **kwargs: Any
+) -> list[CheckMessage]:
+    path = bots_settings.AI_BLOCK_VIEW
+    if path is None:
+        return []
+    try:
+        view = import_string(path)
+    except ImportError as error:
+        problem = str(error)
+    else:
+        if callable(view):
+            return []
+        problem = f"{path!r} is not callable."
+    return [
+        Error(
+            f"BOTS_AI_BLOCK_VIEW can't be used: {problem}",
+            hint="Set it to the dotted path of a view, or to None for the default response.",
+            id="django_bots.E002",
+        )
+    ]

django_bots/conf.py

@@ -0,0 +1,66 @@
+"""Settings for django-bots, each read from Django settings with a default.
+
+Values are looked up on every access, so ``override_settings`` works without any
+cache to clear.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TypeVar
+
+from django.conf import settings
+
+__all__ = ["bots_settings"]
+
+T = TypeVar("T")
+
+
+def _get(name: str, default: T) -> T:
+    value: T = getattr(settings, f"BOTS_{name}", default)
+    return value
+
+
+class BotsSettings:
+    @property
+    def UA_CACHE_SIZE(self) -> int:
+        """LRU size for parsed user agents and bot checks."""
+        return _get("UA_CACHE_SIZE", 2048)
+
+    @property
+    def CRAWLER_EXTRA(self) -> Sequence[str]:
+        """Extra regex patterns to treat as crawlers."""
+        return _get("CRAWLER_EXTRA", ())
+
+    @property
+    def CRAWLER_IGNORE(self) -> Sequence[str]:
+        """Crawler patterns to leave out of detection."""
+        return _get("CRAWLER_IGNORE", ())
+
+    @property
+    def AI_ALLOW(self) -> Sequence[str]:
+        """AI bot names to leave out of robots.txt rules and blocking."""
+        return _get("AI_ALLOW", ())
+
+    @property
+    def AI_EXTRA(self) -> Sequence[str]:
+        """Extra bot names to treat as AI bots."""
+        return _get("AI_EXTRA", ())
+
+    @property
+    def AI_BLOCK_STATUS(self) -> int:
+        """Status code of the default block response."""
+        return _get("AI_BLOCK_STATUS", 403)
+
+    @property
+    def AI_BLOCK_VIEW(self) -> str | None:
+        """Dotted path to a view that renders the block response instead."""
+        return _get("AI_BLOCK_VIEW", None)
+
+    @property
+    def AI_BLOCK_EXEMPT_PATHS(self) -> Sequence[str]:
+        """Paths the blocking middleware never blocks."""
+        return _get("AI_BLOCK_EXEMPT_PATHS", ("/robots.txt",))
+
+
+bots_settings = BotsSettings()

django_bots/crawlers.py

@@ -0,0 +1,56 @@
+"""Detect crawlers by user-agent string.
+
+The patterns come from the `crawler-user-agents
+<https://github.com/monperrus/crawler-user-agents>`_ package. They're compiled into
+one case-insensitive regex on first use, and results are cached per user-agent
+string in an LRU of ``BOTS_UA_CACHE_SIZE`` entries.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from functools import lru_cache
+
+from crawleruseragents import CRAWLER_USER_AGENTS_DATA
+
+from django_bots.conf import bots_settings
+
+__all__ = ["crawler_patterns", "is_crawler"]
+
+
+def crawler_patterns() -> list[str]:
+    """Return the crawler patterns in use, after ``BOTS_CRAWLER_IGNORE`` and ``BOTS_CRAWLER_EXTRA``."""
+    ignore = set(bots_settings.CRAWLER_IGNORE)
+    patterns = [
+        entry["pattern"]
+        for entry in CRAWLER_USER_AGENTS_DATA
+        if entry["pattern"] not in ignore
+    ]
+    return [*patterns, *bots_settings.CRAWLER_EXTRA]
+
+
+@lru_cache(maxsize=1)
+def _get_matcher(
+    cache_size: int, extra: tuple[str, ...], ignore: tuple[str, ...]
+) -> Callable[[str], bool]:
+    """Build the matcher, rebuilt whenever one of the settings changes."""
+    # An empty alternation matches everything, so fall back to a never-matching regex.
+    regex = re.compile("|".join(crawler_patterns()) or "(?!)", re.IGNORECASE)
+
+    def match(ua_string: str) -> bool:
+        return regex.search(ua_string) is not None
+
+    if cache_size > 0:
+        return lru_cache(maxsize=cache_size)(match)
+    return match
+
+
+def is_crawler(ua_string: str) -> bool:
+    """Whether the user-agent string matches a crawler pattern."""
+    matcher = _get_matcher(
+        bots_settings.UA_CACHE_SIZE,
+        tuple(bots_settings.CRAWLER_EXTRA),
+        tuple(bots_settings.CRAWLER_IGNORE),
+    )
+    return matcher(ua_string)

django_bots/data/LICENSE-ai-robots-txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 ai.robots.txt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

django_bots/data/VERSIONS.json

@@ -0,0 +1,3 @@
+{
+  "ai_robots_txt": "v1.52"
+}