agent-census 0.0.1__py3-none-any.whl

agent_census/__init__.py

@@ -0,0 +1,23 @@
+__version__ = "0.0.1"
+__author__ = "Mark Nottingham <mnot@mnot.net>"
+__copyright__ = """\
+Copyright (c) Mark Nottingham
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+"""

agent_census/__main__.py

@@ -0,0 +1,10 @@
+"""Enable ``python -m agent_census``."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

agent_census/classify/__init__.py

@@ -0,0 +1,53 @@
+"""Client classification: independent rule-based classifiers + a combiner.
+
+The public entry point is :func:`classify_client`, which runs every registered
+classifier over a client's features and combines their signals into a single
+:class:`~agent_census.model.Classification` (primary kind plus tags).
+"""
+
+from __future__ import annotations
+
+from ..model import BotVerification, Classification, ClientFeatures, ComplianceReport, Signal
+from .base import Classifier
+from .combiner import DEFAULT_UNKNOWN_THRESHOLD, combine
+from .registry import all_classifiers
+
+
+def run_classifiers(features: ClientFeatures) -> list[Signal]:
+    """Collect signals from every classifier for one client."""
+    signals: list[Signal] = []
+    for classifier in all_classifiers():
+        signals.extend(classifier.evaluate(features))
+    return signals
+
+
+def classify_client(
+    features: ClientFeatures,
+    *,
+    compliance: ComplianceReport | None = None,
+    verification: BotVerification | None = None,
+    datacenter: bool = False,
+    unknown_threshold: float = DEFAULT_UNKNOWN_THRESHOLD,
+    keep_signals: bool = True,
+) -> Classification:
+    """Run all classifiers over ``features`` and combine into a verdict."""
+    signals = run_classifiers(features)
+    return combine(
+        signals,
+        features,
+        compliance=compliance,
+        verification=verification,
+        datacenter=datacenter,
+        unknown_threshold=unknown_threshold,
+        keep_signals=keep_signals,
+    )
+
+
+__all__ = [
+    "Classifier",
+    "classify_client",
+    "run_classifiers",
+    "combine",
+    "all_classifiers",
+    "DEFAULT_UNKNOWN_THRESHOLD",
+]

agent_census/classify/ai_crawler.py

@@ -0,0 +1,13 @@
+"""Declared AI / LLM data-gathering crawlers (GPTBot, ClaudeBot, Google-Extended, ...)."""
+
+from __future__ import annotations
+
+from ..model import Kind
+from .known_bot import KnownBotClassifier
+
+
+class AiCrawlerClassifier(KnownBotClassifier):
+    label = Kind.AI_CRAWLER
+    name = "ai_crawler"
+    category = "ai_crawler"
+    descriptor = "AI / LLM crawler"

agent_census/classify/app.py

@@ -0,0 +1,46 @@
+"""Native mobile / desktop app clients.
+
+Not a browser and not a crawler: a first-party app making requests through a
+platform networking stack (Apple's CFNetwork, Flutter's dart:io, …) or a named
+networking framework. The User-Agent names the stack, not a browser engine, so
+these otherwise fall through to UNKNOWN despite being ordinary app traffic.
+"""
+
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+
+from ..dataload import load_list
+from ..model import ClientFeatures, Kind, Signal
+from .base import Classifier
+from .tags import identifies_as_known_agent
+
+_APP_TOKENS = re.compile("|".join(re.escape(token) for token in load_list("app_clients")), re.I)
+
+
+@lru_cache(maxsize=16384)
+def app_stack_token(ua: str | None) -> str | None:
+    """The native-app networking token in the UA, or None."""
+    if not ua:
+        return None
+    match = _APP_TOKENS.search(ua)
+    return match.group(0) if match else None
+
+
+class AppClientClassifier(Classifier):
+    label = Kind.APP
+    name = "app"
+
+    def evaluate(self, features: ClientFeatures) -> list[Signal]:
+        # A platform networking stack is the *weakest* identity: if the UA also
+        # names a feed reader, crawler, or bot, that more specific identity wins
+        # (a feed reader on CFNetwork is a feed reader, not just "an app").
+        if identifies_as_known_agent(features):
+            return []
+        token = app_stack_token(features.user_agent)
+        if token is None:
+            return []
+        # A platform networking stack is an unambiguous native-app identity, so one
+        # match is enough to carry it past the unknown threshold on its own.
+        return [self._signal(0.65, [f"native-app networking stack in User-Agent ({token})"])]

agent_census/classify/archiver.py

@@ -0,0 +1,13 @@
+"""Web-archiving / preservation crawlers (Internet Archive / Wayback Machine)."""
+
+from __future__ import annotations
+
+from ..model import Kind
+from .known_bot import KnownBotClassifier
+
+
+class ArchiverClassifier(KnownBotClassifier):
+    label = Kind.ARCHIVER
+    name = "archiver"
+    category = "archiver"
+    descriptor = "web archiver"

agent_census/classify/base.py

@@ -0,0 +1,37 @@
+"""The classifier contract.
+
+A classifier is a pure function of :class:`ClientFeatures`: it reads only the
+feature vector (and its own static data lists) and emits zero or more
+:class:`Signal` votes for the kind it argues for. It never imports another
+classifier and never sees the final decision — that keeps each one independently
+testable and free to evolve. "This client is NOT a browser" is expressed simply
+by the browser classifier not firing.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from ..model import ClientFeatures, Kind, Signal
+
+
+class Classifier(ABC):
+    """Argues, from features alone, that a client is of a particular kind."""
+
+    #: the kind this classifier votes for
+    label: Kind = Kind.UNKNOWN
+    #: short stable name, recorded on each signal for provenance
+    name: str = ""
+
+    @abstractmethod
+    def evaluate(self, features: ClientFeatures) -> list[Signal]:
+        """Return signals supporting :attr:`label` (possibly empty)."""
+
+    def _signal(self, confidence: float, evidence: list[str]) -> Signal:
+        """Helper to build a signal for this classifier's label."""
+        return Signal(
+            kind=self.label,
+            confidence=min(confidence, 1.0),
+            evidence=tuple(evidence),
+            classifier=self.name,
+        )

agent_census/classify/browser.py

@@ -0,0 +1,180 @@
+"""Interactive browsers.
+
+The strongest tell is sub-resource co-loading: a real browser, after fetching a
+page, pulls its CSS/JS/images within seconds. Bursty (irregular) timing, on-site
+link navigation, a browser-shaped UA, and a low error rate corroborate it.
+"""
+
+from __future__ import annotations
+
+from .. import uas
+from ..model import ClientFeatures, Kind, Signal
+from .base import Classifier
+from .tags import identifies_as_known_agent
+
+# A browser-shaped UA with positive evidence but no disqualifying behaviour is
+# floored to here, so a brief, asset-less visit (a real person who didn't trigger
+# sub-resource loading) isn't lost to UNKNOWN. It matches the default unknown
+# threshold; the combiner's datacenter discount then keeps the rescue to
+# residential clients -- a hosting "browser" still drops to spoofed_browser.
+_BROWSER_FLOOR = 0.45
+
+
+class BrowserClassifier(Classifier):
+    label = Kind.BROWSER
+    name = "browser"
+
+    def evaluate(self, features: ClientFeatures) -> list[Signal]:
+        # A UA that names a feed reader, crawler, or bot is not a browser, even if
+        # it renders pages and co-loads their sub-resources -- its declared
+        # identity wins.
+        if identifies_as_known_agent(features):
+            return []
+
+        confidence = 0.0
+        evidence: list[str] = []
+        # Set when a signal positively argues *against* a browser (a cap or a
+        # penalty). While clean, an under-supported browser shape is floored at the
+        # end; once disqualified, it is left to fall where its confidence lands. A
+        # stale (but not ancient) version is a mild nudge, not a disqualifier.
+        disqualified = False
+
+        if features.asset_coload_ratio > 0.4:
+            confidence += 0.45
+            evidence.append(
+                f"{features.asset_coload_ratio:.0%} of pages followed by sub-resource loads"
+            )
+
+        if features.ua_looks_like_browser:
+            confidence += 0.2
+            evidence.append("User-Agent matches a real browser profile")
+
+        regularity = features.rate_regularity
+        if regularity is not None and regularity > 0.6:
+            confidence += 0.1
+            evidence.append("irregular, bursty timing (human-like)")
+        elif regularity is not None and regularity < 0.15 and features.request_count >= 5:
+            # Metronomic cadence is a machine, not a person clicking around.
+            confidence -= 0.2
+            disqualified = True
+            evidence.append("metronomic timing — automated, not human")
+
+        if features.referer_following_ratio > 0.3:
+            confidence += 0.1
+            evidence.append(
+                f"{features.referer_following_ratio:.0%} of requests follow on-site links"
+            )
+
+        if features.ratio_404 < 0.1 and features.vuln_path_hits == 0:
+            confidence += 0.1
+            evidence.append("low error rate, no probing")
+
+        if features.static_ratio > 0.3:
+            confidence += 0.05
+            evidence.append(f"{features.static_ratio:.0%} static-asset requests")
+
+        if features.status_counts.get(304, 0) > 0:
+            # Conditional requests answered 304 mean a real cache -- a browser tell.
+            confidence += 0.1
+            evidence.append("revalidates from cache (304 Not Modified)")
+
+        # Chromium-based browsers and Firefox auto-update on a ~monthly cadence, so
+        # a real browser is rarely far behind. A UA claiming a version years old
+        # (measured against when the client was active) is almost always a frozen,
+        # spoofed string; a current one weakly corroborates. Bots can copy a recent
+        # UA, so the fresh bonus is small and the stale penalty is the load-bearing
+        # half. Old but real cases exist (locked fleets, embedded WebViews, ESR),
+        # so only a very old version caps the verdict.
+        band = uas.version_age_band(features.user_agent, features.last_seen)
+        if evidence and band == "current":
+            confidence += 0.1
+            evidence.append("up-to-date browser version")
+        elif evidence and band == "stale":
+            confidence -= 0.15
+            evidence.append("browser version well out of date")
+        elif evidence and band == "ancient":
+            # Years behind on a family that auto-updates: almost always a frozen,
+            # spoofed UA, so cap the browser hypothesis below the threshold.
+            confidence = min(confidence, 0.3)
+            disqualified = True
+            evidence.append("browser version years out of date — modern browsers auto-update")
+        elif evidence and band == "impossible":
+            # Claims a version that doesn't exist yet: a forged UA, not a real
+            # browser, so cap the hypothesis just like an ancient one.
+            confidence = min(confidence, 0.3)
+            disqualified = True
+            evidence.append("browser version is impossibly new — forged User-Agent")
+
+        # A browser never auto-fetches /robots.txt; checking it is a crawler's
+        # habit. Slight on its own (a person could type the URL once), but it
+        # nudges an otherwise browser-shaped client the right way.
+        if evidence and features.fetched_robots_txt:
+            confidence -= 0.15
+            disqualified = True
+            evidence.append("fetched /robots.txt — a crawler's habit, not a browser's")
+
+        # A browser revalidates what it re-requests (earning 304s) and serves
+        # cached assets without hitting the server at all -- so at any real volume
+        # a genuine browser leaves some 304s behind, or simply makes few requests.
+        # A client that re-fetches the same URLs, or just makes a large number of
+        # requests, yet never receives a single 304 holds no cache: not browser
+        # behaviour. Only meaningful once paths are measured, and a 304 can only
+        # arise from a re-request, so distinct-once fetching at low volume is spared.
+        revisits = features.request_count - features.distinct_paths
+        no_304 = features.status_counts.get(304, 0) == 0
+        cold_refetch = revisits >= 20
+        high_volume = features.request_count >= 500
+        if evidence and no_304 and features.distinct_paths > 0 and (cold_refetch or high_volume):
+            dominant = (cold_refetch and revisits >= features.request_count * 0.5) or (
+                features.request_count >= 2000
+            )
+            disqualified = True
+            if dominant:
+                # Re-fetching dominates, or the volume is large enough that zero
+                # revalidations is itself damning: cap below the confident threshold.
+                confidence = min(confidence, 0.3)
+                evidence.append("heavy traffic without a single 304 — holds no browser cache")
+            else:
+                confidence -= 0.2
+                evidence.append("many requests, never revalidated (no 304s)")
+
+        # A person at a browser never fetches attack paths. Vuln probing or
+        # directory traversal means this is automation wearing a browser engine
+        # (e.g. headless Chrome), so cap the browser hypothesis below the unknown
+        # threshold rather than let asset co-loading carry it to a confident
+        # verdict. (Ignoring robots.txt is NOT penalised -- it does not bind a
+        # human browsing by hand.)
+        if features.traversal_hits > 0 or features.vuln_path_hits >= 2:
+            confidence = min(confidence, 0.3)
+            disqualified = True
+            evidence.append("but probes attack paths — not human browsing")
+
+        # Fabricated referers (the Referer is the requested URL itself) are
+        # impossible from real navigation; a client doing this systematically is
+        # faking organic traffic, not browsing.
+        if features.self_referer_ratio >= 0.5 and features.request_count >= 4:
+            confidence = min(confidence, 0.3)
+            disqualified = True
+            evidence.append("but referers are fabricated (Referer = the requested URL)")
+
+        # A browser fetches pages and their sub-resources with GET; it does not
+        # issue HEAD. Meaningful HEAD traffic from something otherwise browser-
+        # shaped is a machine (a monitor, link-checker, or other bot) behind a
+        # browser UA -- cap the hypothesis below the confident threshold. Gated on
+        # an existing browser signal so monitors/feed readers that legitimately
+        # HEAD don't each pick up a spurious browser signal.
+        if evidence and features.head_ratio > 0.1:
+            confidence = min(confidence, 0.3)
+            disqualified = True
+            evidence.append(f"but {features.head_ratio:.0%} HEAD requests — browsers issue GET")
+
+        if not evidence:
+            return []
+        # A browser-shaped UA with positive evidence and nothing arguing against it
+        # is a probable browser even without the asset-loading proof -- a brief
+        # visit. Floor it so it clears the unknown threshold rather than being lost;
+        # ancient/forged UAs and any non-browser behaviour disqualify it above.
+        if features.ua_looks_like_browser and not disqualified and confidence < _BROWSER_FLOOR:
+            confidence = _BROWSER_FLOOR
+            evidence.append("browser-shaped User-Agent with no non-browser behaviour")
+        return [self._signal(confidence, evidence)]

agent_census/classify/combiner.py

@@ -0,0 +1,175 @@
+"""Combine classifier signals into a final classification.
+
+Confidence is treated as an ordinal strength per label, not a probability, so
+signals are aggregated per kind (taking the strongest) rather than multiplied.
+The strongest label wins; ties break by a fixed priority. Below a threshold the
+honest answer is ``UNKNOWN``. Tags are derived separately and can demote a
+falsely-claimed good-bot identity.
+"""
+
+from __future__ import annotations
+
+from .. import uas
+from ..model import (
+    BotVerification,
+    Classification,
+    ClientFeatures,
+    ComplianceReport,
+    Kind,
+    Signal,
+)
+from .tags import derive_tags, impersonation, looks_like_fake_browser
+
+# Tie-break order when two kinds share the top confidence: earlier wins.
+_PRIORITY: tuple[Kind, ...] = (
+    Kind.IMPERSONATOR,
+    Kind.SEARCH_ENGINE,
+    Kind.SOCIAL_PREVIEW,
+    Kind.ARCHIVER,
+    Kind.AI_CRAWLER,
+    Kind.SEO_MARKETING,
+    Kind.VULN_SCANNER,
+    Kind.SPAM_BOT,
+    Kind.FEED_READER,
+    Kind.MONITOR,
+    Kind.BROWSER,
+    Kind.APP,
+    Kind.SCRAPER,
+    Kind.CRAWLER,
+    Kind.SPOOFED_BROWSER,
+    Kind.SINGLETON,
+    Kind.UNKNOWN,
+)
+_RANK = {kind: i for i, kind in enumerate(_PRIORITY)}
+
+DEFAULT_UNKNOWN_THRESHOLD = 0.45
+
+
+def _pick(by_label: dict[Kind, float]) -> Kind:
+    return max(by_label, key=lambda k: (by_label[k], -_RANK.get(k, len(_PRIORITY))))
+
+
+def _top_evidence(signals: tuple[Signal, ...]) -> tuple[str, ...]:
+    if not signals:
+        return ("no classifier produced a signal",)
+    strongest = max(signals, key=lambda s: s.confidence)
+    return strongest.evidence or ("no specific evidence recorded",)
+
+
+def combine(
+    signals: list[Signal],
+    features: ClientFeatures,
+    *,
+    compliance: ComplianceReport | None = None,
+    verification: BotVerification | None = None,
+    datacenter: bool = False,
+    unknown_threshold: float = DEFAULT_UNKNOWN_THRESHOLD,
+    keep_signals: bool = True,
+) -> Classification:
+    """Aggregate ``signals`` into a primary kind plus secondary tags.
+
+    ``keep_signals`` retains every contributing signal on the result for inspect
+    mode's rationale. The ``analyze`` report never reads them, so it passes
+    ``False`` to avoid holding a Signal (and its evidence strings) per client.
+    """
+    by_label: dict[Kind, float] = {}
+    for signal in signals:
+        # Round when aggregating: classifiers build confidence from 0.05-step
+        # increments, and float error (0.3 + 0.15 == 0.44999999996) would otherwise
+        # drop a sum that equals the threshold just below it -- misfiling a clear
+        # client as UNKNOWN while it still displays as the rounded percentage.
+        by_label[signal.kind] = max(by_label.get(signal.kind, 0.0), round(signal.confidence, 3))
+
+    # A person rarely browses from hosting infrastructure, so nudge a datacenter
+    # "browser" verdict down a little -- enough to tip a borderline one, not to
+    # overrule a strongly-behaving real browser.
+    if datacenter and Kind.BROWSER in by_label:
+        by_label[Kind.BROWSER] = round(max(0.0, by_label[Kind.BROWSER] - 0.1), 3)
+
+    tags = derive_tags(features, compliance, verification, datacenter=datacenter)
+    stored = tuple(signals) if keep_signals else ()
+
+    # Impersonation is decisive: a client faking a declared identity is an
+    # impersonator, whatever else it looks like.
+    faking, why = impersonation(verification)
+    if faking:
+        return Classification(
+            primary=Kind.IMPERSONATOR,
+            confidence=0.9,
+            tags=frozenset(tags),
+            evidence=why,
+            all_signals=stored,
+        )
+
+    if not by_label or max(by_label.values()) < unknown_threshold:
+        # A would-be-unknown client wearing a browser UA from a hosting IP, with
+        # no browser behaviour, is automation in disguise -- name it as such.
+        if datacenter and looks_like_fake_browser(features):
+            return Classification(
+                primary=Kind.SPOOFED_BROWSER,
+                confidence=0.6,
+                tags=frozenset(tags),
+                evidence=("browser User-Agent from a datacenter IP, without browser behaviour",),
+                all_signals=stored,
+            )
+        # A generic HTTP library (or no UA) fetching several pages from hosting
+        # infrastructure is harvesting content -- a scraper -- even when no single
+        # signal cleared the bar. The datacenter origin is what tips it: the same
+        # library from a residential IP could be an app or a one-off script.
+        if datacenter and _looks_like_datacenter_scraper(features):
+            return Classification(
+                primary=Kind.SCRAPER,
+                confidence=0.5,
+                tags=frozenset(tags),
+                evidence=("generic HTTP client harvesting pages from a datacenter IP",),
+                all_signals=stored,
+            )
+        # A would-be-unknown client with a single request gets its own bucket:
+        # one hit is too little to characterize, so we file it by volume.
+        if features.request_count == 1:
+            return Classification(
+                primary=Kind.SINGLETON,
+                confidence=1.0,
+                tags=frozenset(tags),
+                evidence=("single request — too little activity to characterize",),
+                all_signals=stored,
+            )
+        confidence = max(by_label.values()) if by_label else 0.0
+        return Classification(
+            primary=Kind.UNKNOWN,
+            confidence=confidence,
+            tags=frozenset(tags),
+            evidence=_top_evidence(tuple(signals)),
+            all_signals=stored,
+        )
+
+    primary = _pick(by_label)
+    if primary is Kind.FEED_READER and _fetches_non_feeds(features):
+        tags.add("fetches-non-feeds")
+    evidence = tuple(e for s in signals if s.kind is primary for e in s.evidence)
+    return Classification(
+        primary=primary,
+        confidence=by_label[primary],
+        tags=frozenset(tags),
+        evidence=evidence,
+        all_signals=stored,
+    )
+
+
+def _looks_like_datacenter_scraper(features: ClientFeatures) -> bool:
+    """A generic-library / UA-less client harvesting several pages, benignly."""
+    return (
+        features.request_count >= 2
+        and features.distinct_paths >= 2
+        and (uas.is_library(features.user_agent) or features.ua_empty)
+        and features.vuln_path_hits == 0
+        and features.traversal_hits == 0
+    )
+
+
+def _fetches_non_feeds(features: ClientFeatures) -> bool:
+    """True if a feed reader also requested non-feed resources (robots.txt aside)."""
+    non_feed = features.request_count - features.feed_requests
+    if features.fetched_robots_txt:
+        non_feed -= 1  # a polite robots.txt fetch does not count as content scraping
+    return non_feed > 0

agent_census/classify/crawler.py

@@ -0,0 +1,56 @@
+"""Generic crawlers that walk the site following links at a steady pace.
+
+Distinguished from a browser by the absence of sub-resource co-loading, and from
+a scraper by actually following on-site links (high referer-following) rather
+than hitting URLs cold.
+"""
+
+from __future__ import annotations
+
+from ..model import ClientFeatures, Kind, Signal
+from .base import Classifier
+
+
+class CrawlerClassifier(Classifier):
+    label = Kind.CRAWLER
+    name = "crawler"
+
+    def evaluate(self, features: ClientFeatures) -> list[Signal]:
+        confidence = 0.0
+        evidence: list[str] = []
+
+        if features.distinct_paths >= 20 and features.coverage > 0.7:
+            confidence += 0.3
+            evidence.append(
+                f"broad coverage: {features.distinct_paths} distinct paths "
+                f"({features.coverage:.0%} unique)"
+            )
+
+        regularity = features.rate_regularity
+        if regularity is not None and regularity < 0.5 and features.request_count >= 10:
+            confidence += 0.2
+            evidence.append("steady, regular request cadence")
+
+        if features.referer_following_ratio > 0.3:
+            confidence += 0.15
+            evidence.append(
+                f"{features.referer_following_ratio:.0%} of requests follow on-site links"
+            )
+
+        if features.ua_declares_bot:
+            confidence += 0.15
+            evidence.append("User-Agent self-identifies as a bot")
+            # A self-declared bot walking many pages with no browser sub-resource
+            # loading is a crawler even without broad unique coverage -- the kind of
+            # "MyBot/1.0 (+url)" client that re-requests a modest path set.
+            if features.distinct_paths >= 20 and features.asset_coload_ratio < 0.1:
+                confidence += 0.3
+                evidence.append("walks many pages without browser sub-resource loading")
+
+        if features.ratio_2xx > 0.7 and features.asset_coload_ratio < 0.1:
+            confidence += 0.1
+            evidence.append("mostly successful page fetches, no browser sub-resource loading")
+
+        if not evidence:
+            return []
+        return [self._signal(confidence, evidence)]

agent_census/classify/data_harvester.py

@@ -0,0 +1,19 @@
+"""Data harvesters: crawl content into a private corpus or dataset.
+
+Commercial crawlers that ingest pages into a proprietary database for their own
+product -- plagiarism/similarity indexes (Turnitin), data brokers and dataset
+builders (Panscient) -- as opposed to public search, preservation, AI/LLM
+training, or SEO. Recognised by a known UA token, like the other declared kinds.
+"""
+
+from __future__ import annotations
+
+from ..model import Kind
+from .known_bot import KnownBotClassifier
+
+
+class DataHarvesterClassifier(KnownBotClassifier):
+    label = Kind.DATA_HARVESTER
+    name = "data_harvester"
+    category = "data_harvester"
+    descriptor = "data harvester"