classifyre-cli 0.4.85__tar.gz → 0.4.86__tar.gz

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 307 packages in 272ms
+Resolved 307 packages in 247ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.85
+Version: 0.4.86
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.85",
+  "version": "0.4.86",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.85"
+version = "0.4.86"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/src/detectors/custom/runners/_gliner2.py

@@ -189,7 +189,17 @@ class GLiNER2Runner(BaseRunner):
         self, model: Any, text: str, task_name: str, labels: list[str]
     ) -> dict[str, object]:
         """Classify text, chunking long inputs and keeping the max-confidence label."""
-        chunks = _chunk_text(text, _CLS_CHUNK_SIZE, _CLS_CHUNK_OVERLAP)[:_MAX_CLS_CHUNKS]
+        all_chunks = _chunk_text(text, _CLS_CHUNK_SIZE, _CLS_CHUNK_OVERLAP)
+        chunks = all_chunks[:_MAX_CLS_CHUNKS]
+        if len(all_chunks) > _MAX_CLS_CHUNKS:
+            logger.warning(
+                "GLiNER2 classification for detector '%s': text of %d chars needs %d chunks, "
+                "capping at %d; the remainder was not classified",
+                self._detector_key,
+                len(text),
+                len(all_chunks),
+                _MAX_CLS_CHUNKS,
+            )
 
         best: dict[str, object] = {}
         for chunk in chunks:

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/src/detectors/custom/runners/_text_classification.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+from collections.abc import Iterator
 from typing import Any
 
 from ....models.generated_detectors import Severity, TextClassificationPipelineSchema
@@ -13,6 +14,22 @@ from ._base import _TEXT_CONTENT_TYPES, BaseRunner, _resolve_pipeline_severity
 logger = logging.getLogger(__name__)
 
 
+# A transformer pipeline called with truncation=True silently keeps only the
+# first model_max_length tokens, so without chunking a long document is
+# classified on its opening paragraph alone -- no error, no log, and everything
+# after it is never looked at. When chunk_size is not configured we derive a
+# window from the model's own token limit instead.
+# Deliberate under-estimate: too small a window only costs compute, too large
+# a one hands the tail back to truncation.
+_CHARS_PER_TOKEN = 3
+_DEFAULT_MODEL_MAX_TOKENS = 512
+# model_max_length is a huge sentinel on tokenizers that declare no limit.
+_MODEL_MAX_TOKENS_SENTINEL = 100_000
+_AUTO_CHUNK_OVERLAP = 128
+# Bound per-asset CPU cost; hitting this is logged, never silent.
+_MAX_AUTO_CHUNKS = 200
+
+
 def _chunk_text(text: str, chunk_size: int | None, chunk_overlap: int) -> list[str]:
     """Split text into chunks. Returns [text] when chunk_size is not set."""
     if not chunk_size:
@@ -21,6 +38,55 @@ def _chunk_text(text: str, chunk_size: int | None, chunk_overlap: int) -> list[s
     return [text[i : i + chunk_size] for i in range(0, len(text), step)]
 
 
+def _model_chunk_chars(pipe: Any, max_length: int | None) -> int:
+    """Characters that comfortably fit one forward pass of *pipe*."""
+    tokens = max_length
+    if tokens is None:
+        tokenizer_limit = getattr(getattr(pipe, "tokenizer", None), "model_max_length", None)
+        if isinstance(tokenizer_limit, int) and 0 < tokenizer_limit < _MODEL_MAX_TOKENS_SENTINEL:
+            tokens = tokenizer_limit
+    if not isinstance(tokens, int) or tokens <= 0:
+        tokens = _DEFAULT_MODEL_MAX_TOKENS
+    return max(256, tokens * _CHARS_PER_TOKEN)
+
+
+def _iter_analysis_chunks(
+    text: str,
+    chunk_size: int | None,
+    chunk_overlap: int,
+    auto_chunk_chars: int,
+) -> Iterator[str]:
+    """Yield the windows to classify, one resident at a time.
+
+    An explicit chunk_size wins. Otherwise the text is passed through whole
+    unless it would overflow the model window, in which case it is auto-chunked
+    so the tail is classified rather than truncated away.
+    """
+    if chunk_size:
+        step = max(1, chunk_size - chunk_overlap)
+        for start in range(0, len(text), step):
+            yield text[start : start + chunk_size]
+        return
+
+    if len(text) <= auto_chunk_chars:
+        yield text
+        return
+
+    overlap = min(_AUTO_CHUNK_OVERLAP, auto_chunk_chars // 4)
+    step = max(1, auto_chunk_chars - overlap)
+    for index, start in enumerate(range(0, len(text), step)):
+        if index >= _MAX_AUTO_CHUNKS:
+            logger.warning(
+                "text_classification: text of %d chars exceeds %d auto-chunks of %d; "
+                "the remainder was not classified (set chunk_size to control this)",
+                len(text),
+                _MAX_AUTO_CHUNKS,
+                auto_chunk_chars,
+            )
+            return
+        yield text[start : start + auto_chunk_chars]
+
+
 class TextClassificationRunner(BaseRunner):
     """Text classification via a single HuggingFace text-classification pipeline."""
 
@@ -94,8 +160,9 @@ class TextClassificationRunner(BaseRunner):
         default_severity = schema.severity if schema.severity is not None else Severity.info
 
         best_scores: dict[str, float] = {}
+        auto_chunk_chars = _model_chunk_chars(pipe, max_length)
         try:
-            for chunk in _chunk_text(text, chunk_size, chunk_overlap):
+            for chunk in _iter_analysis_chunks(text, chunk_size, chunk_overlap, auto_chunk_chars):
                 call_kwargs: dict[str, Any] = {"truncation": True}
                 if max_length is not None:
                     call_kwargs["max_length"] = max_length

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/src/detectors/pii/detector.py

@@ -1,12 +1,14 @@
 """PII detector powered by Microsoft Presidio."""
 
 import asyncio
+import bisect
 import importlib
 import logging
 import re
 import subprocess
 import sys
 import warnings
+from collections.abc import Iterator
 from dataclasses import dataclass
 from typing import Any, ClassVar
 
@@ -239,6 +241,17 @@ class PIIDetector(BaseDetector):
     # Per-cell analysis at scale causes O(rowsxcolumns) Presidio calls per page.
     _TABULAR_CELL_LIMIT: ClassVar[int] = 200
 
+    # spaCy's own ceiling when nlp.max_length is not overridden.
+    _SPACY_DEFAULT_MAX_LENGTH: ClassVar[int] = 1_000_000
+
+    # Window used when a page exceeds the spaCy ceiling and no chunk_size is
+    # configured. The parser/NER need roughly 1GB of scratch memory per 100k
+    # characters, so 200k caps peak usage near 2GB no matter how large the page
+    # is -- whereas simply raising nlp.max_length scales memory with file size
+    # (a 1.4M-char page would need ~14GB).
+    _AUTO_CHUNK_CHARS: ClassVar[int] = 200_000
+    _AUTO_CHUNK_OVERLAP: ClassVar[int] = 512
+
     def __init__(self, config: DetectorConfig | None = None) -> None:
         super().__init__(config)
         self._cfg: PIIDetectorConfig = (
@@ -983,14 +996,64 @@ class PIIDetector(BaseDetector):
         # Offloading to a thread keeps the loop alive and allows I/O to proceed.
         return await asyncio.to_thread(self._detect_sync, content)
 
-    def _chunk_text(self, text: str) -> list[tuple[str, int]]:
-        """Return (chunk, offset) pairs. When chunk_size is null returns the full text at offset 0."""
+    def _spacy_char_limit(self) -> int:
+        """Largest text spaCy will accept in a single call, honouring max_length."""
+        return _unwrap_int(getattr(self._cfg, "max_length", None)) or self._SPACY_DEFAULT_MAX_LENGTH
+
+    def _iter_chunks(self, text: str) -> Iterator[tuple[str, int]]:
+        """Yield (chunk, offset) pairs, one chunk resident at a time.
+
+        With chunk_size configured the text is split on that fixed window. With
+        chunk_size null the text is passed through whole -- unless it exceeds
+        the spaCy ceiling, in which case it is auto-chunked instead of raising
+        E088 and losing every finding on the page.
+        """
         chunk_size = _unwrap_int(getattr(self._cfg, "chunk_size", None))
-        if not chunk_size:
-            return [(text, 0)]
-        overlap = _unwrap_int(getattr(self._cfg, "chunk_overlap", None)) or 0
-        step = max(1, chunk_size - overlap)
-        return [(text[i : i + chunk_size], i) for i in range(0, len(text), step)]
+        if chunk_size:
+            overlap = _unwrap_int(getattr(self._cfg, "chunk_overlap", None)) or 0
+            step = max(1, chunk_size - overlap)
+            for start in range(0, len(text), step):
+                yield (text[start : start + chunk_size], start)
+            return
+
+        if len(text) <= self._spacy_char_limit():
+            yield (text, 0)
+            return
+
+        yield from self._auto_chunks(text)
+
+    def _auto_chunks(self, text: str) -> Iterator[tuple[str, int]]:
+        """Split an oversized page into spaCy-sized windows on whitespace boundaries."""
+        window = min(self._AUTO_CHUNK_CHARS, self._spacy_char_limit())
+        overlap = min(self._AUTO_CHUNK_OVERLAP, window // 4)
+        total = len(text)
+        logger.info(
+            "Page of %d chars exceeds the spaCy limit (%d); auto-chunking at %d chars "
+            "(set chunk_size to control this explicitly)",
+            total,
+            self._spacy_char_limit(),
+            window,
+        )
+
+        start = 0
+        while start < total:
+            end = min(start + window, total)
+            if end < total:
+                # Prefer a line/word boundary inside the trailing overlap zone so
+                # entities are not cut in half; the overlap re-covers the seam.
+                split = text.rfind("\n", end - overlap, end)
+                if split == -1:
+                    split = text.rfind(" ", end - overlap, end)
+                if split > start:
+                    end = split + 1
+            yield (text[start:end], start)
+            if end >= total:
+                return
+            start = max(start + 1, end - overlap)
+
+    def _chunk_text(self, text: str) -> list[tuple[str, int]]:
+        """Materialized :meth:`_iter_chunks` (kept for callers that need a list)."""
+        return list(self._iter_chunks(text))
 
     def _detect_sync(self, content: str) -> list[DetectionResult]:
         tabular_results = self._detect_tabular_content(content)
@@ -1004,8 +1067,11 @@ class PIIDetector(BaseDetector):
         threshold = self._cfg.confidence_threshold or 0.7
         results: list[DetectionResult] = []
         seen: set[tuple[str, int, int]] = set()
+        # Precomputed once: content[:start].count("\n") per finding is O(n) each
+        # and copies the prefix, which is quadratic on multi-megabyte pages.
+        newline_offsets = [m.start() for m in re.finditer("\n", content)]
 
-        for chunk, offset in self._chunk_text(content):
+        for chunk, offset in self._iter_chunks(content):
             for result in self._analyze_content(chunk, entities=entities):
                 if not self._is_entity_enabled(result.entity_type):
                     continue
@@ -1017,7 +1083,7 @@ class PIIDetector(BaseDetector):
                     continue
                 seen.add(dedup_key)
 
-                line_number = content[:abs_start].count("\n") + 1
+                line_number = bisect.bisect_left(newline_offsets, abs_start) + 1
                 matched_content = content[abs_start:abs_end]
 
                 detection = self._build_detection_result(

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/src/models/generated_detectors.py

@@ -432,7 +432,7 @@ class SecretsDetectorConfig(DetectorConfig):
 class MaxLength(RootModel[int]):
     root: int = Field(
         None,
-        description="Override spaCy's nlp.max_length (default 1,000,000 chars). Set higher than your longest expected input to avoid the E088 error. Prefer chunk_size for very large texts.",
+        description="Override spaCy's nlp.max_length (default 1,000,000 chars). Raising it makes memory scale with input size (~1GB of scratch per 100,000 chars), so prefer chunk_size for very large texts. Oversized pages are auto-chunked rather than failing, so this rarely needs setting.",
         ge=1,
     )
 
@@ -440,7 +440,7 @@ class MaxLength(RootModel[int]):
 class ChunkSize(RootModel[int]):
     root: int = Field(
         None,
-        description='Split text into chunks of this many characters before analysis. Findings from all chunks are merged with corrected offsets. When null the full text is passed as-is (subject to max_length).',
+        description='Split text into chunks of this many characters before analysis. Findings from all chunks are merged with corrected offsets. When null the full text is passed as-is, unless it exceeds the spaCy limit, in which case it is auto-chunked at 200,000 characters on whitespace boundaries.',
         ge=1,
     )
 
@@ -483,11 +483,11 @@ class PIIDetectorConfig(DetectorConfig):
     )
     max_length: MaxLength | None = Field(
         None,
-        description="Override spaCy's nlp.max_length (default 1,000,000 chars). Set higher than your longest expected input to avoid the E088 error. Prefer chunk_size for very large texts.",
+        description="Override spaCy's nlp.max_length (default 1,000,000 chars). Raising it makes memory scale with input size (~1GB of scratch per 100,000 chars), so prefer chunk_size for very large texts. Oversized pages are auto-chunked rather than failing, so this rarely needs setting.",
     )
     chunk_size: ChunkSize | None = Field(
         None,
-        description='Split text into chunks of this many characters before analysis. Findings from all chunks are merged with corrected offsets. When null the full text is passed as-is (subject to max_length).',
+        description='Split text into chunks of this many characters before analysis. Findings from all chunks are merged with corrected offsets. When null the full text is passed as-is, unless it exceeds the spaCy limit, in which case it is auto-chunked at 200,000 characters on whitespace boundaries.',
     )
     chunk_overlap: ChunkOverlap | None = Field(
         0,

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/tests/detectors/custom/test_transformer_runners.py

@@ -89,6 +89,56 @@ async def test_text_classification_runner_applies_severity_map() -> None:
     assert findings[0].severity == Severity.high
 
 
+@pytest.mark.asyncio
+async def test_text_classification_short_text_is_a_single_call() -> None:
+    runner = _make_text_cls_runner([{"label": "SPAM", "score": 0.95}])
+
+    runner.detect("Win a free iPhone now!", "text/plain")
+
+    assert runner._pipe.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_text_classification_long_text_is_chunked_not_truncated() -> None:
+    # A pipeline called with truncation=True keeps only the first ~512 tokens,
+    # so an unchunked long document was classified on its opening lines alone
+    # and the rest was silently discarded.
+    runner = _make_text_cls_runner([{"label": "SPAM", "score": 0.95}])
+    text = "spam and eggs " * 5_000  # 70k chars, far past any model window
+
+    runner.detect(text, "text/plain")
+
+    assert runner._pipe.call_count > 1
+    seen = "".join(call.args[0] for call in runner._pipe.call_args_list)
+    # Overlapping windows, so the concatenation is longer than the input -- the
+    # point is that no part of it went unseen.
+    assert len(seen) >= len(text)
+    assert all(len(call.args[0]) <= 4_000 for call in runner._pipe.call_args_list)
+
+
+@pytest.mark.asyncio
+async def test_text_classification_chunk_cap_is_bounded() -> None:
+    from src.detectors.custom.runners._text_classification import _MAX_AUTO_CHUNKS
+
+    runner = _make_text_cls_runner([{"label": "SPAM", "score": 0.95}])
+
+    runner.detect("spam and eggs " * 200_000, "text/plain")
+
+    assert runner._pipe.call_count <= _MAX_AUTO_CHUNKS
+
+
+@pytest.mark.asyncio
+async def test_text_classification_scores_are_max_across_chunks() -> None:
+    runner = _make_text_cls_runner([])
+    runner._pipe = MagicMock(
+        side_effect=[[{"label": "SPAM", "score": s}] for s in (0.1, 0.99, 0.2)] * 200
+    )
+
+    findings = runner.detect("spam and eggs " * 400, "text/plain")
+
+    assert findings[0].confidence == pytest.approx(0.99)
+
+
 def test_text_classification_run_raises_not_implemented() -> None:
     runner = _make_text_cls_runner([])
     with pytest.raises(NotImplementedError):

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/tests/detectors/pii/test_pii_chunking.py

@@ -10,6 +10,8 @@ produced nothing for the whole corpus.
 These tests need no Presidio: they exercise the config plumbing directly.
 """
 
+import itertools
+
 from src.detectors.pii.detector import PIIDetector, _unwrap_int
 from src.models.generated_detectors import ChunkOverlap, ChunkSize, MaxLength
 
@@ -119,3 +121,79 @@ class TestMaxLength:
 
     def test_unset_max_length_is_none(self):
         assert _unwrap_int(None) is None
+
+    def test_limit_defaults_to_spacy_default(self):
+        detector = _detector_with(_Cfg())
+
+        assert detector._spacy_char_limit() == PIIDetector._SPACY_DEFAULT_MAX_LENGTH
+
+    def test_limit_follows_configured_max_length(self):
+        detector = _detector_with(_Cfg(max_length=MaxLength(root=2_000_000)))
+
+        assert detector._spacy_char_limit() == 2_000_000
+
+
+class TestAutoChunking:
+    """A page longer than spaCy's ceiling used to raise E088 inside Presidio,
+    which the detector swallowed -- so every finding on that page was lost.
+    Oversized pages are now split into bounded windows instead."""
+
+    def _oversized(self, chars: int) -> str:
+        # Word-shaped text so boundary snapping has somewhere to cut.
+        return ("lorem ipsum dolor sit amet " * ((chars // 27) + 1))[:chars]
+
+    def test_text_under_the_limit_is_not_chunked(self):
+        detector = _detector_with(_Cfg())
+
+        assert detector._chunk_text("some text") == [("some text", 0)]
+
+    def test_oversized_text_is_split_without_config(self):
+        detector = _detector_with(_Cfg())
+        text = self._oversized(PIIDetector._SPACY_DEFAULT_MAX_LENGTH + 500_000)
+
+        chunks = detector._chunk_text(text)
+
+        assert len(chunks) > 1
+        assert all(len(chunk) <= PIIDetector._AUTO_CHUNK_CHARS for chunk, _ in chunks)
+
+    def test_chunks_cover_the_whole_text_at_correct_offsets(self):
+        detector = _detector_with(_Cfg())
+        text = self._oversized(PIIDetector._SPACY_DEFAULT_MAX_LENGTH + 12_345)
+
+        chunks = detector._chunk_text(text)
+
+        # Every chunk must sit at the offset it claims, or finding positions
+        # reported back to the API point at the wrong bytes.
+        assert all(text[offset : offset + len(chunk)] == chunk for chunk, offset in chunks)
+        assert chunks[0][1] == 0
+        last_chunk, last_offset = chunks[-1]
+        assert last_offset + len(last_chunk) == len(text)
+        # No gaps between consecutive windows.
+        for (chunk, offset), (_, next_offset) in itertools.pairwise(chunks):
+            assert next_offset <= offset + len(chunk)
+
+    def test_a_lower_max_length_tightens_the_window(self):
+        detector = _detector_with(_Cfg(max_length=MaxLength(root=50_000)))
+        text = self._oversized(120_000)
+
+        chunks = detector._chunk_text(text)
+
+        assert all(len(chunk) <= 50_000 for chunk, _ in chunks)
+
+    def test_explicit_chunk_size_still_wins_over_auto(self):
+        detector = _detector_with(_Cfg(chunk_size=ChunkSize(root=10), chunk_overlap=None))
+        text = self._oversized(PIIDetector._SPACY_DEFAULT_MAX_LENGTH + 1)
+
+        first, offset = next(iter(detector._iter_chunks(text)))
+
+        assert offset == 0
+        assert len(first) == 10
+
+    def test_text_without_whitespace_still_terminates(self):
+        detector = _detector_with(_Cfg(max_length=MaxLength(root=1_000)))
+        text = "a" * 5_000
+
+        chunks = detector._chunk_text(text)
+
+        assert chunks[-1][1] + len(chunks[-1][0]) == len(text)
+        assert all(len(chunk) <= 1_000 for chunk, _ in chunks)

{classifyre_cli-0.4.85 → classifyre_cli-0.4.86}/uv.lock

@@ -830,7 +830,7 @@ wheels = [
 
 [[package]]
 name = "classifyre-cli"
-version = "0.4.85"
+version = "0.4.86"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },
@@ -1369,7 +1369,7 @@ youtube = [
 
 [[package]]
 name = "classifyre-schemas"
-version = "0.4.85"
+version = "0.4.86"
 source = { editable = "../../packages/schemas" }
 dependencies = [
     { name = "fastjsonschema" },