classifyre-cli 0.4.86__tar.gz → 0.4.87__tar.gz

classifyre_cli-0.4.87/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 307 packages in 231ms
+Checked 51 packages in 3ms

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.86
+Version: 0.4.87
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.86",
+  "version": "0.4.87",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.86"
+version = "0.4.87"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/src/detectors/pii/detector.py

@@ -672,10 +672,40 @@ class PIIDetector(BaseDetector):
     # ------------------------------------------------------------------
 
     def _analyze_content(self, content: str, *, entities: list[str] | None = None) -> list[Any]:
+        """Analyze *content*, splitting it first if it would overflow spaCy.
+
+        This is the single chokepoint every caller goes through -- full-text
+        pages, individual tabular cells, and pre-chunked windows alike -- so the
+        oversize guard lives here rather than at any one call site. A page that
+        arrives already chunked passes straight through.
+        """
         if self.analyzer is None:
             if self._init_error is not None:
                 raise self._init_error
             return []
+        if len(content) > self._spacy_char_limit():
+            return self._analyze_oversized(content, entities=entities)
+        return self._analyze_once(content, entities=entities)
+
+    def _analyze_oversized(self, content: str, *, entities: list[str] | None) -> list[Any]:
+        """Analyze text past the spaCy ceiling window by window, merged into one result set.
+
+        Results are re-based onto absolute offsets in *content* and deduped
+        across the overlap, keeping the highest-scoring hit for each span.
+        """
+        merged: dict[tuple[str, int, int], Any] = {}
+        for chunk, offset in self._auto_chunks(content):
+            for result in self._analyze_once(chunk, entities=entities):
+                result.start += offset
+                result.end += offset
+                key = (result.entity_type, result.start, result.end)
+                existing = merged.get(key)
+                if existing is None or result.score > existing.score:
+                    merged[key] = result
+        return sorted(merged.values(), key=lambda r: (r.start, r.end))
+
+    def _analyze_once(self, content: str, *, entities: list[str] | None = None) -> list[Any]:
+        """One Presidio pass over text already known to fit the spaCy window."""
         try:
             return self.analyzer.analyze(text=content, language="en", entities=entities)
         except ModuleNotFoundError as exc:

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/src/outputs/base.py

@@ -7,6 +7,12 @@ from pydantic import BaseModel
 
 OutputType = Literal["rest", "file", "console"]
 
+# How long to wait for the API to answer a write-back. Generous enough that a
+# bulk ingest of a large batch finishes well inside it — the old 30 s cut those
+# off mid-flight and made urllib3 re-upload the same body — but finite, so a
+# wedged API fails the run instead of hanging it forever.
+DEFAULT_REST_TIMEOUT_SEC = 300
+
 
 @dataclass(frozen=True)
 class OutputRuntimeContext:
@@ -24,7 +30,8 @@ class OutputSettings:
     runner_id: str | None
     managed_runner: bool
     rest_url: str | None = None
-    rest_timeout_sec: int = 30
+    # Read timeout in seconds; None disables it. See resolve_output_settings.
+    rest_timeout_sec: int | None = DEFAULT_REST_TIMEOUT_SEC
     file_path: str | None = None
 
 

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/src/outputs/factory.py

@@ -4,7 +4,13 @@ import argparse
 import os
 from typing import Any, cast
 
-from .base import OutputRuntimeContext, OutputSettings, OutputSink, OutputType
+from .base import (
+    DEFAULT_REST_TIMEOUT_SEC,
+    OutputRuntimeContext,
+    OutputSettings,
+    OutputSink,
+    OutputType,
+)
 from .console import ConsoleOutputSink
 from .file import FileOutputSink
 from .rest import RestOutputSink
@@ -96,9 +102,17 @@ def resolve_output_settings(
     )
     rest_url = str(rest_url_value) if rest_url_value is not None else None
 
-    rest_timeout_sec = _parse_int(_coalesce(env_rest_timeout, 30), 30)
-    if rest_timeout_sec < 1:
-        rest_timeout_sec = 30
+    # 5 minutes by default (see DEFAULT_REST_TIMEOUT_SEC): long enough that a
+    # bulk ingest of a large batch is never cut off mid-flight, short enough
+    # that a wedged API surfaces as a failure. Override with
+    # CLASSIFYRE_OUTPUT_REST_TIMEOUT_SEC; 0 or negative disables the read
+    # timeout entirely.
+    rest_timeout_sec: int | None = _parse_int(
+        _coalesce(env_rest_timeout, DEFAULT_REST_TIMEOUT_SEC),
+        DEFAULT_REST_TIMEOUT_SEC,
+    )
+    if rest_timeout_sec is not None and rest_timeout_sec < 1:
+        rest_timeout_sec = None
 
     file_path_value = _coalesce(
         getattr(args, "output_file_path", None),

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/src/outputs/rest.py

@@ -55,9 +55,12 @@ class _JitteredRetry(Retry):
 # What we retry and why:
 #   connect=8  — pod restarted / not yet ready (RemoteDisconnected, ConnectionReset,
 #                ConnectTimeout). Request never reached the application.
-#   read=8     — API is under load and slow to respond (ReadTimeout). Safe to retry
-#                because all endpoints are idempotent (bulk ingest is upsert-based,
-#                status/findings updates are set-operations).
+#   read=8     — API is under load and slow to respond (ReadTimeout), i.e. it did
+#                not answer within DEFAULT_REST_TIMEOUT_SEC. Safe to retry because
+#                all endpoints are idempotent (bulk ingest is upsert-based,
+#                status/findings updates are set-operations) — but note that a
+#                retry re-uploads the whole body, which is why that budget is
+#                generous rather than tight.
 #   status=8   — transient HTTP errors from an overloaded or restarting API:
 #                  408 Request Timeout   - API-level timeout
 #                  429 Too Many Requests - rate-limited / backpressure
@@ -76,8 +79,7 @@ class _JitteredRetry(Retry):
 #   attempt 7 → ~60 s  (capped)
 #   attempt 8 → ~60 s  (capped)
 # Total extra wait: ~182 s (~3 min) — covers extended load spikes on a
-# single-node VPS before event-loop pressure drops. Worst-case a single
-# call costs 8 * 120 s + 182 s = ~18 min, acceptable for long-running scans.
+# single-node VPS before event-loop pressure drops.
 #
 # POST and PATCH are explicitly allowed: without this urllib3 only retries
 # idempotent methods (GET/HEAD) by default.
@@ -161,6 +163,12 @@ class ExternalRunnerResponse(BaseModel):
     source_id: str = Field(validation_alias="sourceId")
 
 
+# Applied to connection establishment only, kept short so a refused or
+# unreachable API fails fast and is retried by _RETRY_POLICY. How long the API
+# may then take to answer is a separate budget (timeout_sec, 5 min by default).
+_CONNECT_TIMEOUT_SEC = 30
+
+
 class RestOutputSink:
     output_type: OutputType = "rest"
 
@@ -169,7 +177,7 @@ class RestOutputSink:
         context: OutputRuntimeContext,
         *,
         base_url: str,
-        timeout_sec: int,
+        timeout_sec: int | None,
     ):
         self.context = context
         self.batch_size = context.batch_size
@@ -443,7 +451,9 @@ class RestOutputSink:
             method=method,
             url=url,
             json=payload,
-            timeout=self.timeout_sec,
+            # (connect, read): the read budget is the API's whole processing
+            # time, not just the first byte. None waits indefinitely.
+            timeout=(_CONNECT_TIMEOUT_SEC, self.timeout_sec),
         )
 
         if response.status_code >= 400:

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/src/sources/sandbox/source.py

@@ -10,6 +10,7 @@ from urllib.parse import quote
 import requests
 
 from ...models.generated_input import SandboxInput
+from ...outputs.base import DEFAULT_REST_TIMEOUT_SEC
 from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
 
 logger = logging.getLogger(__name__)
@@ -48,15 +49,19 @@ class SandboxSource(ObjectStorageSourceBase):
         return default
 
     def _max_object_bytes(self) -> int:
-        # Keep the extractor aligned with the API's per-upload limit instead
-        # of the object-storage connector family's conservative 5 MiB default.
+        # Spool threshold (RAM before spilling to disk), not a refusal ceiling —
+        # uploads are no longer size-capped. Raised above the object-storage
+        # family's conservative 5 MiB default because these files are local.
         return 50 * 1024 * 1024
 
     def _request(self, method: str, path: str, *, stream: bool = False) -> requests.Response:
         response = self._session.request(
             method,
             f"{self._api_url}{path}",
-            timeout=120,
+            # (connect, read). The read budget applies per socket read, so 5
+            # minutes is generous for a streamed download of any size; the old
+            # flat 120 s applied to connect and read alike.
+            timeout=(30, DEFAULT_REST_TIMEOUT_SEC),
             stream=stream,
             headers={"Connection": "close"},
         )

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/tests/detectors/pii/test_pii_chunking.py

@@ -32,6 +32,51 @@ class _Cfg:
         self.max_length = max_length
 
 
+class _FakeResult:
+    """Shape of a Presidio RecognizerResult, as far as the detector uses it."""
+
+    def __init__(self, entity_type, start, end, score):
+        self.entity_type = entity_type
+        self.start = start
+        self.end = end
+        self.score = score
+        self.recognition_metadata = None
+
+
+class _FakeAnalyzer:
+    """Finds every occurrence of a needle and reports chunk-relative offsets.
+
+    Real Presidio raises E088 above nlp.max_length; this records what it was
+    handed so the tests can assert nothing was skipped and offsets came back
+    absolute.
+    """
+
+    NEEDLE = "alice@example.com"
+
+    def __init__(self, score=0.9):
+        self.calls: list[str] = []
+        self._score = score
+
+    def analyze(self, *, text, language, entities=None):
+        self.calls.append(text)
+        results = []
+        start = text.find(self.NEEDLE)
+        while start != -1:
+            results.append(
+                _FakeResult("EMAIL_ADDRESS", start, start + len(self.NEEDLE), self._score)
+            )
+            start = text.find(self.NEEDLE, start + 1)
+        return results
+
+
+def _detector_with_analyzer(analyzer, cfg=None) -> PIIDetector:
+    detector = _detector_with(cfg or _Cfg())
+    detector.analyzer = analyzer
+    detector._init_error = None
+    detector._supported_entities_cache = frozenset({"EMAIL_ADDRESS"})
+    return detector
+
+
 class TestUnwrapInt:
     def test_unwraps_root_model(self):
         assert _unwrap_int(ChunkSize(root=500)) == 500
@@ -197,3 +242,86 @@ class TestAutoChunking:
 
         assert chunks[-1][1] + len(chunks[-1][0]) == len(text)
         assert all(len(chunk) <= 1_000 for chunk, _ in chunks)
+
+
+class TestAnalyzeContentGuard:
+    """_analyze_content is the chokepoint every caller shares. Guarding only the
+    full-text loop left the tabular per-cell path calling Presidio with the raw
+    cell, so one multi-megabyte text column still raised E088 -- swallowed, so
+    the page reported zero findings."""
+
+    def _text_with_needles(self, length: int, positions: list[int]) -> str:
+        buf = list("x" * length)
+        for pos in positions:
+            buf[pos : pos + len(_FakeAnalyzer.NEEDLE)] = _FakeAnalyzer.NEEDLE
+        return "".join(buf)
+
+    def test_short_text_is_a_single_pass(self):
+        analyzer = _FakeAnalyzer()
+        detector = _detector_with_analyzer(analyzer)
+
+        detector._analyze_content("contact alice@example.com today")
+
+        assert len(analyzer.calls) == 1
+
+    def test_oversized_text_is_split_before_reaching_presidio(self):
+        analyzer = _FakeAnalyzer()
+        detector = _detector_with_analyzer(analyzer)
+        text = self._text_with_needles(2_023_954, [1_500_000])
+
+        detector._analyze_content(text)
+
+        assert len(analyzer.calls) > 1
+        limit = PIIDetector._SPACY_DEFAULT_MAX_LENGTH
+        assert all(len(call) <= limit for call in analyzer.calls)
+
+    def test_offsets_come_back_absolute(self):
+        analyzer = _FakeAnalyzer()
+        detector = _detector_with_analyzer(analyzer)
+        text = self._text_with_needles(2_023_954, [10, 640_000, 1_500_000, 2_000_000])
+
+        results = detector._analyze_content(text)
+
+        # A finding past the first window is exactly what the old code lost.
+        assert [r.start for r in results] == [10, 640_000, 1_500_000, 2_000_000]
+        assert all(text[r.start : r.end] == _FakeAnalyzer.NEEDLE for r in results)
+
+    def test_overlap_does_not_duplicate_findings(self):
+        analyzer = _FakeAnalyzer()
+        detector = _detector_with_analyzer(analyzer)
+        window = PIIDetector._AUTO_CHUNK_CHARS
+        # Sitting inside the overlap zone, this one is seen by two windows.
+        seam = window - (PIIDetector._AUTO_CHUNK_OVERLAP // 2)
+        text = self._text_with_needles(1_200_000, [seam])
+
+        results = detector._analyze_content(text)
+
+        assert len(results) == 1
+        assert results[0].start == seam
+
+    def test_oversized_tabular_cell_is_analyzed_not_dropped(self):
+        analyzer = _FakeAnalyzer()
+        detector = _detector_with_analyzer(analyzer)
+        cell_value = self._text_with_needles(2_023_954, [1_800_000])
+
+        results = detector._analyze_structured_cell(
+            cell_value, allowed_entity_types={"EMAIL_ADDRESS"}
+        )
+
+        assert [r.start for r in results] == [1_800_000]
+        assert cell_value[results[0].start : results[0].end] == _FakeAnalyzer.NEEDLE
+
+    def test_chunk_size_larger_than_the_spacy_limit_is_still_safe(self):
+        # An explicit chunk_size above the ceiling used to hand Presidio an
+        # oversized window just the same.
+        analyzer = _FakeAnalyzer()
+        detector = _detector_with_analyzer(
+            analyzer, _Cfg(chunk_size=ChunkSize(root=1_500_000), chunk_overlap=ChunkOverlap(root=0))
+        )
+        text = self._text_with_needles(2_023_954, [1_400_000])
+
+        for chunk, _offset in detector._iter_chunks(text):
+            detector._analyze_content(chunk)
+
+        limit = PIIDetector._SPACY_DEFAULT_MAX_LENGTH
+        assert all(len(call) <= limit for call in analyzer.calls)

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/tests/test_outputs.py

@@ -7,7 +7,7 @@ from typing import Any
 
 import pytest
 
-from src.outputs.base import OutputRuntimeContext
+from src.outputs.base import DEFAULT_REST_TIMEOUT_SEC, OutputRuntimeContext
 from src.outputs.console import ConsoleOutputSink
 from src.outputs.factory import resolve_output_settings
 from src.outputs.file import FileOutputSink
@@ -87,6 +87,32 @@ def test_output_resolution_rest_uses_api_url_env_fallback(
     assert settings.rest_url == "http://api-env.example:8000"
 
 
+def test_output_resolution_uses_the_five_minute_read_timeout_by_default() -> None:
+    settings = resolve_output_settings(_args(source_id="source-1"))
+
+    assert settings.rest_timeout_sec == DEFAULT_REST_TIMEOUT_SEC == 300
+
+
+def test_output_resolution_env_overrides_the_read_timeout(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("CLASSIFYRE_OUTPUT_REST_TIMEOUT_SEC", "45")
+
+    settings = resolve_output_settings(_args(source_id="source-1"))
+
+    assert settings.rest_timeout_sec == 45
+
+
+def test_output_resolution_treats_non_positive_read_timeout_as_unlimited(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("CLASSIFYRE_OUTPUT_REST_TIMEOUT_SEC", "0")
+
+    settings = resolve_output_settings(_args(source_id="source-1"))
+
+    assert settings.rest_timeout_sec is None
+
+
 def test_output_resolution_defaults_to_console_without_source_id() -> None:
     settings = resolve_output_settings(_args())
 
@@ -167,7 +193,7 @@ class _FakeSession:
         method: str,
         url: str,
         json: dict[str, Any] | None,
-        timeout: int,
+        timeout: tuple[float, float | None],
     ) -> _FakeResponse:
         self.calls.append(
             {
@@ -199,7 +225,9 @@ async def test_rest_output_managed_runner_flow(monkeypatch: pytest.MonkeyPatch)
         managed_runner=True,
         batch_size=20,
     )
-    sink = RestOutputSink(context, base_url="http://localhost:8000", timeout_sec=30)
+    sink = RestOutputSink(
+        context, base_url="http://localhost:8000", timeout_sec=DEFAULT_REST_TIMEOUT_SEC
+    )
 
     await sink.start()
     await sink.emit_batch([{"hash": "h1"}])
@@ -231,7 +259,7 @@ async def test_rest_output_keeps_namespace_base_for_discovery_and_finding_batche
     sink = RestOutputSink(
         context,
         base_url="http://api.internal/prefix/namespace-id/",
-        timeout_sec=30,
+        timeout_sec=DEFAULT_REST_TIMEOUT_SEC,
     )
 
     await sink.register_discovered_assets(["hash-1", "hash-2"])
@@ -267,7 +295,7 @@ async def test_register_discovered_assets_returns_a_bare_list_without_payload_cu
     sink = RestOutputSink(
         OutputRuntimeContext(source_id="s", runner_id="r", managed_runner=True, batch_size=20),
         base_url="http://localhost:8000",
-        timeout_sec=30,
+        timeout_sec=DEFAULT_REST_TIMEOUT_SEC,
     )
 
     result = await sink.register_discovered_assets(["a", "b"], include_scan_cache=True)
@@ -297,7 +325,7 @@ async def test_register_discovered_assets_collects_payload_cursors(
     sink = RestOutputSink(
         OutputRuntimeContext(source_id="s", runner_id="r", managed_runner=True, batch_size=20),
         base_url="http://localhost:8000",
-        timeout_sec=30,
+        timeout_sec=DEFAULT_REST_TIMEOUT_SEC,
     )
 
     cache, cursors = await sink.register_discovered_assets(
@@ -320,7 +348,7 @@ async def test_asset_payload_carries_the_payload_cursor(
     sink = RestOutputSink(
         OutputRuntimeContext(source_id="s", runner_id="r", managed_runner=True, batch_size=20),
         base_url="http://localhost:8000",
-        timeout_sec=30,
+        timeout_sec=DEFAULT_REST_TIMEOUT_SEC,
     )
 
     cursor = {"v": 1, "offset": 300, "exhausted": False, "checksum": "c1"}
@@ -342,7 +370,9 @@ async def test_rest_output_finalize_includes_sampling_cursor(
         managed_runner=True,
         batch_size=20,
     )
-    sink = RestOutputSink(context, base_url="http://localhost:8000", timeout_sec=30)
+    sink = RestOutputSink(
+        context, base_url="http://localhost:8000", timeout_sec=DEFAULT_REST_TIMEOUT_SEC
+    )
 
     cursor = {"tables": {"db_#_users": {"pk": [42]}}}
     sink.set_sampling_cursor(cursor)
@@ -373,7 +403,9 @@ async def test_rest_output_omits_null_fields_in_nested_payload(
         managed_runner=True,
         batch_size=20,
     )
-    sink = RestOutputSink(context, base_url="http://localhost:8000", timeout_sec=30)
+    sink = RestOutputSink(
+        context, base_url="http://localhost:8000", timeout_sec=DEFAULT_REST_TIMEOUT_SEC
+    )
 
     await sink.start()
     await sink.emit_batch(
@@ -425,7 +457,9 @@ async def test_rest_output_auto_creates_external_runner(monkeypatch: pytest.Monk
         managed_runner=False,
         batch_size=20,
     )
-    sink = RestOutputSink(context, base_url="http://localhost:8000", timeout_sec=30)
+    sink = RestOutputSink(
+        context, base_url="http://localhost:8000", timeout_sec=DEFAULT_REST_TIMEOUT_SEC
+    )
 
     await sink.start()
     await sink.emit_batch([{"hash": "h1"}])
@@ -447,7 +481,9 @@ async def test_rest_output_fail_marks_runner_error(monkeypatch: pytest.MonkeyPat
         managed_runner=False,
         batch_size=20,
     )
-    sink = RestOutputSink(context, base_url="http://localhost:8000", timeout_sec=30)
+    sink = RestOutputSink(
+        context, base_url="http://localhost:8000", timeout_sec=DEFAULT_REST_TIMEOUT_SEC
+    )
 
     await sink.start()
     await sink.fail(RuntimeError("boom"))
@@ -470,7 +506,9 @@ async def test_rest_output_fail_marks_managed_runner_error(
         managed_runner=True,
         batch_size=20,
     )
-    sink = RestOutputSink(context, base_url="http://localhost:8000", timeout_sec=30)
+    sink = RestOutputSink(
+        context, base_url="http://localhost:8000", timeout_sec=DEFAULT_REST_TIMEOUT_SEC
+    )
 
     await sink.start()
     await sink.fail(RuntimeError("boom"))

{classifyre_cli-0.4.86 → classifyre_cli-0.4.87}/uv.lock

@@ -830,7 +830,7 @@ wheels = [
 
 [[package]]
 name = "classifyre-cli"
-version = "0.4.86"
+version = "0.4.87"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },
@@ -1369,7 +1369,7 @@ youtube = [
 
 [[package]]
 name = "classifyre-schemas"
-version = "0.4.86"
+version = "0.4.87"
 source = { editable = "../../packages/schemas" }
 dependencies = [
     { name = "fastjsonschema" },

classifyre_cli-0.4.86/.turbo/turbo-build.log

@@ -1,3 +0,0 @@
-$ uv sync
-Resolved 307 packages in 247ms
-Checked 51 packages in 1ms