classifyre-cli 0.6.5__tar.gz → 0.6.7__tar.gz

classifyre_cli-0.6.7/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 325 packages in 264ms
+Checked 51 packages in 6ms

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.5
+Version: 0.6.7
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.5",
+  "version": "0.6.7",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.5"
+version = "0.6.7"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/src/notebook/sdk.py

@@ -47,6 +47,7 @@ from ..graph.edges import (
     same_as,
     uses,
 )
+from ..utils.sampling_cursor import check_cursor_fits
 from ..utils.urn import Urn
 from .cohort import CohortItem, normalize_bands, select_cohort
 from .files import DEFAULT_PAGE_SIZE, ParsedContent, pages, parse
@@ -700,7 +701,13 @@ class Context:
         no longer implies a deleted one. Worse, the discard was silent: the call
         succeeded, the notebook believed it had saved state, and the next run
         started from nothing.
+
+        Raises ``ValueError`` for a cursor too large to start the next run with
+        (128 KiB compressed). Keep a cursor to positions and counters; state
+        that grows with the corpus belongs in assets, read back with
+        ``ctx.query_assets()``.
         """
+        check_cursor_fits(cursor)
         self._next_cursor = dict(cursor)
 
     @property

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/src/sources/base.py

@@ -1,5 +1,3 @@
-import base64
-import json
 import logging
 import os
 import threading
@@ -13,6 +11,7 @@ from ..outputs.rest import IngestEdge
 if TYPE_CHECKING:
     from ..utils.file_parser import ParsedBytes
 from ..utils.hashing import calculate_checksum, normalize_http_url
+from ..utils.sampling_cursor import decode_sampling_cursor
 from ..utils.validation import validate_output
 from .recipe_normalizer import normalize_source_recipe
 
@@ -34,8 +33,8 @@ class BaseSource(ABC):
     # Default batch size for streaming asset results
     BATCH_SIZE: int = 50
     HAS_SUCCESSFUL_RUN_ENV = "CLASSIFYRE_SOURCE_HAS_SUCCESSFUL_RUN"
-    # The API injects the saved AUTOMATIC sampling cursor here (base64-encoded
-    # JSON) before launching the CLI job. The recipe itself cannot carry it
+    # The API injects the saved cursor here (gzip + base64 JSON, see
+    # utils/sampling_cursor.py) before launching the CLI job. The recipe itself cannot carry it
     # because every source schema sets ``additionalProperties: false``.
     SAMPLING_CURSOR_ENV = "CLASSIFYRE_SAMPLING_CURSOR"
 
@@ -128,12 +127,10 @@ class BaseSource(ABC):
         if not raw:
             return {}
         try:
-            decoded = base64.b64decode(raw).decode("utf-8")
-            data = json.loads(decoded)
+            return decode_sampling_cursor(raw)
         except Exception as exc:
             logger.warning("Ignoring malformed %s: %s", self.SAMPLING_CURSOR_ENV, exc)
             return {}
-        return data if isinstance(data, dict) else {}
 
     def sampling_cursor(self) -> dict[str, Any]:
         """Return the cursor saved by the previous run (empty on first run)."""

classifyre_cli-0.6.7/src/utils/sampling_cursor.py

@@ -0,0 +1,45 @@
+"""How a source's saved cursor travels from the API to a scan.
+
+The API hands the cursor to the CLI in one environment variable, gzip then
+base64. Linux caps a single environment variable at 128 KiB (MAX_ARG_STRLEN);
+over it the process never starts and the kernel's "argument list too long"
+names neither the variable nor its size. So the size is checked on both ends:
+the API refuses to launch with a cursor that does not fit, and a notebook is
+told in the run that tries to *save* one, while it can still do something
+about it.
+"""
+
+from __future__ import annotations
+
+import base64
+import gzip
+import json
+from collections.abc import Mapping
+from typing import Any
+
+#: The limit for one environment variable, and so for an encoded cursor.
+MAX_ENCODED_CURSOR_BYTES = 128 * 1024
+
+
+def encode_sampling_cursor(cursor: Mapping[str, Any]) -> str:
+    """The cursor as it is put into the environment variable."""
+    raw = json.dumps(cursor, separators=(",", ":")).encode("utf-8")
+    return base64.b64encode(gzip.compress(raw)).decode("ascii")
+
+
+def decode_sampling_cursor(encoded: str) -> dict[str, Any]:
+    """The cursor an environment variable carries. Raises on malformed input."""
+    data = json.loads(gzip.decompress(base64.b64decode(encoded)).decode("utf-8"))
+    return data if isinstance(data, dict) else {}
+
+
+def check_cursor_fits(cursor: Mapping[str, Any]) -> None:
+    """Refuse a cursor the next run could not be started with."""
+    size = len(encode_sampling_cursor(cursor))
+    if size > MAX_ENCODED_CURSOR_BYTES:
+        raise ValueError(
+            f"This cursor is {size} bytes compressed, over the "
+            f"{MAX_ENCODED_CURSOR_BYTES}-byte limit: the next run could not be "
+            "started with it. A cursor records where a run stopped; keep bulk "
+            "state in assets and read it back with ctx.query_assets()."
+        )

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_custom_source.py

@@ -20,6 +20,7 @@ from src.sources.asset_metadata import is_open_kind, validate_metadata
 from src.sources.custom.env import ALLOWED_ENV_KEYS, scrubbed_environment
 from src.sources.custom.source import CustomSource, CustomSourceError
 from src.utils.hashing import hash_id
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 SIMPLE_NOTEBOOK = """from classifyre import Asset, ctx
 
@@ -344,12 +345,10 @@ def test_automatic_strategy_pages_across_runs(monkeypatch: pytest.MonkeyPatch) -
 
     # The API carries the cursor to the next run through the environment,
     # because the recipe itself forbids extra keys.
-    import base64
-    import json
 
     monkeypatch.setenv(
         CustomSource.SAMPLING_CURSOR_ENV,
-        base64.b64encode(json.dumps(cursor).encode()).decode(),
+        encode_sampling_cursor(cursor),
     )
     second = get_source(dict(recipe), source_id="s", runner_id="r")
     try:
@@ -378,13 +377,11 @@ def test_an_automatic_window_keeps_the_notebooks_stored_cursor(
 ) -> None:
     # GENESIS field report P15: a window stopped the notebook before it reached
     # ctx.set_cursor(), and the positional offset replaced its whole state.
-    import base64
-    import json
 
     stored = {"no_data": {"51000-0013": "2026-09-13"}, "seed_applied": True}
     monkeypatch.setenv(
         CustomSource.SAMPLING_CURSOR_ENV,
-        base64.b64encode(json.dumps(stored).encode()).decode(),
+        encode_sampling_cursor(stored),
     )
     recipe = build_recipe(counting_notebook(25))
     recipe["sampling"] = {"strategy": "AUTOMATIC", "rows_per_page": 10}
@@ -1020,8 +1017,6 @@ def test_random_is_bounded_and_actually_random_without_cooperation(source) -> No
 
 
 def test_automatic_pages_without_cooperation(monkeypatch) -> None:
-    import base64
-    import json
 
     recipe = build_recipe(IGNORES_CTX)
     recipe["sampling"] = {"strategy": "AUTOMATIC", "rows_per_page": 10}
@@ -1032,7 +1027,7 @@ def test_automatic_pages_without_cooperation(monkeypatch) -> None:
         if cursor is not None:
             monkeypatch.setenv(
                 CustomSource.SAMPLING_CURSOR_ENV,
-                base64.b64encode(json.dumps(cursor).encode()).decode(),
+                encode_sampling_cursor(cursor),
             )
         instance = get_source(dict(recipe), source_id="s", runner_id="r")
         try:
@@ -1050,8 +1045,6 @@ def test_a_notebook_can_take_over_paging_from_the_runtime(monkeypatch) -> None:
     # Without this, run N makes extract() produce N*page items to deliver page
     # of them -- fine for a list in memory, quadratic when each item is a
     # request. Reading ctx.offset hands the skipping to the notebook.
-    import base64
-    import json
 
     notebook = """from classifyre import Asset, ctx
 
@@ -1070,7 +1063,7 @@ def extract():
 
     monkeypatch.setenv(
         CustomSource.SAMPLING_CURSOR_ENV,
-        base64.b64encode(json.dumps({"assets": 20}).encode()).decode(),
+        encode_sampling_cursor({"assets": 20}),
     )
     instance = get_source(dict(recipe), source_id="s", runner_id="r")
     try:

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_dropbox_source.py

@@ -11,6 +11,7 @@ from src.models.generated_single_asset_scan_results import AssetType as OutputAs
 from src.sources.asset_metadata import resolve_fields
 from src.sources.dropbox.source import DropboxObjectRef, DropboxScanTarget, DropboxSource
 from src.sources.object_storage.base import ContentSnapshot
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 
 def _recipe(
@@ -146,12 +147,9 @@ def test_dropbox_sampling_automatic_advances_window_across_runs(monkeypatch):
     assert [item.key for item in first] == [f"/f{index:02d}.txt" for index in range(10)]
     assert cursor == {"objects": 10}
 
-    import base64
-    import json
-
     monkeypatch.setenv(
         DropboxSource.SAMPLING_CURSOR_ENV,
-        base64.b64encode(json.dumps(cursor).encode()).decode(),
+        encode_sampling_cursor(cursor),
     )
     resumed = DropboxSource(_recipe(strategy="AUTOMATIC", rows_per_page=10))
     second = resumed._apply_sampling(refs)

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_elasticsearch_source.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-import base64
-import json
 from typing import Any
 
 import pytest
 import requests
 
 from src.sources.elasticsearch.source import ElasticsearchSource
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 _INDICES = [
     {
@@ -237,7 +236,7 @@ def test_elasticsearch_automatic_strategy_starts_at_offset_zero(
 def test_elasticsearch_automatic_strategy_resumes_and_wraps_on_underfill(
     _patch_requests: dict[str, Any], monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    saved_cursor = base64.b64encode(json.dumps({"index:orders": 10}).encode()).decode()
+    saved_cursor = encode_sampling_cursor({"index:orders": 10})
     monkeypatch.setenv("CLASSIFYRE_SAMPLING_CURSOR", saved_cursor)
 
     src = ElasticsearchSource(_recipe(sampling={"strategy": "AUTOMATIC", "rows_per_page": 10}))
@@ -274,7 +273,7 @@ async def test_elasticsearch_all_strategy_scrolls_the_whole_index(
 def test_elasticsearch_automatic_strategy_wraps_at_max_result_window(
     _patch_requests: dict[str, Any], monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    saved_cursor = base64.b64encode(json.dumps({"index:orders": 10}).encode()).decode()
+    saved_cursor = encode_sampling_cursor({"index:orders": 10})
     monkeypatch.setenv("CLASSIFYRE_SAMPLING_CURSOR", saved_cursor)
 
     src = ElasticsearchSource(_recipe(sampling={"strategy": "AUTOMATIC", "rows_per_page": 10}))

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_google_workspace_source.py

@@ -13,6 +13,7 @@ from src.sources.google_workspace.source import (
     FileRef,
     GoogleWorkspaceSource,
 )
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 
 def _base_recipe(auth_mode: str = "service_account", **overrides: object) -> dict:
@@ -140,11 +141,9 @@ class TestSampling:
         assert cursor_after_first["drive_items:drive-1"] == 10
 
         # Simulate a second run reading the persisted cursor.
-        import base64
-        import json
         import os
 
-        encoded = base64.b64encode(json.dumps(cursor_after_first).encode("utf-8")).decode("utf-8")
+        encoded = encode_sampling_cursor(cursor_after_first)
         os.environ["CLASSIFYRE_SAMPLING_CURSOR"] = encoded
         try:
             source2 = GoogleWorkspaceSource(recipe)
@@ -158,9 +157,7 @@ class TestSampling:
         assert cursor_after_second["drive_items:drive-1"] == 20
 
         # A third page underfills (only 4 items remain) and should wrap to 0.
-        os.environ["CLASSIFYRE_SAMPLING_CURSOR"] = base64.b64encode(
-            json.dumps(cursor_after_second).encode("utf-8")
-        ).decode("utf-8")
+        os.environ["CLASSIFYRE_SAMPLING_CURSOR"] = encode_sampling_cursor(cursor_after_second)
         try:
             source3 = GoogleWorkspaceSource(recipe)
             third = source3._apply_sampling(items, "drive-1")

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_hugging_face_source.py

@@ -11,6 +11,7 @@ from src.models.generated_single_asset_scan_results import AssetType as OutputAs
 from src.sources.asset_metadata import resolve_fields
 from src.sources.hugging_face.source import HuggingFaceObjectRef, HuggingFaceSource
 from src.sources.object_storage.base import ContentSnapshot
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 REPO_SHA = "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c"
 
@@ -462,8 +463,6 @@ def test_hugging_face_sampling_random_is_deterministic():
 
 
 def test_hugging_face_sampling_automatic_advances_window_across_runs(monkeypatch):
-    import base64
-    import json
 
     refs = [_ref(f"data/f{index:02d}.parquet", days_ago=index) for index in range(25)]
 
@@ -476,7 +475,7 @@ def test_hugging_face_sampling_automatic_advances_window_across_runs(monkeypatch
 
     monkeypatch.setenv(
         HuggingFaceSource.SAMPLING_CURSOR_ENV,
-        base64.b64encode(json.dumps(cursor).encode()).decode(),
+        encode_sampling_cursor(cursor),
     )
     resumed = _source(strategy="AUTOMATIC", rows_per_page=10)
     second = resumed._apply_sampling(iter(refs))

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_kafka_source.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
 import base64
-import json
 from typing import Any
 
 import pytest
 
 from src.models.generated_input import SamplingStrategy
 from src.sources.kafka.source import KafkaSource
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 CURSOR_ENV = "CLASSIFYRE_SAMPLING_CURSOR"
 
@@ -19,7 +19,7 @@ _CA_PEM = "-----BEGIN CERTIFICATE-----\nZmFrZS1jYQ==\n-----END CERTIFICATE-----"
 
 
 def _encode_cursor(cursor: dict[str, Any]) -> str:
-    return base64.b64encode(json.dumps(cursor).encode()).decode()
+    return encode_sampling_cursor(cursor)
 
 
 # ── confluent-kafka fakes ────────────────────────────────────────────────

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_meilisearch_source.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-import base64
-import json
 from typing import Any
 
 import pytest
 import requests
 
 from src.sources.meilisearch.source import MeilisearchSource
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 _INDEX_LIST_PAGE1 = {
     "results": [
@@ -227,7 +226,7 @@ def test_meilisearch_automatic_strategy_starts_at_offset_zero(
 def test_meilisearch_automatic_strategy_resumes_and_wraps_on_underfill(
     _patch_requests: dict[str, Any], monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    saved_cursor = base64.b64encode(json.dumps({"index:products": 10}).encode()).decode()
+    saved_cursor = encode_sampling_cursor({"index:products": 10})
     monkeypatch.setenv("CLASSIFYRE_SAMPLING_CURSOR", saved_cursor)
 
     src = MeilisearchSource(_recipe(sampling={"strategy": "AUTOMATIC", "rows_per_page": 10}))

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_opensearch_source.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-import base64
-import json
 from typing import Any
 
 import pytest
 import requests
 
 from src.sources.opensearch.source import OpenSearchSource
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 _INDICES = [
     {
@@ -235,7 +234,7 @@ def test_opensearch_automatic_strategy_starts_at_offset_zero(
 def test_opensearch_automatic_strategy_resumes_and_wraps_on_underfill(
     _patch_requests: dict[str, Any], monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    saved_cursor = base64.b64encode(json.dumps({"index:logs-app": 10}).encode()).decode()
+    saved_cursor = encode_sampling_cursor({"index:logs-app": 10})
     monkeypatch.setenv("CLASSIFYRE_SAMPLING_CURSOR", saved_cursor)
 
     src = OpenSearchSource(_recipe(sampling={"strategy": "AUTOMATIC", "rows_per_page": 10}))

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_sampling_automatic.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-import base64
-import json
 from collections.abc import AsyncGenerator
 from types import SimpleNamespace
 from typing import Any
@@ -12,6 +10,7 @@ import pytest
 
 from src.models.generated_single_asset_scan_results import SingleAssetScanResults
 from src.sources.base import BaseSource
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 CURSOR_ENV = "CLASSIFYRE_SAMPLING_CURSOR"
 
@@ -47,7 +46,7 @@ def _src(rows: int = 3) -> _DummySource:
 
 
 def _encode(cursor: dict[str, Any]) -> str:
-    return base64.b64encode(json.dumps(cursor).encode()).decode()
+    return encode_sampling_cursor(cursor)
 
 
 def test_default_strategy_is_automatic() -> None:

classifyre_cli-0.6.7/tests/test_sampling_cursor.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import base64
+import json
+
+import pytest
+
+from src.notebook.sdk import Context
+from src.sources.base import BaseSource
+from src.utils.sampling_cursor import (
+    MAX_ENCODED_CURSOR_BYTES,
+    check_cursor_fits,
+    decode_sampling_cursor,
+    encode_sampling_cursor,
+)
+
+
+def _person_companies(persons: int) -> dict[str, list[str]]:
+    """The shape that broke a production source: a person -> companies map."""
+    return {
+        f"person-{i:06d}--19{i % 100:02d}-01-01": [f"{i:06d}a", f"{i + 1:06d}b"]
+        for i in range(persons)
+    }
+
+
+def test_cursor_round_trips() -> None:
+    cursor = {"cohort": {"register": {"newest": "606601k"}}, "companies": 1200}
+    assert decode_sampling_cursor(encode_sampling_cursor(cursor)) == cursor
+
+
+def test_a_cursor_too_large_as_plain_base64_fits_once_compressed() -> None:
+    cursor = {"person_companies": _person_companies(2400)}
+    plain = base64.b64encode(json.dumps(cursor).encode()).decode()
+    assert len(plain) > MAX_ENCODED_CURSOR_BYTES
+    assert len(encode_sampling_cursor(cursor)) < MAX_ENCODED_CURSOR_BYTES
+    check_cursor_fits(cursor)
+
+
+def test_a_cursor_that_cannot_start_the_next_run_is_refused() -> None:
+    # Random-looking values, so compression cannot rescue it.
+    cursor = {str(i): base64.b64encode(i.to_bytes(4, "big") * 12).decode() for i in range(40000)}
+    with pytest.raises(ValueError, match="over the 131072-byte limit"):
+        check_cursor_fits(cursor)
+
+
+def test_set_cursor_refuses_an_oversized_cursor_and_keeps_the_last_good_one() -> None:
+    ctx = Context()
+    ctx.set_cursor({"offset": 5})
+    huge = {str(i): base64.b64encode(i.to_bytes(4, "big") * 12).decode() for i in range(40000)}
+    with pytest.raises(ValueError, match=r"ctx\.query_assets\(\)"):
+        ctx.set_cursor(huge)
+    assert ctx.next_cursor == {"offset": 5}
+
+
+def test_a_malformed_cursor_is_ignored(monkeypatch: pytest.MonkeyPatch) -> None:
+    class _Source:
+        SAMPLING_CURSOR_ENV = BaseSource.SAMPLING_CURSOR_ENV
+
+    monkeypatch.setenv(BaseSource.SAMPLING_CURSOR_ENV, "not-a-cursor")
+    assert BaseSource._load_sampling_cursor(_Source()) == {}  # type: ignore[arg-type]

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_sandbox_source.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-import base64
-import json
 from datetime import UTC, datetime, timedelta
 
 import pytest
@@ -9,6 +7,7 @@ import pytest
 from src.sources import get_source, list_available_sources
 from src.sources.object_storage.base import ContentSnapshot, ObjectRef
 from src.sources.sandbox.source import SandboxSource
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 
 def _recipe(strategy: str = "ALL", rows_per_page: int = 10) -> dict:
@@ -76,7 +75,7 @@ def test_sandbox_automatic_sampling_uses_persisted_rolling_window(monkeypatch) -
 
     monkeypatch.setenv(
         "CLASSIFYRE_SAMPLING_CURSOR",
-        base64.b64encode(json.dumps(cursor).encode()).decode(),
+        encode_sampling_cursor(cursor),
     )
     second = SandboxSource(_recipe("AUTOMATIC"), source_id="source-1")
     try:

{classifyre_cli-0.6.5 → classifyre_cli-0.6.7}/tests/test_tabular_automatic_sampling.py

@@ -2,13 +2,12 @@
 
 from __future__ import annotations
 
-import base64
-import json
 from typing import Any
 
 import pytest
 
 from src.sources.mysql.source import MySQLSource, TableRef
+from src.utils.sampling_cursor import encode_sampling_cursor
 
 CURSOR_ENV = "CLASSIFYRE_SAMPLING_CURSOR"
 PAGE = 10  # rows_per_page (MySQL config enforces a minimum of 10)
@@ -94,7 +93,7 @@ def _make_source(
 
 
 def _encode(cursor: dict[str, Any]) -> str:
-    return base64.b64encode(json.dumps(cursor).encode()).decode()
+    return encode_sampling_cursor(cursor)
 
 
 _TABLE = TableRef(database="app_db", schema=None, table="users")