classifyre-cli 0.6.3__tar.gz → 0.6.4__tar.gz

classifyre_cli-0.6.4/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 325 packages in 269ms
+Checked 51 packages in 3ms

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.3
+Version: 0.6.4
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.3",
+  "version": "0.6.4",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.3"
+version = "0.6.4"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/scripts/generate_models.py

@@ -15,6 +15,7 @@ _PIPELINE_TYPE_DEFAULTS: dict[str, str] = {
     "RegexPipelineSchema": "REGEX",
     "LLMPipelineSchema": "LLM",
     "TagPipelineSchema": "TAG",
+    "CodeDetectorPipelineSchema": "CODE_DETECTOR",
 }
 
 # Pipeline schema classes whose `severity` field has a string default from JSON schema
@@ -27,6 +28,7 @@ _SEVERITY_ENUM_DEFAULT_CLASSES = {
     "LLMPipelineSchema": "info",
     "TextClassificationPipelineSchema": "info",
     "TagPipelineSchema": "medium",
+    "CodeDetectorPipelineSchema": "medium",
 }
 
 
@@ -60,6 +62,15 @@ def _patch_severity_enum_defaults(source: str) -> str:
     return source
 
 
+def _patch_category_enum_defaults(source: str) -> str:
+    """Same fix as the severity one, for the code detector's ``category`` default."""
+    pattern = (
+        r"(class CodeDetectorPipelineSchema\(.*?category: DetectorCategory \| None = Field\(\n\s+)"
+        r"'QUALITY'(\s*,)"
+    )
+    return re.sub(pattern, r"\1DetectorCategory.QUALITY\2", source, flags=re.DOTALL)
+
+
 def run_codegen(input_file):
     """Generate Pydantic models from a single JSON schema file."""
     cmd = [
@@ -107,6 +118,7 @@ def main():
     content = run_codegen(detector_schema)
     content = _patch_pipeline_type_defaults(content)
     content = _patch_severity_enum_defaults(content)
+    content = _patch_category_enum_defaults(content)
     (MODEL_DIR / "generated_detectors.py").write_text(content)
     print("Wrote src/models/generated_detectors.py")
 

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/src/augmentation/session.py

@@ -18,7 +18,6 @@ silently-degraded run visible.
 from __future__ import annotations
 
 import asyncio
-import base64
 import json
 import logging
 from collections.abc import Mapping
@@ -29,6 +28,7 @@ from ..graph.edges import edge_from_payload
 from ..notebook.child import ChildProcessError, NotebookChildProcess
 from ..notebook.groups import warm_declared_groups
 from ..notebook.packages import install as install_packages
+from ..pipeline.asset_payload_server import AssetPayloadServer
 from ..utils.hashing import unhash_id
 from ..utils.urn import normalize_urn_or_none
 from .contract import validate_augmentation_notebook
@@ -143,10 +143,6 @@ class AugmentationSession:
         self._warnings: list[str] = []
         self._warning_set: set[str] = set()
         self.stats = AugmentationStats()
-        # Run-scoped memo of fetched payloads, keyed by fetch candidate id.
-        # Bounded: one in-flight asset's worth per entry, each capped at
-        # max_payload_bytes, dropped by evict_asset_cache().
-        self._memo: dict[str, dict[str, Any]] = {}
 
     # -- construction ----------------------------------------------------------
 
@@ -295,7 +291,6 @@ class AugmentationSession:
                 slot.child.terminate()
             except Exception:
                 pass
-        self._memo.clear()
 
     # -- per asset -----------------------------------------------------------------
 
@@ -314,163 +309,34 @@ class AugmentationSession:
             "mime_type": None,
         }
 
-    def _candidates(self, asset: Any) -> list[str]:
-        seen: list[str] = []
-        for candidate in (getattr(asset, "external_url", ""), getattr(asset, "hash", "")):
-            value = str(candidate or "").strip()
-            if value and value not in seen:
-                seen.append(value)
-        return seen
+    @property
+    def _payloads(self) -> AssetPayloadServer:
+        """The source's shared payload server.
+
+        Shared with the code detectors that run on the same asset later in the
+        scan, so a payload augmentation fetched is not fetched again (G1 R20).
+        """
+        return AssetPayloadServer.shared(
+            self._source, max_payload_bytes=self._limits["max_payload_bytes"]
+        )
 
     async def _serve_need(self, asset: Any, memo_key: str, frame: dict[str, Any]) -> Any:
-        op = str(frame.get("op") or "")
-        if op == "payload":
-            fetched = await self._fetch_payload(asset)
-            if fetched is None:
-                return {"bytes_b64": None, "mime": None}
-            raw, mime = fetched
-            return {"bytes_b64": base64.b64encode(raw).decode("ascii"), "mime": mime}
-        if op == "raw_pages":
-            pages = await self._fetch_raw_pages(asset)
-            return {"pages": pages}
-        if op == "text":
-            pages = await self._fetch_text_pages(asset)
-            whole = "".join(pages)
-            return {"text": whole, "pages": pages}
-        raise ChildProcessError(f"Unknown payload op {op!r}")
+        try:
+            return await self._payloads.serve(asset, frame)
+        except ValueError as exc:
+            raise ChildProcessError(str(exc)) from exc
 
     async def _fetch_payload(self, asset: Any) -> tuple[bytes, str] | None:
         """Raw bytes for file-shaped sources; None for row-shaped ones."""
-        memo = self._memo_for(asset)
-        if "payload" in memo:
-            cached = memo["payload"]
-            return cached if cached is None else (bytes(cached[0]), str(cached[1]))
-        fetch = getattr(self._source, "fetch_content_bytes", None)
-        result: tuple[bytes, str] | None = None
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    result = await fetch(candidate)
-                except Exception as exc:
-                    logger.debug("Augmentation payload fetch failed for %s: %s", candidate, exc)
-                    continue
-                if result is not None:
-                    break
-        if result is not None:
-            raw, mime = result
-            cap = self._limits["max_payload_bytes"]
-            if len(raw) > cap:
-                logger.debug("Augmentation payload for %s truncated to %d bytes", candidate, cap)
-                raw = raw[:cap]
-            result = (raw, mime)
-            self._remember_bytes(asset, raw, mime)
-        memo["payload"] = result
-        return result
+        return await self._payloads.payload(asset)
 
     async def _fetch_raw_pages(self, asset: Any) -> list[str]:
         """The connector's own raw representation, page by page (capped)."""
-        memo = self._memo_for(asset)
-        if "raw_pages" in memo:
-            return [str(page) for page in memo["raw_pages"]]
-        pages: list[str] = []
-        fetch = getattr(self._source, "fetch_content_pages", None)
-        cap = self._limits["max_payload_bytes"]
-        used = 0
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    async for raw, _text in fetch(candidate):
-                        if not raw:
-                            continue
-                        chunk = str(raw)
-                        if used + len(chunk) > cap:
-                            chunk = chunk[: max(0, cap - used)]
-                        if chunk:
-                            pages.append(chunk)
-                            used += len(chunk)
-                        if used >= cap or len(pages) >= 50:
-                            break
-                    if pages:
-                        break
-                except Exception as exc:
-                    logger.debug("Augmentation raw fetch failed for %s: %s", candidate, exc)
-                    continue
-        memo["raw_pages"] = pages
-        return pages
+        return await self._payloads.raw_pages(asset)
 
     async def _fetch_text_pages(self, asset: Any) -> list[str]:
         """Extracted text, page by page (capped)."""
-        memo = self._memo_for(asset)
-        if "text_pages" in memo:
-            return [str(page) for page in memo["text_pages"]]
-        pages: list[str] = []
-        fetch = getattr(self._source, "fetch_content_pages", None)
-        cap = self._limits["max_payload_bytes"]
-        used = 0
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    async for _raw, text in fetch(candidate):
-                        if not text:
-                            continue
-                        chunk = str(text)
-                        if used + len(chunk) > cap:
-                            chunk = chunk[: max(0, cap - used)]
-                        if chunk:
-                            pages.append(chunk)
-                            used += len(chunk)
-                        if used >= cap or len(pages) >= 50:
-                            break
-                    if pages:
-                        break
-                except Exception as exc:
-                    logger.debug("Augmentation text fetch failed for %s: %s", candidate, exc)
-                    continue
-        if not pages:
-            provider = getattr(self._source, "fetch_content_bytes", None)
-            if callable(provider):
-                fetched = await self._fetch_payload(asset)
-                if fetched is not None:
-                    raw, mime = fetched
-                    try:
-                        text_pages = await asyncio.to_thread(
-                            list, self._source.iter_asset_pages(raw, mime)
-                        )
-                        for page in text_pages[:50]:
-                            chunk = str(page)
-                            if used + len(chunk) > cap:
-                                chunk = chunk[: max(0, cap - used)]
-                            if chunk:
-                                pages.append(chunk)
-                                used += len(chunk)
-                            if used >= cap:
-                                break
-                    except Exception as exc:
-                        logger.debug("Augmentation text fallback failed: %s", exc)
-        memo["text_pages"] = pages
-        return pages
-
-    def _memo_for(self, asset: Any) -> dict[str, Any]:
-        key = str(getattr(asset, "hash", "") or "")
-        return self._memo.setdefault(key, {})
-
-    def _remember_bytes(self, asset: Any, raw: bytes, mime: str) -> None:
-        """Mirror fetched bytes onto the source so the pipeline reuses them.
-
-        ``ParsedContentProvider.fetch_bytes`` consults this memo first instead
-        of re-downloading — the fetch augmentation already paid for is not
-        paid twice. Registered under every candidate id the pipeline tries
-        (external URL, then hash), and dropped by ``evict`` at the end of the
-        asset.
-        """
-        remember = getattr(self._source, "remember_augmentation_bytes", None)
-        if not callable(remember):
-            return
-        for candidate in self._candidates(asset):
-            try:
-                remember(candidate, raw, mime)
-            except Exception:
-                pass
+        return await self._payloads.text_pages(asset)
 
     async def augment(self, asset: Any) -> Any:
         """Enrich one asset. Never raises: failure degrades to a warning."""
@@ -718,13 +584,4 @@ class AugmentationSession:
 
     def evict(self, asset: Any) -> None:
         """Drop one asset's payload memo. Called alongside evict_asset_cache."""
-        self._memo.pop(str(getattr(asset, "hash", "") or ""), None)
-        forget = getattr(self._source, "forget_augmentation_bytes", None)
-        if not callable(forget):
-            return
-        # The bytes were mirrored under every candidate id; drop them all.
-        for candidate in self._candidates(asset):
-            try:
-                forget(candidate)
-            except Exception:
-                pass
+        self._payloads.evict(asset)

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/src/detectors/custom/runners/__init__.py

@@ -13,6 +13,7 @@ from ._base import (
     BaseRunner,
     _resolve_pipeline_severity,
 )
+from ._custom_detector import CustomDetectorRunner
 from ._factory import create_runner
 from ._gliner2 import (
     GLiNER2Runner,
@@ -35,6 +36,7 @@ __all__ = [
     "_IMAGE_CONTENT_TYPES",
     "_TEXT_CONTENT_TYPES",
     "BaseRunner",
+    "CustomDetectorRunner",
     "GLiNER2Runner",
     "ImageClassificationRunner",
     "LLMRunner",

classifyre_cli-0.6.4/src/detectors/custom/runners/_custom_detector.py

@@ -0,0 +1,72 @@
+"""Runner for the ``CODE_DETECTOR`` pipeline type: a code detector.
+
+Like ``TAG`` it never runs over a page of text: ``detect()`` returns nothing and
+no content type is advertised, so the text, binary and link passes never
+schedule it and the worker pool never receives it. The detector pipeline
+recognises it by this class and calls it once per asset, after every other
+detector, through the :class:`CustomDetectorSession` it owns.
+
+The session is created lazily on first use and bound to the scan's source,
+because it needs the source to serve payload and holds child processes that
+must not exist in the worker pool's processes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ....models.generated_detectors import CodeDetectorPipelineSchema, PipelineResult
+from ....models.generated_single_asset_scan_results import DetectionResult
+from ._base import BaseRunner
+
+
+class CustomDetectorRunner(BaseRunner):
+    """Asset-scoped runner: judged per asset by a notebook, never per page."""
+
+    asset_scoped = True
+
+    def __init__(
+        self,
+        schema: CodeDetectorPipelineSchema,
+        detector_key: str = "",
+        detector_name: str = "",
+    ) -> None:
+        self.schema = schema
+        self._detector_key = detector_key
+        self._detector_name = detector_name
+        self._session: Any = None
+
+    def run(self, text: str) -> PipelineResult:
+        return PipelineResult(metadata={"runner": "CODE_DETECTOR"})
+
+    def detect(self, content: str | bytes, content_type: str) -> list[DetectionResult]:
+        """Never called per page. The pipeline calls ``session().detect(asset)``."""
+        return []
+
+    def get_supported_content_types(self) -> list[str]:
+        return []
+
+    @property
+    def deterministic(self) -> bool:
+        return self.schema.deterministic is not False
+
+    @property
+    def needs_findings(self) -> bool:
+        return bool(self.schema.needs_findings)
+
+    def session(self, source: Any) -> Any:
+        """The run's session, created on first use."""
+        if self._session is None:
+            from ...custom_detector.session import CustomDetectorSession
+
+            self._session = CustomDetectorSession(
+                key=self._detector_key,
+                name=self._detector_name,
+                schema=self.schema,
+                source=source,
+            )
+        return self._session
+
+    def close(self) -> None:
+        if self._session is not None:
+            self._session.close()

{classifyre_cli-0.6.3 → classifyre_cli-0.6.4}/src/detectors/custom/runners/_factory.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from ....models.generated_detectors import (
+    CodeDetectorPipelineSchema,
     GLiNER2PipelineSchema,
     ImageClassificationPipelineSchema,
     LLMPipelineSchema,
@@ -12,6 +13,7 @@ from ....models.generated_detectors import (
     TextClassificationPipelineSchema,
 )
 from ._base import BaseRunner
+from ._custom_detector import CustomDetectorRunner
 from ._gliner2 import GLiNER2Runner
 from ._image_classification import ImageClassificationRunner
 from ._llm import LLMRunner
@@ -30,6 +32,7 @@ def create_runner(
         | ImageClassificationPipelineSchema
         | ObjectDetectionPipelineSchema
         | TagPipelineSchema
+        | CodeDetectorPipelineSchema
     ),
     detector_key: str = "",
     detector_name: str = "",
@@ -50,5 +53,8 @@ def create_runner(
     # detector that is not supposed to run anything at all.
     if isinstance(schema, TagPipelineSchema):
         return TagRunner(schema, detector_key, detector_name)
+    # Same reason as TAG: a code detector must never fall through to GLiNER2.
+    if isinstance(schema, CodeDetectorPipelineSchema):
+        return CustomDetectorRunner(schema, detector_key, detector_name)
     # GLiNER2PipelineSchema is the default / backward-compat path
     return GLiNER2Runner(schema, detector_key, detector_name)

classifyre_cli-0.6.4/src/detectors/custom_detector/__init__.py

@@ -0,0 +1,9 @@
+"""Code detectors: the ``CODE_DETECTOR`` pipeline type.
+
+A notebook that defines ``detect(asset, ctx)`` and yields ``Finding`` objects,
+run in an isolated child process once per asset, after every other detector.
+"""
+
+from .session import CustomDetectorSession, DetectOutcome
+
+__all__ = ["CustomDetectorSession", "DetectOutcome"]

classifyre_cli-0.6.4/src/detectors/custom_detector/contract.py

@@ -0,0 +1,45 @@
+"""The interface a code detector's notebook must cover.
+
+A code detector answers one question -- "what is wrong with this asset?" -- so
+``detect(asset, ctx)`` is required and ``setup(ctx)`` optional. Checked at the
+three moments the connector contract is: on save (the API's cheap check),
+before a preview, and before every scan, because the recipe travels to a job
+that has to defend itself.
+
+Only the shape is checked, through the shared ``validate_module`` /
+``validate_notebook``. Isolation is the child process and its scrubbed
+environment, not anything an AST can decide.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+
+from ...notebook.contract import ContractReport, validate_module, validate_notebook
+from ...notebook.serialize import ModuleSource
+
+REQUIRED_FUNCTIONS = ("detect",)
+
+#: ``setup(ctx)`` runs once per worker before the first asset: load a list,
+#: a model, a lookup table into ``ctx.state``.
+OPTIONAL_FUNCTIONS = ("setup",)
+
+
+def validate_detector_module(module: ModuleSource) -> ContractReport:
+    """Parse the assembled module and check ``detect()`` exists."""
+    return validate_module(module, required=REQUIRED_FUNCTIONS)
+
+
+def validate_detector_notebook(
+    cells: Iterable[Mapping[str, object] | object],
+) -> ContractReport:
+    """Validate a code detector's cells."""
+    return validate_notebook(cells, required=REQUIRED_FUNCTIONS)
+
+
+__all__ = [
+    "OPTIONAL_FUNCTIONS",
+    "REQUIRED_FUNCTIONS",
+    "validate_detector_module",
+    "validate_detector_notebook",
+]

classifyre_cli-0.6.4/src/detectors/custom_detector/files.py

@@ -0,0 +1,102 @@
+"""Materialize a code detector's uploaded files for ``ctx.file(name)``.
+
+Downloaded by the parent, never by the child: the child is started without the
+API URL or the callback key, and handing it either would undo the isolation it
+exists for. The API injects ``files_runtime`` -- one entry per file, with a
+content hash and a download URL -- into the pipeline schema at dispatch.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import shutil
+import tempfile
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+import requests
+
+from ...outputs.base import DEFAULT_REST_TIMEOUT_SEC
+from ...outputs.rest import INTERNAL_KEY_HEADER
+from ...utils.source_files import CHUNK_BYTES, api_base_url
+
+logger = logging.getLogger(__name__)
+
+#: A directory prepared by the caller (the preview job's init container, a
+#: test) wins over downloading.
+FILES_DIR_ENV = "CLASSIFYRE_DETECTOR_FILES_DIR"
+
+
+def _safe_name(name: str) -> str:
+    base = Path(name).name or "file"
+    return re.sub(r"[^A-Za-z0-9._ -]", "_", base)[:200] or "file"
+
+
+def _entries(files_runtime: Iterable[Any] | None) -> list[dict[str, Any]]:
+    entries: list[dict[str, Any]] = []
+    for raw in files_runtime or []:
+        entry = raw.model_dump() if hasattr(raw, "model_dump") else raw
+        if isinstance(entry, Mapping) and entry.get("id") and entry.get("name"):
+            entries.append(dict(entry))
+    return entries
+
+
+def materialize_detector_files(
+    detector_id: str | None,
+    files_runtime: Iterable[Any] | None,
+) -> Path | None:
+    """Download every file into a fresh temp dir; None when there are none.
+
+    A file that fails to download is logged and skipped: a rule that reads one
+    list out of three should still see the two that arrived, and a rule that
+    needs the missing one fails loudly on ``ctx.file(name)`` with its name.
+    """
+    prepared = os.environ.get(FILES_DIR_ENV)
+    if prepared and Path(prepared).is_dir():
+        return Path(prepared)
+    entries = _entries(files_runtime)
+    if not entries:
+        return None
+    destination = Path(tempfile.mkdtemp(prefix="classifyre-detector-files-"))
+    headers = {"Connection": "close"}
+    key = os.environ.get("CLASSIFYRE_INTERNAL_KEY", "").strip()
+    if key:
+        headers[INTERNAL_KEY_HEADER] = key
+    written = 0
+    with requests.Session() as session:
+        for entry in entries:
+            url = str(entry.get("url") or "").strip()
+            if not url:
+                if not detector_id:
+                    continue
+                url = (
+                    f"{api_base_url()}/custom-detectors/{quote(detector_id, safe='')}"
+                    f"/files/{quote(str(entry['id']), safe='')}/content"
+                )
+            target = destination / _safe_name(str(entry["name"]))
+            try:
+                with session.get(
+                    url, headers=headers, stream=True, timeout=(30, DEFAULT_REST_TIMEOUT_SEC)
+                ) as response:
+                    response.raise_for_status()
+                    with target.open("wb") as handle:
+                        for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
+                            if chunk:
+                                handle.write(chunk)
+            except (requests.RequestException, OSError) as exc:
+                logger.warning("Could not download detector file %s: %s", entry.get("name"), exc)
+                target.unlink(missing_ok=True)
+                continue
+            written += 1
+    if not written:
+        shutil.rmtree(destination, ignore_errors=True)
+        return None
+    logger.info("Made %d detector file(s) available to ctx.files", written)
+    return destination
+
+
+__all__ = ["FILES_DIR_ENV", "materialize_detector_files"]