classifyre-cli 0.6.3__tar.gz → 0.6.5__tar.gz

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 325 packages in 183ms
+Resolved 325 packages in 219ms
 Checked 51 packages in 1ms

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.3
+Version: 0.6.5
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.3"
+version = "0.6.5"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/scripts/generate_models.py

@@ -15,6 +15,7 @@ _PIPELINE_TYPE_DEFAULTS: dict[str, str] = {
     "RegexPipelineSchema": "REGEX",
     "LLMPipelineSchema": "LLM",
     "TagPipelineSchema": "TAG",
+    "CodeDetectorPipelineSchema": "CODE_DETECTOR",
 }
 
 # Pipeline schema classes whose `severity` field has a string default from JSON schema
@@ -27,6 +28,7 @@ _SEVERITY_ENUM_DEFAULT_CLASSES = {
     "LLMPipelineSchema": "info",
     "TextClassificationPipelineSchema": "info",
     "TagPipelineSchema": "medium",
+    "CodeDetectorPipelineSchema": "medium",
 }
 
 
@@ -60,6 +62,15 @@ def _patch_severity_enum_defaults(source: str) -> str:
     return source
 
 
+def _patch_category_enum_defaults(source: str) -> str:
+    """Same fix as the severity one, for the code detector's ``category`` default."""
+    pattern = (
+        r"(class CodeDetectorPipelineSchema\(.*?category: DetectorCategory \| None = Field\(\n\s+)"
+        r"'QUALITY'(\s*,)"
+    )
+    return re.sub(pattern, r"\1DetectorCategory.QUALITY\2", source, flags=re.DOTALL)
+
+
 def run_codegen(input_file):
     """Generate Pydantic models from a single JSON schema file."""
     cmd = [
@@ -107,6 +118,7 @@ def main():
     content = run_codegen(detector_schema)
     content = _patch_pipeline_type_defaults(content)
     content = _patch_severity_enum_defaults(content)
+    content = _patch_category_enum_defaults(content)
     (MODEL_DIR / "generated_detectors.py").write_text(content)
     print("Wrote src/models/generated_detectors.py")
 

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/scripts/generate_sdk_completions.py

@@ -35,6 +35,7 @@ from src.notebook.contract import (  # noqa: E402
 from src.notebook.sdk import (  # noqa: E402
     Asset,
     Context,
+    Entity,
     FieldMapping,
     FlowType,
     NotebookFile,
@@ -259,6 +260,13 @@ def build() -> dict[str, Any]:
                 "documentation": _summary(Tag),
                 "insertText": "Tag",
             },
+            {
+                "label": "Entity",
+                "kind": "class",
+                "detail": 'Entity(name, type="ORGANIZATION", identifiers={...})',
+                "documentation": _summary(Entity),
+                "insertText": "Entity",
+            },
             {
                 "label": "Ref",
                 "kind": "class",
@@ -304,6 +312,10 @@ def build() -> dict[str, Any]:
                 "documentation": _summary(Tag),
                 "fields": _fields(Tag),
             },
+            "Entity": {
+                "documentation": _summary(Entity),
+                "fields": _fields(Entity),
+            },
             "AugmentedAsset": {
                 "documentation": _summary(augmentation_sdk.AugmentedAsset),
                 "members": _members(augmentation_sdk.AugmentedAsset),

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/src/augmentation/sdk.py

@@ -44,11 +44,13 @@ from ..graph.edges import (
     FlowType,
     Ref,
     contains,
+    edge_to_payload,
     flow,
     references,
     same_as,
     uses,
 )
+from ..graph.edges import means as means_edge
 from ..notebook.files import ParsedContent, pages, parse
 from ..notebook.sdk import (
     MODULE_NAME,
@@ -141,6 +143,7 @@ class AugmentedAsset:
         self._added_metadata: dict[str, Any] = {}
         self._tags: dict[str, str] = {}
         self._links: list[str] = []
+        self._means: list[dict[str, Any]] = []
         self._new_urn: str | None = None
         self.warnings: list[str] = []
 
@@ -313,6 +316,33 @@ class AugmentedAsset:
         if target not in self._links:
             self._links.append(target)
 
+    def means(
+        self,
+        key: str,
+        *,
+        evidence: Mapping[str, Any] | None = None,
+        confidence: float | None = None,
+    ) -> None:
+        """Declare that this asset is about a glossary concept or entity, by key.
+
+        Additive, like ``tag()`` and ``link()``: a DECLARED semantic link. The
+        term need not exist yet; the declaration links once it is defined. An
+        invalid key is skipped with a warning.
+        """
+        try:
+            edge = means_edge(
+                Ref.asset(self._hash),
+                Ref.term(str(key or "")),
+                evidence=dict(evidence or {}),
+                confidence=confidence,
+            )
+        except ValueError as exc:
+            self.warnings.append(f"Ignoring asset.means({key!r}): {exc}")
+            return
+        payload = edge_to_payload(edge)
+        if payload not in self._means:
+            self._means.append(payload)
+
     def set_urn(self, urn: str) -> None:
         """Set the asset's cross-system identity, only if the connector left it empty.
 
@@ -337,6 +367,7 @@ class AugmentedAsset:
             "metadata": dict(self._added_metadata),
             "tags": dict(self._tags),
             "links": list(self._links),
+            "means": list(self._means),
             "urn": self._new_urn,
             "warnings": list(self.warnings),
         }
@@ -418,6 +449,7 @@ def build_augmentation_module(context: AugmentContext) -> types.ModuleType:
     module.flow = flow  # type: ignore[attr-defined]
     module.contains = contains  # type: ignore[attr-defined]
     module.references = references  # type: ignore[attr-defined]
+    module.means = means_edge  # type: ignore[attr-defined]
     module.same_as = same_as  # type: ignore[attr-defined]
     module.uses = uses  # type: ignore[attr-defined]
     module.urn_for = urn_for  # type: ignore[attr-defined]
@@ -439,6 +471,7 @@ def build_augmentation_module(context: AugmentContext) -> types.ModuleType:
         "contains",
         "ctx",
         "flow",
+        "means",
         "pages",
         "parse",
         "references",
@@ -470,6 +503,7 @@ def augmentation_namespace(context: AugmentContext) -> dict[str, Any]:
         "flow": flow,
         "contains": contains,
         "references": references,
+        "means": means_edge,
         "same_as": same_as,
         "uses": uses,
         "urn_for": urn_for,

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/src/augmentation/session.py

@@ -18,7 +18,6 @@ silently-degraded run visible.
 from __future__ import annotations
 
 import asyncio
-import base64
 import json
 import logging
 from collections.abc import Mapping
@@ -29,6 +28,7 @@ from ..graph.edges import edge_from_payload
 from ..notebook.child import ChildProcessError, NotebookChildProcess
 from ..notebook.groups import warm_declared_groups
 from ..notebook.packages import install as install_packages
+from ..pipeline.asset_payload_server import AssetPayloadServer
 from ..utils.hashing import unhash_id
 from ..utils.urn import normalize_urn_or_none
 from .contract import validate_augmentation_notebook
@@ -143,10 +143,6 @@ class AugmentationSession:
         self._warnings: list[str] = []
         self._warning_set: set[str] = set()
         self.stats = AugmentationStats()
-        # Run-scoped memo of fetched payloads, keyed by fetch candidate id.
-        # Bounded: one in-flight asset's worth per entry, each capped at
-        # max_payload_bytes, dropped by evict_asset_cache().
-        self._memo: dict[str, dict[str, Any]] = {}
 
     # -- construction ----------------------------------------------------------
 
@@ -295,7 +291,6 @@ class AugmentationSession:
                 slot.child.terminate()
             except Exception:
                 pass
-        self._memo.clear()
 
     # -- per asset -----------------------------------------------------------------
 
@@ -314,163 +309,34 @@ class AugmentationSession:
             "mime_type": None,
         }
 
-    def _candidates(self, asset: Any) -> list[str]:
-        seen: list[str] = []
-        for candidate in (getattr(asset, "external_url", ""), getattr(asset, "hash", "")):
-            value = str(candidate or "").strip()
-            if value and value not in seen:
-                seen.append(value)
-        return seen
+    @property
+    def _payloads(self) -> AssetPayloadServer:
+        """The source's shared payload server.
+
+        Shared with the code detectors that run on the same asset later in the
+        scan, so a payload augmentation fetched is not fetched again (G1 R20).
+        """
+        return AssetPayloadServer.shared(
+            self._source, max_payload_bytes=self._limits["max_payload_bytes"]
+        )
 
     async def _serve_need(self, asset: Any, memo_key: str, frame: dict[str, Any]) -> Any:
-        op = str(frame.get("op") or "")
-        if op == "payload":
-            fetched = await self._fetch_payload(asset)
-            if fetched is None:
-                return {"bytes_b64": None, "mime": None}
-            raw, mime = fetched
-            return {"bytes_b64": base64.b64encode(raw).decode("ascii"), "mime": mime}
-        if op == "raw_pages":
-            pages = await self._fetch_raw_pages(asset)
-            return {"pages": pages}
-        if op == "text":
-            pages = await self._fetch_text_pages(asset)
-            whole = "".join(pages)
-            return {"text": whole, "pages": pages}
-        raise ChildProcessError(f"Unknown payload op {op!r}")
+        try:
+            return await self._payloads.serve(asset, frame)
+        except ValueError as exc:
+            raise ChildProcessError(str(exc)) from exc
 
     async def _fetch_payload(self, asset: Any) -> tuple[bytes, str] | None:
         """Raw bytes for file-shaped sources; None for row-shaped ones."""
-        memo = self._memo_for(asset)
-        if "payload" in memo:
-            cached = memo["payload"]
-            return cached if cached is None else (bytes(cached[0]), str(cached[1]))
-        fetch = getattr(self._source, "fetch_content_bytes", None)
-        result: tuple[bytes, str] | None = None
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    result = await fetch(candidate)
-                except Exception as exc:
-                    logger.debug("Augmentation payload fetch failed for %s: %s", candidate, exc)
-                    continue
-                if result is not None:
-                    break
-        if result is not None:
-            raw, mime = result
-            cap = self._limits["max_payload_bytes"]
-            if len(raw) > cap:
-                logger.debug("Augmentation payload for %s truncated to %d bytes", candidate, cap)
-                raw = raw[:cap]
-            result = (raw, mime)
-            self._remember_bytes(asset, raw, mime)
-        memo["payload"] = result
-        return result
+        return await self._payloads.payload(asset)
 
     async def _fetch_raw_pages(self, asset: Any) -> list[str]:
         """The connector's own raw representation, page by page (capped)."""
-        memo = self._memo_for(asset)
-        if "raw_pages" in memo:
-            return [str(page) for page in memo["raw_pages"]]
-        pages: list[str] = []
-        fetch = getattr(self._source, "fetch_content_pages", None)
-        cap = self._limits["max_payload_bytes"]
-        used = 0
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    async for raw, _text in fetch(candidate):
-                        if not raw:
-                            continue
-                        chunk = str(raw)
-                        if used + len(chunk) > cap:
-                            chunk = chunk[: max(0, cap - used)]
-                        if chunk:
-                            pages.append(chunk)
-                            used += len(chunk)
-                        if used >= cap or len(pages) >= 50:
-                            break
-                    if pages:
-                        break
-                except Exception as exc:
-                    logger.debug("Augmentation raw fetch failed for %s: %s", candidate, exc)
-                    continue
-        memo["raw_pages"] = pages
-        return pages
+        return await self._payloads.raw_pages(asset)
 
     async def _fetch_text_pages(self, asset: Any) -> list[str]:
         """Extracted text, page by page (capped)."""
-        memo = self._memo_for(asset)
-        if "text_pages" in memo:
-            return [str(page) for page in memo["text_pages"]]
-        pages: list[str] = []
-        fetch = getattr(self._source, "fetch_content_pages", None)
-        cap = self._limits["max_payload_bytes"]
-        used = 0
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    async for _raw, text in fetch(candidate):
-                        if not text:
-                            continue
-                        chunk = str(text)
-                        if used + len(chunk) > cap:
-                            chunk = chunk[: max(0, cap - used)]
-                        if chunk:
-                            pages.append(chunk)
-                            used += len(chunk)
-                        if used >= cap or len(pages) >= 50:
-                            break
-                    if pages:
-                        break
-                except Exception as exc:
-                    logger.debug("Augmentation text fetch failed for %s: %s", candidate, exc)
-                    continue
-        if not pages:
-            provider = getattr(self._source, "fetch_content_bytes", None)
-            if callable(provider):
-                fetched = await self._fetch_payload(asset)
-                if fetched is not None:
-                    raw, mime = fetched
-                    try:
-                        text_pages = await asyncio.to_thread(
-                            list, self._source.iter_asset_pages(raw, mime)
-                        )
-                        for page in text_pages[:50]:
-                            chunk = str(page)
-                            if used + len(chunk) > cap:
-                                chunk = chunk[: max(0, cap - used)]
-                            if chunk:
-                                pages.append(chunk)
-                                used += len(chunk)
-                            if used >= cap:
-                                break
-                    except Exception as exc:
-                        logger.debug("Augmentation text fallback failed: %s", exc)
-        memo["text_pages"] = pages
-        return pages
-
-    def _memo_for(self, asset: Any) -> dict[str, Any]:
-        key = str(getattr(asset, "hash", "") or "")
-        return self._memo.setdefault(key, {})
-
-    def _remember_bytes(self, asset: Any, raw: bytes, mime: str) -> None:
-        """Mirror fetched bytes onto the source so the pipeline reuses them.
-
-        ``ParsedContentProvider.fetch_bytes`` consults this memo first instead
-        of re-downloading — the fetch augmentation already paid for is not
-        paid twice. Registered under every candidate id the pipeline tries
-        (external URL, then hash), and dropped by ``evict`` at the end of the
-        asset.
-        """
-        remember = getattr(self._source, "remember_augmentation_bytes", None)
-        if not callable(remember):
-            return
-        for candidate in self._candidates(asset):
-            try:
-                remember(candidate, raw, mime)
-            except Exception:
-                pass
+        return await self._payloads.text_pages(asset)
 
     async def augment(self, asset: Any) -> Any:
         """Enrich one asset. Never raises: failure degrades to a warning."""
@@ -597,6 +463,11 @@ class AugmentationSession:
 
         self._ingest_edges(patch, asset_hash=asset_hash or None)
 
+        # asset.means(): declarations, already built as MEANS edges by the SDK.
+        declared = patch.get("means")
+        if isinstance(declared, list) and declared:
+            self._ingest_edges({"edges": declared}, asset_hash=asset_hash or None)
+
         for message in patch.get("warnings") or []:
             self._warn(f"Augmentation on {getattr(asset, 'name', asset_hash)}: {message}")
 
@@ -718,13 +589,4 @@ class AugmentationSession:
 
     def evict(self, asset: Any) -> None:
         """Drop one asset's payload memo. Called alongside evict_asset_cache."""
-        self._memo.pop(str(getattr(asset, "hash", "") or ""), None)
-        forget = getattr(self._source, "forget_augmentation_bytes", None)
-        if not callable(forget):
-            return
-        # The bytes were mirrored under every candidate id; drop them all.
-        for candidate in self._candidates(asset):
-            try:
-                forget(candidate)
-            except Exception:
-                pass
+        self._payloads.evict(asset)

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/src/detectors/custom/runners/__init__.py

@@ -13,6 +13,7 @@ from ._base import (
     BaseRunner,
     _resolve_pipeline_severity,
 )
+from ._custom_detector import CustomDetectorRunner
 from ._factory import create_runner
 from ._gliner2 import (
     GLiNER2Runner,
@@ -35,6 +36,7 @@ __all__ = [
     "_IMAGE_CONTENT_TYPES",
     "_TEXT_CONTENT_TYPES",
     "BaseRunner",
+    "CustomDetectorRunner",
     "GLiNER2Runner",
     "ImageClassificationRunner",
     "LLMRunner",

classifyre_cli-0.6.5/src/detectors/custom/runners/_custom_detector.py

@@ -0,0 +1,72 @@
+"""Runner for the ``CODE_DETECTOR`` pipeline type: a code detector.
+
+Like ``TAG`` it never runs over a page of text: ``detect()`` returns nothing and
+no content type is advertised, so the text, binary and link passes never
+schedule it and the worker pool never receives it. The detector pipeline
+recognises it by this class and calls it once per asset, after every other
+detector, through the :class:`CustomDetectorSession` it owns.
+
+The session is created lazily on first use and bound to the scan's source,
+because it needs the source to serve payload and holds child processes that
+must not exist in the worker pool's processes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ....models.generated_detectors import CodeDetectorPipelineSchema, PipelineResult
+from ....models.generated_single_asset_scan_results import DetectionResult
+from ._base import BaseRunner
+
+
+class CustomDetectorRunner(BaseRunner):
+    """Asset-scoped runner: judged per asset by a notebook, never per page."""
+
+    asset_scoped = True
+
+    def __init__(
+        self,
+        schema: CodeDetectorPipelineSchema,
+        detector_key: str = "",
+        detector_name: str = "",
+    ) -> None:
+        self.schema = schema
+        self._detector_key = detector_key
+        self._detector_name = detector_name
+        self._session: Any = None
+
+    def run(self, text: str) -> PipelineResult:
+        return PipelineResult(metadata={"runner": "CODE_DETECTOR"})
+
+    def detect(self, content: str | bytes, content_type: str) -> list[DetectionResult]:
+        """Never called per page. The pipeline calls ``session().detect(asset)``."""
+        return []
+
+    def get_supported_content_types(self) -> list[str]:
+        return []
+
+    @property
+    def deterministic(self) -> bool:
+        return self.schema.deterministic is not False
+
+    @property
+    def needs_findings(self) -> bool:
+        return bool(self.schema.needs_findings)
+
+    def session(self, source: Any) -> Any:
+        """The run's session, created on first use."""
+        if self._session is None:
+            from ...custom_detector.session import CustomDetectorSession
+
+            self._session = CustomDetectorSession(
+                key=self._detector_key,
+                name=self._detector_name,
+                schema=self.schema,
+                source=source,
+            )
+        return self._session
+
+    def close(self) -> None:
+        if self._session is not None:
+            self._session.close()

{classifyre_cli-0.6.3 → classifyre_cli-0.6.5}/src/detectors/custom/runners/_factory.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from ....models.generated_detectors import (
+    CodeDetectorPipelineSchema,
     GLiNER2PipelineSchema,
     ImageClassificationPipelineSchema,
     LLMPipelineSchema,
@@ -12,6 +13,7 @@ from ....models.generated_detectors import (
     TextClassificationPipelineSchema,
 )
 from ._base import BaseRunner
+from ._custom_detector import CustomDetectorRunner
 from ._gliner2 import GLiNER2Runner
 from ._image_classification import ImageClassificationRunner
 from ._llm import LLMRunner
@@ -30,6 +32,7 @@ def create_runner(
         | ImageClassificationPipelineSchema
         | ObjectDetectionPipelineSchema
         | TagPipelineSchema
+        | CodeDetectorPipelineSchema
     ),
     detector_key: str = "",
     detector_name: str = "",
@@ -50,5 +53,8 @@ def create_runner(
     # detector that is not supposed to run anything at all.
     if isinstance(schema, TagPipelineSchema):
         return TagRunner(schema, detector_key, detector_name)
+    # Same reason as TAG: a code detector must never fall through to GLiNER2.
+    if isinstance(schema, CodeDetectorPipelineSchema):
+        return CustomDetectorRunner(schema, detector_key, detector_name)
     # GLiNER2PipelineSchema is the default / backward-compat path
     return GLiNER2Runner(schema, detector_key, detector_name)

classifyre_cli-0.6.5/src/detectors/custom_detector/__init__.py

@@ -0,0 +1,9 @@
+"""Code detectors: the ``CODE_DETECTOR`` pipeline type.
+
+A notebook that defines ``detect(asset, ctx)`` and yields ``Finding`` objects,
+run in an isolated child process once per asset, after every other detector.
+"""
+
+from .session import CustomDetectorSession, DetectOutcome
+
+__all__ = ["CustomDetectorSession", "DetectOutcome"]

classifyre_cli-0.6.5/src/detectors/custom_detector/contract.py

@@ -0,0 +1,45 @@
+"""The interface a code detector's notebook must cover.
+
+A code detector answers one question -- "what is wrong with this asset?" -- so
+``detect(asset, ctx)`` is required and ``setup(ctx)`` optional. Checked at the
+three moments the connector contract is: on save (the API's cheap check),
+before a preview, and before every scan, because the recipe travels to a job
+that has to defend itself.
+
+Only the shape is checked, through the shared ``validate_module`` /
+``validate_notebook``. Isolation is the child process and its scrubbed
+environment, not anything an AST can decide.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+
+from ...notebook.contract import ContractReport, validate_module, validate_notebook
+from ...notebook.serialize import ModuleSource
+
+REQUIRED_FUNCTIONS = ("detect",)
+
+#: ``setup(ctx)`` runs once per worker before the first asset: load a list,
+#: a model, a lookup table into ``ctx.state``.
+OPTIONAL_FUNCTIONS = ("setup",)
+
+
+def validate_detector_module(module: ModuleSource) -> ContractReport:
+    """Parse the assembled module and check ``detect()`` exists."""
+    return validate_module(module, required=REQUIRED_FUNCTIONS)
+
+
+def validate_detector_notebook(
+    cells: Iterable[Mapping[str, object] | object],
+) -> ContractReport:
+    """Validate a code detector's cells."""
+    return validate_notebook(cells, required=REQUIRED_FUNCTIONS)
+
+
+__all__ = [
+    "OPTIONAL_FUNCTIONS",
+    "REQUIRED_FUNCTIONS",
+    "validate_detector_module",
+    "validate_detector_notebook",
+]