classifyre-cli 0.4.135__tar.gz → 0.4.136__tar.gz

classifyre_cli-0.4.136/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 322 packages in 257ms
+Checked 51 packages in 0.67ms

{classifyre_cli-0.4.135 → classifyre_cli-0.4.136}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.4.135
+Version: 0.4.136
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.135 → classifyre_cli-0.4.136}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.135",
+  "version": "0.4.136",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.135 → classifyre_cli-0.4.136}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.135"
+version = "0.4.136"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.135 → classifyre_cli-0.4.136}/scripts/generate_sdk_completions.py

@@ -26,6 +26,8 @@ from typing import Any
 CLI_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(CLI_ROOT))
 
+from src.augmentation import contract as augmentation_contract  # noqa: E402
+from src.augmentation import sdk as augmentation_sdk  # noqa: E402
 from src.notebook.contract import (  # noqa: E402
     OPTIONAL_FUNCTIONS,
     REQUIRED_FUNCTIONS,
@@ -126,7 +128,17 @@ def _snippet(name: str, func: Any) -> str:
 #: Public on Context because the runtime reads them, but not part of what a
 #: notebook author writes -- offering them would be noise at best and a
 #: misleading suggestion at worst.
-RUNTIME_ONLY_MEMBERS = frozenset({"set_offset", "offset_consumed", "next_cursor"})
+RUNTIME_ONLY_MEMBERS = frozenset(
+    {
+        "set_offset",
+        "offset_consumed",
+        "next_cursor",
+        # The adapter reads these back off the run; the author only ever calls
+        # set_partial_coverage().
+        "partial_coverage",
+        "partial_coverage_reason",
+    }
+)
 
 
 def _members(cls: type) -> list[dict[str, Any]]:
@@ -220,7 +232,8 @@ def build() -> dict[str, Any]:
     return {
         "$comment": (
             "Generated by apps/cli/scripts/generate_sdk_completions.py from "
-            "apps/cli/src/notebook/sdk.py. Do not edit by hand."
+            "apps/cli/src/notebook/sdk.py and apps/cli/src/augmentation/sdk.py. "
+            "Do not edit by hand."
         ),
         "module": "classifyre",
         "globals": [
@@ -263,12 +276,26 @@ def build() -> dict[str, Any]:
         ],
         "objects": {
             "ctx": {"type": "Context", "members": _members(Context)},
+            "augmentation": {
+                "ctx": {
+                    "type": "AugmentContext",
+                    "members": _members(augmentation_sdk.AugmentContext),
+                },
+                "asset": {
+                    "type": "AugmentedAsset",
+                    "members": _members(augmentation_sdk.AugmentedAsset),
+                },
+            },
         },
         "classes": {
             "Asset": {
                 "documentation": _summary(Asset),
                 "fields": _fields(Asset),
             },
+            "AugmentedAsset": {
+                "documentation": _summary(augmentation_sdk.AugmentedAsset),
+                "members": _members(augmentation_sdk.AugmentedAsset),
+            },
             # Returned by `parse()` and reached as `ctx.file(...)`, so an author
             # who never writes their name still needs their members offered.
             "ParsedContent": {
@@ -288,6 +315,10 @@ def build() -> dict[str, Any]:
         "contract": {
             "required": list(REQUIRED_FUNCTIONS),
             "optional": list(OPTIONAL_FUNCTIONS),
+            "augmentation": {
+                "required": list(augmentation_contract.REQUIRED_FUNCTIONS),
+                "optional": list(augmentation_contract.OPTIONAL_FUNCTIONS),
+            },
         },
     }
 

classifyre_cli-0.4.136/src/augmentation/__init__.py

@@ -0,0 +1,5 @@
+"""Source augmentation: per-asset enrichment for any known source."""
+
+from .session import AugmentationSession
+
+__all__ = ["AugmentationSession"]

classifyre_cli-0.4.136/src/augmentation/contract.py

@@ -0,0 +1,70 @@
+"""The interface an augmentation notebook must cover.
+
+An augmentation is not a connector: it never answers "can you connect?" or
+"what is there?", so the connector contract's ``test_connection``/``extract``
+would be the wrong shape to check. What it must answer is "what do you add to
+this asset?", which is ``augment(asset)`` — required — plus the two lifecycle
+hooks ``setup()`` (once, before the first asset) and ``finalize()`` (once, at
+the end, when both sides of a computed relationship are finally known).
+
+The check itself is the connector one, parametrised: ``validate_module`` and
+``validate_notebook`` already take a ``required`` iterable rather than
+hardcoding the connector names, so this module passes its own names instead of
+copying the parser. Only the *shape* is checked here as well — isolation is a
+process boundary and a scrubbed environment, not a blocklist.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+
+from ..notebook.contract import (
+    ContractReport,
+    NotebookContractError,
+    validate_module,
+    validate_notebook,
+)
+from ..notebook.serialize import ModuleSource
+
+#: Without this the notebook cannot answer "what do you add to this asset?",
+#: which is the whole of the augmentation contract.
+REQUIRED_FUNCTIONS = ("augment",)
+
+#: Implemented when the notebook needs them; absence is not an error. ``setup``
+#: builds per-run state (lookup tables in ``ctx.state``) before the first
+#: asset; ``finalize`` runs once at the end and may yield edges that needed
+#: every asset to have been seen first.
+OPTIONAL_FUNCTIONS = ("setup", "finalize")
+
+
+def validate_augmentation_module(module: ModuleSource) -> ContractReport:
+    """Parse the assembled module and check ``augment()`` exists."""
+    return validate_module(module, required=REQUIRED_FUNCTIONS)
+
+
+def validate_augmentation_notebook(
+    cells: Iterable[Mapping[str, object] | object],
+    *,
+    required: Iterable[str] = REQUIRED_FUNCTIONS,
+) -> ContractReport:
+    """Validate a notebook's cells against the augmentation contract."""
+    return validate_notebook(cells, required=required)
+
+
+def defined_functions(cells: Iterable[Mapping[str, object] | object]) -> frozenset[str]:
+    """Names the notebook defines, without failing a notebook that parses badly."""
+    try:
+        return validate_notebook(list(cells), required=()).defined_functions
+    except Exception:
+        # A notebook that will not parse has already failed louder elsewhere.
+        return frozenset()
+
+
+__all__ = [
+    "OPTIONAL_FUNCTIONS",
+    "REQUIRED_FUNCTIONS",
+    "NotebookContractError",
+    "defined_functions",
+    "validate_augmentation_module",
+    "validate_augmentation_notebook",
+]

classifyre_cli-0.4.136/src/augmentation/preview.py

@@ -0,0 +1,280 @@
+"""Preview augmentation against real assets: ``preview_augment``.
+
+Unlike ``preview_extract`` — which replays cells in-process — augmentation
+needs the **real connector**: the assets it enriches only exist after
+extraction. So this flow builds the source from the recipe with sampling
+forced to RANDOM and a cap of N (default 5, max 25), takes the first N assets
+from ``extract_raw()``, and runs ``setup()`` / ``augment(asset)`` /
+``finalize()`` over them.
+
+No sink, no run record, no ingestion — read-only. The answer is an
+``ExecutionResponse`` whose ``assets`` carry per-asset **diffs**: metadata
+added, tags asserted (and whether each key matches a live Tag detector), links
+added, URN change, edges yielded, and warnings.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+import time
+from typing import Any
+
+from ..notebook.protocol import (
+    ExecutionError,
+    ExecutionMode,
+    ExecutionResponse,
+    ExecutionStatus,
+)
+from ..notebook.redact import Redactor
+from .contract import validate_augmentation_notebook
+from .session import AugmentationSession, augmentation_config_of
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PREVIEW_ASSETS = 5
+MAX_PREVIEW_ASSETS = 25
+
+
+def _tag_keys_from_recipe(recipe: dict[str, Any], source: Any) -> set[str]:
+    """Keys a live Tag detector would resolve, from this run's recipe.
+
+    The API hydrates runtime TAG detectors into ``recipe.detectors`` before
+    dispatch (see cli-runner), so the dispatched recipe is authoritative. A
+    detector entry names its tag key via ``custom_detector_key`` (top level or
+    under ``config``); a bare ``TAG`` pipeline entry carries ``key``.
+    """
+    keys: set[str] = set()
+    detectors = recipe.get("detectors")
+    if not isinstance(detectors, list):
+        return keys
+    for entry in detectors:
+        if not isinstance(entry, dict):
+            continue
+        if str(entry.get("type") or "").strip().upper() not in {"TAG", "CUSTOM"}:
+            continue
+        for holder in (entry, entry.get("config")):
+            if not isinstance(holder, dict):
+                continue
+            for field in ("custom_detector_key", "key"):
+                value = holder.get(field)
+                if isinstance(value, str) and value.strip():
+                    keys.add(value.strip())
+    if not keys:
+        # Fall back to the detectors the pipeline would actually build: exact,
+        # at the cost of constructing them.
+        try:
+            from ..pipeline.detector_pipeline import DetectorPipeline
+
+            pipeline = DetectorPipeline.from_recipe(recipe, source, "preview")
+            runners = DetectorPipeline._tag_runners(pipeline.detectors)
+            keys.update(runners)
+        except Exception as exc:
+            logger.debug("Could not resolve live Tag detector keys: %s", exc)
+    return keys
+
+
+async def preview_augment(
+    recipe: dict[str, Any],
+    *,
+    max_assets: int = DEFAULT_PREVIEW_ASSETS,
+    execution_id: str | None = None,
+    revision: int | None = None,
+) -> ExecutionResponse:
+    """Run augmentation over a bounded sample of real assets and diff each one."""
+    started = time.monotonic()
+    mode = ExecutionMode.PREVIEW_AUGMENT
+    redactor = Redactor.from_recipe(recipe)
+
+    section = augmentation_config_of(recipe)
+    cells = (
+        section.get("notebook", {}).get("cells")
+        if isinstance(section.get("notebook"), dict)
+        else None
+    )
+    if not section.get("enabled", False) or not cells:
+        return ExecutionResponse(
+            status=ExecutionStatus.ERROR,
+            mode=mode,
+            execution_id=execution_id,
+            revision=revision,
+            error=_error(
+                "AugmentationNotEnabled",
+                "This source has no enabled augmentation notebook to preview.",
+            ),
+        )
+
+    report = validate_augmentation_notebook(cells)
+    if not report.ok:
+        violation = report.violations[0]
+        return ExecutionResponse(
+            status=ExecutionStatus.ERROR,
+            mode=mode,
+            execution_id=execution_id,
+            revision=revision,
+            contract=report.to_dict(),
+            error=_error("NotebookContractError", violation.message),
+        )
+
+    count = max(1, min(MAX_PREVIEW_ASSETS, int(max_assets or DEFAULT_PREVIEW_ASSETS)))
+
+    # The real connector, sampled: RANDOM with a cap, so the preview shows real
+    # assets without ingesting anything. No sink is ever built.
+    sampled = copy.deepcopy(recipe)
+    sampled["sampling"] = {"strategy": "RANDOM", "rows_per_page": count}
+
+    from ..sources import get_source
+
+    try:
+        source = get_source(sampled)
+    except Exception as exc:
+        return ExecutionResponse(
+            status=ExecutionStatus.ERROR,
+            mode=mode,
+            execution_id=execution_id,
+            revision=revision,
+            error=_error("SourceBuildFailed", f"Could not build the source: {exc}"),
+        )
+
+    session = AugmentationSession.from_recipe(recipe, source)
+    if session is None:  # pragma: no cover - guarded above, kept for safety
+        return ExecutionResponse(
+            status=ExecutionStatus.ERROR,
+            mode=mode,
+            execution_id=execution_id,
+            revision=revision,
+            error=_error("AugmentationNotEnabled", "Augmentation is not enabled."),
+        )
+    source.attach_augmentation(session)
+
+    samples: list[dict[str, Any]] = []
+    tag_keys = _tag_keys_from_recipe(recipe, source)
+    try:
+        await session.startup()
+        if not session.enabled:
+            return ExecutionResponse(
+                status=ExecutionStatus.ERROR,
+                mode=mode,
+                execution_id=execution_id,
+                revision=revision,
+                error=_error(
+                    "AugmentationSetupFailed",
+                    session.warnings[-1] if session.warnings else "setup() failed",
+                ),
+            )
+
+        seen = 0
+        async for batch in source.extract_raw():
+            for asset in batch:
+                if seen >= count:
+                    break
+                samples.append(await _preview_one(session, source, asset, tag_keys, redactor))
+                seen += 1
+            if seen >= count:
+                break
+
+        finalize_edges = await _finalize_edges(session, source)
+        response = ExecutionResponse(
+            status=ExecutionStatus.SUCCESS,
+            mode=mode,
+            execution_id=execution_id,
+            revision=revision or session.revision,
+            assets=samples,
+            result={
+                "sampled": len(samples),
+                "finalizeEdges": finalize_edges,
+                "session": session.summary(),
+            },
+        )
+    except Exception as exc:
+        logger.warning("preview_augment failed: %s", exc)
+        response = ExecutionResponse(
+            status=ExecutionStatus.ERROR,
+            mode=mode,
+            execution_id=execution_id,
+            revision=revision,
+            assets=samples,
+            error=_error(type(exc).__name__, str(exc)),
+        )
+    finally:
+        session.close()
+        try:
+            source.cleanup()
+        except Exception:
+            pass
+
+    response.duration_ms = int((time.monotonic() - started) * 1000)
+    return response
+
+
+async def _preview_one(
+    session: AugmentationSession,
+    source: Any,
+    asset: Any,
+    tag_keys: set[str],
+    redactor: Redactor,
+) -> dict[str, Any]:
+    asset_hash = str(getattr(asset, "hash", "") or "")
+    before_links = list(getattr(asset, "links", None) or [])
+    before_urn = getattr(asset, "urn", None)
+    try:
+        before_tags = dict(source.asset_tags(asset_hash))
+    except Exception:
+        before_tags = {}
+    warnings_before = len(session.warnings)
+    edges_before = session.stats.edges
+
+    await session.augment(asset)
+
+    try:
+        after_tags = dict(source.asset_tags(asset_hash))
+    except Exception:
+        after_tags = {}
+    captured_tags = {key: value for key, value in after_tags.items() if key not in before_tags}
+
+    after = getattr(asset, "metadata", None) or {}
+    added_metadata = after.get("augmentation") if isinstance(after, dict) else {}
+    if not isinstance(added_metadata, dict):
+        added_metadata = {}
+    after_links = list(getattr(asset, "links", None) or [])
+
+    try:
+        drained = source.drain_edges()
+    except Exception:
+        drained = []
+
+    redacted: dict[str, Any] = redactor.redact_deep(
+        {
+            "hash": str(getattr(asset, "hash", "") or ""),
+            "name": str(getattr(asset, "name", "") or ""),
+            "kind": str(getattr(asset, "asset_kind", "") or ""),
+            "metadataAdded": dict(added_metadata),
+            "tags": {
+                key: {"value": value, "matchesTagDetector": key in tag_keys}
+                for key, value in captured_tags.items()
+            },
+            "linksAdded": [link for link in after_links if link not in before_links],
+            "urn": {"before": before_urn, "after": getattr(asset, "urn", None)},
+            "edges": len(drained),
+            "warnings": session.warnings[warnings_before:],
+            "augmentedEdges": session.stats.edges - edges_before,
+        }
+    )
+    return redacted
+
+
+async def _finalize_edges(session: AugmentationSession, source: Any) -> int:
+    before = session.stats.edges
+    await session.finish()
+    try:
+        leftover = source.drain_edges()
+    except Exception:
+        leftover = []
+    return (session.stats.edges - before) + len(leftover or [])
+
+
+def _error(type_: str, message: str) -> ExecutionError:
+    return ExecutionError(type=type_, message=message)
+
+
+__all__ = ["DEFAULT_PREVIEW_ASSETS", "MAX_PREVIEW_ASSETS", "preview_augment"]