equiframe 0.12.4__py3-none-any.whl

equiframe/detectors.py

@@ -0,0 +1,389 @@
+"""Fit, score and calibrate family detectors independently from rules and the oracle."""
+
+import math
+import platform
+import struct
+from fractions import Fraction
+from pathlib import Path
+from typing import Literal
+
+from equiframe._document_io import read_document as _read_document
+from equiframe._document_io import write_document as _write_document
+from equiframe._regular_io import local_input_path
+from equiframe.detector_types import (
+    ComponentDecision,
+    ComponentDetectionResult,
+    DetectionResult,
+    DetectorBundle,
+    DetectorCalibration,
+    DetectorDecision,
+    DetectorFit,
+    DetectorModel,
+    DetectorScoreResult,
+    DetectorScores,
+    DetectorSupport,
+    DetectorVersions,
+    IsolationForestConfig,
+    IsolationNode,
+    IsolationTree,
+    NormalCalibrationProfiles,
+    ScoreValue,
+    detector_fingerprint,
+    numeric_score,
+    quantile_rank,
+)
+from equiframe.errors import ArtifactIntegrityError, MissingDependencyError, ResourceLimitError
+from equiframe.preprocessing import _compatible, fit_preprocessor, transform_profiles
+from equiframe.profile_types import (
+    FeatureMatrix,
+    NormalTrainingProfiles,
+    ProfileSpec,
+    ProfileVector,
+    profile_digest,
+)
+
+
+def _unavailable(method: Literal["isolation_forest", "mad"], reason: str) -> DetectorModel:
+    return DetectorModel(method=method, state="not_estimable", reason=reason)
+
+
+def _float32(row: tuple[float, ...]) -> tuple[float, ...]:
+    try:
+        converted = tuple(struct.unpack("f", struct.pack("f", v))[0] for v in row)
+    except (OverflowError, struct.error) as exc:
+        raise ValueError("Isolation Forest input is outside finite float32 range") from exc
+    if not all(math.isfinite(v) for v in converted):
+        raise ValueError("Isolation Forest input is outside finite float32 range")
+    return converted
+
+
+def _fit_forest(
+    matrix: FeatureMatrix, config: IsolationForestConfig
+) -> tuple[DetectorModel, str | None, str | None]:
+    converted = tuple(_float32(row) for row in matrix.values)
+    if len(set(converted)) < 2:
+        return _unavailable("isolation_forest", "no_float32_variation"), None, None
+    try:
+        import numpy as np
+        import sklearn
+        from sklearn.ensemble import IsolationForest
+    except ImportError as exc:
+        raise MissingDependencyError("Isolation Forest fit requires equiframe[ml]") from exc
+    samples = min(config.max_samples, len(matrix.values))
+    model = IsolationForest(
+        n_estimators=config.n_estimators,
+        max_samples=samples,
+        random_state=config.random_state,
+        contamination="auto",
+        max_features=1.0,
+        bootstrap=False,
+        n_jobs=1,
+    ).fit(np.asarray(converted, dtype=np.float32))
+    trees = []
+    for estimator in model.estimators_:
+        tree = estimator.tree_
+        nodes = []
+        for i in range(tree.node_count):
+            leaf = tree.children_left[i] == -1
+            nodes.append(
+                IsolationNode(
+                    samples=int(tree.n_node_samples[i]),
+                    left=None if leaf else int(tree.children_left[i]),
+                    right=None if leaf else int(tree.children_right[i]),
+                    feature=None if leaf else int(tree.feature[i]),
+                    threshold=None if leaf else float(tree.threshold[i]),
+                )
+            )
+        trees.append(IsolationTree(nodes=tuple(nodes)))
+    if not any(len(tree.nodes) > 1 for tree in trees):
+        return (
+            _unavailable("isolation_forest", "no_isolation_splits"),
+            sklearn.__version__,
+            np.__version__,
+        )
+    return (
+        DetectorModel(
+            method="isolation_forest", state="fitted", trees=tuple(trees), effective_samples=samples
+        ),
+        sklearn.__version__,
+        np.__version__,
+    )
+
+
+def _median(values: tuple[float, ...]) -> float:
+    # statistics.median can overflow while averaging two finite positive numbers.
+    ordered = sorted(values)
+    middle = len(ordered) // 2
+    if len(ordered) % 2:
+        return ordered[middle]
+    return float((Fraction(ordered[middle - 1]) + Fraction(ordered[middle])) / 2)
+
+
+def fit_detectors(
+    training: NormalTrainingProfiles,
+    *,
+    spec: ProfileSpec,
+    config: IsolationForestConfig,
+    support: DetectorSupport,
+    empty_policy: Literal["drop", "not_estimable"] = "drop",
+) -> DetectorFit:
+    """Fit M and B on the same matrix. Eligibility is verified by the caller/benchmark."""
+    config = IsolationForestConfig.model_validate(config)
+    support = DetectorSupport.model_validate(support)
+    training = NormalTrainingProfiles.model_validate(training)
+    preprocessor = fit_preprocessor(training, spec=spec, empty_policy=empty_policy)
+    matrix = (
+        transform_profiles(preprocessor, training.profiles)
+        if preprocessor.state == "ready"
+        else None
+    )
+    reason = (
+        "preprocessor_not_estimable"
+        if matrix is None
+        else "insufficient_training_rows"
+        if len(matrix.values) < support.min_training_rows
+        else "no_training_variation"
+        if len(set(matrix.values)) < 2
+        else None
+    )
+    sklearn_version = numpy_version = None
+    if reason is not None:
+        m = _unavailable("isolation_forest", reason)
+        b = _unavailable("mad", reason)
+    else:
+        if not (matrix is not None):
+            raise AssertionError("Internal invariant check failed.")
+        columns = tuple(zip(*matrix.values, strict=True))
+        medians = tuple(_median(c) for c in columns)
+        deviations = tuple(
+            _median(tuple(abs(v - center) for v in values))
+            for center, values in zip(medians, columns, strict=True)
+        )
+        b = DetectorModel(method="mad", state="fitted", medians=medians, deviations=deviations)
+        m, sklearn_version, numpy_version = _fit_forest(matrix, config)
+    return DetectorFit(
+        preprocessor=preprocessor,
+        config=config,
+        support=support,
+        versions=DetectorVersions(
+            python=platform.python_version(), sklearn=sklearn_version, numpy=numpy_version
+        ),
+        training_matrix_sha256=profile_digest(matrix.model_dump(mode="json")) if matrix else None,
+        m=m,
+        b=b,
+    )
+
+
+def _average_path(samples: int) -> float:
+    if samples <= 1:
+        return 0.0
+    if samples == 2:
+        return 1.0
+    # The approximation used by sklearn IsolationForest, not the harmonic series.
+    return 2 * (math.log(samples - 1) + 0.5772156649015329) - 2 * (samples - 1) / samples
+
+
+def _forest_score(model: DetectorModel, row: tuple[float, ...]) -> float:
+    row = _float32(row)
+    total = 0.0
+    for tree in model.trees:
+        index = depth = 0
+        node = tree.nodes[index]
+        while node.left is not None:
+            if not (
+                node.feature is not None and node.threshold is not None and node.right is not None
+            ):
+                raise AssertionError("Internal invariant check failed.")
+            index = node.left if row[node.feature] <= node.threshold else node.right
+            depth += 1
+            node = tree.nodes[index]
+        total += depth + _average_path(node.samples)
+    if not (model.effective_samples is not None):
+        raise AssertionError("Internal invariant check failed.")
+    return float(2 ** (-total / (len(model.trees) * _average_path(model.effective_samples))))
+
+
+def _mad_score(model: DetectorModel, row: tuple[float, ...]) -> ScoreValue:
+    maximum = 0.0
+    for value, center, scale in zip(row, model.medians, model.deviations, strict=True):
+        if scale == 0:
+            if value != center:
+                return "+infinity"
+            continue
+        deviation = abs(value - center) / scale
+        if not math.isfinite(deviation):
+            # Mathematical infinity is reserved for MAD=0, not floating overflow.
+            raise ValueError("finite MAD score exceeds numeric range")
+        maximum = max(maximum, deviation)
+    return maximum
+
+
+def _inference_matrix(
+    fit: DetectorFit, profiles: tuple[ProfileVector, ...]
+) -> FeatureMatrix | None:
+    profiles = tuple(_compatible(p, fit.preprocessor.spec) for p in profiles)
+    return (
+        transform_profiles(fit.preprocessor, profiles)
+        if fit.preprocessor.state == "ready"
+        else None
+    )
+
+
+def _score_model(model: DetectorModel, matrix: FeatureMatrix | None) -> DetectorScores:
+    if model.state == "not_estimable":
+        return DetectorScores(state="not_estimable", reason=model.reason)
+    if not (matrix is not None):
+        raise AssertionError("Internal invariant check failed.")
+    scorer = _forest_score if model.method == "isolation_forest" else _mad_score
+    return DetectorScores(state="scored", values=tuple(scorer(model, row) for row in matrix.values))
+
+
+def score_detectors(fit: DetectorFit, profiles: tuple[ProfileVector, ...]) -> DetectorScoreResult:
+    """Frozen inference. Incompatible/failed profiles raise, never become no-alert."""
+    fit = DetectorFit.model_validate(fit)
+    matrix = _inference_matrix(fit, profiles)
+    return DetectorScoreResult(
+        fit_sha256=detector_fingerprint(fit),
+        matrix_sha256=profile_digest(matrix.model_dump(mode="json")) if matrix else None,
+        rows=len(profiles),
+        m=_score_model(fit.m, matrix),
+        b=_score_model(fit.b, matrix),
+    )
+
+
+def _calibrate(scores: DetectorScores, alpha: str, n: int, minimum: int) -> DetectorCalibration:
+    if scores.state == "not_estimable" or n < minimum:
+        return DetectorCalibration(
+            alpha=alpha,
+            sample_count=n,
+            state="not_estimable",
+            reason=scores.reason or "insufficient_calibration_rows",
+        )
+    rank = quantile_rank(alpha, n)
+    threshold = sorted(scores.values, key=numeric_score)[rank - 1]
+    infinite = threshold == "+infinity"
+    return DetectorCalibration(
+        alpha=alpha,
+        sample_count=n,
+        scores=scores.values,
+        rank=rank,
+        threshold=threshold,
+        state="not_estimable" if infinite else "ready",
+        reason="infinite_calibration_threshold" if infinite else None,
+    )
+
+
+def calibrate_detectors(
+    fit: DetectorFit, calibration: NormalCalibrationProfiles, *, alpha: str
+) -> DetectorBundle:
+    """Use separate normal calibration; the benchmark enforces disjoint group lineage."""
+    fit = DetectorFit.model_validate(fit)
+    calibration = NormalCalibrationProfiles.model_validate(calibration)
+    quantile_rank(alpha, max(1, len(calibration.profiles)))
+    if calibration.selection_manifest_sha256 == fit.preprocessor.selection_manifest_sha256:
+        raise ValueError("training and calibration selection manifests must differ")
+    scores = score_detectors(fit, calibration.profiles)
+    return DetectorBundle(
+        fit=fit,
+        calibration_manifest_sha256=calibration.selection_manifest_sha256,
+        calibration_matrix_sha256=scores.matrix_sha256,
+        m=_calibrate(scores.m, alpha, scores.rows, fit.support.min_calibration_rows),
+        b=_calibrate(scores.b, alpha, scores.rows, fit.support.min_calibration_rows),
+    )
+
+
+def _decisions(
+    calibrated: DetectorCalibration, scored: DetectorScores, count: int
+) -> tuple[DetectorDecision, ...]:
+    rows = []
+    for index in range(count):
+        score = scored.values[index] if scored.state == "scored" else None
+        if calibrated.state != "ready":
+            rows.append(
+                DetectorDecision(
+                    result=ComponentDecision(
+                        decision="not_evaluable", complete=False, reason=calibrated.reason
+                    ),
+                    score=score,
+                    threshold=None,
+                )
+            )
+            continue
+        if not (isinstance(calibrated.threshold, float) and score is not None):
+            raise AssertionError("Internal invariant check failed.")
+        rows.append(
+            DetectorDecision(
+                result=ComponentDecision(
+                    decision="alert" if numeric_score(score) > calibrated.threshold else "no_alert",
+                    complete=True,
+                ),
+                score=score,
+                threshold=calibrated.threshold,
+            )
+        )
+    return tuple(rows)
+
+
+def detect(bundle: DetectorBundle, profiles: tuple[ProfileVector, ...]) -> DetectionResult:
+    bundle = DetectorBundle.model_validate(bundle)
+    scores = score_detectors(bundle.fit, profiles)
+    return DetectionResult(
+        bundle_sha256=detector_fingerprint(bundle),
+        scores=scores,
+        m=_decisions(bundle.m, scores.m, scores.rows),
+        b=_decisions(bundle.b, scores.b, scores.rows),
+    )
+
+
+def detect_component(
+    bundle: DetectorBundle,
+    profiles: tuple[ProfileVector, ...],
+    *,
+    method: Literal["isolation_forest", "mad"],
+) -> ComponentDetectionResult:
+    """Score and decide one frozen detector, without evaluating the other method.
+
+    The shared preprocessor and strict decision threshold are identical to
+    ``detect``. Invalid or technically failed profiles raise; missing model or
+    calibration support produces explicit not-evaluable decisions.
+    """
+    if method not in ("isolation_forest", "mad"):
+        raise ValueError("method must be isolation_forest or mad")
+    bundle = DetectorBundle.model_validate(bundle)
+    matrix = _inference_matrix(bundle.fit, profiles)
+    model, calibrated = (
+        (bundle.fit.m, bundle.m) if method == "isolation_forest" else (bundle.fit.b, bundle.b)
+    )
+    scores = _score_model(model, matrix)
+    return ComponentDetectionResult(
+        method=method,
+        bundle_sha256=detector_fingerprint(bundle),
+        fit_sha256=detector_fingerprint(bundle.fit),
+        profiles_sha256=profile_digest([p.model_dump(mode="json") for p in profiles]),
+        matrix_sha256=profile_digest(matrix.model_dump(mode="json")) if matrix else None,
+        rows=len(profiles),
+        scores=scores,
+        decisions=_decisions(calibrated, scores, len(profiles)),
+    )
+
+
+def write_detector_bundle(bundle: DetectorBundle, path: Path) -> str:
+    """Append-only JSON. Returns semantic digest for a trusted external manifest."""
+    bundle = DetectorBundle.model_validate(bundle)
+    payload = bundle.model_dump_json().encode("utf-8")
+    if len(payload) > 64 * 1024 * 1024:
+        raise ValueError("detector bundle exceeds the 64 MiB document limit")
+    _write_document(path, payload)
+    return detector_fingerprint(bundle)
+
+
+def read_detector_bundle(path: Path, *, expected_sha256: str | None = None) -> DetectorBundle:
+    """Read bounded declarative data only; no pickle, estimator restoration or imports."""
+    try:
+        raw = _read_document(local_input_path(path))
+    except ResourceLimitError:
+        raise ValueError("detector bundle exceeds the 64 MiB document limit") from None
+    bundle = DetectorBundle.model_validate_json(raw)
+    if expected_sha256 is not None and detector_fingerprint(bundle) != expected_sha256:
+        raise ArtifactIntegrityError("detector bundle fingerprint mismatch")
+    return bundle

equiframe/diff.py

@@ -0,0 +1,192 @@
+"""A direct-input facade over the existing complete comparison kernel.
+
+Actual DuckDB relations explicitly supplied by the caller are consumed as native
+inputs. SQL strings, connections and Polars lazy plans are not accepted. Capture
+and inference failures before a valid contract exists raise; once configured,
+the kernel owns semantic failure states.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal
+
+from typing_extensions import TypeAliasType
+
+from equiframe._base import FrozenModel
+from equiframe._diff_capture import (
+    DiffCaptureOptions,
+    _CapturedInput,
+    _NativeInput,
+    capture_native,
+)
+from equiframe._document_io import read_document
+from equiframe._regular_io import local_input_path
+from equiframe.comparison import ComparisonInput, _compare, comparison_fingerprint
+from equiframe.comparison_types import ComparisonOptions, ComparisonPolicy, ComparisonResult
+from equiframe.contracts import Contract
+from equiframe.diff_contracts import DiffContract, infer_diff_contract
+from equiframe.errors import ArtifactIntegrityError, CapabilityError, ExecutionCancelled
+from equiframe.execution import CancellationToken
+from equiframe.execution_types import ExecutionResult, TableArtifact
+from equiframe.local import local_path
+from equiframe.tables import ArrowInput
+from equiframe.types import ArtifactRef, Sha256
+
+if TYPE_CHECKING:
+    import duckdb
+    import pandas as pd
+    import polars as pl
+    import pyarrow as pa
+
+    DiffInput = TypeAliasType(
+        "DiffInput",
+        ComparisonInput | pd.DataFrame | pl.DataFrame | pa.Table | duckdb.DuckDBPyRelation,
+    )
+else:
+    # Introspection must not import optional engines. Capture performs actual
+    # class admission; this runtime alias is not an admission validator.
+    DiffInput = TypeAliasType("DiffInput", ComparisonInput | object)
+
+
+class _DiffReceipt(FrozenModel):
+    schema_version: Literal["1.0"] = "1.0"
+    comparison_id: str
+    spec_sha256: Sha256
+    declaration: DiffContract
+    capture_options: DiffCaptureOptions
+
+
+def _input(
+    value: DiffInput, options: DiffCaptureOptions, cancellation: CancellationToken | None = None
+) -> ComparisonInput | _NativeInput:
+    if isinstance(value, (ArrowInput, ArtifactRef, TableArtifact, ExecutionResult, Path)):
+        return value
+    return _NativeInput(value, options, cancellation)
+
+
+def diff(
+    candidate: DiffInput,
+    expected: DiffInput,
+    *,
+    contract: Contract | DiffContract | None = None,
+    keys: Sequence[str] | None = None,
+    policy: ComparisonPolicy | None = None,
+    reference: Literal["peer", "expected"] | None = None,
+    options: ComparisonOptions | None = None,
+    capture_options: DiffCaptureOptions | None = None,
+    work_dir: Path | None = None,
+    cancellation: CancellationToken | None = None,
+) -> ComparisonResult:
+    """Compare native tables, using the right-hand expected schema by default.
+
+    Passing a real DuckDBPyRelation authorizes consuming its caller-owned native
+    plan once per input. Capture is bounded by rows and Arrow batch bytes, with
+    cooperative cancellation between batches. SQL strings/connections and
+    Polars LazyFrame objects are not executed through this facade.
+
+    With no contract, infer types only from the right-hand captured table; keys
+    are always explicit. Read all resulting policies and their origins through
+    :func:`read_diff_contract`. Explicit contracts also accept the existing
+    materialized comparison inputs. DiffContract forbids overrides: construct a
+    new declaration to change its policies. Native row/byte capture bounds are
+    separate from cooperative comparison bounds; neither is a hard RSS quota.
+
+    Raises ValueError/Pydantic ValidationError/InvalidExecutionInput for invalid configuration;
+    CapabilityError for unsupported or ambiguous reference inference;
+    MissingDependencyError, ResourceLimitError or ExecutionCancelled during
+    pre-contract capture; and OSError if evidence cannot be published. These
+    failures do not guarantee a persisted final report. Configured comparisons
+    retain the core's equivalent/different/not_evaluable states and diagnostics.
+    """
+    capture = DiffCaptureOptions.model_validate(
+        DiffCaptureOptions() if capture_options is None else capture_options
+    )
+    checked_options = ComparisonOptions.model_validate(
+        ComparisonOptions() if options is None else options
+    )
+    if reference not in (None, "peer", "expected"):
+        raise ValueError("reference must be peer or expected")
+    if contract is not None and keys is not None:
+        raise ValueError("keys cannot override an explicit contract")
+    right: ComparisonInput | _NativeInput | _CapturedInput
+    if isinstance(contract, DiffContract):
+        if policy is not None or reference is not None:
+            raise ValueError("policy/reference cannot override a DiffContract")
+        declaration = DiffContract.model_validate(contract)
+        right = _input(expected, capture, cancellation)
+    elif contract is not None:
+        if reference is None:
+            declaration = DiffContract.from_contract(contract, policy=policy)
+        else:
+            declaration = DiffContract.from_contract(contract, policy=policy, reference=reference)
+        right = _input(expected, capture, cancellation)
+    else:
+        if isinstance(expected, (ArtifactRef, TableArtifact, ExecutionResult, Path)):
+            raise CapabilityError(
+                "schema inference requires an eager table or an explicit DuckDB relation; "
+                "supply an explicit contract for materialized artifact/execution inputs"
+            )
+        if cancellation is not None and cancellation.cancelled:
+            raise ExecutionCancelled("cancelled before reference capture and inference")
+        observed = expected.table if isinstance(expected, ArrowInput) else expected
+        loaded = capture_native(observed, capture, cancellation=cancellation)
+        if reference is None:
+            declaration = infer_diff_contract(loaded.table.schema, keys=keys, policy=policy)
+        else:
+            declaration = infer_diff_contract(
+                loaded.table.schema, keys=keys, policy=policy, reference=reference
+            )
+        right = _CapturedInput(loaded)
+    result = _compare(
+        _input(candidate, capture, cancellation),
+        right,
+        contract=declaration.spec.contract,
+        policy=declaration.spec.policy,
+        reference=declaration.reference,
+        options=checked_options,
+        work_dir=work_dir,
+        cancellation=cancellation,
+    )
+    receipt = _DiffReceipt(
+        comparison_id=result.comparison_id,
+        spec_sha256=result.spec_sha256,
+        declaration=declaration,
+        capture_options=capture,
+    )
+    directory = local_path(result.differences).parent
+    temporary = directory / "diff-contract.json.part"
+    try:
+        with temporary.open("xb") as target:
+            target.write(receipt.model_dump_json(indent=2).encode("utf-8"))
+            target.flush()
+            os.fsync(target.fileno())
+        os.link(temporary, directory / "diff-contract.json")
+    finally:
+        temporary.unlink(missing_ok=True)
+    return result
+
+
+def read_diff_contract(result: ComparisonResult) -> DiffContract:
+    """Read a diff's companion declaration, bound to its result and policies.
+
+    This supplementary provenance is not a signature or producer attestation.
+    Comparisons made directly with compare have no such companion. Reading
+    does not execute the source or query, or verify all comparison evidence.
+    """
+    result = ComparisonResult.model_validate(result)
+    path = local_path(result.differences).parent / "diff-contract.json"
+    raw = read_document(local_input_path(path))
+    receipt = _DiffReceipt.model_validate_json(raw)
+    declaration = receipt.declaration
+    if (
+        receipt.comparison_id != result.comparison_id
+        or receipt.spec_sha256 != result.spec_sha256
+        or comparison_fingerprint(declaration.spec) != result.spec_sha256
+        or declaration.spec != result.spec
+        or declaration.reference != result.reference
+    ):
+        raise ArtifactIntegrityError("diff declaration does not match the comparison")
+    return declaration