equiframe-benchmark 0.12.4__py3-none-any.whl

equiframe_benchmark/_scale_workload.py

@@ -0,0 +1,375 @@
+"""Trusted isolated synthetic cost worker. No source/service/network access."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import shutil
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal, TypeVar
+
+import equiframe as ef
+from equiframe.evidence import describe_file, inventory, load_json, publish_json, verify_files
+
+from equiframe_benchmark.costs import _Monitor, _PsutilProbe
+from equiframe_benchmark.models import digest
+from equiframe_benchmark.scale_io import save_scale_document
+from equiframe_benchmark.scale_types import (
+    SCALE_SAMPLE_INTERVAL_SECONDS,
+    BaselineCost,
+    ScaleSpec,
+    ScaleTrial,
+    ScaleWorkerCompletion,
+    StageCost,
+)
+
+_T = TypeVar("_T")
+
+
+def make_contract(spec: ScaleSpec) -> ef.Contract:
+    columns = tuple(
+        ef.ColumnSpec(
+            name="id" if i == 0 else f"value_{i}",
+            logical_type=ef.StringType()
+            if i > 1 and i % 2 == 0
+            else ef.IntegerType(bits=64, signed=True),
+            nullable=i > 0 and spec.null_every is not None,
+            numeric_policy=None
+            if i > 1 and i % 2 == 0
+            else ef.NumericPolicy(comparison="exact", unit="count"),
+            null_policy=ef.NullPolicy(representations=("null",)),
+        )
+        for i in range(spec.width)
+    )
+    keys = (
+        (
+            ef.KeySpec(
+                name="id",
+                columns=("id",),
+                unique=True,
+                incomplete="reject",
+                scope=ef.TableKeyScope(),
+            ),
+        )
+        if spec.alignment == "keyed"
+        else ()
+    )
+    schema = ef.TableSchema(
+        columns=columns, keys=keys, multiplicity="preserve", order=ef.IgnoredOrder()
+    )
+    return ef.Contract(
+        schema_version="1.0",
+        contract_id="engineering.scale",
+        revision=1,
+        family="duplicates_order",
+        inputs=(ef.TableSpec(name="source", schema_spec=schema),),
+        output=ef.TableSpec(name="result", schema_spec=schema),
+        preconditions=(),
+        rules=(),
+    )
+
+
+def _schema(spec: ScaleSpec) -> Any:
+    pa = importlib.import_module("pyarrow")
+    return pa.schema(
+        [
+            pa.field(
+                "id" if i == 0 else f"value_{i}",
+                pa.string() if i > 1 and i % 2 == 0 else pa.int64(),
+                nullable=i > 0 and spec.null_every is not None,
+            )
+            for i in range(spec.width)
+        ]
+    )
+
+
+def _value(spec: ScaleSpec, row: int, column: int, *, changed: bool) -> int | str | None:
+    identity = row % (spec.distinct_rows or spec.rows)
+    if column == 0:
+        return identity
+    # Alter the first payload column after NULL generation, so even NULL is a real change.
+    if changed and column == 1:
+        return -(identity + 1)
+    if spec.null_every is not None and identity % spec.null_every == 0:
+        return None
+    number = 0 if identity % 100 < spec.skew_percent else identity % spec.cardinality
+    if column > 1 and column % 2 == 0:
+        return ("x" * (1 + number % spec.text_max_length)) + str(number)
+    return number
+
+
+def capture(spec: ScaleSpec, directory: Path) -> tuple[Path, Path]:
+    """Generate both complete sides in bounded batches directly into Arrow IPC files."""
+    pa = importlib.import_module("pyarrow")
+    schema = _schema(spec)
+    paths = directory / "candidate.arrow", directory / "expected.arrow"
+    for side, path in enumerate(paths):
+        with pa.OSFile(str(path), "wb") as sink, pa.ipc.new_file(sink, schema) as writer:
+            for start in range(0, spec.rows, spec.batch_rows):
+                rows = range(start, min(start + spec.batch_rows, spec.rows))
+                arrays = [
+                    pa.array(
+                        [
+                            _value(
+                                spec,
+                                row,
+                                column,
+                                changed=side == 0
+                                and (
+                                    (spec.difference_last and row == spec.rows - 1)
+                                    or (
+                                        spec.difference_every is not None
+                                        and row % spec.difference_every == 0
+                                    )
+                                ),
+                            )
+                            for row in rows
+                        ],
+                        type=schema.field(column).type,
+                    )
+                    for column in range(spec.width)
+                ]
+                writer.write_batch(pa.RecordBatch.from_arrays(arrays, schema=schema))
+    return paths
+
+
+def _snapshot(paths: tuple[Path, Path], directory: Path) -> None:
+    pa = importlib.import_module("pyarrow")
+    records = []
+    for path in paths:
+        with pa.memory_map(str(path), "r") as source:
+            reader = pa.ipc.open_file(source)
+            rows = sum(reader.get_batch(i).num_rows for i in range(reader.num_record_batches))
+            records.append(
+                {"artifact": describe_file(path, directory).model_dump(mode="json"), "rows": rows}
+            )
+    publish_json({"inputs": records}, directory / "snapshot.json")
+
+
+def _baseline(spec: ScaleSpec, paths: tuple[Path, Path], method: str) -> BaselineCost:
+    name: Any = method
+    if method == "datacompy":
+        return BaselineCost(
+            method=name,
+            state="not_available",
+            wall_seconds=None,
+            reason=(
+                "DataComPy is not a pinned dependency of this candidate; "
+                "no untested semantic or version claims"
+            ),
+        )
+    if method == "pandas_assert" and spec.alignment != "keyed":
+        return BaselineCost(
+            method=name,
+            state="not_applicable",
+            wall_seconds=None,
+            reason=(
+                "Native assertion baseline is restricted to declared unique keys; "
+                "no silent row sorting or multiset policy substitution"
+            ),
+        )
+    start = time.perf_counter()
+    state: Literal["equivalent", "different"]
+    try:
+        pa = importlib.import_module("pyarrow")
+        engine = importlib.import_module("pandas" if method == "pandas_assert" else "duckdb")
+        tables = []
+        for path in paths:
+            with pa.memory_map(str(path), "r") as source:
+                tables.append(pa.ipc.open_file(source).read_all())
+        if not tables[0].schema.equals(tables[1].schema, check_metadata=True):
+            state = "different"
+        elif method == "pandas_assert":
+            frames = [
+                table.to_pandas(types_mapper=engine.ArrowDtype)
+                .sort_values("id")
+                .reset_index(drop=True)
+                for table in tables
+            ]
+            try:
+                engine.testing.assert_frame_equal(
+                    frames[0], frames[1], check_exact=True, check_dtype=True, check_names=True
+                )
+                state = "equivalent"
+            except AssertionError:
+                state = "different"
+        else:
+            with engine.connect(config={"threads": "1", "memory_limit": "256MB"}) as connection:
+                connection.register("lhs", tables[0])
+                connection.register("rhs", tables[1])
+                delta = connection.execute(
+                    "SELECT EXISTS(SELECT * FROM (SELECT * FROM lhs EXCEPT ALL SELECT * FROM rhs) "
+                    "UNION ALL SELECT * FROM (SELECT * FROM rhs EXCEPT ALL SELECT * FROM lhs))"
+                ).fetchone()
+                state = "different" if delta[0] else "equivalent"
+        return BaselineCost(
+            method=name,
+            state=state,
+            wall_seconds=time.perf_counter() - start,
+            reason=(
+                "Exact integers/text/NULL and equal Arrow schemas; includes native loading/"
+                "conversion/query; excludes full differences, durable snapshots, reporting "
+                "and integrity verification"
+            ),
+        )
+    except ImportError:
+        return BaselineCost(
+            method=name,
+            state="not_available",
+            wall_seconds=None,
+            reason="Optional native engine is not installed",
+        )
+    except Exception as error:
+        return BaselineCost(
+            method=name,
+            state="failed",
+            wall_seconds=time.perf_counter() - start,
+            reason=type(error).__name__,
+        )
+
+
+def run_trial(spec: ScaleSpec, directory: Path, repetition: int) -> ScaleTrial:
+    directory.mkdir()
+    stages: list[StageCost] = []
+    monitor = _Monitor(_PsutilProbe, SCALE_SAMPLE_INTERVAL_SECONDS)
+    monitor.start()
+    start = time.perf_counter()
+
+    def measured(name: str, scope: str, action: Callable[[], _T]) -> _T:
+        began = time.perf_counter()
+        value = action()
+        stages.append(StageCost(name=name, wall_seconds=time.perf_counter() - began, scope=scope))
+        return value
+
+    try:
+        paths = measured(
+            "capture",
+            "Synthetic generation and batch Arrow IPC writing; no external database",
+            lambda: capture(spec, directory),
+        )
+        measured(
+            "snapshot",
+            "Complete input hashing and batch row counting",
+            lambda: _snapshot(paths, directory),
+        )
+        transferred = directory / "left.arrow", directory / "right.arrow"
+        measured(
+            "transfer",
+            "Local file copies only, no network transfer",
+            lambda: tuple(shutil.copyfile(a, b) for a, b in zip(paths, transferred, strict=True)),
+        )
+        contract = measured(
+            "plan",
+            "Declare and validate the exact contract; "
+            "comparator internal planning remains in compare",
+            lambda: make_contract(spec),
+        )
+        result = measured(
+            "compare",
+            "Public compare end-to-end including its input snapshots, planning, "
+            "exact SQLite work, serialization and persistence",
+            lambda: ef.compare(
+                *transferred,
+                contract=contract,
+                reference="expected",
+                work_dir=directory / "comparison",
+                options=spec.comparison_options,
+            ),
+        )
+        serialized = measured(
+            "serialize",
+            "Additional standalone serialization of the returned comparison document",
+            lambda: result.model_dump_json().encode("utf-8"),
+        )
+
+        def persist() -> None:
+            target = directory / "serialized-comparison.json"
+            partial = directory / "serialized-comparison.json.part"
+            with partial.open("xb") as output:
+                output.write(serialized)
+                output.flush()
+                os.fsync(output.fileno())
+            os.link(partial, target)
+            partial.unlink()
+
+        measured(
+            "persist",
+            "Additional standalone fsync plus exclusive atomic manifest publication",
+            persist,
+        )
+        measured(
+            "render",
+            "Privacy-filtered report construction and HTML/text/JSON publication "
+            "without reexecuting inputs",
+            lambda: ef.write_comparison_report(result, directory / "report"),
+        )
+
+        def verify() -> int:
+            restored = ef.read_comparison(directory / "serialized-comparison.json")
+            if restored != result:
+                raise ValueError("comparison roundtrip changed")
+            count = sum(1 for _ in ef.iter_differences(result.differences))
+            verify_files(directory, inventory(directory))
+            return count
+
+        count = measured(
+            "verify",
+            "Comparison reader, all difference records, and complete evidence SHA-256 reread",
+            verify,
+        )
+        elapsed = time.perf_counter() - start
+    finally:
+        resources = monitor.finish(time.perf_counter() - start)
+    # Baselines are separate services and outside EquiFrame pipeline duration/RSS samples.
+    baselines = (
+        tuple(
+            _baseline(spec, transferred, name)
+            for name in ("pandas_assert", "duckdb_sql", "datacompy")
+        )
+        if spec.run_baselines
+        else ()
+    )
+    evidence = inventory(directory)
+    return ScaleTrial(
+        case_sha256=digest(spec),
+        repetition=repetition,
+        cache_state="fresh_process_os_cache_uncontrolled"
+        if repetition == 0
+        else "warm_process_os_cache_uncontrolled",
+        stages=tuple(stages),
+        end_to_end_seconds=elapsed,
+        resources=resources,
+        comparison=result,
+        verified_difference_records=count,
+        evidence_bytes=sum(item.size_bytes for item in evidence),
+        evidence=evidence,
+        baselines=baselines,
+    )
+
+
+def main() -> None:
+    from equiframe_benchmark.scaling import _environment
+
+    spec = load_json(ScaleSpec, Path(sys.argv[1]))
+    directory = Path(sys.argv[2])
+    for repetition in range(spec.repetitions):
+        trial = run_trial(spec, directory / f"trial-{repetition:03d}", repetition)
+        save_scale_document(trial, directory / f"trial-{repetition:03d}.json")
+    files = tuple(
+        item
+        for item in inventory(directory)
+        if item.path in {f"trial-{i:03d}.json" for i in range(spec.repetitions)}
+    )
+    save_scale_document(
+        ScaleWorkerCompletion(
+            environment=_environment(), case_sha256=digest(spec), trial_files=files
+        ),
+        directory / "worker-completed.json",
+    )
+
+
+if __name__ == "__main__":
+    main()

equiframe_benchmark/calibration_types.py

@@ -0,0 +1,112 @@
+"""Reviewed calibration lineage and group support stay outside detector features."""
+
+from typing import Literal
+
+from equiframe import NormalCalibrationProfiles
+from pydantic import model_validator
+from typing_extensions import Self
+
+from equiframe_benchmark.models import Digest, Document, Text, digest
+from equiframe_benchmark.training_types import TrainingRecord, TrainingSelection
+
+
+class CalibrationManifest(Document):
+    purpose: Literal["normal_calibration"] = "normal_calibration"
+    catalog_sha256: Digest
+    splits_sha256: Digest
+    records: tuple[TrainingRecord, ...]
+
+    @model_validator(mode="after")
+    def unique(self) -> Self:
+        if not self.records:
+            raise ValueError("nonempty normal calibration selection required")
+        for field in ("candidate_id", "execution_sha256", "profile_sha256"):
+            if len({getattr(r, field) for r in self.records}) != len(self.records):
+                raise ValueError("duplicate calibration evidence")
+        return self
+
+
+class CalibrationSelection(Document):
+    manifest: CalibrationManifest
+    calibration: NormalCalibrationProfiles
+
+    @model_validator(mode="after")
+    def consistent(self) -> Self:
+        if digest(self.manifest) != self.calibration.selection_manifest_sha256:
+            raise ValueError("calibration manifest mismatch")
+        if len(self.manifest.records) != len(self.calibration.profiles):
+            raise ValueError("calibration selection size mismatch")
+        if any(
+            record.vector_sha256 != digest(vector)
+            for record, vector in zip(self.manifest.records, self.calibration.profiles, strict=True)
+        ):
+            raise ValueError("calibration vectors do not match recorded evidence")
+        return self
+
+
+class DetectorSelectionAudit(Document):
+    training_sha256: Digest
+    calibration_sha256: Digest
+    training_groups: int
+    calibration_groups: int
+    min_training_groups: int
+    min_calibration_groups: int
+    rationale: Text
+    state: Literal["eligible", "not_estimable"]
+
+    @model_validator(mode="after")
+    def consistent(self) -> Self:
+        if min(self.min_training_groups, self.min_calibration_groups) < 1:
+            raise ValueError("explicit positive group minima required")
+        if min(self.training_groups, self.calibration_groups) < 1:
+            raise ValueError("selections must contain groups")
+        eligible = (
+            self.training_groups >= self.min_training_groups
+            and self.calibration_groups >= self.min_calibration_groups
+        )
+        if (self.state == "eligible") != eligible:
+            raise ValueError("audit state contradicts group support")
+        return self
+
+
+def audit_detector_selections(
+    training: TrainingSelection,
+    calibration: CalibrationSelection,
+    *,
+    min_training_groups: int,
+    min_calibration_groups: int,
+    rationale: str,
+) -> DetectorSelectionAudit:
+    """Check pair provenance/support before fitting; eligibility is not a power guarantee."""
+    training = TrainingSelection.model_validate(training)
+    calibration = CalibrationSelection.model_validate(calibration)
+    t, c = training.manifest, calibration.manifest
+    if t.catalog_sha256 != c.catalog_sha256 or t.splits_sha256 != c.splits_sha256:
+        raise ValueError("training and calibration require the same catalog and split revision")
+    if (
+        len(
+            {
+                p.spec_sha256
+                for p in (*training.training.profiles, *calibration.calibration.profiles)
+            }
+        )
+        != 1
+    ):
+        raise ValueError("a detector pair requires one family and feature revision")
+    for field in ("group_id", "scenario_id", "candidate_id", "execution_sha256", "profile_sha256"):
+        if {getattr(r, field) for r in t.records} & {getattr(r, field) for r in c.records}:
+            raise ValueError("training/calibration leakage in " + field)
+    train_groups = len({r.group_id for r in t.records})
+    calibration_groups = len({r.group_id for r in c.records})
+    return DetectorSelectionAudit(
+        training_sha256=digest(training),
+        calibration_sha256=digest(calibration),
+        training_groups=train_groups,
+        calibration_groups=calibration_groups,
+        min_training_groups=min_training_groups,
+        min_calibration_groups=min_calibration_groups,
+        rationale=rationale,
+        state="eligible"
+        if train_groups >= min_training_groups and calibration_groups >= min_calibration_groups
+        else "not_estimable",
+    )

equiframe_benchmark/cli.py

@@ -0,0 +1,196 @@
+"""Benchmark commands keep experimental labels outside the operational library."""
+
+import importlib
+import json
+import re
+from collections.abc import Callable
+from pathlib import Path
+from uuid import uuid4
+
+import typer
+from equiframe._cli_support import exit_with_failure
+
+from equiframe_benchmark.commands import register as _register_delivery
+from equiframe_benchmark.commands_scale import register as _register_scale
+from equiframe_benchmark.commands_step10 import register as _register_step10
+from equiframe_benchmark.commands_v2 import register_v2 as _register_v2
+from equiframe_benchmark.io import load_document, save_document
+from equiframe_benchmark.models import Catalog, SplitManifest
+from equiframe_benchmark.splits import dependency_groups, make_splits, split_counts, validate_splits
+
+app = typer.Typer(no_args_is_help=True, add_completion=False, pretty_exceptions_enable=False)
+
+
+def _failure(error: Exception, *, input_validation: bool = True) -> None:
+    invalid = input_validation and isinstance(error, ValueError)
+    exit_with_failure(
+        error,
+        code="invalid_input" if invalid else "operation_failed",
+        exit_code=2 if invalid else 3,
+        input_validation=input_validation,
+    )
+
+
+@app.command("validate")
+def validate_command(catalog: Path, splits: Path | None = typer.Option(None)) -> None:
+    """Validate a catalog and optional grouped partition manifest; never create labels."""
+    try:
+        document = load_document(Catalog, catalog)
+        summary: dict[str, object] = {
+            "catalog_id": document.catalog_id,
+            "synthetic": document.synthetic,
+            "scenarios": len(document.scenarios),
+            "groups": len(dependency_groups(document)),
+            "scientific_review_confirmed": False,
+        }
+        if splits:
+            manifest = load_document(SplitManifest, splits)
+            validate_splits(document, manifest)
+            summary["partitions"] = split_counts(document, manifest)
+    except Exception as error:
+        _failure(error)
+        return
+    typer.echo(json.dumps(summary, sort_keys=True))
+
+
+@app.command("split")
+def split_command(
+    catalog: Path,
+    output: Path = typer.Option(...),
+    seed: int = typer.Option(...),
+    training: int = typer.Option(3),
+    calibration: int = typer.Option(1),
+    test: int = typer.Option(2),
+) -> None:
+    """Create an exclusive grouped split with an explicit seed and integer weights."""
+    try:
+        document = load_document(Catalog, catalog)
+        manifest = make_splits(document, seed=seed, weights=(training, calibration, test))
+        validate_splits(document, manifest)
+        save_document(manifest, output)
+    except Exception as error:
+        _failure(error)
+        return
+    typer.echo(json.dumps(split_counts(document, manifest), sort_keys=True))
+
+
+@app.command("pilot")
+def pilot_command(output: Path = typer.Option(Path("equiframe-benchmark-runs"))) -> None:
+    """Run a new synthetic native pilot; human reviews remain pending."""
+    from equiframe_benchmark.pilot import run_pilot
+
+    directory = output.resolve() / ("pilot_" + uuid4().hex)
+    try:
+        summary = run_pilot(directory)
+    except Exception as error:
+        _failure(error)
+        return
+    typer.echo(json.dumps({"pilot_id": directory.name, **summary}, sort_keys=True))
+
+
+def _factory(reference: str) -> Callable[[], object]:
+    from typing import cast
+
+    if not re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:[A-Za-z_]\w*", reference):
+        raise ValueError("explicit module:factory reference is required")
+    module, name = reference.split(":")
+    factory = getattr(importlib.import_module(module), name)
+    if not callable(factory):
+        raise ValueError("registered factory must be callable")
+    return cast(Callable[[], object], factory)
+
+
+@app.command("integrate")
+def integrate_command(
+    spec: Path,
+    snapshot_id: str = typer.Option(...),
+    client: str = typer.Option(...),
+    registry: str = typer.Option(...),
+    output: Path = typer.Option(Path("equiframe-integration")),
+    expected: Path | None = typer.Option(None),
+    bundle: Path | None = typer.Option(None),
+) -> None:
+    """Consume a pinned snapshot through explicitly supplied trusted client/code."""
+    from typing import cast
+
+    import equiframe as ef
+    from equiframe.evidence import load_json
+
+    from equiframe_benchmark.integration import IntegrationSpec, run_integration
+    from equiframe_benchmark.snapshot import SnapshotClient, SnapshotDataSource
+
+    try:
+        configuration = load_json(IntegrationSpec, spec)
+        detector = load_json(ef.DetectorBundle, bundle) if bundle else None
+        registered = _factory(registry)()
+        if not isinstance(registered, ef.TransformationRegistry):
+            raise ValueError("registry factory must return a TransformationRegistry")
+        transport = _factory(client)()
+        if not all(
+            callable(getattr(transport, name, None))
+            for name in ("get_snapshot", "download_artifact")
+        ):
+            raise ValueError("client factory must expose the snapshot client protocol")
+        source = SnapshotDataSource(
+            snapshot_id,
+            client=cast(SnapshotClient, transport),
+            expected_domain_schema_revision=configuration.domain_schema_revision,
+            table_names=tuple(
+                table.name for table in configuration.pipelines[0].execution.contract.inputs
+            ),
+        )
+        result = run_integration(
+            configuration,
+            source=source,
+            registry=registered,
+            detector_bundle=detector,
+            expected=expected,
+            work_dir=output,
+        )
+    except Exception as error:
+        _failure(error)
+        return
+    typer.echo(
+        json.dumps(
+            {
+                "integration_id": result.integration_id,
+                "complete": result.complete,
+                "inputs_verified": result.inputs_verified,
+                "scientific_equivalence_confirmed": False,
+            },
+            sort_keys=True,
+        )
+    )
+    raise typer.Exit(code=result.exit_code)
+
+
+@app.command("verify-integration")
+def verify_integration_command(directory: Path) -> None:
+    """Verify the existing receipt and evidence without contacting the service."""
+    from equiframe_benchmark.integration import read_integration
+
+    try:
+        result = read_integration(directory)
+    except Exception as error:
+        _failure(error, input_validation=False)
+        return
+    typer.echo(
+        json.dumps(
+            {
+                "integration_id": result.integration_id,
+                "evidence_verified": True,
+                "complete": result.complete,
+            },
+            sort_keys=True,
+        )
+    )
+
+
+_register_scale(app)
+_register_step10(app)
+_register_v2(app)
+_register_delivery(app)
+
+
+def main() -> None:
+    app()