equiframe-benchmark 0.12.4__py3-none-any.whl

equiframe_benchmark/commands.py

@@ -0,0 +1,149 @@
+"""Readiness and reproduction commands; JSON never chooses executable code."""
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+from typing import cast
+
+import typer
+
+
+def register(app: typer.Typer) -> None:
+    """Attach commands to the caller's app without an eager circular CLI import."""
+    from equiframe_benchmark.cli import _factory, _failure
+
+    @app.command("distribution-readiness")
+    def distribution_command(
+        package_directory: Path,
+        wheel: Path | None = typer.Option(None),
+        name_check: Path | None = typer.Option(None),
+        output: Path | None = typer.Option(None),
+    ) -> None:
+        """Inventory local metadata; no PyPI network requests or publication actions."""
+        from equiframe_benchmark.distribution import NameCheck, inspect_distribution
+        from equiframe_benchmark.io import load_document, save_document
+
+        try:
+            observation = load_document(NameCheck, name_check) if name_check else None
+            result = inspect_distribution(package_directory, wheel, name_check=observation)
+            if output is not None:
+                save_document(result, output)
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(result.model_dump_json())
+
+    @app.command("verify-bundle")
+    def verify_command(
+        directory: Path,
+        expected_sha256: str | None = typer.Option(None),
+    ) -> None:
+        """Verify closed bundle bytes and inventory without importing a regenerator."""
+        from equiframe_benchmark.reproduction import read_bundle
+
+        try:
+            result = read_bundle(directory, expected_sha256=expected_sha256)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "bundle_id": result.bundle_id,
+                    "evidence_verified": True,
+                    "files": len(result.files),
+                    "reproduction_record_available": result.record_path is not None,
+                    "independent_person_reproduced": False,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("reproduce-bundle")
+    def reproduce_command(
+        directory: Path,
+        regenerator: str = typer.Option(
+            ..., help="Explicit trusted module:factory, never from JSON."
+        ),
+        expected_sha256: str | None = typer.Option(None),
+        output: Path | None = typer.Option(None),
+    ) -> None:
+        """Run an explicitly supplied trusted callable; compare complete logical content."""
+        from equiframe_benchmark.io import save_document
+        from equiframe_benchmark.reproduction import (
+            ReproductionRecord,
+            read_bundle,
+            reproduce_bundle,
+        )
+
+        try:
+            # Check persisted evidence before executing even the explicit factory.
+            read_bundle(directory, expected_sha256=expected_sha256)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            supplied = _factory(regenerator)()
+            if not callable(supplied):
+                raise ValueError("regenerator factory must return a callable")
+            callback = cast(Callable[[ReproductionRecord], ReproductionRecord], supplied)
+        except Exception as error:
+            _failure(error)
+            return
+        try:
+            receipt = reproduce_bundle(
+                directory,
+                regenerator=callback,
+                expected_sha256=expected_sha256,
+            )
+            if output is not None:
+                save_document(receipt, output)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(receipt.model_dump_json())
+        if receipt.status == "different":
+            raise typer.Exit(code=1)
+
+    @app.command("bundle-study")
+    def bundle_study_command(directory: Path, output: Path = typer.Option(...)) -> None:
+        """Copy a verified study into a closed bundle without rewriting original evidence."""
+        from equiframe_benchmark.study_bundle import bundle_study
+
+        try:
+            result = bundle_study(directory, output=output)
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "bundle_id": result.bundle_id,
+                    "files": len(result.files),
+                    "independent_person_reproduced": False,
+                    "etl_repeated": False,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("reproduce-study")
+    def reproduce_study_command(
+        directory: Path,
+        expected_sha256: str | None = typer.Option(None),
+        output: Path | None = typer.Option(None),
+    ) -> None:
+        """Regenerate metrics/tables from captured study decisions; never rerun ETL."""
+        from equiframe_benchmark.io import save_document
+        from equiframe_benchmark.study_bundle import reproduce_study_bundle
+
+        try:
+            result = reproduce_study_bundle(directory, expected_sha256=expected_sha256)
+            if output is not None:
+                save_document(result, output)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(result.model_dump_json())
+        if result.status == "different":
+            raise typer.Exit(code=1)

equiframe_benchmark/commands_scale.py

@@ -0,0 +1,38 @@
+"""Engineering scale commands do not change the scientific benchmark protocol."""
+
+import json
+from pathlib import Path
+
+import typer
+from equiframe._cli_support import exit_with_failure
+from equiframe.evidence import load_json
+
+from equiframe_benchmark.scale_types import ScaleSpec
+from equiframe_benchmark.scaling import measure_scale, read_scale, summarize_scale
+
+
+def register(app: typer.Typer) -> None:
+    @app.command("scale")
+    def scale_command(spec: Path, output: Path = typer.Option(...)) -> None:
+        """Measure one bounded synthetic case with safe process completion checks."""
+        try:
+            result = measure_scale(load_json(ScaleSpec, spec), work_dir=output)
+        except Exception as error:
+            exit_with_failure(error, code="scale_operation_failed", exit_code=3)
+        typer.echo(json.dumps(summarize_scale(result), sort_keys=True))
+        raise typer.Exit(code=0 if result.complete else 3)
+
+    @app.command("verify-scale")
+    def verify_scale_command(directory: Path) -> None:
+        """Verify all retained bytes and completion; never rerun a workload."""
+        try:
+            result = read_scale(directory)
+        except Exception as error:
+            exit_with_failure(
+                error,
+                code="scale_evidence_invalid_or_incomplete",
+                exit_code=3,
+                input_validation=False,
+            )
+        typer.echo(json.dumps(summarize_scale(result), sort_keys=True))
+        raise typer.Exit(code=0 if result.complete else 3)

equiframe_benchmark/commands_step10.py

@@ -0,0 +1,275 @@
+"""Explicit experimental commands; persisted JSON never selects executable code."""
+
+import json
+from pathlib import Path
+
+import typer
+
+
+def register(app: typer.Typer) -> None:
+    from equiframe_benchmark.cli import _factory, _failure
+
+    @app.command("freeze")
+    def freeze_command(definition: Path, output: Path = typer.Option(...)) -> None:
+        """Audit and seal a declared pilot/final protocol before test execution."""
+        from equiframe_benchmark.freeze import FreezeDefinition, audit_freeze, create_freeze
+        from equiframe_benchmark.io import load_document, save_document
+
+        try:
+            configuration = load_document(FreezeDefinition, definition)
+            audit = audit_freeze(configuration)
+            if not audit.ready:
+                typer.echo(
+                    json.dumps(
+                        {
+                            "mode": configuration.mode,
+                            "technical_ready": audit.technical_ready,
+                            "scientific_ready": audit.scientific_ready,
+                            "blockers": audit.blockers,
+                            "sealed": False,
+                        },
+                        sort_keys=True,
+                    )
+                )
+                raise typer.Exit(code=2)
+            manifest = create_freeze(configuration)
+            save_document(manifest, output)
+        except typer.Exit:
+            raise
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "mode": manifest.definition.mode,
+                    "technical_ready": manifest.audit.technical_ready,
+                    "scientific_ready": manifest.audit.scientific_ready,
+                    "blockers": manifest.audit.blockers,
+                    "sealed": True,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("verify-freeze")
+    def verify_freeze_command(path: Path) -> None:
+        from equiframe_benchmark.freeze import read_freeze
+
+        try:
+            manifest = read_freeze(path)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "evidence_verified": True,
+                    "mode": manifest.definition.mode,
+                    "technical_ready": manifest.audit.technical_ready,
+                    "scientific_ready": manifest.audit.scientific_ready,
+                    "blockers": manifest.audit.blockers,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("run-study")
+    def study_command(
+        freeze: Path,
+        registry: str = typer.Option(...),
+        output: Path = typer.Option(Path("equiframe-studies")),
+    ) -> None:
+        """Run sealed cases with an explicitly trusted registry, without fitting."""
+        import equiframe as ef
+
+        from equiframe_benchmark.freeze import read_freeze
+        from equiframe_benchmark.study import read_study, run_study
+
+        try:
+            manifest = read_freeze(freeze)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            registered = _factory(registry)()
+            if not isinstance(registered, ef.TransformationRegistry):
+                raise ValueError("registry factory must return a TransformationRegistry")
+            result = run_study(manifest, registry=registered, work_dir=output)
+            if read_study(output.resolve() / result.study_id) != result:
+                raise ValueError("published study differs from its verified receipt")
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "study_id": result.study_id,
+                    "scientific": result.scientific,
+                    "complete": result.complete,
+                    "cases": len(result.evaluation.records),
+                    "evidence_verified": True,
+                },
+                sort_keys=True,
+            )
+        )
+        raise typer.Exit(code=result.exit_code)
+
+    @app.command("verify-study")
+    def verify_study_command(directory: Path) -> None:
+        from equiframe_benchmark.study import read_study
+
+        try:
+            result = read_study(directory)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "study_id": result.study_id,
+                    "scientific": result.scientific,
+                    "complete": result.complete,
+                    "cases": len(result.evaluation.records),
+                    "evidence_verified": True,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("evaluate")
+    def evaluate_command(study: Path, output: Path = typer.Option(...)) -> None:
+        """Regenerate a verified study's report from its complete captured records."""
+        from equiframe_benchmark.evaluation import evaluate_records
+        from equiframe_benchmark.io import save_document
+        from equiframe_benchmark.study import read_study
+
+        try:
+            captured = read_study(study).evaluation
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            report = evaluate_records(captured.records, config=captured.config)
+            if report != captured:
+                raise ValueError("evaluation must reproduce the captured report")
+            save_document(report, output)
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(json.dumps({"scientific": report.scientific, "regenerated": True}))
+
+    @app.command("tables")
+    def tables_command(study: Path, output: Path = typer.Option(...)) -> None:
+        from equiframe_benchmark.study import read_study
+        from equiframe_benchmark.tables import write_tables
+
+        try:
+            report = read_study(study).evaluation
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            receipt = write_tables(report, output)
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(json.dumps({"tables": len(receipt.tables), "scientific": report.scientific}))
+
+    @app.command("costs")
+    def costs_command(
+        spec: Path,
+        bundle: Path = typer.Option(...),
+        output: Path = typer.Option(Path("equiframe-costs")),
+    ) -> None:
+        """Measure profile+M and comparison on previously captured artifacts."""
+        import equiframe as ef
+
+        from equiframe_benchmark.costs import CostSpec, ProfileMCostResult, measure_costs
+        from equiframe_benchmark.io import load_document
+
+        try:
+            configuration = load_document(CostSpec, spec)
+            detector = ef.read_detector_bundle(bundle)
+            if ef.detector_fingerprint(detector) != configuration.detector_bundle_sha256:
+                raise ValueError("costs require the declared frozen detector bundle")
+
+            def profile_m(
+                directory: Path, cancellation: ef.CancellationToken
+            ) -> ProfileMCostResult:
+                profile = ef.profile_execution(
+                    configuration.execution,
+                    spec=configuration.profile,
+                    options=configuration.profile_options,
+                    work_dir=directory,
+                    cancellation=cancellation,
+                )
+                detection = (
+                    ef.detect_component(detector, (profile.vector,), method="isolation_forest")
+                    if profile.complete
+                    else None
+                )
+                return ProfileMCostResult(profile=profile, detection=detection)
+
+            def comparison(
+                directory: Path, cancellation: ef.CancellationToken
+            ) -> ef.ComparisonResult:
+                return ef.compare(
+                    configuration.execution,
+                    configuration.reference,
+                    contract=configuration.execution.plan.contract,
+                    policy=configuration.comparison_policy,
+                    reference=configuration.comparison_reference,
+                    options=configuration.comparison_options,
+                    work_dir=directory,
+                    cancellation=cancellation,
+                )
+
+            result = measure_costs(
+                configuration, profile_m=profile_m, comparison=comparison, work_dir=output
+            )
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "cost_id": result.cost_id,
+                    "complete": result.complete,
+                    "scientific_h3_confirmed": False,
+                },
+                sort_keys=True,
+            )
+        )
+        raise typer.Exit(code=0 if result.complete else 3)
+
+    @app.command("verify-costs")
+    def verify_costs_command(directory: Path) -> None:
+        from equiframe_benchmark.costs import read_costs
+
+        try:
+            result = read_costs(directory)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "cost_id": result.cost_id,
+                    "complete": result.complete,
+                    "evidence_verified": True,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("cost-table")
+    def cost_table_command(directory: Path, output: Path = typer.Option(...)) -> None:
+        from equiframe_benchmark.cost_table import write_cost_table
+
+        try:
+            write_cost_table(directory, output=output)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(json.dumps({"observations": 12, "scientific_h3_confirmed": False}))

equiframe_benchmark/commands_v2.py

@@ -0,0 +1,206 @@
+"""Explicit experimental commands; persisted JSON never selects executable code."""
+
+import json
+from pathlib import Path
+
+import typer
+
+
+def register_v2(app: typer.Typer) -> None:
+    from equiframe_benchmark.cli import _factory, _failure
+
+    @app.command("freeze-v2")
+    def freeze_command(definition: Path, output: Path = typer.Option(...)) -> None:
+        """Audit and seal a declared pilot/final protocol before test execution."""
+        from equiframe_benchmark.freeze_v2 import (
+            FreezeDefinitionV2,
+            audit_freeze_v2,
+            create_freeze_v2,
+        )
+        from equiframe_benchmark.io import load_document, save_document
+
+        try:
+            configuration = load_document(FreezeDefinitionV2, definition)
+            audit = audit_freeze_v2(configuration)
+            if not audit.ready:
+                typer.echo(
+                    json.dumps(
+                        {
+                            "mode": configuration.mode,
+                            "technical_ready": audit.technical_ready,
+                            "scientific_ready": audit.scientific_ready,
+                            "blockers": audit.blockers,
+                            "sealed": False,
+                        },
+                        sort_keys=True,
+                    )
+                )
+                raise typer.Exit(code=2)
+            manifest = create_freeze_v2(configuration)
+            save_document(manifest, output)
+        except typer.Exit:
+            raise
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "mode": manifest.definition.mode,
+                    "technical_ready": manifest.audit.technical_ready,
+                    "scientific_ready": manifest.audit.scientific_ready,
+                    "blockers": manifest.audit.blockers,
+                    "sealed": True,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("verify-freeze-v2")
+    def verify_freeze_command(path: Path) -> None:
+        from equiframe_benchmark.freeze_v2 import read_freeze_v2
+
+        try:
+            manifest = read_freeze_v2(path)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "evidence_verified": True,
+                    "mode": manifest.definition.mode,
+                    "technical_ready": manifest.audit.technical_ready,
+                    "scientific_ready": manifest.audit.scientific_ready,
+                    "blockers": manifest.audit.blockers,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("run-study-v2")
+    def study_command(
+        freeze: Path,
+        registry: str = typer.Option(...),
+        output: Path = typer.Option(Path("equiframe-studies")),
+    ) -> None:
+        """Run sealed cases with an explicitly trusted registry, without fitting."""
+        import equiframe as ef
+
+        from equiframe_benchmark.freeze_v2 import read_freeze_v2
+        from equiframe_benchmark.study_v2 import read_study_v2, run_study_v2
+
+        try:
+            manifest = read_freeze_v2(freeze)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            registered = _factory(registry)()
+            if not isinstance(registered, ef.TransformationRegistry):
+                raise ValueError("registry factory must return a TransformationRegistry")
+            result = run_study_v2(manifest, registry=registered, work_dir=output)
+            if read_study_v2(output.resolve() / result.study_id) != result:
+                raise ValueError("published study differs from its verified receipt")
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "study_id": result.study_id,
+                    "scientific": result.scientific,
+                    "complete": result.complete,
+                    "cases": len(result.evaluation.records),
+                    "evidence_verified": True,
+                },
+                sort_keys=True,
+            )
+        )
+        raise typer.Exit(code=result.exit_code)
+
+    @app.command("verify-study-v2")
+    def verify_study_command(directory: Path) -> None:
+        from equiframe_benchmark.study_v2 import read_study_v2
+
+        try:
+            result = read_study_v2(directory)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(
+            json.dumps(
+                {
+                    "study_id": result.study_id,
+                    "scientific": result.scientific,
+                    "complete": result.complete,
+                    "cases": len(result.evaluation.records),
+                    "evidence_verified": True,
+                },
+                sort_keys=True,
+            )
+        )
+
+    @app.command("evaluate-v2")
+    def evaluate_command(study: Path, output: Path = typer.Option(...)) -> None:
+        """Regenerate a verified study's report from its complete captured records."""
+        from equiframe_benchmark.evaluation_v2 import evaluate_records_v2
+        from equiframe_benchmark.io import save_document
+        from equiframe_benchmark.study_v2 import read_study_v2
+
+        try:
+            captured = read_study_v2(study).evaluation
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            report = evaluate_records_v2(captured.records, config=captured.config)
+            if report != captured:
+                raise ValueError("evaluation must reproduce the captured report")
+            save_document(report, output)
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(json.dumps({"scientific": report.scientific, "regenerated": True}))
+
+    @app.command("tables-v2")
+    def tables_command(study: Path, output: Path = typer.Option(...)) -> None:
+        from equiframe_benchmark.study_v2 import read_study_v2
+        from equiframe_benchmark.tables_v2 import write_tables_v2
+
+        try:
+            report = read_study_v2(study).evaluation
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        try:
+            receipt = write_tables_v2(report, output)
+        except Exception as error:
+            _failure(error)
+            return
+        typer.echo(json.dumps({"tables": len(receipt.tables), "scientific": report.scientific}))
+
+    @app.command("bundle-study-v2")
+    def bundle_command(study: Path, output: Path = typer.Option(...)) -> None:
+        """Copy the V2 captured study and original sources without rewriting provenance."""
+        from equiframe_benchmark.study_bundle_v2 import bundle_study_v2
+
+        try:
+            result = bundle_study_v2(study, output=output)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(json.dumps({"protocol_version": "2.0", "bundle_id": result.bundle_id}))
+
+    @app.command("reproduce-study-v2")
+    def reproduce_command(directory: Path) -> None:
+        """Regenerate paired tables from captured decisions, without executing native ETL."""
+        from equiframe_benchmark.study_bundle_v2 import reproduce_study_bundle_v2
+
+        try:
+            result = reproduce_study_bundle_v2(directory)
+        except Exception as error:
+            _failure(error, input_validation=False)
+            return
+        typer.echo(json.dumps({"protocol_version": "2.0", "status": result.status}))
+        raise typer.Exit(code=0 if result.status == "reproduced" else 1)