equiframe 0.12.4__py3-none-any.whl

equiframe/examples/journeys/e_telemetry.py

@@ -0,0 +1,135 @@
+"""E: timezone-aware nanoseconds, NULL/NaN, duplicates and sequence semantics."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+import equiframe as ef
+
+if TYPE_CHECKING or __package__:
+    from .common import cli, column, comparison_record, new_output, write_summary
+else:
+    from common import cli, column, comparison_record, new_output, write_summary
+
+SCENARIOS = ("good", "different", "not_evaluable", "null_nan", "duplicate_loss")
+
+
+def declaration() -> ef.DiffContract:
+    return ef.sequence_contract(
+        (
+            column("sensor", ef.StringType()),
+            column("at", ef.TimestampType(unit="ns", timezone="UTC"), nullable=True),
+            column("reading", ef.FloatType(bits=64), nullable=True, unit="degrees_C"),
+        ),
+        by=(ef.SortSpec(column="at", direction="ascending", nulls="last"),),
+        ties="preserve",
+        contract_id="journey.E.telemetry",
+        policy=ef.ComparisonPolicy(
+            physical_types="lossless",
+            justification="Zones may encode the same instant; retain nanoseconds exactly.",
+        ),
+    )
+
+
+def fixture(scenario: str) -> tuple[pa.Table, pa.Table]:
+    import pyarrow as pa
+
+    expected = pa.table(
+        {
+            "sensor": ["s1", "s1", "s1", "s2"],
+            "at": pa.array(
+                [1704067200000000001, 1704067200000000001, 1704067201000000002, None],
+                type=pa.timestamp("ns", tz="UTC"),
+            ),
+            "reading": pa.array(
+                [1.5, 1.5, float("nan"), None], type=pa.float64(), from_pandas=False
+            ),
+        }
+    )
+    candidate = pa.table(
+        {
+            "sensor": ["s1", "s1", "s1", "s2"],
+            "at": pa.array(
+                [1704067200000000001, 1704067200000000001, 1704067201000000002, None],
+                type=pa.timestamp("ns", tz="America/Guayaquil"),
+            ),
+            "reading": pa.array(
+                [
+                    1.5,
+                    1.5,
+                    None if scenario == "null_nan" else float("nan"),
+                    float("nan") if scenario == "null_nan" else None,
+                ],
+                type=pa.float64(),
+                from_pandas=False,
+            ),
+        }
+    )
+    if scenario == "different":
+        candidate = candidate.take([2, 0, 1, 3])
+    elif scenario == "duplicate_loss":
+        candidate = candidate.take([0, 2, 3])
+    return candidate, expected
+
+
+def run(output: Path, scenario: str = "good") -> dict[str, object]:
+    private = new_output(output, scenario, SCENARIOS)
+    candidate, expected = fixture(scenario)
+    declared = declaration()
+    result = ef.diff(
+        candidate,
+        expected,
+        contract=declared,
+        options=ef.ComparisonOptions(max_rows=1 if scenario == "not_evaluable" else 100),
+        work_dir=private,
+    )
+    # This Scan has no final Sort matching the significant-order contract.
+    schema = declared.spec.contract.output.schema_spec
+    contract = ef.Contract(
+        schema_version="1.0",
+        contract_id="journey.E.migration-limit",
+        revision=1,
+        family="dates",
+        inputs=(ef.TableSpec(name="telemetry", schema_spec=schema),),
+        output=ef.TableSpec(name="result", schema_spec=schema),
+        preconditions=(),
+        rules=(),
+    )
+    checks = ef.check_migration(
+        ef.MigrationSpec(
+            contract=contract,
+            source_engine="pandas",
+            target_engine="duckdb",
+            ir=ef.RelationalPlan(
+                nodes=(ef.Scan(node_id="scan", table="telemetry"),), output="scan"
+            ),
+            require_rules=False,
+        )
+    )
+    if any(check.supported for check in checks):
+        raise RuntimeError(
+            "Review the matching-final-Sort preflight contract before changing this example"
+        )
+    return write_summary(
+        output,
+        {
+            "journey": "E",
+            "synthetic": True,
+            "scenario": scenario,
+            "timezone_policy": "instant; UTC and America/Guayaquil",
+            "timestamp_unit": "ns",
+            "duplicate_policy": "preserve",
+            "null_nan_policy": "distinct",
+            "order_policy": "significant",
+            "migration_preflight": [check.model_dump(mode="json") for check in checks],
+            **comparison_record(result, output, "telemetry"),
+        },
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(cli(run, __doc__ or "E", SCENARIOS))

equiframe/examples/journeys/f_data_science_preparation.py

@@ -0,0 +1,84 @@
+"""F: prove feature preparation preserves records; do not infer anomaly quality."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+import equiframe as ef
+
+if TYPE_CHECKING or __package__:
+    from .common import cli, column, comparison_record, new_output, write_summary
+else:
+    from common import cli, column, comparison_record, new_output, write_summary
+
+SCENARIOS = ("good", "different", "not_evaluable", "dropped_row", "changed_id")
+
+
+def materialize(scenario: str) -> tuple[pd.DataFrame, pd.DataFrame]:
+    import pandas as pd
+
+    source = pd.DataFrame(
+        {
+            "sample_id": pd.Series(["0001", "0002", "0003", "0004"], dtype="string[pyarrow]"),
+            "duration_ms": [1000, 2500, 0, 1000000],
+            "events": [2, 3, 0, 99],
+        }
+    )
+    # Explicit predetermined scale; it is not fitted using labels or test outputs.
+    candidate = source[["sample_id", "events"]].assign(duration_s=source["duration_ms"] / 1000.0)
+    candidate = candidate[["sample_id", "duration_s", "events"]]
+    # The large fourth value is intentionally conserved, not called an anomaly.
+    expected = pd.DataFrame(
+        {
+            "sample_id": pd.Series(["0001", "0002", "0003", "0004"], dtype="string[pyarrow]"),
+            "duration_s": [1.0, 2.5, 0.0, 1000.0],
+            "events": [2, 3, 0, 99],
+        }
+    )
+    if scenario == "different":
+        candidate["duration_s"] = candidate["duration_s"].clip(upper=100.0)
+    elif scenario == "dropped_row":
+        candidate = candidate.iloc[:3].reset_index(drop=True)
+    elif scenario == "changed_id":
+        candidate.loc[0, "sample_id"] = "1"
+    return candidate, expected
+
+
+def run(output: Path, scenario: str = "good") -> dict[str, object]:
+    private = new_output(output, scenario, SCENARIOS)
+    candidate, expected = materialize(scenario)
+    declaration = ef.keyed_contract(
+        (
+            column("sample_id", ef.StringType()),
+            column("duration_s", ef.FloatType(bits=64), unit="seconds"),
+            column("events", ef.IntegerType(bits=64, signed=True), unit="events"),
+        ),
+        keys=["sample_id"],
+        contract_id="journey.F.features",
+    )
+    result = ef.diff(
+        candidate,
+        expected,
+        contract=declaration,
+        options=ef.ComparisonOptions(max_rows=1 if scenario == "not_evaluable" else 100),
+        work_dir=private,
+    )
+    return write_summary(
+        output,
+        {
+            "journey": "F",
+            "synthetic": True,
+            "scenario": scenario,
+            "anomaly_detection": "not_requested",
+            "model_training": "not_requested",
+            **comparison_record(result, output, "features"),
+        },
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(cli(run, __doc__ or "F", SCENARIOS))

equiframe/examples/journeys/g_external_snapshots.py

@@ -0,0 +1,197 @@
+"""G: fixed-identity injected client, provenance and byte/schema integrity errors."""
+
+from __future__ import annotations
+
+import hashlib
+from decimal import Decimal
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+import equiframe as ef
+
+if TYPE_CHECKING or __package__:
+    from .common import cli, column, comparison_record, new_output, write_summary
+else:
+    from common import cli, column, comparison_record, new_output, write_summary
+
+SCENARIOS = ("good", "different", "error", "schema_error")
+SNAPSHOT_ID = "synthetic.snapshot.20261006.v1"
+
+
+class MemoryClient:
+    """A trusted transport fixture. No HTTP request, credentials or hidden dataset."""
+
+    def __init__(self, manifest: ef.SnapshotManifest, payload: bytes) -> None:
+        self.manifest = manifest
+        self.payload = payload
+        self.requests: list[str] = []
+        self.downloads = 0
+
+    def get_snapshot(self, snapshot_id: str) -> ef.SnapshotManifest:
+        if snapshot_id != SNAPSHOT_ID:
+            raise LookupError("Unknown fixed snapshot identity")
+        self.requests.append(snapshot_id)
+        return self.manifest
+
+    def download_artifact(self, ref: ef.ArtifactRef, destination: Path) -> None:
+        if ref != self.manifest.tables[0].artifact:
+            raise LookupError("Unknown artifact")
+        self.downloads += 1
+        with destination.open("xb") as stream:
+            stream.write(self.payload)
+
+
+def fixture(scenario: str) -> tuple[MemoryClient, pa.Table, ef.Contract]:
+    import pyarrow as pa
+
+    schema = pa.schema([("id", pa.int64()), ("amount", pa.decimal128(8, 2))])
+    expected = pa.Table.from_pylist(
+        [{"id": 1, "amount": Decimal("1.10")}, {"id": 2, "amount": None}], schema=schema
+    )
+    actual = pa.Table.from_pylist(
+        [
+            {"id": 1, "amount": Decimal("1.11" if scenario == "different" else "1.10")},
+            {"id": 2, "amount": None},
+        ],
+        schema=schema,
+    )
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_file(sink, actual.schema) as writer:
+        writer.write_table(actual)
+    payload = sink.getvalue().to_pybytes()
+    kinds: tuple[ef.LogicalType, ...] = (
+        ef.IntegerType(bits=64, signed=True),
+        ef.DecimalType(precision=8, scale=2),
+    )
+    manifest = ef.SnapshotManifest(
+        snapshot_id=SNAPSHOT_ID,
+        domain_schema_revision="synthetic-ledger.1",
+        extraction_revision="synthetic-generator.1",
+        provenance=("Inline synthetic fixture authored for journey G; no live warehouse",),
+        tables=(
+            ef.SnapshotTable(
+                name="items",
+                artifact=ef.ArtifactRef(
+                    schema_version="1.0",
+                    uri="https://example.invalid/synthetic.arrow",
+                    sha256=hashlib.sha256(payload).hexdigest(),
+                    format="arrow",
+                    media_type="application/vnd.apache.arrow.file",
+                    size_bytes=len(payload),
+                ),
+                row_count=2,
+                columns=tuple(
+                    ef.SnapshotColumn(
+                        name=field.name,
+                        physical_type="int32"
+                        if scenario == "schema_error" and field.name == "id"
+                        else str(field.type),
+                        logical_type=kind,
+                        nullable=field.name == "amount",
+                    )
+                    for field, kind in zip(schema, kinds, strict=True)
+                ),
+            ),
+        ),
+    )
+    # Alter one known synthetic byte after recording the manifest; preserve size.
+    if scenario == "error":
+        payload = bytes([payload[0] ^ 1]) + payload[1:]
+    client = MemoryClient(manifest, payload)
+    declaration = ef.keyed_contract(
+        (
+            column("id", kinds[0], unit="identifier"),
+            column("amount", kinds[1], nullable=True, unit="USD"),
+        ),
+        keys=["id"],
+    )
+    output_schema = declaration.spec.contract.output.schema_spec
+    contract = ef.Contract(
+        schema_version="1.0",
+        contract_id="journey.G.snapshot",
+        revision=1,
+        family="nulls_types",
+        inputs=(ef.TableSpec(name="items", schema_spec=output_schema),),
+        output=ef.TableSpec(name="result", schema_spec=output_schema),
+        preconditions=(),
+        rules=(),
+    )
+    return client, expected, contract
+
+
+def run(output: Path, scenario: str = "good") -> dict[str, object]:
+    private = new_output(output, scenario, SCENARIOS)
+    client, expected, contract = fixture(scenario)
+    source = ef.SnapshotDataSource(
+        SNAPSHOT_ID, client=client, expected_domain_schema_revision="synthetic-ledger.1"
+    )
+    registry = ef.TransformationRegistry()
+    registry.register(ef.DuckDBTransformation("identity", 'SELECT * FROM "items"', "journey.G.v1"))
+    execution = ef.execute(
+        ef.ExecutionPlan(
+            contract=contract,
+            engine="duckdb",
+            transformation_id="identity",
+            options=ef.ExecutionOptions(duckdb_threads=1),
+        ),
+        source=source,
+        registry=registry,
+        work_dir=private,
+    )
+    configuration = {
+        parameter.name: parameter.value.value
+        for parameter in execution.effective_configuration
+        if isinstance(parameter.value, ef.StringValue)
+    }
+    provenance_preserved = configuration.get("snapshot_id") == SNAPSHOT_ID and configuration.get(
+        "snapshot_manifest_sha256"
+    ) == ef.snapshot_fingerprint(client.manifest)
+    record: dict[str, object]
+    if execution.state == "completed":
+        comparison = ef.compare(
+            execution,
+            ef.ArrowInput(expected),
+            contract=contract,
+            reference="expected",
+            work_dir=private,
+        )
+        record = comparison_record(comparison, output, "snapshot")
+    else:
+        record = {
+            "accepted": False,
+            "exit_code": 2,
+            "state": "error",
+            "problem_code": execution.problem.code if execution.problem is not None else None,
+        }
+    if client.requests != [SNAPSHOT_ID] or client.downloads != 1 or not provenance_preserved:
+        raise RuntimeError("Snapshot identity/provenance was not retained")
+    return write_summary(
+        output,
+        {
+            "journey": "G",
+            "synthetic": True,
+            "scenario": scenario,
+            "snapshot_id": SNAPSHOT_ID,
+            "manifest_requests": len(client.requests),
+            "download_count": client.downloads,
+            "provenance_preserved": provenance_preserved,
+            "execution_state": execution.state,
+            "output_available": execution.output is not None,
+            "integrity": (
+                "passed"
+                if execution.state == "completed"
+                else "rejected"
+                if execution.problem is not None
+                and execution.problem.code == "artifact_integrity_error"
+                else "not_evaluable"
+            ),
+            **record,
+        },
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(cli(run, __doc__ or "G", SCENARIOS))

equiframe/examples/journeys/h_portable_ci.py

@@ -0,0 +1,129 @@
+"""H: portable verified evidence, minimized report and a fail-closed CI gate."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+import equiframe as ef
+
+if TYPE_CHECKING or __package__:
+    from .common import cli, column, comparison_record, new_output, write_summary
+else:
+    from common import cli, column, comparison_record, new_output, write_summary
+
+SCENARIOS = ("good", "different", "not_evaluable", "error")
+
+
+def capture_expected(table: pa.Table, directory: Path) -> ef.ArtifactRef:
+    import pyarrow as pa
+
+    path = directory / "literal-expected.arrow"
+    with pa.OSFile(str(path), "wb") as sink, pa.ipc.new_file(sink, table.schema) as writer:
+        writer.write_table(table)
+    return ef.LocalArtifactStore(directory / "expected-store").put(
+        path, media_type="application/vnd.apache.arrow.file"
+    )
+
+
+def run(output: Path, scenario: str = "good") -> dict[str, object]:
+    import pyarrow as pa
+
+    private = new_output(output, scenario, SCENARIOS)
+    original = private / "original"
+    original.mkdir()
+    # Trusted synthetic SQL executes once in an owned/closed in-memory connection.
+    captured = ef.capture_duckdb(
+        "SELECT id, value FROM (VALUES ('SECRET_SYNTH_A', 10::BIGINT), "
+        "('SECRET_SYNTH_B', $amount::BIGINT)) AS t(id, value)",
+        parameters={"amount": 21 if scenario == "different" else 20},
+        work_dir=original / "capture",
+    )
+    expected = capture_expected(
+        pa.table({"id": ["SECRET_SYNTH_A", "SECRET_SYNTH_B"], "value": [10, 20]}), original
+    )
+    contract = ef.keyed_contract(
+        (
+            column("id", ef.StringType()),
+            column("value", ef.IntegerType(bits=64, signed=True), unit="items"),
+        ),
+        keys=["id"],
+        contract_id="journey.H.portable",
+    ).spec.contract
+    verified = ef.verify(
+        {"candidate": captured.data},
+        contract=contract,
+        expected=expected,
+        require_expected=True,
+        options=ef.ComparisonOptions(max_rows=1 if scenario == "not_evaluable" else 100),
+        work_dir=original / "verify",
+    )
+    portable = private / "portable"
+    bundle = ef.export_bundle(original / "verify" / verified.verification_id, portable)
+    manifest_sha256 = hashlib.sha256((portable / "bundle.json").read_bytes()).hexdigest()
+    # The old absolute artifact paths cease to exist before import or inspection.
+    original.rename(private / "retired-original")
+    imported = private / "imported"
+    imported_bundle = ef.import_bundle(portable, imported, expected_sha256=manifest_sha256)
+    if imported_bundle != bundle:
+        raise RuntimeError("Portable inventory changed during import")
+    if scenario == "error":
+        (imported / bundle.root_document).write_text("{}", encoding="utf-8")
+        try:
+            ef.read_bundle(imported, expected_sha256=manifest_sha256)
+        except ef.ArtifactIntegrityError:
+            return write_summary(
+                output,
+                {
+                    "journey": "H",
+                    "synthetic": True,
+                    "scenario": scenario,
+                    "accepted": False,
+                    "exit_code": 2,
+                    "integrity": "rejected",
+                    "report_created": False,
+                    "original_paths_available": original.exists(),
+                },
+            )
+        raise RuntimeError("Synthetic corrupted bundle was unexpectedly accepted")
+    inspected = ef.inspect_bundle(imported, expected_sha256=manifest_sha256)
+    # read/inspect already verified the closed inventory and typed linked records.
+    # Reopen only its validated root to render the imported comparison models.
+    reread = ef.VerificationResult.model_validate_json(
+        (imported / bundle.root_document).read_bytes()
+    )
+    comparisons = [
+        comparison_record(comparison, output, f"comparison-{index}")
+        for index, comparison in enumerate(reread.comparisons)
+    ]
+    accepted = (
+        reread.publishable
+        and bool(comparisons)
+        and all(record["accepted"] for record in comparisons)
+    )
+    exit_code = 0 if accepted else 1 if reread.operational_state == "failed" else 3
+    return write_summary(
+        output,
+        {
+            "journey": "H",
+            "synthetic": True,
+            "scenario": scenario,
+            "accepted": accepted,
+            "exit_code": exit_code,
+            "integrity": inspected["integrity"],
+            "operational_state": reread.operational_state,
+            "bundle_manifest_sha256": manifest_sha256,
+            "original_paths_available": original.exists(),
+            "report_created": True,
+            "independent_review": reread.independent_review.state,
+            "comparisons": comparisons,
+        },
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(cli(run, __doc__ or "H", SCENARIOS))