equiframe 0.12.4__py3-none-any.whl

equiframe/examples/integrations/eda_generic.py

@@ -0,0 +1,60 @@
+"""Reproducible synthetic EDA; standalone, no migration or detector training."""
+
+import argparse
+import csv
+import hashlib
+from pathlib import Path
+
+from equiframe.eda import Conversion, EdaConfig, NumericRule, Source, analyze, read_result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--work-dir", type=Path, required=True)
+    args = parser.parse_args()
+    args.work_dir.mkdir(parents=True, exist_ok=False)
+    inputs = args.work_dir / "input"
+    inputs.mkdir()
+    path = inputs / "sales.csv"
+    with path.open("x", encoding="utf8", newline="") as stream:
+        writer = csv.writer(stream)
+        writer.writerow(["id", "amount", "currency"])
+        writer.writerows(
+            (f"{i:03}", f"{v}.00", "USD") for i, v in enumerate([1, 2, 3, 4, 5, 6, 7, 100], 1)
+        )
+    fingerprint = hashlib.sha256(path.read_bytes()).hexdigest()
+    result = analyze(
+        Source(
+            path=str(path),
+            format="csv",
+            snapshot_id="synthetic-sales-v1",
+            expected_sha256=fingerprint,
+        ),
+        config=EdaConfig(
+            charts=True,
+            identifiers=("id",),
+            conversions=(Conversion(column="amount", target="money", kind="decimal", scale=2),),
+            numeric=(
+                NumericRule(
+                    column="money", groups=("currency",), monetary=True, currency_column="currency"
+                ),
+            ),
+        ),
+        output=args.work_dir / "eda",
+    )
+    if (
+        result.state != "completed"
+        or result.rows != 8
+        or read_result(args.work_dir / "eda") != result
+    ):
+        raise RuntimeError("synthetic EDA incomplete or inconsistent")
+    if result.numeric[0]["groups"][0]["outliers"]["count"] != 1:
+        raise RuntimeError("independent expected outlier count differs")
+    if hashlib.sha256(path.read_bytes()).hexdigest() != fingerprint:
+        raise RuntimeError("input changed")
+    print("PASS: eight synthetic rows, exact Decimal, one investigation candidate; input unchanged")
+    print(args.work_dir / "eda/report.html")
+
+
+if __name__ == "__main__":
+    main()

equiframe/examples/integrations/outcome_gates.py

@@ -0,0 +1,94 @@
+"""Synthetic installed-package smoke: reports and empty outputs never grant approval.
+
+Requires the identified private wheel with the arrow extra. Run this copied file
+outside the repository. Every case uses a new child directory; no prior evidence
+is overwritten. Exit zero means the expected gate behavior was observed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import tempfile
+from pathlib import Path
+
+import pyarrow as pa
+
+import equiframe as ef
+
+
+def reject_truth_value(value: object) -> None:
+    """Treat failure to reject implicit approval as a broken example, even with -O."""
+    try:
+        bool(value)
+    except TypeError as error:
+        if "has no truth value" not in str(error):
+            raise
+    else:
+        raise RuntimeError(f"{type(value).__name__} incorrectly accepted a truth test")
+
+
+def run(work_dir: Path) -> list[dict[str, str | bool | int]]:
+    work_dir.mkdir(parents=True, exist_ok=True)
+    expected = pa.table({"id": ["one", "two"], "quantity": [10, 20]})
+    observations: list[dict[str, str | bool | int]] = []
+    for case in ("equivalent", "different", "not_evaluable", "empty"):
+        reference = expected.slice(0, 0) if case == "empty" else expected
+        candidate = (
+            pa.table({"id": ["one", "two"], "quantity": [10, 21]})
+            if case == "different"
+            else reference
+        )
+        directory = Path(tempfile.mkdtemp(prefix=case + "-", dir=work_dir))
+        result = ef.diff(
+            candidate,
+            reference,
+            keys=["id"],
+            options=ef.ComparisonOptions(max_work=1) if case == "not_evaluable" else None,
+            work_dir=directory / "comparison",
+        )
+        report = ef.write_comparison_report(result, directory / "report")
+        reject_truth_value(result)
+        reject_truth_value(report)
+        wanted_state = "equivalent" if case == "empty" else case
+        if result.state != wanted_state or report.state != result.state:
+            raise RuntimeError("comparison/report state differs from the synthetic expectation")
+        try:
+            ef.assert_equivalent(result)
+        except ef.EquivalenceDifferentError:
+            gate = "different"
+        except ef.EquivalenceNotEvaluableError:
+            gate = "not_evaluable"
+        else:
+            gate = "equivalent"
+        if gate != wanted_state:
+            raise RuntimeError("the explicit assertion did not preserve the comparison verdict")
+        # Emptiness is legitimate only for this captured-output contract. It does
+        # not establish a business requirement that any records must exist.
+        observations.append(
+            {
+                "case": case,
+                "state": result.state,
+                "complete": result.complete,
+                "candidate_rows": candidate.num_rows,
+                "explicit_gate": gate,
+                "report_generated": True,
+                "implicit_approval_rejected": True,
+            }
+        )
+    for state in ("passed", "failed", "not_evaluable", "not_requested", "pending_review"):
+        reject_truth_value(ef.VerificationCheck(state=state, reason="Synthetic gate example"))
+    reject_truth_value(ef.CompilationCheck(engine="synthetic", supported=True))
+    reject_truth_value(ef.DetectorScores(state="not_estimable", reason="No input support"))
+    return observations
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--work-dir", type=Path, required=True)
+    args = parser.parse_args()
+    print(json.dumps(run(args.work_dir), sort_keys=True, indent=2))
+
+
+if __name__ == "__main__":
+    main()

equiframe/examples/integrations/sqlite_snapshot.py

@@ -0,0 +1,246 @@
+"""Small trusted SQLite producer for EquiFrame's existing SnapshotClient port.
+
+This external example is deliberately not an equiframe module, general DB-API
+adapter, remote data service or SQL sandbox. See docs/sqlite-snapshots.md.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+import sqlite3
+from collections.abc import Mapping
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+from types import MappingProxyType
+from urllib.parse import quote
+from uuid import uuid4
+
+import pyarrow as pa
+
+import equiframe as ef
+
+
+@dataclass(frozen=True)
+class SQLiteQuery:
+    """Trusted SELECT/WITH and its independently declared output Arrow schema.
+
+    Positional parameters are bound by SQLite, never interpolated. Only int64,
+    string and decimal128 with nonnegative scale are supported by this example.
+    Decimal input must be exact decimal TEXT, never SQLite REAL or NUMERIC.
+    """
+
+    name: str
+    sql: str
+    schema: pa.Schema
+    parameters: tuple[str | int | float | bytes | None, ...] = ()
+
+
+@dataclass(frozen=True)
+class SQLiteSnapshotClient:
+    """Detached immutable bytes; no live connection or database re-query on fetch."""
+
+    manifest: ef.SnapshotManifest
+    _payloads: Mapping[str, bytes]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "_payloads", MappingProxyType(dict(self._payloads)))
+
+    def get_snapshot(self, snapshot_id: str) -> ef.SnapshotManifest:
+        if snapshot_id != self.manifest.snapshot_id:
+            raise LookupError("unknown snapshot identity; request the captured ID")
+        return self.manifest
+
+    def download_artifact(self, ref: ef.ArtifactRef, destination: Path) -> None:
+        if not any(table.artifact == ref for table in self.manifest.tables):
+            raise LookupError("artifact is not part of this captured snapshot")
+        # The URI is an identifier, not a network location this client visits.
+        with destination.open("xb") as stream:
+            stream.write(self._payloads[ref.uri])
+
+
+def _logical_type(data_type: pa.DataType) -> ef.LogicalType:
+    if pa.types.is_int64(data_type):
+        return ef.IntegerType(bits=64, signed=True)
+    if pa.types.is_string(data_type):
+        return ef.StringType()
+    if pa.types.is_decimal128(data_type) and 0 <= data_type.scale <= data_type.precision:
+        return ef.DecimalType(precision=data_type.precision, scale=data_type.scale)
+    raise ValueError("SQLite example supports only int64, string and decimal128(p, 0..p)")
+
+
+def _decode(value: object, field: pa.Field) -> object:
+    if value is None:
+        if not field.nullable:
+            raise ValueError("SQLite returned NULL for a nonnullable declared field")
+        return None
+    if pa.types.is_int64(field.type):
+        if type(value) is not int:
+            raise ValueError("declared int64 requires SQLite INTEGER; no implicit cast")
+        return value
+    if pa.types.is_string(field.type):
+        if type(value) is not str:
+            raise ValueError("declared string requires SQLite TEXT; no implicit cast")
+        return value
+    if type(value) is not str or re.fullmatch(r"-?[0-9]+(?:\.[0-9]+)?", value) is None:
+        raise ValueError("declared decimal requires plain exact decimal TEXT, never REAL")
+    result = Decimal(value)
+    if result.is_zero() and result.is_signed():
+        raise ValueError("Arrow decimal cannot preserve negative zero; revise the source schema")
+    # pa.array(..., safe=True) below rejects overflow and any nonzero digits
+    # beyond the declared scale. No float intermediary or rounding is used.
+    return result
+
+
+def _read_query(
+    connection: sqlite3.Connection,
+    query: SQLiteQuery,
+    *,
+    max_rows_per_table: int,
+    batch_rows: int,
+) -> pa.Table:
+    cursor = connection.cursor()
+    try:
+        cursor.execute(query.sql, query.parameters)
+        if cursor.description is None or tuple(item[0] for item in cursor.description) != tuple(
+            query.schema.names
+        ):
+            raise ValueError("query names/order differ from the declared Arrow schema")
+        batches = []
+        count = 0
+        while rows := cursor.fetchmany(batch_rows):
+            count += len(rows)
+            if count > max_rows_per_table:
+                raise ef.ResourceLimitError("SQLite capture exceeds its per-table row budget")
+            arrays = [
+                pa.array([_decode(row[index], field) for row in rows], type=field.type, safe=True)
+                for index, field in enumerate(query.schema)
+            ]
+            batches.append(pa.RecordBatch.from_arrays(arrays, schema=query.schema))
+        if not connection.in_transaction:
+            raise RuntimeError("the native SQLite read transaction ended before capture finished")
+        return pa.Table.from_batches(batches, schema=query.schema)
+    finally:
+        cursor.close()
+
+
+def capture_sqlite_snapshot(
+    database: Path,
+    *,
+    snapshot_id: str,
+    queries: tuple[SQLiteQuery, ...],
+    domain_schema_revision: str,
+    extraction_revision: str,
+    provenance: tuple[str, ...],
+    max_rows_per_table: int = 10_000,
+    max_payload_bytes: int = 16 * 1024 * 1024,
+    batch_rows: int = 256,
+    lock_timeout_seconds: float = 5.0,
+) -> SQLiteSnapshotClient:
+    """Capture every query in one owned read transaction; publish only on success.
+
+    The database must already exist. No caller connection is borrowed, committed
+    or closed. The snapshot is anchored by the first native read, shared across
+    queries, then detached. All cursors and the connection close on every path.
+    Budget checks are cooperative, not hard memory, query-time or process limits.
+    """
+    for value in (max_rows_per_table, max_payload_bytes, batch_rows):
+        if type(value) is not int or value <= 0:
+            raise ValueError("capture budgets and batch_rows must be positive integers")
+    if (
+        type(lock_timeout_seconds) not in (int, float)
+        or not math.isfinite(lock_timeout_seconds)
+        or lock_timeout_seconds < 0
+    ):
+        raise ValueError("lock_timeout_seconds must be finite and nonnegative")
+    if not queries or len({query.name for query in queries}) != len(queries):
+        raise ValueError("capture requires nonempty queries with unique table names")
+    declarations = []
+    # Validate the complete declaration before opening a connection or doing SQL.
+    for query in queries:
+        if re.match(r"\s*(?:SELECT|WITH)\b", query.sql, re.IGNORECASE) is None:
+            raise ValueError("provide one trusted SELECT/WITH query per table")
+        if not query.schema.names or len(set(query.schema.names)) != len(query.schema.names):
+            raise ValueError("each Arrow schema requires unique nonempty column names")
+        if query.schema.metadata is not None or any(field.metadata for field in query.schema):
+            raise ValueError("this example does not interpret Arrow schema/field metadata")
+        declarations.append(
+            tuple(
+                ef.SnapshotColumn(
+                    name=field.name,
+                    physical_type=str(field.type),
+                    logical_type=_logical_type(field.type),
+                    nullable=field.nullable,
+                )
+                for field in query.schema
+            )
+        )
+    # An opaque per-capture namespace avoids identifying two captures as the same
+    # artifact when their human labels are reused. It is not authentication.
+    namespace = uuid4().hex
+    refs = tuple(
+        ef.ArtifactRef(
+            schema_version="1.0",
+            uri=f"https://example.invalid/sqlite/{namespace}/{quote(query.name, safe='')}.arrow",
+            sha256="0" * 64,
+            format="arrow",
+            media_type="application/vnd.apache.arrow.file",
+            size_bytes=0,
+        )
+        for query in queries
+    )
+    declaration = ef.SnapshotManifest(
+        snapshot_id=snapshot_id,
+        domain_schema_revision=domain_schema_revision,
+        extraction_revision=extraction_revision,
+        provenance=provenance,
+        tables=tuple(
+            ef.SnapshotTable(name=query.name, artifact=ref, row_count=0, columns=columns)
+            for query, ref, columns in zip(queries, refs, declarations, strict=True)
+        ),
+    )
+    payloads: dict[str, bytes] = {}
+    tables = []
+    total_bytes = 0
+    # URI mode=ro refuses a missing database rather than creating an empty one.
+    connection = sqlite3.connect(
+        database.resolve().as_uri() + "?mode=ro",
+        uri=True,
+        isolation_level=None,
+        timeout=lock_timeout_seconds,
+    )
+    try:
+        connection.execute("BEGIN").close()
+        for query, table_declaration in zip(queries, declaration.tables, strict=True):
+            table = _read_query(
+                connection, query, max_rows_per_table=max_rows_per_table, batch_rows=batch_rows
+            )
+            sink = pa.BufferOutputStream()
+            with pa.ipc.new_file(sink, table.schema) as writer:
+                writer.write_table(table)
+            payload = sink.getvalue().to_pybytes()
+            total_bytes += len(payload)
+            if total_bytes > max_payload_bytes:
+                raise ef.ResourceLimitError(
+                    "SQLite capture exceeds its total serialized-byte budget"
+                )
+            ref = table_declaration.artifact.model_copy(
+                update={"sha256": hashlib.sha256(payload).hexdigest(), "size_bytes": len(payload)}
+            )
+            tables.append(
+                table_declaration.model_copy(update={"artifact": ref, "row_count": len(table)})
+            )
+            payloads[ref.uri] = payload
+    finally:
+        try:
+            if connection.in_transaction:
+                connection.rollback()
+        finally:
+            connection.close()
+    # No manifest/client is exposed until ALL tables succeeded and SQLite closed.
+    manifest = ef.SnapshotManifest.model_validate(
+        {**declaration.model_dump(), "tables": tuple(tables)}
+    )
+    return SQLiteSnapshotClient(manifest, payloads)

equiframe/examples/integrations/sqlite_snapshot_demo.py

@@ -0,0 +1,226 @@
+"""Run a synthetic SQLite -> pinned snapshot -> native DuckDB -> comparison gate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+from decimal import Decimal
+from pathlib import Path
+from typing import Literal
+
+import pyarrow as pa
+from sqlite_snapshot import SQLiteQuery, SQLiteSnapshotClient, capture_sqlite_snapshot
+
+import equiframe as ef
+
+
+def declarations() -> tuple[SQLiteQuery, ...]:
+    return (
+        SQLiteQuery(
+            "accounts",
+            "SELECT account_id, label FROM accounts ORDER BY account_id",
+            pa.schema(
+                [pa.field("account_id", pa.int64(), nullable=False), pa.field("label", pa.string())]
+            ),
+        ),
+        SQLiteQuery(
+            "entries",
+            "SELECT id, account_id, amount, currency FROM entries WHERE cohort = ? ORDER BY id",
+            pa.schema(
+                [
+                    pa.field("id", pa.string(), nullable=False),
+                    pa.field("account_id", pa.int64(), nullable=False),
+                    pa.field("amount", pa.decimal128(10, 2)),
+                    pa.field("currency", pa.string(), nullable=False),
+                ]
+            ),
+            parameters=("synthetic",),
+        ),
+    )
+
+
+def create_synthetic_database(database: Path, *, discrepant: bool) -> None:
+    """Write only this demo's new synthetic database; never reuse an existing one."""
+    with database.open("xb"):
+        pass
+    connection = sqlite3.connect(database)
+    try:
+        with connection:
+            connection.execute("CREATE TABLE accounts (account_id INTEGER, label TEXT)")
+            # TEXT affinity is intentional: SQLite REAL would lose decimal meaning.
+            connection.execute(
+                "CREATE TABLE entries "
+                "(id TEXT, account_id INTEGER, amount TEXT, currency TEXT, cohort TEXT)"
+            )
+            connection.executemany("INSERT INTO accounts VALUES (?, ?)", [(1, "Café"), (2, None)])
+            connection.executemany(
+                "INSERT INTO entries VALUES (?, ?, ?, ?, ?)",
+                [
+                    ("A", 1, "1.11" if discrepant else "1.10", "USD", "synthetic"),
+                    ("B", 1, None, "USD", "synthetic"),
+                    ("C", 2, "-0.01", "EUR", "synthetic"),
+                    ("excluded", 2, "99.99", "EUR", "other"),
+                ],
+            )
+    finally:
+        connection.close()
+
+
+def expected_output() -> pa.Table:
+    """Literal expectation authored separately; no query/source output is its oracle."""
+    return pa.table(
+        {
+            "id": pa.array(["A", "B", "C"], pa.string()),
+            "label": pa.array(["Café", "Café", None], pa.string()),
+            "amount": pa.array([Decimal("1.10"), None, Decimal("-0.01")], pa.decimal128(10, 2)),
+            "currency": pa.array(["USD", "USD", "EUR"], pa.string()),
+        }
+    )
+
+
+def contract_for(manifest: ef.SnapshotManifest) -> ef.Contract:
+    def schema(columns: tuple[ef.SnapshotColumn, ...]) -> ef.TableSchema:
+        declarations = tuple(
+            ef.ColumnSpec(
+                name=column.name,
+                logical_type=column.logical_type,
+                nullable=column.nullable,
+                null_policy=ef.NullPolicy(representations=("null",)),
+                numeric_policy=(
+                    ef.NumericPolicy(
+                        comparison="exact",
+                        unit=(
+                            "currency_units"
+                            if isinstance(column.logical_type, ef.DecimalType)
+                            else "identifier"
+                        ),
+                    )
+                    if isinstance(column.logical_type, (ef.DecimalType, ef.IntegerType))
+                    else None
+                ),
+            )
+            for column in columns
+        )
+        return ef.bag_contract(declarations).spec.contract.output.schema_spec
+
+    output_columns = (
+        manifest.tables[1].columns[0],
+        manifest.tables[0].columns[1],
+        *manifest.tables[1].columns[2:],
+    )
+    return ef.Contract(
+        schema_version="1.0",
+        contract_id="sqlite.synthetic.join",
+        revision=1,
+        family="joins",
+        inputs=tuple(
+            ef.TableSpec(name=table.name, schema_spec=schema(table.columns))
+            for table in manifest.tables
+        ),
+        output=ef.TableSpec(name="result", schema_spec=schema(output_columns)),
+        preconditions=(),
+        rules=(),
+    )
+
+
+class CorruptingTransport:
+    """Synthetic functional failure: change one byte without changing its manifest."""
+
+    def __init__(self, client: SQLiteSnapshotClient) -> None:
+        self.client = client
+
+    def get_snapshot(self, snapshot_id: str) -> ef.SnapshotManifest:
+        return self.client.get_snapshot(snapshot_id)
+
+    def download_artifact(self, ref: ef.ArtifactRef, destination: Path) -> None:
+        self.client.download_artifact(ref, destination)
+        with destination.open("r+b") as stream:
+            first = stream.read(1)
+            stream.seek(0)
+            stream.write(bytes([first[0] ^ 1]))
+
+
+def run_demo(output: Path, case: Literal["good", "different", "corrupt"]) -> int:
+    """Exit 0 only for an equivalent capture; 1 difference, 3 failed/incomplete."""
+    if case not in ("good", "different", "corrupt"):
+        raise ValueError("unknown demonstration case")
+    output.mkdir(parents=True, exist_ok=False)
+    database = output / "synthetic.sqlite"
+    create_synthetic_database(database, discrepant=case == "different")
+    client = capture_sqlite_snapshot(
+        database,
+        snapshot_id="synthetic.sqlite.001",
+        queries=declarations(),
+        domain_schema_revision="synthetic.accounts.1",
+        extraction_revision="sqlite.exact-text-decimal.1",
+        provenance=(
+            "Synthetic accounts and entries; no real service or DW acceptance",
+            f"SQLite {sqlite3.sqlite_version}; one owned native read transaction",
+        ),
+    )
+    source = ef.SnapshotDataSource(
+        client.manifest.snapshot_id,
+        client=CorruptingTransport(client) if case == "corrupt" else client,
+        expected_domain_schema_revision="synthetic.accounts.1",
+    )
+    registry = ef.TransformationRegistry()
+    registry.register(
+        ef.DuckDBTransformation(
+            "join",
+            "SELECT e.id, a.label, e.amount, e.currency "
+            "FROM entries e LEFT JOIN accounts a USING (account_id)",
+            "sqlite-demo.join.1",
+        )
+    )
+    contract = contract_for(source.manifest)
+    execution = ef.execute(
+        ef.ExecutionPlan(contract=contract, engine="duckdb", transformation_id="join"),
+        source=source,
+        registry=registry,
+        work_dir=output / "execution",
+    )
+    receipt = {
+        "case": case,
+        "snapshot_id": source.snapshot_id,
+        "snapshot_manifest_sha256": ef.snapshot_fingerprint(source.manifest),
+        "execution_state": execution.state,
+        "comparison_state": "not_evaluable",
+    }
+    code = 3
+    if execution.state == "completed":
+        comparison = ef.compare(
+            execution,
+            ef.ArrowInput(expected_output()),
+            contract=contract,
+            reference="expected",
+            work_dir=output / "comparison",
+        )
+        # Persist a redacted report BEFORE enforcing the gate.
+        ef.write_comparison_report(comparison, output / "report")
+        receipt["comparison_state"] = comparison.state
+        try:
+            ef.assert_equivalent(comparison)
+        except ef.EquivalenceDifferentError:
+            code = 1
+        except ef.EquivalenceNotEvaluableError:
+            code = 3
+        else:
+            code = 0
+    with (output / "receipt.json").open("x", encoding="utf-8") as stream:
+        json.dump({**receipt, "exit_code": code}, stream, indent=2)
+        stream.write("\n")
+    print(f"execution={execution.state}; comparison={receipt['comparison_state']}; exit={code}")
+    return code
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--case", choices=("good", "different", "corrupt"), default="good")
+    args = parser.parse_args()
+    raise SystemExit(run_demo(args.output, args.case))
+
+
+if __name__ == "__main__":
+    main()