equiframe 0.12.4__py3-none-any.whl

equiframe/examples/integrations/universal_first_use.py

@@ -0,0 +1,296 @@
+"""One exact contract, six input variants and three independent native engines.
+
+All data and the injected API response are synthetic. No remote service, paid
+resource, credentials, benchmark or GitHub Actions are required.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import io
+import json
+import sqlite3
+from contextlib import closing
+from decimal import Decimal
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+import equiframe as ef
+
+INT = ef.IntegerType(bits=64, signed=True)
+MONEY = ef.DecimalType(precision=12, scale=2)
+
+
+def column(
+    name: str, dtype: ef.LogicalType, *, nullable: bool = True, unit: str | None = None
+) -> ef.ColumnSpec:
+    return ef.ColumnSpec(
+        name=name,
+        logical_type=dtype,
+        nullable=nullable,
+        null_policy=ef.NullPolicy(representations=("null",)),
+        numeric_policy=ef.NumericPolicy(comparison="exact", unit=unit)
+        if unit is not None
+        else None,
+    )
+
+
+def declarations() -> tuple[ef.Contract, ef.RelationalPlan, ef.IngestionSpec]:
+    columns = (
+        column("id", ef.StringType(), nullable=False),
+        column("sequence", INT, nullable=False, unit="sequence"),
+        column("amount", MONEY, unit="USD"),
+    )
+    outputs = (
+        *columns,
+        column("position", INT, nullable=False, unit="rows"),
+        column("previous_amount", MONEY, unit="USD"),
+    )
+
+    def table(name: str, fields: tuple[ef.ColumnSpec, ...]) -> ef.TableSpec:
+        return ef.TableSpec(
+            name=name,
+            schema_spec=ef.TableSchema(
+                columns=fields, keys=(), multiplicity="preserve", order=ef.IgnoredOrder()
+            ),
+        )
+
+    contract = ef.Contract(
+        schema_version="1.0",
+        contract_id="universal.synthetic",
+        revision=1,
+        family="nulls_types",
+        inputs=(table("input", columns),),
+        output=table("output", outputs),
+        preconditions=(),
+        rules=(),
+    )
+    plan = ef.RelationalPlan(
+        nodes=(
+            ef.Scan(node_id="s", table="input"),
+            ef.Window(
+                node_id="w",
+                source="s",
+                partition_by=("id",),
+                by=(ef.SortKey(column="sequence", direction="ascending", nulls="last"),),
+                measures=(
+                    ef.WindowMeasure(
+                        name="position", operation="row_number", logical_type=INT, unit="rows"
+                    ),
+                    ef.WindowMeasure(
+                        name="previous_amount", operation="lag", column="amount", logical_type=MONEY
+                    ),
+                ),
+            ),
+        ),
+        output="w",
+    )
+    spec = ef.IngestionSpec(
+        format="jsonl",
+        fields=tuple(ef.FieldMapping(column=field, path=(field.name,)) for field in columns),
+    )
+    return contract, plan, spec
+
+
+def source_table() -> pa.Table:
+    return pa.table(
+        {
+            "id": ["001", "001", "002", "001"],
+            "sequence": [2, 1, 1, 3],
+            "amount": pa.array(
+                [None, Decimal("12.30"), Decimal("-0.01"), Decimal("2.00")],
+                type=pa.decimal128(12, 2),
+            ),
+        }
+    )
+
+
+def expected_output() -> pa.Table:
+    """Literal expected authored independently of source loading/native execution."""
+    return pa.table(
+        {
+            "id": ["001", "001", "001", "002"],
+            "sequence": [1, 2, 3, 1],
+            "amount": pa.array(
+                [Decimal("12.30"), None, Decimal("2.00"), Decimal("-0.01")],
+                type=pa.decimal128(12, 2),
+            ),
+            "position": [1, 2, 3, 1],
+            "previous_amount": pa.array(
+                [None, Decimal("12.30"), None, None], type=pa.decimal128(12, 2)
+            ),
+        }
+    )
+
+
+def inputs(root: Path, spec: ef.IngestionSpec, case: str) -> dict[str, ef.DataSource]:
+    root.mkdir(exist_ok=False)
+    csv = root / "input.csv"
+    csv.write_text(
+        "id,sequence,amount\n001,2,NULL\n001,1,12.30\n002,1,-0.01\n001,3,2.00\n", encoding="utf-8"
+    )
+    csv_spec = ef.IngestionSpec(
+        format="csv",
+        fields=tuple(
+            mapping.model_copy(update={"null_tokens": ("NULL",)})
+            if mapping.column.nullable
+            else mapping
+            for mapping in spec.fields
+        ),
+    )
+    database = root / "input.sqlite"
+    with sqlite3.connect(database) as db:
+        db.execute("CREATE TABLE entries (id TEXT, sequence INTEGER, amount TEXT, cohort TEXT)")
+        db.executemany(
+            "INSERT INTO entries VALUES (?, ?, ?, ?)",
+            [
+                ("001", 2, None, "demo"),
+                ("001", 1, "12.31" if case == "different" else "12.30", "demo"),
+                ("002", 1, "-0.01", "demo"),
+                ("001", 3, "2.00", "demo"),
+                ("excluded", 1, "99.99", "other"),
+            ],
+        )
+    records = [
+        {"id": "001", "sequence": 2, "amount": None},
+        {"id": "001", "sequence": 1, "amount": "12.30"},
+        {"id": "002", "sequence": 1, "amount": "-0.01"},
+        {"id": "001", "sequence": 3, "amount": "2.00"},
+    ]
+    json_path = root / "input.json"
+    json_path.write_text(json.dumps({"records": records}), encoding="utf-8")
+    json_spec = ef.IngestionSpec(format="json", records_path=("records", "*"), fields=spec.fields)
+    response = ("\n".join(json.dumps(row) for row in records) + "\n").encode()
+    response_sha = "0" * 64 if case == "corrupt" else hashlib.sha256(response).hexdigest()
+    raw = source_table()
+    parquet = root / "input.parquet"
+    pq.write_table(raw, parquet)
+    ref = ef.LocalArtifactStore(root / "artifacts").put(
+        parquet, media_type="application/vnd.apache.parquet"
+    )
+    batch_input = ef.RecordBatchInput(
+        lambda: closing(
+            pa.RecordBatchReader.from_batches(raw.schema, raw.to_batches(max_chunksize=2))
+        ),
+        raw.schema,
+    )
+    items = {
+        "csv": ef.FileInput(csv, csv_spec, batch_rows=2),
+        "sqlite": ef.SQLiteInput(
+            database,
+            "SELECT id, sequence, amount FROM entries WHERE cohort = ?",
+            spec,
+            ("demo",),
+            batch_rows=2,
+        ),
+        "json": ef.FileInput(json_path, json_spec),
+        "api_response": ef.StreamInput(lambda: closing(io.BytesIO(response)), spec, response_sha),
+        "parquet": ef.ArtifactInput(ref),
+        "arrow_batches": batch_input,
+    }
+    return {
+        name: ef.CompositeDataSource(
+            {"input": (ef.LocalDataSource({"provider_table": item}), "provider_table")}
+        )
+        for name, item in items.items()
+    }
+
+
+def run_demo(output: Path, case: str = "good") -> int:
+    if case not in ("good", "different", "corrupt"):
+        raise ValueError("unknown synthetic case")
+    output = output.resolve()
+    output.mkdir(parents=True, exist_ok=False)
+    contract, plan, spec = declarations()
+    sources = inputs(output / "inputs", spec, case)
+    receipts: list[dict[str, object]] = []
+    exit_code = 0
+    for engine in ("pandas", "polars", "duckdb"):
+        compiled = ef.native_compiler_registry().resolve(engine).compile(plan, contract)
+        registry = ef.TransformationRegistry()
+        registry.register(compiled.transformation)
+        for name, source in sources.items():
+            directory = output / engine / name
+            execution = ef.execute(
+                ef.ExecutionPlan(
+                    contract=contract,
+                    engine=engine,
+                    transformation_id=compiled.transformation.transformation_id,
+                    options=ef.ExecutionOptions(
+                        max_input_rows=100,
+                        max_output_rows=100,
+                        max_transport_bytes=1_000_000,
+                        timeout_seconds=30,
+                    ),
+                ),
+                source=source,
+                registry=registry,
+                work_dir=directory / "execution",
+            )
+            receipt: dict[str, object] = {
+                "engine": engine,
+                "source": name,
+                "execution_state": execution.state,
+                "comparison_state": None,
+                "problem": execution.problem.model_dump(mode="json") if execution.problem else None,
+            }
+            if execution.state != "completed":
+                exit_code = 3
+            else:
+                comparison = ef.compare(
+                    execution,
+                    ef.ArrowInput(expected_output()),
+                    contract=contract,
+                    reference="expected",
+                    work_dir=directory / "comparison",
+                    policy=ef.ComparisonPolicy(
+                        index="ignore",
+                        justification=(
+                            "This declared relational bag has no native index identity."
+                        ),
+                    ),
+                )
+                ef.write_comparison_report(comparison, directory / "report")
+                receipt["comparison_state"] = comparison.state
+                receipt["comparison_complete"] = comparison.complete
+                try:
+                    ef.assert_equivalent(comparison)
+                except ef.EquivalenceDifferentError:
+                    exit_code = max(exit_code, 1)
+                except ef.EquivalenceNotEvaluableError:
+                    exit_code = 3
+            receipts.append(receipt)
+            print(
+                f"{engine}/{name}: execution={execution.state}; "
+                f"comparison={receipt['comparison_state']}"
+            )
+    (output / "receipt.json").write_text(
+        json.dumps(
+            {
+                "equiframe": ef.__version__,
+                "synthetic": True,
+                "case": case,
+                "exit_code": exit_code,
+                "runs": receipts,
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return exit_code
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--case", choices=("good", "different", "corrupt"), default="good")
+    args = parser.parse_args()
+    raise SystemExit(run_demo(args.output, args.case))
+
+
+if __name__ == "__main__":
+    main()

equiframe/examples/journeys/README.md

@@ -0,0 +1,31 @@
+# Eight complete synthetic journeys
+
+Read the [packaged journey guide](../../docs/journeys.md) for installation,
+independent expected values, acceptance criteria, artifacts, privacy and limits.
+Everything is inline synthetic data; no credentials or network data sources.
+
+After installing an identified private wheel and the exact dependencies in
+`requirements.lock`, copy this entire directory outside the checkout:
+
+```bash
+python journeys/a_pandas_refactor.py --case good --output evidence-a
+python journeys/verify_journeys.py --output all-journey-evidence
+```
+
+The first command is an equivalence gate. The second is an **acceptance harness**:
+it expects both good cases and intentional failures, and returns 0 only when all
+31 scenario exits match their declared meaning. It does not approve every input.
+
+When these examples are installed inside the candidate wheel, the isolated route
+requires no copied scripts:
+
+```bash
+python -I -m equiframe.examples.journeys.a_pandas_refactor --case good --output evidence-a
+python -I -m equiframe.examples.journeys.verify_journeys --output all-journey-evidence
+```
+
+Use a fresh output directory every time. `private/` or `raw/` contains raw
+captures. Only the `reports/` or `public_reports/` subtree is data-minimized.
+A portable bundle, `summary.json`, and `environment.json` are not redacted sharing
+packages. Local Parquet output is local file publication, never package/service
+publication. Keep raw captures and local evidence out of public commits.

equiframe/examples/journeys/__init__.py

@@ -0,0 +1 @@
+"""Runnable synthetic product journeys; not an additional public core API."""

equiframe/examples/journeys/a_pandas_refactor.py

@@ -0,0 +1,88 @@
+"""A: pandas refactor, exact money, composite keys and visible duplicate failures."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+import equiframe as ef
+
+if TYPE_CHECKING or __package__:
+    from .common import cli, column, comparison_record, new_output, write_summary
+else:
+    from common import cli, column, comparison_record, new_output, write_summary
+
+SCENARIOS = ("good", "different", "not_evaluable", "duplicate")
+
+
+def materialize(scenario: str) -> tuple[pd.DataFrame, pd.DataFrame]:
+    import pandas as pd
+
+    source = pd.DataFrame(
+        {
+            "order_id": pd.Series(["001", "001", "002"], dtype="string[pyarrow]"),
+            "line": [1, 2, 1],
+            "quantity": [2, 1, 3],
+            "unit_cents": [125, 50, 100],
+        }
+    )
+    # Independent literal, not computed using old or optimized transformation.
+    expected = pd.DataFrame(
+        {
+            "order_id": pd.Series(["001", "001", "002"], dtype="string[pyarrow]"),
+            "line": [1, 2, 1],
+            "total_cents": [250, 50, 300],
+        }
+    )
+    old = source[["order_id", "line"]].copy()
+    old["total_cents"] = [
+        int(quantity) * int(price)
+        for quantity, price in zip(source["quantity"], source["unit_cents"], strict=True)
+    ]
+    pd.testing.assert_frame_equal(old, expected)
+    candidate = source[["order_id", "line"]].assign(
+        total_cents=source["quantity"] * source["unit_cents"]
+    )
+    if scenario == "different":
+        candidate["total_cents"] += candidate["order_id"].eq("002").astype("int64")
+    if scenario == "duplicate":
+        candidate = pd.concat([candidate, candidate.iloc[[0]]], ignore_index=True)
+    else:
+        candidate = candidate.iloc[[2, 0, 1]].reset_index(drop=True)
+    return candidate, expected
+
+
+def declaration() -> ef.DiffContract:
+    return ef.keyed_contract(
+        (
+            column("order_id", ef.StringType()),
+            column("line", ef.IntegerType(bits=64, signed=True), unit="line_identifier"),
+            column("total_cents", ef.IntegerType(bits=64, signed=True), unit="USD_cent"),
+        ),
+        keys=["order_id", "line"],
+        contract_id="journey.A.refactor",
+    )
+
+
+def run(output: Path, scenario: str = "good") -> dict[str, object]:
+    private = new_output(output, scenario, SCENARIOS)
+    candidate, expected = materialize(scenario)
+    result = ef.diff(
+        candidate,
+        expected,
+        contract=declaration(),
+        options=ef.ComparisonOptions(max_rows=1 if scenario == "not_evaluable" else 100),
+        work_dir=private,
+    )
+    record = comparison_record(result, output, "refactor")
+    return write_summary(
+        output,
+        {"journey": "A", "synthetic": True, "scenario": scenario, **record},
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(cli(run, __doc__ or "A", SCENARIOS))

equiframe/examples/journeys/b_native_migration.py

@@ -0,0 +1,181 @@
+"""B: real pandas and DuckDB compilation, one input capture, independent expected."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pass
+
+import equiframe as ef
+
+if TYPE_CHECKING or __package__:
+    from .common import cli, column, comparison_record, new_output, write_summary
+else:
+    from common import cli, column, comparison_record, new_output, write_summary
+
+
+class CountedSource:
+    """Observe actual source resolution without changing the production source port."""
+
+    def __init__(self, source: ef.LocalDataSource) -> None:
+        self.source = source
+        self.calls = 0
+
+    @property
+    def table_names(self) -> tuple[str, ...]:
+        return self.source.table_names
+
+    def resolve(self, name: str, *, context: ef.RunContext) -> ef.LoadedTable:
+        self.calls += 1
+        return self.source.resolve(name, context=context)
+
+
+def configuration(scenario: str) -> ef.MigrationSpec:
+    currency = column("currency", ef.StringType())
+    amount = column(
+        "amount", ef.DecimalType(precision=18, scale=2), nullable=True, unit="currency_amount"
+    )
+    total = column(
+        "total", ef.DecimalType(precision=20, scale=2), nullable=True, unit="currency_amount"
+    )
+    output_schema = ef.keyed_contract(
+        (currency, total), keys=["currency"]
+    ).spec.contract.output.schema_spec
+    contract = ef.Contract(
+        schema_version="1.0",
+        contract_id="journey.B.native-totals",
+        revision=1,
+        family="aggregations",
+        inputs=(
+            ef.TableSpec(
+                name="sales",
+                schema_spec=ef.bag_contract((currency, amount)).spec.contract.output.schema_spec,
+            ),
+        ),
+        output=ef.TableSpec(name="result", schema_spec=output_schema),
+        preconditions=(),
+        rules=(
+            ef.RuleSpec(
+                rule_id="schema",
+                revision=1,
+                severity="error",
+                required=True,
+                specification=ef.SchemaCheck(table="result"),
+            ),
+        ),
+    )
+    return ef.MigrationSpec(
+        contract=contract,
+        source_engine="pandas",
+        target_engine="duckdb",
+        ir=ef.RelationalPlan(
+            nodes=(
+                ef.Scan(node_id="source", table="sales"),
+                ef.Aggregate(
+                    node_id="totals",
+                    source="source",
+                    group_by=("currency",),
+                    measures=(
+                        ef.Measure(
+                            name="total",
+                            operation="sum",
+                            column="amount",
+                            logical_type=ef.DecimalType(precision=20, scale=2),
+                            unit="currency_amount",
+                        ),
+                    ),
+                    null_groups="retain",
+                    empty_sum="zero" if scenario == "different" else "null",
+                ),
+            ),
+            output="totals",
+        ),
+        comparison_policy=ef.ComparisonPolicy(
+            index="ignore", justification="Native generated RangeIndex is not row identity."
+        ),
+        comparison_options=ef.ComparisonOptions(max_rows=1 if scenario == "not_evaluable" else 100),
+        execution_options=ef.ExecutionOptions(
+            duckdb_threads=1, duckdb_timezone="UTC", max_input_rows=100, max_output_rows=100
+        ),
+    )
+
+
+def run(output: Path, scenario: str = "good") -> dict[str, object]:
+    import pyarrow as pa
+
+    private = new_output(output, scenario)
+    source = CountedSource(
+        ef.LocalDataSource(
+            {
+                "sales": ef.ArrowInput(
+                    pa.table(
+                        {
+                            "currency": ["USD", "USD", "EUR", "EUR"],
+                            "amount": pa.array(
+                                [Decimal("1.10"), Decimal("2.20"), None, None],
+                                type=pa.decimal128(18, 2),
+                            ),
+                        }
+                    )
+                )
+            }
+        )
+    )
+    # This literal is independent of both native implementations and their IR.
+    expected = pa.table(
+        {
+            "currency": ["USD", "EUR"],
+            "total": pa.array([Decimal("3.30"), None], type=pa.decimal128(20, 2)),
+        }
+    )
+    spec = configuration(scenario)
+    checks = ef.check_migration(spec)
+    if source.calls != 0 or not all(check.supported for check in checks):
+        raise RuntimeError("Preflight unexpectedly read input or refused the documented plan")
+    result = ef.migrate(spec, source=source, expected=ef.ArrowInput(expected), work_dir=private)
+    if ef.read_migration(private / result.migration_id) != result:
+        raise RuntimeError("Migration readback failed")
+    comparisons = {}
+    for label, comparison in (
+        ("peers", result.comparison),
+        ("pandas-expected", result.verification_source),
+        ("duckdb-expected", result.verification_target),
+    ):
+        if comparison is not None:
+            comparisons[label] = comparison_record(comparison, output, label)
+    executions = [
+        pipeline.execution
+        for pipeline in (result.source, result.target)
+        if pipeline is not None and pipeline.execution is not None
+    ]
+    same_input = (
+        len(executions) == 2
+        and executions[0].inputs[0].data.sha256 == executions[1].inputs[0].data.sha256
+    )
+    accepted = result.exit_code == 0 and source.calls == 1 and same_input
+    return write_summary(
+        output,
+        {
+            "journey": "B",
+            "synthetic": True,
+            "scenario": scenario,
+            "preflight": [check.model_dump(mode="json") for check in checks],
+            "source_resolutions": source.calls,
+            "same_input_sha256": same_input,
+            "native_engines": [execution.plan.engine for execution in executions],
+            "execution_states": [execution.state for execution in executions],
+            "state": result.state,
+            "verification": result.verification,
+            "complete": result.complete,
+            "accepted": accepted,
+            "exit_code": result.exit_code if not (result.exit_code == 0 and not accepted) else 3,
+            "comparisons": comparisons,
+        },
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(cli(run, __doc__ or "B"))