equiframe 0.12.4__py3-none-any.whl

equiframe/migration/backends/polars_decimal.py

@@ -0,0 +1,231 @@
+"""Exact native Decimal coefficient arithmetic with explicit overflow checks."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from equiframe.errors import CapabilityError
+
+if TYPE_CHECKING:
+    import polars as pl
+
+    from equiframe.execution import RunContext
+
+
+def divide(
+    frame: pl.DataFrame,
+    left: pl.Expr,
+    right: pl.Expr,
+    *,
+    left_scale: int,
+    right_scale: int,
+    precision: int,
+    scale: int,
+    rounding: str,
+    prefix: str,
+    context: RunContext,
+) -> tuple[pl.DataFrame, pl.Expr]:
+    """Materialize native quotient/remainder steps; never use floats or row callbacks.
+
+    Each operand coefficient fits 38 decimal digits. Modular doubling computes
+    the next digit without multiplying a potentially 38-digit remainder by ten.
+    Decimal(38,0) carries exact coefficient arithmetic: Polars 1.30 panics on
+    Int128 column/column addition and subtraction. Integer floor/modulo remain
+    native Int128 operations; every resulting coefficient still fits 38 digits.
+    Only finite native boolean reductions cross back to the coordinator to reject
+    invalid arithmetic; neither operands nor answers are computed in Python.
+    """
+    import polars as pl
+
+    def col(name: str) -> pl.Expr:
+        return pl.col(prefix + name)
+
+    def named(value: pl.Expr, name: str) -> pl.Expr:
+        return value.alias(prefix + name)
+
+    wide = pl.Decimal(38, 0)
+    zero = pl.lit(0, dtype=wide)
+
+    def constant(value: int) -> pl.Expr:
+        return pl.lit(str(value)).cast(wide)
+
+    def floor(left: pl.Expr, right: pl.Expr) -> pl.Expr:
+        # The native direct Int128 -> Decimal cast rejects valid large values
+        # in Polars 1.30. Its native integer text and Decimal parsers are exact.
+        return (left.cast(pl.Int128) // right.cast(pl.Int128)).cast(pl.String).cast(wide)
+
+    def modulo(left: pl.Expr, right: pl.Expr) -> pl.Expr:
+        return (left.cast(pl.Int128) % right.cast(pl.Int128)).cast(pl.String).cast(wide)
+
+    frame = frame.with_columns(
+        named(left.is_not_null() & right.is_not_null(), "active"),
+        named(left.cast(pl.String).str.replace_all(r"\.", "").cast(wide), "a"),
+        named(right.cast(pl.String).str.replace_all(r"\.", "").cast(wide), "b"),
+    )
+
+    def reject(condition: pl.Expr, reason: str) -> None:
+        context.check()
+        if frame.select((col("active") & condition).any()).item():
+            raise CapabilityError("Polars Decimal division: " + reason)
+
+    reject(col("b") == 0, "zero denominator")
+    frame = frame.with_columns(
+        named((col("a") < 0).xor(col("b") < 0), "negative"),
+        named(pl.when(col("active")).then(col("a").abs()).otherwise(zero), "a"),
+        named(
+            pl.when(col("active")).then(col("b").abs()).otherwise(constant(1)),
+            "b",
+        ),
+    )
+    frame = frame.with_columns(
+        named(floor(col("a"), col("b")), "q"), named(modulo(col("a"), col("b")), "r")
+    )
+    exponent = scale + right_scale - left_scale
+    limit = 10**precision - 1
+    if exponent >= 0:
+        reject(col("q") > constant(limit), "output precision exceeded")
+        for _ in range(exponent):
+            reject(col("q") > constant(limit // 10), "output precision exceeded")
+            remainder, denominator = col("r"), col("b")
+            carry = remainder >= denominator - remainder
+            frame = frame.with_columns(
+                named(carry.cast(pl.Int64).cast(wide), "c2"),
+                named(
+                    remainder
+                    - pl.when(carry).then(denominator - remainder).otherwise(zero - remainder),
+                    "r2",
+                ),
+            )
+            for first, second in ((2, 4), (4, 8)):
+                remainder = col("r" + str(first))
+                carry = remainder >= denominator - remainder
+                frame = frame.with_columns(
+                    named(
+                        col("c" + str(first)) * constant(2) + carry.cast(pl.Int64).cast(wide),
+                        "c" + str(second),
+                    ),
+                    named(
+                        remainder
+                        - pl.when(carry).then(denominator - remainder).otherwise(zero - remainder),
+                        "r" + str(second),
+                    ),
+                )
+            r8, r2 = col("r8"), col("r2")
+            carry = r8 >= denominator - r2
+            frame = frame.with_columns(
+                named(
+                    col("q") * constant(10)
+                    + col("c8")
+                    + col("c2")
+                    + carry.cast(pl.Int64).cast(wide),
+                    "q",
+                ),
+                named(r8 - pl.when(carry).then(denominator - r2).otherwise(zero - r2), "r"),
+            )
+        remainder = col("r")
+        exact = remainder == 0
+        above = remainder > col("b") - remainder
+        tie = remainder == col("b") - remainder
+    else:
+        factor = pl.lit(10 ** (-exponent), dtype=pl.Int128)
+        frame = frame.with_columns(
+            named(modulo(col("q"), factor), "dropped"), named(floor(col("q"), factor), "q")
+        )
+        dropped, remainder = col("dropped"), col("r")
+        half = constant(5 * 10 ** (-exponent - 1))
+        exact = (dropped == 0) & (remainder == 0)
+        above = (dropped > half) | ((dropped == half) & (remainder > 0))
+        tie = (dropped == half) & (remainder == 0)
+    if rounding == "exact":
+        reject(~exact, "declared exact scale is insufficient")
+        increment = pl.lit(False)
+    elif rounding == "half_even":
+        increment = above | (tie & (modulo(col("q"), constant(2)) == 1))
+    elif rounding == "half_away_from_zero":
+        increment = above | tie
+    else:
+        raise CapabilityError("unsupported Decimal division rounding")
+    reject(
+        (col("q") > constant(limit)) | ((col("q") == constant(limit)) & increment),
+        "output precision exceeded",
+    )
+    frame = frame.with_columns(named(col("q") + increment.cast(pl.Int64).cast(wide), "q"))
+    text = col("q").cast(pl.String)
+    if scale:
+        text = text.str.pad_start(scale + 1, "0")
+        text = pl.concat_str(
+            text.str.slice(0, text.str.len_chars() - scale), pl.lit("."), text.str.slice(-scale)
+        )
+    text = pl.concat_str(
+        pl.when(col("negative") & (col("q") != 0)).then(pl.lit("-")).otherwise(pl.lit("")),
+        text,
+    )
+    output = prefix + "result"
+    frame = frame.with_columns(
+        pl.when(col("active"))
+        .then(text.cast(pl.Decimal(precision, scale), strict=True))
+        .otherwise(None)
+        .alias(output)
+    )
+    return frame, pl.col(output)
+
+
+def multiply(
+    frame: pl.DataFrame,
+    left: pl.Expr,
+    right: pl.Expr,
+    *,
+    precision: int,
+    scale: int,
+    prefix: str,
+    context: RunContext,
+) -> tuple[pl.DataFrame, pl.Expr]:
+    """Bound native coefficient multiplication before the product can overflow."""
+    import polars as pl
+
+    def col(name: str) -> pl.Expr:
+        return pl.col(prefix + name)
+
+    def named(value: pl.Expr, name: str) -> pl.Expr:
+        return value.alias(prefix + name)
+
+    wide = pl.Decimal(38, 0)
+    zero, one = pl.lit(0, dtype=wide), pl.lit(1, dtype=wide)
+    limit = pl.lit(str(10**precision - 1)).cast(wide)
+    context.check()
+    frame = frame.with_columns(
+        named(left.is_not_null() & right.is_not_null(), "active"),
+        named(left.cast(pl.String).str.replace_all(r"\.", "").cast(wide), "a"),
+        named(right.cast(pl.String).str.replace_all(r"\.", "").cast(wide), "b"),
+    ).with_columns(
+        named((col("a") < 0).xor(col("b") < 0), "negative"),
+        named(pl.when(col("active")).then(col("a").abs()).otherwise(zero), "a"),
+        named(pl.when(col("active")).then(col("b").abs()).otherwise(zero), "b"),
+    )
+    denominator = pl.when(col("b") != 0).then(col("b")).otherwise(one)
+    bound = (limit.cast(pl.Int128) // denominator.cast(pl.Int128)).cast(pl.String).cast(wide)
+    too_large = col("active") & (col("b") != 0) & (col("a") > bound)
+    if frame.select(too_large.any()).item():
+        raise CapabilityError("Polars Decimal multiply: output precision exceeded")
+    context.check()
+    # Both magnitudes fit the declared coefficient bound before this product.
+    # No intermediate is scaled up to the final scale or widened through float.
+    frame = frame.with_columns(named(col("a") * col("b"), "value"))
+    text = col("value").cast(pl.String)
+    if scale:
+        text = text.str.pad_start(scale + 1, "0")
+        text = pl.concat_str(
+            text.str.slice(0, text.str.len_chars() - scale), pl.lit("."), text.str.slice(-scale)
+        )
+    text = pl.concat_str(
+        pl.when(col("negative") & (col("value") != 0)).then(pl.lit("-")).otherwise(pl.lit("")),
+        text,
+    )
+    output = prefix + "result"
+    frame = frame.with_columns(
+        pl.when(col("active"))
+        .then(text.cast(pl.Decimal(precision, scale), strict=True))
+        .otherwise(None)
+        .alias(output)
+    )
+    return frame, pl.col(output)

equiframe/migration/io.py

@@ -0,0 +1,198 @@
+"""Read and verify migration evidence without executing ETL, compilers or plugins."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from equiframe._physical import physical_size, verify_table_metadata
+from equiframe._report_format_v1 import (
+    migration_html_v1,
+    migration_legacy_summary_v1,
+    migration_summary_v1,
+)
+from equiframe.comparison import iter_differences, read_comparison
+from equiframe.errors import ArtifactIntegrityError
+from equiframe.evidence import (
+    describe_file,
+    load_json,
+    parse_json,
+    read_pipeline,
+    verify_events,
+    verify_files,
+)
+from equiframe.local import local_path
+from equiframe.types import ArtifactRef
+
+from .runner_types import MigrationResult, MigrationSpec
+
+
+def _legacy_summary(result: MigrationResult) -> dict[str, object]:
+    return migration_legacy_summary_v1(result)
+
+
+def migration_summary(result: MigrationResult) -> dict[str, object]:
+    return migration_summary_v1(result)
+
+
+def _render_report(result: MigrationResult, *, legacy: bool = False) -> str:
+    return migration_html_v1(result, legacy=legacy)
+
+
+def render_migration_report(result: MigrationResult) -> str:
+    return _render_report(result)
+
+
+def read_migration(path: Path, *, verify: bool = True) -> MigrationResult:
+    """Check a closed bundle; explicit False reads metadata without byte verification."""
+    if type(verify) is not bool:
+        raise TypeError("verify must be a bool; only explicit False skips integrity checks")
+    path = path / "migration.json" if path.is_dir() else path
+    result = load_json(MigrationResult, path)
+    if not verify:
+        return result
+    directory = path.parent.resolve()
+    listed = {i.path: i for i in result.evidence}
+    verify_files(directory, result.evidence)
+
+    def bind(ref: ArtifactRef) -> None:
+        ref_path = local_path(ref).resolve()
+        if not ref_path.is_relative_to(directory):
+            raise ArtifactIntegrityError("migration artifact escapes its operation")
+        item = listed.get(ref_path.relative_to(directory).as_posix())
+        if item is None or (item.sha256, item.size_bytes) != (ref.sha256, ref.size_bytes):
+            raise ArtifactIntegrityError("migration artifact contradicts its inventory")
+
+    actual = {
+        p.relative_to(directory).as_posix()
+        for p in directory.rglob("*")
+        if p.is_file() and p != path.resolve()
+    }
+    if actual != set(listed):
+        raise ArtifactIntegrityError("migration files contradict its closed inventory")
+    for name in ("events.jsonl", "spec.json", "report.json", "report.html"):
+        if name not in listed:
+            raise ArtifactIntegrityError("required migration evidence is missing")
+    if load_json(MigrationSpec, directory / "spec.json") != result.spec:
+        raise ArtifactIntegrityError("migration configuration contradicts the manifest")
+    events = verify_events(directory / "events.jsonl", operation_id=result.migration_id)
+    if (
+        not events
+        or len(events) != result.event_count
+        or events[-1].sha256 != result.event_head_sha256
+        or (events[0].stage, events[0].state) != ("migration", "started")
+        or (events[-1].stage, events[-1].state) != ("migration", "finished")
+    ):
+        raise ArtifactIntegrityError("migration journal contradicts its manifest")
+    terminal = tuple(
+        (e.stage, e.state, e.code)
+        for e in events
+        if e.stage != "migration" and e.state != "started"
+    )
+    if terminal != tuple((s.stage, s.state, s.code) for s in result.stages):
+        raise ArtifactIntegrityError("migration stage outcomes contradict its journal")
+    refs = []
+    for table in result.inputs:
+        bind(table.data)
+        if table.index:
+            bind(table.index)
+        verify_table_metadata(table)
+        refs.append(table.data)
+        if table.index:
+            refs.append(table.index)
+    if result.expected:
+        bind(result.expected)
+        refs.append(result.expected)
+    if result.stages[1].state == "completed":
+        if "capture.json" not in listed:
+            raise ArtifactIntegrityError("complete capture receipt is missing")
+        captured = parse_json((directory / "capture.json").read_bytes())
+        if (
+            not isinstance(captured, dict)
+            or set(captured)
+            != {
+                "schema_version",
+                "inputs",
+                "expected",
+                "rows",
+                "arrow_buffer_bytes",
+            }
+            or (
+                captured.get("schema_version") != "1.0"
+                or captured.get("inputs") != [t.model_dump(mode="json") for t in result.inputs]
+                or captured.get("expected")
+                != (result.expected.model_dump(mode="json") if result.expected else None)
+            )
+        ):
+            raise ArtifactIntegrityError("capture receipt contradicts fixed inputs")
+        sizes = [physical_size(t.data) for t in result.inputs]
+        expected_size = physical_size(result.expected) if result.expected else (0, 0)
+        observed_rows = sum(n for n, _ in sizes) + expected_size[0]
+        observed_bytes = sum(n for _, n in sizes) + expected_size[1]
+        observed_bytes += sum(physical_size(t.index)[1] for t in result.inputs if t.index)
+        if (
+            type(captured.get("rows")) is not int
+            or type(captured.get("arrow_buffer_bytes")) is not int
+            or captured.get("rows") != observed_rows
+            or captured.get("arrow_buffer_bytes") != observed_bytes
+            or observed_rows > result.spec.options.max_capture_rows
+            or observed_bytes > result.spec.options.max_capture_bytes
+        ):
+            raise ArtifactIntegrityError("capture physical statistics contradict the receipt")
+    for p in (result.source, result.target):
+        if p is not None and read_pipeline(directory / p.pipeline_id) != p:
+            raise ArtifactIntegrityError("native pipeline contradicts migration manifest")
+        if p is not None and p.execution is not None:
+            for t in (*p.execution.inputs, *((p.execution.output,) if p.execution.output else ())):
+                bind(t.data)
+                if t.index:
+                    bind(t.index)
+                verify_table_metadata(t)
+    for c in (result.comparison, result.verification_source, result.verification_target):
+        if c is None:
+            continue
+        if read_comparison(directory / c.comparison_id / "comparison.json") != c:
+            raise ArtifactIntegrityError("comparison contradicts migration manifest")
+        bind(c.differences)
+        count = violations = 0
+        for difference in iter_differences(c.differences):
+            if count < len(c.examples) and difference != c.examples[count]:
+                raise ArtifactIntegrityError("comparison examples contradict complete differences")
+            count += 1
+            violations += int(difference.category in ("contract", "schema"))
+        if count != c.counts.difference_records or violations != c.counts.contract_violations:
+            raise ArtifactIntegrityError("comparison differences contradict recorded counts")
+        if len(c.examples) != min(count, c.options.max_examples):
+            raise ArtifactIntegrityError("comparison example count contradicts configured budget")
+        refs.append(c.differences)
+        refs.extend(s.data for s in (c.left, c.right) if s is not None)
+    for ref in refs:
+        ref_path = local_path(ref).resolve()
+        if not ref_path.is_relative_to(directory):
+            raise ArtifactIntegrityError("migration artifact escapes its operation")
+        item = listed.get(ref_path.relative_to(directory).as_posix())
+        if item is None or (item.sha256, item.size_bytes) != (ref.sha256, ref.size_bytes):
+            raise ArtifactIntegrityError("migration artifact contradicts its inventory")
+        if describe_file(ref_path, directory) != item:
+            raise ArtifactIntegrityError("migration artifact changed")
+    stored_summary = parse_json((directory / "report.json").read_bytes())
+    legacy = stored_summary == migration_legacy_summary_v1(result)
+    if not legacy and stored_summary != migration_summary_v1(result):
+        raise ArtifactIntegrityError("migration report contradicts its manifest")
+    if (directory / "report.html").read_bytes() != migration_html_v1(result, legacy=legacy).encode(
+        "utf-8"
+    ):
+        raise ArtifactIntegrityError("migration HTML contradicts its manifest")
+    if result.detection_requested:
+        from equiframe.detector_types import DetectorBundle
+
+        if "detector-bundle.json" not in listed:
+            raise ArtifactIntegrityError("migration detector bundle is missing")
+        bundle = load_json(DetectorBundle, directory / "detector-bundle.json")
+        for p in (result.source, result.target):
+            if (
+                p is not None
+                and load_json(DetectorBundle, directory / p.pipeline_id / "detector-bundle.json")
+                != bundle
+            ):
+                raise ArtifactIntegrityError("native pipelines used different detector bundles")
+    return result

equiframe/migration/ports.py

@@ -0,0 +1,96 @@
+"""Explicit compiler registry; JSON never imports or registers a plugin."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from equiframe.contracts import Contract
+from equiframe.errors import CapabilityError
+from equiframe.registry import Transformation
+
+from .types import CompilationCheck, CompilationDiagnostic, RelationalPlan
+from .validation import ValidatedIR, validate_ir
+
+
+@dataclass(frozen=True)
+class CompiledPlan:
+    """Lower-level native implementation for already validated physical inputs.
+
+    Compilation checks IR/contract metadata, not data. The caller must establish
+    exact input types and value constraints before executing this transformation;
+    ``migrate`` performs that admission automatically. Passing the transformation
+    directly to ``execute`` uses native conversion behavior and records it, but
+    completion does not establish input conformance or semantic equivalence.
+    """
+
+    engine: str
+    transformation: Transformation
+    ir_sha256: str
+    contract_sha256: str
+    check: CompilationCheck
+
+
+class PlanCompiler(Protocol):
+    """Compile trusted IR metadata; actual input admission belongs to the caller.
+
+    ``check`` and ``compile`` do not load or validate tables. Use ``migrate`` for
+    checked input capture, or independently validate exact physical input types
+    and constraints before using a returned ``CompiledPlan`` with ``execute``.
+    """
+
+    @property
+    def engine(self) -> str: ...
+
+    def check(self, plan: RelationalPlan, contract: Contract) -> CompilationCheck: ...
+
+    def compile(self, plan: RelationalPlan, contract: Contract) -> CompiledPlan: ...
+
+
+class CompilerRegistry:
+    def __init__(self) -> None:
+        self._compilers: dict[str, PlanCompiler] = {}
+
+    def register(self, compiler: PlanCompiler) -> None:
+        if not compiler.engine or compiler.engine in self._compilers:
+            raise ValueError("compiler engine must be nonempty and registered once")
+        self._compilers[compiler.engine] = compiler
+
+    def resolve(self, engine: str) -> PlanCompiler:
+        if engine not in self._compilers:
+            raise CapabilityError("no explicitly registered compiler for engine: " + engine)
+        return self._compilers[engine]
+
+
+def compiler_check(engine: str, plan: RelationalPlan, contract: Contract) -> CompilationCheck:
+    try:
+        validate_ir(plan, contract)
+    except (ValueError, CapabilityError) as error:
+        return CompilationCheck(
+            engine=engine,
+            supported=False,
+            diagnostics=(
+                CompilationDiagnostic(
+                    code="unsupported_ir", message=str(error).strip() or "invalid IR"
+                ),
+            ),
+        )
+    return CompilationCheck(engine=engine, supported=True)
+
+
+def require_supported(engine: str, plan: RelationalPlan, contract: Contract) -> ValidatedIR:
+    check = compiler_check(engine, plan, contract)
+    if not check.supported:
+        raise CapabilityError(check.diagnostics[0].message)
+    return validate_ir(plan, contract)
+
+
+def native_compiler_registry() -> CompilerRegistry:
+    from .backends.duckdb import DuckDBCompiler
+    from .backends.pandas import PandasCompiler
+    from .backends.polars import PolarsCompiler
+
+    registry = CompilerRegistry()
+    for compiler in (PandasCompiler(), PolarsCompiler(), DuckDBCompiler()):
+        registry.register(compiler)
+    return registry