equiframe 0.12.4__py3-none-any.whl

equiframe/_duckdb_transport.py

@@ -0,0 +1,149 @@
+"""Closed native-to-Arrow admission shared by DuckDB execution and capture.
+
+Arrow validation cannot prove native fidelity: DuckDB can export a UHUGEINT as
+a valid decimal128 with a different value. Admit native types before export,
+then require their exact Arrow representation and reject temporal sentinels.
+Optional engines are imported only when a transport check actually runs.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from decimal import Decimal
+from typing import TYPE_CHECKING
+
+from equiframe.errors import CapabilityError
+from equiframe.tables import check_arrow
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+
+def duckdb_parameters(parameters: Mapping[str, object]) -> dict[str, object]:
+    """Bind admitted scalars without DuckDB's lossy Python Decimal inference.
+
+    Decimal is rendered exactly (no context arithmetic), then explicitly typed
+    before native execution. In particular, leading fractional zeros can make
+    DuckDB's automatic Decimal inference exceed precision 38 and use DOUBLE.
+    """
+    import duckdb
+
+    bound = {}
+    for name, value in parameters.items():
+        if value is None or type(value) in (bool, str, float):
+            bound[name] = value
+            continue
+        if type(value) is int and -(2**63) <= value < 2**64:
+            bound[name] = value
+            continue
+        if type(value) is Decimal:
+            if not value.is_finite() or (value.is_zero() and value.is_signed()):
+                raise CapabilityError("DuckDB parameters cannot retain this Decimal representation")
+            digits = value.as_tuple()
+            exponent = int(digits.exponent)
+            precision = max(len(digits.digits), -exponent)
+            if exponent <= 0 and precision <= 38:
+                bound[name] = duckdb.Value(
+                    format(value, "f"), duckdb.decimal_type(precision, -exponent)
+                )
+                continue
+        raise CapabilityError(
+            "DuckDB parameters admit plain NULL/bool/text/float, 64-bit integers and "
+            "finite nonnegative-scale Decimal up to precision 38; use text with explicit "
+            "SQL casts for temporal or other intentionally converted literals"
+        )
+    return bound
+
+
+def duckdb_arrow_schema(native_types: tuple[tuple[str, str], ...], timezone: str) -> pa.Schema:
+    """Return the exact admitted schema, without casting or erasing native types.
+
+    The names are observed result names and types are DuckDB's canonical native
+    descriptions, not user SQL aliases. DuckDB result fields are always nullable.
+    HUGEINT, UHUGEINT and unknown/extended types deliberately have no mapping.
+    """
+    import pyarrow as pa
+
+    if not isinstance(timezone, str) or not timezone:
+        raise CapabilityError("DuckDB result transport requires its effective session timezone")
+    if not native_types:
+        raise CapabilityError("DuckDB result transport requires a nonempty native schema")
+    scalar_types = {
+        "BOOLEAN": pa.bool_(),
+        "TINYINT": pa.int8(),
+        "SMALLINT": pa.int16(),
+        "INTEGER": pa.int32(),
+        "BIGINT": pa.int64(),
+        "UTINYINT": pa.uint8(),
+        "USMALLINT": pa.uint16(),
+        "UINTEGER": pa.uint32(),
+        "UBIGINT": pa.uint64(),
+        "FLOAT": pa.float32(),
+        "DOUBLE": pa.float64(),
+        "VARCHAR": pa.string(),
+        "DATE": pa.date32(),
+        "TIMESTAMP_S": pa.timestamp("s"),
+        "TIMESTAMP_MS": pa.timestamp("ms"),
+        "TIMESTAMP": pa.timestamp("us"),
+        "TIMESTAMP_NS": pa.timestamp("ns"),
+        "TIMESTAMP WITH TIME ZONE": pa.timestamp("us", tz=timezone),
+    }
+    fields = []
+    names = set()
+    for name, native_type in native_types:
+        if not isinstance(name, str) or not name or name in names:
+            raise CapabilityError("DuckDB result column names must be nonempty and unique")
+        names.add(name)
+        dtype = scalar_types.get(native_type)
+        if dtype is None:
+            decimal = re.fullmatch(r"DECIMAL\(([0-9]{1,2}),([0-9]{1,2})\)", native_type)
+            if decimal is not None:
+                precision, scale = (int(value) for value in decimal.groups())
+                if 1 <= precision <= 38 and 0 <= scale <= precision:
+                    dtype = pa.decimal128(precision, scale)
+        if dtype is None:
+            if native_type in {"HUGEINT", "UHUGEINT"}:
+                raise CapabilityError(
+                    "DuckDB HUGEINT/UHUGEINT results have no faithful admitted Arrow transport; "
+                    "use an explicit checked BIGINT or DECIMAL cast in the caller's SQL "
+                    "when its range and semantics permit it"
+                )
+            raise CapabilityError("unsupported faithful DuckDB native result type")
+        fields.append(pa.field(name, dtype, nullable=True))
+    return pa.schema(fields)
+
+
+def duckdb_temporal_capabilities(table: pa.Table) -> None:
+    """Reject native infinity markers that Arrow exposes as ordinary ticks."""
+    import pyarrow as pa
+    import pyarrow.compute as pc
+
+    for field in table.schema:
+        if pa.types.is_timestamp(field.type):
+            bounds = pc.min_max(table[field.name].cast(pa.int64())).as_py()
+            if bounds["min"] is not None and (
+                bounds["min"] <= -(2**63 - 1) or bounds["max"] >= 2**63 - 1
+            ):
+                raise CapabilityError(
+                    "DuckDB timestamp infinity/sentinel values cannot be represented "
+                    "faithfully in Arrow"
+                )
+        elif pa.types.is_date32(field.type):
+            values = table[field.name].cast(pa.int32())
+            sentinels = pa.array([-(2**31 - 1), 2**31 - 1], type=pa.int32())
+            if pc.any(pc.is_in(values, value_set=sentinels)).as_py():
+                raise CapabilityError(
+                    "DuckDB date infinity values cannot be represented faithfully in Arrow"
+                )
+
+
+def verify_duckdb_table(
+    table: pa.Table, native_types: tuple[tuple[str, str], ...], timezone: str
+) -> None:
+    """Validate physical data, exact native mapping and finite temporal values."""
+    expected = duckdb_arrow_schema(native_types, timezone)
+    check_arrow(table)
+    if not table.schema.equals(expected, check_metadata=True):
+        raise CapabilityError("DuckDB Arrow result schema differs from its admitted native schema")
+    duckdb_temporal_capabilities(table)

equiframe/_error_reporting.py

@@ -0,0 +1,52 @@
+"""Safe runtime failure summaries and bounded, value-free debug frames."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+
+_LOG = logging.getLogger("equiframe.diagnostics")
+_LOG.addHandler(logging.NullHandler())
+_NAME = re.compile(r"[A-Za-z_][A-Za-z_0-9.]{0,127}\Z")
+
+
+def report_failure(error: BaseException, message: str) -> str:
+    """Return an authored public message; never stringify the caught exception.
+
+    Debug output contains owned module/function/line metadata only. Exception
+    values, source lines, filenames, locals and external callback names are
+    excluded, including when an exception's __str__ itself is unsafe.
+    """
+    try:
+        enabled = _LOG.isEnabledFor(logging.DEBUG)
+    except Exception:
+        return message
+    if enabled:
+        try:
+            _debug_frames(error)
+        except Exception:
+            # Logging is best effort; a caller's failing handler must not
+            # expose values or replace the original operational outcome.
+            return message
+    return message
+
+
+def _debug_frames(error: BaseException) -> None:
+    frames: list[tuple[str, str, int]] = []
+    cursor = error.__traceback__
+    while cursor is not None and len(frames) < 32:
+        frame = cursor.tb_frame
+        module = frame.f_globals.get("__name__")
+        function = frame.f_code.co_name
+        if (
+            type(module) is str
+            and module.startswith("equiframe.")
+            and _NAME.fullmatch(module)
+            and _NAME.fullmatch(function)
+        ):
+            frames.append((module, function, cursor.tb_lineno))
+        else:
+            frames.append(("external", "callback", 0))
+        cursor = cursor.tb_next
+    _LOG.debug("Operation failed; safe frames=%s", json.dumps(frames, ensure_ascii=True))

equiframe/_first_use.py

@@ -0,0 +1,102 @@
+"""Small synthetic role exercises using the existing public comparison APIs."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+
+class Role(str, Enum):
+    architect = "architect"
+    scientist = "scientist"
+
+
+class Case(str, Enum):
+    good = "good"
+    different = "different"
+    not_evaluable = "not-evaluable"
+
+
+def run_demo(role: Role, case: Case, output: Path) -> dict[str, Any]:
+    """Save both report views before deciding; never replace existing evidence."""
+    import equiframe as ef
+    from equiframe.tables import require_dependency
+
+    require_dependency("pyarrow", "arrow")
+    import pyarrow as pa
+
+    # Reserve the caller's new destination before performing any native work.
+    output.mkdir(parents=True, exist_ok=False)
+    if role == Role.architect:
+        expected = pa.table({"id": ["A", "B"], "quantity": [10, 20]})
+        candidate = ef.capture_duckdb(
+            "SELECT 'A' AS id, 10::BIGINT AS quantity UNION ALL SELECT 'B', $quantity::BIGINT",
+            parameters={"quantity": 21 if case == Case.different else 20},
+            work_dir=output / "capture",
+        )
+        declaration = ef.infer_diff_contract(expected.schema, keys=["id"])
+    else:
+        expected = pa.table(
+            {
+                "sample_id": ["0001", "0002", "0003"],
+                "duration_s": pa.array(
+                    [Decimal("1.250"), None, Decimal("1000000.001")], type=pa.decimal128(16, 3)
+                ),
+                "observed_at": pa.array([0, 1000000, None], type=pa.timestamp("us", tz="UTC")),
+            }
+        )
+        # Candidate calculation is separate from the literal expected durations.
+        durations = [Decimal("1250") / Decimal("1000"), None, Decimal("1000000001") / 1000]
+        if case == Case.different:
+            durations[0] = Decimal("1.251")
+        candidate = pa.table(
+            {
+                "sample_id": ["0001", "0002", "0003"],
+                "duration_s": pa.array(durations, type=pa.decimal128(16, 3)),
+                "observed_at": pa.array([0, 1000000, None], type=pa.timestamp("us", tz="UTC")),
+            }
+        )
+        duration = ef.ColumnSpec(
+            name="duration_s",
+            logical_type=ef.DecimalType(precision=16, scale=3),
+            nullable=True,
+            null_policy=ef.NullPolicy(representations=("null",)),
+            numeric_policy=ef.NumericPolicy(comparison="exact", unit="seconds"),
+        )
+        declaration = ef.infer_diff_contract(
+            expected.schema, keys=["sample_id"], column_overrides=(duration,)
+        )
+    result = ef.diff(
+        candidate,
+        ef.ArrowInput(expected),
+        contract=declaration,
+        work_dir=output / "private",
+        options=ef.ComparisonOptions(max_rows=1 if case == Case.not_evaluable else 100),
+    )
+    report = ef.write_comparison_report(result, output / "report")
+    detail = ef.write_comparison_report(
+        result,
+        output / "synthetic-detail",
+        options=ef.ReportOptions(include_values=True, include_metadata=True),
+    )
+    decision = ef.equivalence_decision(
+        result,
+        policy=ef.EquivalencePolicy(require_persisted_evidence=True),
+        evidence_root=output / "private",
+    )
+    return {
+        "role": role.value,
+        "case": case.value,
+        "synthetic": True,
+        "state": decision.state,
+        "complete": decision.complete,
+        "accepted": decision.accepted,
+        "code": decision.code,
+        "next_action": report.next_action,
+        "report": report.to_text(),
+        "synthetic_detail": detail.to_text(),
+        "report_html": str(output / "report" / "report.html"),
+        "synthetic_detail_html": str(output / "synthetic-detail" / "report.html"),
+    }

equiframe/_path_compat.py

@@ -0,0 +1,26 @@
+"""Filesystem compatibility for the supported CPython versions.
+
+Windows junctions are mount-point reparse points. The lstat tag has existed
+since Python 3.8, so older Python must inspect it rather than assume no junction.
+This is not a race-free sandbox and does not extend the supported filesystems.
+"""
+
+import stat
+import sys
+from pathlib import Path
+
+
+def is_junction(path: Path) -> bool:
+    """Identify a Windows junction without following it, including on 3.10/3.11.
+
+    POSIX has no Windows junctions. Missing paths are not junctions; other I/O
+    failures propagate so inaccessible paths cannot be silently accepted.
+    Symlinks are checked separately by the caller, preserving its error contract.
+    """
+    if sys.platform != "win32":
+        return False
+    try:
+        result = path.lstat()
+    except FileNotFoundError:
+        return False
+    return result.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT

equiframe/_physical.py

@@ -0,0 +1,133 @@
+"""Verify captured physical metadata without executing or normalizing a transformation."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from equiframe.errors import ArtifactIntegrityError, CapabilityError
+from equiframe.local import local_path
+from equiframe.tables import require_dependency
+
+if TYPE_CHECKING:
+    from equiframe.execution_types import PhysicalColumn, TableArtifact
+    from equiframe.types import ArtifactRef
+
+
+@dataclass(frozen=True)
+class _Observation:
+    schema: tuple[tuple[str, str], ...]
+    rows: int
+    buffer_bytes: int
+    nulls: tuple[int, ...]
+    nans: tuple[int, ...]
+
+
+def _observe(ref: ArtifactRef, *, check: Callable[[], None] | None = None) -> _Observation:
+    """Hash first, then decode batches; buffer bytes are distinct from file bytes."""
+    require_dependency("pyarrow", "arrow")
+    import pyarrow as pa
+    import pyarrow.compute as pc
+
+    if ref.format not in ("arrow", "parquet"):
+        raise ArtifactIntegrityError("physical table requires Arrow IPC or Parquet")
+    try:
+        with local_path(ref).open("rb") as source:
+            if check:
+                check()
+            digest = hashlib.sha256()
+            file_bytes = 0
+            while block := source.read(1024 * 1024):
+                if check:
+                    check()
+                digest.update(block)
+                file_bytes += len(block)
+                if check:
+                    check()
+            if (digest.hexdigest(), file_bytes) != (ref.sha256, ref.size_bytes):
+                raise ArtifactIntegrityError("physical table SHA-256 or size mismatch")
+            source.seek(0)
+            if ref.format == "arrow":
+                reader = pa.ipc.open_file(source)
+                schema = reader.schema
+                batches = (reader.get_batch(i) for i in range(reader.num_record_batches))
+            else:
+                import pyarrow.parquet as pq
+
+                parquet = pq.ParquetFile(source)
+                schema = parquet.schema_arrow
+                # Preserve row-group boundaries. A 65536-row subdivision is byte
+                # aligned for validity bitmaps and does not change materialized
+                # Arrow nbytes through additional partial bitmap boundaries.
+                batches = (
+                    batch
+                    for group in range(parquet.num_row_groups)
+                    for batch in parquet.iter_batches(
+                        batch_size=65536, row_groups=[group], use_threads=False
+                    )
+                )
+            nulls = [0] * len(schema)
+            nans = [0] * len(schema)
+            rows = buffer_bytes = 0
+            batch_iterator = iter(batches)
+            while True:
+                if check:
+                    check()
+                try:
+                    batch = next(batch_iterator)
+                except StopIteration:
+                    break
+                batch.validate(full=True)
+                if batch.schema != schema:
+                    raise ArtifactIntegrityError("physical batch schema changed")
+                rows += batch.num_rows
+                buffer_bytes += batch.nbytes
+                for i, array in enumerate(batch.columns):
+                    nulls[i] += array.null_count
+                    if pa.types.is_floating(array.type):
+                        nans[i] += int(pc.sum(pc.is_nan(array)).as_py() or 0)
+                if check:
+                    check()
+            return _Observation(
+                schema=tuple((field.name, str(field.type)) for field in schema),
+                rows=rows,
+                buffer_bytes=buffer_bytes,
+                nulls=tuple(nulls),
+                nans=tuple(nans),
+            )
+    except (OSError, ValueError, CapabilityError, pa.ArrowException) as error:
+        raise ArtifactIntegrityError("cannot verify captured physical table") from error
+
+
+def physical_size(ref: ArtifactRef, *, check: Callable[[], None] | None = None) -> tuple[int, int]:
+    """Return rows and materialized Arrow buffer bytes, with verified source bytes.
+
+    Decoding is by record batch. An IPC record batch can itself be large; this
+    function does not impose a process-memory quota or execute native engines.
+    """
+    observed = _observe(ref, check=check)
+    return observed.rows, observed.buffer_bytes
+
+
+def _verify_columns(observed: _Observation, columns: tuple[PhysicalColumn, ...], rows: int) -> None:
+    if observed.rows != rows or observed.schema != tuple(
+        (column.name, column.arrow_type) for column in columns
+    ):
+        raise ArtifactIntegrityError("captured physical rows or schema contradict metadata")
+    if observed.nulls != tuple(column.null_count for column in columns) or (
+        observed.nans != tuple(column.nan_count for column in columns)
+    ):
+        raise ArtifactIntegrityError("captured physical NULL or NaN counts contradict metadata")
+
+
+def verify_table_metadata(table: TableArtifact) -> None:
+    """Verify schema, counts and a separate index against captured table bytes.
+
+    Native ``engine_type`` is provenance and cannot be inferred from IPC/Parquet.
+    No casts, contract normalization, ETL, plugins or engine imports occur here.
+    """
+    _verify_columns(_observe(table.data), table.columns, table.row_count)
+    if table.index is not None:
+        _verify_columns(_observe(table.index), table.index_columns, table.row_count)