mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/assets/logo-ink.svg

@@ -0,0 +1,31 @@
+<!-- Vendored from mostly-right-landing/src/assets/logo-ink.svg on 2026-08-11.
+     Facet and wordmark classes expose the canonical Logo.astro motion without a runtime dependency. -->
+<svg class="mr-logo" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 332.3 49.3" width="332.3" height="49.3" role="img" aria-label="Mostly Right">
+  <title>Mostly Right</title>
+  <g class="mr-logo-mark" transform="translate(0 4.45) scale(0.841458)">
+    <path class="mr-logo-facet mr-logo-facet--1" fill="#1A1813" d="M40,24h-24v16h8c1.5-5,5.1-8,8-8s6.5,3,8,8h8v-8c-2.9,0-6.5-3-8-8Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--2" fill="#F15B22" d="M40,40h-16c-1.5,5-5.1,8-8,8h32c-2.9,0-6.5-3-8-8Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--3" fill="#FFC613" d="M32,0c2.9,0,6.5,3,8,8h8V0h-16Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--4" fill="#2B5FE3" d="M40,8h-24v8h16c2.9,0,6.5,3,8,8h8v-8c-2.9,0-6.5-3-8-8Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--5" fill="#F15B22" d="M0,16c2.9,0,6.5-3,8-8H0v8Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--6" fill="#1A1813" d="M8,8h8V0c-2.9,0-6.5,3-8,8Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--7" fill="#2B5FE3" d="M0,32c2.9,0,6.5-3,8-8H0v8Z"></path>
+    <path class="mr-logo-facet mr-logo-facet--8" fill="#FFC613" d="M8,24h8v-8c-2.9,0-6.5,3-8,8Z"></path>
+    <rect class="mr-logo-facet mr-logo-facet--9" fill="#FFC613" x="0" y="40" width="16" height="8"></rect>
+  </g>
+  <g class="mr-logo-word" fill="#1A1813" transform="translate(-6.86 0)">
+    <path d="M101.06,24.06v12.85h-7.58v-12.64c0-3.69-1.84-4.86-3.79-4.86-2.35,0-4.1,1.64-4.1,5.84v11.67h-7.58v-12.49c0-3.69-1.84-5.02-3.74-5.02-2.05,0-4.15,1.02-4.15,5.02v12.49h-7.58V13h7.58v3.79c.92-3.02,4.2-4.4,6.45-4.4,3.53,0,6.14,1.69,7.58,4.56,1.64-3.17,5.32-4.56,7.83-4.56,5.99,0,9.06,4.35,9.06,11.67Z"></path>
+    <path d="M105.72,24.98c0-7.58,6.14-12.59,12.95-12.59s13,5.02,13,12.59-6.19,12.54-13,12.54-12.95-5.02-12.95-12.54ZM123.94,24.98c0-3.28-2.35-5.58-5.27-5.58s-5.27,2.3-5.27,5.58,2.35,5.53,5.27,5.53,5.27-2.25,5.27-5.53Z"></path>
+    <path d="M135.25,29.18h7.17c0,1.18.87,2.56,3.02,2.56,1.59,0,2.92-.82,2.92-1.95,0-.77-.56-1.13-2.35-1.54l-3.12-.82c-7.01-1.89-7.22-5.89-7.22-7.88,0-3.53,3.99-7.17,9.78-7.17,4.51,0,9.98,2.1,9.88,8.09h-7.17c0-1.54-1.08-2.3-2.56-2.3-1.28,0-2.35.61-2.35,1.64,0,.92.82,1.43,2.05,1.74l4.15,1.13c5.99,1.64,6.5,5.43,6.5,7.06,0,5.27-5.48,7.78-10.29,7.78-5.32,0-10.39-2.92-10.39-8.34Z"></path>
+    <path d="M175.8,13v5.73h-5.02v18.17h-7.58v-18.17h-4.35v-5.73h4.35V4.45h7.58v8.55h5.02Z"></path>
+    <path d="M179.94,36.91V0h7.58v36.91h-7.58Z"></path>
+    <path d="M208.97,13h8.14l-14.74,35.68h-8.09l5.89-14.64-9.42-21.04h8.4l4.97,12.39,4.86-12.39Z"></path>
+  </g>
+  <g class="mr-logo-word" fill="#1A1813" transform="translate(-10.87 0)">
+    <path d="M249.61,13.11l-1.79,7.22c-1.48-.87-2.66-.92-3.43-.92-2.97,0-5.43,2.51-5.43,9.57v7.94h-7.58V13h7.47v5.07c1.38-3.94,4.25-5.68,7.17-5.68,1.13,0,2.35.1,3.58.72Z"></path>
+    <path d="M252.02,5.02c0-2.51,2.15-4.4,4.71-4.4s4.61,1.89,4.61,4.4-2.05,4.4-4.61,4.4-4.71-2-4.71-4.4ZM252.94,36.91V13h7.58v23.91h-7.58Z"></path>
+    <path d="M284.52,13h7.53v21.04c0,10.14-5.32,15.26-13.98,15.26-5.43,0-10.03-3.17-11.93-7.68l6.25-2.51c.87,1.95,3.17,3.53,5.68,3.53,3.94,0,6.45-2.2,6.45-7.68v-.72c-1.79,2.25-4.97,3.28-7.42,3.28-5.94,0-11.57-4.86-11.57-12.59s5.63-12.54,11.57-12.54c2.46,0,5.63.97,7.42,3.23v-2.61ZM284.21,24.93c0-3.07-2.61-5.53-5.48-5.53s-5.53,2.51-5.53,5.53,2.56,5.58,5.53,5.58,5.48-2.51,5.48-5.58Z"></path>
+    <path d="M322.3,24.06v12.85h-7.58v-12.39c0-3.63-2.25-5.12-4.15-5.12-2.05,0-4.61,1.13-4.61,5.12v12.39h-7.58V0h7.58v16.84c.87-3.07,4.71-4.45,6.86-4.45,6.35,0,9.47,4.35,9.47,11.67Z"></path>
+    <path d="M343.08,13v5.73h-5.02v18.17h-7.58v-18.17h-4.35v-5.73h4.35V4.45h7.58v8.55h5.02Z"></path>
+  </g>
+</svg>

mostlyright/data_harness/backends/__init__.py

@@ -0,0 +1,28 @@
+"""The ``backends`` package: named backend protocol, reference backend, registry.
+
+This is the engine seam of the candidate build path. ``pipeline.build_candidate``
+resolves its engine by name through ``get_backend`` and runs the inner
+``clean -> join -> select`` row transforms on the returned backend; ``reference`` is the
+default and the byte-parity authority every other engine must reproduce exactly.
+
+This seam belongs to ``pipeline`` and ``local_contracts.TablePlan``. It is unrelated to
+the ``preparation`` package, whose ``execute_preparation`` is a separate stack with its own
+plan type and no backend dispatch.
+"""
+
+from __future__ import annotations
+
+from mostlyright.data_harness.backends.pandas_backend import PandasBackend
+from mostlyright.data_harness.backends.polars_backend import PolarsBackend
+from mostlyright.data_harness.backends.protocol import Backend
+from mostlyright.data_harness.backends.reference import ReferenceBackend
+from mostlyright.data_harness.backends.registry import get_backend, registered_backends
+
+__all__ = [
+    "Backend",
+    "PandasBackend",
+    "PolarsBackend",
+    "ReferenceBackend",
+    "get_backend",
+    "registered_backends",
+]

mostlyright/data_harness/backends/pandas_backend.py

@@ -0,0 +1,350 @@
+"""The pandas backend: ``PandasBackend`` over Arrow-backed pandas DataFrames.
+
+``PandasBackend`` re-owns the narrow ``clean -> join -> select`` triple behind the
+``Backend`` protocol; every other candidate byte is still produced by the shared layer (the
+one pyarrow sink, the canonical schema, ``_coerce_native_rows``, the evidence serializer).
+Because the shared layer downstream is unchanged, the backend's job is to reproduce the
+reference compute byte-identically while genuinely exercising a pandas object.
+
+Design (the load-bearing dtype strategy):
+
+- The row-shape transforms (``clean``/``join``/``select``) are computed **value-wise** in
+  plain Python, mirroring ``pipeline._apply_cleaning`` / ``_join`` / ``_select`` exactly so
+  unicode-whitespace, empty-vs-null, ordering, and the typed ``BuildError`` refusals match
+  the reference bit-for-bit. Every value-level cast routes through the shared
+  ``pipeline._cast`` — never an engine-native parser, which is what keeps cast results and
+  cast refusals identical across engines.
+- The op's output is then materialized as an **Arrow-backed** pandas DataFrame — one
+  explicit ``pd.ArrowDtype`` per column drawn from the closed ``pipeline._ARROW_BY_LOGICAL``
+  vocabulary the sink uses, constructed from the already-typed native values (never
+  inferred, never a whole-frame ``pa.Table`` pandas conversion into the sink, never a native
+  cast token). Arrow-backed columns
+  carry true Arrow nulls (never ``np.nan``/``NaT``) and keep an ``int64`` column ``int64``
+  under nulls (no int->float promotion).
+- The frame is converted back to an ordered native ``list[dict]`` at the boundary
+  (``_rows_from_frame``): the index is dropped (positional order), and each column is read
+  through pyarrow ``to_pylist()`` so Arrow nulls become Python ``None`` and every scalar is
+  a native Python builtin (``int``/``float``/``str``/``datetime.date``) — no numpy scalar,
+  no ``pd.NA``. ``pipeline._coerce_native_rows`` remains the shared backstop.
+
+``import pandas`` / ``import pyarrow`` are **function-local** (mirroring ``reference.py``'s
+lazy-engine-import ethos), so ``import backends`` and constructing ``PandasBackend()`` at
+registry load never import pandas. The module is named ``pandas_backend`` (not ``pandas``)
+to avoid the ``import pandas`` self-shadow footgun.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mostlyright.data_harness.local_contracts import CleaningStep, TablePlan
+
+
+class PandasBackend:
+    """pandas engine for the ``clean -> join -> select`` triple over Arrow-backed frames."""
+
+    # A plain literal (not a lazy ``pipeline`` import), so constructing the backend at
+    # ``registry`` module load never imports ``pipeline`` or pandas. The seam resolves the
+    # backend by this exact name; any drift fails closed at ``get_backend`` (BACKEND_UNKNOWN).
+    name: str = "pandas"
+
+    def versions(self) -> dict[str, str]:
+        # Provenance only (engine-runtime.v2 ``backend_versions``). The pandas import is
+        # function-local (only reached during a build, when pandas is already dispatched), so
+        # ``import backends`` / constructing the backend stay pandas-free. The version is read
+        # from the module attribute ``pd.__version__`` rather than the stdlib metadata-version
+        # helper: that helper's module name is one of the dynamic-import tokens banned under
+        # ``backends/`` by ``test_backend_registry.py``. For a pinned release the two
+        # spellings are identical.
+        import pandas as pd
+
+        return {"pandas": str(pd.__version__)}
+
+    def materialize_graph_rows(
+        self,
+        columns: tuple[str, ...],
+        rows: tuple[tuple[Any, ...], ...],
+    ) -> tuple[tuple[Any, ...], ...]:
+        """Exercise an object-typed pandas frame without allowing dtype inference to coerce."""
+
+        import pandas as pd
+
+        if any(len(row) != len(columns) for row in rows):
+            raise ValueError("graph row width differs from its columns")
+        frame = pd.DataFrame(
+            {
+                column: pd.Series([row[index] for row in rows], dtype="object")
+                for index, column in enumerate(columns)
+            },
+            columns=list(columns),
+            index=range(len(rows)),
+        )
+        return tuple(tuple(row) for row in frame.itertuples(index=False, name=None))
+
+    def clean(
+        self,
+        rows: list[dict[str, Any]],
+        columns: list[str],
+        lineage: dict[str, dict[str, Any]],
+        step: CleaningStep,
+    ) -> tuple[list[dict[str, Any]], list[str], dict[str, dict[str, Any]]]:
+        # Value-wise reproduction of ``pipeline._apply_cleaning`` (the DataFrame is a shaping
+        # container, not a transform engine): shared ``_require_columns`` refusals, Python
+        # shared ASCII-only trim, ``== ""`` -> None, positional
+        # rename with ``RENAME_COLLISION``, and every cast through the shared ``_cast``.
+        from mostlyright.data_harness import pipeline
+
+        output_rows = [dict(row) for row in rows]
+        output_columns = list(columns)
+        output_lineage = {
+            column: {**entry, "operations": list(entry["operations"])}
+            for column, entry in lineage.items()
+        }
+        if step.operation in {"trim", "empty_to_null"}:
+            target_columns = tuple(step.columns)
+            pipeline._require_columns(output_columns, target_columns, step.operation)
+            for row in output_rows:
+                for column in target_columns:
+                    value = row[column]
+                    if step.operation == "trim":
+                        if value is not None and not isinstance(value, str):
+                            raise pipeline.BuildError(
+                                "CLEAN_TYPE", f"trim requires strings in {column!r}"
+                            )
+                        row[column] = None if value is None else pipeline._trim_ascii(value)
+                    elif value == "":
+                        row[column] = None
+            for column in target_columns:
+                output_lineage[column]["operations"].append(step.operation)
+        elif step.operation == "rename":
+            mapping = dict(step.columns)
+            pipeline._require_columns(output_columns, tuple(mapping), "rename")
+            final_columns = [mapping.get(column, column) for column in output_columns]
+            if len(final_columns) != len(set(final_columns)):
+                raise pipeline.BuildError("RENAME_COLLISION", "rename creates a column collision")
+            output_rows = [
+                {mapping.get(column, column): row[column] for column in output_columns}
+                for row in output_rows
+            ]
+            new_lineage: dict[str, dict[str, Any]] = {}
+            for column in output_columns:
+                target = mapping.get(column, column)
+                entry = output_lineage[column]
+                if target != column:
+                    entry["operations"].append(f"rename:{column}->{target}")
+                new_lineage[target] = entry
+            output_columns = final_columns
+            output_lineage = new_lineage
+        elif step.operation == "cast":
+            mapping = dict(step.columns)
+            pipeline._require_columns(output_columns, tuple(mapping), "cast")
+            for row_index, row in enumerate(output_rows, start=1):
+                for column, target in mapping.items():
+                    row[column] = pipeline._cast(row[column], target, column, row_index)
+            for column, target in mapping.items():
+                output_lineage[column]["operations"].append(f"cast:{target}")
+        else:  # pragma: no cover - contract parser makes this unreachable
+            raise pipeline.ContractError(
+                "plan.cleaning.operation",
+                "CLEAN_OPERATION_UNSUPPORTED",
+                f"unsupported operation: {step.operation}",
+            )
+
+        shaped = _rows_from_frame(
+            _frame_from_rows(
+                output_rows, output_columns, _logical_by_column(output_lineage, output_columns)
+            )
+        )
+        return shaped, output_columns, output_lineage
+
+    def join(
+        self,
+        plan: TablePlan,
+        rows_by_source: dict[str, list[dict[str, Any]]],
+        columns_by_source: dict[str, list[str]],
+        lineage_by_source: dict[str, dict[str, dict[str, Any]]],
+    ) -> tuple[
+        list[dict[str, Any]],
+        list[str],
+        dict[str, dict[str, Any]],
+        dict[str, Any],
+    ]:
+        # Value-wise reproduction of ``pipeline._join``. Left-source order is imposed
+        # positionally (iterate the left rows; look up the right by an equality index) — never
+        # ``DataFrame.merge``'s incidental ordering. Same guards and typed refusals; output
+        # columns ``[*left_columns, *right_nonkeys]``; merged lineage; and a ``report``
+        # byte-identical to the reference one. The report counts stay native ``int`` because
+        # they are pure-Python ``len(...)``; a numpy scalar there would change the evidence
+        # bytes.
+        from mostlyright.data_harness import pipeline
+
+        spec = plan.join
+        left_rows = rows_by_source[spec.left]
+        right_rows = rows_by_source[spec.right]
+        left_columns = columns_by_source[spec.left]
+        right_columns = columns_by_source[spec.right]
+        pipeline._require_columns(left_columns, spec.on, "join.left keys")
+        pipeline._require_columns(right_columns, spec.on, "join.right keys")
+        for key in spec.on:
+            left_types = {type(row[key]) for row in left_rows if row[key] is not None}
+            right_types = {type(row[key]) for row in right_rows if row[key] is not None}
+            if any(row[key] is None for row in left_rows + right_rows):
+                raise pipeline.BuildError("JOIN_NULL_KEY", f"join key {key!r} contains null")
+            if len(left_types) != 1 or left_types != right_types:
+                raise pipeline.BuildError(
+                    "JOIN_KEY_TYPE", f"join key {key!r} types do not match exactly"
+                )
+        right_nonkeys = [column for column in right_columns if column not in spec.on]
+        collisions = set(left_columns) & set(right_nonkeys)
+        if collisions:
+            raise pipeline.BuildError(
+                "JOIN_COLUMN_COLLISION", f"join column collision: {sorted(collisions)}"
+            )
+        right_index: dict[tuple[Any, ...], dict[str, Any]] = {}
+        for row in right_rows:
+            key = tuple(row[column] for column in spec.on)
+            if key in right_index:
+                raise pipeline.BuildError("JOIN_CARDINALITY", "right join keys are not unique")
+            right_index[key] = row
+        if spec.cardinality == "one_to_one":
+            left_keys = [tuple(row[column] for column in spec.on) for row in left_rows]
+            if len(left_keys) != len(set(left_keys)):
+                raise pipeline.BuildError("JOIN_CARDINALITY", "left join keys are not unique")
+        output: list[dict[str, Any]] = []
+        unmatched = 0
+        for left_row in left_rows:
+            key = tuple(left_row[column] for column in spec.on)
+            right_row = right_index.get(key)
+            if right_row is None:
+                unmatched += 1
+                merged = {**left_row, **{column: None for column in right_nonkeys}}
+            else:
+                merged = {**left_row, **{column: right_row[column] for column in right_nonkeys}}
+            output.append(merged)
+        multiplier = len(output) / len(left_rows)
+        if unmatched:
+            raise pipeline.BuildError("JOIN_UNMATCHED", f"left join has {unmatched} unmatched rows")
+        if multiplier != 1.0:
+            raise pipeline.BuildError("JOIN_MULTIPLIER", f"join row multiplier is {multiplier}")
+        columns = [*left_columns, *right_nonkeys]
+        lineage = {
+            **lineage_by_source[spec.left],
+            **{column: lineage_by_source[spec.right][column] for column in right_nonkeys},
+        }
+        report = {
+            "schema_version": "join-evidence.v1",
+            "left_source": spec.left,
+            "right_source": spec.right,
+            "keys": list(spec.on),
+            "cardinality": spec.cardinality,
+            "left_rows": len(left_rows),
+            "right_rows": len(right_rows),
+            "output_rows": len(output),
+            "matched_left_rows": len(output) - unmatched,
+            "unmatched_left_rows": unmatched,
+            "row_multiplier": multiplier,
+            "stable_order": "left_source_order",
+        }
+        shaped = _rows_from_frame(
+            _frame_from_rows(output, columns, _logical_by_column(lineage, columns))
+        )
+        return shaped, columns, lineage, report
+
+    def select(
+        self,
+        plan: TablePlan,
+        rows: list[dict[str, Any]],
+        columns: list[str],
+        lineage: dict[str, dict[str, Any]],
+    ) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]:
+        # Value-wise reproduction of ``pipeline._select``: project/order columns to
+        # ``plan.select`` (shared ``_require_columns`` for the missing-column refusal);
+        # ``selected_lineage`` in ``plan.select`` order.
+        from mostlyright.data_harness import pipeline
+
+        pipeline._require_columns(columns, plan.select, "select")
+        selected_rows = [{column: row[column] for column in plan.select} for row in rows]
+        selected_lineage = {column: lineage[column] for column in plan.select}
+        shaped = _rows_from_frame(
+            _frame_from_rows(
+                selected_rows,
+                list(plan.select),
+                _logical_by_column(selected_lineage, plan.select),
+            )
+        )
+        return shaped, selected_lineage
+
+
+def _logical_by_column(lineage: dict[str, dict[str, Any]], columns: Any) -> dict[str, str]:
+    """Map each column to its closed logical type via the shared ``_logical_type_for``.
+
+    The logical type is drawn only from the column's lineage ``operations`` (the last
+    ``cast:<target>``, else ``"string"``) — never from an engine scalar's class name — so
+    the Arrow-backed dtype the frame is built with matches the sink's imposed schema.
+    """
+
+    from mostlyright.data_harness import pipeline
+
+    return {column: pipeline._logical_type_for(lineage[column]) for column in columns}
+
+
+def _arrow_dtype_for(logical: str) -> Any:
+    """Return the ``pd.ArrowDtype`` for one authoritative local logical type.
+
+    Uses the same closed ``pipeline._ARROW_BY_LOGICAL`` map the shared sink imposes, so the
+    frame's per-column dtype is the canonical one and can never be inferred/lenient.
+    """
+
+    import pandas as pd
+
+    from mostlyright.data_harness import pipeline
+
+    return pd.ArrowDtype(pipeline._ARROW_BY_LOGICAL[logical])
+
+
+def _frame_from_rows(
+    rows: list[dict[str, Any]], columns: Any, logical_by_column: dict[str, str]
+) -> Any:
+    """Build an Arrow-backed pandas DataFrame with an explicit per-column dtype.
+
+    Each column is constructed with ``pd.array(values, dtype=pd.ArrowDtype(pa_type))`` from
+    the already-typed native values — never inferred, never a whole-frame conversion into the
+    sink. Arrow-backed columns carry true Arrow nulls (never ``np.nan``) and keep ``int64``
+    as ``int64`` under
+    nulls (no float promotion). The explicit ``index`` preserves the row count even for a
+    zero-column frame (an unreachable but defensively-handled shape).
+    """
+
+    import pandas as pd
+
+    column_order = list(columns)
+    data = {
+        column: pd.array(
+            [row[column] for row in rows],
+            dtype=_arrow_dtype_for(logical_by_column[column]),
+        )
+        for column in column_order
+    }
+    return pd.DataFrame(data, columns=column_order, index=range(len(rows)))
+
+
+def _rows_from_frame(frame: Any) -> list[dict[str, Any]]:
+    """Convert an Arrow-backed DataFrame back to an ordered native ``list[dict]``.
+
+    The index is dropped (positional order); each column is read through pyarrow
+    ``to_pylist()`` so an Arrow null becomes Python ``None`` and every scalar is a native
+    Python builtin (``bool``/``int``/``float``/``str``/``datetime.date``/UTC
+    ``datetime.datetime``) — no numpy scalar, no ``pd.NA``. Column (key) order follows the
+    frame's column order. ``pipeline``'s
+    ``_coerce_native_rows`` remains the shared backstop downstream.
+    """
+
+    column_order = list(frame.columns)
+    native_by_column = {
+        column: frame[column].array.__arrow_array__().to_pylist() for column in column_order
+    }
+    row_count = len(frame)
+    return [
+        {column: native_by_column[column][index] for column in column_order}
+        for index in range(row_count)
+    ]