mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/preparation/profile.py

@@ -0,0 +1,384 @@
+"""Exact bounded before/after table profiles."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass
+from decimal import Decimal
+from typing import Any
+
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.preparation.contracts import PROFILE_VERSION
+from mostlyright.data_harness.preparation.errors import PreparationError
+from mostlyright.data_harness.preparation.table import TableState, validate_columns
+
+MAX_DISTRIBUTION_VALUES = 32
+_DIGEST = re.compile(r"\A[0-9a-f]{64}\Z")
+
+
+@dataclass(frozen=True)
+class ValueCount:
+    value: str
+    count: int
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.value, str) or len(self.value) > 1_000_000:
+            raise PreparationError("PROFILE_VALUE", "profile.distribution.value", "is invalid")
+        if type(self.count) is not int or self.count < 1:
+            raise PreparationError(
+                "PROFILE_COUNT", "profile.distribution.count", "must be positive"
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"value": self.value, "count": self.count}
+
+
+@dataclass(frozen=True)
+class FieldProfile:
+    name: str
+    logical_type: str
+    physical_type: str
+    null_count: int
+    non_null_count: int
+    distinct_count: int
+    minimum: str | None
+    maximum: str | None
+    minimum_utf8_bytes: int | None
+    maximum_utf8_bytes: int | None
+    quantiles: tuple[tuple[str, str], ...]
+    distribution: tuple[ValueCount, ...]
+    other_value_count: int
+    outlier_count: int
+    outlier_lower_bound: str | None
+    outlier_upper_bound: str | None
+
+    def __post_init__(self) -> None:
+        counts = (
+            self.null_count,
+            self.non_null_count,
+            self.distinct_count,
+            self.other_value_count,
+            self.outlier_count,
+        )
+        if any(type(value) is not int or value < 0 for value in counts):
+            raise PreparationError(
+                "PROFILE_COUNT", f"profile.fields.{self.name}", "has invalid counts"
+            )
+        if self.distinct_count > self.non_null_count:
+            raise PreparationError(
+                "PROFILE_COUNT",
+                f"profile.fields.{self.name}",
+                "distinct count exceeds non-null count",
+            )
+        if (self.minimum is None) != (self.maximum is None):
+            raise PreparationError(
+                "PROFILE_RANGE",
+                f"profile.fields.{self.name}",
+                "minimum and maximum must be paired",
+            )
+        if self.logical_type == "string":
+            if (self.minimum_utf8_bytes is None) != (self.maximum_utf8_bytes is None):
+                raise PreparationError(
+                    "PROFILE_STRING_RANGE",
+                    f"profile.fields.{self.name}",
+                    "string byte bounds must be paired",
+                )
+        elif self.minimum_utf8_bytes is not None or self.maximum_utf8_bytes is not None:
+            raise PreparationError(
+                "PROFILE_STRING_RANGE",
+                f"profile.fields.{self.name}",
+                "non-string profile cannot carry string byte bounds",
+            )
+        if tuple(key for key, _ in self.quantiles) not in {
+            (),
+            ("0.25", "0.50", "0.75"),
+        }:
+            raise PreparationError(
+                "PROFILE_QUANTILES",
+                f"profile.fields.{self.name}",
+                "quantile set/order is not exact",
+            )
+        if len(self.distribution) > MAX_DISTRIBUTION_VALUES:
+            raise PreparationError(
+                "PROFILE_DISTRIBUTION",
+                f"profile.fields.{self.name}",
+                "distribution exceeds its bound",
+            )
+        distribution_values = tuple(item.value for item in self.distribution)
+        if len(set(distribution_values)) != len(distribution_values):
+            raise PreparationError(
+                "PROFILE_DISTRIBUTION",
+                f"profile.fields.{self.name}",
+                "distribution contains duplicate values",
+            )
+        if (self.outlier_lower_bound is None) != (self.outlier_upper_bound is None):
+            raise PreparationError(
+                "PROFILE_OUTLIER",
+                f"profile.fields.{self.name}",
+                "outlier bounds must be paired",
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "logical_type": self.logical_type,
+            "physical_type": self.physical_type,
+            "null_count": self.null_count,
+            "non_null_count": self.non_null_count,
+            "distinct_count": self.distinct_count,
+            "minimum": self.minimum,
+            "maximum": self.maximum,
+            "minimum_utf8_bytes": self.minimum_utf8_bytes,
+            "maximum_utf8_bytes": self.maximum_utf8_bytes,
+            "quantiles": [{"quantile": key, "value": value} for key, value in self.quantiles],
+            "distribution": [item.to_dict() for item in self.distribution],
+            "other_value_count": self.other_value_count,
+            "outlier_count": self.outlier_count,
+            "outlier_lower_bound": self.outlier_lower_bound,
+            "outlier_upper_bound": self.outlier_upper_bound,
+        }
+
+
+@dataclass(frozen=True)
+class TableProfile:
+    schema_version: str
+    table_id: str
+    schema_digest: str
+    row_count: int
+    column_count: int
+    duplicate_row_count: int
+    key_columns: tuple[str, ...]
+    duplicate_key_count: int
+    temporal_coverage: tuple[tuple[str, str | None, str | None], ...]
+    fields: tuple[FieldProfile, ...]
+
+    def __post_init__(self) -> None:
+        if self.schema_version != PROFILE_VERSION:
+            raise PreparationError("PROFILE_VERSION", "profile.schema_version", "is unsupported")
+        if not isinstance(self.table_id, str) or not self.table_id:
+            raise PreparationError("PROFILE_TABLE", "profile.table_id", "must be non-empty")
+        if not isinstance(self.schema_digest, str) or _DIGEST.fullmatch(self.schema_digest) is None:
+            raise PreparationError("PROFILE_DIGEST", "profile.schema_digest", "is invalid")
+        counts = (
+            self.row_count,
+            self.column_count,
+            self.duplicate_row_count,
+            self.duplicate_key_count,
+        )
+        if any(type(value) is not int or value < 0 for value in counts):
+            raise PreparationError("PROFILE_COUNT", "profile", "contains invalid counts")
+        if self.column_count != len(self.fields):
+            raise PreparationError(
+                "PROFILE_COUNT",
+                "profile.column_count",
+                "does not equal the field count",
+            )
+        if self.duplicate_row_count > self.row_count or self.duplicate_key_count > self.row_count:
+            raise PreparationError("PROFILE_COUNT", "profile", "duplicate count exceeds row count")
+        if any(item.null_count + item.non_null_count != self.row_count for item in self.fields):
+            raise PreparationError(
+                "PROFILE_COUNT",
+                "profile.fields",
+                "field counts do not reconcile the table row count",
+            )
+        field_names = tuple(item.name for item in self.fields)
+        if len(set(field_names)) != len(field_names):
+            raise PreparationError("PROFILE_FIELDS", "profile.fields", "contains duplicate names")
+        if any(column not in field_names for column in self.key_columns):
+            raise PreparationError(
+                "PROFILE_KEYS", "profile.key_columns", "references unknown fields"
+            )
+        temporal_names = tuple(item[0] for item in self.temporal_coverage)
+        if len(set(temporal_names)) != len(temporal_names) or any(
+            column not in field_names for column in temporal_names
+        ):
+            raise PreparationError(
+                "PROFILE_TEMPORAL",
+                "profile.temporal_coverage",
+                "is duplicate or references an unknown field",
+            )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "table_id": self.table_id,
+            "schema_digest": self.schema_digest,
+            "row_count": self.row_count,
+            "column_count": self.column_count,
+            "duplicate_row_count": self.duplicate_row_count,
+            "key_columns": list(self.key_columns),
+            "duplicate_key_count": self.duplicate_key_count,
+            "temporal_coverage": [
+                {"column": column, "minimum": minimum, "maximum": maximum}
+                for column, minimum, maximum in self.temporal_coverage
+            ],
+            "fields": [field.to_dict() for field in self.fields],
+        }
+
+
+def profile_table(state: TableState, *, key_columns: tuple[str, ...] = ()) -> TableProfile:
+    """Calculate an exact deterministic bounded profile."""
+
+    key_indexes: tuple[int, ...] = ()
+    if key_columns:
+        key_indexes = validate_columns(state, key_columns, "profile.key_columns")
+    duplicate_rows = sum(
+        count - 1 for count in Counter(_row_key(row) for row in state.rows).values()
+    )
+    duplicate_keys = 0
+    if key_indexes:
+        duplicate_keys = sum(
+            count - 1
+            for count in Counter(
+                tuple(_value_key(row[index]) for index in key_indexes) for row in state.rows
+            ).values()
+        )
+    fields = tuple(
+        _profile_field(field.name, field.logical_type, field.physical_type, values)
+        for field, values in (
+            (field, tuple(row[index] for row in state.rows))
+            for index, field in enumerate(state.fields)
+        )
+    )
+    temporal = tuple(
+        (profile.name, profile.minimum, profile.maximum)
+        for field, profile in zip(state.fields, fields, strict=True)
+        if field.temporal_role != "none"
+    )
+    schema_digest = canonical_sha256([field.to_dict() for field in state.fields])
+    return TableProfile(
+        schema_version=PROFILE_VERSION,
+        table_id=state.table_id,
+        schema_digest=schema_digest,
+        row_count=len(state.rows),
+        column_count=len(state.fields),
+        duplicate_row_count=duplicate_rows,
+        key_columns=key_columns,
+        duplicate_key_count=duplicate_keys,
+        temporal_coverage=temporal,
+        fields=fields,
+    )
+
+
+def _profile_field(
+    name: str,
+    logical_type: str,
+    physical_type: str,
+    values: tuple[Any, ...],
+) -> FieldProfile:
+    non_null = tuple(value for value in values if value is not None)
+    keyed = Counter(_value_key(value) for value in non_null)
+    distribution_items = sorted(
+        keyed.items(),
+        key=lambda item: (-item[1], item[0]),
+    )
+    distribution = tuple(
+        ValueCount(value=key, count=count)
+        for key, count in distribution_items[:MAX_DISTRIBUTION_VALUES]
+    )
+    other = sum(count for _, count in distribution_items[MAX_DISTRIBUTION_VALUES:])
+    ordered = sorted(non_null, key=_ordered_key)
+    minimum = _evidence_scalar(ordered[0]) if ordered else None
+    maximum = _evidence_scalar(ordered[-1]) if ordered else None
+    minimum_bytes: int | None = None
+    maximum_bytes: int | None = None
+    if logical_type == "string" and non_null:
+        sizes = tuple(len(value.encode("utf-8")) for value in non_null)
+        minimum_bytes = min(sizes)
+        maximum_bytes = max(sizes)
+    quantiles: tuple[tuple[str, str], ...] = ()
+    outlier_count = 0
+    lower: str | None = None
+    upper: str | None = None
+    if logical_type in {"int64", "float64"} and non_null:
+        numeric = sorted(_decimal(value) for value in non_null)
+        q1 = _nearest_rank(numeric, 1, 4)
+        q2 = _nearest_rank(numeric, 1, 2)
+        q3 = _nearest_rank(numeric, 3, 4)
+        quantiles = (
+            ("0.25", _decimal_text(q1)),
+            ("0.50", _decimal_text(q2)),
+            ("0.75", _decimal_text(q3)),
+        )
+        iqr = q3 - q1
+        lower_value = q1 - (iqr * Decimal(3) / Decimal(2))
+        upper_value = q3 + (iqr * Decimal(3) / Decimal(2))
+        lower = _decimal_text(lower_value)
+        upper = _decimal_text(upper_value)
+        outlier_count = sum(value < lower_value or value > upper_value for value in numeric)
+    return FieldProfile(
+        name=name,
+        logical_type=logical_type,
+        physical_type=physical_type,
+        null_count=len(values) - len(non_null),
+        non_null_count=len(non_null),
+        distinct_count=len(keyed),
+        minimum=minimum,
+        maximum=maximum,
+        minimum_utf8_bytes=minimum_bytes,
+        maximum_utf8_bytes=maximum_bytes,
+        quantiles=quantiles,
+        distribution=distribution,
+        other_value_count=other,
+        outlier_count=outlier_count,
+        outlier_lower_bound=lower,
+        outlier_upper_bound=upper,
+    )
+
+
+def _nearest_rank(values: list[Decimal], numerator: int, denominator: int) -> Decimal:
+    index = ((len(values) - 1) * numerator) // denominator
+    return values[index]
+
+
+def _decimal(value: Any) -> Decimal:
+    if type(value) is int:
+        return Decimal(value)
+    if type(value) is float and math.isfinite(value):
+        return Decimal(str(value))
+    raise PreparationError("PROFILE_NUMERIC", "profile", "encountered a non-finite numeric value")
+
+
+def _decimal_text(value: Decimal) -> str:
+    normalized = value.normalize()
+    text = format(normalized, "f")
+    if "." in text:
+        text = text.rstrip("0").rstrip(".")
+    return "0" if text in {"", "-0"} else text
+
+
+def _evidence_scalar(value: Any) -> str:
+    if value is None:
+        return "null"
+    if type(value) is bool:
+        return "true" if value else "false"
+    if type(value) is int:
+        return str(value)
+    if type(value) is float:
+        return _decimal_text(_decimal(value))
+    return value
+
+
+def _value_key(value: Any) -> str:
+    return f"{type(value).__name__}:{_evidence_scalar(value)}"
+
+
+def _ordered_key(value: Any) -> tuple[int, Any]:
+    if type(value) is bool:
+        return (0, value)
+    if type(value) in {int, float}:
+        return (1, _decimal(value))
+    return (2, value)
+
+
+def _row_key(row: tuple[Any, ...]) -> tuple[str, ...]:
+    return tuple(_value_key(value) for value in row)
+
+
+__all__ = ["FieldProfile", "TableProfile", "ValueCount", "profile_table"]

mostlyright/data_harness/preparation/table.py

@@ -0,0 +1,217 @@
+"""Immutable in-memory tables at the acquisition-to-preparation boundary."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable
+from dataclasses import dataclass, replace
+from datetime import date
+from typing import Any
+
+from mostlyright.data_harness.acquisition import ParsedTable
+from mostlyright.data_harness.preparation.contracts import (
+    MAX_FIELDS,
+    MAX_ROWS,
+    ColumnLineage,
+    FieldSpec,
+    LineageInput,
+    parse_utc,
+)
+from mostlyright.data_harness.preparation.errors import PreparationError
+
+
+@dataclass(frozen=True)
+class TableState:
+    """An immutable typed table with complete source-to-current lineage."""
+
+    table_id: str
+    snapshot_digests: tuple[str, ...]
+    fields: tuple[FieldSpec, ...]
+    rows: tuple[tuple[Any, ...], ...]
+    lineage: tuple[ColumnLineage, ...]
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.table_id, str) or not self.table_id:
+            raise PreparationError("TABLE_ID", "table.table_id", "must be non-empty")
+        if (
+            not isinstance(self.snapshot_digests, tuple)
+            or not self.snapshot_digests
+            or tuple(sorted(set(self.snapshot_digests))) != self.snapshot_digests
+        ):
+            raise PreparationError(
+                "TABLE_SNAPSHOTS",
+                "table.snapshot_digests",
+                "must be a non-empty sorted unique tuple",
+            )
+        if not isinstance(self.fields, tuple) or not 1 <= len(self.fields) <= MAX_FIELDS:
+            raise PreparationError("TABLE_FIELDS", "table.fields", "field count is outside bounds")
+        names = tuple(field.name for field in self.fields)
+        if len(set(names)) != len(names):
+            raise PreparationError("TABLE_FIELDS", "table.fields", "field names are not unique")
+        if not isinstance(self.rows, tuple) or len(self.rows) > MAX_ROWS:
+            raise PreparationError("TABLE_ROWS", "table.rows", "row count is outside bounds")
+        for row_index, row in enumerate(self.rows):
+            if not isinstance(row, tuple) or len(row) != len(self.fields):
+                raise PreparationError(
+                    "TABLE_ROW_WIDTH",
+                    f"table.rows[{row_index}]",
+                    "row width differs from the schema",
+                )
+            for column_index, (field, value) in enumerate(zip(self.fields, row, strict=True)):
+                _validate_cell(value, field, f"table.rows[{row_index}][{column_index}]")
+        if not isinstance(self.lineage, tuple) or len(self.lineage) != len(self.fields):
+            raise PreparationError(
+                "TABLE_LINEAGE",
+                "table.lineage",
+                "must contain exactly one record per field",
+            )
+        if tuple(item.output_column for item in self.lineage) != names:
+            raise PreparationError(
+                "TABLE_LINEAGE",
+                "table.lineage",
+                "must be ordered exactly like the fields",
+            )
+        if any(
+            item.snapshot_digest not in self.snapshot_digests
+            for lineage in self.lineage
+            for item in lineage.inputs
+        ):
+            raise PreparationError(
+                "TABLE_LINEAGE",
+                "table.lineage",
+                "references an unbound snapshot digest",
+            )
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        return tuple(field.name for field in self.fields)
+
+    def index(self, column: str) -> int:
+        try:
+            return self.columns.index(column)
+        except ValueError:
+            raise PreparationError("TABLE_COLUMN", column, "column does not exist") from None
+
+    def field(self, column: str) -> FieldSpec:
+        return self.fields[self.index(column)]
+
+    def dictionaries(self) -> tuple[dict[str, Any], ...]:
+        return tuple(dict(zip(self.columns, row, strict=True)) for row in self.rows)
+
+    def replace(
+        self,
+        *,
+        table_id: str | None = None,
+        fields: tuple[FieldSpec, ...] | None = None,
+        rows: tuple[tuple[Any, ...], ...] | None = None,
+        lineage: tuple[ColumnLineage, ...] | None = None,
+        snapshot_digests: tuple[str, ...] | None = None,
+    ) -> TableState:
+        return replace(
+            self,
+            table_id=self.table_id if table_id is None else table_id,
+            fields=self.fields if fields is None else fields,
+            rows=self.rows if rows is None else rows,
+            lineage=self.lineage if lineage is None else lineage,
+            snapshot_digests=(
+                self.snapshot_digests if snapshot_digests is None else snapshot_digests
+            ),
+        )
+
+
+def table_from_parsed(
+    *,
+    source_id: str,
+    parsed: ParsedTable,
+    fields: tuple[FieldSpec, ...],
+) -> TableState:
+    """Bind one parsed acquisition snapshot into a typed preparation table."""
+
+    if not isinstance(parsed, ParsedTable):
+        raise PreparationError(
+            "TABLE_PARSED", "parsed", "must be verified acquisition ParsedTable evidence"
+        )
+    if tuple(field.name for field in fields) != parsed.columns:
+        raise PreparationError(
+            "TABLE_SCHEMA",
+            "fields",
+            "declared fields do not exactly match parsed columns",
+        )
+    if any(field.source_id != source_id for field in fields):
+        raise PreparationError("TABLE_SOURCE", "fields", "field source IDs do not match")
+    lineage = tuple(
+        ColumnLineage(
+            output_column=field.name,
+            inputs=(
+                LineageInput(
+                    source_id=source_id,
+                    snapshot_digest=parsed.input_sha256,
+                    column=field.name,
+                ),
+            ),
+            operation_ids=(),
+        )
+        for field in fields
+    )
+    return TableState(
+        table_id=source_id,
+        snapshot_digests=(parsed.input_sha256,),
+        fields=fields,
+        rows=parsed.rows,
+        lineage=lineage,
+    )
+
+
+def validate_columns(state: TableState, columns: Iterable[str], path: str) -> tuple[int, ...]:
+    indexes: list[int] = []
+    seen: set[str] = set()
+    for column in columns:
+        if column in seen:
+            raise PreparationError("TABLE_COLUMN", path, "contains duplicate columns")
+        seen.add(column)
+        indexes.append(state.index(column))
+    if not indexes:
+        raise PreparationError("TABLE_COLUMN", path, "must contain at least one column")
+    return tuple(indexes)
+
+
+def _validate_cell(value: Any, field: FieldSpec, path: str) -> None:
+    if value is None:
+        if not field.nullable:
+            raise PreparationError("TABLE_NULL", path, f"{field.name} is not nullable")
+        return
+    valid = False
+    if field.logical_type == "string":
+        valid = (
+            isinstance(value, str)
+            and len(value.encode("utf-8", errors="surrogatepass")) <= 1_048_576
+            and not any(0xD800 <= ord(character) <= 0xDFFF for character in value)
+        )
+    elif field.logical_type == "int64":
+        valid = type(value) is int and -(1 << 63) <= value <= (1 << 63) - 1
+    elif field.logical_type == "float64":
+        valid = type(value) in {int, float} and math.isfinite(float(value))
+    elif field.logical_type == "boolean":
+        valid = type(value) is bool
+    elif field.logical_type == "date":
+        if isinstance(value, str):
+            try:
+                valid = date.fromisoformat(value).isoformat() == value and len(value) == 10
+            except ValueError:
+                valid = False
+    elif field.logical_type == "timestamp_utc":
+        if isinstance(value, str):
+            try:
+                parse_utc(value, path)
+                valid = True
+            except PreparationError:
+                valid = False
+    if not valid:
+        raise PreparationError(
+            "TABLE_CELL_TYPE",
+            path,
+            f"value does not satisfy declared {field.logical_type}",
+        )
+
+
+__all__ = ["TableState", "table_from_parsed", "validate_columns"]