qavf 0.4.0__py3-none-any.whl

qavf/__init__.py

@@ -0,0 +1,128 @@
+"""QAVF: explicit QA contracts for files, frames and queries.
+
+Quick start::
+
+    from qavf import validate
+
+    report = validate("counts.parquet", schema={"columns": {"count": {"ge": 0}}})
+    report.write_json("qa_report.json")
+
+Public objects are imported when requested, so importing QAVF's version does
+not initialize dataframe, regression or report-rendering dependencies.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from qavf._version import __version__
+
+if TYPE_CHECKING:
+    from qavf.api import validate
+    from qavf.engine.compilation import ConfigurationError, build_registry
+    from qavf.engine.context import DataContext
+    from qavf.engine.planning import DataRequirements
+    from qavf.engine.registry import Finding, Rule, RuleParams, RuleRegistry, register
+    from qavf.engine.runner import QARunner
+    from qavf.engine.severity import Severity, Status
+    from qavf.execution.errors import ExecutionLimitError
+    from qavf.inputs.descriptors import Dataset, File, Loader, Query
+    from qavf.inputs.resources import ResourceLimits
+    from qavf.profiling import ProfileResult, profile_input
+    from qavf.regression.compare_outputs import (
+        assert_outputs_match,
+        compare_frames,
+        compare_outputs,
+    )
+    from qavf.regression.models import ComparisonResult
+    from qavf.reports.html import render_html, write_html
+    from qavf.reports.models import QAReport, RuleResult
+    from qavf.settings import QAVFSettings, load_settings
+    from qavf.sources import DataSource, SourceParams
+
+_EXPORTS = {
+    "ComparisonResult": "qavf.regression.models",
+    "ConfigurationError": "qavf.engine.compilation",
+    "DataContext": "qavf.engine.context",
+    "DataRequirements": "qavf.engine.planning",
+    "DataSource": "qavf.sources",
+    "Dataset": "qavf.inputs.descriptors",
+    "ExecutionLimitError": "qavf.execution.errors",
+    "File": "qavf.inputs.descriptors",
+    "Finding": "qavf.engine.registry",
+    "Loader": "qavf.inputs.descriptors",
+    "ProfileResult": "qavf.profiling",
+    "QAReport": "qavf.reports.models",
+    "QARunner": "qavf.engine.runner",
+    "QAVFSettings": "qavf.settings",
+    "Query": "qavf.inputs.descriptors",
+    "ResourceLimits": "qavf.inputs.resources",
+    "Rule": "qavf.engine.registry",
+    "RuleParams": "qavf.engine.registry",
+    "RuleRegistry": "qavf.engine.registry",
+    "RuleResult": "qavf.reports.models",
+    "Severity": "qavf.engine.severity",
+    "SourceParams": "qavf.sources",
+    "Status": "qavf.engine.severity",
+    "assert_outputs_match": "qavf.regression.compare_outputs",
+    "build_registry": "qavf.engine.compilation",
+    "compare_frames": "qavf.regression.compare_outputs",
+    "compare_outputs": "qavf.regression.compare_outputs",
+    "load_settings": "qavf.settings",
+    "profile_input": "qavf.profiling",
+    "register": "qavf.engine.registry",
+    "render_html": "qavf.reports.html",
+    "validate": "qavf.api",
+    "write_html": "qavf.reports.html",
+}
+
+__all__ = [
+    "ComparisonResult",
+    "ConfigurationError",
+    "DataContext",
+    "DataRequirements",
+    "DataSource",
+    "Dataset",
+    "ExecutionLimitError",
+    "File",
+    "Finding",
+    "Loader",
+    "ProfileResult",
+    "QAReport",
+    "QARunner",
+    "QAVFSettings",
+    "Query",
+    "ResourceLimits",
+    "Rule",
+    "RuleParams",
+    "RuleRegistry",
+    "RuleResult",
+    "Severity",
+    "SourceParams",
+    "Status",
+    "__version__",
+    "assert_outputs_match",
+    "build_registry",
+    "compare_frames",
+    "compare_outputs",
+    "load_settings",
+    "profile_input",
+    "register",
+    "render_html",
+    "validate",
+    "write_html",
+]
+
+
+def __getattr__(name: str) -> Any:
+    module = _EXPORTS.get(name)
+    if module is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    value = getattr(import_module(module), name)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted(set(globals()) | set(__all__))

qavf/_audit_temporal.py

@@ -0,0 +1,43 @@
+"""Lossless admission of optional temporal scalar implementations.
+
+The audit format has native NumPy temporal tags and Python datetime tags.
+Choose one before subclass checks can silently discard nanoseconds. This
+module never imports pandas, so core validation keeps its optional dependency.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+
+def normalize_temporal_scalar(value: Any) -> Any:
+    """Map pandas scalars to existing lossless tags, or reject unsupported values.
+
+    Naive timestamps and durations retain their native NumPy units. Zoned
+    timestamps use Python datetime only when microseconds represent them
+    exactly, retaining the zone and DST fold. The existing format cannot
+    represent a named zone together with a nanosecond remainder.
+    """
+    pandas = sys.modules.get("pandas")
+    if pandas is None:
+        return value
+    # Another thread can be importing the optional adapter. Its module enters
+    # sys.modules before these public classes have been assigned.
+    if value is getattr(pandas, "NaT", None):
+        raise ValueError("audit pandas NaT is ambiguous; use None or a typed NumPy NaT")
+    duration_type = getattr(pandas, "Timedelta", None)
+    timestamp_type = getattr(pandas, "Timestamp", None)
+    scalar: Any = value
+    if isinstance(duration_type, type) and isinstance(value, duration_type):
+        return scalar.to_timedelta64()
+    if isinstance(timestamp_type, type) and isinstance(value, timestamp_type):
+        if scalar.tzinfo is None:
+            return scalar.to_datetime64()
+        if scalar.nanosecond:
+            raise ValueError(
+                "audit pandas Timestamp with both a timezone and nanosecond precision "
+                "is unsupported"
+            )
+        return scalar.to_pydatetime()
+    return value

qavf/_float_accumulator.py

@@ -0,0 +1,52 @@
+"""Mergeable exact accumulation of stored binary floating-point observations."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable
+from dataclasses import dataclass
+from fractions import Fraction
+
+
+@dataclass(slots=True)
+class ExactFloatAccumulator:
+    """Retain an integer total and count, independent of input/chunk ordering.
+
+    Every Float64 is an integer multiple of 2**-1074. Storage grows only with
+    the bit length of the total and count; no observation buffer is retained.
+    Null/NaN observations are missing; infinities cannot be aggregated.
+    """
+
+    units: int = 0
+    count: int = 0
+
+    def update(self, values: Iterable[float | None]) -> None:
+        units, count = self.units, self.count
+        for value in values:
+            if value is None or math.isnan(value):
+                continue
+            if not math.isfinite(value):
+                raise ValueError("non-finite observations cannot be aggregated")
+            numerator, denominator = float(value).as_integer_ratio()
+            units += numerator << (1074 - (denominator.bit_length() - 1))
+            count += 1
+        self.units, self.count = units, count
+
+    def total(self) -> Fraction | None:
+        return Fraction(self.units, 1 << 1074) if self.count else None
+
+    def update_counted(self, values: Iterable[tuple[float, int]]) -> None:
+        """Accumulate exact value frequencies without expanding repeated observations."""
+        units, count = self.units, self.count
+        for value, frequency in values:
+            if isinstance(frequency, bool) or not isinstance(frequency, int) or frequency < 0:
+                raise ValueError("observation frequencies must be nonnegative integers")
+            if not math.isfinite(value):
+                raise ValueError("non-finite observations cannot be aggregated")
+            numerator, denominator = float(value).as_integer_ratio()
+            units += (numerator << (1074 - (denominator.bit_length() - 1))) * frequency
+            count += frequency
+        self.units, self.count = units, count
+
+    def mean(self) -> Fraction | None:
+        return Fraction(self.units, (1 << 1074) * self.count) if self.count else None

qavf/_observations.py

@@ -0,0 +1,71 @@
+"""Lossless admission of caller observations before Float64 statistical kernels."""
+
+from __future__ import annotations
+
+import math
+from decimal import Decimal
+
+import numpy as np
+import numpy.typing as npt
+
+
+def _scalar(value: object) -> float:
+    """Validate the original scalar, before mixed-container dtype inference."""
+    if value is None:
+        return math.nan
+    if isinstance(value, bool | np.bool_):
+        raise ValueError("observations must be numeric, not booleans")
+    if isinstance(value, float):
+        return float(value)  # Already stored at Float64 precision, including NaN/inf.
+    if not isinstance(value, int | float | np.integer | np.floating | Decimal):
+        raise ValueError("observations must be numeric, not coercible text or objects")
+    try:
+        converted = float(value)
+    except (OverflowError, ValueError) as exc:
+        raise ValueError("observations cannot be represented exactly as Float64") from exc
+    if isinstance(value, int | np.integer):
+        exact = math.isfinite(converted) and int(converted) == value
+    elif isinstance(value, Decimal):
+        exact = not value.is_finite() or (
+            math.isfinite(converted) and Decimal.from_float(converted) == value
+        )
+    else:
+        # Preserve the original NumPy precision: converting the source to Python
+        # float before comparing would erase the very difference being tested.
+        exact = not np.isfinite(value) or (
+            math.isfinite(converted) and type(value)(converted) == value
+        )
+    if not exact:
+        raise ValueError("observations cannot be represented exactly as Float64")
+    return converted
+
+
+def float_observations(values: npt.ArrayLike) -> npt.NDArray[np.float64]:
+    """Return a 1-D Float64 view/copy only after original values prove lossless.
+
+    Homogeneous NumPy arrays retain vectorized admission. Other array-like inputs
+    first retain their original scalar objects, so a mixed float/integer list
+    cannot round integers or disguise booleans during dtype inference.
+    """
+    raw = values if isinstance(values, np.ndarray) else np.asarray(values, dtype=object)
+    if raw.ndim != 1:
+        raise ValueError(f"values must be 1-D, got an array with {raw.ndim} dimension(s)")
+    if raw.dtype.kind not in "fiuO":
+        raise ValueError("observations must be real numeric values")
+    if raw.dtype.kind == "O":
+        with np.errstate(over="ignore", invalid="ignore"):
+            return np.fromiter((_scalar(item) for item in raw), dtype=np.float64, count=len(raw))
+    with np.errstate(over="ignore", invalid="ignore"):
+        converted = np.asarray(raw, dtype=np.float64)
+        if raw.dtype.kind in "iu":
+            # A value that rounds to the integer type's limit is not exact, and the
+            # out-of-range cast back is architecture dependent (x86 wraps, ARM
+            # saturates to the original value), so reject it before casting.
+            limit = 2.0 ** (raw.dtype.itemsize * 8 - (raw.dtype.kind == "i"))
+            if np.any(converted >= limit) or not np.array_equal(
+                raw, converted.astype(raw.dtype)
+            ):
+                raise ValueError("integer observations cannot be represented exactly as Float64")
+        elif raw.dtype.itemsize > 8 and np.any(np.isfinite(raw) & (raw != converted)):
+            raise ValueError("observations cannot be represented exactly as Float64")
+    return converted

qavf/_version.py

@@ -0,0 +1,3 @@
+"""Single source of the package version."""
+
+__version__ = "0.4.0"

qavf/aggregation.py

@@ -0,0 +1,84 @@
+"""Exact accumulator selection and typed reductions shared by execution backends."""
+
+from __future__ import annotations
+
+from decimal import Decimal, localcontext
+
+import polars as pl
+
+from qavf.numeric import exact_number, stable_sum_scalar
+
+
+def accumulator_dtype(dtype: pl.DataType) -> pl.DataType:
+    """Storage for integer/Decimal sums, retaining the input decimal scale."""
+    return pl.Decimal(38, dtype.scale or 0) if isinstance(dtype, pl.Decimal) else pl.Int128()
+
+
+def accumulator_sql(dtype: pl.DataType) -> str:
+    """Native SQL storage with the same width and scale as typed accumulation."""
+    storage = accumulator_dtype(dtype)
+    return f"DECIMAL(38,{storage.scale})" if isinstance(storage, pl.Decimal) else "HUGEINT"
+
+
+def checked_accumulator_dtype(
+    dtype: pl.DataType,
+    minimum: object,
+    maximum: object,
+    count: int,
+) -> pl.DataType:
+    """Prove a conservative sum bound from scalar metadata, without reading rows.
+
+    Callers supply their existing bound on contributing rows. This guard deliberately
+    rejects a potentially overflowing sum even when cancellation could make it fit.
+    """
+    if not (minimum is None or isinstance(minimum, int | float | Decimal)) or not (
+        maximum is None or isinstance(maximum, int | float | Decimal)
+    ):
+        raise TypeError("numeric aggregation returned a nonnumeric extremum")
+    magnitude = max(exact_number(minimum or 0).copy_abs(), exact_number(maximum or 0).copy_abs())
+    storage = accumulator_dtype(dtype)
+    with localcontext() as context:
+        context.prec = 100
+        ceiling = (
+            Decimal(10) ** (38 - (storage.scale or 0))
+            if isinstance(storage, pl.Decimal)
+            else 2**127
+        )
+        if magnitude and magnitude * count >= ceiling:
+            raise ValueError("exact sum may exceed accumulator precision; partition the data")
+    return storage
+
+
+def numeric_expr(column: str, dtype: pl.DataType) -> pl.Expr:
+    """Preserve exact integers/decimals and normalize floating missing values."""
+    value = pl.col(column)
+    return value.fill_nan(None) if dtype.is_float() else value
+
+
+def exact_sum_expr(series: pl.Series) -> pl.Expr:
+    """Overflow-checked sum; null-only input remains an undefined measurement."""
+    dtype = series.dtype
+    value = numeric_expr(series.name, dtype)
+    if dtype.is_float():
+        return value.map_batches(stable_sum_scalar, return_dtype=pl.Float64, returns_scalar=True)
+    if dtype.is_integer() or dtype.is_decimal():
+        value = value.cast(
+            checked_accumulator_dtype(dtype, series.min(), series.max(), series.len())
+        )
+    return pl.when(value.count() > 0).then(value.sum()).otherwise(None)
+
+
+def narrow_integer(frame: pl.DataFrame, column: str) -> pl.DataFrame:
+    """Keep ordinary integer results compact, retaining Int128 when required."""
+    if frame.schema[column] != pl.Int128:
+        return frame
+    lo, hi = frame[column].min(), frame[column].max()
+    if lo is None:
+        return frame.with_columns(pl.col(column).cast(pl.Int64))
+    if not isinstance(lo, int) or not isinstance(hi, int):
+        raise TypeError("integer aggregation returned a noninteger extremum")
+    if lo >= -(2**63) and hi < 2**63:
+        return frame.with_columns(pl.col(column).cast(pl.Int64))
+    if lo >= 0 and hi < 2**64:
+        return frame.with_columns(pl.col(column).cast(pl.UInt64))
+    return frame

qavf/api.py

@@ -0,0 +1,34 @@
+"""Public validation orchestration over explicit input descriptions."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from qavf.engine.runner import QARunner
+from qavf.inputs.normalize import Configuration, prepare
+from qavf.inputs.resources import ResourceLimits
+from qavf.reports.models import QAReport
+from qavf.settings import SchemaSpec
+
+
+def validate(
+    data: object | Mapping[str, object] | None = None,
+    *,
+    schema: SchemaSpec | Mapping[str, Any] | None = None,
+    config: Configuration = None,
+    resources: ResourceLimits | None = None,
+) -> QAReport:
+    """Validate explicit contracts and return a report without writing files.
+
+    Pass a path, File, frame, Query, Loader, or named mapping of those inputs.
+    Dataset wraps an input with its own schema, foreign keys, expressions and
+    loading safeguards. Plain mappings always name datasets, never columns.
+    All input names and contracts are checked before adapting or executing data.
+    Lazy frames, deferred loaders and caller-owned handles keep their ownership.
+    """
+    settings, frames = prepare(data, schema=schema, config=config, resources=resources)
+    return QARunner(settings).run(frames=frames)
+
+
+__all__ = ["validate"]