datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/engine/reports.py

@@ -0,0 +1,206 @@
+"""Report assembly (03 §4 stage 8, 06 §3.5, 05 §7).
+
+Builds the sectioned ``Report`` payload the Results screen and the ``reports``
+table bind to. Pure engine code — no DB/web imports. The sections the engine can
+honestly produce are populated:
+
+* ``compliance_score`` / ``distribution`` — from the KS compliance pass.
+* ``correlation`` — Pearson matrix over numeric columns (cheap, honest).
+* ``mutual_information`` — pairwise MI (nats) over discretized columns (05 §7).
+* ``causal_truth`` — the true generating DAG + interventions (P2).
+* ``failures`` — per-mode realized diffs when failures were injected (P3).
+* ``difficulty`` — target band, achieved metric, probe, iterations, knobs (P4).
+* ``determinism`` — spec_hash, seed, per-namespace key digests, checksums.
+
+Sections the engine cannot honestly produce for a given run stay ``None``; the
+schema is stable so the UI is coherent from day one.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pandas as pd
+
+from .causal.execute import resolve_interventions
+from .dist import ComplianceReport
+from .profile import build_profile
+
+if TYPE_CHECKING:
+    from .causal.graph import CausalDag
+    from .spec.models import CausalGraph, Spec
+
+# Columns with more distinct values than this are treated as free-text / id-like
+# and excluded from the mutual-information matrix (binning would be meaningless).
+_MI_MAX_CARDINALITY = 50
+_MI_NUMERIC_BINS = 10
+
+
+@dataclass
+class ReportBundle:
+    compliance_score: float
+    distribution: dict[str, Any]
+    correlation: dict[str, Any] | None = None
+    mutual_information: dict[str, Any] | None = None
+    causal_truth: dict[str, Any] | None = None
+    difficulty: dict[str, Any] | None = None
+    failures: dict[str, Any] | None = None
+    profile: dict[str, Any] | None = None
+    determinism: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def correlation_pearson(frame: pd.DataFrame) -> dict[str, Any] | None:
+    """Pearson correlation over numeric columns, JSON-serializable.
+
+    Returns ``None`` when fewer than two numeric columns exist (a matrix would
+    be degenerate). NaNs (e.g. a constant column) are rendered as ``None``.
+    """
+    numeric = frame.select_dtypes(include=[np.number])
+    if numeric.shape[1] < 2:
+        return None
+    corr = numeric.corr(method="pearson")
+    matrix = [[None if pd.isna(v) else float(v) for v in row] for row in corr.to_numpy()]
+    return {"method": "pearson", "columns": list(corr.columns), "matrix": matrix}
+
+
+def _discretize(series: pd.Series) -> np.ndarray | None:
+    """Map a column to integer codes for MI; ``None`` if it isn't usable.
+
+    Numeric/datetime columns are quantile-binned; low-cardinality
+    categorical/boolean columns are factorized; free-text/id-like columns
+    (cardinality above the cap) are skipped.
+    """
+    nunique = series.nunique(dropna=True)
+    if nunique < 2:
+        return None
+    if pd.api.types.is_bool_dtype(series) or pd.api.types.is_object_dtype(series) or isinstance(
+        series.dtype, pd.CategoricalDtype
+    ):
+        if nunique > _MI_MAX_CARDINALITY:
+            return None
+        return pd.factorize(series, sort=True)[0]
+    values = series
+    if pd.api.types.is_datetime64_any_dtype(series):
+        values = series.astype("int64")
+    if pd.api.types.is_numeric_dtype(values):
+        bins = min(_MI_NUMERIC_BINS, int(nunique))
+        codes = pd.qcut(values, q=bins, duplicates="drop").cat.codes
+        return codes.to_numpy() if codes.nunique() >= 2 else None
+    return None
+
+
+def _mutual_information(a: np.ndarray, b: np.ndarray) -> float:
+    """MI in nats between two integer-code arrays via the joint histogram."""
+    contingency = pd.crosstab(a, b).to_numpy(dtype=float)
+    n = contingency.sum()
+    if n == 0:
+        return 0.0
+    pxy = contingency / n
+    px = pxy.sum(axis=1, keepdims=True)
+    py = pxy.sum(axis=0, keepdims=True)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        terms = pxy * (np.log(pxy) - np.log(px) - np.log(py))
+    return float(np.nansum(np.where(pxy > 0, terms, 0.0)))
+
+
+def mutual_information_matrix(frame: pd.DataFrame) -> dict[str, Any] | None:
+    """Pairwise mutual information (nats) over discretizable columns (05 §7).
+
+    Returns ``None`` when fewer than two columns are usable. The diagonal is the
+    column's entropy ``H(X) = I(X;X)``. Symmetric by construction.
+    """
+    codes: dict[str, np.ndarray] = {}
+    for col in frame.columns:
+        disc = _discretize(frame[col])
+        if disc is not None:
+            codes[col] = disc
+    cols = list(codes)
+    if len(cols) < 2:
+        return None
+    size = len(cols)
+    matrix = [[0.0] * size for _ in range(size)]
+    for i in range(size):
+        for j in range(i, size):
+            mi = _mutual_information(codes[cols[i]], codes[cols[j]])
+            matrix[i][j] = mi
+            matrix[j][i] = mi
+    return {"method": "histogram", "units": "nats", "columns": cols, "matrix": matrix}
+
+
+def causal_truth(
+    causal: CausalGraph | None, dag: CausalDag | None
+) -> dict[str, Any] | None:
+    """The true generating graph: edges (with params), interventions, topo order.
+
+    Edges whose destination is intervened are reported ``active: false`` — an
+    intervention ``do(X)`` detaches X's incoming edges (05 §3.1).
+    """
+    if causal is None:
+        return None
+    interventions = resolve_interventions(causal.interventions)
+    edges: list[dict[str, Any]] = []
+    for e in causal.edges:
+        edge: dict[str, Any] = {"from": e.src, "to": e.dst, "fn": e.fn}
+        for key, val in (
+            ("weight", e.weight),
+            ("bias", e.bias),
+            ("coeffs", e.coeffs),
+            ("mapping", e.mapping),
+        ):
+            if val is not None:
+                edge[key] = val
+        edge["active"] = e.dst not in interventions
+        edges.append(edge)
+    return {
+        "nodes": dag.topological_order() if dag is not None else None,
+        "edges": edges,
+        "interventions": interventions,
+        "topological_order": dag.topological_order() if dag is not None else None,
+    }
+
+
+def failures_section(diffs: list[dict[str, Any]] | None) -> dict[str, Any] | None:
+    """Wrap the per-mode failure diffs into the report's ``failures`` section.
+
+    ``None`` when no failures were injected, so the UI can tell "no corruption"
+    apart from "corruption with empty effect".
+    """
+    if not diffs:
+        return None
+    return {"count": len(diffs), "modes": diffs}
+
+
+def build_report(
+    *,
+    compliance: ComplianceReport,
+    frame: pd.DataFrame,
+    determinism: dict[str, Any],
+    spec: Spec | None = None,
+    causal: CausalGraph | None = None,
+    causal_dag: CausalDag | None = None,
+    failures: list[dict[str, Any]] | None = None,
+    injected: pd.DataFrame | None = None,
+    difficulty: dict[str, Any] | None = None,
+) -> ReportBundle:
+    """Assemble the report bundle from the realized frame and compliance pass."""
+    profile = (
+        build_profile(spec, frame, injected=injected, failure_diffs=failures)
+        if spec is not None
+        else None
+    )
+    return ReportBundle(
+        compliance_score=compliance.score,
+        distribution=compliance.to_dict(),
+        correlation=correlation_pearson(frame),
+        mutual_information=mutual_information_matrix(frame),
+        causal_truth=causal_truth(causal, causal_dag),
+        failures=failures_section(failures),
+        profile=profile,
+        difficulty=difficulty,
+        determinism=determinism,
+    )

datadoom/engine/rng.py

@@ -0,0 +1,79 @@
+"""Seeded RNG factory — the determinism invariant (05 §1.2).
+
+All randomness in the engine MUST flow through an :class:`RNGFactory`. No stdlib
+``random``, ``uuid4``, ``time``, or ``np.random.*`` *global* calls are allowed in
+the data path. Each logical stream gets its own independent generator keyed by
+``namespace`` so that adding a feature never perturbs another's draws.
+
+    key(ns) = SHA256(spec_hash || ":" || seed || ":" || ns)[:8] -> uint64
+    RNG(ns) = numpy.random.Generator(PCG64(key(ns)))
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+import numpy as np
+
+# Stable namespace prefixes (documented in 05 §1.2). Helpers below build the
+# full namespace string so call sites stay consistent.
+NS_FEATURE = "feature"
+NS_NOISE = "noise"
+NS_FAILURE = "failure"
+NS_PROBE = "probe"
+NS_DIFFICULTY = "difficulty"
+NS_SHUFFLE = "shuffle"
+
+
+def _derive_key(spec_hash: str, seed: int, namespace: str) -> int:
+    """Derive a uint64 PCG64 key from (spec_hash, seed, namespace)."""
+    payload = f"{spec_hash}:{seed}:{namespace}".encode()
+    digest = hashlib.sha256(payload).digest()
+    # First 8 bytes, big-endian, as an unsigned 64-bit integer.
+    return int.from_bytes(digest[:8], byteorder="big", signed=False)
+
+
+class RNGFactory:
+    """Produces independent, deterministic generators per namespace.
+
+    Two factories with identical ``(spec_hash, seed)`` yield identical draws for
+    the same namespace, and different namespaces are statistically independent.
+    """
+
+    def __init__(self, spec_hash: str, seed: int) -> None:
+        self.spec_hash = spec_hash
+        self.seed = int(seed)
+
+    def key(self, namespace: str) -> int:
+        """Return the raw uint64 key for a namespace (used in determinism reports)."""
+        return _derive_key(self.spec_hash, self.seed, namespace)
+
+    def generator(self, namespace: str) -> np.random.Generator:
+        """Return an independent ``numpy.random.Generator`` for ``namespace``."""
+        return np.random.Generator(np.random.PCG64(self.key(namespace)))
+
+    # Convenience namespace builders -------------------------------------------------
+
+    def feature(self, name: str) -> np.random.Generator:
+        return self.generator(f"{NS_FEATURE}:{name}")
+
+    def noise(self, name: str) -> np.random.Generator:
+        return self.generator(f"{NS_NOISE}:{name}")
+
+    def failure(self, index: int) -> np.random.Generator:
+        return self.generator(f"{NS_FAILURE}:{index}")
+
+    def difficulty(self, name: str) -> np.random.Generator:
+        """Stream for difficulty-calibration perturbations (feature/label noise)."""
+        return self.generator(f"{NS_DIFFICULTY}:{name}")
+
+    def probe(self, name: str) -> np.random.Generator:
+        """Stream for difficulty probe-model seeds (train/test split, estimator)."""
+        return self.generator(f"{NS_PROBE}:{name}")
+
+    def shuffle(self) -> np.random.Generator:
+        return self.generator(NS_SHUFFLE)
+
+    def key_digests(self, namespaces: list[str]) -> dict[str, str]:
+        """Hex key digests for the determinism report section (06 §3.5)."""
+        return {ns: format(self.key(ns), "016x") for ns in namespaces}

datadoom/engine/spec/__init__.py

@@ -0,0 +1,45 @@
+"""Spec parsing, validation, canonicalization and hashing."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import yaml
+
+from ..errors import SpecValidationError
+from .hashing import canonical_json, spec_hash
+from .models import Spec
+from .validate import validate_spec
+
+__all__ = [
+    "Spec",
+    "validate_spec",
+    "canonical_json",
+    "spec_hash",
+    "load_spec",
+    "parse_spec",
+]
+
+
+def parse_spec(data: dict[str, Any]) -> Spec:
+    """Parse a raw dict into a validated :class:`Spec` (shape + cross-field)."""
+    from pydantic import ValidationError
+
+    try:
+        spec = Spec.model_validate(data)
+    except ValidationError as exc:
+        # Surface the first error with a dotted locator the UI/CLI can use.
+        first = exc.errors()[0]
+        locator = ".".join(str(p) for p in first["loc"])
+        raise SpecValidationError(first["msg"], locator=locator) from exc
+    validate_spec(spec)
+    return spec
+
+
+def load_spec(path: str) -> Spec:
+    """Load and validate a spec from a YAML (or JSON) file."""
+    with open(path, encoding="utf-8") as fh:
+        data = yaml.safe_load(fh)
+    if not isinstance(data, dict):
+        raise SpecValidationError("spec file must be a mapping at the top level")
+    return parse_spec(data)

datadoom/engine/spec/hashing.py

@@ -0,0 +1,57 @@
+"""Canonical serialization & spec hashing (05 §1.1).
+
+The spec hash is the identity of a dataset's *design*. It must be stable across
+machines and runs, so we define a strict canonical JSON form:
+
+- object keys sorted lexicographically,
+- no insignificant whitespace,
+- numbers normalized (integral floats collapse to ints; shortest float repr),
+- arrays preserve author order (order is semantic),
+- the ``seed`` field is excluded (seed is not part of the design identity).
+
+    spec_hash = SHA256(canonical_json(spec_without_seed))   # hex
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+
+def _normalize(value: Any) -> Any:
+    """Recursively normalize numbers and drop nothing else.
+
+    Integral floats (``1.0``) become ints (``1``) so that a value authored as an
+    int and one authored as a float-but-whole hash identically.
+    """
+    if isinstance(value, bool):
+        # bool is a subclass of int — keep it as-is (True/False -> JSON booleans).
+        return value
+    if isinstance(value, float):
+        if value.is_integer():
+            return int(value)
+        return value
+    if isinstance(value, dict):
+        return {k: _normalize(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_normalize(v) for v in value]
+    return value
+
+
+def canonical_json(spec_body: dict[str, Any]) -> str:
+    """Return the canonical JSON string for a spec dict (``seed`` removed)."""
+    body = {k: v for k, v in spec_body.items() if k != "seed"}
+    normalized = _normalize(body)
+    return json.dumps(
+        normalized,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+        allow_nan=False,
+    )
+
+
+def spec_hash(spec_body: dict[str, Any]) -> str:
+    """Compute the hex SHA256 of the canonical (seed-excluded) spec."""
+    return hashlib.sha256(canonical_json(spec_body).encode("utf-8")).hexdigest()

datadoom/engine/spec/models.py

@@ -0,0 +1,238 @@
+"""Pydantic v2 spec models — the parsed, validated form of doc 04.
+
+These are pure (no DB/framework imports). Shape/type validation happens here;
+cross-entity semantic checks (acyclicity, references) live in ``validate.py``.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from .hashing import canonical_json, spec_hash
+
+FEATURE_NAME_PATTERN = r"^[A-Za-z_][A-Za-z0-9_]*$"
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+# --- Feature definitions (04 §4) -----------------------------------------------------
+
+
+class _FeatureBase(_Model):
+    """Fields shared by every feature type.
+
+    ``emit=False`` marks a **latent** feature: it is sampled / computed and may
+    drive the causal SEM (and appears in the true causal graph), but it is **not
+    shipped** — excluded from the output CSV, the difficulty probe, compliance,
+    and correlation/MI. It models hidden variables (unobserved confounders, a
+    latent score that generates an observed label). The default is ``None`` ==
+    emitted; only an explicit ``False`` is canonicalized, so adding the field
+    never changes the hash of an existing spec (invariant #5/#6).
+    """
+
+    description: str | None = None
+    emit: bool | None = None
+
+
+class NumericFeature(_FeatureBase):
+    type: Literal["numeric"]
+    dist: str | None = None  # None => derived via causal
+    params: dict[str, float] = Field(default_factory=dict)
+    min: float | None = None
+    max: float | None = None
+    dtype: Literal["int", "float"] = "float"
+
+
+class CategoricalFeature(_FeatureBase):
+    type: Literal["categorical"]
+    categories: list[str] = Field(min_length=1)
+    weights: list[float] | None = None
+
+    @field_validator("weights")
+    @classmethod
+    def _weights_nonneg(cls, v: list[float] | None) -> list[float] | None:
+        if v is not None and any(w < 0 for w in v):
+            raise ValueError("categorical weights must be non-negative")
+        return v
+
+
+class BooleanFeature(_FeatureBase):
+    type: Literal["boolean"]
+    rate: float = 0.5
+
+    @field_validator("rate")
+    @classmethod
+    def _rate_in_unit(cls, v: float) -> float:
+        if not 0.0 <= v <= 1.0:
+            raise ValueError("boolean rate must be in [0, 1]")
+        return v
+
+
+class DatetimeFeature(_FeatureBase):
+    type: Literal["datetime"]
+    start: str
+    end: str
+    granularity: Literal["second", "minute", "hour", "day"] = "day"
+    dist: str = "uniform"
+
+
+class TextFeature(_FeatureBase):
+    type: Literal["text"]
+    # "lorem" emits filler words; any realistic provider key (name, email,
+    # address, company, …) is served by mimesis. See dist/providers.py.
+    generator: str = "lorem"
+    locale: str = "en"
+    length: dict[str, int] = Field(default_factory=lambda: {"min": 5, "max": 30})
+
+
+class TrendSpec(_Model):
+    """Linear trend ``slope·t + intercept`` over the row index."""
+
+    slope: float = 0.0
+    intercept: float = 0.0
+
+
+class SeasonalitySpec(_Model):
+    """One sinusoidal season ``amplitude·sin(2π·t/period + phase)``."""
+
+    amplitude: float
+    period: float = Field(gt=0.0)
+    phase: float = 0.0
+
+
+class TimeseriesFeature(_FeatureBase):
+    """An ordered series ``Xₜ = T(t) + S(t) + AR(p) + εₜ`` (05 §6).
+
+    Generated over the row index (row order is preserved end-to-end). It is a
+    *root* feature — it may be a causal **parent** but is never a causal target,
+    and it is not assessed by distribution compliance (KS/GoF don't apply to a
+    trended, autocorrelated series).
+    """
+
+    type: Literal["timeseries"]
+    trend: TrendSpec | None = None
+    seasonality: list[SeasonalitySpec] = Field(default_factory=list)
+    ar: list[float] = Field(default_factory=list)  # AR coefficients φ₁…φ_p
+    noise_std: float = Field(default=1.0, ge=0.0)  # σ of εₜ ~ N(0, σ²)
+    min: float | None = None
+    max: float | None = None
+    dtype: Literal["int", "float"] = "float"
+
+
+Feature = Annotated[
+    NumericFeature
+    | CategoricalFeature
+    | BooleanFeature
+    | DatetimeFeature
+    | TextFeature
+    | TimeseriesFeature,
+    Field(discriminator="type"),
+]
+
+
+# --- Causal graph (04 §5) ------------------------------------------------------------
+
+
+class CausalEdge(_Model):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    src: str = Field(alias="from")
+    dst: str = Field(alias="to")
+    fn: str  # linear | logistic | polynomial | map | identity | <plugin>
+    weight: float | None = None
+    bias: float | None = None
+    coeffs: list[float] | None = None
+    mapping: dict[str, float] | None = None
+
+
+class CausalGraph(_Model):
+    edges: list[CausalEdge] = Field(default_factory=list)
+    noise: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    interventions: list[dict[str, Any]] = Field(default_factory=list)
+
+
+# --- Difficulty / failures / export (04 §6-8) ----------------------------------------
+
+
+class Difficulty(_Model):
+    target: str | dict[str, Any]
+    label: str
+    probe: str = "logreg"
+    max_iters: int = Field(default=8, ge=1)
+    # Lean-default knobs: feature-observation noise (primary) + label flips
+    # (deep end). `causal` shrink / `imbalance` are planned (status.md backlog).
+    knobs: list[str] = Field(default_factory=lambda: ["noise", "label_noise"])
+
+
+class Failure(BaseModel):
+    # type-specific fields are validated by the FailureMode handler (P3).
+    model_config = ConfigDict(extra="allow")
+
+    type: str
+
+
+ExportVersion = Literal["clean", "injected"]
+
+
+def _default_versions() -> list[ExportVersion]:
+    return ["clean"]
+
+
+class ExportSpec(_Model):
+    formats: list[str] = Field(default_factory=lambda: ["csv"])
+    versions: list[ExportVersion] = Field(default_factory=_default_versions)
+    splits: dict[str, float] | None = None
+    shuffle: bool = True
+    metadata: bool = True
+
+
+# --- Top-level spec (04 §2) ----------------------------------------------------------
+
+
+class Spec(_Model):
+    datadoom_version: str
+    name: str
+    description: str | None = None
+    seed: int | None = None
+    rows: int = Field(ge=1)
+    features: dict[str, Feature]
+    causal: CausalGraph | None = None
+    difficulty: Difficulty | None = None
+    failures: list[Failure] = Field(default_factory=list)
+    export: ExportSpec = Field(default_factory=ExportSpec)
+    meta: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("name")
+    @classmethod
+    def _name_slug(cls, v: str) -> str:
+        import re
+
+        if not re.match(r"^[A-Za-z0-9_-]+$", v):
+            raise ValueError("name must be slug-friendly ([A-Za-z0-9_-]+)")
+        return v
+
+    # --- Canonical form & identity (05 §1) -------------------------------------------
+
+    def body(self) -> dict[str, Any]:
+        """Serializable spec document (aliases applied, None fields dropped)."""
+        return self.model_dump(mode="json", by_alias=True, exclude_none=True)
+
+    def canonical(self) -> str:
+        """Canonical JSON string (seed excluded) used for hashing."""
+        return canonical_json(self.body())
+
+    def spec_hash(self) -> str:
+        """sha256 of the canonical, seed-excluded spec."""
+        return spec_hash(self.body())
+
+    def latent_names(self) -> set[str]:
+        """Feature names marked ``emit: false`` (computed but not shipped)."""
+        return {name for name, feat in self.features.items() if feat.emit is False}
+
+    def emitted_names(self) -> list[str]:
+        """Feature names that are shipped, in spec order."""
+        return [name for name, feat in self.features.items() if feat.emit is not False]