datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/engine/difficulty/knobs.py

@@ -0,0 +1,171 @@
+"""The difficulty dial: knobs that make a dataset harder, baked into the frame.
+
+The adaptive loop (``calibrate.py``) is a 1-D root-find — it can only bisect a
+single monotone scalar. So the lean-default knob set is composed into **one
+bisectable dial** ``d``:
+
+    d ∈ [0, 1):  feature-observation noise η ramps 0 → η_max
+    d ∈ [1, 2]:  η held at η_max, label-flip rate ρ ramps 0 → ρ_max
+
+Feature noise is the primary lever — it adds Gaussian observation noise to the
+numeric predictors while **leaving the authored causal graph untouched** (so the
+``causal_truth`` report stays honest: the label still depends on the true
+signal, the features are just noisier observations of it). Label flipping is the
+deep-end extension used when feature noise saturates before reaching a very low
+target band.
+
+Determinism: the per-column noise draws and the label-flip draws are taken
+*once* (independent of ``d``) and merely *scaled* by the dial. This makes the
+realized frame a continuous, monotone function of ``d`` and guarantees the
+flipped-row set is **nested** as ρ grows — clean monotonicity for the bisection,
+and byte-identical output for a given final dial (invariant #6).
+
+`causal` (coefficient shrink) and `imbalance` are recognized knob *names* in the
+spec but are not active levers in v0.1; the calibrator reports which knobs it
+actually used. See the difficulty backlog in ``status.md``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pandas as pd
+
+if TYPE_CHECKING:
+    from ..rng import RNGFactory
+
+# At η = η_max each numeric predictor receives additive noise with std equal to
+# η_max × the feature's own (clean) standard deviation — a 1.5σ observation
+# blur, enough to wash most linear signal out of a strong feature.
+FEATURE_NOISE_MAX = 1.5
+# Label flips top out at 0.5 — at which point a binary label is pure coin-flip
+# (AUROC → 0.5), the hardest any classification task can be.
+LABEL_FLIP_MAX = 0.5
+# The dial spans the feature-noise region [0,1) then the label-noise region [1,2].
+DIAL_MAX = 2.0
+
+# Knob names the dial actively implements; others are accepted-but-inactive.
+ACTIVE_KNOBS = ("noise", "label_noise")
+
+
+@dataclass
+class KnobState:
+    """The decomposed knob values realized at a given dial position."""
+
+    dial: float
+    feature_noise: float  # realized η
+    label_flip: float  # realized ρ
+
+    def to_dict(self) -> dict[str, float]:
+        return {"dial": self.dial, "feature_noise": self.feature_noise, "label_flip": self.label_flip}
+
+
+class DifficultyDial:
+    """Pre-draws the perturbations once, then realizes any dial position cheaply.
+
+    Holding the draws fixed and scaling by the dial is what makes ``μ(d)``
+    monotone (nested label flips, proportional feature blur) so the calibrator's
+    bisection is well-posed.
+    """
+
+    def __init__(
+        self,
+        base: pd.DataFrame,
+        label: str,
+        rng: RNGFactory,
+        knobs: list[str],
+        used_namespaces: list[str],
+    ) -> None:
+        self.base = base
+        self.label = label
+        self.n = len(base)
+        self.use_feature_noise = "noise" in knobs
+        self.use_label_noise = "label_noise" in knobs
+
+        # Numeric predictors only (booleans/categoricals/datetimes are left to
+        # the label-flip lever; the label column itself is never blurred).
+        self.numeric_cols: list[str] = [
+            c
+            for c in base.columns
+            if c != label
+            and pd.api.types.is_numeric_dtype(base[c])
+            and not pd.api.types.is_bool_dtype(base[c])
+        ]
+        self._z: dict[str, np.ndarray] = {}
+        self._sd: dict[str, float] = {}
+        self._is_int: dict[str, bool] = {}
+        if self.use_feature_noise:
+            for c in self.numeric_cols:
+                col = base[c]
+                self._sd[c] = float(np.nanstd(col.to_numpy(dtype=float)))
+                self._is_int[c] = pd.api.types.is_integer_dtype(col)
+                self._z[c] = rng.difficulty(f"feature:{c}").standard_normal(self.n)
+                used_namespaces.append(f"difficulty:feature:{c}")
+
+        self._u: np.ndarray | None = None
+        if self.use_label_noise:
+            self._u = rng.difficulty("label").random(self.n)
+            used_namespaces.append("difficulty:label")
+
+    # --- dial → knob magnitudes -----------------------------------------------------
+    def feature_noise_at(self, dial: float) -> float:
+        if not self.use_feature_noise:
+            return 0.0
+        return min(dial, 1.0) * FEATURE_NOISE_MAX
+
+    def label_flip_at(self, dial: float) -> float:
+        if not self.use_label_noise:
+            return 0.0
+        return max(0.0, dial - 1.0) * LABEL_FLIP_MAX
+
+    # --- realize a frame at a dial position -----------------------------------------
+    def realize(self, dial: float) -> tuple[pd.DataFrame, KnobState]:
+        eta = self.feature_noise_at(dial)
+        rho = self.label_flip_at(dial)
+        frame = self.base.copy()
+
+        if eta > 0.0:
+            for c in self.numeric_cols:
+                sd = self._sd[c]
+                if sd <= 0.0:
+                    continue  # constant column — nothing to blur
+                noisy = frame[c].to_numpy(dtype=float) + eta * sd * self._z[c]
+                if self._is_int[c]:
+                    frame[c] = np.rint(noisy).astype("int64")  # preserve int dtype
+                else:
+                    frame[c] = noisy
+
+        if rho > 0.0 and self._u is not None:
+            flip = self._u < rho  # nested as ρ grows → monotone
+            frame[self.label] = _flip_label(frame[self.label], flip)
+
+        return frame, KnobState(dial=dial, feature_noise=eta, label_flip=rho)
+
+    def noise_to_signal(self, eta: float) -> float:
+        """Var(ε)/Var(signal) for the feature noise (05 §5.4): equals η²."""
+        return float(eta * eta) if self.numeric_cols else 0.0
+
+
+def _flip_label(series: pd.Series, flip: np.ndarray) -> pd.Series:
+    """Flip the selected label rows to a *different* class.
+
+    Boolean → logical NOT; binary categorical → swap to the other category;
+    k-ary categorical → rotate to the next category (deterministic, always a
+    genuine change). Mirrors the ``label_noise`` failure mode's "flip to a
+    different class" guarantee.
+    """
+    if pd.api.types.is_bool_dtype(series):
+        out = series.to_numpy().copy()
+        out[flip] = ~out[flip]
+        return pd.Series(out, index=series.index)
+
+    out = series.to_numpy().copy()
+    categories = sorted(pd.unique(series.dropna()))
+    if len(categories) < 2:
+        return series  # nothing to flip into
+    nxt = {c: categories[(i + 1) % len(categories)] for i, c in enumerate(categories)}
+    flipped = np.array([nxt.get(v, v) for v in out[flip]], dtype=out.dtype)
+    out[flip] = flipped
+    return pd.Series(out, index=series.index)

datadoom/engine/difficulty/probes.py

@@ -0,0 +1,181 @@
+"""Baseline probe models for difficulty calibration (05 §5.1, 17 step 15).
+
+Difficulty is defined *operationally*: a dataset is "as hard as" the score a
+standard baseline model achieves on it. A :class:`ProbeModel` trains a baseline
+classifier on a seeded train split and reports the task metric on the holdout —
+AUROC for binary classification. The probe never refits or "helps" the data; it
+just measures how separable the label is from the features (honest statistics,
+invariant #3).
+
+scikit-learn powers the estimators. The probe metric drives the adaptive loop's
+knob selection, so it sits on the determinism-critical path — every estimator is
+either intrinsically deterministic (lbfgs logistic regression) or seeded
+(decision tree, train/test split). Same `(spec_hash, seed)` on the pinned
+environment → identical metric → identical calibrated bytes (invariant #6).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import accuracy_score, roc_auc_score
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+from sklearn.tree import DecisionTreeClassifier
+
+# Object columns with more distinct values than this are treated as free-text /
+# id-like and dropped from the probe design matrix (one-hot would explode).
+_MAX_CATEGORY_CARDINALITY = 50
+_TEST_SIZE = 0.3
+
+
+@dataclass
+class ProbeResult:
+    """What a probe measured on one realized frame."""
+
+    metric: float  # the task metric the band targets (AUROC for binary)
+    metric_name: str  # "auroc"
+    task: str  # "classification"
+    linear_separability: float  # holdout accuracy of a linear probe (05 §5.4)
+    class_balance: float  # fraction of the positive class
+    n_features: int  # design-matrix width actually fed to the probe
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "metric": self.metric,
+            "metric_name": self.metric_name,
+            "task": self.task,
+            "linear_separability": self.linear_separability,
+            "class_balance": self.class_balance,
+            "n_features": self.n_features,
+        }
+
+
+class ProbeModel(ABC):
+    """A baseline model whose holdout score *defines* the dataset's difficulty."""
+
+    name: str
+    # Optional JSON-schema fragment for probe options (09 §6); ``None`` for built-ins.
+    param_schema: Mapping[str, Any] | None = None
+
+    @abstractmethod
+    def estimator(self, seed: int) -> Any:
+        """Return a fresh scikit-learn classifier exposing ``predict_proba``."""
+
+
+class LogRegProbe(ProbeModel):
+    name = "logreg"
+
+    def estimator(self, seed: int) -> Any:
+        # lbfgs is deterministic given the data; no random_state needed.
+        return LogisticRegression(max_iter=1000)
+
+
+class TreeProbe(ProbeModel):
+    name = "tree"
+
+    def estimator(self, seed: int) -> Any:
+        # Depth-capped so it stays a *baseline*, not a memorizer; seeded for repro.
+        return DecisionTreeClassifier(max_depth=8, random_state=seed)
+
+
+PROBES: dict[str, ProbeModel] = {p.name: p for p in (LogRegProbe(), TreeProbe())}
+
+
+def _encode_label(series: pd.Series) -> tuple[np.ndarray, int]:
+    """Map a classification label to 0/1 codes; return (codes, n_classes).
+
+    The positive class is the lexicographically larger category (so booleans map
+    True→1) — a stable convention so AUROC orientation is reproducible.
+    """
+    if pd.api.types.is_bool_dtype(series):
+        return series.to_numpy().astype(int), 2
+    codes, uniques = pd.factorize(series, sort=True)
+    return codes.astype(int), len(uniques)
+
+
+def _design_matrix(frame: pd.DataFrame, label: str) -> tuple[np.ndarray, list[str]]:
+    """Build a numeric feature matrix from every column except the label.
+
+    Numeric/boolean columns pass through (median-imputed); low-cardinality
+    categoricals are one-hot encoded; datetimes become ordinals; free-text /
+    high-cardinality columns are dropped (uninformative to a baseline probe).
+    """
+    parts: list[pd.DataFrame] = []
+    for col in frame.columns:
+        if col == label:
+            continue
+        s = frame[col]
+        if pd.api.types.is_bool_dtype(s):
+            parts.append(s.astype(float).to_frame(col))
+        elif pd.api.types.is_numeric_dtype(s):
+            filled = s.astype(float)
+            median = filled.median()
+            parts.append(filled.fillna(0.0 if pd.isna(median) else median).to_frame(col))
+        elif pd.api.types.is_datetime64_any_dtype(s):
+            parts.append(s.astype("int64").astype(float).to_frame(col))
+        else:  # object / categorical
+            if s.nunique(dropna=True) <= _MAX_CATEGORY_CARDINALITY:
+                dummies = pd.get_dummies(s, prefix=col, dummy_na=False, dtype=float)
+                if dummies.shape[1] > 0:
+                    parts.append(dummies)
+    if not parts:
+        return np.empty((len(frame), 0), dtype=float), []
+    design = pd.concat(parts, axis=1)
+    return design.to_numpy(dtype=float), list(design.columns)
+
+
+def evaluate(
+    probe: ProbeModel,
+    frame: pd.DataFrame,
+    label: str,
+    *,
+    split_seed: int,
+    est_seed: int,
+) -> ProbeResult:
+    """Train ``probe`` on a seeded split and score it on the holdout (05 §5.1).
+
+    Binary classification only in v0.1: returns holdout AUROC. Degenerate cases
+    (no usable features, a single realized class) score at chance (0.5) — honest
+    "no signal" rather than a misleading number or a crash.
+    """
+    X, feat_names = _design_matrix(frame, label)
+    y, _ = _encode_label(frame[label])
+    n_features = X.shape[1]
+    positive_rate = float(np.mean(y == 1)) if len(y) else 0.0
+
+    classes = np.unique(y)
+    if n_features == 0 or classes.size < 2:
+        # Nothing to learn from, or label collapsed to one class → chance.
+        return ProbeResult(0.5, "auroc", "classification", 0.5, positive_rate, n_features)
+
+    # Stratify only when every class can appear on both sides of the split.
+    counts = np.bincount(y)
+    stratify = y if counts.min() >= 2 else None
+    X_tr, X_te, y_tr, y_te = train_test_split(
+        X, y, test_size=_TEST_SIZE, random_state=split_seed, stratify=stratify
+    )
+    if np.unique(y_te).size < 2 or np.unique(y_tr).size < 2:
+        return ProbeResult(0.5, "auroc", "classification", 0.5, positive_rate, n_features)
+
+    scaler = StandardScaler().fit(X_tr)
+    X_tr_s = scaler.transform(X_tr)
+    X_te_s = scaler.transform(X_te)
+
+    model = probe.estimator(est_seed)
+    model.fit(X_tr_s, y_tr)
+    proba = model.predict_proba(X_te_s)[:, 1]
+    auroc = float(roc_auc_score(y_te, proba))
+
+    # Linear-separability reference (05 §5.4): a plain logistic probe's holdout
+    # accuracy, regardless of which probe scored the AUROC.
+    linear = LogisticRegression(max_iter=1000).fit(X_tr_s, y_tr)
+    lin_acc = float(accuracy_score(y_te, linear.predict(X_te_s)))
+
+    return ProbeResult(auroc, "auroc", "classification", lin_acc, positive_rate, n_features)

datadoom/engine/dist/__init__.py

@@ -0,0 +1,35 @@
+"""Distributions + honest compliance reporting."""
+
+from __future__ import annotations
+
+from .base import Distribution
+from .builtins import (
+    REGISTRY,
+    sample_boolean,
+    sample_categorical,
+    sample_datetime,
+    sample_text,
+)
+from .compliance import ComplianceReport, FeatureCompliance, assess_numeric
+from .providers import (
+    REALISTIC_GENERATORS,
+    is_realistic_generator,
+    resolve_locale,
+    sample_provider,
+)
+
+__all__ = [
+    "Distribution",
+    "REGISTRY",
+    "ComplianceReport",
+    "FeatureCompliance",
+    "assess_numeric",
+    "sample_boolean",
+    "sample_categorical",
+    "sample_datetime",
+    "sample_text",
+    "sample_provider",
+    "is_realistic_generator",
+    "resolve_locale",
+    "REALISTIC_GENERATORS",
+]

datadoom/engine/dist/base.py

@@ -0,0 +1,46 @@
+"""Distribution ABC (05 §2).
+
+A :class:`Distribution` knows how to *sample* from a target ``D(theta)`` using an
+injected RNG, *validate* its parameters, and expose its theoretical *cdf* so the
+compliance layer can run a KS test. Sampling is correct by construction — we
+never refit parameters to the realized sample (05 §2.3).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Mapping
+
+import numpy as np
+
+from ..errors import SpecValidationError
+
+
+class Distribution(ABC):
+    name: str
+    required_params: tuple[str, ...] = ()
+    # Optional JSON-schema fragment for the feature `params`. Built-ins leave this
+    # ``None`` (the Canvas renders their native controls); plugins declare one so
+    # the UI can render config controls with no frontend work (09 §6).
+    param_schema: Mapping[str, object] | None = None
+
+    @abstractmethod
+    def sample(self, rng: np.random.Generator, n: int, params: Mapping[str, float]) -> np.ndarray:
+        """Draw ``n`` samples from the target distribution."""
+
+    @abstractmethod
+    def cdf(self, x: np.ndarray, params: Mapping[str, float]) -> np.ndarray:
+        """Theoretical CDF F(x; params) — used for KS reporting."""
+
+    def validate(self, params: Mapping[str, float], locator: str | None = None) -> None:
+        """Check required params are present and satisfy domain constraints."""
+        missing = [p for p in self.required_params if p not in params]
+        if missing:
+            raise SpecValidationError(
+                f"distribution {self.name!r} missing params: {missing}", locator=locator
+            )
+        self._validate_domain(params, locator)
+
+    def _validate_domain(self, params: Mapping[str, float], locator: str | None) -> None:
+        """Override for per-distribution constraints (e.g. std > 0)."""
+        return None

datadoom/engine/dist/builtins.py

@@ -0,0 +1,172 @@
+"""Built-in distributions and feature samplers (04 §4, 05 §2).
+
+Numeric distributions referenced by ``dist:`` live in :data:`REGISTRY`. Sampling
+flows entirely through the injected ``numpy.random.Generator`` so results are
+deterministic on the pinned path. Non-numeric feature kinds (categorical,
+boolean, datetime, text) have dedicated samplers used by the pipeline.
+"""
+
+from __future__ import annotations
+
+from typing import Mapping
+
+import numpy as np
+from scipy import stats
+
+from ..errors import SpecValidationError
+from .base import Distribution
+
+
+def _require_positive(params: Mapping[str, float], keys: tuple[str, ...], locator: str | None) -> None:
+    for k in keys:
+        if k in params and params[k] <= 0:
+            raise SpecValidationError(f"param {k!r} must be > 0", locator=locator)
+
+
+class Normal(Distribution):
+    name = "normal"
+    required_params = ("mean", "std")
+
+    def _validate_domain(self, params, locator):
+        _require_positive(params, ("std",), locator)
+
+    def sample(self, rng, n, params):
+        return rng.normal(params["mean"], params["std"], size=n)
+
+    def cdf(self, x, params):
+        return stats.norm.cdf(x, loc=params["mean"], scale=params["std"])
+
+
+class LogNormal(Distribution):
+    name = "lognormal"
+    required_params = ("mu", "sigma")
+
+    def _validate_domain(self, params, locator):
+        _require_positive(params, ("sigma",), locator)
+
+    def sample(self, rng, n, params):
+        return rng.lognormal(params["mu"], params["sigma"], size=n)
+
+    def cdf(self, x, params):
+        return stats.lognorm.cdf(x, s=params["sigma"], scale=np.exp(params["mu"]))
+
+
+class Poisson(Distribution):
+    name = "poisson"
+    required_params = ("lam",)
+
+    def _validate_domain(self, params, locator):
+        _require_positive(params, ("lam",), locator)
+
+    def sample(self, rng, n, params):
+        return rng.poisson(params["lam"], size=n)
+
+    def cdf(self, x, params):
+        return stats.poisson.cdf(x, mu=params["lam"])
+
+
+class Pareto(Distribution):
+    name = "pareto"
+    required_params = ("alpha", "xm")
+
+    def _validate_domain(self, params, locator):
+        _require_positive(params, ("alpha", "xm"), locator)
+
+    def sample(self, rng, n, params):
+        # numpy draws Lomax (Pareto II); classical Pareto I = (lomax + 1) * xm.
+        return (rng.pareto(params["alpha"], size=n) + 1.0) * params["xm"]
+
+    def cdf(self, x, params):
+        return stats.pareto.cdf(x, b=params["alpha"], scale=params["xm"])
+
+
+class Uniform(Distribution):
+    name = "uniform"
+    required_params = ("low", "high")
+
+    def _validate_domain(self, params, locator):
+        if params["low"] >= params["high"]:
+            raise SpecValidationError("uniform requires low < high", locator=locator)
+
+    def sample(self, rng, n, params):
+        return rng.uniform(params["low"], params["high"], size=n)
+
+    def cdf(self, x, params):
+        return stats.uniform.cdf(x, loc=params["low"], scale=params["high"] - params["low"])
+
+
+class Exponential(Distribution):
+    name = "exponential"
+    required_params = ("scale",)
+
+    def _validate_domain(self, params, locator):
+        _require_positive(params, ("scale",), locator)
+
+    def sample(self, rng, n, params):
+        return rng.exponential(params["scale"], size=n)
+
+    def cdf(self, x, params):
+        return stats.expon.cdf(x, scale=params["scale"])
+
+
+REGISTRY: dict[str, Distribution] = {
+    d.name: d
+    for d in (Normal(), LogNormal(), Poisson(), Pareto(), Uniform(), Exponential())
+}
+
+
+# --- Non-numeric feature samplers ----------------------------------------------------
+
+
+def sample_categorical(
+    rng: np.random.Generator, n: int, categories: list[str], weights: list[float] | None
+) -> np.ndarray:
+    if weights is None:
+        probs = None
+    else:
+        total = float(sum(weights))
+        probs = [w / total for w in weights]
+    return rng.choice(np.asarray(categories, dtype=object), size=n, p=probs)
+
+
+def sample_boolean(rng: np.random.Generator, n: int, rate: float) -> np.ndarray:
+    return rng.random(size=n) < rate
+
+
+def sample_datetime(
+    rng: np.random.Generator,
+    n: int,
+    start: str,
+    end: str,
+    granularity: str,
+) -> np.ndarray:
+    """Uniformly sample timestamps in [start, end] at the given granularity."""
+    unit = {"second": "s", "minute": "m", "hour": "h", "day": "D"}[granularity]
+    # Cast both endpoints to the chosen unit so their difference is an integer
+    # number of steps — avoids unit-mismatch and stays fully deterministic.
+    start_dt = np.datetime64(start).astype(f"datetime64[{unit}]")
+    end_dt = np.datetime64(end).astype(f"datetime64[{unit}]")
+    span_steps = int((end_dt - start_dt).astype("int64"))
+    if span_steps < 0:
+        raise SpecValidationError("datetime end must be >= start")
+    offsets = rng.integers(0, span_steps + 1, size=n)
+    return start_dt + offsets.astype(f"timedelta64[{unit}]")
+
+
+_LOREM = (
+    "lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor "
+    "incididunt ut labore et dolore magna aliqua ut enim ad minim veniam quis nostrud "
+    "exercitation ullamco laboris nisi aliquip ex ea commodo consequat"
+).split()
+
+
+def sample_text(
+    rng: np.random.Generator, n: int, min_len: int, max_len: int
+) -> np.ndarray:
+    words = np.asarray(_LOREM, dtype=object)
+    lengths = rng.integers(min_len, max_len + 1, size=n)
+    out = np.empty(n, dtype=object)
+    for i in range(n):
+        idx = rng.integers(0, len(words), size=int(lengths[i]))
+        out[i] = " ".join(words[idx])
+    return out