datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/engine/profile.py

@@ -0,0 +1,327 @@
+"""Per-column data profile — exploratory analysis the engine can do for you.
+
+A synthetic dataset is special: the engine knows the ground truth of every column
+— its declared type, how a derived column was generated, and exactly which
+failure modes corrupted it and by how much. This module turns that into a
+**per-column report card** so an engineer or student opening the Results screen
+gets, at a glance, what each column is, its summary statistics, and — crucially —
+*what's wrong with it and how to handle that when building an ML model*.
+
+Pure engine code: deterministic pandas aggregation on the realized frame plus a
+static advice lookup (:mod:`datadoom.engine.advice`). No randomness, no model
+fitting — same ``(spec_hash, seed)`` → identical profile (invariant #6).
+
+The profile is computed from the **clean** shipped frame (the canonical
+artifact); when an **injected** variant exists, each column also carries its
+post-corruption missing rate / moments so the realized impact of the failures is
+visible next to the pristine baseline.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pandas as pd
+
+from .advice import build_issue, severity_rank
+
+if TYPE_CHECKING:
+    from .spec.models import Spec
+
+# Categorical columns with more distinct values than this are summarised by their
+# top categories only (a full breakdown would be noise for the reader).
+_TOP_CATEGORIES = 12
+# A target whose minority class is below this share is flagged as imbalanced.
+_IMBALANCE_MINORITY = 0.35
+
+
+def _num(x: Any) -> float | None:
+    """Coerce to a JSON-safe float, mapping NaN/inf to ``None``."""
+    try:
+        v = float(x)
+    except (TypeError, ValueError):
+        return None
+    return v if np.isfinite(v) else None
+
+
+def _parents_of(spec: Spec, col: str) -> list[str]:
+    """Causal parents (edge sources) feeding a derived column, in spec order."""
+    if spec.causal is None:
+        return []
+    return [e.src for e in spec.causal.edges if e.dst == col]
+
+
+def _derived_names(spec: Spec) -> set[str]:
+    return set() if spec.causal is None else {e.dst for e in spec.causal.edges}
+
+
+def _label_column(spec: Spec) -> str | None:
+    """Best guess at the target column.
+
+    Authoritative when a difficulty block names a ``label``; otherwise a
+    best-effort heuristic: a boolean/categorical causal *sink* (a derived column
+    that no other edge consumes) is almost always the prediction target. Returns
+    ``None`` when the guess is ambiguous (zero or several candidates).
+    """
+    if spec.difficulty is not None and getattr(spec.difficulty, "label", None):
+        return spec.difficulty.label
+    if spec.causal is None:
+        return None
+    sources = {e.src for e in spec.causal.edges}
+    candidates = [
+        e.dst
+        for e in spec.causal.edges
+        if e.dst not in sources
+        and (feat := spec.features.get(e.dst)) is not None
+        and feat.type in ("boolean", "categorical")
+    ]
+    unique = list(dict.fromkeys(candidates))
+    return unique[0] if len(unique) == 1 else None
+
+
+def _numeric_stats(series: pd.Series) -> dict[str, Any]:
+    """Summary statistics for a numeric column (NaN-aware, JSON-safe)."""
+    values = pd.to_numeric(series, errors="coerce").to_numpy(dtype=float)
+    clean = values[np.isfinite(values)]
+    if clean.size == 0:
+        return {}
+    q = np.quantile(clean, [0.25, 0.5, 0.75])
+    return {
+        "mean": _num(clean.mean()),
+        "std": _num(clean.std()),
+        "min": _num(clean.min()),
+        "p25": _num(q[0]),
+        "median": _num(q[1]),
+        "p75": _num(q[2]),
+        "max": _num(clean.max()),
+        "skew": _num(pd.Series(clean).skew()) if clean.size > 2 else None,
+    }
+
+
+def _category_breakdown(series: pd.Series) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
+    """Top categories (value/count/pct) and an imbalance summary for a discrete column."""
+    counts = series.dropna().value_counts()
+    n = int(counts.sum())
+    if n == 0:
+        return [], None
+    top = [
+        {"value": _stringify(val), "count": int(c), "pct": c / n}
+        for val, c in counts.head(_TOP_CATEGORIES).items()
+    ]
+    majority = int(counts.iloc[0])
+    minority = int(counts.iloc[-1])
+    imbalance = {
+        "classes": int(counts.size),
+        "majority_pct": majority / n,
+        "minority_pct": minority / n,
+        "ratio": majority / minority if minority else None,
+    }
+    return top, imbalance
+
+
+def _stringify(val: Any) -> str:
+    if isinstance(val, (bool, np.bool_)):
+        return "true" if val else "false"
+    return str(val)
+
+
+def _failures_by_column(diffs: list[dict[str, Any]] | None) -> dict[str, list[dict[str, Any]]]:
+    """Invert the per-mode failure diffs into a per-column list of (mode, magnitude).
+
+    Each entry carries the realized magnitude (authoritative — the engine measured
+    it) so :func:`datadoom.engine.advice.build_issue` can size severity and the UI
+    can show the concrete effect.
+    """
+    out: dict[str, list[dict[str, Any]]] = {}
+
+    def add(col: str, mode: str, magnitude: str, fraction: float | None, detail: dict[str, Any]) -> None:
+        out.setdefault(col, []).append(
+            {"mode": mode, "magnitude": magnitude, "fraction": fraction, "detail": detail}
+        )
+
+    for d in diffs or []:
+        mode = str(d.get("type") or d.get("mechanism") or "")
+        if mode == "mcar":
+            for col, frac in (d.get("nullified_fraction") or {}).items():
+                f = _num(frac) or 0.0
+                add(col, "mcar", f"{f * 100:.1f}% of values missing", f, {"rate": f})
+        elif mode in ("mar", "mnar"):
+            col = str(d.get("column"))
+            f = _num(d.get("realized_rate")) or 0.0
+            detail = {"rate": f, "driver": d.get("driver"), "self_dependent": d.get("self_dependent")}
+            add(col, mode, f"{f * 100:.1f}% of values missing", f, detail)
+        elif mode == "label_noise":
+            col = str(d.get("column"))
+            f = _num(d.get("flipped_fraction")) or 0.0
+            add(col, "label_noise", f"{f * 100:.1f}% of labels flipped", f, {"rate": f})
+        elif mode == "feature_noise":
+            col = str(d.get("column"))
+            sd = _num(d.get("realized_noise_std"))
+            add(
+                col,
+                "feature_noise",
+                f"σ≈{sd:.3g} noise added" if sd is not None else "noise added",
+                None,
+                {"noise_std": sd, "mean_shift": _num(d.get("realized_mean_shift"))},
+            )
+        elif mode == "drift":
+            col = str(d.get("column"))
+            shift = _num(d.get("total_shift"))
+            kind = d.get("kind", "linear")
+            add(
+                col,
+                "drift",
+                f"{shift:.3g} total {kind} shift" if shift is not None else f"{kind} drift",
+                None,
+                {"total_shift": shift, "kind": kind},
+            )
+        elif mode == "covariate_shift":
+            col = str(d.get("column"))
+            before, after = d.get("before") or {}, d.get("after") or {}
+            bm, am = _num(before.get("mean")), _num(after.get("mean"))
+            mag = f"mean {bm:.3g}→{am:.3g}" if bm is not None and am is not None else "distribution shifted"
+            add(col, "covariate_shift", mag, None, {"before": before, "after": after})
+        elif mode == "leakage":
+            col = str(d.get("into"))
+            corr = _num(d.get("realized_correlation"))
+            mag = f"corr={corr:.3f} with {d.get('target')}" if corr is not None else "high-MI proxy"
+            add(col, "leakage", mag, None, {"target": d.get("target"), "correlation": corr})
+    return out
+
+
+def _role(name: str, derived: set[str], label: str | None, planted: bool) -> str:
+    """How the column functions for modelling: label / leakage proxy / derived / feature."""
+    if planted:
+        return "leakage_proxy"
+    if name == label:
+        return "label"
+    if name in derived:
+        return "derived"
+    return "feature"
+
+
+def _column_profile(
+    name: str,
+    *,
+    spec: Spec,
+    clean: pd.DataFrame,
+    injected: pd.DataFrame | None,
+    derived: set[str],
+    label: str | None,
+    col_failures: list[dict[str, Any]],
+) -> dict[str, Any]:
+    """Assemble the report card for a single column."""
+    planted = name not in clean.columns  # e.g. a leakage proxy lives only in injected
+    base = injected if planted and injected is not None else clean
+    series = base[name]
+    feat = spec.features.get(name)
+    feature_type = feat.type if feat is not None else "synthetic"
+
+    n = int(len(series))
+    missing = int(series.isna().sum())
+    profile: dict[str, Any] = {
+        "name": name,
+        "role": _role(name, derived, label, planted),
+        "feature_type": feature_type,
+        "dtype": str(series.dtype),
+        "count": n,
+        "missing": missing,
+        "missing_pct": missing / n if n else 0.0,
+        "unique": int(series.nunique(dropna=True)),
+        "derived": name in derived,
+        "parents": _parents_of(spec, name),
+        "description": getattr(feat, "description", None),
+    }
+
+    if pd.api.types.is_numeric_dtype(series) and not pd.api.types.is_bool_dtype(series):
+        profile["stats"] = _numeric_stats(series)
+        profile["categories"] = None
+        profile["imbalance"] = None
+    else:
+        top, imbalance = _category_breakdown(series)
+        profile["stats"] = None
+        profile["categories"] = top
+        profile["imbalance"] = imbalance
+
+    # Post-corruption snapshot: how the column actually looks in the injected variant.
+    if injected is not None and name in injected.columns and not planted:
+        inj = injected[name]
+        inj_missing = int(inj.isna().sum())
+        post: dict[str, Any] = {"missing_pct": inj_missing / n if n else 0.0}
+        if pd.api.types.is_numeric_dtype(inj) and not pd.api.types.is_bool_dtype(inj):
+            vals = pd.to_numeric(inj, errors="coerce").to_numpy(dtype=float)
+            vals = vals[np.isfinite(vals)]
+            if vals.size:
+                post["mean"] = _num(vals.mean())
+                post["std"] = _num(vals.std())
+        profile["injected"] = post
+    else:
+        profile["injected"] = None
+
+    # Issues: failure-mode corruptions + (for the label) class imbalance.
+    issues = [
+        build_issue(f["mode"], magnitude=f["magnitude"], fraction=f["fraction"], detail=f["detail"]).to_dict()
+        for f in col_failures
+    ]
+    imbalance = profile.get("imbalance")
+    if name == label and imbalance and imbalance["minority_pct"] < _IMBALANCE_MINORITY:
+        ratio = imbalance.get("ratio")
+        mag = (
+            f"{imbalance['majority_pct'] * 100:.1f}% / {imbalance['minority_pct'] * 100:.1f}%"
+            + (f" ({ratio:.1f}:1)" if ratio else "")
+        )
+        issues.append(
+            build_issue("class_imbalance", magnitude=mag, fraction=None, detail=imbalance).to_dict()
+        )
+    issues.sort(key=lambda i: severity_rank(i["severity"]), reverse=True)
+    profile["issues"] = issues
+    return profile
+
+
+def build_profile(
+    spec: Spec,
+    clean: pd.DataFrame,
+    *,
+    injected: pd.DataFrame | None = None,
+    failure_diffs: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    """Build the full per-column data profile for the Results screen.
+
+    Returns a JSON-serialisable dict with a top-level ``summary`` and a
+    ``columns`` list (one report card each). Columns appear in shipped order,
+    with any injected-only columns (e.g. leakage proxies) appended.
+    """
+    derived = _derived_names(spec)
+    label = _label_column(spec)
+    by_col = _failures_by_column(failure_diffs)
+
+    names: list[str] = list(clean.columns)
+    if injected is not None:
+        names += [c for c in injected.columns if c not in clean.columns]
+
+    columns = [
+        _column_profile(
+            name,
+            spec=spec,
+            clean=clean,
+            injected=injected,
+            derived=derived,
+            label=label,
+            col_failures=by_col.get(name, []),
+        )
+        for name in names
+    ]
+
+    n_issue_cols = sum(1 for c in columns if c["issues"])
+    severities = [i["severity"] for c in columns for i in c["issues"]]
+    summary = {
+        "n_rows": int(len(clean)),
+        "n_columns": len(columns),
+        "label": label,
+        "columns_with_issues": n_issue_cols,
+        "total_issues": len(severities),
+        "critical_issues": sum(1 for s in severities if s == "critical"),
+        "high_issues": sum(1 for s in severities if s == "high"),
+    }
+    return {"summary": summary, "columns": columns}

datadoom/engine/progress.py

@@ -0,0 +1,14 @@
+"""Progress emission contract.
+
+The engine stays framework-free: it emits stage events to a sink. In P0 the sink
+is a no-op; later phases wire a WebSocket hub behind the same interface.
+"""
+
+from __future__ import annotations
+
+
+class ProgressEmitter:
+    """No-op progress sink. Subclasses publish events elsewhere."""
+
+    def emit(self, stage: str, pct: int, message: str = "") -> None:  # noqa: D401
+        return None

datadoom/engine/reference.py

@@ -0,0 +1,338 @@
+"""Machine-readable spec capabilities manifest (the AI-authoring contract).
+
+`build_capabilities()` returns a JSON-serializable dict that enumerates **every**
+knob a DataDoom spec accepts, with its exact valid values and constraints. It is
+built from the **live engine registries** (distributions, structural functions,
+failure modes, exporters, text providers, difficulty tiers) so it is always in
+sync with the running build *and* automatically reflects any registered plugin.
+
+The authoritative *names* come from the registries; richer per-item *annotations*
+(parameter domains, failure-mode fields, prose) are curated here and merged in by
+name. An item with no annotation (e.g. a third-party plugin distribution) still
+appears, carrying whatever the engine ABC exposes (required params, schema).
+
+This is what you feed an LLM/agent so it can emit a valid `*.datadoom.yaml`
+without guessing. The CLI surfaces it via ``datadoom spec-reference`` and the API
+via ``GET /api/spec-reference``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..version import __version__
+
+# --- curated annotations (merged onto registry-derived names by key) ----------------
+
+_DIST_ANNOTATIONS: dict[str, dict[str, Any]] = {
+    "normal": {
+        "summary": "Symmetric bell curve over all real numbers.",
+        "params": {"mean": "center (any real)", "std": "spread, must be > 0"},
+    },
+    "lognormal": {
+        "summary": "Right-skewed, positive-only (income, prices). mu/sigma are of the underlying normal (log space).",
+        "params": {"mu": "mean of ln(X) (any real)", "sigma": "std of ln(X), must be > 0"},
+    },
+    "uniform": {
+        "summary": "Flat — every value in [low, high] equally likely.",
+        "params": {"low": "lower bound", "high": "upper bound, must be > low"},
+    },
+    "exponential": {
+        "summary": "Decaying, non-negative (waiting times). Mean == scale.",
+        "params": {"scale": "mean of the distribution, must be > 0"},
+    },
+    "poisson": {
+        "summary": "Discrete counts 0,1,2,… ; lam is the mean. Output is integer.",
+        "params": {"lam": "mean count, must be > 0"},
+        "discrete": True,
+    },
+    "pareto": {
+        "summary": "Heavy-tailed power law; values are >= xm. Smaller alpha = heavier tail.",
+        "params": {"alpha": "tail index, must be > 0", "xm": "minimum value (scale), must be > 0"},
+    },
+}
+
+_FN_ANNOTATIONS: dict[str, dict[str, Any]] = {
+    "linear": {
+        "summary": "weight·parent + bias.",
+        "fields": {"weight": "number (required)", "bias": "number (optional, default 0)"},
+    },
+    "logistic": {
+        "summary": "1/(1+e^-(weight·parent+bias)) — squash a driver to 0..1; typically the last edge into a boolean target.",
+        "fields": {"weight": "number (required)", "bias": "number (optional, default 0)"},
+    },
+    "polynomial": {
+        "summary": "Σ coeffs[i]·parent^i — curved/non-linear effect.",
+        "fields": {"coeffs": "non-empty list of numbers (required)"},
+    },
+    "map": {
+        "summary": "Look up mapping[parent_category] — turns a categorical parent into a number. Must cover every category.",
+        "fields": {"mapping": "object {category: number} covering all parent categories (required)"},
+    },
+    "identity": {"summary": "Pass the parent value through unchanged.", "fields": {}},
+}
+
+#: Failure modes: each is a list item under top-level ``failures`` with ``type`` + these fields.
+_FAILURE_MODES: dict[str, dict[str, Any]] = {
+    "mcar": {
+        "category": "missingness",
+        "summary": "Missing Completely At Random — blanks chosen independently of the data.",
+        "fields": {
+            "column": "feature name (or use 'columns')",
+            "columns": "list of feature names (alternative to 'column')",
+            "rate": "fraction in [0,1] to blank (required)",
+        },
+    },
+    "mar": {
+        "category": "missingness",
+        "summary": "Missing At Random — blank probability depends on another observed column.",
+        "fields": {
+            "column": "feature to blank (required)",
+            "driver": "observed numeric/boolean feature that drives missingness (required)",
+            "rate": "expected fraction blanked, calibrated (required)",
+            "strength": "driver skew, number (optional, default 2.0)",
+        },
+    },
+    "mnar": {
+        "category": "missingness",
+        "summary": "Missing Not At Random — blank probability depends on the column's own value.",
+        "fields": {
+            "column": "feature to blank (required)",
+            "driver": "optional numeric/boolean driver (defaults to the column itself)",
+            "rate": "expected fraction blanked, calibrated (required)",
+            "strength": "skew, number (optional, default 2.0)",
+        },
+    },
+    "label_noise": {
+        "category": "noise",
+        "summary": "Flip a boolean / reassign a categorical label to a different class.",
+        "fields": {
+            "column": "boolean or categorical feature (required)",
+            "rate": "fraction in [0,1] to corrupt (required)",
+        },
+    },
+    "feature_noise": {
+        "category": "noise",
+        "summary": "Additive noise on a numeric column: x' = x + ε.",
+        "fields": {
+            "column": "numeric feature (required)",
+            "dist": "noise distribution name (required, e.g. normal)",
+            "params": "params for that distribution (e.g. {mean: 0, std: 1})",
+        },
+    },
+    "drift": {
+        "category": "shift",
+        "summary": "Gradually shift a numeric column across the row index (concept drift).",
+        "fields": {
+            "column": "numeric feature (required)",
+            "schedule": "object: {kind: linear|step, magnitude: number (total shift) OR rate: per-row slope, at: 0..1 (step only, default 0.5)}",
+        },
+    },
+    "covariate_shift": {
+        "category": "shift",
+        "summary": "Affine-rescale a numeric column to a target mean/std.",
+        "fields": {
+            "column": "numeric feature (required)",
+            "target": "object {mean?: number, std?: number} (at least one required)",
+        },
+    },
+    "leakage": {
+        "category": "leakage",
+        "summary": "Plant a NEW column that is a near-perfect proxy for a target.",
+        "fields": {
+            "target": "numeric/boolean feature to leak (required)",
+            "into": "new column name, must differ from target (required)",
+            "noise": "proxy noise level relative to target spread (optional, default 0.05; smaller = stronger leak)",
+        },
+    },
+}
+
+_FEATURE_TYPES: dict[str, dict[str, Any]] = {
+    "numeric": {
+        "summary": "Numbers from a distribution, optionally clamped and/or rounded to int. Omit 'dist' to make it a causal-derived column.",
+        "fields": {
+            "dist": "distribution name (see 'distributions'); omit for a causal target",
+            "params": "object of distribution parameters",
+            "min": "lower clamp (optional)",
+            "max": "upper clamp (optional)",
+            "dtype": "'float' (default) or 'int' (rounds to whole numbers)",
+        },
+    },
+    "categorical": {
+        "summary": "One label per row from a fixed set.",
+        "fields": {
+            "categories": "non-empty list of strings (required)",
+            "weights": "list of non-negative numbers, positionally matched; normalized (optional, default uniform)",
+        },
+    },
+    "boolean": {
+        "summary": "True/false column.",
+        "fields": {"rate": "probability of true, in [0,1] (default 0.5)"},
+    },
+    "datetime": {
+        "summary": "Timestamps drawn uniformly in a range.",
+        "fields": {
+            "start": "ISO date string, e.g. '2023-01-01' (required)",
+            "end": "ISO date string >= start (required)",
+            "granularity": "'second' | 'minute' | 'hour' | 'day' (default 'day')",
+        },
+    },
+    "text": {
+        "summary": "Strings: 'lorem' filler or a realistic provider (see 'text_generators'). Realistic providers are seeded/reproducible.",
+        "fields": {
+            "generator": "'lorem' (default) or a realistic provider key",
+            "locale": "locale for realistic providers (default 'en')",
+            "length": "object {min, max} word-count range — lorem only (default {min:5,max:30})",
+        },
+    },
+    "timeseries": {
+        "summary": "Ordered additive series Xt = trend + seasonality + AR(p) + noise over the row index. Row order is the time axis (preserved). May be a causal parent; never a causal target; not distribution-compliance assessed.",
+        "fields": {
+            "trend": "object {slope, intercept} — linear trend (optional)",
+            "seasonality": "list of {amplitude, period (>0), phase} sinusoids, summed (optional)",
+            "ar": "list of AR coefficients [phi1..phip]; sum(|phi|) must be < 1 (stationarity)",
+            "noise_std": "sigma of Gaussian innovations, >= 0 (default 1.0)",
+            "min": "lower clamp (optional)",
+            "max": "upper clamp (optional)",
+            "dtype": "'float' (default) or 'int'",
+        },
+    },
+}
+
+_SHARED_FEATURE_FIELDS = {
+    "description": "free-text doc (optional)",
+    "emit": "boolean; false = latent (computed/drives the SEM but NOT exported, and excluded from probe/compliance/correlation). Default true.",
+}
+
+_RULES = [
+    "Top-level required keys: datadoom_version (always \"1\"), name (slug [A-Za-z0-9_-]+), rows (int >= 1), features.",
+    "A causal-derived feature (numeric or boolean) is declared WITHOUT a 'dist'/'rate' and MUST be the 'to' of at least one causal edge.",
+    "A feature cannot be both sampled (has dist) and a causal target.",
+    "The causal graph must be acyclic. Only numeric/boolean features can be causal targets.",
+    "'map' edges require a categorical parent and a mapping covering every category; other fns require a numeric/boolean/timeseries parent.",
+    "A difficulty 'label' must be a boolean or 2-class categorical feature, and must not be latent (emit:false).",
+    "difficulty.knobs ⊆ {noise, label_noise}. target is a named tier or {band:[a,b]}.",
+    "Failures are an ordered list applied after the clean baseline is captured; export versions must include 'injected' to write the corrupted variant.",
+    "A failure cannot reference a latent (emit:false) feature.",
+    "export.splits ratios must sum to 1.0. export.formats must be known formats.",
+    "time-series AR must satisfy sum(|coefficients|) < 1 (stationarity).",
+    "Determinism: same (spec, seed) -> identical bytes. Seed is NOT part of the spec hash.",
+]
+
+
+def _distributions() -> list[dict[str, Any]]:
+    from .dist.builtins import REGISTRY
+
+    out: list[dict[str, Any]] = []
+    for name in sorted(REGISTRY):
+        dist = REGISTRY[name]
+        entry: dict[str, Any] = {
+            "name": name,
+            "required_params": list(dist.required_params),
+            "builtin": name in _DIST_ANNOTATIONS,
+        }
+        entry.update(_DIST_ANNOTATIONS.get(name, {}))
+        if getattr(dist, "param_schema", None) is not None:
+            entry["param_schema"] = dist.param_schema
+        out.append(entry)
+    return out
+
+
+def _structural_fns() -> list[dict[str, Any]]:
+    from .causal.functions import STRUCTURAL_FNS
+
+    out: list[dict[str, Any]] = []
+    for name in sorted(STRUCTURAL_FNS):
+        entry: dict[str, Any] = {"name": name, "builtin": name in _FN_ANNOTATIONS}
+        entry.update(_FN_ANNOTATIONS.get(name, {}))
+        out.append(entry)
+    return out
+
+
+def _failure_modes() -> list[dict[str, Any]]:
+    from .failure import FAILURE_MODES
+
+    out: list[dict[str, Any]] = []
+    for name in sorted(FAILURE_MODES):
+        entry: dict[str, Any] = {"type": name, "builtin": name in _FAILURE_MODES}
+        entry.update(_FAILURE_MODES.get(name, {}))
+        out.append(entry)
+    return out
+
+
+def _difficulty() -> dict[str, Any]:
+    from .difficulty import PROBES, TIER_BANDS
+
+    return {
+        "tiers": {name: list(band) for name, band in TIER_BANDS.items()},
+        "probes": sorted(PROBES),
+        "knobs": ["noise", "label_noise"],
+        "target": "a named tier (e.g. 'advanced') or an explicit {band: [a, b]} of AUROC",
+        "label": "the boolean / 2-class categorical column the baseline probe predicts",
+        "max_iters": "calibration steps, int >= 1 (default 8)",
+    }
+
+
+def _exporters() -> dict[str, Any]:
+    from .export import EXPORTERS
+
+    return {
+        "formats": sorted(EXPORTERS),
+        "versions": ["clean", "injected"],
+        "fields": {
+            "formats": "list of output formats (default [csv]); parquet needs the optional extra",
+            "versions": "subset of {clean, injected} (default [clean])",
+            "splits": "object {name: ratio} whose ratios sum to 1.0 (optional)",
+            "shuffle": "boolean (default true)",
+            "metadata": "boolean — write metadata.json (default true)",
+        },
+    }
+
+
+def _text_generators() -> dict[str, Any]:
+    from .dist.providers import REALISTIC_GENERATORS
+
+    return {
+        "lorem": "filler words (uses 'length' {min,max})",
+        "realistic": sorted(REALISTIC_GENERATORS),
+    }
+
+
+def build_capabilities() -> dict[str, Any]:
+    """Return the full, JSON-serializable spec capabilities manifest."""
+    return {
+        "datadoom_version": "1",
+        "package_version": __version__,
+        "summary": (
+            "DataDoom spec capabilities. A spec is a YAML/JSON document describing a "
+            "reproducible synthetic dataset. Use the exact names/fields below; same "
+            "(spec, seed) regenerates identical data."
+        ),
+        "top_level_keys": {
+            "datadoom_version": 'required, always "1"',
+            "name": "required, slug [A-Za-z0-9_-]+",
+            "description": "optional string",
+            "seed": "optional int (reproducibility; not part of the spec hash)",
+            "rows": "required int >= 1",
+            "features": "required object {name: feature} — see feature_types",
+            "causal": "optional DAG {edges, noise, interventions}",
+            "difficulty": "optional classification difficulty target",
+            "failures": "optional ordered list of corruptions",
+            "export": "optional output config",
+            "meta": "optional free-form object (ignored by the engine)",
+        },
+        "shared_feature_fields": _SHARED_FEATURE_FIELDS,
+        "feature_types": _FEATURE_TYPES,
+        "distributions": _distributions(),
+        "structural_fns": _structural_fns(),
+        "causal": {
+            "edges": "list of {from, to, fn, ...fn params} — see structural_fns",
+            "noise": "object {derived_node: {dist: <name|none>, params: {...}}}",
+            "interventions": "list of {do: {feature: value}} — fix a node to a constant",
+        },
+        "failure_modes": _failure_modes(),
+        "difficulty": _difficulty(),
+        "export": _exporters(),
+        "text_generators": _text_generators(),
+        "rules": _RULES,
+    }