datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/engine/export/metadata.py

@@ -0,0 +1,58 @@
+"""Deterministic ``metadata.json`` writer (04 §8, 06 §3.5).
+
+The metadata document is intentionally free of timestamps or other ambient state
+so that it is itself reproducible: the same ``(spec_hash, seed)`` produces an
+identical metadata file. Human-facing run timestamps live in the persistence
+layer, not in the reproducible artifact bundle.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .base import ArtifactInfo
+from .checksums import sha256_bytes
+
+
+def build_metadata(
+    *,
+    spec_body: dict[str, Any],
+    spec_hash: str,
+    seed: int,
+    rows: int,
+    package_version: str,
+    artifacts: list[ArtifactInfo],
+    compliance: dict[str, Any],
+    determinism: dict[str, Any],
+    failures: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+    metadata: dict[str, Any] = {
+        "datadoom_package_version": package_version,
+        "spec_hash": spec_hash,
+        "seed": seed,
+        "rows": rows,
+        "spec": spec_body,
+        "artifacts": [a.to_dict() for a in artifacts],
+        "compliance": compliance,
+        "determinism": determinism,
+    }
+    if failures is not None:
+        metadata["failures"] = failures
+    return metadata
+
+
+def write_metadata(metadata: dict[str, Any], path: str | Path) -> ArtifactInfo:
+    text = json.dumps(metadata, sort_keys=True, indent=2, ensure_ascii=False)
+    data = (text + "\n").encode("utf-8")
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "wb") as fh:
+        fh.write(data)
+    return ArtifactInfo(
+        path=str(path),
+        format="json",
+        checksum_sha256=sha256_bytes(data),
+        size_bytes=len(data),
+    )

datadoom/engine/export/parquet_exporter.py

@@ -0,0 +1,45 @@
+"""Parquet writer (17 step 18, 09 §8).
+
+Parquet is a columnar format ML tooling consumes directly (pandas/Polars/Spark,
+``datasets``). ``pyarrow`` is an **optional** dependency (``pip install
+datadoom[parquet]``) — it's a large wheel and the core stays light — so the import
+is deferred to ``write`` with an actionable error if it is missing. Within a
+pinned environment the same ``(spec_hash, seed)`` yields identical bytes; the only
+embedded ambient value is pyarrow's constant ``created_by`` build string.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+from ..errors import DataDoomError
+from .base import ArtifactInfo, Exporter
+from .checksums import sha256_file
+
+
+class ParquetExporter(Exporter):
+    format = "parquet"
+
+    def write(self, df: pd.DataFrame, path: str | Path) -> ArtifactInfo:
+        try:
+            import pyarrow as pa
+            import pyarrow.parquet as pq
+        except ImportError as exc:  # pragma: no cover - exercised only without the extra
+            raise DataDoomError(  # noqa: TRY003
+                "the 'parquet' export format needs pyarrow; install it with "
+                "`pip install datadoom[parquet]`"
+            ) from exc
+
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        table = pa.Table.from_pandas(df, preserve_index=False)
+        # Fixed options so the bytes are reproducible on the pinned path.
+        pq.write_table(table, path, compression="snappy", version="2.6")
+        return ArtifactInfo(
+            path=str(path),
+            format=self.format,
+            checksum_sha256=sha256_file(path),
+            size_bytes=path.stat().st_size,
+        )

datadoom/engine/failure/__init__.py

@@ -0,0 +1,18 @@
+"""Failure injection — deterministic corruption transforms (05 §4, 04 §7).
+
+A clean baseline is captured before injection and always preserved alongside the
+injected variant. Each mode draws from ``RNG(failure:i)`` so the injected frame
+is itself reproducible on the pinned path.
+"""
+
+from __future__ import annotations
+
+from .apply import apply_failures
+from .base import FailureMode
+from .modes import FAILURE_MODES
+
+__all__ = [
+    "FailureMode",
+    "FAILURE_MODES",
+    "apply_failures",
+]

datadoom/engine/failure/apply.py

@@ -0,0 +1,37 @@
+"""Failure-injection orchestration (03 §4 stage 6, 05 §4).
+
+Captures the clean baseline, then applies the spec's ordered failure list to a
+copy, each mode drawing from its own ``RNG(failure:i)``. Returns the injected
+frame and the per-mode diff summaries. The clean frame is never mutated.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pandas as pd
+
+from .modes import FAILURE_MODES
+
+if TYPE_CHECKING:  # avoid a runtime import cycle with pipeline
+    from ..pipeline import RunContext
+
+
+def apply_failures(ctx: RunContext, clean: pd.DataFrame) -> tuple[pd.DataFrame, list[dict[str, Any]]]:
+    """Apply the spec's failures in order to a copy of ``clean``.
+
+    Returns ``(injected_frame, diffs)`` where each diff records the mode and its
+    realized effect. ``ctx.used_namespaces`` gains a ``failure:i`` entry per mode
+    so the determinism report covers the injected stream too.
+    """
+    injected = clean.copy(deep=True)
+    diffs: list[dict[str, Any]] = []
+    for i, failure in enumerate(ctx.spec.failures):
+        mode = FAILURE_MODES[failure.type]
+        params = failure.model_dump()
+        params.pop("type", None)
+        rng = ctx.rng.failure(i)
+        ctx.used_namespaces.append(f"failure:{i}")
+        summary = mode.apply(rng, injected, params, ctx.spec.features)
+        diffs.append({"index": i, "type": failure.type, **summary})
+    return injected, diffs

datadoom/engine/failure/base.py

@@ -0,0 +1,116 @@
+"""FailureMode ABC + shared helpers (05 §4, 04 §7).
+
+A failure mode is a deterministic corruption transform applied *after* the clean
+baseline is captured. Each mode reads its config (the failure spec entry minus
+``type``), mutates the working ``injected`` frame in place, and returns a **diff
+summary** describing what it changed (fraction nullified, realized rate, shift
+magnitude, leakage correlation…). The clean baseline is never touched.
+
+All randomness flows through an injected ``numpy.random.Generator`` (``RNG(failure:i)``)
+so the injected variant is itself reproducible on the pinned path.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from ..errors import SpecValidationError
+from ..spec.models import Feature
+
+
+def sigmoid(z: np.ndarray) -> np.ndarray:
+    """Numerically-stable logistic function."""
+    return np.where(z >= 0, 1.0 / (1.0 + np.exp(-z)), np.exp(z) / (1.0 + np.exp(z)))
+
+
+def standardize(values: np.ndarray) -> np.ndarray:
+    """Z-score a numeric (or boolean) driver; constant/degenerate columns map to
+    zeros. NaN-robust: previously-injected missing values contribute a 0 score
+    rather than poisoning the whole column."""
+    x = np.asarray(values, dtype=float)
+    std = float(np.nanstd(x))
+    if std == 0.0 or not np.isfinite(std):
+        return np.zeros_like(x)
+    z = (x - float(np.nanmean(x))) / std
+    return np.nan_to_num(z, nan=0.0)
+
+
+def calibrate_logistic_intercept(scores: np.ndarray, target_rate: float) -> float:
+    """Find ``a`` so that ``mean(sigmoid(a + scores)) == target_rate``.
+
+    The mean of the logistic is monotonic in the intercept, so a bisection
+    converges. This lets MAR/MNAR honor the requested *expected* missing rate
+    while keeping the missingness **dependent on the driver/value** (the whole
+    point of those mechanisms).
+    """
+    if target_rate <= 0.0:
+        return float("-inf")
+    if target_rate >= 1.0:
+        return float("inf")
+    lo, hi = -60.0, 60.0
+    for _ in range(80):
+        mid = 0.5 * (lo + hi)
+        if float(np.mean(sigmoid(mid + scores))) < target_rate:
+            lo = mid
+        else:
+            hi = mid
+    return 0.5 * (lo + hi)
+
+
+# --- validation helpers --------------------------------------------------------------
+
+
+def require_rate(params: Mapping[str, Any], locator: str) -> float:
+    rate = params.get("rate")
+    if rate is None:
+        raise SpecValidationError("missing required 'rate'", locator=f"{locator}.rate")
+    rate = float(rate)
+    if not 0.0 <= rate <= 1.0:
+        raise SpecValidationError("rate must be in [0, 1]", locator=f"{locator}.rate")
+    return rate
+
+
+def require_feature(
+    params: Mapping[str, Any],
+    key: str,
+    features: Mapping[str, Feature],
+    locator: str,
+) -> str:
+    ref = params.get(key)
+    if not isinstance(ref, str):
+        raise SpecValidationError(f"missing required '{key}'", locator=f"{locator}.{key}")
+    if ref not in features:
+        raise SpecValidationError(
+            f"{key} {ref!r} is not a declared feature", locator=f"{locator}.{key}"
+        )
+    return ref
+
+
+class FailureMode(ABC):
+    """ABC for a corruption transform (05 §4)."""
+
+    name: str
+    # Optional JSON-schema fragment for the failure params (09 §6); ``None`` for built-ins.
+    param_schema: Mapping[str, Any] | None = None
+
+    def validate(
+        self, params: Mapping[str, Any], features: Mapping[str, Feature], locator: str
+    ) -> None:
+        """Check the config carries the fields this mode needs and they reference
+        real features. Raise :class:`SpecValidationError` on the first problem."""
+        return None
+
+    @abstractmethod
+    def apply(
+        self,
+        rng: np.random.Generator,
+        frame: pd.DataFrame,
+        params: Mapping[str, Any],
+        features: Mapping[str, Feature],
+    ) -> dict[str, Any]:
+        """Mutate ``frame`` in place and return a diff summary."""