datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/__init__.py

@@ -0,0 +1,23 @@
+"""DataDoom — local-first engine for reproducible synthetic data.
+
+Public API:
+
+    >>> import datadoom
+    >>> spec = datadoom.load_spec("dataset.datadoom.yaml")
+    >>> result = datadoom.generate(spec, seed=42)
+    >>> result.frame.head()
+"""
+
+from __future__ import annotations
+
+from .engine import Spec, generate, load_spec, parse_spec, validate_spec
+from .version import __version__
+
+__all__ = [
+    "Spec",
+    "generate",
+    "load_spec",
+    "parse_spec",
+    "validate_spec",
+    "__version__",
+]

datadoom/adapters/__init__.py

@@ -0,0 +1,29 @@
+"""Framework adapters — load a generated DataDoom run into ML frameworks (18.4).
+
+A run directory (the ``--out`` of ``datadoom run``) holds ``data.csv`` (and any
+``data.injected.*`` / other formats). These helpers turn it into the in-memory
+object each framework expects:
+
+* :func:`load_dataframe` — a **pandas** ``DataFrame`` (no extra needed; pandas is
+  a core dep). Auto-detects csv / parquet / json and the clean/injected variant.
+* :func:`to_torch_dataset` — a ``torch.utils.data.TensorDataset`` (extra: ``torch``).
+* :func:`to_tf_dataset` — a ``tf.data.Dataset`` (extra: ``tf``).
+* :func:`to_hf_dataset` — a HuggingFace ``datasets.Dataset`` (extra: ``hf``).
+
+The framework loaders **lazy-import** their backend and raise a clear install
+hint if it is missing, so the core install stays light. This package depends only
+on the engine (for nothing heavyweight) + pandas; the engine never imports it.
+"""
+
+from __future__ import annotations
+
+from .frameworks import to_hf_dataset, to_tf_dataset, to_torch_dataset
+from .loaders import load_dataframe, numeric_feature_columns
+
+__all__ = [
+    "load_dataframe",
+    "numeric_feature_columns",
+    "to_torch_dataset",
+    "to_tf_dataset",
+    "to_hf_dataset",
+]

datadoom/adapters/frameworks.py

@@ -0,0 +1,94 @@
+"""Convert a pandas DataFrame into torch / tensorflow / HuggingFace datasets.
+
+Each converter lazy-imports its backend so the core install stays light; a
+missing backend raises an actionable install hint naming the right extra.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+import pandas as pd
+
+from .loaders import numeric_feature_columns
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    pass
+
+
+def _require(module: str, extra: str) -> Any:
+    try:
+        return import_module(module)
+    except ImportError as exc:  # pragma: no cover - exercised only without the extra
+        raise ImportError(
+            f"{module!r} is required for this adapter. Install it with: "
+            f"pip install 'datadoom[{extra}]'"
+        ) from exc
+
+
+def _feature_matrix(
+    df: pd.DataFrame, feature_columns: list[str] | None, target: str | None
+) -> tuple[list[str], np.ndarray]:
+    cols = feature_columns or numeric_feature_columns(df, exclude=[target] if target else None)
+    if not cols:
+        raise ValueError(
+            "no numeric/boolean feature columns found; pass feature_columns explicitly "
+            "or encode categorical/text columns first"
+        )
+    x = df[cols].to_numpy(dtype="float32")
+    return cols, x
+
+
+def to_torch_dataset(
+    df: pd.DataFrame,
+    *,
+    target: str | None = None,
+    feature_columns: list[str] | None = None,
+) -> Any:
+    """Build a ``torch.utils.data.TensorDataset`` from ``df`` (extra: ``torch``).
+
+    Features are the numeric/boolean columns (or ``feature_columns``); if
+    ``target`` is given it becomes the second tensor.
+    """
+    torch = _require("torch", "torch")
+    _, x = _feature_matrix(df, feature_columns, target)
+    x_t = torch.as_tensor(x)
+    if target is None:
+        return torch.utils.data.TensorDataset(x_t)
+    y = df[target].to_numpy()
+    y_t = torch.as_tensor(y.astype("float32") if y.dtype != object else y)
+    return torch.utils.data.TensorDataset(x_t, y_t)
+
+
+def to_tf_dataset(
+    df: pd.DataFrame,
+    *,
+    target: str | None = None,
+    feature_columns: list[str] | None = None,
+    batch_size: int | None = None,
+) -> Any:
+    """Build a ``tf.data.Dataset`` from ``df`` (extra: ``tf``).
+
+    Yields feature rows, or ``(features, label)`` pairs when ``target`` is set.
+    Optionally batched.
+    """
+    tf = _require("tensorflow", "tf")
+    _, x = _feature_matrix(df, feature_columns, target)
+    if target is None:
+        ds = tf.data.Dataset.from_tensor_slices(x)
+    else:
+        ds = tf.data.Dataset.from_tensor_slices((x, df[target].to_numpy()))
+    if batch_size:
+        ds = ds.batch(batch_size)
+    return ds
+
+
+def to_hf_dataset(df: pd.DataFrame) -> Any:
+    """Build a HuggingFace ``datasets.Dataset`` from ``df`` (extra: ``hf``).
+
+    Keeps every column (including categorical/text) — HF datasets are schema-rich.
+    """
+    datasets = _require("datasets", "hf")
+    return datasets.Dataset.from_pandas(df, preserve_index=False)

datadoom/adapters/loaders.py

@@ -0,0 +1,72 @@
+"""Read generated run artifacts into a pandas DataFrame."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+# Preferred read order: CSV is the canonical artifact, then parquet, then json.
+_READERS: list[tuple[str, str]] = [("csv", "csv"), ("parquet", "parquet"), ("json", "json")]
+
+
+def _read(path: Path, kind: str) -> pd.DataFrame:
+    if kind == "csv":
+        return pd.read_csv(path)
+    if kind == "parquet":
+        return pd.read_parquet(path)
+    return pd.read_json(path, orient="records")
+
+
+def load_dataframe(
+    run_dir: str | Path,
+    *,
+    version: str = "clean",
+    split: str | None = None,
+) -> pd.DataFrame:
+    """Load a generated dataset variant into a pandas ``DataFrame``.
+
+    Args:
+        run_dir: the run output directory (``datadoom run --out <dir>``).
+        version: ``"clean"`` (default) or ``"injected"`` (the corrupted variant).
+        split: optional split name (e.g. ``"train"``) if split files were written.
+
+    Returns:
+        The dataset as a ``DataFrame``, columns in spec order.
+
+    Raises:
+        FileNotFoundError: if no matching data artifact exists in ``run_dir``.
+    """
+    base = Path(run_dir)
+    stem = "data" if version == "clean" else "data.injected"
+    if split:
+        stem = f"{stem}.{split}"
+
+    for ext, kind in _READERS:
+        candidate = base / f"{stem}.{ext}"
+        if candidate.exists():
+            return _read(candidate, kind)
+
+    tried = ", ".join(f"{stem}.{ext}" for ext, _ in _READERS)
+    raise FileNotFoundError(
+        f"no data artifact for version={version!r}"
+        + (f", split={split!r}" if split else "")
+        + f" in {base} (looked for: {tried})"
+    )
+
+
+def numeric_feature_columns(df: pd.DataFrame, *, exclude: list[str] | None = None) -> list[str]:
+    """Return the numeric/boolean columns of ``df`` (model-ready features).
+
+    Categorical/text/datetime columns are skipped — encode them yourself if you
+    need them. Pass ``exclude`` to drop e.g. the target column.
+    """
+    drop = set(exclude or [])
+    cols: list[str] = []
+    for name in df.columns:
+        if name in drop:
+            continue
+        s = df[name]
+        if pd.api.types.is_numeric_dtype(s) or pd.api.types.is_bool_dtype(s):
+            cols.append(name)
+    return cols

datadoom/api/__init__.py

@@ -0,0 +1,11 @@
+"""DataDoom HTTP API (FastAPI). Thin layer over ``jobs`` + ``store`` + ``engine``.
+
+``create_app()`` is the entry point; ``datadoom serve`` (CLI) runs it under
+uvicorn.
+"""
+
+from __future__ import annotations
+
+from .app import create_app
+
+__all__ = ["create_app"]

datadoom/api/app.py

@@ -0,0 +1,109 @@
+"""FastAPI application factory (08, 17 step 9).
+
+Assembles config -> store -> jobs -> api into one app: opens the DB (running
+Alembic to head), wires the worker + event hub, mounts the REST routes and the
+WebSocket/SSE transport, installs the error envelope, and serves the bundled SPA
+from ``webdist/`` (so ``datadoom serve`` is a complete app with no Node needed).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from datadoom.config import Config, load_config
+from datadoom.jobs import EventHub, WorkerPool
+from datadoom.plugins import load_plugins
+from datadoom.store import LocalArtifactStore, init_database
+from datadoom.version import __version__
+
+from . import ws
+from .errors import install_error_handlers
+from .routes import artifacts, datasets, meta, plugins, runs, specs, templates
+from .state import AppState
+
+WEBDIST = Path(__file__).resolve().parent.parent / "webdist"
+
+
+def create_app(config: Config | None = None) -> FastAPI:
+    config = config or load_config()
+    config.ensure_dirs()
+
+    # Discover plugins (entry points + local dir) into the engine's lookup tables;
+    # conflicts fail loudly here rather than silently shadowing a capability (09 §3).
+    load_plugins(local_dir=config.home / "plugins")
+
+    db = init_database(config.db_url)
+    artifact_store = LocalArtifactStore(config.artifacts_dir)
+    hub = EventHub()
+    worker = WorkerPool(db, artifact_store, hub, __version__)
+    state = AppState(
+        config=config, db=db, artifacts=artifact_store, hub=hub, worker=worker
+    )
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):  # noqa: ANN202
+        # Bind the running loop so worker threads can marshal WS events onto it.
+        hub.bind_loop(asyncio.get_running_loop())
+        yield
+        worker.shutdown()
+        db.dispose()
+
+    app = FastAPI(
+        title="DataDoom",
+        version=__version__,
+        description="Local-first engine for controllable, reproducible synthetic data.",
+        openapi_url="/api/openapi.json",
+        docs_url="/api/docs",
+        redoc_url=None,
+        lifespan=lifespan,
+    )
+    app.state.dd = state
+
+    install_error_handlers(app)
+
+    for module in (meta, specs, datasets, runs, artifacts, templates, plugins):
+        app.include_router(module.router)
+    app.include_router(ws.router)
+
+    _mount_spa(app)
+    return app
+
+
+def _mount_spa(app: FastAPI) -> None:
+    """Serve the built SPA at ``/`` with client-side-routing fallback.
+
+    If ``webdist/`` is absent (dev before the frontend is built), ``/`` returns a
+    friendly JSON pointer instead of 404 so the API is still usable.
+    """
+    if not (WEBDIST / "index.html").exists():
+
+        @app.get("/", include_in_schema=False)
+        async def _no_spa() -> JSONResponse:  # noqa: ANN202
+            return JSONResponse(
+                {
+                    "status": "ok",
+                    "message": "DataDoom API is running. The web UI is not built; "
+                    "run `cd frontend && npm install && npm run build`.",
+                    "docs": "/api/docs",
+                }
+            )
+
+        return
+
+    assets = WEBDIST / "assets"
+    if assets.exists():
+        app.mount("/assets", StaticFiles(directory=assets), name="assets")
+
+    @app.get("/{full_path:path}", include_in_schema=False)
+    async def _spa(full_path: str) -> FileResponse:  # noqa: ANN202
+        # Serve real files when they exist; otherwise the SPA entry (client routing).
+        candidate = WEBDIST / full_path
+        if full_path and candidate.is_file():
+            return FileResponse(candidate)
+        return FileResponse(WEBDIST / "index.html")

datadoom/api/deps.py

@@ -0,0 +1,30 @@
+"""FastAPI dependencies: app state, DB sessions, and (no-op local) auth.
+
+Auth is a no-op dependency in local mode (08 §1); team mode swaps in a real
+bearer-token dependency without changing any route signature.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from fastapi import Request
+from sqlalchemy.orm import Session
+
+from .state import AppState
+
+
+def get_state(request: Request) -> AppState:
+    return request.app.state.dd
+
+
+def get_session(request: Request) -> Iterator[Session]:
+    """Yield a transactional session for the request (commit/rollback handled)."""
+    state: AppState = request.app.state.dd
+    with state.db.session() as session:
+        yield session
+
+
+def current_owner() -> None:
+    """No-op auth: local mode has a single implicit owner (``owner_id = None``)."""
+    return None

datadoom/api/errors.py

@@ -0,0 +1,89 @@
+"""Consistent error envelope + exception handlers (08 §1, §12).
+
+Every error response is ``{ "error": { code, message, locator? } }``. Spec
+validation failures map to 422 with the offending field's ``locator``; other
+engine errors to 400; anything unexpected to 500 (traceback logged, not leaked).
+"""
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from datadoom.engine.errors import (
+    DataDoomError,
+    DistributionError,
+    SpecValidationError,
+)
+
+log = logging.getLogger("datadoom.api")
+
+
+def _envelope(code: str, message: str, locator: str | None = None) -> dict:
+    detail = {"code": code, "message": message}
+    if locator is not None:
+        detail["locator"] = locator
+    return {"error": detail}
+
+
+# Status-code -> default error code for bare HTTPExceptions.
+_CODE_FOR_STATUS = {400: "bad_request", 404: "not_found", 409: "conflict", 422: "validation_error"}
+
+
+def http_error(status: int, code: str, message: str, locator: str | None = None):  # noqa: ANN201
+    """Build an HTTPException whose detail carries our envelope fields."""
+    from fastapi import HTTPException
+
+    return HTTPException(status_code=status, detail={"code": code, "message": message, "locator": locator})
+
+
+def install_error_handlers(app: FastAPI) -> None:
+    @app.exception_handler(SpecValidationError)
+    async def _spec_invalid(_req: Request, exc: SpecValidationError):  # noqa: ANN202
+        return JSONResponse(
+            status_code=422,
+            content=_envelope("validation_error", str(exc), getattr(exc, "locator", None)),
+        )
+
+    @app.exception_handler(DistributionError)
+    async def _dist_error(_req: Request, exc: DistributionError):  # noqa: ANN202
+        return JSONResponse(
+            status_code=422,
+            content=_envelope("distribution_error", str(exc), getattr(exc, "locator", None)),
+        )
+
+    @app.exception_handler(DataDoomError)
+    async def _domain_error(_req: Request, exc: DataDoomError):  # noqa: ANN202
+        return JSONResponse(
+            status_code=400, content=_envelope("error", str(exc), getattr(exc, "locator", None))
+        )
+
+    @app.exception_handler(StarletteHTTPException)
+    async def _http_exc(_req: Request, exc: StarletteHTTPException):  # noqa: ANN202
+        detail = exc.detail
+        if isinstance(detail, dict) and "code" in detail:
+            content = _envelope(detail["code"], detail.get("message", ""), detail.get("locator"))
+        else:
+            code = _CODE_FOR_STATUS.get(exc.status_code, "error")
+            content = _envelope(code, str(detail))
+        return JSONResponse(status_code=exc.status_code, content=content)
+
+    @app.exception_handler(RequestValidationError)
+    async def _req_invalid(_req: Request, exc: RequestValidationError):  # noqa: ANN202
+        first = exc.errors()[0] if exc.errors() else {}
+        locator = ".".join(str(p) for p in first.get("loc", []) if p != "body")
+        return JSONResponse(
+            status_code=422,
+            content=_envelope("validation_error", first.get("msg", "invalid request"), locator or None),
+        )
+
+    @app.exception_handler(Exception)
+    async def _unexpected(_req: Request, exc: Exception):  # noqa: ANN202
+        log.exception("unhandled server error")
+        return JSONResponse(
+            status_code=500, content=_envelope("internal_error", "internal server error")
+        )

datadoom/api/estimate.py

@@ -0,0 +1,82 @@
+"""Resource estimator (doc 12) — heuristic runtime / RAM / output-size guess.
+
+Local-first means **no cost, no GPU, no quotas** — we estimate only so the UI can
+warn before a heavy run. Pure function of the spec + fixed calibration constants,
+so it is deterministic and reproducible (doc 12 §9). Never blocks a run.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from datadoom.engine import Spec
+
+# Per-cell byte estimates by type (doc 12 §3).
+_BYTES_NUMERIC = 8
+_BYTES_BOOL = 1
+_BYTES_DATETIME = 19  # ISO-8601 string in CSV
+_CSV_FORMAT_FACTOR = 1.2
+
+# Calibrated throughput constants (rows/sec-ish) — reference-laptop defaults (doc 12 §6).
+_KAPPA_SAMPLE = 4_000_000.0  # vectorized sampling cells/sec
+_KAPPA_IO = 80_000_000.0  # bytes/sec write
+_T_FIXED = 0.15  # process/setup overhead seconds
+
+
+@dataclass
+class Estimate:
+    estimated_runtime_seconds: float
+    estimated_ram_mb: float
+    estimated_size_bytes: int
+    features: int
+    edges: int
+    gpu_required: bool = False
+
+
+def _avg_bytes(feat) -> float:  # noqa: ANN001 — duck-typed over the feature union
+    t = feat.type
+    if t == "numeric":
+        return _BYTES_NUMERIC
+    if t == "boolean":
+        return _BYTES_BOOL
+    if t == "datetime":
+        return _BYTES_DATETIME
+    if t == "categorical":
+        labels = feat.categories or [""]
+        return sum(len(c) for c in labels) / len(labels) + 1
+    if t == "text":
+        length = getattr(feat, "length", {}) or {}
+        avg_tokens = (length.get("min", 5) + length.get("max", 30)) / 2
+        return avg_tokens * 5  # ~5 bytes/token incl. spaces
+    return _BYTES_NUMERIC
+
+
+def estimate(spec: Spec) -> Estimate:
+    n = spec.rows
+    feats = list(spec.features.values())
+    f = len(feats)
+    edges = len(spec.causal.edges) if spec.causal else 0
+
+    bytes_per_row = sum(_avg_bytes(ft) for ft in feats)
+    versions = len(spec.export.versions) or 1
+    formats = len(spec.export.formats) or 1
+    size_clean = n * bytes_per_row * _CSV_FORMAT_FACTOR
+    size_total = int(size_clean * versions * formats)
+
+    f_num = sum(1 for ft in feats if ft.type == "numeric")
+    t_base = (n * max(f_num, 1)) / _KAPPA_SAMPLE
+    t_io = size_total / _KAPPA_IO
+    runtime = round(_T_FIXED + t_base + t_io, 3)
+
+    # One float64 working frame, with clean (+possible injected) copies (doc 12 §4).
+    frame_multiplier = 2 + (1 if "injected" in spec.export.versions else 0)
+    ram_mb = round((n * max(f, 1) * 8 * frame_multiplier) / (1024 * 1024), 2)
+
+    return Estimate(
+        estimated_runtime_seconds=runtime,
+        estimated_ram_mb=ram_mb,
+        estimated_size_bytes=size_total,
+        features=f,
+        edges=edges,
+        gpu_required=False,
+    )

datadoom/api/routes/__init__.py

@@ -0,0 +1,7 @@
+"""API route modules (doc 08). Each exposes an ``APIRouter`` named ``router``."""
+
+from __future__ import annotations
+
+from . import artifacts, datasets, meta, plugins, runs, specs, templates
+
+__all__ = ["artifacts", "datasets", "meta", "plugins", "runs", "specs", "templates"]