datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/api/schemas.py

@@ -0,0 +1,230 @@
+"""Request/response models — the typed surface FastAPI turns into OpenAPI.
+
+The frontend generates its API client from ``/api/openapi.json``, so these
+shapes ARE the contract (doc 08). Spec bodies travel as open ``dict`` payloads
+(the authoritative validation lives in ``engine.spec``); these models describe
+the persistence/metadata envelope around them.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+SpecBody = dict[str, Any]
+
+
+# --- errors -------------------------------------------------------------------
+class ErrorDetail(BaseModel):
+    code: str
+    message: str
+    locator: str | None = None
+
+
+class ErrorResponse(BaseModel):
+    error: ErrorDetail
+
+
+# --- specs (stateless helpers) ------------------------------------------------
+class ValidateResponse(BaseModel):
+    valid: bool = True
+    spec_hash: str
+    warnings: list[str] = Field(default_factory=list)
+
+
+class ParseTextRequest(BaseModel):
+    """Raw spec text (YAML or JSON) for the web 'New from YAML' import flow."""
+
+    text: str
+
+
+class ParseResponse(BaseModel):
+    valid: bool = True
+    spec_hash: str
+    spec: SpecBody  # the parsed, validated spec body (JSON form)
+
+
+class HashResponse(BaseModel):
+    spec_hash: str
+
+
+class EstimateResponse(BaseModel):
+    estimated_runtime_seconds: float
+    estimated_ram_mb: float
+    estimated_size_bytes: int
+    features: int
+    edges: int
+    gpu_required: bool = False
+
+
+# --- specs of a dataset -------------------------------------------------------
+class SpecSummary(BaseModel):
+    spec_id: str
+    spec_hash: str
+    version: int
+    datadoom_version: str
+    created_at: str
+
+
+class SpecDetail(SpecSummary):
+    body: SpecBody
+
+
+class SaveSpecResponse(BaseModel):
+    spec_id: str
+    spec_hash: str
+    version: int
+
+
+# --- runs ---------------------------------------------------------------------
+class RunSummary(BaseModel):
+    run_id: str
+    dataset_id: str
+    spec_id: str
+    spec_hash: str | None = None
+    name: str | None = None
+    seed: int
+    status: str
+    stage: str | None = None
+    progress_pct: int = 0
+    compliance_score: float | None = None
+    error: dict[str, Any] | None = None
+    metrics: dict[str, Any] | None = None
+    started_at: str | None = None
+    finished_at: str | None = None
+    created_at: str
+
+
+class CreateRunRequest(BaseModel):
+    seed: int | None = None
+    name: str | None = None
+
+
+class UpdateRunRequest(BaseModel):
+    name: str
+
+
+class CreateRunResponse(BaseModel):
+    run_id: str
+    status: str
+    seed: int
+    ws: str
+
+
+class CancelResponse(BaseModel):
+    status: str
+
+
+# --- datasets -----------------------------------------------------------------
+class LatestRun(BaseModel):
+    run_id: str
+    status: str
+    compliance_score: float | None = None
+
+
+class DatasetSummary(BaseModel):
+    dataset_id: str
+    name: str
+    description: str | None = None
+    status: str
+    rows: int | None = None
+    features: int | None = None
+    compliance_score: float | None = None
+    created_at: str
+    updated_at: str
+
+
+class DatasetList(BaseModel):
+    items: list[DatasetSummary]
+    total: int
+
+
+class Dataset(BaseModel):
+    dataset_id: str
+    name: str
+    description: str | None = None
+    status: str
+    current_spec: SpecDetail | None = None
+    latest_run: LatestRun | None = None
+    created_at: str
+    updated_at: str
+
+
+class CreateDatasetRequest(BaseModel):
+    name: str
+    description: str | None = None
+    spec: SpecBody | None = None
+
+
+class UpdateDatasetRequest(BaseModel):
+    name: str | None = None
+    description: str | None = None
+
+
+# --- artifacts & reports ------------------------------------------------------
+class Artifact(BaseModel):
+    artifact_id: str
+    run_id: str
+    version: str
+    split: str | None = None
+    format: str
+    filename: str
+    size_bytes: int
+    checksum_sha256: str
+    created_at: str
+
+
+class Report(BaseModel):
+    report_id: str
+    run_id: str
+    compliance_score: float | None = None
+    distribution: dict[str, Any] | None = None
+    correlation: dict[str, Any] | None = None
+    mutual_information: dict[str, Any] | None = None
+    causal_truth: dict[str, Any] | None = None
+    difficulty: dict[str, Any] | None = None
+    failures: dict[str, Any] | None = None
+    profile: dict[str, Any] | None = None
+    determinism: dict[str, Any] | None = None
+
+
+class PreviewResponse(BaseModel):
+    columns: list[str]
+    rows: list[list[Any]]
+    total: int
+
+
+# --- templates & plugins & meta ----------------------------------------------
+class TemplateSummary(BaseModel):
+    id: str
+    name: str
+    domain: str
+    description: str
+    tags: list[str] = Field(default_factory=list)
+    level: str = "starter"  # "starter" | "hackathon"
+
+
+class TemplateDetail(TemplateSummary):
+    spec: dict[str, Any]
+
+
+class PluginInfo(BaseModel):
+    name: str
+    kind: str
+    version: str | None = None
+    schema_: dict[str, Any] | None = Field(default=None, alias="schema")
+    source: str = "builtin"  # builtin | entrypoint | local
+    builtin: bool = True
+    enabled: bool = True
+
+
+class HealthResponse(BaseModel):
+    status: str = "ok"
+
+
+class VersionResponse(BaseModel):
+    version: str
+    datadoom_version: str
+    python: str
+    platform: str

datadoom/api/serializers.py

@@ -0,0 +1,143 @@
+"""ORM row -> API schema converters (keeps route bodies thin)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from datadoom.store import (
+    ArtifactRow,
+    DatasetRow,
+    GenerationRunRow,
+    ReportRow,
+    SpecRow,
+)
+
+from . import schemas
+
+
+def spec_summary(row: SpecRow) -> schemas.SpecSummary:
+    return schemas.SpecSummary(
+        spec_id=row.spec_id,
+        spec_hash=row.spec_hash,
+        version=row.version,
+        datadoom_version=row.datadoom_version,
+        created_at=row.created_at,
+    )
+
+
+def spec_detail(row: SpecRow) -> schemas.SpecDetail:
+    return schemas.SpecDetail(
+        spec_id=row.spec_id,
+        spec_hash=row.spec_hash,
+        version=row.version,
+        datadoom_version=row.datadoom_version,
+        created_at=row.created_at,
+        body=dict(row.body),
+    )
+
+
+def run_summary(row: GenerationRunRow) -> schemas.RunSummary:
+    compliance = None
+    if row.metrics:
+        compliance = row.metrics.get("compliance_score")
+    return schemas.RunSummary(
+        run_id=row.run_id,
+        dataset_id=row.dataset_id,
+        spec_id=row.spec_id,
+        spec_hash=row.spec.spec_hash if row.spec is not None else None,
+        name=row.name,
+        seed=row.seed,
+        status=row.status,
+        stage=row.stage,
+        progress_pct=row.progress_pct,
+        compliance_score=compliance,
+        error=row.error,
+        metrics=row.metrics,
+        started_at=row.started_at,
+        finished_at=row.finished_at,
+        created_at=row.created_at,
+    )
+
+
+def latest_run(row: GenerationRunRow | None) -> schemas.LatestRun | None:
+    if row is None:
+        return None
+    compliance = row.metrics.get("compliance_score") if row.metrics else None
+    return schemas.LatestRun(
+        run_id=row.run_id, status=row.status, compliance_score=compliance
+    )
+
+
+def _spec_stats(body: dict[str, Any] | None) -> tuple[int | None, int | None]:
+    if not body:
+        return None, None
+    rows = body.get("rows")
+    features = body.get("features")
+    return rows, (len(features) if isinstance(features, dict) else None)
+
+
+def dataset_summary(
+    row: DatasetRow, current_spec: SpecRow | None, latest: GenerationRunRow | None
+) -> schemas.DatasetSummary:
+    rows, features = _spec_stats(current_spec.body if current_spec else None)
+    compliance = latest.metrics.get("compliance_score") if latest and latest.metrics else None
+    return schemas.DatasetSummary(
+        dataset_id=row.dataset_id,
+        name=row.name,
+        description=row.description,
+        status=row.status,
+        rows=rows,
+        features=features,
+        compliance_score=compliance,
+        created_at=row.created_at,
+        updated_at=row.updated_at,
+    )
+
+
+def dataset(
+    row: DatasetRow, current_spec: SpecRow | None, latest: GenerationRunRow | None
+) -> schemas.Dataset:
+    return schemas.Dataset(
+        dataset_id=row.dataset_id,
+        name=row.name,
+        description=row.description,
+        status=row.status,
+        current_spec=spec_detail(current_spec) if current_spec else None,
+        latest_run=latest_run(latest),
+        created_at=row.created_at,
+        updated_at=row.updated_at,
+    )
+
+
+def artifact(row: ArtifactRow) -> schemas.Artifact:
+    # The real on-disk filename is the basename of the storage URI — the
+    # authoritative name (data.csv, data.injected.csv, metadata.json, …) so the
+    # UI never has to guess clean-vs-injected from version/format.
+    filename = row.storage_uri.replace("\\", "/").rsplit("/", 1)[-1]
+    return schemas.Artifact(
+        artifact_id=row.artifact_id,
+        run_id=row.run_id,
+        version=row.version,
+        split=row.split,
+        format=row.format,
+        filename=filename,
+        size_bytes=row.size_bytes,
+        checksum_sha256=row.checksum_sha256,
+        created_at=row.created_at,
+    )
+
+
+def report(row: ReportRow) -> schemas.Report:
+    return schemas.Report(
+        report_id=row.report_id,
+        run_id=row.run_id,
+        compliance_score=row.compliance_score,
+        distribution=row.distribution,
+        correlation=row.correlation,
+        mutual_information=row.mutual_information,
+        causal_truth=row.causal_truth,
+        difficulty=row.difficulty,
+        failures=row.failures,
+        profile=row.profile,
+        determinism=row.determinism,
+    )

datadoom/api/state.py

@@ -0,0 +1,24 @@
+"""Shared application state, assembled by the app factory and hung on
+``app.state.dd``. Holds the singletons the routes need: config, the database,
+the artifact store, the event hub, and the worker pool.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from datadoom.config import Config
+from datadoom.jobs import EventHub, WorkerPool
+from datadoom.store import ArtifactStore, Database
+
+
+@dataclass
+class AppState:
+    config: Config
+    db: Database
+    artifacts: ArtifactStore
+    hub: EventHub
+    worker: WorkerPool
+    # In-process idempotency map: (dataset_id, key) -> run_id (08 §1). Sufficient
+    # for the single-process local server; team mode would persist this.
+    idempotency: dict[tuple[str, str], str] = field(default_factory=dict)

datadoom/api/store_helpers.py

@@ -0,0 +1,56 @@
+"""Thin helpers shared by routes: dataset loading + latest-run lookup.
+
+Re-exports the store repositories so route modules import them from one place,
+and centralizes the "404 if missing" and "latest run" patterns.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+from datadoom.store import (
+    ArtifactRepository,
+    DatasetRepository,
+    DatasetRow,
+    GenerationRunRow,
+    ReportRepository,
+    RunRepository,
+    SpecRepository,
+)
+
+from .errors import http_error
+
+__all__ = [
+    "DatasetRepository",
+    "SpecRepository",
+    "RunRepository",
+    "ArtifactRepository",
+    "ReportRepository",
+    "load_dataset",
+    "load_run",
+    "latest_run_row",
+]
+
+
+def load_dataset(session: Session, dataset_id: str) -> DatasetRow:
+    row = DatasetRepository(session).get(dataset_id)
+    if row is None:
+        raise http_error(404, "not_found", f"dataset {dataset_id} not found")
+    return row
+
+
+def load_run(session: Session, run_id: str) -> GenerationRunRow:
+    row = RunRepository(session).get(run_id)
+    if row is None:
+        raise http_error(404, "not_found", f"run {run_id} not found")
+    return row
+
+
+def latest_run_row(runs: RunRepository, dataset: DatasetRow) -> GenerationRunRow | None:
+    """Prefer the dataset's recorded latest run, else the most recent by time."""
+    if dataset.latest_run_id:
+        found = runs.get(dataset.latest_run_id)
+        if found is not None:
+            return found
+    rows = runs.list_for_dataset(dataset.dataset_id)
+    return rows[0] if rows else None

datadoom/api/ws.py

@@ -0,0 +1,72 @@
+"""Live progress transport (08 §7): WebSocket primary, SSE fallback.
+
+Both subscribe to the :class:`~datadoom.jobs.progress.EventHub`, which replays
+the stage events so far to a late subscriber, then streams live updates until a
+terminal event (``completed`` / ``failed`` / ``cancelled``). The WS channel also
+accepts ``{"type":"cancel"}`` from the client.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+
+from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import StreamingResponse
+
+from .state import AppState
+
+router = APIRouter(tags=["ws"])
+
+_TERMINAL = {"completed", "failed", "cancelled"}
+
+
+@router.websocket("/api/ws/runs/{run_id}")
+async def ws_run(websocket: WebSocket, run_id: str) -> None:
+    await websocket.accept()
+    state: AppState = websocket.app.state.dd
+    hub = state.hub
+
+    async def pump_client() -> None:
+        # Listen for client -> server messages (only "cancel" is meaningful).
+        try:
+            while True:
+                msg = await websocket.receive_text()
+                try:
+                    data = json.loads(msg)
+                except ValueError:
+                    continue
+                if data.get("type") == "cancel":
+                    hub.request_cancel(run_id)
+        except WebSocketDisconnect:
+            return
+
+    client_task = asyncio.create_task(pump_client())
+    try:
+        async for event in hub.subscribe(run_id):
+            await websocket.send_json(event)
+            if event.get("type") in _TERMINAL:
+                break
+    except WebSocketDisconnect:
+        pass
+    finally:
+        client_task.cancel()
+        with contextlib.suppress(RuntimeError):
+            await websocket.close()
+
+
+@router.get("/api/runs/{run_id}/events")
+async def sse_run(run_id: str, request: Request) -> StreamingResponse:
+    state: AppState = request.app.state.dd
+    hub = state.hub
+
+    async def event_stream():  # noqa: ANN202
+        async for event in hub.subscribe(run_id):
+            if await request.is_disconnected():
+                break
+            yield f"data: {json.dumps(event)}\n\n"
+            if event.get("type") in _TERMINAL:
+                break
+
+    return StreamingResponse(event_stream(), media_type="text/event-stream")

datadoom/cli/__init__.py

@@ -0,0 +1 @@
+"""DataDoom command-line interface."""