datadoom 0.1.0.dev0__py3-none-any.whl

datadoom/api/routes/artifacts.py

@@ -0,0 +1,147 @@
+"""Artifacts, preview, report, and bundle download (08 §8).
+
+These power the Results screen: list output files, stream a download (with the
+reproducibility checksum in a header), preview the first rows, fetch the full
+report, or download a zip bundle (artifacts + metadata + spec).
+"""
+
+from __future__ import annotations
+
+import io
+import zipfile
+from pathlib import Path
+
+import pandas as pd
+from fastapi import APIRouter, Depends, Query
+from fastapi.responses import FileResponse, StreamingResponse
+from sqlalchemy.orm import Session
+
+from .. import serializers
+from ..deps import get_session, get_state
+from ..errors import http_error
+from ..schemas import Artifact, PreviewResponse, Report
+from ..state import AppState
+from ..store_helpers import (
+    ArtifactRepository,
+    ReportRepository,
+    SpecRepository,
+    load_run,
+)
+
+router = APIRouter(prefix="/api", tags=["artifacts"])
+
+
+@router.get("/runs/{run_id}/artifacts", response_model=list[Artifact])
+def list_artifacts(run_id: str, s: Session = Depends(get_session)) -> list[Artifact]:
+    load_run(s, run_id)
+    return [serializers.artifact(a) for a in ArtifactRepository(s).list_for_run(run_id)]
+
+
+@router.get("/artifacts/{artifact_id}/download")
+def download_artifact(
+    artifact_id: str,
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> FileResponse:
+    art = ArtifactRepository(s).get(artifact_id)
+    if art is None:
+        raise http_error(404, "not_found", f"artifact {artifact_id} not found")
+    path = state.artifacts.open_uri(art.storage_uri)
+    if not path.exists():
+        raise http_error(404, "not_found", "artifact file is missing on disk")
+    return FileResponse(
+        path,
+        filename=path.name,
+        headers={"X-Checksum-SHA256": art.checksum_sha256},
+    )
+
+
+@router.get("/runs/{run_id}/spec.yaml")
+def spec_yaml(
+    run_id: str,
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> FileResponse:
+    """Download the locked, resolved spec YAML (spec + baked-in seed) for a run.
+
+    This is the version-control / reproducibility record: the exact spec, with the
+    resolved seed, that produced this generation. Regenerating from it yields
+    byte-identical data.
+    """
+    run = load_run(s, run_id)
+    path = state.artifacts.run_dir(run.dataset_id, run_id) / "spec.resolved.yaml"
+    if not path.exists():
+        raise http_error(404, "not_found", "resolved spec is not available for this run")
+    return FileResponse(
+        path,
+        filename=f"{run_id[:8]}.spec.datadoom.yaml",
+        media_type="application/x-yaml",
+    )
+
+
+@router.get("/runs/{run_id}/report", response_model=Report)
+def get_report(run_id: str, s: Session = Depends(get_session)) -> Report:
+    load_run(s, run_id)
+    rep = ReportRepository(s).get_for_run(run_id)
+    if rep is None:
+        raise http_error(404, "not_found", "no report (run not completed)")
+    return serializers.report(rep)
+
+
+@router.get("/runs/{run_id}/preview", response_model=PreviewResponse)
+def preview(
+    run_id: str,
+    version: str = "clean",
+    split: str = "full",
+    limit: int = Query(100, ge=1, le=5000),
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> PreviewResponse:
+    run = load_run(s, run_id)
+    arts = [
+        a
+        for a in ArtifactRepository(s).list_for_run(run_id)
+        if a.version == version and (a.split or "full") == split
+    ]
+    # Prefer CSV (always readable); fall back to JSON, then Parquet if that's all
+    # the spec exported. Keeps preview working for non-CSV format selections.
+    priority = {"csv": 0, "json": 1, "parquet": 2}
+    candidates = sorted(
+        (a for a in arts if a.format in priority), key=lambda a: priority[a.format]
+    )
+    target = candidates[0] if candidates else None
+    if target is None:
+        raise http_error(404, "not_found", "no matching data artifact to preview")
+
+    path = state.artifacts.open_uri(target.storage_uri)
+    if not path.exists():
+        raise http_error(404, "not_found", "artifact file is missing on disk")
+
+    if target.format == "json":
+        frame = pd.read_json(path).head(limit)
+    elif target.format == "parquet":
+        frame = pd.read_parquet(path).head(limit)
+    else:
+        frame = pd.read_csv(path, nrows=limit)
+    spec_row = SpecRepository(s).get(run.spec_id)
+    total = (spec_row.body.get("rows") if spec_row else None) or len(frame)
+    rows = frame.where(pd.notna(frame), None).values.tolist()
+    return PreviewResponse(columns=list(frame.columns), rows=rows, total=int(total))
+
+
+@router.get("/runs/{run_id}/bundle")
+def bundle(
+    run_id: str,
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> StreamingResponse:
+    run = load_run(s, run_id)
+    run_dir: Path = state.artifacts.run_dir(run.dataset_id, run_id)
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        for f in sorted(run_dir.glob("*")):
+            if f.is_file():
+                zf.write(f, arcname=f.name)
+    buf.seek(0)
+    headers = {"Content-Disposition": f'attachment; filename="{run_id}.zip"'}
+    return StreamingResponse(buf, media_type="application/zip", headers=headers)

datadoom/api/routes/datasets.py

@@ -0,0 +1,180 @@
+"""Dataset CRUD + spec versioning (08 §4-5).
+
+Editing a spec never mutates a row — it creates a new immutable version and
+repoints ``current_spec_id`` (the immutability invariant, 06 §5).
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Query, Response
+from sqlalchemy.orm import Session
+
+from datadoom.engine import parse_spec
+
+from .. import serializers
+from ..deps import get_session, get_state
+from ..errors import http_error
+from ..schemas import (
+    CreateDatasetRequest,
+    Dataset,
+    DatasetList,
+    SaveSpecResponse,
+    SpecBody,
+    SpecDetail,
+    SpecSummary,
+    UpdateDatasetRequest,
+)
+from ..state import AppState
+from ..store_helpers import (
+    DatasetRepository,
+    RunRepository,
+    SpecRepository,
+    latest_run_row,
+    load_dataset,
+)
+
+router = APIRouter(prefix="/api/datasets", tags=["datasets"])
+
+
+@router.get("", response_model=DatasetList)
+def list_datasets(
+    status: str | None = None,
+    q: str | None = None,
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+    s: Session = Depends(get_session),
+) -> DatasetList:
+    rows, total = DatasetRepository(s).list(status=status, q=q, limit=limit, offset=offset)
+    specs = SpecRepository(s)
+    runs = RunRepository(s)
+    items = [
+        serializers.dataset_summary(r, specs.current(r), latest_run_row(runs, r))
+        for r in rows
+    ]
+    return DatasetList(items=items, total=total)
+
+
+@router.post("", response_model=Dataset, status_code=201)
+def create_dataset(
+    req: CreateDatasetRequest, s: Session = Depends(get_session)
+) -> Dataset:
+    datasets = DatasetRepository(s)
+    if datasets.get_by_name(req.name) is not None:
+        raise http_error(409, "conflict", f"a dataset named {req.name!r} already exists")
+
+    row = datasets.create(name=req.name, description=req.description)
+    current_spec = None
+    if req.spec is not None:
+        spec = parse_spec(req.spec)  # raises 422 with locator on invalid
+        current_spec = SpecRepository(s).create_version(
+            row, spec.body(), spec.spec_hash(), spec.datadoom_version
+        )
+    return serializers.dataset(row, current_spec, None)
+
+
+@router.get("/{dataset_id}", response_model=Dataset)
+def get_dataset(dataset_id: str, s: Session = Depends(get_session)) -> Dataset:
+    row = load_dataset(s, dataset_id)
+    current_spec = SpecRepository(s).current(row)
+    latest = latest_run_row(RunRepository(s), row)
+    return serializers.dataset(row, current_spec, latest)
+
+
+@router.patch("/{dataset_id}", response_model=Dataset)
+def update_dataset(
+    dataset_id: str, req: UpdateDatasetRequest, s: Session = Depends(get_session)
+) -> Dataset:
+    datasets = DatasetRepository(s)
+    row = load_dataset(s, dataset_id)
+    if (
+        req.name is not None
+        and req.name != row.name
+        and datasets.get_by_name(req.name) is not None
+    ):
+        raise http_error(409, "conflict", f"a dataset named {req.name!r} already exists")
+    datasets.update(row, name=req.name, description=req.description)
+    current_spec = SpecRepository(s).current(row)
+    latest = latest_run_row(RunRepository(s), row)
+    return serializers.dataset(row, current_spec, latest)
+
+
+@router.delete("/{dataset_id}", status_code=204)
+def delete_dataset(
+    dataset_id: str,
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> Response:
+    row = load_dataset(s, dataset_id)
+    DatasetRepository(s).delete(row)  # ORM cascade -> specs/runs/artifacts/reports
+    state.artifacts.remove_dataset(dataset_id)  # removes the artifact directory
+    return Response(status_code=204)
+
+
+@router.post("/{dataset_id}/duplicate", response_model=Dataset, status_code=201)
+def duplicate_dataset(dataset_id: str, s: Session = Depends(get_session)) -> Dataset:
+    datasets = DatasetRepository(s)
+    specs = SpecRepository(s)
+    src = load_dataset(s, dataset_id)
+    src_spec = specs.current(src)
+
+    new_name = _unique_copy_name(datasets, src.name)
+    clone = datasets.create(name=new_name, description=src.description)
+    new_spec = None
+    if src_spec is not None:
+        new_spec = specs.create_version(
+            clone, dict(src_spec.body), src_spec.spec_hash, src_spec.datadoom_version
+        )
+    return serializers.dataset(clone, new_spec, None)
+
+
+# --- spec versioning ----------------------------------------------------------
+@router.put("/{dataset_id}/spec", response_model=SaveSpecResponse)
+def save_spec(
+    dataset_id: str, body: SpecBody, s: Session = Depends(get_session)
+) -> SaveSpecResponse:
+    row = load_dataset(s, dataset_id)
+    spec = parse_spec(body)  # 422 with locator on invalid
+    new_spec = SpecRepository(s).create_version(
+        row, spec.body(), spec.spec_hash(), spec.datadoom_version
+    )
+    # A fresh edit returns the dataset to a draft state (a new run regenerates it).
+    if row.status in {"completed", "failed"}:
+        DatasetRepository(s).set_status(row, "draft")
+    return SaveSpecResponse(
+        spec_id=new_spec.spec_id, spec_hash=new_spec.spec_hash, version=new_spec.version
+    )
+
+
+@router.get("/{dataset_id}/spec", response_model=SpecDetail)
+def get_current_spec(dataset_id: str, s: Session = Depends(get_session)) -> SpecDetail:
+    row = load_dataset(s, dataset_id)
+    spec = SpecRepository(s).current(row)
+    if spec is None:
+        raise http_error(404, "not_found", "dataset has no spec yet")
+    return serializers.spec_detail(spec)
+
+
+@router.get("/{dataset_id}/spec/history", response_model=list[SpecSummary])
+def spec_history(dataset_id: str, s: Session = Depends(get_session)) -> list[SpecSummary]:
+    load_dataset(s, dataset_id)
+    return [serializers.spec_summary(r) for r in SpecRepository(s).history(dataset_id)]
+
+
+@router.get("/{dataset_id}/spec/{version}", response_model=SpecDetail)
+def get_spec_version(
+    dataset_id: str, version: int, s: Session = Depends(get_session)
+) -> SpecDetail:
+    load_dataset(s, dataset_id)
+    spec = SpecRepository(s).by_version(dataset_id, version)
+    if spec is None:
+        raise http_error(404, "not_found", f"no spec version {version}")
+    return serializers.spec_detail(spec)
+
+
+def _unique_copy_name(datasets: DatasetRepository, base: str) -> str:
+    candidate = f"{base}-copy"
+    i = 2
+    while datasets.get_by_name(candidate) is not None:
+        candidate = f"{base}-copy-{i}"
+        i += 1
+    return candidate

datadoom/api/routes/meta.py

@@ -0,0 +1,45 @@
+"""Meta endpoints (08 §11): health + version."""
+
+from __future__ import annotations
+
+import platform
+import sys
+from typing import Any
+
+from fastapi import APIRouter
+
+from datadoom.engine.reference import build_capabilities
+from datadoom.version import __version__
+
+from ..schemas import HealthResponse, VersionResponse
+
+router = APIRouter(prefix="/api", tags=["meta"])
+
+# The spec format version DataDoom currently authors/reads (independent of the
+# HTTP API version, 08 §13).
+DATADOOM_SPEC_VERSION = "1"
+
+
+@router.get("/health", response_model=HealthResponse)
+def health() -> HealthResponse:
+    return HealthResponse(status="ok")
+
+
+@router.get("/version", response_model=VersionResponse)
+def version() -> VersionResponse:
+    return VersionResponse(
+        version=__version__,
+        datadoom_version=DATADOOM_SPEC_VERSION,
+        python=platform.python_version(),
+        platform=f"{platform.system()} {platform.release()} ({sys.platform})",
+    )
+
+
+@router.get("/spec-reference")
+def spec_reference() -> dict[str, Any]:
+    """Machine-readable spec capabilities manifest (for AI/tooling authoring).
+
+    Built from the live registries, so plugin-registered capabilities are
+    included. Mirrors the ``datadoom spec-reference`` CLI.
+    """
+    return build_capabilities()

datadoom/api/routes/plugins.py

@@ -0,0 +1,22 @@
+"""Plugins endpoint (08 §10).
+
+Returns the live plugin registry — core built-ins plus anything discovered from
+entry points or the local plugins directory at startup (09 §3). The Canvas reads
+each entry's ``schema`` fragment to render config controls for third-party
+capabilities with no frontend changes (09 §6).
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from datadoom.plugins import get_registry
+
+from ..schemas import PluginInfo
+
+router = APIRouter(prefix="/api/plugins", tags=["plugins"])
+
+
+@router.get("", response_model=list[PluginInfo])
+def list_plugins() -> list[PluginInfo]:
+    return [PluginInfo(**record.to_info()) for record in get_registry().records()]

datadoom/api/routes/runs.py

@@ -0,0 +1,144 @@
+"""Generation run endpoints (08 §6, §9).
+
+Creating a run returns ``202`` immediately with a resolved seed and a WebSocket
+path; the worker executes it asynchronously and streams progress. Repeated
+``Idempotency-Key`` headers return the existing run (``200``).
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Header, Response
+from sqlalchemy.orm import Session
+
+from datadoom.engine import parse_spec, resolve_seed
+
+from .. import serializers
+from ..deps import get_session, get_state
+from ..errors import http_error
+from ..schemas import (
+    CancelResponse,
+    CreateRunRequest,
+    CreateRunResponse,
+    RunSummary,
+    UpdateRunRequest,
+)
+from ..state import AppState
+from ..store_helpers import (
+    RunRepository,
+    SpecRepository,
+    load_dataset,
+    load_run,
+)
+
+router = APIRouter(prefix="/api", tags=["runs"])
+
+
+def _ws_path(run_id: str) -> str:
+    return f"/api/ws/runs/{run_id}"
+
+
+@router.post("/datasets/{dataset_id}/runs", response_model=CreateRunResponse, status_code=202)
+def create_run(
+    dataset_id: str,
+    req: CreateRunRequest,
+    response: Response,
+    idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"),
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> CreateRunResponse:
+    dataset = load_dataset(s, dataset_id)
+    spec_row = SpecRepository(s).current(dataset)
+    if spec_row is None:
+        raise http_error(400, "bad_request", "dataset has no spec to generate from")
+
+    # Idempotency replay (08 §1): same key -> the existing run, 200.
+    if idempotency_key is not None:
+        existing_id = state.idempotency.get((dataset_id, idempotency_key))
+        if existing_id is not None:
+            existing = RunRepository(s).get(existing_id)
+            if existing is not None:
+                response.status_code = 200
+                return CreateRunResponse(
+                    run_id=existing.run_id,
+                    status=existing.status,
+                    seed=existing.seed,
+                    ws=_ws_path(existing.run_id),
+                )
+
+    spec = parse_spec(dict(spec_row.body))
+    seed = resolve_seed(spec, req.seed)
+    name = req.name.strip() if req.name and req.name.strip() else None
+    run = RunRepository(s).create(dataset.dataset_id, spec_row.spec_id, seed, name=name)
+    run_id = run.run_id
+    if idempotency_key is not None:
+        state.idempotency[(dataset_id, idempotency_key)] = run_id
+
+    # Commit the queued row before handing off to the worker thread so it is
+    # visible when the worker opens its own session (the request's own context
+    # manager will no-op commit again at the end).
+    s.commit()
+    state.worker.submit(run_id)
+
+    return CreateRunResponse(run_id=run_id, status="queued", seed=seed, ws=_ws_path(run_id))
+
+
+@router.get("/runs/{run_id}", response_model=RunSummary)
+def get_run(run_id: str, s: Session = Depends(get_session)) -> RunSummary:
+    return serializers.run_summary(load_run(s, run_id))
+
+
+@router.get("/datasets/{dataset_id}/runs", response_model=list[RunSummary])
+def list_runs(dataset_id: str, s: Session = Depends(get_session)) -> list[RunSummary]:
+    load_dataset(s, dataset_id)
+    rows = RunRepository(s).list_for_dataset(dataset_id)
+    return [serializers.run_summary(r) for r in rows]
+
+
+@router.patch("/runs/{run_id}", response_model=RunSummary)
+def update_run(
+    run_id: str, req: UpdateRunRequest, s: Session = Depends(get_session)
+) -> RunSummary:
+    run = load_run(s, run_id)
+    name = req.name.strip()
+    if not name:
+        raise http_error(422, "validation_error", "a generation name is required")
+    RunRepository(s).set_name(run, name)
+    return serializers.run_summary(run)
+
+
+@router.delete("/runs/{run_id}", status_code=204)
+def delete_run(
+    run_id: str,
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> Response:
+    run = load_run(s, run_id)
+    if run.status in {"queued", "running"}:
+        raise http_error(409, "conflict", "cancel the run before deleting it")
+    dataset_id = run.dataset_id
+    RunRepository(s).delete(run)  # ORM cascade -> artifacts/report rows
+    state.artifacts.remove_run(dataset_id, run_id)  # remove the run's files
+    return Response(status_code=204)
+
+
+@router.post("/runs/{run_id}/cancel", response_model=CancelResponse)
+def cancel_run(
+    run_id: str,
+    s: Session = Depends(get_session),
+    state: AppState = Depends(get_state),
+) -> CancelResponse:
+    run = load_run(s, run_id)
+    if run.status in {"completed", "failed", "cancelled"}:
+        return CancelResponse(status=run.status)
+    # Cooperative: flag the run; the worker aborts at the next stage boundary.
+    state.hub.request_cancel(run_id)
+    return CancelResponse(status="cancelling")
+
+
+@router.post("/runs/{run_id}/inject", status_code=501)
+def inject_failures(run_id: str, s: Session = Depends(get_session)) -> Response:
+    """Failure-injected variant (08 §9). The failure engine lands in P3 (task 13)."""
+    load_run(s, run_id)
+    raise http_error(
+        501, "not_implemented", "failure injection arrives in Phase 3 (engine/failure)"
+    )

datadoom/api/routes/specs.py

@@ -0,0 +1,73 @@
+"""Stateless spec helpers (08 §3): validate, hash, estimate.
+
+These never touch the DB — they parse the posted spec through ``engine.spec``
+(the single source of validation truth) and return derived facts.
+"""
+
+from __future__ import annotations
+
+import yaml
+from fastapi import APIRouter
+
+from datadoom.engine import parse_spec
+from datadoom.engine.errors import SpecValidationError
+
+from ..estimate import estimate as estimate_spec
+from ..schemas import (
+    EstimateResponse,
+    HashResponse,
+    ParseResponse,
+    ParseTextRequest,
+    SpecBody,
+    ValidateResponse,
+)
+
+router = APIRouter(prefix="/api/specs", tags=["specs"])
+
+
+@router.post("/validate", response_model=ValidateResponse)
+def validate(body: SpecBody) -> ValidateResponse:
+    # parse_spec raises SpecValidationError -> 422 with locator (handled centrally).
+    spec = parse_spec(body)
+    return ValidateResponse(valid=True, spec_hash=spec.spec_hash(), warnings=[])
+
+
+@router.post("/parse", response_model=ParseResponse)
+def parse(body: ParseTextRequest) -> ParseResponse:
+    """Parse raw YAML/JSON spec text → validated spec body (web 'New from YAML').
+
+    YAML is parsed by the same PyYAML loader the CLI uses, then validated through
+    the single ``engine.spec`` path, so the web import accepts exactly what
+    ``datadoom run file.yaml`` would. Syntax and validation errors come back as a
+    422 with a ``locator`` (handled centrally).
+    """
+    try:
+        data = yaml.safe_load(body.text)
+    except yaml.YAMLError as exc:
+        mark = getattr(exc, "problem_mark", None)
+        loc = f"line {mark.line + 1}" if mark is not None else None
+        raise SpecValidationError(f"invalid YAML: {exc}", locator=loc) from exc
+    if not isinstance(data, dict):
+        raise SpecValidationError("spec must be a mapping at the top level (key: value …)")
+    spec = parse_spec(data)
+    return ParseResponse(valid=True, spec_hash=spec.spec_hash(), spec=spec.body())
+
+
+@router.post("/hash", response_model=HashResponse)
+def spec_hash(body: SpecBody) -> HashResponse:
+    spec = parse_spec(body)
+    return HashResponse(spec_hash=spec.spec_hash())
+
+
+@router.post("/estimate", response_model=EstimateResponse)
+def estimate(body: SpecBody) -> EstimateResponse:
+    spec = parse_spec(body)
+    est = estimate_spec(spec)
+    return EstimateResponse(
+        estimated_runtime_seconds=est.estimated_runtime_seconds,
+        estimated_ram_mb=est.estimated_ram_mb,
+        estimated_size_bytes=est.estimated_size_bytes,
+        features=est.features,
+        edges=est.edges,
+        gpu_required=est.gpu_required,
+    )

datadoom/api/routes/templates.py

@@ -0,0 +1,30 @@
+"""Templates endpoints (08 §10).
+
+Surfaces the built-in domain templates (17 step 18). The gallery lists them; the
+detail endpoint returns the full spec so the Canvas can create a dataset from it
+in one click (the existing create flow accepts a ``spec``).
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from datadoom.templates import get_template, list_templates, load_template_body
+
+from ..errors import http_error
+from ..schemas import TemplateDetail, TemplateSummary
+
+router = APIRouter(prefix="/api/templates", tags=["templates"])
+
+
+@router.get("", response_model=list[TemplateSummary])
+def list_all() -> list[TemplateSummary]:
+    return [TemplateSummary(**t.to_summary()) for t in list_templates()]
+
+
+@router.get("/{template_id}", response_model=TemplateDetail)
+def get_one(template_id: str) -> TemplateDetail:
+    meta = get_template(template_id)
+    if meta is None:
+        raise http_error(404, "not_found", f"template {template_id!r} not found")
+    return TemplateDetail(**meta.to_summary(), spec=load_template_body(template_id))