misbar 1.0.0__py3-none-any.whl

misbar/__init__.py

@@ -0,0 +1,38 @@
+"""misbar (مسبار) — foundational workflow spine, tracing, and human review toolkit."""
+
+from misbar.capture import JsonlRecorder
+from misbar.db import MisbarStore
+from misbar.lineage import apply_payload_patch, canonical_json, compute_input_digest, sha256_digest
+from misbar.runner import PartitionedStepResult, Pipeline, StepContext
+from misbar.tracing import (
+    CompositeSink,
+    JsonlSink,
+    MlflowSink,
+    NoopSink,
+    SqliteSink,
+    TraceSink,
+    ingest_capture_bundle,
+)
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "CompositeSink",
+    "JsonlRecorder",
+    "JsonlSink",
+    "MisbarStore",
+    "MlflowSink",
+    "NoopSink",
+    "PartitionedStepResult",
+    "Pipeline",
+    "SqliteSink",
+    "StepContext",
+    "TraceSink",
+    "__version__",
+    "apply_payload_patch",
+    "canonical_json",
+    "compute_input_digest",
+    "ingest_capture_bundle",
+    "sha256_digest",
+]
+

misbar/api.py

@@ -0,0 +1,629 @@
+"""FastAPI router and server-enforced blindness gates for misbar."""
+
+from __future__ import annotations
+
+import hashlib
+import random
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+from typing import Any, Literal
+
+from fastapi import APIRouter, FastAPI, HTTPException, Query
+from fastapi.responses import HTMLResponse, PlainTextResponse, Response
+from pydantic import BaseModel, Field
+
+from misbar.db import MisbarStore
+from misbar.tracing import MlflowSink, TraceSink
+
+BlindMode = Literal["full_blind", "reveal_after_submit", "ab_randomized", "unblinded"]
+
+
+@dataclass
+class QueueSpec:
+    """Declarative queue configuration enforcing blindness and action boundaries."""
+
+    queue_key: str
+    task_key: str
+    artifact_key: str
+    title: str = ""
+    blind_mode: BlindMode = "reveal_after_submit"
+    allow_mode_override: bool = False
+    secret_meta_keys: tuple[str, ...] = ("silver", "gold", "reference", "teacher_score")
+    ab_candidate_keys: tuple[str, str] = ("A", "B")
+    verdict_options: tuple[str, ...] = ("yes", "no")
+    allow_revisions: bool = False
+    revision_artifact_key: str | None = None
+    base_filter: Callable[[dict[str, Any]], bool] | None = None
+    filter_presets: dict[str, Callable[[dict[str, Any]], bool]] = field(default_factory=dict)
+
+
+def compute_ab_permutation(
+    reviewer_id: str,
+    item_key: str,
+    candidate_keys: tuple[str, str],
+) -> dict[str, str]:
+    """Deterministic 50/50 slot assignment per `(reviewer_id, item_key)`."""
+    seed_hex = hashlib.sha256(f"{reviewer_id}:{item_key}".encode("utf-8")).hexdigest()
+    rng = random.Random(int(seed_hex[:16], 16))
+    keys = [candidate_keys[0], candidate_keys[1]]
+    rng.shuffle(keys)
+    return {"slot_1": keys[0], "slot_2": keys[1]}
+
+
+def sanitize_item_for_client(
+    item_row: Mapping[str, Any],
+    spec: QueueSpec,
+    *,
+    effective_mode: BlindMode,
+    reviewer_id: str = "local",
+    mlflow_sink: MlflowSink | None = None,
+) -> dict[str, Any]:
+    """Strip model predictions and secret metadata on the server when blinded."""
+    ann = item_row.get("annotation")
+    has_ann = ann is not None
+    raw_meta = dict(item_row.get("meta") or {})
+    safe_meta = {k: v for k, v in raw_meta.items() if k not in spec.secret_meta_keys}
+    raw_output = item_row.get("output")
+
+    base = {
+        "item_row_id": item_row["item_row_id"],
+        "artifact_version_id": item_row["artifact_version_id"],
+        "run_id": item_row.get("run_id"),
+        "item_key": item_row["item_key"],
+        "partition_key": item_row.get("partition_key"),
+        "input_digest": item_row["input_digest"],
+        "input": item_row["input"],
+        "annotation": ann,
+        "annotation_status": item_row.get("annotation_status", "ungraded"),
+        "blind_mode": effective_mode,
+    }
+
+    if effective_mode == "full_blind":
+        return {
+            **base,
+            "blinded": True,
+            "output": None,
+            "meta": safe_meta,
+            "item_trace_id": None,
+            "item_span_id": None,
+            "mlflow_url": None,
+        }
+
+    if effective_mode == "reveal_after_submit":
+        if not has_ann:
+            return {
+                **base,
+                "blinded": True,
+                "output": None,
+                "meta": safe_meta,
+                "item_trace_id": None,
+                "item_span_id": None,
+                "mlflow_url": None,
+            }
+        trace_id = item_row.get("item_trace_id")
+        return {
+            **base,
+            "blinded": False,
+            "output": raw_output,
+            "meta": raw_meta,
+            "item_trace_id": trace_id,
+            "item_span_id": item_row.get("item_span_id"),
+            "mlflow_url": mlflow_sink.trace_ui_url(trace_id) if mlflow_sink else None,
+        }
+
+    if effective_mode == "ab_randomized":
+        perm = compute_ab_permutation(reviewer_id, str(item_row["item_key"]), spec.ab_candidate_keys)
+        candidates_src = (
+            raw_output.get("candidates", raw_output)
+            if isinstance(raw_output, Mapping)
+            else {}
+        )
+        ab_slots = {
+            "slot_1": candidates_src.get(perm["slot_1"]) if isinstance(candidates_src, Mapping) else None,
+            "slot_2": candidates_src.get(perm["slot_2"]) if isinstance(candidates_src, Mapping) else None,
+        }
+        if not has_ann:
+            return {
+                **base,
+                "blinded": True,
+                "output": {"ab_slots": ab_slots},
+                "meta": safe_meta,
+                "item_trace_id": None,
+                "item_span_id": None,
+                "mlflow_url": None,
+            }
+        trace_id = item_row.get("item_trace_id")
+        revealed_out = dict(raw_output) if isinstance(raw_output, Mapping) else {"raw": raw_output}
+        revealed_out["ab_slots"] = ab_slots
+        revealed_out["ab_mapping"] = perm
+        return {
+            **base,
+            "blinded": False,
+            "output": revealed_out,
+            "meta": raw_meta,
+            "item_trace_id": trace_id,
+            "item_span_id": item_row.get("item_span_id"),
+            "mlflow_url": mlflow_sink.trace_ui_url(trace_id) if mlflow_sink else None,
+        }
+
+    # unblinded
+    trace_id = item_row.get("item_trace_id")
+    return {
+        **base,
+        "blinded": False,
+        "output": raw_output,
+        "meta": raw_meta,
+        "item_trace_id": trace_id,
+        "item_span_id": item_row.get("item_span_id"),
+        "mlflow_url": mlflow_sink.trace_ui_url(trace_id) if mlflow_sink else None,
+    }
+
+
+class AnnotationRequest(BaseModel):
+    mutation_id: str | None = None
+    queue_key: str | None = None
+    task_key: str | None = None
+    item_key: str
+    input_digest: str
+    target_artifact_version_id: str | None = None
+    reviewer_id: str = "local"
+    blind_mode: BlindMode | None = None
+    verdict: str | None = None
+    scores: dict[str, Any] = Field(default_factory=dict)
+    corrected_output: Any = None
+    note: str | None = None
+
+
+class RevisionRequest(BaseModel):
+    mutation_id: str | None = None
+    queue_key: str | None = None
+    base_artifact_version_id: str
+    patch: dict[str, Any]
+    reason: str | None = None
+    source_annotation_id: str | None = None
+    created_by: str = "local"
+    trigger_rerun: bool = False
+    parent_run_id: str | None = None
+    config_overrides: dict[str, Any] | None = None
+
+
+def create_misbar_router(
+    store: MisbarStore,
+    *,
+    queues: Sequence[QueueSpec] | None = None,
+    sink: TraceSink | None = None,
+    mlflow_sink: MlflowSink | None = None,
+    rerun_handler: Callable[[str, str, dict[str, Any] | None], dict[str, Any]] | None = None,
+) -> APIRouter:
+    """Create the FastAPI `/api` router backed by `MisbarStore`."""
+    router = APIRouter(prefix="/api")
+    queue_map: dict[str, QueueSpec] = {q.queue_key: q for q in (queues or [])}
+
+    def _resolve_queues() -> dict[str, QueueSpec]:
+        if queue_map:
+            return queue_map
+        # Dynamic default queue discovery for zero-config `misbar serve`
+        discovered: dict[str, QueueSpec] = {}
+        seen_keys: set[str] = set()
+        for av in store.list_artifact_versions():
+            ak = av["artifact_key"]
+            if ak in seen_keys or av["item_count"] == 0:
+                continue
+            seen_keys.add(ak)
+            discovered[ak] = QueueSpec(
+                queue_key=ak,
+                task_key=ak,
+                artifact_key=ak,
+                title=f"{ak} (default)",
+                blind_mode="reveal_after_submit",
+                allow_mode_override=True,
+                allow_revisions=True,
+            )
+        return discovered
+
+    def _get_queue_and_av(
+        queue_key: str,
+        artifact_version_id: str | None = None,
+    ) -> tuple[QueueSpec, dict[str, Any]]:
+        qmap = _resolve_queues()
+        if queue_key not in qmap:
+            raise HTTPException(status_code=404, detail=f"Unknown queue '{queue_key}'")
+        spec = qmap[queue_key]
+        av = (
+            store.get_artifact_version(artifact_version_id)
+            if artifact_version_id
+            else store.get_latest_artifact_version(spec.artifact_key)
+        )
+        if not av:
+            raise HTTPException(
+                status_code=404,
+                detail=f"No artifact versions found for '{spec.artifact_key}'",
+            )
+        return spec, av
+
+    def _effective_mode(spec: QueueSpec, requested: BlindMode | None) -> BlindMode:
+        if requested and spec.allow_mode_override:
+            return requested
+        return spec.blind_mode
+
+    @router.get("/health")
+    def health() -> dict[str, Any]:
+        runs = store.list_runs(limit=5)
+        return {
+            "status": "ok",
+            "db_path": str(store.db_path),
+            "offline_ready": True,
+            "recent_runs": len(runs),
+            "mlflow_configured": mlflow_sink is not None,
+            "mlflow_ui_url": mlflow_sink.ui_base_url if mlflow_sink else None,
+        }
+
+    @router.get("/runs")
+    def list_runs(pipeline_key: str | None = None, limit: int = 50) -> dict[str, Any]:
+        runs = store.list_runs(pipeline_key=pipeline_key, limit=limit)
+        return {"runs": runs}
+
+    @router.get("/runs/{run_id}")
+    def get_run(run_id: str) -> dict[str, Any]:
+        run = store.get_run(run_id)
+        if not run:
+            raise HTTPException(status_code=404, detail=f"Unknown run '{run_id}'")
+        return {"run": run, "steps": store.list_steps(run_id)}
+
+    @router.get("/artifacts")
+    def list_artifacts(artifact_key: str | None = None) -> dict[str, Any]:
+        return {"artifact_versions": store.list_artifact_versions(artifact_key)}
+
+    @router.get("/queues")
+    def list_queues(reviewer_id: str = "local") -> dict[str, Any]:
+        qmap = _resolve_queues()
+        summaries = []
+        for qk, spec in qmap.items():
+            av = store.get_latest_artifact_version(spec.artifact_key)
+            counts = {"total": 0, "ungraded": 0, "graded_this_version": 0, "carried_forward": 0}
+            if av:
+                _, counts = store.query_queue_items(
+                    artifact_version_id=av["artifact_version_id"],
+                    task_key=spec.task_key,
+                    reviewer_id=reviewer_id,
+                    include_graded=True,
+                    filter_fn=spec.base_filter,
+                )
+            rev_av = (
+                store.get_latest_artifact_version(spec.revision_artifact_key)
+                if spec.revision_artifact_key
+                else av
+            )
+            summaries.append(
+                {
+                    "queue_key": qk,
+                    "title": spec.title or qk,
+                    "task_key": spec.task_key,
+                    "artifact_key": spec.artifact_key,
+                    "latest_artifact_version_id": av["artifact_version_id"] if av else None,
+                    "latest_version_num": av["version_num"] if av else None,
+                    "blind_mode": spec.blind_mode,
+                    "allow_mode_override": spec.allow_mode_override,
+                    "verdict_options": list(spec.verdict_options),
+                    "allow_revisions": spec.allow_revisions,
+                    "revision_artifact_key": spec.revision_artifact_key or spec.artifact_key,
+                    "revision_base_artifact_version_id": (
+                        rev_av["artifact_version_id"] if rev_av else None
+                    ),
+                    "filter_presets": sorted(spec.filter_presets.keys()),
+                    "counts": counts,
+                }
+            )
+        return {"queues": summaries}
+
+    @router.get("/queues/{queue_key}/items")
+    def get_queue_items(
+        queue_key: str,
+        reviewer_id: str = "local",
+        artifact_version_id: str | None = None,
+        include_graded: bool = True,
+        preset: str | None = None,
+        search: str | None = None,
+        blind_mode: BlindMode | None = None,
+        offset: int = Query(default=0, ge=0),
+        limit: int = Query(default=100, ge=1, le=500),
+    ) -> dict[str, Any]:
+        spec, av = _get_queue_and_av(queue_key, artifact_version_id)
+        mode = _effective_mode(spec, blind_mode)
+
+        preset_fn = None
+        if preset:
+            if preset not in spec.filter_presets:
+                raise HTTPException(status_code=400, detail=f"Unknown preset '{preset}'")
+            # Disallow output-dependent presets when queue is blinded to prevent side-channel leaks
+            if mode in ("full_blind", "reveal_after_submit"):
+                raise HTTPException(
+                    status_code=403,
+                    detail="Output-based filter presets are disabled on blind queues",
+                )
+            preset_fn = spec.filter_presets[preset]
+
+        def combined_filter(row: dict[str, Any]) -> bool:
+            if spec.base_filter is not None and not spec.base_filter(row):
+                return False
+            if preset_fn is not None and not preset_fn(row):
+                return False
+            if search:
+                needle = search.lower()
+                hay = f"{row['item_key']} {row['input']}".lower()
+                if needle not in hay:
+                    return False
+            return True
+
+        rows, counts = store.query_queue_items(
+            artifact_version_id=av["artifact_version_id"],
+            task_key=spec.task_key,
+            reviewer_id=reviewer_id,
+            include_graded=include_graded,
+            filter_fn=combined_filter if (spec.base_filter or preset_fn or search) else None,
+        )
+        sliced = rows[offset : offset + limit]
+        sanitized = [
+            sanitize_item_for_client(
+                r,
+                spec,
+                effective_mode=mode,
+                reviewer_id=reviewer_id,
+                mlflow_sink=mlflow_sink,
+            )
+            for r in sliced
+        ]
+        return {
+            "queue_key": queue_key,
+            "artifact_version": av,
+            "blind_mode": mode,
+            "counts": counts,
+            "offset": offset,
+            "limit": limit,
+            "items": sanitized,
+        }
+
+    @router.get("/queues/{queue_key}/items/{item_key}/spans")
+    def get_item_spans(
+        queue_key: str,
+        item_key: str,
+        reviewer_id: str = "local",
+        artifact_version_id: str | None = None,
+        blind_mode: BlindMode | None = None,
+    ) -> dict[str, Any]:
+        spec, av = _get_queue_and_av(queue_key, artifact_version_id)
+        mode = _effective_mode(spec, blind_mode)
+        rows, _ = store.query_queue_items(
+            artifact_version_id=av["artifact_version_id"],
+            task_key=spec.task_key,
+            reviewer_id=reviewer_id,
+            include_graded=True,
+            filter_fn=lambda r: r["item_key"] == item_key,
+        )
+        if not rows:
+            raise HTTPException(status_code=404, detail=f"Item '{item_key}' not found")
+        row = rows[0]
+        sanitized = sanitize_item_for_client(
+            row,
+            spec,
+            effective_mode=mode,
+            reviewer_id=reviewer_id,
+            mlflow_sink=mlflow_sink,
+        )
+        if sanitized["blinded"]:
+            raise HTTPException(
+                status_code=403,
+                detail="Execution traces are hidden until human judgment is submitted",
+            )
+        spans = store.get_item_spans(
+            item_key,
+            run_id=row.get("run_id"),
+            trace_id=row.get("item_trace_id"),
+        )
+        return {
+            "item_key": item_key,
+            "run_id": row.get("run_id"),
+            "mlflow_url": sanitized.get("mlflow_url"),
+            "spans": spans,
+        }
+
+    @router.post("/annotations")
+    def post_annotation(req: AnnotationRequest) -> dict[str, Any]:
+        """Mode A: Record human evaluation/gold ONLY — never mutates artifacts or triggers runs."""
+        spec: QueueSpec | None = None
+        av: dict[str, Any] | None = None
+        if req.queue_key:
+            spec, av = _get_queue_and_av(req.queue_key, req.target_artifact_version_id)
+
+        task_key = req.task_key or (spec.task_key if spec else None)
+        if not task_key:
+            raise HTTPException(status_code=400, detail="Either queue_key or task_key is required")
+
+        mode: BlindMode = (
+            _effective_mode(spec, req.blind_mode)
+            if spec
+            else (req.blind_mode or "unblinded")
+        )
+        target_avid = req.target_artifact_version_id or (av["artifact_version_id"] if av else None)
+
+        permutation = None
+        if spec and mode == "ab_randomized":
+            permutation = compute_ab_permutation(
+                req.reviewer_id, req.item_key, spec.ab_candidate_keys
+            )
+
+        ann = store.save_annotation(
+            mutation_id=req.mutation_id,
+            task_key=task_key,
+            item_key=req.item_key,
+            input_digest=req.input_digest,
+            target_artifact_version_id=target_avid,
+            reviewer_id=req.reviewer_id,
+            blind_mode=mode,
+            verdict=req.verdict,
+            scores=req.scores,
+            corrected_output=req.corrected_output,
+            permutation=permutation,
+            note=req.note,
+        )
+
+        revealed_item: dict[str, Any] | None = None
+        updated_counts: dict[str, int] | None = None
+        if spec and av:
+            raw_it = store.get_item(av["artifact_version_id"], req.item_key)
+            if raw_it:
+                enriched = {
+                    **raw_it,
+                    "annotation": ann,
+                    "annotation_status": "graded_this_version",
+                }
+                revealed_item = sanitize_item_for_client(
+                    enriched,
+                    spec,
+                    effective_mode=mode,
+                    reviewer_id=req.reviewer_id,
+                    mlflow_sink=mlflow_sink,
+                )
+                if sink is not None and raw_it.get("item_trace_id") and req.verdict is not None:
+                    sink.log_feedback(
+                        trace_id=raw_it["item_trace_id"],
+                        name=task_key,
+                        value=req.verdict,
+                        rationale=req.note,
+                        reviewer_id=req.reviewer_id,
+                        metadata={"annotation_id": ann["annotation_id"]},
+                    )
+            _, updated_counts = store.query_queue_items(
+                artifact_version_id=av["artifact_version_id"],
+                task_key=spec.task_key,
+                reviewer_id=req.reviewer_id,
+                include_graded=True,
+                filter_fn=spec.base_filter,
+            )
+
+        return {"annotation": ann, "item": revealed_item, "counts": updated_counts}
+
+    @router.post("/revisions")
+    def post_revision(req: RevisionRequest) -> dict[str, Any]:
+        """Mode B: Mint explicit `artifact_version = v+1` and optionally rerun dirty downstream."""
+        if req.queue_key:
+            spec, _ = _get_queue_and_av(req.queue_key)
+            if not spec.allow_revisions:
+                raise HTTPException(
+                    status_code=403,
+                    detail=f"Queue '{req.queue_key}' does not allow artifact revisions",
+                )
+
+        try:
+            rev, new_av = store.create_revision(
+                mutation_id=req.mutation_id,
+                base_artifact_version_id=req.base_artifact_version_id,
+                patch=req.patch,
+                reason=req.reason,
+                source_annotation_id=req.source_annotation_id,
+                created_by=req.created_by,
+            )
+        except KeyError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+        rerun_result: dict[str, Any] | None = None
+        if req.trigger_rerun:
+            if rerun_handler is None:
+                raise HTTPException(
+                    status_code=400,
+                    detail="No rerun_handler is configured on this server",
+                )
+            parent_run_id = req.parent_run_id
+            if not parent_run_id:
+                runs = store.list_runs(limit=1)
+                if not runs:
+                    raise HTTPException(
+                        status_code=400,
+                        detail="parent_run_id is required when no prior runs exist",
+                    )
+                parent_run_id = runs[0]["run_id"]
+            rerun_result = rerun_handler(
+                parent_run_id, rev["revision_id"], req.config_overrides
+            )
+
+        return {
+            "revision": rev,
+            "new_artifact_version": new_av,
+            "rerun": rerun_result,
+        }
+
+    @router.get("/exports/{task_key}/gold.jsonl", response_class=PlainTextResponse)
+    def export_gold_jsonl(task_key: str, reviewer_id: str | None = None) -> str:
+        import json
+
+        rows = store.export_gold(task_key, reviewer_id=reviewer_id)
+        return "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows)
+
+    @router.get("/exports/{task_key}/preferences.jsonl", response_class=PlainTextResponse)
+    def export_preferences_jsonl(
+        task_key: str,
+        artifact_version_id: str,
+        reviewer_id: str | None = None,
+    ) -> str:
+        import json
+
+        rows = store.export_preferences(
+            task_key, artifact_version_id, reviewer_id=reviewer_id
+        )
+        return "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in rows)
+
+    return router
+
+
+def mount_builtin_micro_ui(app: FastAPI, *, custom_ui_dir: str | Path | None = None) -> None:
+    """Serve zero-build reviewer assets (`/`, `/assets/rtl.css`, `/assets/primitives.js`)."""
+    ui_pkg = resources.files("misbar").joinpath("ui_assets")
+    custom_dir = Path(custom_ui_dir) if custom_ui_dir else None
+
+    def _read_asset(filename: str) -> str:
+        if custom_dir and (custom_dir / filename).exists():
+            return (custom_dir / filename).read_text(encoding="utf-8")
+        return ui_pkg.joinpath(filename).read_text(encoding="utf-8")
+
+    @app.get("/", response_class=HTMLResponse)
+    def index_page() -> str:
+        return _read_asset("index.html")
+
+    @app.get("/assets/{asset_name}")
+    def asset_file(asset_name: str) -> Response:
+        if asset_name not in {"rtl.css", "primitives.js", "app.js"}:
+            if not (custom_dir and (custom_dir / asset_name).is_file()):
+                raise HTTPException(status_code=404, detail="Asset not found")
+        media = (
+            "text/css; charset=utf-8"
+            if asset_name.endswith(".css")
+            else "application/javascript; charset=utf-8"
+        )
+        return Response(content=_read_asset(asset_name), media_type=media)
+
+
+def create_misbar_app(
+    store: MisbarStore,
+    *,
+    title: str = "misbar Reviewer Workbench",
+    queues: Sequence[QueueSpec] | None = None,
+    sink: TraceSink | None = None,
+    mlflow_sink: MlflowSink | None = None,
+    rerun_handler: Callable[[str, str, dict[str, Any] | None], dict[str, Any]] | None = None,
+    custom_ui_dir: str | Path | None = None,
+) -> FastAPI:
+    """Create a complete FastAPI app with `/api` routes and zero-build micro-UI."""
+    app = FastAPI(title=title)
+    router = create_misbar_router(
+        store,
+        queues=queues,
+        sink=sink,
+        mlflow_sink=mlflow_sink,
+        rerun_handler=rerun_handler,
+    )
+    app.include_router(router)
+    mount_builtin_micro_ui(app, custom_ui_dir=custom_ui_dir)
+    return app
+