agent-loop-guard-runtime 0.6.0a2__py3-none-any.whl

app/api/replay_routes.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.responses import JSONResponse, PlainTextResponse
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from app.db.repository import (
+    Repository,
+    trace_event_dict,
+    trace_run_dict,
+    trace_span_dict,
+)
+from app.db.session import get_db
+from app.replay.formats import trace_to_jsonl, trace_to_otel
+
+router = APIRouter(prefix="/api/v1")
+
+
+class TraceEventPayload(BaseModel):
+    event_id: str | None = None
+    span_id: str | None = None
+    name: str = "event"
+    severity: str = "info"
+    timestamp_ns: int | None = None
+    attributes: dict[str, Any] = Field(default_factory=dict)
+
+
+class TraceSpanPayload(BaseModel):
+    span_id: str | None = None
+    trace_id: str | None = None
+    parent_span_id: str | None = None
+    name: str = "span"
+    status: str = "ok"
+    start_ns: int | None = None
+    end_ns: int | None = None
+    duration_ms: int = 0
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    events: list[TraceEventPayload] = Field(default_factory=list)
+
+
+class TraceCreatePayload(BaseModel):
+    trace_id: str | None = None
+    project_id: str = "default"
+    task_id: str | None = None
+    task_fingerprint: str | None = None
+    agent_name: str | None = None
+    model: str | None = None
+    status: str = "running"
+    failure_tag: str | None = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+    total_tokens: int = 0
+    total_cost_micros: int = 0
+    pinned: bool = False
+    start_ns: int | None = None
+    end_ns: int | None = None
+    duration_ms: int = 0
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    spans: list[TraceSpanPayload] = Field(default_factory=list)
+
+
+class EventBatchPayload(BaseModel):
+    trace_id: str
+    events: list[TraceEventPayload]
+
+
+class ComparePayload(BaseModel):
+    left_trace_id: str
+    right_trace_id: str
+
+
+class PinPayload(BaseModel):
+    pinned: bool = True
+
+
+class TraceImportPayload(BaseModel):
+    bundle: dict[str, Any]
+
+
+@router.post("/traces")
+def create_trace(payload: TraceCreatePayload, db: Session = Depends(get_db)) -> dict:
+    repo = Repository(db)
+    trace_data = payload.model_dump(exclude={"spans"})
+    span_starts = [span.start_ns for span in payload.spans if span.start_ns is not None]
+    span_ends = [span.end_ns for span in payload.spans if span.end_ns is not None]
+    if trace_data.get("start_ns") is None and span_starts:
+        trace_data["start_ns"] = min(span_starts)
+    if trace_data.get("end_ns") is None and span_ends:
+        trace_data["end_ns"] = max(span_ends)
+    if not trace_data.get("duration_ms") and span_starts and span_ends:
+        trace_data["duration_ms"] = max(0, int((max(span_ends) - min(span_starts)) / 1_000_000))
+    run = repo.create_trace_run(trace_data)
+    for span_payload in payload.spans:
+        span_data = span_payload.model_dump(exclude={"events"})
+        span = repo.add_trace_span(run.id, span_data)
+        if span_payload.events:
+            events = [
+                {**event.model_dump(), "span_id": event.span_id or (span.id if span else None)}
+                for event in span_payload.events
+            ]
+            repo.add_trace_events(run.id, events)
+    exported = repo.trace_export(run.id)
+    if exported is None:
+        raise HTTPException(500, "Trace was not created.")
+    return exported
+
+
+@router.post("/spans")
+def create_span(payload: TraceSpanPayload, db: Session = Depends(get_db)) -> dict:
+    if not payload.trace_id:
+        raise HTTPException(400, "trace_id is required.")
+    repo = Repository(db)
+    span = repo.add_trace_span(payload.trace_id, payload.model_dump(exclude={"events"}))
+    if span is None:
+        raise HTTPException(404, "Trace not found.")
+    if payload.events:
+        repo.add_trace_events(
+            payload.trace_id,
+            [
+                {**event.model_dump(), "span_id": event.span_id or span.id}
+                for event in payload.events
+            ],
+        )
+    return trace_span_dict(span)
+
+
+@router.post("/events/batch")
+def create_events(payload: EventBatchPayload, db: Session = Depends(get_db)) -> dict:
+    events = Repository(db).add_trace_events(
+        payload.trace_id, [event.model_dump() for event in payload.events]
+    )
+    if events is None:
+        raise HTTPException(404, "Trace not found.")
+    return {"events": [trace_event_dict(item) for item in events]}
+
+
+@router.get("/runs")
+def runs(
+    limit: int = 50,
+    project_id: str | None = None,
+    q: str | None = None,
+    db: Session = Depends(get_db),
+) -> list[dict]:
+    return [
+        trace_run_dict(item)
+        for item in Repository(db).list_trace_runs(limit=limit, project_id=project_id, query=q)
+    ]
+
+
+@router.get("/runs/{trace_id}")
+def run_detail(trace_id: str, db: Session = Depends(get_db)) -> dict:
+    exported = Repository(db).trace_export(trace_id)
+    if exported is None:
+        raise HTTPException(404, "Trace not found.")
+    return exported
+
+
+@router.get("/runs/{trace_id}/export")
+def run_export(
+    trace_id: str,
+    format: str = Query(default="json", pattern="^(json|jsonl|otel)$"),
+    db: Session = Depends(get_db),
+):
+    exported = Repository(db).trace_export(trace_id)
+    if exported is None:
+        raise HTTPException(404, "Trace not found.")
+    if format == "jsonl":
+        return PlainTextResponse(trace_to_jsonl(exported), media_type="application/x-ndjson")
+    if format == "otel":
+        return JSONResponse(trace_to_otel(exported))
+    return JSONResponse(exported)
+
+
+@router.post("/runs/import")
+def import_run(payload: TraceImportPayload, db: Session = Depends(get_db)) -> dict:
+    repo = Repository(db)
+    try:
+        run = repo.import_trace_bundle(payload.bundle)
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    exported = repo.trace_export(run.id)
+    assert exported is not None
+    return exported
+
+
+@router.post("/runs/{trace_id}/pin")
+def pin_run(trace_id: str, payload: PinPayload, db: Session = Depends(get_db)) -> dict:
+    run = Repository(db).pin_trace(trace_id, payload.pinned)
+    if run is None:
+        raise HTTPException(404, "Trace not found.")
+    return trace_run_dict(run)
+
+
+@router.post("/compare")
+def compare(payload: ComparePayload, db: Session = Depends(get_db)) -> dict:
+    result = Repository(db).compare_traces(payload.left_trace_id, payload.right_trace_id)
+    if result is None:
+        raise HTTPException(404, "One or both traces were not found.")
+    return result

app/api/ui_routes.py

@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.responses import RedirectResponse
+from fastapi.templating import Jinja2Templates
+from sqlalchemy.orm import Session
+
+from app.core.demo import run_demo
+from app.db.repository import (
+    Repository,
+    agent_dict,
+    event_dict,
+    mcp_approval_dict,
+    mcp_event_dict,
+    mcp_server_dict,
+    policy_dict,
+    request_dict,
+    session_dict,
+    trace_artifact_dict,
+    trace_event_dict,
+    trace_run_dict,
+    trace_span_dict,
+)
+from app.db.session import get_db
+
+router = APIRouter()
+templates = Jinja2Templates(directory=str(Path(__file__).resolve().parents[1] / "templates"))
+
+
+@router.get("/")
+def dashboard(request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    approvals = repo.list_mcp_approvals(pending_only=True)
+    return templates.TemplateResponse(
+        request,
+        "dashboard.html",
+        {
+            "stats": repo.aggregate_stats(),
+            "sessions": [session_dict(item) for item in repo.list_sessions(10)],
+            "pending_approvals": len(approvals),
+            "docker_available": shutil.which("docker") is not None,
+        },
+    )
+
+
+@router.get("/sessions")
+def sessions_page(request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    return templates.TemplateResponse(
+        request,
+        "sessions.html",
+        {"sessions": [session_dict(item) for item in repo.list_sessions(100)]},
+    )
+
+
+@router.get("/sessions/{session_id}")
+def session_page(session_id: str, request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    session = repo.get_session(session_id)
+    if session is None:
+        raise HTTPException(404, "Session not found.")
+    return templates.TemplateResponse(
+        request,
+        "session_detail.html",
+        {
+            "session": session_dict(session),
+            "requests": [request_dict(item) for item in repo.requests_for_session(session_id)],
+            "events": [event_dict(item) for item in repo.events_for_session(session_id)],
+        },
+    )
+
+
+@router.get("/replay")
+def replay_page(
+    request: Request,
+    q: str | None = None,
+    project_id: str | None = None,
+    db: Session = Depends(get_db),
+):
+    repo = Repository(db)
+    return templates.TemplateResponse(
+        request,
+        "replay.html",
+        {
+            "runs": [
+                trace_run_dict(item)
+                for item in repo.list_trace_runs(limit=100, project_id=project_id, query=q)
+            ],
+            "q": q or "",
+            "project_id": project_id or "",
+        },
+    )
+
+
+@router.get("/mcp")
+def mcp_page(request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    return templates.TemplateResponse(
+        request,
+        "mcp.html",
+        {
+            "servers": [mcp_server_dict(item) for item in repo.list_mcp_servers()],
+            "events": [mcp_event_dict(item) for item in repo.list_mcp_events(100)],
+            "approvals": [mcp_approval_dict(item) for item in repo.list_mcp_approvals()],
+            "policy_path": request.app.state.config.mcp_policy_path,
+        },
+    )
+
+
+@router.post("/mcp/approvals/{approval_id}/decision")
+async def mcp_approval_ui(
+    approval_id: str, request: Request, db: Session = Depends(get_db)
+):
+    form = await request.form()
+    action = str(form.get("action") or "deny")
+    scope = str(form.get("scope") or "once")
+    Repository(db).decide_mcp_approval(approval_id, action, scope)
+    return RedirectResponse("/mcp", status_code=303)
+
+
+@router.post("/replay/compare")
+async def replay_compare_page(request: Request, db: Session = Depends(get_db)):
+    form = await request.form()
+    left_id = str(form.get("left_trace_id") or "")
+    right_id = str(form.get("right_trace_id") or "")
+    result = Repository(db).compare_traces(left_id, right_id)
+    if result is None:
+        raise HTTPException(404, "One or both traces were not found.")
+    return templates.TemplateResponse(
+        request,
+        "replay_compare.html",
+        {"result": result, "left_id": left_id, "right_id": right_id},
+    )
+
+
+@router.get("/replay/{trace_id}")
+def replay_detail_page(trace_id: str, request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    run = repo.get_trace_run(trace_id)
+    if run is None:
+        raise HTTPException(404, "Trace not found.")
+    spans = [trace_span_dict(item) for item in repo.trace_spans(trace_id)]
+    if spans:
+        first_start = min(span["start_ns"] for span in spans)
+        last_end = max((span["end_ns"] or span["start_ns"]) for span in spans)
+        total = max(1, last_end - first_start)
+        parents = {span["id"]: span.get("parent_span_id") for span in spans}
+        for span in spans:
+            depth = 0
+            parent = span.get("parent_span_id")
+            while parent and depth < 8:
+                depth += 1
+                parent = parents.get(parent)
+            span["depth"] = depth
+            span["offset_pct"] = ((span["start_ns"] - first_start) / total) * 100
+            span["width_pct"] = max(1.5, (max(1, span["duration_ms"] * 1_000_000) / total) * 100)
+    return templates.TemplateResponse(
+        request,
+        "replay_detail.html",
+        {
+            "run": trace_run_dict(run),
+            "spans": spans,
+            "events": [trace_event_dict(item) for item in repo.trace_events(trace_id)],
+            "artifacts": [trace_artifact_dict(item) for item in repo.trace_artifacts(trace_id)],
+        },
+    )
+
+
+@router.post("/replay/{trace_id}/pin")
+async def replay_pin_page(trace_id: str, request: Request, db: Session = Depends(get_db)):
+    form = await request.form()
+    Repository(db).pin_trace(trace_id, str(form.get("pinned") or "true") == "true")
+    return RedirectResponse(f"/replay/{trace_id}", status_code=303)
+
+
+@router.post("/sessions/{session_id}/pause")
+def pause_session_ui(session_id: str, db: Session = Depends(get_db)):
+    Repository(db).pause_session(session_id)
+    return RedirectResponse(f"/sessions/{session_id}", status_code=303)
+
+
+@router.post("/sessions/{session_id}/resume")
+def resume_session_ui(session_id: str, db: Session = Depends(get_db)):
+    Repository(db).resume_session(session_id)
+    return RedirectResponse(f"/sessions/{session_id}", status_code=303)
+
+
+@router.post("/sessions/{session_id}/edit")
+async def edit_session_ui(session_id: str, request: Request, db: Session = Depends(get_db)):
+    form = await request.form()
+    Repository(db).update_session(
+        session_id,
+        name=str(form.get("name") or ""),
+        note=str(form.get("note") or ""),
+    )
+    return RedirectResponse(f"/sessions/{session_id}", status_code=303)
+
+
+@router.get("/policies")
+def policies_page(request: Request, project_id: str = "default", db: Session = Depends(get_db)):
+    repo = Repository(db)
+    return templates.TemplateResponse(
+        request,
+        "policies.html",
+        {"project_id": project_id, "policies": [policy_dict(item) for item in repo.policies(project_id)]},
+    )
+
+
+@router.post("/policies/{project_id}/update")
+async def update_policies_ui(project_id: str, request: Request, db: Session = Depends(get_db)):
+    form = await request.form()
+    rows = []
+    for policy in Repository(db).policies(project_id):
+        prefix = policy.id
+        rows.append(
+            {
+                "id": policy.id,
+                "rule_type": policy.rule_type,
+                "threshold": int(form.get(f"{prefix}_threshold") or policy.threshold),
+                "window_size": int(form.get(f"{prefix}_window_size") or policy.window_size),
+                "action": str(form.get(f"{prefix}_action") or policy.action),
+                "enabled": form.get(f"{prefix}_enabled") == "on",
+            }
+        )
+    Repository(db).upsert_policies(project_id, rows)
+    return RedirectResponse(f"/policies?project_id={project_id}", status_code=303)
+
+
+@router.get("/agents")
+def agents_page(request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    return templates.TemplateResponse(
+        request,
+        "agents.html",
+        {"agents": [agent_dict(item) for item in repo.list_agents()], "created_key": None},
+    )
+
+
+@router.post("/agents")
+async def create_agent_ui(request: Request, db: Session = Depends(get_db)):
+    form = await request.form()
+    repo = Repository(db)
+    agent, raw_key = repo.create_agent(
+        str(form.get("project_id") or "default"),
+        str(form.get("name") or "Local agent"),
+        str(form.get("protocol") or "openai"),
+    )
+    return templates.TemplateResponse(
+        request,
+        "agents.html",
+        {"agents": [agent_dict(item) for item in repo.list_agents()], "created_key": raw_key, "created_agent": agent_dict(agent)},
+    )
+
+
+@router.post("/agents/{agent_id}/pause")
+def pause_agent_ui(agent_id: str, db: Session = Depends(get_db)):
+    Repository(db).pause_agent(agent_id)
+    return RedirectResponse("/agents", status_code=303)
+
+
+@router.post("/agents/{agent_id}/resume")
+def resume_agent_ui(agent_id: str, db: Session = Depends(get_db)):
+    Repository(db).resume_agent(agent_id)
+    return RedirectResponse("/agents", status_code=303)
+
+
+@router.get("/demo")
+def demo_page(request: Request, db: Session = Depends(get_db)):
+    repo = Repository(db)
+    return templates.TemplateResponse(
+        request,
+        "demo.html",
+        {"sessions": [session_dict(item) for item in repo.list_sessions(10)], "result": None},
+    )
+
+
+@router.post("/demo/run")
+async def demo_run_ui(request: Request, db: Session = Depends(get_db)):
+    form = await request.form()
+    scenario = str(form.get("scenario") or "exact-loop")
+    mode = str(form.get("mode") or "shadow")
+    repo = Repository(db)
+    agents = repo.list_agents()
+    if not agents:
+        raise HTTPException(500, "No agent is available.")
+    session = run_demo(repo, agents[-1], scenario, mode)
+    return RedirectResponse(f"/sessions/{session.id}", status_code=303)
+
+
+@router.get("/settings")
+def settings_page(request: Request):
+    return templates.TemplateResponse(request, "settings.html", {"config": request.app.state.config})

app/benchmark/__init__.py

@@ -0,0 +1,2 @@
+"""Deterministic local benchmark and regression tools."""
+

app/benchmark/adapters.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import json
+import random
+import shlex
+import subprocess
+from abc import ABC, abstractmethod
+from typing import Any
+
+import httpx
+
+from app.benchmark.models import AdapterResult, BenchmarkTask
+
+
+def _tokens(value: str) -> int:
+    return max(1, (len(value) + 3) // 4)
+
+
+class BenchmarkAdapter(ABC):
+    @abstractmethod
+    def execute(self, task: BenchmarkTask, seed: int, timeout: float) -> AdapterResult:
+        raise NotImplementedError
+
+
+class MockAdapter(BenchmarkAdapter):
+    def __init__(self, variant: str = "baseline") -> None:
+        self.variant = variant
+
+    def execute(self, task: BenchmarkTask, seed: int, timeout: float) -> AdapterResult:
+        del timeout
+        output = json.dumps(task.expected, sort_keys=True) if task.scorer == "json_equal" else str(task.expected)
+        if self.variant == "regressed" and task.difficulty in {"medium", "hard"}:
+            output = f"incorrect-{random.Random(f'{seed}:{task.id}').randrange(1000)}"
+        return AdapterResult(output, _tokens(task.prompt), _tokens(output))
+
+
+class HTTPAdapter(BenchmarkAdapter):
+    def __init__(self, endpoint: str, model: str, api_key: str | None = None) -> None:
+        self.endpoint = endpoint
+        self.model = model
+        self.api_key = api_key
+
+    def execute(self, task: BenchmarkTask, seed: int, timeout: float) -> AdapterResult:
+        headers = {"authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+        try:
+            response = httpx.post(
+                self.endpoint,
+                headers=headers,
+                json={"model": self.model, "input": task.prompt, "seed": seed},
+                timeout=timeout,
+            )
+            response.raise_for_status()
+            payload = response.json()
+            output = _response_text(payload)
+            usage = payload.get("usage") or {}
+            return AdapterResult(
+                output=output,
+                input_tokens=int(usage.get("input_tokens") or _tokens(task.prompt)),
+                output_tokens=int(usage.get("output_tokens") or _tokens(output)),
+            )
+        except (httpx.HTTPError, ValueError) as exc:
+            return AdapterResult("", error=str(exc))
+
+
+class CLIAdapter(BenchmarkAdapter):
+    def __init__(self, command: str | list[str]) -> None:
+        self.command = shlex.split(command) if isinstance(command, str) else command
+
+    def execute(self, task: BenchmarkTask, seed: int, timeout: float) -> AdapterResult:
+        try:
+            process = subprocess.run(
+                self.command,
+                input=json.dumps({"prompt": task.prompt, "seed": seed}),
+                text=True,
+                capture_output=True,
+                timeout=timeout,
+                check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            return AdapterResult("", error=str(exc))
+        if process.returncode:
+            return AdapterResult("", error=process.stderr.strip() or f"exit {process.returncode}")
+        output = process.stdout.strip()
+        return AdapterResult(output, _tokens(task.prompt), _tokens(output))
+
+
+def _response_text(payload: dict[str, Any]) -> str:
+    if isinstance(payload.get("output_text"), str):
+        return payload["output_text"]
+    for item in payload.get("output") or []:
+        for content in item.get("content") or []:
+            if isinstance(content.get("text"), str):
+                return content["text"]
+    choices = payload.get("choices") or []
+    if choices:
+        return str((choices[0].get("message") or {}).get("content") or choices[0].get("text") or "")
+    return ""

app/benchmark/data/starter-v1.json

@@ -0,0 +1,38 @@
+{
+  "version": "starter-v1",
+  "description": "Thirty deterministic tasks for local regression smoke tests.",
+  "tasks": [
+    {"id":"easy-01","difficulty":"easy","prompt":"Return exactly: alpha","expected":"alpha","scorer":"exact","tags":["format"]},
+    {"id":"easy-02","difficulty":"easy","prompt":"Return exactly: beta","expected":"beta","scorer":"exact","tags":["format"]},
+    {"id":"easy-03","difficulty":"easy","prompt":"What is 2 + 2? Return only the number.","expected":"4","scorer":"exact","tags":["math"]},
+    {"id":"easy-04","difficulty":"easy","prompt":"Return the lowercase form of HELLO.","expected":"hello","scorer":"exact","tags":["text"]},
+    {"id":"easy-05","difficulty":"easy","prompt":"Name the boolean opposite of true.","expected":"false","scorer":"contains","tags":["logic"]},
+    {"id":"easy-06","difficulty":"easy","prompt":"Return JSON with ok=true.","expected":{"ok":true},"scorer":"json_equal","tags":["json"]},
+    {"id":"easy-07","difficulty":"easy","prompt":"What extension is commonly used for Python files?","expected":".py","scorer":"contains","tags":["coding"]},
+    {"id":"easy-08","difficulty":"easy","prompt":"Return exactly three.","expected":"three","scorer":"exact","tags":["format"]},
+    {"id":"easy-09","difficulty":"easy","prompt":"Return the first letter of agent.","expected":"a","scorer":"exact","tags":["text"]},
+    {"id":"easy-10","difficulty":"easy","prompt":"Return JSON null.","expected":null,"scorer":"json_equal","tags":["json"]},
+
+    {"id":"medium-01","difficulty":"medium","prompt":"Return the sorted numbers 3,1,2 joined by commas.","expected":"1,2,3","scorer":"exact","tags":["reasoning"]},
+    {"id":"medium-02","difficulty":"medium","prompt":"Return the SHA abbreviation as lowercase text.","expected":"sha","scorer":"exact","tags":["coding"]},
+    {"id":"medium-03","difficulty":"medium","prompt":"Return JSON array containing 1, 2, 3.","expected":[1,2,3],"scorer":"json_equal","tags":["json"]},
+    {"id":"medium-04","difficulty":"medium","prompt":"A command exits with code zero. Include the word success.","expected":"success","scorer":"contains","tags":["shell"]},
+    {"id":"medium-05","difficulty":"medium","prompt":"Return 5 factorial as digits only.","expected":"120","scorer":"exact","tags":["math"]},
+    {"id":"medium-06","difficulty":"medium","prompt":"Return the normalized path a/c for input a/b/../c.","expected":"a/c","scorer":"exact","tags":["filesystem"]},
+    {"id":"medium-07","difficulty":"medium","prompt":"Return JSON with count 3 and valid true.","expected":{"count":3,"valid":true},"scorer":"json_equal","tags":["json"]},
+    {"id":"medium-08","difficulty":"medium","prompt":"Which HTTP status means not found? Return digits.","expected":"404","scorer":"exact","tags":["http"]},
+    {"id":"medium-09","difficulty":"medium","prompt":"Include the term least privilege in the answer.","expected":"least privilege","scorer":"contains","tags":["security"]},
+    {"id":"medium-10","difficulty":"medium","prompt":"Return the result of 0b1010 in decimal.","expected":"10","scorer":"exact","tags":["math"]},
+
+    {"id":"hard-01","difficulty":"hard","prompt":"Return JSON describing a denied delete: action deny, tool delete_file.","expected":{"action":"deny","tool":"delete_file"},"scorer":"json_equal","tags":["policy"]},
+    {"id":"hard-02","difficulty":"hard","prompt":"Return the stable topological order A,B,C for edges A->B and B->C.","expected":"A,B,C","scorer":"exact","tags":["reasoning"]},
+    {"id":"hard-03","difficulty":"hard","prompt":"A paired test lacks enough samples. Include inconclusive.","expected":"inconclusive","scorer":"contains","tags":["statistics"]},
+    {"id":"hard-04","difficulty":"hard","prompt":"Return JSON with protocol jsonrpc and version 2.0.","expected":{"protocol":"jsonrpc","version":"2.0"},"scorer":"json_equal","tags":["mcp"]},
+    {"id":"hard-05","difficulty":"hard","prompt":"Return the hexadecimal SHA-256 digest length in characters.","expected":"64","scorer":"exact","tags":["security"]},
+    {"id":"hard-06","difficulty":"hard","prompt":"A sandbox has no network. Include offline and blocked.","expected":"offline","scorer":"contains","tags":["sandbox"]},
+    {"id":"hard-07","difficulty":"hard","prompt":"Return JSON with ci_low -0.2 and ci_high -0.1.","expected":{"ci_high":-0.1,"ci_low":-0.2},"scorer":"json_equal","tags":["statistics"]},
+    {"id":"hard-08","difficulty":"hard","prompt":"Return the SQL operation that reads rows.","expected":"SELECT","scorer":"exact","tags":["database"]},
+    {"id":"hard-09","difficulty":"hard","prompt":"Include both rate and limit in a two-word phrase.","expected":"rate limit","scorer":"contains","tags":["resilience"]},
+    {"id":"hard-10","difficulty":"hard","prompt":"Return JSON indicating passed true and score 1.","expected":{"passed":true,"score":1},"scorer":"json_equal","tags":["benchmark"]}
+  ]
+}