provekit 0.2.0__tar.gz → 0.6.0__tar.gz

{provekit-0.2.0 → provekit-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: provekit
-Version: 0.2.0
+Version: 0.6.0
 Summary: Drop-in agent tracing — add one decorator, get a project key, review every run your agent makes.
 Author: ProveKit contributors
 License: MIT
@@ -38,6 +38,29 @@ Requires-Dist: opentelemetry-api<2,>=1.20; extra == "trace"
 Requires-Dist: opentelemetry-sdk<2,>=1.20; extra == "trace"
 Requires-Dist: openinference-instrumentation-openai>=0.1; extra == "trace"
 Requires-Dist: openinference-instrumentation-anthropic>=0.1; extra == "trace"
+Provides-Extra: http
+Requires-Dist: opentelemetry-instrumentation-httpx>=0.40b0; extra == "http"
+Requires-Dist: opentelemetry-instrumentation-requests>=0.40b0; extra == "http"
+Requires-Dist: opentelemetry-instrumentation-urllib>=0.40b0; extra == "http"
+Provides-Extra: trace-all
+Requires-Dist: opentelemetry-api<2,>=1.20; extra == "trace-all"
+Requires-Dist: opentelemetry-sdk<2,>=1.20; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-openai>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-anthropic>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-langchain>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-llama-index>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-crewai>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-bedrock>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-litellm>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-groq>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-mistralai>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-dspy>=0.1; extra == "trace-all"
+Requires-Dist: openinference-instrumentation-haystack>=0.1; extra == "trace-all"
+Requires-Dist: opentelemetry-instrumentation-httpx>=0.40b0; extra == "trace-all"
+Requires-Dist: opentelemetry-instrumentation-requests>=0.40b0; extra == "trace-all"
+Requires-Dist: opentelemetry-instrumentation-urllib>=0.40b0; extra == "trace-all"
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.2; extra == "mcp"
 Provides-Extra: dev
 Requires-Dist: pytest==9.1.1; extra == "dev"
 Requires-Dist: pytest-cov==6.0.0; extra == "dev"

provekit-0.6.0/provekit/__init__.py

@@ -0,0 +1,18 @@
+"""ProveKit — drop-in agent tracing. One SDK on the client captures the whole flow.
+
+    import provekit.trace as pk          # the canonical import
+    pk.init()                            # or, zero-code: import provekit.auto
+
+    @pk.trace(name="my-agent")           # optional: group one run under a named root
+    def run(q): ...
+
+Everything beneath your code — LLM providers, agent frameworks, outbound HTTP — is
+captured automatically. `init`, `configure`, and `span` are also re-exported here for
+`import provekit as pk` convenience; the `trace` decorator lives on the `provekit.trace`
+submodule (a same-named top-level export would shadow that module).
+"""
+from provekit import scorers
+from provekit.eval import evaluate
+from provekit.trace import configure, init, span
+
+__all__ = ["init", "configure", "span", "evaluate", "scorers"]

provekit-0.6.0/provekit/auto.py

@@ -0,0 +1,11 @@
+"""Zero-code activation: `import provekit.auto` turns tracing on with no other calls.
+
+    import provekit.auto      # reads PROVEKIT_API_KEY / PROVEKIT_ENDPOINT and instruments
+
+Put this once at your program's entrypoint (before the libraries you want traced do their
+work). Everything instrumented below it is then captured automatically. Fail-open: if the
+env vars are unset or OpenTelemetry isn't installed, this is a silent no-op.
+"""
+from provekit.trace import init
+
+init()

provekit-0.6.0/provekit/config.py

@@ -0,0 +1,68 @@
+"""ProveKit configuration."""
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    database_url: str = "sqlite:///./provekit.db"
+    cors_origins: str = "http://localhost:3001,http://localhost:3000"
+
+    # Hosted mode: a login wall, secure cookies, and stricter outbound-URL guarding.
+    hosted: bool = False
+
+    # Signs session/reset/verify tokens (any string → a Fernet key). Optional for local
+    # SQLite (a key file is auto-generated); required for any other database.
+    secret_key: str = ""
+
+    # Optional OTLP/HTTP collector to mirror captured runs to another backend.
+    otel_export_url: str = ""
+
+    # Optional Sentry DSN for error reporting (off when unset).
+    sentry_dsn: str = ""
+
+    # Web app base URL (for links in emails).
+    web_base_url: str = "http://localhost:3001"
+
+    # Email (account recovery / verification). No SMTP host → links are logged, not sent.
+    smtp_host: str = ""
+    smtp_port: int = 587
+    smtp_user: str = ""
+    smtp_password: str = ""
+    smtp_from: str = ""
+    smtp_starttls: bool = True
+    require_email_verification: bool = False
+
+    # Optional Redis (shared rate-limit windows across workers). Unset → in-memory.
+    redis_url: str = ""
+
+    thread_pool_size: int = 200
+
+    # Mask PII (emails, cards, SSNs, phones, common secret keys) in captured span
+    # input/output/error before it's stored. Off by default; a safety net for hosted use.
+    redact_pii: bool = False
+
+    # Bootstrap platform operators: any account whose email is in this comma-list is treated
+    # as a superuser (admin console), even before the DB flag is set. e.g. "you@co.com".
+    superuser_emails: str = ""
+
+    @property
+    def superuser_email_list(self) -> list[str]:
+        return [e.strip().lower() for e in self.superuser_emails.split(",") if e.strip()]
+
+    # Quotas (protect shared infra). 0 disables a given limit.
+    ingest_rate_per_min: int = 600     # trace-ingest requests per project per minute
+    login_attempts_per_min: int = 10   # login attempts per email+IP per minute
+    runs_retention: int = 10000        # keep last N spans per project (older ones pruned)
+    max_body_bytes: int = 2_000_000    # reject request bodies larger than this
+
+    @property
+    def cors_origin_list(self) -> list[str]:
+        return [o.strip() for o in self.cors_origins.split(",") if o.strip()]
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

provekit-0.6.0/provekit/demo.py

@@ -0,0 +1,118 @@
+"""`provekit-demo` — a zero-setup smoke test.
+
+Sends a small gallery of traces to your portal so a fresh install + project key can be
+verified in ~10 seconds. No LLM key needed: the model is mocked, but recorded as a real
+gen_ai LLM span (model, tokens, temperature) so the portal renders it as a proper chat call.
+
+    pip install "provekit[trace]"
+    export PROVEKIT_API_KEY=pk_...        # a project key from the portal
+    export PROVEKIT_ENDPOINT=https://your-provekit-host
+    provekit-demo
+
+For richer, framework-native examples (LangGraph, a deep multi-agent orchestrator) see the
+examples/ folder: https://github.com/MobirizerServices/ProveKit/tree/main/examples
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import time
+
+from . import trace as pk
+
+log = logging.getLogger("provekit-demo")
+
+_KB = {
+    "refund": "I've started your refund — it lands in 3–5 business days.",
+    "vpn": "Forget the VPN profile and re-add it; that clears the stale auth token.",
+    "pricing": "The Pro plan is $49/seat/mo; annual billing saves ~15%.",
+    "export": "Settings → Data → Export builds a downloadable JSON of your workspace.",
+}
+
+
+def _mock_llm(span, model: str, prompt: str, temperature: float = 0.7) -> str:
+    """A deterministic, offline stand-in for an LLM — recorded as a real gen_ai LLM span."""
+    key = next((k for k in _KB if k in prompt.lower()), None)
+    answer = _KB.get(key, "Thanks for reaching out — a teammate will follow up shortly.")
+    if span:
+        span.set_attribute("gen_ai.request.model", model)
+        span.set_attribute("gen_ai.request.temperature", temperature)
+        span.set_attribute("gen_ai.input.messages", prompt)
+        span.set_attribute("gen_ai.usage.input_tokens", 40 + len(prompt) // 4)
+        span.set_attribute("gen_ai.usage.output_tokens", 12 + len(answer) // 4)
+        span.set_attribute("gen_ai.output.messages", answer)
+        span.set_attribute("gen_ai.response.finish_reasons", "stop")
+    time.sleep(0.01)
+    return answer
+
+
+@pk.trace(name="support-agent")
+def _support_agent(question: str, model: str = "gpt-4o") -> str:
+    with pk.span("retrieve") as s:
+        if s:
+            s.set_attribute("gen_ai.tool.name", "kb_search")
+            s.set_attribute("gen_ai.input.messages", question)
+        log.info("retrieved 3 candidate docs")
+    with pk.span("chat") as s:
+        answer = _mock_llm(s, model, question)
+    pk.score("helpfulness", score=0.9, comment="demo auto-score")
+    return answer
+
+
+@pk.trace(name="support-agent", session_id="conv-demo-1")
+def _support_turn(question: str) -> str:
+    with pk.span("chat") as s:
+        return _mock_llm(s, "gpt-4o-mini", question)
+
+
+@pk.trace(name="flaky-agent")
+def _flaky_agent(question: str) -> str:
+    with pk.span("call-upstream") as s:
+        if s:
+            s.set_attribute("gen_ai.tool.name", "billing_api")
+        raise RuntimeError("upstream billing_api returned 503")
+
+
+def main(argv=None) -> None:
+    p = argparse.ArgumentParser(
+        prog="provekit-demo",
+        description="Send a small gallery of demo traces to your ProveKit portal (no LLM key needed).")
+    p.add_argument("--api-key", default=os.environ.get("PROVEKIT_API_KEY"),
+                   help="project key (default: $PROVEKIT_API_KEY)")
+    p.add_argument("--endpoint", default=os.environ.get("PROVEKIT_ENDPOINT"),
+                   help="portal URL (default: $PROVEKIT_ENDPOINT)")
+    args = p.parse_args(argv)
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
+
+    if not (args.api_key and args.endpoint):
+        raise SystemExit("Set PROVEKIT_API_KEY and PROVEKIT_ENDPOINT (or pass --api-key / --endpoint).\n"
+                         "Grab a key from your portal → Project keys.")
+    if not pk.configure(api_key=args.api_key, endpoint=args.endpoint):
+        raise SystemExit("ProveKit could not start — check your key/endpoint.")
+
+    for q, model in (("I want a refund for my order", "gpt-4o"),
+                     ("VPN won't connect after the update", "gpt-4o"),
+                     ("What's your pricing for a 20-person team?", "claude-sonnet-5"),
+                     ("How do I export my data?", "gpt-4o-mini")):
+        print("→", _support_agent(q, model=model)[:60])
+
+    # a two-turn conversation (same session) so sessions light up
+    _support_turn("Is the Pro plan monthly?")
+    _support_turn("And does annual billing save money?")
+
+    # a failed run so the dashboard error-rate + a red trace show up
+    try:
+        _flaky_agent("charge dispute")
+    except RuntimeError as exc:
+        print("✗ flaky-agent failed as designed:", exc)
+
+    from opentelemetry import trace as _t
+    provider = _t.get_tracer_provider()
+    if hasattr(provider, "force_flush"):
+        provider.force_flush()
+    print("\nDone — open your ProveKit portal:", args.endpoint.rstrip("/") + "/traces")
+
+
+if __name__ == "__main__":
+    main()

provekit-0.6.0/provekit/eval.py

@@ -0,0 +1,74 @@
+"""Offline evaluation from the client: run a target over a dataset, score each output, and
+record it as an experiment in your portal — the basis of a CI regression gate.
+
+    import provekit as pk
+
+    def target(question: str) -> str:
+        return my_agent(question)
+
+    summary = pk.evaluate("qa-golden", target, scorers=["exact_match", "contains"])
+    assert summary["mean_score"] >= 0.8       # fail the build on a regression
+
+`scorers` are names from provekit.scorers (exact_match/contains/regex_match/json_valid) or
+your own `fn(output, expected) -> float`. The dataset is a name or id from your portal.
+"""
+from __future__ import annotations
+
+import json
+
+import httpx
+
+from . import scorers as _scorers
+from . import trace as _trace
+
+_TIMEOUT = 60
+
+
+def _get(endpoint, headers, path):
+    r = httpx.get(f"{endpoint}{path}", headers=headers, timeout=_TIMEOUT, follow_redirects=False)
+    r.raise_for_status()
+    return r.json()
+
+
+def _post(endpoint, headers, path, body):
+    r = httpx.post(f"{endpoint}{path}", headers=headers, json=body, timeout=_TIMEOUT, follow_redirects=False)
+    r.raise_for_status()
+    return r.json()
+
+
+def _resolve_dataset_id(endpoint, headers, dataset) -> int:
+    if isinstance(dataset, int):
+        return dataset
+    for d in _get(endpoint, headers, "/v1/datasets"):
+        if d.get("name") == dataset:
+            return d["id"]
+    raise ValueError(f"dataset {dataset!r} not found")
+
+
+def evaluate(dataset, target, scorers, *, name: str = "experiment") -> dict:
+    """Run `target(input)` over every item in `dataset`, score each output with `scorers`,
+    post the results as a new experiment, and return its summary (id, result_count,
+    scorer_means, mean_score). Raises if ProveKit isn't configured."""
+    if not _trace.configure():
+        raise RuntimeError("ProveKit not configured: set PROVEKIT_API_KEY and PROVEKIT_ENDPOINT")
+    endpoint, headers = _trace._endpoint, {"Authorization": f"Bearer {_trace._api_key}"}
+
+    dsid = _resolve_dataset_id(endpoint, headers, dataset)
+    items = _get(endpoint, headers, f"/v1/datasets/{dsid}/items")
+    exp = _post(endpoint, headers, "/v1/experiments", {"name": name, "dataset_id": dsid})
+    eid = exp["id"]
+
+    for it in items:
+        try:
+            output = target(it["input"])
+        except Exception as exc:   # a target failure is a data point, not a crash
+            output = f"ERROR: {exc}"
+        if not isinstance(output, str):
+            output = json.dumps(output, default=str)
+        expected = it.get("expected", "")
+        scores = _scorers.run_scorers(scorers, output, expected)
+        _post(endpoint, headers, f"/v1/experiments/{eid}/results",
+              {"item_id": it.get("id"), "input": it["input"], "output": output,
+               "expected": expected, "scores": scores})
+
+    return _get(endpoint, headers, f"/v1/experiments/{eid}")

{provekit-0.2.0 → provekit-0.6.0}/provekit/main.py

@@ -16,7 +16,7 @@ from .observability import (
     init_sentry,
     setup_logging,
 )
-from .routers import apikeys, auth, traces
+from .routers import admin, alerts, apikeys, auth, datasets, experiments, metrics, projects, traces
 
 logging.basicConfig(level=logging.INFO)
 settings = get_settings()
@@ -81,6 +81,14 @@ app.include_router(apikeys.router)
 app.include_router(traces.router)
 app.include_router(traces.ws_router)
 app.include_router(traces.runs_router)
+app.include_router(datasets.router)
+app.include_router(datasets.key_router)
+app.include_router(experiments.router)
+app.include_router(experiments.key_router)
+app.include_router(metrics.router)
+app.include_router(alerts.router)
+app.include_router(projects.router)
+app.include_router(admin.router)
 
 
 @app.get("/")

provekit-0.6.0/provekit/mcp.py

@@ -0,0 +1,83 @@
+"""ProveKit MCP server — debug your traces from any MCP client (Claude Desktop, Cursor, …).
+
+This is a *debug channel*, not a second client SDK: your agent still installs only the one
+`provekit` tracing SDK. This server runs wherever you like (your laptop, CI) and reads the
+already-captured traces from your portal over HTTP, authenticated by the same project key.
+
+    pip install "provekit[mcp]"
+    export PROVEKIT_API_KEY=pk_...        # your project key
+    export PROVEKIT_ENDPOINT=https://provekit.your-co.com
+    provekit-mcp                          # speaks MCP over stdio
+
+Then point an MCP client at the `provekit-mcp` command. It exposes tools to list recent
+traces, surface failures, and pull a full span tree — so the assistant can reason over an
+agent run and tell you what broke, with no code added to the traced app.
+"""
+from __future__ import annotations
+
+import os
+
+import httpx
+
+_TIMEOUT = 15
+
+
+def _base() -> tuple[str, dict]:
+    """Resolve the portal URL + auth header from the environment, or raise a clear error."""
+    key = os.environ.get("PROVEKIT_API_KEY")
+    endpoint = os.environ.get("PROVEKIT_ENDPOINT")
+    if not key or not endpoint:
+        raise RuntimeError(
+            "Set PROVEKIT_API_KEY (your pk_ project key) and PROVEKIT_ENDPOINT "
+            "(your portal URL) before starting the ProveKit MCP server."
+        )
+    return endpoint.rstrip("/"), {"Authorization": f"Bearer {key}"}
+
+
+def _get(path: str, **params):
+    url, headers = _base()
+    r = httpx.get(f"{url}/v1{path}", headers=headers, params=params, timeout=_TIMEOUT,
+                  follow_redirects=False)
+    r.raise_for_status()
+    return r.json()
+
+
+# ---- the callable data layer (import-safe; used by the MCP tools and by tests) ----
+def list_traces(limit: int = 20, status: str | None = None) -> list[dict]:
+    """Recent traces, newest first. Pass status='failed' to see only failed runs."""
+    return _get("/traces", limit=limit, **({"status": status} if status else {}))
+
+
+def list_failures(limit: int = 20) -> list[dict]:
+    """Just the traces that ended in a failure — the ones worth debugging first."""
+    return list_traces(limit=limit, status="failed")
+
+
+def get_trace(trace_id: str) -> list[dict]:
+    """Every span of one trace (the full nested flow), in start order."""
+    return _get(f"/traces/{trace_id}")
+
+
+def build_server():  # pragma: no cover - needs the optional `mcp` dep; glue over the tested data layer
+    """Construct the FastMCP server with the trace tools registered. Imported lazily so the
+    module stays importable (and testable) without the optional `mcp` dependency. The three
+    data-layer functions above are registered directly as MCP tools."""
+    from mcp.server.fastmcp import FastMCP
+
+    server = FastMCP("provekit")
+    server.add_tool(list_traces, name="provekit_list_traces",
+                    description="List recent ProveKit traces (agent runs). status='failed' filters to failures.")
+    server.add_tool(list_failures, name="provekit_list_failures",
+                    description="List recent traces that ended in a failure — start debugging here.")
+    server.add_tool(get_trace, name="provekit_get_trace",
+                    description="Fetch the full span tree of one trace by trace_id to inspect what happened.")
+    return server
+
+
+def main() -> None:  # pragma: no cover - blocks serving MCP over stdio; exercised by hand
+    """Console-script entrypoint: `provekit-mcp`. Speaks MCP over stdio."""
+    build_server().run()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

provekit-0.6.0/provekit/migrations/versions/a1b2c3d4e5f6_feedback_sessions.py

@@ -0,0 +1,49 @@
+"""feedback table + runs.session_id
+
+Revision ID: a1b2c3d4e5f6
+Revises: 9d2c06bfa4e6
+Create Date: 2026-07-19 00:00:00.000000
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'a1b2c3d4e5f6'
+down_revision = '9d2c06bfa4e6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table('feedback',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workspace_id', sa.Integer(), nullable=True),
+    sa.Column('trace_id', sa.String(length=32), nullable=False),
+    sa.Column('name', sa.String(length=120), nullable=False),
+    sa.Column('score', sa.Float(), nullable=True),
+    sa.Column('value', sa.String(length=200), nullable=False),
+    sa.Column('comment', sa.Text(), nullable=False),
+    sa.Column('source', sa.String(length=16), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], name=op.f('fk_feedback_workspace_id_workspaces')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_feedback'))
+    )
+    with op.batch_alter_table('feedback', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_feedback_trace_id'), ['trace_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_feedback_workspace_id'), ['workspace_id'], unique=False)
+
+    with op.batch_alter_table('runs', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('session_id', sa.String(length=64), nullable=False,
+                                      server_default=''))
+        batch_op.create_index(batch_op.f('ix_runs_session_id'), ['session_id'], unique=False)
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('runs', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_runs_session_id'))
+        batch_op.drop_column('session_id')
+
+    with op.batch_alter_table('feedback', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_feedback_workspace_id'))
+        batch_op.drop_index(batch_op.f('ix_feedback_trace_id'))
+    op.drop_table('feedback')

provekit-0.6.0/provekit/migrations/versions/b2c3d4e5f6a7_datasets.py

@@ -0,0 +1,54 @@
+"""datasets + dataset_items
+
+Revision ID: b2c3d4e5f6a7
+Revises: a1b2c3d4e5f6
+Create Date: 2026-07-19 01:00:00.000000
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'b2c3d4e5f6a7'
+down_revision = 'a1b2c3d4e5f6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table('datasets',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workspace_id', sa.Integer(), nullable=True),
+    sa.Column('name', sa.String(length=160), nullable=False),
+    sa.Column('description', sa.Text(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], name=op.f('fk_datasets_workspace_id_workspaces')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_datasets'))
+    )
+    with op.batch_alter_table('datasets', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_datasets_workspace_id'), ['workspace_id'], unique=False)
+
+    op.create_table('dataset_items',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workspace_id', sa.Integer(), nullable=True),
+    sa.Column('dataset_id', sa.Integer(), nullable=False),
+    sa.Column('input', sa.Text(), nullable=False),
+    sa.Column('expected', sa.Text(), nullable=False),
+    sa.Column('meta', sa.JSON(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['dataset_id'], ['datasets.id'], name=op.f('fk_dataset_items_dataset_id_datasets')),
+    sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], name=op.f('fk_dataset_items_workspace_id_workspaces')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_dataset_items'))
+    )
+    with op.batch_alter_table('dataset_items', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_dataset_items_dataset_id'), ['dataset_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_dataset_items_workspace_id'), ['workspace_id'], unique=False)
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('dataset_items', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_dataset_items_workspace_id'))
+        batch_op.drop_index(batch_op.f('ix_dataset_items_dataset_id'))
+    op.drop_table('dataset_items')
+    with op.batch_alter_table('datasets', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_datasets_workspace_id'))
+    op.drop_table('datasets')

provekit-0.6.0/provekit/migrations/versions/c3d4e5f6a7b8_experiments.py

@@ -0,0 +1,59 @@
+"""experiments + experiment_results
+
+Revision ID: c3d4e5f6a7b8
+Revises: b2c3d4e5f6a7
+Create Date: 2026-07-19 02:00:00.000000
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'c3d4e5f6a7b8'
+down_revision = 'b2c3d4e5f6a7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table('experiments',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workspace_id', sa.Integer(), nullable=True),
+    sa.Column('name', sa.String(length=160), nullable=False),
+    sa.Column('dataset_id', sa.Integer(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['dataset_id'], ['datasets.id'], name=op.f('fk_experiments_dataset_id_datasets')),
+    sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], name=op.f('fk_experiments_workspace_id_workspaces')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_experiments'))
+    )
+    with op.batch_alter_table('experiments', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_experiments_dataset_id'), ['dataset_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_experiments_workspace_id'), ['workspace_id'], unique=False)
+
+    op.create_table('experiment_results',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workspace_id', sa.Integer(), nullable=True),
+    sa.Column('experiment_id', sa.Integer(), nullable=False),
+    sa.Column('item_id', sa.Integer(), nullable=True),
+    sa.Column('input', sa.Text(), nullable=False),
+    sa.Column('output', sa.Text(), nullable=False),
+    sa.Column('expected', sa.Text(), nullable=False),
+    sa.Column('scores', sa.JSON(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['experiment_id'], ['experiments.id'], name=op.f('fk_experiment_results_experiment_id_experiments')),
+    sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], name=op.f('fk_experiment_results_workspace_id_workspaces')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_experiment_results'))
+    )
+    with op.batch_alter_table('experiment_results', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_experiment_results_experiment_id'), ['experiment_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_experiment_results_workspace_id'), ['workspace_id'], unique=False)
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('experiment_results', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_experiment_results_workspace_id'))
+        batch_op.drop_index(batch_op.f('ix_experiment_results_experiment_id'))
+    op.drop_table('experiment_results')
+    with op.batch_alter_table('experiments', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_experiments_workspace_id'))
+        batch_op.drop_index(batch_op.f('ix_experiments_dataset_id'))
+    op.drop_table('experiments')

provekit-0.6.0/provekit/migrations/versions/d4e5f6a7b8c9_alerts.py

@@ -0,0 +1,40 @@
+"""alerts
+
+Revision ID: d4e5f6a7b8c9
+Revises: c3d4e5f6a7b8
+Create Date: 2026-07-19 03:00:00.000000
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'd4e5f6a7b8c9'
+down_revision = 'c3d4e5f6a7b8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table('alerts',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('workspace_id', sa.Integer(), nullable=True),
+    sa.Column('name', sa.String(length=160), nullable=False),
+    sa.Column('metric', sa.String(length=32), nullable=False),
+    sa.Column('comparator', sa.String(length=4), nullable=False),
+    sa.Column('threshold', sa.Float(), nullable=False),
+    sa.Column('window_hours', sa.Integer(), nullable=False),
+    sa.Column('email', sa.String(length=255), nullable=False),
+    sa.Column('enabled', sa.Boolean(), nullable=False),
+    sa.Column('last_triggered_at', sa.DateTime(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], name=op.f('fk_alerts_workspace_id_workspaces')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_alerts'))
+    )
+    with op.batch_alter_table('alerts', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_alerts_workspace_id'), ['workspace_id'], unique=False)
+
+
+def downgrade() -> None:
+    with op.batch_alter_table('alerts', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_alerts_workspace_id'))
+    op.drop_table('alerts')