provekit 0.1.0__tar.gz

provekit-0.1.0/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: provekit
+Version: 0.1.0
+Summary: Prove your agent works — test any AI agent (LLM/MCP/HTTP/A2A) and run the suite in CI.
+Author: ProveKit contributors
+License: MIT
+Project-URL: Homepage, https://github.com/MobirizerServices/ProveKit
+Project-URL: Repository, https://github.com/MobirizerServices/ProveKit
+Project-URL: Issues, https://github.com/MobirizerServices/ProveKit/issues
+Keywords: agent,llm,testing,evals,mcp,ci,prompt,openai,anthropic
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi==0.115.6
+Requires-Dist: uvicorn[standard]==0.34.0
+Requires-Dist: sqlalchemy==2.0.36
+Requires-Dist: pydantic==2.10.4
+Requires-Dist: pydantic-settings==2.7.1
+Requires-Dist: httpx==0.28.1
+Requires-Dist: jsonschema==4.23.0
+Requires-Dist: cryptography==49.0.0
+Requires-Dist: pyyaml==6.0.3
+Requires-Dist: alembic==1.18.5
+Requires-Dist: psycopg[binary]==3.3.4
+Requires-Dist: email-validator==2.2.0
+Requires-Dist: redis==5.2.1
+Requires-Dist: sentry-sdk[fastapi]==2.20.0
+Provides-Extra: dev
+Requires-Dist: pytest==9.1.1; extra == "dev"
+Requires-Dist: ruff==0.9.2; extra == "dev"
+
+# ProveKit
+
+**Prove your agent works.** ProveKit tests any AI agent — an LLM API, an MCP server, an
+HTTP agent, or an A2A agent — turns a run into a regression test, and runs the suite
+headless in CI. Open-source, local-first, no account.
+
+This package is the **`provekit` CLI** — the headless test runner. The full visual app
+(console, flow builder, live step-debugger) runs as a server; see the
+[repo](https://github.com/MobirizerServices/ProveKit).
+
+## Install
+
+```bash
+pip install provekit
+```
+
+## Use in CI
+
+Write plain-text, git-diffable tests under `.provekit/` (connections referenced by name,
+secrets via `${ENV_VAR}` — never in the files), commit them, and run:
+
+```bash
+provekit run .provekit/tests/                      # pretty output, non-zero exit on failure
+provekit run .provekit/tests/ --format junit -o results.xml
+provekit import-promptfoo promptfooconfig.yaml -o .provekit/tests/   # migrate from promptfoo
+```
+
+Connections resolve from `.provekit/connections.yaml`:
+
+```yaml
+connections:
+  OpenAI (prod):
+    provider: openai
+    api_key: ${OPENAI_API_KEY}
+    models: [gpt-4o-mini]
+```
+
+## Assertions
+
+`contains` · `equals` · `regex` · `json_path` · `json_schema` · `tool_called` ·
+`latency_lt` · `llm_judge` — pass/fail per assertion, non-zero exit on failure.
+
+## Links
+
+- Repository & full app: https://github.com/MobirizerServices/ProveKit
+- File format: see `docs/FILE_FORMAT.md` in the repo
+- License: MIT

provekit-0.1.0/README.md

@@ -0,0 +1,47 @@
+# ProveKit
+
+**Prove your agent works.** ProveKit tests any AI agent — an LLM API, an MCP server, an
+HTTP agent, or an A2A agent — turns a run into a regression test, and runs the suite
+headless in CI. Open-source, local-first, no account.
+
+This package is the **`provekit` CLI** — the headless test runner. The full visual app
+(console, flow builder, live step-debugger) runs as a server; see the
+[repo](https://github.com/MobirizerServices/ProveKit).
+
+## Install
+
+```bash
+pip install provekit
+```
+
+## Use in CI
+
+Write plain-text, git-diffable tests under `.provekit/` (connections referenced by name,
+secrets via `${ENV_VAR}` — never in the files), commit them, and run:
+
+```bash
+provekit run .provekit/tests/                      # pretty output, non-zero exit on failure
+provekit run .provekit/tests/ --format junit -o results.xml
+provekit import-promptfoo promptfooconfig.yaml -o .provekit/tests/   # migrate from promptfoo
+```
+
+Connections resolve from `.provekit/connections.yaml`:
+
+```yaml
+connections:
+  OpenAI (prod):
+    provider: openai
+    api_key: ${OPENAI_API_KEY}
+    models: [gpt-4o-mini]
+```
+
+## Assertions
+
+`contains` · `equals` · `regex` · `json_path` · `json_schema` · `tool_called` ·
+`latency_lt` · `llm_judge` — pass/fail per assertion, non-zero exit on failure.
+
+## Links
+
+- Repository & full app: https://github.com/MobirizerServices/ProveKit
+- File format: see `docs/FILE_FORMAT.md` in the repo
+- License: MIT

provekit-0.1.0/provekit/cli.py

@@ -0,0 +1,256 @@
+"""`provekit` — headless runner for .provekit test files (CI).
+
+    provekit run .provekit/tests/            # run every test file under a dir
+    provekit run support-bot.yaml --format junit -o results.xml
+    provekit import-promptfoo promptfooconfig.yaml -o .provekit/tests/
+
+Connections resolve from a local file (default: .provekit/connections.yaml, then
+~/.provekit/connections.yaml), NEVER from the test files. Secrets in that file may
+be written as ${ENV_VAR} and are expanded from the environment at run time.
+
+Exit code is non-zero when any assertion fails or any run errors — so CI gates on it.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+from xml.sax.saxutils import escape
+
+import yaml
+
+from .services import assertions as ae
+from .services import dispatch, testfile
+from .services.promptfoo import import_promptfoo
+
+_ENV_REF = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+
+# ---- local connection registry (stands in for the DB in CLI mode) ----
+class _Conn:
+    def __init__(self, config):
+        self.config = config
+
+
+class _Registry:
+    """Minimal object graph dispatch/assertions expect from a `db`: `.get(model, id)`."""
+    def __init__(self, by_name: dict):
+        self._by_name = by_name
+        self._ids = {name: i + 1 for i, name in enumerate(by_name)}
+        self._by_id = {i: _Conn(by_name[name]) for name, i in self._ids.items()}
+
+    def id_of(self, name: str) -> int | None:
+        return self._ids.get(name)
+
+    def get(self, _model, cid):
+        return self._by_id.get(cid)
+
+
+def _expand_env(obj):
+    if isinstance(obj, str):
+        return _ENV_REF.sub(lambda m: os.environ.get(m.group(1), ""), obj)
+    if isinstance(obj, dict):
+        return {k: _expand_env(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_expand_env(v) for v in obj]
+    return obj
+
+
+def _load_connections(explicit: str | None) -> _Registry:
+    paths = [explicit] if explicit else [".provekit/connections.yaml",
+                                         str(Path.home() / ".provekit/connections.yaml")]
+    for p in paths:
+        if p and Path(p).exists():
+            doc = yaml.safe_load(Path(p).read_text()) or {}
+            return _Registry(_expand_env(doc.get("connections") or {}))
+    return _Registry({})
+
+
+# ---- running ----
+def _run_case(reg, request, variables):
+    rd = _collect(reg, request, variables)
+    asserts = ae.evaluate(reg, request.get("assertions") or [], rd)
+    ok = all(a["ok"] for a in asserts) if asserts else rd["status"] == "completed"
+    return {"status": rd["status"], "error": rd["error"], "duration_ms": rd["duration_ms"],
+            "text": rd["result"].get("text"), "assertions": asserts, "ok": ok}
+
+
+def _collect(reg, req, variables):
+    # dispatch is async; the CLI is sync, so drive it via the sync bridge (its own loop).
+    r = dispatch.run_collect_sync(reg, req, variables)
+    return {"result": {"text": r["text"], "output": r["output"], "meta": r["meta"]},
+            "status": r["status"], "error": r["error"], "duration_ms": r.get("duration_ms", 0)}
+
+
+def _resolve_request(doc, reg) -> tuple[dict, str | None]:
+    """Build the runnable request from a test doc; returns (request, unresolved_connection)."""
+    req = dict(doc["request"])
+    name = doc.get("connection")
+    cid = reg.id_of(name) if name else None
+    if cid:
+        req["connection_id"] = cid
+    if doc.get("assertions"):
+        req["assertions"] = doc["assertions"]
+    # llm_judge assertions may name their own judge connection
+    for a in req.get("assertions") or []:
+        if a.get("type") == "llm_judge" and a.get("connection"):
+            a["connection_id"] = reg.id_of(a["connection"])
+    return req, (name if name and not cid else None)
+
+
+def _iter_test_files(paths):
+    for p in paths:
+        path = Path(p)
+        if path.is_dir():
+            yield from sorted(path.rglob("*.yaml"))
+            yield from sorted(path.rglob("*.yml"))
+        elif path.exists():
+            yield path
+        else:
+            print(f"warning: path not found: {p}", file=sys.stderr)
+
+
+def cmd_run(args) -> int:
+    reg = _load_connections(args.connections)
+    cli_vars = dict(kv.split("=", 1) for kv in args.env) if args.env else {}
+    files = list(_iter_test_files(args.paths))
+    if not files:
+        print("no .provekit test files found", file=sys.stderr)
+        return 2
+
+    suites = []
+    for f in files:
+        try:
+            doc = testfile.load(f.read_text())
+        except ValueError as exc:
+            suites.append({"file": str(f), "error": f"parse error: {exc}", "cases": []})
+            continue
+        if doc.get("kind") != "test":
+            continue  # flows aren't CI test units
+        request, unresolved = _resolve_request(doc, reg)
+        rows = doc.get("dataset") or [{"name": None, "variables": {}}]
+        cases = []
+        for i, row in enumerate(rows):
+            variables = {**cli_vars, **(row.get("variables") or {})}
+            res = _run_case(reg, request, variables)
+            res["name"] = row.get("name") or (f"row {i + 1}" if len(rows) > 1 else doc["name"])
+            cases.append(res)
+        suites.append({"file": str(f), "name": doc["name"], "cases": cases,
+                       "error": (f"connection '{unresolved}' not found" if unresolved else None)})
+
+    _render(suites, args.format, args.output)
+    failed = sum(1 for s in suites for c in s["cases"] if not c["ok"]) + \
+        sum(1 for s in suites if s.get("error"))
+    return 1 if failed else 0
+
+
+# ---- output renderers ----
+_G, _R, _Y, _D, _X = "\033[32m", "\033[31m", "\033[33m", "\033[2m", "\033[0m"
+
+
+def _color(on: bool):
+    return ("", "", "", "", "") if not on else (_G, _R, _Y, _D, _X)
+
+
+def _render(suites, fmt, output):
+    if fmt == "json":
+        import json
+        text = json.dumps({"suites": suites}, indent=2)
+    elif fmt == "junit":
+        text = _junit(suites)
+    else:
+        text = _pretty(suites, color=output is None and sys.stdout.isatty())
+    if output:
+        Path(output).write_text(text)
+        print(f"wrote {fmt} report to {output}", file=sys.stderr)
+    else:
+        print(text)
+
+
+def _pretty(suites, color: bool) -> str:
+    g, r, y, d, x = _color(color)
+    lines, total, passed = [], 0, 0
+    for s in suites:
+        if s.get("error") and not s["cases"]:
+            lines.append(f"{r}✕{x} {s['file']} — {s['error']}")
+            continue
+        lines.append(f"{d}{s['file']}{x}")
+        if s.get("error"):
+            lines.append(f"  {y}! {s['error']}{x}")
+        for c in s["cases"]:
+            total += 1; passed += c["ok"]
+            mark = f"{g}✓{x}" if c["ok"] else f"{r}✕{x}"
+            lines.append(f"  {mark} {c['name']} {d}{c['duration_ms']}ms{x}")
+            for a in c["assertions"]:
+                am = f"{g}✓{x}" if a["ok"] else f"{r}✕{x}"
+                lines.append(f"      {am} {a['type']}: {a['detail']}")
+            if c["error"]:
+                lines.append(f"      {r}error: {c['error']}{x}")
+    head = f"{g if passed == total else r}{passed}/{total} passed{x}"
+    lines.append("")
+    lines.append(head)
+    return "\n".join(lines)
+
+
+def _junit(suites) -> str:
+    out = ['<?xml version="1.0" encoding="UTF-8"?>', "<testsuites>"]
+    for s in suites:
+        cases = s["cases"]
+        fails = sum(1 for c in cases if not c["ok"])
+        errs = 1 if s.get("error") and not cases else 0
+        out.append(f'  <testsuite name="{escape(s.get("name") or s["file"])}" '
+                   f'tests="{len(cases)}" failures="{fails}" errors="{errs}">')
+        if errs:
+            out.append(f'    <testcase name="{escape(s["file"])}">'
+                       f'<error message="{escape(s["error"])}"/></testcase>')
+        for c in cases:
+            t = (c["duration_ms"] or 0) / 1000
+            out.append(f'    <testcase name="{escape(c["name"])}" time="{t:.3f}">')
+            if not c["ok"]:
+                detail = c["error"] or "; ".join(f"{a['type']}: {a['detail']}"
+                                                 for a in c["assertions"] if not a["ok"])
+                out.append(f'      <failure message="{escape(detail)}"/>')
+            out.append("    </testcase>")
+        out.append("  </testsuite>")
+    out.append("</testsuites>")
+    return "\n".join(out)
+
+
+def cmd_import_promptfoo(args) -> int:
+    docs, warnings = import_promptfoo(Path(args.config).read_text())
+    outdir = Path(args.output or ".")
+    outdir.mkdir(parents=True, exist_ok=True)
+    for name, text in docs:
+        (outdir / name).write_text(text)
+        print(f"wrote {outdir / name}")
+    for w in warnings:  # never drop anything silently
+        print(f"warning: {w}", file=sys.stderr)
+    print(f"imported {len(docs)} test(s) from {args.config}", file=sys.stderr)
+    return 0
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(prog="provekit", description="Run .provekit agent tests.")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    run = sub.add_parser("run", help="run test files or directories")
+    run.add_argument("paths", nargs="+")
+    run.add_argument("--connections", help="connections YAML (default: .provekit/connections.yaml)")
+    run.add_argument("--env", action="append", metavar="KEY=VAL", help="extra template variables")
+    run.add_argument("--format", choices=["pretty", "json", "junit"], default="pretty")
+    run.add_argument("-o", "--output", help="write the report to a file instead of stdout")
+    run.set_defaults(func=cmd_run)
+
+    imp = sub.add_parser("import-promptfoo", help="convert a promptfoo config to .provekit tests")
+    imp.add_argument("config")
+    imp.add_argument("-o", "--output", help="output directory (default: cwd)")
+    imp.set_defaults(func=cmd_import_promptfoo)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

provekit-0.1.0/provekit/config.py

@@ -0,0 +1,77 @@
+"""ProveKit configuration."""
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    database_url: str = "sqlite:///./provekit.db"
+    cors_origins: str = "http://localhost:3001,http://localhost:3000"
+
+    # Hosted mode: stricter outbound-URL guarding (block private/internal ranges).
+    # Leave false for local single-user use.
+    hosted: bool = False
+
+    # Encryption key for secrets at rest (any string; derived to a Fernet key).
+    # Optional for local SQLite (a key file is auto-generated next to the db);
+    # required for any other database.
+    secret_key: str = ""
+
+    # Optional OTLP/HTTP collector to mirror ProveKit's own runs as gen_ai spans.
+    otel_export_url: str = ""
+
+    # Public base URL shown in deployment endpoints (where /v1/d/{slug} is reachable).
+    public_base_url: str = "http://localhost:8100"
+
+    # Wall-clock cap for a single deployment invocation (seconds); 0 disables.
+    deployment_timeout_s: float = 120.0
+
+    # Optional Sentry DSN for error reporting (off when unset).
+    sentry_dsn: str = ""
+
+    # Web app base URL (for links in emails). Same-origin in prod.
+    web_base_url: str = "http://localhost:3001"
+
+    # Email (account recovery/verification). No SMTP host → links are logged, not sent.
+    smtp_host: str = ""
+    smtp_port: int = 587
+    smtp_user: str = ""
+    smtp_password: str = ""
+    smtp_from: str = ""
+    smtp_starttls: bool = True
+    require_email_verification: bool = False
+
+    # Optional Redis for paused flow-run contexts (survives workers/restarts).
+    # Unset → in-memory (single-process local use).
+    redis_url: str = ""
+
+    # Max concurrent sync workers (streams run in this threadpool). Lifts Starlette's
+    # ~40-token default so many more streams run at once without an async rewrite.
+    thread_pool_size: int = 200
+
+    # Quotas (protect shared infra). 0 disables a given limit.
+    rate_limit_per_min: int = 120     # run requests per workspace per minute
+    login_attempts_per_min: int = 10  # login attempts per email+IP per minute
+    dataset_max_rows: int = 200       # rows per dataset run
+    max_tokens_cap: int = 0           # hard cap on prompt max_tokens (0 = no cap)
+    runs_retention: int = 1000        # keep last N runs per workspace
+    max_body_bytes: int = 2_000_000   # reject request bodies larger than this
+    max_flow_nodes: int = 200         # cap nodes per flow
+
+    # Seed empty OpenAI + Anthropic example connections on first run. The keyless
+    # "Demo Assistant (mock)" connection is seeded regardless of this setting.
+    seed_examples: bool = True
+
+    # A default OpenAI key can be provided to prefill the example LLM connection.
+    openai_api_key: str = ""
+
+    @property
+    def cors_origin_list(self) -> list[str]:
+        return [o.strip() for o in self.cors_origins.split(",") if o.strip()]
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

provekit-0.1.0/provekit/database.py

@@ -0,0 +1,132 @@
+"""SQLAlchemy engine + session (SQLite)."""
+from sqlalchemy import MetaData, create_engine, event
+from sqlalchemy.orm import DeclarativeBase, sessionmaker
+
+from .config import get_settings
+
+settings = get_settings()
+_is_sqlite = settings.database_url.startswith("sqlite")
+connect_args = {"check_same_thread": False} if _is_sqlite else {}
+# A concurrent stream holds a pooled connection for its lifetime; if the pool is exhausted,
+# the next (synchronous) checkout blocks the event loop. Size the pool to the stream
+# concurrency so checkouts never block the loop.
+if _is_sqlite and ":memory:" in settings.database_url:
+    _pool = {}
+else:
+    _pool = {"pool_size": 20, "max_overflow": max(10, settings.thread_pool_size)}
+    if not _is_sqlite:
+        _pool["pool_pre_ping"] = True  # Postgres: drop stale connections (harmful w/ SQLite WAL pragma)
+engine = create_engine(settings.database_url, connect_args=connect_args, **_pool)
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
+
+if _is_sqlite:
+    @event.listens_for(engine, "connect")
+    def _sqlite_pragmas(dbapi_conn, _record):
+        # WAL lets readers and a writer coexist; busy_timeout makes a contended write wait
+        # briefly instead of erroring — important now that many async streams write runs.
+        cur = dbapi_conn.cursor()
+        cur.execute("PRAGMA journal_mode=WAL")
+        cur.execute("PRAGMA busy_timeout=5000")
+        cur.close()
+
+# Named constraints so Alembic's SQLite batch mode (ALTER via table-rebuild) can re-add them.
+_NAMING = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}
+
+
+class Base(DeclarativeBase):
+    metadata = MetaData(naming_convention=_NAMING)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+def init_db() -> None:
+    from . import models  # noqa: F401
+    if settings.database_url.startswith("sqlite") and ":memory:" in settings.database_url:
+        # Ephemeral in-memory DB (tests only): create_all is fast and needs no migration tooling.
+        Base.metadata.create_all(bind=engine)
+    else:
+        # Every persistent DB — file-based SQLite included — is owned by Alembic so that
+        # future column/index changes actually apply on upgrade. create_all only ever adds
+        # whole tables and would silently skip new columns on an existing local database.
+        _run_migrations()
+
+
+# Revision that first introduced each column, newest first — used to adopt a pre-migration
+# database (built by the old create_all path) at the revision matching its actual columns.
+_ADOPT_BY_COLUMN = [
+    ("users", "token_version", "a1b2c3d4e5f6"),
+    ("users", "email_verified", "323eb73d463c"),
+    ("workspaces", "ingest_key_hash", "819ed5ff183e"),
+]
+_BASELINE_REVISION = "49e8ab812556"
+
+
+# Postgres advisory-lock key ("AGMN") used to serialize migrations across uvicorn workers.
+_MIGRATION_LOCK_ID = 0x4147_4D4E
+
+
+def _migration_config():
+    """Alembic config built in code rather than read from alembic.ini.
+
+    The migrations ship *inside* the package, so script_location is resolved absolutely
+    from `__file__`. Reading alembic.ini instead would only work when the process runs
+    from a source checkout — a `pip install`ed wheel has no alembic.ini next to the
+    package, and the app would fail to boot. alembic.ini stays for `alembic` CLI use.
+    """
+    import os
+
+    from alembic.config import Config
+
+    cfg = Config()
+    cfg.set_main_option("script_location", os.path.join(os.path.dirname(__file__), "migrations"))
+    # No sqlalchemy.url here on purpose: env.py reads it from settings and builds the engine
+    # itself, which keeps a '%' in the password out of ConfigParser's interpolation.
+    return cfg
+
+
+def _run_migrations() -> None:
+    cfg = _migration_config()
+    if settings.database_url.startswith("sqlite"):
+        _migrate_to_head(cfg)  # single process — no cross-worker race
+        return
+    # With `--workers N` every worker runs the lifespan and would race through upgrade
+    # (DuplicateTable / duplicate alembic_version). A Postgres session advisory lock lets
+    # exactly one run at a time; the rest block, then find the schema already at head.
+    with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
+        conn.exec_driver_sql("SELECT pg_advisory_lock(%s)", (_MIGRATION_LOCK_ID,))
+        try:
+            _migrate_to_head(cfg)
+        finally:
+            conn.exec_driver_sql("SELECT pg_advisory_unlock(%s)", (_MIGRATION_LOCK_ID,))
+
+
+def _migrate_to_head(cfg) -> None:
+    from alembic import command
+    from alembic.runtime.migration import MigrationContext
+    from sqlalchemy import inspect
+
+    insp = inspect(engine)
+    with engine.connect() as conn:
+        current = MigrationContext.configure(conn).get_current_revision()
+    if current is None and insp.has_table("users"):
+        # A database created by create_all before migrations existed has the tables but no
+        # alembic version. Stamp it at the revision matching its columns so `upgrade` applies
+        # only what's genuinely missing instead of trying to re-create existing tables.
+        stamp_at = _BASELINE_REVISION
+        for table, column, revision in _ADOPT_BY_COLUMN:
+            if any(c["name"] == column for c in insp.get_columns(table)):
+                stamp_at = revision
+                break
+        command.stamp(cfg, stamp_at)
+    command.upgrade(cfg, "head")