assay-server 0.1.0__py3-none-any.whl

assay/__init__.py

@@ -0,0 +1,2 @@
+"""Assay: evaluation and observability for document-intelligence pipelines."""
+__version__ = "0.1.0"

assay/__main__.py

@@ -0,0 +1,180 @@
+"""Command line: python -m assay <command>"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from sqlalchemy import select
+
+from assay import runner, store
+from assay.config import Settings
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(prog="assay", description="Evaluation and observability for document-intelligence pipelines.")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("serve", help="Run the API and dashboard")
+    s.add_argument("--host", default="127.0.0.1")
+    s.add_argument("--port", type=int, default=8400)
+    s.add_argument("--every", type=int, metavar="MINUTES",
+                   help="Also run measures every N minutes (overrides ASSAY_SCHEDULE_MINUTES)")
+    s.add_argument("--source", action="append", dest="sources", metavar="SOURCE",
+                   help="Source to schedule, repeatable (overrides ASSAY_SCHEDULE_SOURCES)")
+    s.add_argument("--window-days", type=float, help="Window each scheduled run covers (default 1)")
+
+    r = sub.add_parser("run", help="Compute all measures once and store the results")
+    r.add_argument("--source", default="sql", help="sql (your pipeline database) or events:<tenant>")
+    r.add_argument("--days", type=int, default=7)
+
+    sub.add_parser("check-source", help="Test every mapped field against your pipeline database")
+
+    k = sub.add_parser("keys", help="Create, list and revoke API keys")
+    ks = k.add_subparsers(dest="keys_cmd", required=True)
+    kc = ks.add_parser("create", help="Create a key; the secret is printed once")
+    kc.add_argument("--tenant", required=True, help="Tenant the key belongs to, or '*' for a platform key")
+    kc.add_argument("--scopes", required=True, help="Comma-separated: ingest, read, manage, admin")
+    kc.add_argument("--name", required=True, help="What uses it, e.g. 'invoice pipeline (prod)'")
+    kc.add_argument("--expires-in-days", type=int)
+    kl = ks.add_parser("list", help="List keys (never shows secrets)")
+    kl.add_argument("--tenant")
+    kr = ks.add_parser("revoke", help="Revoke a key immediately")
+    kr.add_argument("id", type=int)
+
+    b = sub.add_parser("backfill", help="Replay past days so baselines and alerts work from day one")
+    b.add_argument("--source", default="sql", help="sql or events:<tenant>")
+    b.add_argument("--days", type=int, default=30)
+    b.add_argument("--window-days", type=float, default=1.0)
+
+    c = sub.add_parser("coverage", help="Which measures your data can answer, and what would unlock the rest")
+    c.add_argument("--source", default="sql", help="sql or events:<tenant>")
+    c.add_argument("--days", type=float, default=7)
+    sub.add_parser("demo", help="Load a synthetic demo tenant and backfill 7 weeks of daily runs")
+    sub.add_parser("schema", help="Print the v1 event schema as JSON Schema")
+
+    args = p.parse_args(argv)
+    if args.cmd == "schema":
+        from assay.schema import json_schema
+        print(json.dumps(json_schema(), indent=1))
+        return 0
+    settings = Settings.from_env()
+
+    if args.cmd == "serve":
+        if args.every is not None:
+            settings.schedule_minutes = args.every
+        if args.sources:
+            settings.schedule_sources = args.sources
+        if args.window_days is not None:
+            settings.schedule_window_days = args.window_days
+        import uvicorn
+        from assay.api import create_app
+        uvicorn.run(create_app(settings), host=args.host, port=args.port)
+        return 0
+
+    engine = store.make_engine(settings.store_url)
+    from assay import integrations
+
+    if args.cmd == "run":
+        try:
+            source = runner.resolve_source(args.source, engine, settings)
+        except ValueError as exc:
+            print(exc, file=sys.stderr)
+            return 2
+        run_id = runner.run_measures(engine, source, runner.window_for_days(args.days),
+                                     notify=integrations.notifier(engine, source.name, settings.public_url,
+                                                                  settings.notifier()),
+                                     alert_min_n=settings.alert_min_n,
+                                     alert_after_runs=settings.alert_after_runs)
+        out = runner.latest_run(engine, source.name)
+        for mid, m in out["measures"].items():
+            val = m["overall"]["value"] if m["overall"] else None
+            shown = "—" if val is None else f"{val:.4g}"
+            print(f"{mid:24} {m['status']:10} {shown:>10}  {m['reason'] or ''}")
+        with engine.connect() as conn:
+            a = store.alerts
+            live = conn.execute(select(a).where((a.c.source == source.name) & (a.c.state == "open"))).all()
+        print(f"run {run_id} stored · {len(live)} open alerts")
+        for r in live:
+            print(f"  [{r.kind}] {r.message}")
+        return 0
+
+    if args.cmd == "keys":
+        from assay import auth
+        if args.keys_cmd == "create":
+            try:
+                row, secret = auth.create_key(engine, args.tenant, args.name,
+                                              [s.strip() for s in args.scopes.split(",") if s.strip()],
+                                              args.expires_in_days)
+            except ValueError as exc:
+                print(exc, file=sys.stderr)
+                return 2
+            print(f"Created key {row['id']} for tenant {row['tenant']} with scopes {', '.join(row['scopes'])}.")
+            print(f"\n  {secret}\n\nStore it now: it isn't shown again. Send it as 'Authorization: Bearer <key>'.")
+            print("Authentication is now required on this server." if not settings.admin_key else "")
+            return 0
+        if args.keys_cmd == "list":
+            for r in auth.list_keys(engine, args.tenant):
+                state = "revoked" if r["revoked_at"] else "active"
+                print(f"{r['id']:>4}  {r['prefix']}…  {r['tenant']:12} {','.join(r['scopes']):22} {state:8} "
+                      f"last used {r['last_used_at'] or 'never'}  {r['name']}")
+            return 0
+        if args.keys_cmd == "revoke":
+            ok = auth.revoke_key(engine, args.id)
+            print("Revoked." if ok else f"No active key {args.id}.")
+            return 0 if ok else 1
+
+    if args.cmd in ("backfill", "coverage"):
+        try:
+            source = runner.resolve_source(args.source, engine, settings)
+        except ValueError as exc:
+            print(exc, file=sys.stderr)
+            return 2
+        if args.cmd == "backfill":
+            out = runner.backfill(engine, source, args.days, args.window_days,
+                                  settings.alert_min_n, settings.alert_after_runs)
+            print(f"{out['runs_created']} runs created, {out['skipped']} days already had one.")
+            return 0
+        from assay.coverage import compute
+        rep = compute(source, runner.window_for_days(args.days), runner.load_rates(engine, source.name))
+        for name, p in rep["records"].items():
+            print(f"{name:11} {'%d rows' % p['rows'] if p['available'] else 'not provided'}")
+        print()
+        for m in rep["measures"]:
+            print(f"{m['status']:8} {m['name']}")
+            for f in m["missing"]:
+                print(f"         needs {f}")
+            for i in m["improve"]:
+                print(f"         better with {i['field']}: {i['why']}")
+        c = rep["counts"]
+        print(f"\n{c['live']} live, {c['partial']} partial, {c['blocked']} blocked")
+        return 0
+
+    if args.cmd == "check-source":
+        if not settings.source_url:
+            print("Set ASSAY_SOURCE_URL first.", file=sys.stderr)
+            return 2
+        from assay.sources.sql import SQLSource
+        report = SQLSource(settings.source_url).check()
+        bad = 0
+        for table, fields in report.items():
+            for field, err in fields.items():
+                print(f"{'ok ' if err is None else 'ERR'} {table}.{field}{'' if err is None else '  ' + err}")
+                bad += err is not None
+        if bad:
+            print(f"\n{bad} field(s) failed. Fix them in your mapping file (ASSAY_SOURCE_MAPPING), "
+                  "or set them to \"NULL\" if your schema doesn't record them.", file=sys.stderr)
+            return 1
+        print("\nAll mapped fields work.")
+        return 0
+
+    if args.cmd == "demo":
+        from assay.demo import seed
+        print(json.dumps(seed(engine), indent=2))
+        print("Demo loaded. Start the dashboard with: python -m assay serve")
+        return 0
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())