ragops 1.6.0__py3-none-any.whl

ragops/__init__.py

@@ -0,0 +1,16 @@
+"""RAGOps public API."""
+
+from ragops.engine import compare, evaluate
+from ragops.loader import load_responses, load_scenario, responses_from_data, scenario_from_dict
+
+__version__ = "1.6.0"
+
+__all__ = [
+    "compare",
+    "evaluate",
+    "load_responses",
+    "load_scenario",
+    "responses_from_data",
+    "scenario_from_dict",
+    "__version__",
+]

ragops/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""Optional dependency-free integration adapters."""
+
+from .http import AdapterError, post_json
+
+__all__ = ["AdapterError", "post_json"]

ragops/adapters/http.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import Any
+
+
+class AdapterError(RuntimeError):
+    """Raised when a remote application adapter cannot return valid JSON."""
+
+
+def post_json(
+    url: str,
+    payload: dict[str, Any],
+    *,
+    headers: dict[str, str] | None = None,
+    timeout: float = 30.0,
+) -> dict[str, Any]:
+    request_headers = {"content-type": "application/json", **(headers or {})}
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers=request_headers,
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:  # noqa: S310
+            data = json.loads(response.read().decode("utf-8"))
+    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
+        raise AdapterError(f"HTTP adapter failed for {url}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise AdapterError(f"HTTP adapter expected a JSON object from {url}")
+    return data

ragops/benchmarks.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from ragops.models import Scenario
+
+
+def scenario_summary(scenario: Scenario) -> dict[str, Any]:
+    """Return deterministic benchmark coverage metadata for authoring and review."""
+    categories = Counter(case.category for case in scenario.cases)
+    severities = Counter(case.severity for case in scenario.cases)
+    languages = Counter(case.language for case in scenario.cases)
+    attacks = Counter(
+        case.attack_category for case in scenario.cases if case.attack_category is not None
+    )
+    return {
+        "schema_version": scenario.schema_version,
+        "scenario_id": scenario.id,
+        "case_count": len(scenario.cases),
+        "categories": dict(sorted(categories.items())),
+        "severities": dict(sorted(severities.items())),
+        "languages": dict(sorted(languages.items())),
+        "attack_categories": dict(sorted(attacks.items())),
+    }

ragops/cli.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from ragops.benchmarks import scenario_summary
+from ragops.config import load_regression_policy
+from ragops.control_plane import ControlPlane
+from ragops.demo import write_demo
+from ragops.engine import compare, evaluate
+from ragops.loader import ContractError, load_responses, load_scenario
+from ragops.plugins import (
+    CaseEvaluator,
+    CitationCorrectnessEvaluator,
+    ClaimSupportEvaluator,
+    RetrievalRecallEvaluator,
+)
+from ragops.reporters import comparison_html, comparison_markdown, evaluation_markdown
+from ragops.store import ExperimentStore
+from ragops.traces import load_trace_jsonl
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="ragops")
+    commands = parser.add_subparsers(dest="command", required=True)
+    demo_parser = commands.add_parser("demo", help="Generate a credential-free release-gate demo")
+    demo_parser.add_argument("--output", default="ragops-demo")
+    demo_parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Replace regular demo files in an existing non-symlink directory",
+    )
+    evaluate_parser = commands.add_parser("evaluate", help="Evaluate recorded responses")
+    evaluate_parser.add_argument("--scenario", required=True)
+    evaluate_input = evaluate_parser.add_mutually_exclusive_group(required=True)
+    evaluate_input.add_argument("--responses")
+    evaluate_input.add_argument("--traces", help="Portable JSONL trace input")
+    evaluate_parser.add_argument("--output")
+    evaluate_parser.add_argument("--format", choices=("json", "markdown"), default="json")
+    evaluate_parser.add_argument("--store")
+    evaluate_parser.add_argument("--label", default="")
+    evaluate_parser.add_argument(
+        "--evaluator",
+        action="append",
+        choices=("retrieval_recall", "citation_correctness", "claim_support"),
+        default=[],
+    )
+    compare_parser = commands.add_parser("compare", help="Compare candidate responses to baseline")
+    compare_parser.add_argument("--scenario", required=True)
+    baseline_input = compare_parser.add_mutually_exclusive_group(required=True)
+    baseline_input.add_argument("--baseline")
+    baseline_input.add_argument("--baseline-traces")
+    candidate_input = compare_parser.add_mutually_exclusive_group(required=True)
+    candidate_input.add_argument("--candidate")
+    candidate_input.add_argument("--candidate-traces")
+    compare_parser.add_argument("--output")
+    compare_parser.add_argument("--format", choices=("json", "markdown", "html"), default="markdown")
+    compare_parser.add_argument("--store")
+    compare_parser.add_argument("--label", default="")
+    compare_parser.add_argument("--policy", help="TOML regression policy")
+    history_parser = commands.add_parser("history", help="List saved experiment runs")
+    history_parser.add_argument("--store", required=True)
+    history_parser.add_argument("--limit", type=int, default=20)
+    review_parser = commands.add_parser("review", help="Review a saved experiment run")
+    review_parser.add_argument("--store", required=True)
+    review_parser.add_argument("--run-id", required=True)
+    review_parser.add_argument(
+        "--status", required=True, choices=("accepted", "rejected", "needs_changes")
+    )
+    review_parser.add_argument("--reviewer", required=True)
+    review_parser.add_argument("--note", default="")
+    trend_parser = commands.add_parser("trend", help="Read a saved metric trend")
+    trend_parser.add_argument("--store", required=True)
+    trend_parser.add_argument("--scenario-id", required=True)
+    trend_parser.add_argument("--metric", required=True)
+    trend_parser.add_argument("--limit", type=int, default=50)
+    inspect_parser = commands.add_parser("inspect", help="Inspect scenario benchmark coverage")
+    inspect_parser.add_argument("--scenario", required=True)
+    workspace_create = commands.add_parser("workspace-create", help="Create an alpha workspace")
+    workspace_create.add_argument("--root", required=True)
+    workspace_create.add_argument("--workspace-id", required=True)
+    workspace_create.add_argument("--name", required=True)
+    workspace_rotate = commands.add_parser("workspace-rotate-key", help="Rotate a workspace key")
+    workspace_rotate.add_argument("--root", required=True)
+    workspace_rotate.add_argument("--workspace-id", required=True)
+    workspace_rotate.add_argument("--current-key", required=True)
+    workspace_audit = commands.add_parser("workspace-audit", help="Read workspace audit events")
+    workspace_audit.add_argument("--root", required=True)
+    workspace_audit.add_argument("--workspace-id", required=True)
+    workspace_audit.add_argument("--limit", type=int, default=100)
+    return parser
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+    if args.command == "demo":
+        try:
+            summary = write_demo(args.output, force=args.force)
+        except OSError as exc:
+            raise SystemExit(f"demo output error: {exc}") from exc
+        print(json.dumps(summary, ensure_ascii=False, indent=2))
+        return 0
+    if args.command == "inspect":
+        try:
+            summary = scenario_summary(load_scenario(args.scenario))
+        except ContractError as exc:
+            raise SystemExit(f"contract error: {exc}") from exc
+        print(json.dumps(summary, ensure_ascii=False, indent=2))
+        return 0
+    if args.command == "workspace-create":
+        try:
+            key = ControlPlane(args.root).create_workspace(args.workspace_id, args.name)
+        except ValueError as exc:
+            raise SystemExit(str(exc)) from exc
+        print(json.dumps({"workspace_id": args.workspace_id, "api_key": key}))
+        return 0
+    if args.command == "workspace-rotate-key":
+        try:
+            key = ControlPlane(args.root).rotate_key(args.workspace_id, args.current_key)
+        except (PermissionError, ValueError) as exc:
+            raise SystemExit(str(exc)) from exc
+        print(json.dumps({"workspace_id": args.workspace_id, "api_key": key}))
+        return 0
+    if args.command == "workspace-audit":
+        events = ControlPlane(args.root).audit_events(args.workspace_id, limit=args.limit)
+        print(json.dumps(events, ensure_ascii=False, indent=2))
+        return 0
+    if args.command == "history":
+        runs = ExperimentStore(args.store).list_runs(limit=args.limit)
+        print(json.dumps(runs, ensure_ascii=False, indent=2))
+        return 0
+    if args.command == "review":
+        try:
+            ExperimentStore(args.store).review(
+                args.run_id,
+                status=args.status,
+                reviewer=args.reviewer,
+                note=args.note,
+            )
+        except (KeyError, ValueError) as exc:
+            raise SystemExit(str(exc)) from exc
+        print(json.dumps({"run_id": args.run_id, "review_status": args.status}))
+        return 0
+    if args.command == "trend":
+        points = ExperimentStore(args.store).metric_trend(
+            args.scenario_id,
+            args.metric,
+            limit=args.limit,
+        )
+        print(json.dumps(points, ensure_ascii=False, indent=2))
+        return 0
+    try:
+        scenario = load_scenario(args.scenario)
+        if args.command == "evaluate":
+            report = evaluate(
+                scenario,
+                load_trace_jsonl(args.traces) if args.traces else load_responses(args.responses),
+                evaluators=_evaluators_from_names(args.evaluator),
+            )
+        else:
+            report = compare(
+                scenario,
+                (
+                    load_trace_jsonl(args.baseline_traces)
+                    if args.baseline_traces
+                    else load_responses(args.baseline)
+                ),
+                (
+                    load_trace_jsonl(args.candidate_traces)
+                    if args.candidate_traces
+                    else load_responses(args.candidate)
+                ),
+                policy=load_regression_policy(args.policy) if args.policy else None,
+            )
+    except (ContractError, ValueError) as exc:
+        raise SystemExit(f"contract error: {exc}") from exc
+    if args.format == "json":
+        rendered = json.dumps(report.to_dict(), ensure_ascii=False, indent=2)
+    elif args.command == "evaluate":
+        rendered = evaluation_markdown(report)
+    elif args.format == "html":
+        rendered = comparison_html(report)
+    else:
+        rendered = comparison_markdown(report)
+    if args.store:
+        ExperimentStore(args.store).save(report, label=args.label)
+    if args.output:
+        output_path = Path(args.output)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(rendered.rstrip() + "\n", encoding="utf-8")
+    else:
+        print(rendered)
+    return 0 if report.passed else 2
+
+
+def _evaluators_from_names(names: list[str]) -> tuple[CaseEvaluator, ...]:
+    factories = {
+        "retrieval_recall": RetrievalRecallEvaluator,
+        "citation_correctness": CitationCorrectnessEvaluator,
+        "claim_support": ClaimSupportEvaluator,
+    }
+    return tuple(factories[name]() for name in names)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ragops/config.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import tomllib
+from pathlib import Path
+
+from ragops.models import RegressionPolicy
+
+
+def load_regression_policy(path: str | Path) -> RegressionPolicy:
+    try:
+        data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
+        return RegressionPolicy(**data["regression"])
+    except (OSError, tomllib.TOMLDecodeError, KeyError, TypeError) as exc:
+        raise ValueError(f"Invalid regression policy {path}: {exc}") from exc

ragops/control_plane.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import re
+import secrets
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ragops.store import ExperimentStore
+
+WORKSPACE_ID = re.compile(r"^[a-z0-9][a-z0-9-]{2,62}$")
+
+
+class ControlPlane:
+    """Local control-plane alpha with workspace-isolated experiment stores."""
+
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.index = self.root / "control-plane.db"
+        self._initialize()
+
+    def create_workspace(self, workspace_id: str, name: str) -> str:
+        if not WORKSPACE_ID.fullmatch(workspace_id):
+            raise ValueError("workspace ID must be a lowercase slug of 3-63 characters")
+        if not name.strip():
+            raise ValueError("workspace name is required")
+        api_key = f"rgo_{secrets.token_urlsafe(32)}"
+        digest = _digest(api_key)
+        created_at = datetime.now(timezone.utc).isoformat()
+        try:
+            with self._connect() as connection:
+                connection.execute(
+                    "INSERT INTO workspaces (id, name, key_digest, created_at) VALUES (?, ?, ?, ?)",
+                    (workspace_id, name, digest, created_at),
+                )
+                connection.execute(
+                    "INSERT INTO audit_events (created_at, workspace_id, action, detail) VALUES (?, ?, ?, ?)",
+                    (created_at, workspace_id, "workspace.created", name),
+                )
+        except sqlite3.IntegrityError as exc:
+            raise ValueError(f"workspace already exists: {workspace_id}") from exc
+        return api_key
+
+    def authenticate(self, workspace_id: str, api_key: str) -> bool:
+        with self._connect() as connection:
+            row = connection.execute(
+                "SELECT key_digest FROM workspaces WHERE id = ?", (workspace_id,)
+            ).fetchone()
+        return bool(row and hmac.compare_digest(row[0], _digest(api_key)))
+
+    def workspace_store(self, workspace_id: str, api_key: str) -> ExperimentStore:
+        if not self.authenticate(workspace_id, api_key):
+            raise PermissionError("invalid workspace credentials")
+        path = self.root / "workspaces" / workspace_id / "runs.db"
+        self.audit(workspace_id, "workspace.store_accessed", "experiment store")
+        return ExperimentStore(path)
+
+    def rotate_key(self, workspace_id: str, current_key: str) -> str:
+        if not self.authenticate(workspace_id, current_key):
+            raise PermissionError("invalid workspace credentials")
+        new_key = f"rgo_{secrets.token_urlsafe(32)}"
+        with self._connect() as connection:
+            connection.execute(
+                "UPDATE workspaces SET key_digest = ? WHERE id = ?",
+                (_digest(new_key), workspace_id),
+            )
+        self.audit(workspace_id, "workspace.key_rotated", "credential rotated")
+        return new_key
+
+    def audit(self, workspace_id: str, action: str, detail: str) -> None:
+        with self._connect() as connection:
+            connection.execute(
+                "INSERT INTO audit_events (created_at, workspace_id, action, detail) VALUES (?, ?, ?, ?)",
+                (datetime.now(timezone.utc).isoformat(), workspace_id, action, detail),
+            )
+
+    def audit_events(self, workspace_id: str, *, limit: int = 100) -> list[dict[str, str]]:
+        if limit < 1 or limit > 1000:
+            raise ValueError("limit must be between 1 and 1000")
+        with self._connect() as connection:
+            rows = connection.execute(
+                """
+                SELECT created_at, action, detail FROM audit_events
+                WHERE workspace_id = ? ORDER BY id DESC LIMIT ?
+                """,
+                (workspace_id, limit),
+            ).fetchall()
+        return [
+            {"created_at": row[0], "action": row[1], "detail": row[2]} for row in rows
+        ]
+
+    def _connect(self) -> sqlite3.Connection:
+        return sqlite3.connect(self.index)
+
+    def _initialize(self) -> None:
+        with self._connect() as connection:
+            connection.execute(
+                """
+                CREATE TABLE IF NOT EXISTS workspaces (
+                    id TEXT PRIMARY KEY,
+                    name TEXT NOT NULL,
+                    key_digest TEXT NOT NULL,
+                    created_at TEXT NOT NULL
+                )
+                """
+            )
+            connection.execute(
+                """
+                CREATE TABLE IF NOT EXISTS audit_events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    created_at TEXT NOT NULL,
+                    workspace_id TEXT NOT NULL,
+                    action TEXT NOT NULL,
+                    detail TEXT NOT NULL,
+                    FOREIGN KEY(workspace_id) REFERENCES workspaces(id)
+                )
+                """
+            )
+
+
+def _digest(api_key: str) -> str:
+    return hashlib.sha256(api_key.encode("utf-8")).hexdigest()

ragops/demo.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from uuid import uuid4
+
+from ragops.engine import compare
+from ragops.loader import responses_from_data, scenario_from_dict
+from ragops.reporters import comparison_html, comparison_markdown
+
+
+DEMO_SCENARIO = {
+    "schema_version": "0.2",
+    "id": "ragops-five-minute-demo",
+    "name": "RAGOps five-minute release-gate demo",
+    "thresholds": {
+        "citation_coverage": 1.0,
+        "citation_precision": 1.0,
+        "lexical_groundedness": 0.7,
+        "max_latency_ms": 3000,
+        "max_cost_usd": 0.03,
+    },
+    "redteam": {
+        "forbidden_output_terms": ["DEMO_SECRET"],
+        "require_human_approval_for_external_actions": True,
+        "external_action_markers": ["sent email"],
+    },
+    "cases": [
+        {
+            "id": "demo-procedure",
+            "question": "A1000型でE-42が発生した場合の一次対応は？",
+            "evidence": [
+                "Stop A1000, inspect the cooling fan guard, remove debris, and restart below 60C."
+            ],
+            "required_citation_ids": ["manual-a1000-e42"],
+            "category": "direct_procedure",
+            "severity": "high",
+            "language": "ja",
+        },
+        {
+            "id": "demo-escalation",
+            "question": "E-42をいつエスカレーションする？",
+            "evidence": [
+                "Escalate E-42 after two repeats within 24 hours or temperature remaining above 60C."
+            ],
+            "required_citation_ids": ["policy-e42-escalation"],
+            "category": "escalation_decision",
+            "severity": "high",
+            "language": "ja",
+        },
+    ],
+}
+
+DEMO_BASELINE = [
+    {
+        "case_id": "demo-procedure",
+        "answer": "Stop A1000, inspect the cooling fan guard, remove debris, and restart below 60C.",
+        "citation_ids": ["manual-a1000-e42"],
+        "latency_ms": 850,
+        "cost_usd": 0.008,
+    },
+    {
+        "case_id": "demo-escalation",
+        "answer": "Escalate E-42 after two repeats within 24 hours or temperature remaining above 60C.",
+        "citation_ids": ["policy-e42-escalation"],
+        "latency_ms": 900,
+        "cost_usd": 0.009,
+    },
+]
+
+DEMO_CANDIDATE = [
+    DEMO_BASELINE[0],
+    {
+        "case_id": "demo-escalation",
+        "answer": "Escalate whenever the issue seems serious.",
+        "citation_ids": [],
+        "latency_ms": 920,
+        "cost_usd": 0.009,
+    },
+]
+
+
+def write_demo(output_dir: str | Path, *, force: bool = False) -> dict[str, object]:
+    """Write a credential-free demo bundle and return its release summary."""
+
+    destination = Path(output_dir)
+    if destination.is_symlink():
+        raise FileExistsError(f"refusing symlinked demo output directory: {destination}")
+    if destination.exists():
+        if not force:
+            raise FileExistsError(
+                f"demo output already exists: {destination}; pass --force to replace regular files"
+            )
+        if not destination.is_dir():
+            raise NotADirectoryError(f"demo output is not a directory: {destination}")
+    else:
+        destination.mkdir(parents=True, exist_ok=False)
+    scenario = scenario_from_dict(DEMO_SCENARIO)
+    report = compare(
+        scenario,
+        responses_from_data(DEMO_BASELINE),
+        responses_from_data(DEMO_CANDIDATE),
+    )
+    files = {
+        "scenario": destination / "scenario.json",
+        "baseline": destination / "baseline.json",
+        "candidate": destination / "candidate.json",
+        "markdown_report": destination / "release-report.md",
+        "html_report": destination / "release-report.html",
+    }
+    _write_demo_file(
+        files["scenario"],
+        json.dumps(DEMO_SCENARIO, ensure_ascii=False, indent=2) + "\n",
+        force=force,
+    )
+    _write_demo_file(
+        files["baseline"],
+        json.dumps(DEMO_BASELINE, ensure_ascii=False, indent=2) + "\n",
+        force=force,
+    )
+    _write_demo_file(
+        files["candidate"],
+        json.dumps(DEMO_CANDIDATE, ensure_ascii=False, indent=2) + "\n",
+        force=force,
+    )
+    _write_demo_file(files["markdown_report"], comparison_markdown(report), force=force)
+    _write_demo_file(files["html_report"], comparison_html(report), force=force)
+    return {
+        "demo_completed": True,
+        "candidate_decision": "BLOCK" if not report.passed else "PASS",
+        "failed_gates": list(report.failed_gates),
+        "output_dir": str(destination),
+        "files": {name: str(path) for name, path in files.items()},
+    }
+
+
+def _write_demo_file(path: Path, content: str, *, force: bool) -> None:
+    if path.is_symlink():
+        raise FileExistsError(f"refusing symlinked demo output file: {path}")
+    if not force:
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        descriptor = os.open(path, flags, 0o644)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(content)
+        return
+
+    if path.exists() and not path.is_file():
+        raise FileExistsError(f"refusing non-file demo output target: {path}")
+    temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    descriptor = os.open(temporary, flags, 0o644)
+    try:
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(content)
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)