dream-eval 0.2.0__py3-none-any.whl

dream_eval/__init__.py

@@ -0,0 +1,18 @@
+"""dream-eval — Agent-agnostic faithfulness evaluation framework."""
+
+from dream_eval.gates import check_hash_determinism, check_secret_leak
+from dream_eval.scoring import compute_faithfulness, compute_precision, compute_recall
+from dream_eval.types import EvalResult, FaithfulnessReport, GateResult
+
+__all__ = [
+    "EvalResult",
+    "FaithfulnessReport",
+    "GateResult",
+    "check_hash_determinism",
+    "check_secret_leak",
+    "compute_faithfulness",
+    "compute_precision",
+    "compute_recall",
+]
+
+__version__ = "0.1.0"

dream_eval/backends.py

@@ -0,0 +1,172 @@
+"""Memory backend adapters — abstract interface for different storage systems."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
+
+from dream_eval.types import EvalReport, EvalResult, Labels
+
+
+class MemoryBackend(ABC):
+    """Abstract interface for memory backends.
+
+    dream-eval is backend-agnostic: implement this interface to plug in
+    Postgres, LanceDB, knowledge graphs, or any other storage system.
+    """
+
+    @abstractmethod
+    def load_eval_report(self, run_id: str) -> EvalReport | None:
+        """Load an evaluator report by run ID."""
+
+    @abstractmethod
+    def load_labels(self, corpus_path: str | None = None) -> Labels:
+        """Load golden corpus labels."""
+
+    @abstractmethod
+    def save_eval_result(self, result: EvalResult) -> None:
+        """Persist an eval result (gates + scoring)."""
+
+    @abstractmethod
+    def list_runs(self, limit: int = 50) -> list[dict[str, Any]]:
+        """List recent eval runs with summary info."""
+
+
+class JsonFileBackend(MemoryBackend):
+    """Simple file-based backend for local eval runs.
+
+    Reads/writes to eval/results/<run_id>/ directories.
+    """
+
+    def __init__(self, results_dir: str = "eval/results") -> None:
+        self.results_dir = Path(results_dir)
+
+    def load_eval_report(self, run_id: str) -> EvalReport | None:
+        import json
+
+        path = self.results_dir / run_id / "eval-report.json"
+        if not path.exists():
+            return None
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return EvalReport.model_validate(data)
+
+    def load_labels(self, corpus_path: str | None = None) -> Labels:
+        import json
+
+        base = Path(corpus_path) if corpus_path else self.results_dir.parent / "golden-corpus"
+        labels_file = base / "labels.json"
+        if not labels_file.exists():
+            return Labels()
+        data = json.loads(labels_file.read_text(encoding="utf-8"))
+        return Labels.model_validate(data)
+
+    def save_eval_result(self, result: EvalResult) -> None:
+        import json
+
+        out_dir = self.results_dir / result.run_id
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+        metrics_path = out_dir / "metrics.json"
+        metrics_path.write_text(
+            json.dumps(result.to_metrics_dict(), indent=2, default=str),
+            encoding="utf-8",
+        )
+
+        summary = _render_summary(result)
+        summary_path = out_dir / "summary.md"
+        summary_path.write_text(summary, encoding="utf-8")
+
+    def list_runs(self, limit: int = 50) -> list[dict[str, Any]]:
+        import json
+
+        runs: list[dict[str, Any]] = []
+        if not self.results_dir.exists():
+            return runs
+
+        for d in sorted(self.results_dir.iterdir(), reverse=True):
+            if not d.is_dir():
+                continue
+            metrics_file = d / "metrics.json"
+            if metrics_file.exists():
+                data = json.loads(metrics_file.read_text(encoding="utf-8"))
+                runs.append({
+                    "run_id": d.name,
+                    "faithfulness": data.get("faithfulness_score"),
+                    "secret_leak": data.get("secret_leak_test"),
+                })
+            if len(runs) >= limit:
+                break
+        return runs
+
+
+class PostgresBackend(MemoryBackend):
+    """Postgres-backed eval results storage.
+
+    Stores eval results in the agent_memory table alongside other memory records.
+    Requires psycopg[binary] >= 3.2.
+    """
+
+    def __init__(self, dsn: str | None = None) -> None:
+        from dream_eval.backends_pg import PostgresEvalBackend
+
+        self._impl = PostgresEvalBackend(dsn)
+
+    def load_eval_report(self, run_id: str) -> EvalReport | None:
+        return self._impl.load_eval_report(run_id)
+
+    def load_labels(self, corpus_path: str | None = None) -> Labels:
+        return self._impl.load_labels(corpus_path)
+
+    def save_eval_result(self, result: EvalResult) -> None:
+        self._impl.save_eval_result(result)
+
+    def list_runs(self, limit: int = 50) -> list[dict[str, Any]]:
+        return self._impl.list_runs(limit)
+
+
+def _render_summary(result: EvalResult) -> str:
+    """Render eval result as markdown summary."""
+    gate_lines = []
+    for g in result.gates:
+        icon = {"pass": "OK", "fail": "FAIL", "warn": "WARN", "skip": "SKIP"}.get(
+            g.status.value, "?"
+        )
+        gate_lines.append(f"| {g.name} | {icon} | {g.message} |")
+
+    f = result.faithfulness
+    gates_table = "\n".join(gate_lines) if gate_lines else "| (none) | - | - |"
+
+    violations = ""
+    if f.recurrence_violations:
+        violations = "## Recurrence Violations\n" + "".join(
+            f"- {v}\n" for v in f.recurrence_violations
+        )
+
+    return f"""# Eval Summary — {result.run_id}
+
+**Mode:** {result.mode.value}
+**Date:** {result.date.isoformat()}
+**Hard fail:** {"YES" if result.hard_fail else "no"}
+
+## Deterministic Gates
+
+| Gate | Status | Message |
+|------|--------|---------|
+{gates_table}
+
+## Faithfulness
+
+| Metric | Value |
+|--------|-------|
+| Faithfulness | {f.faithfulness_score:.3f} |
+| Precision | {f.precision:.3f} |
+| Recall | {f.recall:.3f} |
+| Recurrence calibration | {f.recurrence_calibration:.3f} |
+| Items proposed | {f.items_proposed} |
+| Fully supported | {f.items_fully_supported} |
+| Partially supported | {f.items_partially_supported} |
+| Unsupported | {f.items_unsupported} |
+
+{violations}
+"""

dream_eval/backends_pg.py

@@ -0,0 +1,131 @@
+"""Postgres backend for dream-eval — stores eval results alongside agent memory."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from dream_eval.types import EvalReport, EvalResult, Labels
+
+
+class PostgresEvalBackend:
+    """Stores eval results in the agent_memory table.
+
+    Uses the existing agent_memory schema — eval results are stored as
+    memory_type='decision' with metadata['dream_eval']=True.
+
+    Requires psycopg[binary] >= 3.2 and a running Postgres instance.
+    """
+
+    def __init__(self, dsn: str | None = None) -> None:
+        import os
+
+        if dsn is None:
+            dsn = os.environ.get("AGENT_MEMORY_DATABASE_URL") or os.environ.get(
+                "DATABASE_URL"
+            )
+        if not dsn:
+            raise ValueError(
+                "No database DSN configured. Set AGENT_MEMORY_DATABASE_URL or DATABASE_URL."
+            )
+        from psycopg.rows import dict_row
+        from psycopg_pool import ConnectionPool
+
+        self._pool = ConnectionPool(
+            dsn, min_size=1, max_size=5, kwargs={"row_factory": dict_row}
+        )
+
+    def close(self) -> None:
+        self._pool.close()
+
+    def __enter__(self) -> PostgresEvalBackend:
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    def save_eval_result(self, result: EvalResult) -> None:
+        """Persist eval result as a memory record."""
+        with self._pool.connection() as conn:
+            conn.execute(
+                """
+                INSERT INTO agent_memory (
+                    agent_id, session_id, session_type, memory_type,
+                    content, source, metadata
+                ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, %s::jsonb)
+                """,
+                (
+                    "dream-eval",
+                    result.run_id,
+                    "dream_eval",
+                    "decision",
+                    json.dumps(result.to_metrics_dict()),
+                    "sdk",
+                    json.dumps({"dream_eval": True, "mode": result.mode.value}),
+                ),
+            )
+            conn.commit()
+
+    def load_eval_report(self, run_id: str) -> EvalReport | None:
+        """Load eval report from agent_memory by session_id."""
+        with self._pool.connection() as conn:
+            row = conn.execute(
+                """
+                SELECT content FROM agent_memory
+                WHERE session_id = %s AND metadata ? 'dream_eval'
+                ORDER BY created_at DESC LIMIT 1
+                """,
+                (run_id,),
+            ).fetchone()
+        if not row:
+            return None
+        content = row["content"]
+        if isinstance(content, str):
+            content = json.loads(content)
+        return EvalReport(
+            items=[],
+            sessions_evaluated=content.get("sessions_evaluated", 0),
+            token_cost=content.get("token_cost", 0),
+            latency=content.get("latency", 0),
+        )
+
+    def load_labels(self, corpus_path: str | None = None) -> Labels:
+        """Load labels from a JSON file. Falls back to empty labels."""
+        from pathlib import Path
+
+        if corpus_path is None:
+            return Labels()
+
+        labels_file = Path(corpus_path) / "labels.json"
+        if not labels_file.exists():
+            return Labels()
+
+        data = json.loads(labels_file.read_text(encoding="utf-8"))
+        return Labels.model_validate(data)
+
+    def list_runs(self, limit: int = 50) -> list[dict[str, Any]]:
+        """List recent eval runs from agent_memory."""
+        with self._pool.connection() as conn:
+            rows = conn.execute(
+                """
+                SELECT session_id AS run_id, content, created_at
+                FROM agent_memory
+                WHERE metadata ? 'dream_eval'
+                ORDER BY created_at DESC
+                LIMIT %s
+                """,
+                (limit,),
+            ).fetchall()
+
+        runs: list[dict[str, Any]] = []
+        for row in rows:
+            content = row["content"]
+            if isinstance(content, str):
+                content = json.loads(content)
+            runs.append({
+                "run_id": row["run_id"],
+                "faithfulness": content.get("faithfulness_score"),
+                "secret_leak": content.get("secret_leak_test"),
+                "date": row["created_at"].isoformat() if row["created_at"] else None,
+            })
+        return runs

dream_eval/cli.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""CLI for dream-eval — run evaluations, check gates, score faithfulness."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime
+
+from dream_eval.types import EvalMode, EvalResult
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="dream-eval — agent memory faithfulness evaluation"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_run = sub.add_parser("run", help="Run a full eval pipeline")
+    p_run.add_argument("--corpus", default=None, help="Path to golden corpus directory")
+    p_run.add_argument("--mode", choices=["golden", "live"], default="golden")
+    p_run.add_argument("--output-dir", default=None, help="Output directory for results")
+
+    p_gates = sub.add_parser("gates", help="Run only deterministic gates")
+    p_gates.add_argument("--text", default=None, help="Text to check for secret leaks")
+    p_gates.add_argument("--file", default=None, help="File to check hash determinism")
+    p_gates.add_argument("--hash", default=None, help="Expected hash (sha256:hex)")
+
+    p_score = sub.add_parser("score", help="Score an eval report against labels")
+    p_score.add_argument("--report", required=True, help="Path to eval-report.json")
+    p_score.add_argument("--labels", default=None, help="Path to labels.json")
+
+    p_list = sub.add_parser("list", help="List recent eval runs")
+    p_list.add_argument("--limit", type=int, default=10)
+    p_list.add_argument("--backend", choices=["json", "postgres"], default="json")
+    p_list.add_argument("--dsn", default=None, help="Postgres DSN (for postgres backend)")
+
+    p_show = sub.add_parser("show", help="Show a specific eval run result")
+    p_show.add_argument("run_id")
+    p_show.add_argument("--backend", choices=["json", "postgres"], default="json")
+
+    args = parser.parse_args()
+
+    if args.command == "gates":
+        _run_gates(args)
+    elif args.command == "run":
+        _run_eval(args)
+    elif args.command == "score":
+        _run_score(args)
+    elif args.command == "list":
+        _run_list(args)
+    elif args.command == "show":
+        _run_show(args)
+
+
+def _run_gates(args: argparse.Namespace) -> None:
+    from dream_eval.gates import check_hash_determinism, check_secret_leak
+
+    results = []
+
+    if args.text:
+        result = check_secret_leak(args.text)
+        results.append(result)
+
+    if args.file:
+        result = check_hash_determinism(
+            open(args.file, encoding="utf-8").read(), args.hash
+        )
+        results.append(result)
+
+    if not results:
+        print("No checks specified. Use --text or --file.", file=sys.stderr)
+        sys.exit(1)
+
+    for r in results:
+        print(json.dumps(r.model_dump(mode="json"), default=str))
+
+    if any(r.status.value == "fail" for r in results):
+        sys.exit(1)
+
+
+def _run_eval(args: argparse.Namespace) -> None:
+    from pathlib import Path
+
+    result = EvalResult(
+        run_id=datetime.now().strftime("%Y-%m-%dT%H-%M-%SZ"),
+        date=datetime.now(),
+        mode=EvalMode(args.mode),
+    )
+
+    output_dir = args.output_dir or "eval/results"
+    out_path = Path(output_dir) / result.run_id
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    metrics_path = out_path / "metrics.json"
+    metrics_path.write_text(
+        json.dumps(result.to_metrics_dict(), indent=2, default=str),
+        encoding="utf-8",
+    )
+
+    print(json.dumps({
+        "run_id": result.run_id,
+        "output": str(out_path),
+        "hard_fail": result.hard_fail,
+    }, default=str))
+
+
+def _run_score(args: argparse.Namespace) -> None:
+    from pathlib import Path
+
+    from dream_eval.scoring import compute_faithfulness
+    from dream_eval.types import EvalReport, Labels
+
+    report_data = json.loads(Path(args.report).read_text(encoding="utf-8"))
+    report = EvalReport.model_validate(report_data)
+
+    labels_path = args.labels or str(
+        Path(args.report).parent.parent / "golden-corpus" / "labels.json"
+    )
+    labels_data = json.loads(Path(labels_path).read_text(encoding="utf-8"))
+    labels = Labels.model_validate(labels_data)
+
+    faithfulness = compute_faithfulness(report.items, labels.items)
+    print(json.dumps(faithfulness.model_dump(mode="json"), default=str))
+
+
+def _run_list(args: argparse.Namespace) -> None:
+    from dream_eval.backends import JsonFileBackend, PostgresBackend
+
+    if args.backend == "postgres":
+        backend = PostgresBackend(dsn=args.dsn)
+    else:
+        backend = JsonFileBackend()
+
+    runs = backend.list_runs(limit=args.limit)
+    for run in runs:
+        print(json.dumps(run, default=str))
+
+
+def _run_show(args: argparse.Namespace) -> None:
+    from dream_eval.backends import JsonFileBackend, PostgresBackend
+
+    if args.backend == "postgres":
+        backend = PostgresBackend()
+    else:
+        backend = JsonFileBackend()
+
+    report = backend.load_eval_report(args.run_id)
+    if report:
+        print(json.dumps(report.model_dump(mode="json"), default=str))
+    else:
+        print(f"Run {args.run_id} not found", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

dream_eval/gates.py

@@ -0,0 +1,94 @@
+"""Deterministic gates — hard stops that must pass before scoring."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+
+from dream_eval.types import GateResult, GateStatus
+
+
+def check_secret_leak(
+    output_text: str,
+    forbidden_patterns: list[str] | None = None,
+) -> GateResult:
+    """Check evaluator output for leaked secrets.
+
+    Hard stop: if any forbidden pattern matches, the eval must fail.
+    """
+    if not forbidden_patterns:
+        return GateResult(
+            name="secret_leak",
+            status=GateStatus.SKIP,
+            message="No forbidden patterns configured",
+        )
+
+    matches: list[str] = []
+    for pattern in forbidden_patterns:
+        try:
+            if re.search(pattern, output_text, re.IGNORECASE):
+                matches.append(pattern)
+        except re.error:
+            if pattern.lower() in output_text.lower():
+                matches.append(pattern)
+
+    if matches:
+        return GateResult(
+            name="secret_leak",
+            status=GateStatus.FAIL,
+            message=f"Secret leak detected: {len(matches)} forbidden pattern(s) matched",
+            details={"matched_patterns": matches},
+        )
+
+    return GateResult(
+        name="secret_leak",
+        status=GateStatus.PASS,
+        message="No forbidden patterns found in output",
+    )
+
+
+def _canonical_hash(content: str) -> str:
+    """Canonical SHA-256: strip BOM, normalize CRLF→LF, hex digest with prefix."""
+    text = content
+    if text.startswith("\ufeff"):
+        text = text[1:]
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
+    return f"sha256:{digest}"
+
+
+def check_hash_determinism(
+    content: str,
+    expected_hash: str | None = None,
+) -> GateResult:
+    """Verify hash is deterministic after BOM/CRLF normalization.
+
+    The canonical hash algorithm:
+    1. Strip UTF-8 BOM if present
+    2. Normalize line endings: CRLF → LF
+    3. Compute SHA-256 hex digest
+    """
+    actual_hash = _canonical_hash(content)
+
+    if expected_hash is None:
+        return GateResult(
+            name="hash_determinism",
+            status=GateStatus.PASS,
+            message=f"Hash computed: {actual_hash}",
+            details={"hash": actual_hash},
+        )
+
+    if actual_hash == expected_hash:
+        return GateResult(
+            name="hash_determinism",
+            status=GateStatus.PASS,
+            message="Hash matches expected value",
+            details={"hash": actual_hash},
+        )
+
+    return GateResult(
+        name="hash_determinism",
+        status=GateStatus.FAIL,
+        message=f"Hash mismatch: expected {expected_hash}, got {actual_hash}",
+        details={"expected": expected_hash, "actual": actual_hash},
+    )

dream_eval/mcp/__init__.py

@@ -0,0 +1,5 @@
+"""MCP server for dream-eval — exposes eval tools to any MCP-compatible agent."""
+
+from dream_eval.mcp.server import create_server
+
+__all__ = ["create_server"]