dream-eval 0.2.0__tar.gz

dream_eval-0.2.0/.claude-plugin/SKILL.md

@@ -0,0 +1,88 @@
+---
+name: dream-eval
+description: Agent-agnostic faithfulness evaluation for agent memory — scoring, deterministic gates, and metrics. Works with any memory backend (Postgres, LanceDB, knowledge graphs).
+---
+
+# dream-eval
+
+Faithfulness evaluation framework for agent memory quality scoring.
+
+## Usage
+
+### Python API
+
+```python
+from dream_eval.scoring import compute_faithfulness
+from dream_eval.types import ProposedItem, LabeledItem
+
+proposed = [ProposedItem(id="pref-1", category="pref", content={"key": "ci-merge-gate"})]
+labels = [LabeledItem(id="pref-1", category="pref", content={"key": "ci-merge-gate"})]
+
+report = compute_faithfulness(proposed, labels)
+print(report.faithfulness_score)  # 1.0
+```
+
+### CLI
+
+```bash
+# Score an eval report
+dream-eval score --report eval/results/run-1/eval-report.json
+
+# Check for secret leaks
+dream-eval gates --text "output to check"
+
+# Check hash determinism
+dream-eval gates --file input.txt
+
+# List recent runs
+dream-eval list --limit 10
+```
+
+### Deterministic Gates
+
+These are **hard stops** — if either fails, the eval must stop:
+
+| Gate | What it checks |
+|------|----------------|
+| `secret_leak` | Forbidden patterns (API keys, DSNs) in evaluator output |
+| `hash_determinism` | BOM/CRLF normalization produces consistent SHA-256 hashes |
+
+## Installation
+
+```bash
+pip install dream-eval
+
+# With Postgres backend
+pip install dream-eval[postgres]
+
+# With MCP server
+pip install dream-eval[mcp]
+```
+
+## Architecture
+
+dream-eval is **backend-agnostic**. The `MemoryBackend` abstract class defines the interface:
+
+```python
+class MemoryBackend(ABC):
+    def load_eval_report(self, run_id: str) -> EvalReport | None: ...
+    def load_labels(self, corpus_path: str | None = None) -> Labels: ...
+    def save_eval_result(self, result: EvalResult) -> None: ...
+    def list_runs(self, limit: int = 50) -> list[dict[str, Any]]: ...
+```
+
+Built-in backends:
+- `JsonFileBackend` — reads/writes to `eval/results/<run_id>/`
+- `PostgresBackend` — stores in `agent_memory` table alongside other memory records
+
+## MCP Server
+
+```bash
+# Run as MCP server
+dream-eval-mcp
+
+# Or via uvx
+uvx dream-eval[mcp]
+```
+
+Exposes tools: `dream_score`, `dream_check_secrets`, `dream_check_hash`, `dream_metrics_schema`.

dream_eval-0.2.0/.claude-plugin/plugin.json

@@ -0,0 +1,32 @@
+{
+  "name": "dream-eval",
+  "version": "0.1.0",
+  "description": "Agent-agnostic faithfulness evaluation for agent memory — scoring, deterministic gates, and metrics.",
+  "author": "OnlineChefGroep",
+  "license": "MIT",
+  "skills": [
+    {
+      "name": "dream-eval",
+      "description": "Faithfulness evaluation pipeline for agent memory quality scoring",
+      "trigger": "Apply the dream-eval skill to score agent memory faithfulness",
+      "entry_point": "SKILL.md"
+    }
+  ],
+  "tools": [
+    {
+      "name": "dream_score",
+      "description": "Score evaluator output against golden corpus labels",
+      "command": "uvx dream-eval score"
+    },
+    {
+      "name": "dream_check_secrets",
+      "description": "Check text for leaked secrets",
+      "command": "uvx dream-eval gates --text"
+    },
+    {
+      "name": "dream_check_hash",
+      "description": "Verify content produces a deterministic hash",
+      "command": "uvx dream-eval gates --file"
+    }
+  ]
+}

dream_eval-0.2.0/.codex-plugin/SKILL.md

@@ -0,0 +1,50 @@
+---
+name: dream-eval
+description: Agent-agnostic faithfulness evaluation for agent memory — scoring, deterministic gates, and metrics. Works with any memory backend.
+---
+
+# dream-eval (Codex)
+
+Faithfulness evaluation framework for agent memory quality scoring.
+
+## Trigger
+
+```
+Apply the dream-eval skill to score agent memory faithfulness.
+```
+
+## Usage
+
+```bash
+# Score an eval report
+uvx dream-eval score --report eval/results/run-1/eval-report.json
+
+# Check for secret leaks
+uvx dream-eval gates --text "output to check"
+
+# Check hash determinism
+uvx dream-eval gates --file input.txt
+
+# List recent runs
+uvx dream-eval list --limit 10
+```
+
+## Python API
+
+```python
+from dream_eval.scoring import compute_faithfulness
+from dream_eval.types import ProposedItem, LabeledItem
+
+proposed = [ProposedItem(id="pref-1", category="pref", content={"key": "ci-merge-gate"})]
+labels = [LabeledItem(id="pref-1", category="pref", content={"key": "ci-merge-gate"})]
+
+report = compute_faithfulness(proposed, labels)
+print(report.faithfulness_score)  # 1.0
+```
+
+## Deterministic Gates
+
+| Gate | What it checks |
+|------|----------------|
+| `secret_leak` | Forbidden patterns (API keys, DSNs) in evaluator output |
+| `hash_determinism` | BOM/CRLF normalization produces consistent SHA-256 hashes |

dream_eval-0.2.0/.codex-plugin/plugin.json

@@ -0,0 +1,32 @@
+{
+  "name": "dream-eval",
+  "version": "0.1.0",
+  "description": "Agent-agnostic faithfulness evaluation for agent memory — scoring, deterministic gates, and metrics.",
+  "author": "OnlineChefGroep",
+  "license": "MIT",
+  "skills": [
+    {
+      "name": "dream-eval",
+      "description": "Faithfulness evaluation pipeline for agent memory quality scoring",
+      "trigger": "Apply the dream-eval skill to score agent memory faithfulness",
+      "entry_point": "SKILL.md"
+    }
+  ],
+  "tools": [
+    {
+      "name": "dream_score",
+      "description": "Score evaluator output against golden corpus labels",
+      "command": "uvx dream-eval score"
+    },
+    {
+      "name": "dream_check_secrets",
+      "description": "Check text for leaked secrets",
+      "command": "uvx dream-eval gates --text"
+    },
+    {
+      "name": "dream_check_hash",
+      "description": "Verify content produces a deterministic hash",
+      "command": "uvx dream-eval gates --file"
+    }
+  ]
+}

dream_eval-0.2.0/.cursor-plugin/SKILL.md

@@ -0,0 +1,57 @@
+---
+name: dream-eval
+description: Agent-agnostic faithfulness evaluation for agent memory — scoring, deterministic gates, and metrics. Works with any memory backend.
+disable-model-invocation: true
+---
+
+# dream-eval (Cursor)
+
+Faithfulness evaluation framework for agent memory quality scoring.
+
+## Trigger
+
+`/dream-eval` or "Apply the dream-eval skill to score agent memory faithfulness."
+
+## Usage
+
+```bash
+# Score an eval report
+uvx dream-eval score --report eval/results/run-1/eval-report.json
+
+# Check for secret leaks
+uvx dream-eval gates --text "output to check"
+
+# Check hash determinism
+uvx dream-eval gates --file input.txt
+
+# List recent runs
+uvx dream-eval list --limit 10
+```
+
+## Python API
+
+```python
+from dream_eval.scoring import compute_faithfulness
+from dream_eval.types import ProposedItem, LabeledItem
+
+proposed = [ProposedItem(id="pref-1", category="pref", content={"key": "ci-merge-gate"})]
+labels = [LabeledItem(id="pref-1", category="pref", content={"key": "ci-merge-gate"})]
+
+report = compute_faithfulness(proposed, labels)
+print(report.faithfulness_score)  # 1.0
+```
+
+## Deterministic Gates
+
+| Gate | What it checks |
+|------|----------------|
+| `secret_leak` | Forbidden patterns (API keys, DSNs) in evaluator output |
+| `hash_determinism` | BOM/CRLF normalization produces consistent SHA-256 hashes |
+
+## MCP Server
+
+```bash
+dream-eval-mcp
+```
+
+Exposes tools: `dream_score`, `dream_check_secrets`, `dream_check_hash`, `dream_metrics_schema`.

dream_eval-0.2.0/.cursor-plugin/plugin.json

@@ -0,0 +1,32 @@
+{
+  "name": "dream-eval",
+  "version": "0.1.0",
+  "description": "Agent-agnostic faithfulness evaluation for agent memory — scoring, deterministic gates, and metrics.",
+  "author": "OnlineChefGroep",
+  "license": "MIT",
+  "skills": [
+    {
+      "name": "dream-eval",
+      "description": "Faithfulness evaluation pipeline for agent memory quality scoring",
+      "trigger": "Apply the dream-eval skill to score agent memory faithfulness",
+      "entry_point": "SKILL.md"
+    }
+  ],
+  "tools": [
+    {
+      "name": "dream_score",
+      "description": "Score evaluator output against golden corpus labels",
+      "command": "uvx dream-eval score"
+    },
+    {
+      "name": "dream_check_secrets",
+      "description": "Check text for leaked secrets",
+      "command": "uvx dream-eval gates --text"
+    },
+    {
+      "name": "dream_check_hash",
+      "description": "Verify content produces a deterministic hash",
+      "command": "uvx dream-eval gates --file"
+    }
+  ]
+}

dream_eval-0.2.0/.gitignore

@@ -0,0 +1,9 @@
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+uv.lock

dream_eval-0.2.0/CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+## 0.2.0 (2026-06-26)
+
+### Added
+- **NLI claim verification** via Vectara HHEM-2.1-Open (`dream-eval[nli]`)
+  - `verify_claim()` for single claim verification
+  - `verify_content_nli()` for batch content matching
+  - Automatic fallback to fuzzy matching if NLI extra not installed
+- **Fuzzy content matching** with `fuzzy=True` parameter
+  - Uses difflib SequenceMatcher (no LLM dependency)
+  - Configurable threshold (default 0.85)
+- **Async parallel scoring** via `score_transcripts_parallel()`
+  - ThreadPoolExecutor-based parallelism
+  - Preserves input order
+- **Hypothesis property-based tests** for scoring invariants
+  - Deterministic regression tests with `@example` decorator
+  - Bounds checking for all scoring functions
+- **MCP server** for Claude/Copilot/etc integration (`dream-eval-mcp`)
+- **Agent Skills plugins** (`.claude-plugin`, `.codex-plugin`, `.cursor-plugin`)
+- **Materialized views** for dashboard metrics (schema_v3.sql)
+
+### Changed
+- Pool tuning: `min_size=2, max_size=20, max_waiting=100`
+- Faithfulness baseline target: 0.63 → 0.75
+
+## 0.1.0 (2026-06-26)
+
+### Added
+- Initial release
+- Scoring algorithms: precision, recall, faithfulness, recurrence calibration
+- Deterministic gates: secret_leak, hash_determinism
+- Backends: JsonFileBackend, PostgresBackend
+- CLI: `dream-eval run/gates/score/list/show`

dream_eval-0.2.0/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: dream-eval
+Version: 0.2.0
+Summary: Agent-agnostic faithfulness evaluation framework — evaluator → judge → curator pipeline for agent memory quality scoring
+Author: OnlineChefGroep
+License: MIT
+Keywords: agent-eval,faithfulness,llm-eval,memory-evaluation,quality-gate
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: <3.15,>=3.11
+Requires-Dist: pydantic>=2.13
+Provides-Extra: dev
+Requires-Dist: hypothesis>=6.100; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.15; extra == 'dev'
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.0; extra == 'mcp'
+Provides-Extra: nli
+Requires-Dist: torch>=2.0; extra == 'nli'
+Requires-Dist: transformers>=4.35; extra == 'nli'
+Provides-Extra: postgres
+Requires-Dist: psycopg-pool>=3.1; extra == 'postgres'
+Requires-Dist: psycopg[binary]>=3.2; extra == 'postgres'
+Description-Content-Type: text/markdown
+
+# dream-eval
+
+Agent-agnostic faithfulness evaluation framework for agent memory quality scoring.
+
+## What it does
+
+dream-eval implements the **evaluator → judge → curator** pipeline pattern:
+
+- **Evaluator** reads transcripts + soul (interpretive lens), proposes items
+- **Judge** scores against labels WITHOUT reading soul (enforcing objectivity)
+- **Curator** writes results (enforcing separation of concerns)
+
+This pattern is unique in the agent memory space — no competitor (mem0, Cognee, LangMem) offers automated faithfulness evaluation.
+
+## Install
+
+```bash
+pip install dream-eval
+```
+
+## Quick start
+
+```python
+from dream_eval import compute_faithfulness
+from dream_eval.types import ProposedItem, LabeledItem
+
+proposed = [
+    ProposedItem(id="pref-1", category="pref", content={"key": "dark_mode"}),
+    ProposedItem(id="workflow-1", category="workflow", content={"key": "ci_merge"}),
+]
+labels = [
+    LabeledItem(id="pref-1", category="pref"),
+    LabeledItem(id="workflow-1", category="workflow"),
+]
+
+report = compute_faithfulness(proposed, labels)
+print(f"Faithfulness: {report.faithfulness_score}")
+```
+
+## CLI
+
+```bash
+# Score evaluator report against labels
+dream-eval score --report report.json --labels labels.json
+
+# Run deterministic gates
+dream-eval gate --labels labels.json --output evaluator_output.txt
+
+# Export to metrics.json format
+dream-eval export --input eval_result.json --output metrics.json
+```
+
+## Deterministic gates
+
+These fail the eval regardless of LLM scores:
+
+- **secret_leak** — checks for forbidden patterns (API keys, tokens, passwords)
+- **hash_determinism** — verifies BOM/CRLF normalization produces stable hashes
+
+## Memory backend adapter
+
+dream-eval works with any memory backend via `BaseMemoryBackend`:
+
+```python
+from dream_eval.adapter import BaseMemoryBackend
+
+class MyBackend(BaseMemoryBackend):
+    def read_transcripts(self, corpus_path=None):
+        # Read from your storage
+        ...
+
+    def read_labels(self, labels_path=None):
+        # Read ground truth labels
+        ...
+
+    def write_eval_result(self, result):
+        # Write evaluation results
+        ...
+```
+
+Built-in `DictMemoryBackend` for testing.
+
+## Architecture
+
+```
+dream-eval/
+├── src/dream_eval/
+│   ├── __init__.py      # Package exports
+│   ├── types.py         # Pydantic models (EvalResult, FaithfulnessReport, etc.)
+│   ├── scoring.py       # Faithfulness, precision, recall algorithms
+│   ├── gates.py         # Deterministic gates (secret_leak, hash_determinism)
+│   ├── adapter.py       # Abstract BaseMemoryBackend + DictMemoryBackend
+│   └── cli.py           # CLI entry point
+└── tests/               # Test suite
+```
+
+## License
+
+MIT

dream_eval-0.2.0/README.md

@@ -0,0 +1,99 @@
+# dream-eval
+
+Agent-agnostic faithfulness evaluation framework for agent memory quality scoring.
+
+## What it does
+
+dream-eval implements the **evaluator → judge → curator** pipeline pattern:
+
+- **Evaluator** reads transcripts + soul (interpretive lens), proposes items
+- **Judge** scores against labels WITHOUT reading soul (enforcing objectivity)
+- **Curator** writes results (enforcing separation of concerns)
+
+This pattern is unique in the agent memory space — no competitor (mem0, Cognee, LangMem) offers automated faithfulness evaluation.
+
+## Install
+
+```bash
+pip install dream-eval
+```
+
+## Quick start
+
+```python
+from dream_eval import compute_faithfulness
+from dream_eval.types import ProposedItem, LabeledItem
+
+proposed = [
+    ProposedItem(id="pref-1", category="pref", content={"key": "dark_mode"}),
+    ProposedItem(id="workflow-1", category="workflow", content={"key": "ci_merge"}),
+]
+labels = [
+    LabeledItem(id="pref-1", category="pref"),
+    LabeledItem(id="workflow-1", category="workflow"),
+]
+
+report = compute_faithfulness(proposed, labels)
+print(f"Faithfulness: {report.faithfulness_score}")
+```
+
+## CLI
+
+```bash
+# Score evaluator report against labels
+dream-eval score --report report.json --labels labels.json
+
+# Run deterministic gates
+dream-eval gate --labels labels.json --output evaluator_output.txt
+
+# Export to metrics.json format
+dream-eval export --input eval_result.json --output metrics.json
+```
+
+## Deterministic gates
+
+These fail the eval regardless of LLM scores:
+
+- **secret_leak** — checks for forbidden patterns (API keys, tokens, passwords)
+- **hash_determinism** — verifies BOM/CRLF normalization produces stable hashes
+
+## Memory backend adapter
+
+dream-eval works with any memory backend via `BaseMemoryBackend`:
+
+```python
+from dream_eval.adapter import BaseMemoryBackend
+
+class MyBackend(BaseMemoryBackend):
+    def read_transcripts(self, corpus_path=None):
+        # Read from your storage
+        ...
+
+    def read_labels(self, labels_path=None):
+        # Read ground truth labels
+        ...
+
+    def write_eval_result(self, result):
+        # Write evaluation results
+        ...
+```
+
+Built-in `DictMemoryBackend` for testing.
+
+## Architecture
+
+```
+dream-eval/
+├── src/dream_eval/
+│   ├── __init__.py      # Package exports
+│   ├── types.py         # Pydantic models (EvalResult, FaithfulnessReport, etc.)
+│   ├── scoring.py       # Faithfulness, precision, recall algorithms
+│   ├── gates.py         # Deterministic gates (secret_leak, hash_determinism)
+│   ├── adapter.py       # Abstract BaseMemoryBackend + DictMemoryBackend
+│   └── cli.py           # CLI entry point
+└── tests/               # Test suite
+```
+
+## License
+
+MIT

dream_eval-0.2.0/examples/quick_start.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Quick start example for dream-eval."""
+
+from dream_eval.scoring import compute_faithfulness
+from dream_eval.gates import check_secret_leak, check_hash_determinism
+from dream_eval.types import ProposedItem, LabeledItem
+
+
+def main() -> None:
+    # --- 1. Faithfulness scoring ---
+    proposed = [
+        ProposedItem(
+            id="pref-1",
+            category="pref",
+            content={"key": "ci-merge-gate", "value": "require-reviews"},
+            recurrence=3,
+        ),
+        ProposedItem(
+            id="rule-1",
+            category="rule",
+            content={"key": "no-secrets-in-code", "value": "true"},
+        ),
+        ProposedItem(
+            id="unknown-1",
+            category="pref",
+            content={"key": "mystery-pref"},
+        ),
+    ]
+
+    labels = [
+        LabeledItem(
+            id="pref-1",
+            category="pref",
+            content={"key": "ci-merge-gate", "value": "require-reviews"},
+            max_recurrence=5,
+        ),
+        LabeledItem(
+            id="rule-1",
+            category="rule",
+            content={"key": "no-secrets-in-code", "value": "true"},
+        ),
+    ]
+
+    report = compute_faithfulness(proposed, labels)
+
+    print("=== Faithfulness Report ===")
+    print(f"  Score:        {report.faithfulness_score:.3f}")
+    print(f"  Precision:    {report.precision:.3f}")
+    print(f"  Recall:       {report.recall:.3f}")
+    print(f"  Proposed:     {report.items_proposed}")
+    print(f"  Supported:    {report.items_fully_supported}")
+    print(f"  Unsupported:  {report.items_unsupported}")
+    print()
+
+    # --- 2. Secret leak detection ---
+    clean = check_secret_leak(
+        "The config uses environment variables for all secrets.",
+        forbidden_patterns=[r"sk-.*", r"password\s*=\s*\S+"],
+    )
+    leaked = check_secret_leak(
+        "Set password=secret123 in the config file.",
+        forbidden_patterns=[r"sk-.*", r"password\s*=\s*\S+"],
+    )
+
+    print("=== Secret Leak Gates ===")
+    print(f"  Clean text:  {clean.status.value}")
+    print(f"  Leaked text: {leaked.status.value}")
+    print()
+
+    # --- 3. Hash determinism ---
+    content_v1 = "Hello World\n"
+    content_v2 = "Hello World\r\n"  # CRLF variant
+    content_v3 = "\ufeffHello World\n"  # BOM variant
+
+    h1 = check_hash_determinism(content_v1)
+    h2 = check_hash_determinism(content_v2)
+    h3 = check_hash_determinism(content_v3)
+
+    print("=== Hash Determinism ===")
+    print(f"  LF:      {h1.details['hash']}")
+    print(f"  CRLF:    {h2.details['hash']}")
+    print(f"  BOM+LF:  {h3.details['hash']}")
+    print(f"  All match: {h1.details['hash'] == h2.details['hash'] == h3.details['hash']}")
+
+
+if __name__ == "__main__":
+    main()