memor-cli 0.1.0__py3-none-any.whl

memor/distill/distiller.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+import hashlib, json, re
+from memor.types import Artifact, Scope
+from memor.llm.base import DISTILL_PROMPT
+from memor.distill.extractive import extract_key_chunks
+from memor.tokencount import count_tokens
+
+DEDUP_SIM_THRESHOLD = 0.92
+SUPERSEDE_SIM_THRESHOLD = 0.80
+
+_REPLACEMENT_RE = re.compile(
+    r"(instead of|no longer|switched from|ripped out|replaced .+ with|"
+    r"deprecated|migrated from|removed .+ in favor|moved away from|"
+    r"changed .+ to|swapped .+ for|dropped .+ for)", re.I)
+
+def _extract_json(raw: str) -> dict:
+    m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", raw, re.DOTALL)
+    text = m.group(1).strip() if m else raw.strip()
+    return json.loads(text)
+
+
+def _store_memory(store, embedder, text: str, mem_type: str, session_id: str,
+                  project: str, created: float, source_chunks: list[Artifact]) -> str | None:
+    """Store a single memory with dedup/supersession check and provenance edges."""
+    mid = f"mem:{session_id}:{hashlib.sha1(text.encode()).hexdigest()[:8]}"
+    vec = embedder.embed([text])[0]
+    existing = store.search(vec, Scope(project=project, kinds=["memory"]), k=1)
+    if existing:
+        old_art, sim = existing[0]
+        if sim >= DEDUP_SIM_THRESHOLD:
+            return None
+        if (sim >= SUPERSEDE_SIM_THRESHOLD
+                and created > old_art.created_at
+                and _REPLACEMENT_RE.search(text)):
+            art = Artifact(
+                id=mid, kind="memory", project=project, source="distill",
+                text=text, token_count=max(1, count_tokens(text)), created_at=created,
+                meta={"mem_type": mem_type, "session_id": session_id},
+            )
+            store.add_artifacts([art], [vec])
+            store.deactivate(old_art.id, superseded_by=mid)
+            for c in source_chunks:
+                store.add_edge(mid, c.id, "derived_from")
+            return mid
+    art = Artifact(
+        id=mid, kind="memory", project=project, source="distill",
+        text=text, token_count=max(1, count_tokens(text)), created_at=created,
+        meta={"mem_type": mem_type, "session_id": session_id},
+    )
+    store.add_artifacts([art], [vec])
+    for c in source_chunks:
+        store.add_edge(mid, c.id, "derived_from")
+    return mid
+
+
+class Distiller:
+    """Two-step distiller: extractive pre-filter (free) → LLM abstractive (paid).
+    The extractive step reduces LLM input by ~90%."""
+
+    def __init__(self, store, embedder, llm):
+        self.store, self.embedder, self.llm = store, embedder, llm
+
+    def distill_session(
+        self, session_id: str, chunks: list[Artifact], project: str
+    ) -> list[str]:
+        # Step 1: extractive pre-filter (local, free)
+        key_chunks = extract_key_chunks(chunks, self.embedder)
+        # Step 2: LLM abstractive distillation on the filtered set
+        session_text = "\n".join(c.text for c in key_chunks)
+        raw = self.llm.complete(DISTILL_PROMPT.format(session_text=session_text))
+        data = _extract_json(raw)
+        created = max((c.created_at for c in chunks), default=0.0)
+        new_ids: list[str] = []
+        for m in data.get("memories", []):
+            text = m["text"].strip()
+            mid = _store_memory(self.store, self.embedder, text, m["type"],
+                                session_id, project, created, key_chunks)
+            if mid is None:
+                continue
+            sup = m.get("supersedes_text")
+            if sup:
+                prior = self.store.search(
+                    self.embedder.embed([sup])[0],
+                    Scope(project=project, kinds=["memory"]), k=1,
+                )
+                if prior and prior[0][1] >= 0.8 and prior[0][0].id != mid:
+                    self.store.deactivate(prior[0][0].id, superseded_by=mid)
+            new_ids.append(mid)
+        return new_ids
+
+
+class ExtractiveDistiller:
+    """LLM-free distiller. Stores the key extracted chunks as memories directly.
+    Used as automatic fallback when no API key is available."""
+
+    def __init__(self, store, embedder):
+        self.store, self.embedder = store, embedder
+
+    def distill_session(
+        self, session_id: str, chunks: list[Artifact], project: str
+    ) -> list[str]:
+        from memor.distill.extractive import classify_chunk
+        key_chunks = extract_key_chunks(chunks, self.embedder)
+        created = max((c.created_at for c in chunks), default=0.0)
+        new_ids: list[str] = []
+        for c in key_chunks:
+            mem_type = classify_chunk(c.text)
+            mid = _store_memory(self.store, self.embedder, c.text, mem_type,
+                                session_id, project, created, [c])
+            if mid is not None:
+                new_ids.append(mid)
+        return new_ids

memor/distill/extractive.py

@@ -0,0 +1,161 @@
+"""Extractive distillation — LLM-free, local-only.
+
+Picks the highest-signal chunks from a session using:
+1. TF-IDF scoring (rare/specific terms score higher)
+2. Embedding clustering (one representative per topic cluster)
+3. Heuristic rules (drop filler, prefer long structured responses + user questions)
+"""
+from __future__ import annotations
+import hashlib, math, re
+from collections import Counter
+from memor.types import Artifact
+
+_DECISION_RE = re.compile(
+    r"(we decided|the approach is|instead of|switched to|chose .+ over|"
+    r"trade-?off|architecture:|design decision)", re.I)
+_BUGFIX_RE = re.compile(
+    r"(the fix is|root cause|the bug was|the issue was|caused by|"
+    r"the problem is|this fails because|the error occurs)", re.I)
+_LESSON_RE = re.compile(
+    r"(always use|never use|never do|important:|note:|pattern:|"
+    r"best practice|lesson learned|rule of thumb|should always|should never)", re.I)
+_SNIPPET_RE = re.compile(r"```")
+
+
+def classify_chunk(text: str) -> str:
+    """Classify a chunk into a memory type based on content patterns."""
+    if _BUGFIX_RE.search(text):
+        return "bugfix"
+    if _DECISION_RE.search(text):
+        return "decision"
+    if _LESSON_RE.search(text):
+        return "lesson"
+    if _SNIPPET_RE.search(text) and len(text) > 200:
+        return "snippet"
+    return "extract"
+
+MIN_CHUNK_TOKENS = 15
+MAX_EXTRACTS = 12
+
+_FILLER = re.compile(
+    r"^(Let me |Now let me |Now I|Good[,.]|Great[,.]|Perfect[!,.]|Done[!,.]"
+    r"|Sure[,.]|OK[,.]|Alright|Looking at |Checking |I'll )", re.I)
+
+_SIGNAL_PATTERNS = re.compile(
+    r"(we decided|the fix is|the solution|use .+ instead of|should always|"
+    r"never use|important:|note:|bug:|pattern:|architecture:)", re.I)
+
+
+def _tfidf_scores(chunks: list[Artifact]) -> list[float]:
+    """Score each chunk by TF-IDF — chunks with rare, specific terms rank higher."""
+    tokenize = lambda t: re.findall(r"[a-zA-Z_][a-zA-Z0-9_]{2,}", t.lower())
+    doc_freq: Counter[str] = Counter()
+    chunk_tokens: list[list[str]] = []
+    for c in chunks:
+        toks = tokenize(c.text)
+        chunk_tokens.append(toks)
+        doc_freq.update(set(toks))
+    n = len(chunks) or 1
+    scores = []
+    for toks in chunk_tokens:
+        if not toks:
+            scores.append(0.0)
+            continue
+        tf = Counter(toks)
+        score = sum(
+            (tf[t] / len(toks)) * math.log((n + 1) / (doc_freq[t] + 1))
+            for t in tf
+        )
+        scores.append(score)
+    return scores
+
+
+def _heuristic_score(chunk: Artifact) -> float:
+    """Bonus/penalty based on content heuristics."""
+    score = 0.0
+    if chunk.token_count < MIN_CHUNK_TOKENS:
+        return -1.0
+    if _FILLER.match(chunk.text):
+        score -= 0.5
+    if _SIGNAL_PATTERNS.search(chunk.text):
+        score += 1.0
+    role = chunk.meta.get("role", "")
+    if role == "user" and chunk.token_count > 20:
+        score += 0.3
+    if role == "assistant" and chunk.token_count > 100:
+        score += 0.5
+    if "```" in chunk.text:
+        score += 0.2
+    return score
+
+
+def _cluster_select(chunks: list[Artifact], embedder, max_clusters: int) -> list[int]:
+    """Cluster chunks by embedding similarity, return index of chunk nearest each centroid."""
+    if len(chunks) <= max_clusters:
+        return list(range(len(chunks)))
+    vecs = embedder.embed([c.text for c in chunks])
+    n = len(vecs)
+    dim = len(vecs[0])
+    # Simple k-means (few iterations, good enough for <300 chunks)
+    k = min(max_clusters, n)
+    centroids = [list(vecs[i]) for i in range(0, n, max(1, n // k))][:k]
+    assignments = [0] * n
+    for _ in range(8):
+        for i in range(n):
+            best_d, best_j = float("inf"), 0
+            for j, c in enumerate(centroids):
+                d = sum((a - b) ** 2 for a, b in zip(vecs[i], c))
+                if d < best_d:
+                    best_d, best_j = d, j
+            assignments[i] = best_j
+        new_centroids = [[0.0] * dim for _ in range(k)]
+        counts = [0] * k
+        for i in range(n):
+            j = assignments[i]
+            counts[j] += 1
+            for d_idx in range(dim):
+                new_centroids[j][d_idx] += vecs[i][d_idx]
+        for j in range(k):
+            if counts[j] > 0:
+                for d_idx in range(dim):
+                    new_centroids[j][d_idx] /= counts[j]
+        centroids = new_centroids
+    # Pick chunk nearest each centroid
+    selected = []
+    for j in range(k):
+        members = [i for i in range(n) if assignments[i] == j]
+        if not members:
+            continue
+        best_i = min(members, key=lambda i: sum(
+            (vecs[i][d] - centroids[j][d]) ** 2 for d in range(dim)))
+        selected.append(best_i)
+    return selected
+
+
+def extract_key_chunks(
+    chunks: list[Artifact], embedder, *, max_extracts: int = MAX_EXTRACTS
+) -> list[Artifact]:
+    """Select the highest-signal chunks from a session. Pure local, no LLM."""
+    if not chunks:
+        return []
+    # Score each chunk: TF-IDF + heuristic
+    tfidf = _tfidf_scores(chunks)
+    max_tf = max(tfidf) or 1.0
+    combined = []
+    for i, c in enumerate(chunks):
+        h = _heuristic_score(c)
+        if h <= -1.0:
+            combined.append(-999.0)
+            continue
+        combined.append((tfidf[i] / max_tf) + h)
+    # Pre-filter: drop anything scored below 0
+    viable_idx = [i for i, s in enumerate(combined) if s > 0]
+    if not viable_idx:
+        viable_idx = list(range(len(chunks)))
+    viable = [chunks[i] for i in viable_idx]
+    # Cluster the viable chunks and pick representatives
+    cluster_idx = _cluster_select(viable, embedder, max_extracts)
+    selected = [viable[i] for i in cluster_idx]
+    # Sort by original order (temporal coherence)
+    selected.sort(key=lambda c: c.meta.get("ord", 0))
+    return selected[:max_extracts]

memor/embed/api.py

@@ -0,0 +1,15 @@
+import httpx
+
+
+class APIEmbedder:
+    """OpenAI-compatible embedding API."""
+    def __init__(self, base_url: str = "https://api.openai.com/v1", api_key: str = "",
+                 model: str = "text-embedding-3-small", dim: int = 1536):
+        self.base_url, self.api_key, self.model, self.dim = base_url, api_key, model, dim
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        r = httpx.post(f"{self.base_url}/embeddings",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={"model": self.model, "input": texts}, timeout=60)
+        r.raise_for_status()
+        return [d["embedding"] for d in sorted(r.json()["data"], key=lambda d: d["index"])]

memor/embed/fake.py

@@ -0,0 +1,16 @@
+import hashlib, math
+
+class FakeEmbedder:
+    """Deterministic hash-based embedder for tests. No model download."""
+    def __init__(self, dim: int = 16):
+        self.dim = dim
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        out = []
+        for t in texts:
+            vec = [0.0] * self.dim
+            for tok in t.lower().split():
+                h = int(hashlib.sha256(tok.encode()).hexdigest(), 16)
+                vec[h % self.dim] += 1.0
+            norm = math.sqrt(sum(x*x for x in vec)) or 1.0
+            out.append([x / norm for x in vec])
+        return out

memor/embed/local.py

@@ -0,0 +1,16 @@
+from model2vec import StaticModel
+
+
+class LocalEmbedder:
+    """Local embedding via model2vec. Default potion-base-8M (dim 256).
+
+    Static token embeddings — no transformer inference needed.
+    ~62MB model download on first use, 500x faster than ONNX transformers.
+    """
+    def __init__(self, model_name: str = "minishlab/potion-base-8M"):
+        self._model = StaticModel.from_pretrained(model_name)
+        self.dim = self._model.dim
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        vecs = self._model.encode(texts)
+        return vecs.tolist()

memor/eval/baselines/__init__.py

@@ -0,0 +1,5 @@
+from memor.eval.baselines.base import ExternalBaseline
+from memor.eval.baselines.claude_mem import ClaudeMemBaseline
+from memor.eval.baselines.graphiti import GraphitiBaseline
+
+__all__ = ["ExternalBaseline", "ClaudeMemBaseline", "GraphitiBaseline"]

memor/eval/baselines/base.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+from memor.types import Artifact
+
+
+class ExternalBaseline:
+    name: str = ""
+
+    def available(self) -> bool:
+        raise NotImplementedError
+
+    def index(self, artifacts: list[Artifact]) -> None:
+        raise NotImplementedError
+
+    def retrieve(self, query: str, project: str, k: int) -> list[str]:
+        raise NotImplementedError

memor/eval/baselines/claude_mem.py

@@ -0,0 +1,19 @@
+from memor.eval.baselines.base import ExternalBaseline
+from memor.types import Artifact
+import os
+
+
+class ClaudeMemBaseline(ExternalBaseline):
+    """Adapter for the claude-mem plugin store. Behind a flag; skipped when its
+    store path is absent so the suite still runs."""
+
+    name = "claude-mem"
+
+    def available(self) -> bool:
+        return os.path.exists(os.path.expanduser("~/.claude-mem"))
+
+    def index(self, artifacts: list[Artifact]) -> None:
+        raise NotImplementedError("point at the claude-mem store or re-ingest via its API")
+
+    def retrieve(self, query: str, project: str, k: int) -> list[str]:
+        raise NotImplementedError("query claude-mem; map results back to artifact ids")

memor/eval/baselines/graphiti.py

@@ -0,0 +1,25 @@
+from memor.eval.baselines.base import ExternalBaseline
+from memor.types import Artifact
+
+
+class GraphitiBaseline(ExternalBaseline):
+    """Adapter for Zep Graphiti. Requires `pip install graphiti-core` + a Neo4j/FalkorDB
+    instance. Behind a flag; skipped when unavailable so the suite still runs."""
+
+    name = "graphiti"
+
+    def __init__(self):
+        self._client = None
+
+    def available(self) -> bool:
+        try:
+            import graphiti_core  # noqa
+            return True
+        except Exception:
+            return False
+
+    def index(self, artifacts: list[Artifact]) -> None:
+        raise NotImplementedError("wire to graphiti_core.add_episode per artifact")
+
+    def retrieve(self, query: str, project: str, k: int) -> list[str]:
+        raise NotImplementedError("wire to graphiti_core search; map results back to artifact ids")

memor/eval/dataset.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+from dataclasses import dataclass
+
+@dataclass
+class EvalCase:
+    query: str
+    scope_project: str
+    relevant_ids: set[str]
+    baseline_full_tokens: int   # tokens if you dumped full history instead
+
+@dataclass
+class CaseResult:
+    recall: float
+    ndcg: float
+    tokens_sent: int
+    latency_ms: float
+
+import re
+from memor.types import Artifact
+
+_WORD = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]{3,}")
+
+def build_counterfactual_cases(artifacts: list["Artifact"], *, project: str,
+                               min_prior_sessions: int = 1) -> list[EvalCase]:
+    """For each session N (by start time), use its opening turn as the query and
+    label prior-session chunks that share salient tokens as the context it needed."""
+    by_session: dict[str, list[Artifact]] = {}
+    for a in artifacts:
+        by_session.setdefault(a.meta["session_id"], []).append(a)
+    for v in by_session.values():
+        v.sort(key=lambda a: a.meta.get("ord", 0))
+    sessions = sorted(by_session.items(), key=lambda kv: kv[1][0].created_at)
+
+    cases: list[EvalCase] = []
+    for idx, (sid, chunks) in enumerate(sessions):
+        if idx < min_prior_sessions:
+            continue
+        opening = chunks[0].text
+        need_tokens = {w.lower() for w in _WORD.findall(opening)}
+        prior = [a for j, (_, cs) in enumerate(sessions) if j < idx for a in cs]
+        relevant = {a.id for a in prior
+                    if need_tokens & {w.lower() for w in _WORD.findall(a.text)}}
+        if not relevant:
+            continue
+        full_tokens = sum(a.token_count for a in prior)
+        cases.append(EvalCase(query=opening, scope_project=project,
+                              relevant_ids=relevant, baseline_full_tokens=full_tokens))
+    return cases

memor/eval/embed_benchmark.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+import time
+from dataclasses import dataclass
+from memor.store.sqlite_store import SqliteStore
+from memor.eval.runner import run_suite
+from memor.eval.dataset import EvalCase
+from memor.types import Artifact
+
+CANDIDATE_MODELS = [
+    {"name": "potion-base-8M", "model_name": "minishlab/potion-base-8M"},
+    {"name": "potion-base-32M", "model_name": "minishlab/potion-base-32M"},
+    {"name": "potion-code-16M", "model_name": "minishlab/potion-code-16M"},
+]
+
+
+@dataclass
+class EmbedBenchmarkResult:
+    model_name: str
+    dim: int
+    recall_at_k: float
+    ndcg_at_k: float
+    tokens_sent: float
+    embed_latency_ms: float
+    retrieval_latency_ms: float
+
+
+def run_embed_benchmark(
+    artifacts: list[Artifact],
+    cases: list[EvalCase],
+    *,
+    model_specs: list[dict] | None = None,
+    embedder_factory=None,
+    db_dir: str = "/tmp",
+    k: int = 8,
+) -> list[EmbedBenchmarkResult]:
+    specs = model_specs or CANDIDATE_MODELS
+    results = []
+
+    for spec in specs:
+        name = spec["name"]
+        if embedder_factory:
+            embedder = embedder_factory(spec)
+        else:
+            from memor.embed.local import LocalEmbedder
+            embedder = LocalEmbedder(model_name=spec["model_name"])
+
+        db_path = f"{db_dir}/bench_{name.replace('/', '_')}.db"
+        store = SqliteStore(db_path, dim=embedder.dim)
+
+        t0 = time.perf_counter()
+        vecs = embedder.embed([a.text for a in artifacts])
+        embed_ms = (time.perf_counter() - t0) * 1000
+
+        store.add_artifacts(artifacts, vecs)
+        summary = run_suite(cases, store=store, embedder=embedder, k=k)
+        mem = summary.get("memory", {})
+
+        results.append(EmbedBenchmarkResult(
+            model_name=name,
+            dim=embedder.dim,
+            recall_at_k=mem.get("recall@k", 0.0),
+            ndcg_at_k=mem.get("ndcg@k", 0.0),
+            tokens_sent=mem.get("tokens_sent", 0),
+            embed_latency_ms=embed_ms,
+            retrieval_latency_ms=mem.get("latency_ms_p50", 0.0),
+        ))
+    return results

memor/eval/judge.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+import json, re, time
+from dataclasses import dataclass
+from memor.types import Artifact, Scope
+from memor.retrieve.retriever import Retriever
+
+JUDGE_PROMPT = """You are evaluating whether recalled memory context would help a coding agent.
+
+The agent is about to work on this task:
+---
+{query}
+---
+
+Here is what actually happened next in the session (the agent doesn't see this — you use it to judge):
+---
+{holdout}
+---
+
+Here is the context that was recalled from prior sessions:
+---
+{recalled_context}
+---
+
+Score how relevant and useful the recalled context is for the task, given what actually happened.
+- 1.0 = recalled context directly addresses or anticipates what happened
+- 0.7 = recalled context is clearly relevant and would save the agent time
+- 0.4 = recalled context is somewhat related but not directly useful
+- 0.1 = recalled context is mostly irrelevant
+- 0.0 = no recalled context, or completely irrelevant
+
+Return STRICT JSON: {{"relevance_score": <float 0-1>, "reasoning": "<one sentence>"}}"""
+
+
+@dataclass
+class JudgeCase:
+    query: str
+    holdout_texts: list[str]
+    scope_project: str
+    session_id: str
+    baseline_full_tokens: int
+
+
+@dataclass
+class JudgeVerdict:
+    relevance_score: float
+    reasoning: str
+    tokens_recalled: int
+    latency_ms: float
+
+
+def build_judge_cases(
+    artifacts: list[Artifact], *, project: str,
+    holdout_turns: int = 2, min_session_turns: int = 4,
+    min_prior_sessions: int = 1,
+) -> list[JudgeCase]:
+    by_session: dict[str, list[Artifact]] = {}
+    for a in artifacts:
+        by_session.setdefault(a.meta.get("session_id", "?"), []).append(a)
+    for v in by_session.values():
+        v.sort(key=lambda a: a.meta.get("ord", 0))
+    sessions = sorted(by_session.items(), key=lambda kv: kv[1][0].created_at)
+
+    cases: list[JudgeCase] = []
+    for idx, (sid, chunks) in enumerate(sessions):
+        if idx < min_prior_sessions:
+            continue
+        if len(chunks) < min_session_turns:
+            continue
+        query = chunks[0].text
+        holdout = [c.text for c in chunks[-holdout_turns:]]
+        prior_chunks = [a for j, (_, cs) in enumerate(sessions) if j < idx for a in cs]
+        full_tokens = sum(a.token_count for a in prior_chunks)
+        cases.append(JudgeCase(
+            query=query, holdout_texts=holdout, scope_project=project,
+            session_id=sid, baseline_full_tokens=full_tokens,
+        ))
+    return cases
+
+
+def _extract_json(raw: str) -> dict:
+    m = re.search(r"\{.*\}", raw, re.DOTALL)
+    if m:
+        return json.loads(m.group())
+    return json.loads(raw)
+
+
+def run_judge_case(
+    case: JudgeCase, *, store, embedder, llm, k: int = 8
+) -> JudgeVerdict:
+    scope = Scope(project=case.scope_project)
+    r = Retriever(store, embedder, k=k)
+    t0 = time.perf_counter()
+    trace = r.query(case.query, scope)
+    latency = (time.perf_counter() - t0) * 1000
+
+    if not trace.hits:
+        return JudgeVerdict(relevance_score=0.0, reasoning="No context recalled",
+                            tokens_recalled=0, latency_ms=latency)
+
+    recalled = "\n\n".join(
+        f"[{h.artifact.kind}] {h.artifact.text}" for h in trace.hits
+    )
+    tokens_recalled = sum(h.artifact.token_count for h in trace.hits)
+    holdout = "\n".join(case.holdout_texts)
+
+    prompt = JUDGE_PROMPT.format(
+        query=case.query, holdout=holdout, recalled_context=recalled,
+    )
+    raw = llm.complete(prompt)
+    data = _extract_json(raw)
+    return JudgeVerdict(
+        relevance_score=float(data.get("relevance_score", 0.0)),
+        reasoning=data.get("reasoning", ""),
+        tokens_recalled=tokens_recalled,
+        latency_ms=latency,
+    )
+
+
+def run_judge_suite(
+    cases: list[JudgeCase], *, store, embedder, llm, k: int = 8
+) -> dict:
+    verdicts = []
+    for c in cases:
+        v = run_judge_case(c, store=store, embedder=embedder, llm=llm, k=k)
+        verdicts.append(v)
+    n = len(verdicts) or 1
+    return {
+        "mean_relevance": sum(v.relevance_score for v in verdicts) / n,
+        "mean_tokens_recalled": sum(v.tokens_recalled for v in verdicts) / n,
+        "mean_latency_ms": sum(v.latency_ms for v in verdicts) / n,
+        "n_cases": len(verdicts),
+        "verdicts": [
+            {"score": v.relevance_score, "reasoning": v.reasoning,
+             "tokens": v.tokens_recalled, "latency_ms": v.latency_ms}
+            for v in verdicts
+        ],
+    }

memor/eval/metrics.py

@@ -0,0 +1,13 @@
+import math
+
+def recall_at_k(retrieved: list[str], relevant: set[str], k: int) -> float:
+    if not relevant:
+        return 0.0
+    topk = retrieved[:k]
+    return len(set(topk) & relevant) / len(relevant)
+
+def ndcg_at_k(retrieved: list[str], relevant: set[str], k: int) -> float:
+    dcg = sum((1.0 / math.log2(i + 2)) for i, x in enumerate(retrieved[:k]) if x in relevant)
+    ideal_n = min(len(relevant), k)
+    idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_n)) or 1.0
+    return dcg / idcg