memor-cli 0.4.1__tar.gz → 0.5.0__tar.gz

{memor_cli-0.4.1/memor_cli.egg-info → memor_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memor-cli
-Version: 0.4.1
+Version: 0.5.0
 Summary: Measured memory for coding agents. Fire and forget — no API keys needed.
 Author-email: Nimit Bhandari <nimitbhandari17@gmail.com>
 License-Expression: MIT
@@ -46,7 +46,7 @@ Dynamic: license-file
 ```
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-215%20passing-brightgreen.svg)]()
+[![Tests](https://img.shields.io/badge/tests-227%20passing-brightgreen.svg)]()
 [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)]()
 [![PyPI](https://img.shields.io/pypi/v/memor-cli.svg)](https://pypi.org/project/memor-cli/)
 
@@ -200,6 +200,7 @@ memor ingest-project <dir>           Bulk ingest a project directory
 memor ingest-doc <file>              Ingest a markdown document
 memor distill --project <name>       Run distillation manually
 memor eval <cases.json>              Run eval suite
+memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
 memor bench-embed --project <name>   Compare embedding models
 ```
 

{memor_cli-0.4.1 → memor_cli-0.5.0}/README.md

@@ -9,7 +9,7 @@
 ```
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-215%20passing-brightgreen.svg)]()
+[![Tests](https://img.shields.io/badge/tests-227%20passing-brightgreen.svg)]()
 [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)]()
 [![PyPI](https://img.shields.io/pypi/v/memor-cli.svg)](https://pypi.org/project/memor-cli/)
 
@@ -163,6 +163,7 @@ memor ingest-project <dir>           Bulk ingest a project directory
 memor ingest-doc <file>              Ingest a markdown document
 memor distill --project <name>       Run distillation manually
 memor eval <cases.json>              Run eval suite
+memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
 memor bench-embed --project <name>   Compare embedding models
 ```
 

{memor_cli-0.4.1 → memor_cli-0.5.0}/memor/cli.py

@@ -78,9 +78,10 @@ MAINTENANCE
   memor version                    Print the installed version
 
 EVALUATION
-  memor eval <cases.json>          Run eval suite
-  memor eval-judge --project <name>  LLM-as-judge evaluation
-  memor bench-embed --project <name> Compare embedding models
+  memor eval <cases.json>              Run eval suite
+  memor eval-judge --project <name>    LLM-as-judge evaluation
+  memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
+  memor bench-embed --project <name>   Compare embedding models
 
 CONFIGURATION
   Everything works locally with zero API keys.
@@ -191,6 +192,38 @@ def eval_judge_cmd(project: str = typer.Option(...), db: str = "memor.db",
     typer.echo(f"Judge eval complete: mean relevance = {summary['mean_relevance']:.3f}")
 
 
+@app.command("eval-counterfactual")
+def eval_counterfactual_cmd(project: str = typer.Option(...), db: str = "memor.db",
+                            k: int = 8, fake: bool = False,
+                            llm_provider: str = "anthropic",
+                            llm_model: str = "claude-sonnet-4-6",
+                            holdout: int = 2):
+    """Counterfactual eval: win/tie/loss vs no-memory baseline. Requires an LLM API key."""
+    from memor.eval.counterfactual import build_cases_from_store, run_suite
+    e = _embedder(fake); s = SqliteStore(_db_path(db), dim=e.dim)
+    cases = build_cases_from_store(s, project=project, holdout_turns=holdout)
+    if not cases:
+        typer.echo("No cases — need at least 2 sessions with >= 4 turns each.")
+        raise typer.Exit(1)
+    typer.echo(f"Built {len(cases)} counterfactual cases. Running evaluation...")
+    if llm_provider == "anthropic":
+        from memor.llm.anthropic import AnthropicLLM
+        llm = AnthropicLLM(model=llm_model)
+    else:
+        from memor.llm.openai_compat import OpenAICompatLLM
+        import os
+        llm = OpenAICompatLLM(base_url=os.environ.get("OPENAI_BASE_URL", "http://localhost:11434/v1"),
+                              api_key=os.environ.get("OPENAI_API_KEY", ""), model=llm_model)
+    summary = run_suite(cases, store=s, embedder=e, llm=llm, k=k)
+    typer.echo(json.dumps({k: v for k, v in summary.items() if k != "cases"}, indent=2))
+    typer.echo("")
+    typer.echo(f"  Win:  {summary['win_count']}/{summary['n_cases']} ({summary['win_pct']}%)")
+    typer.echo(f"  Tie:  {summary['tie_count']}/{summary['n_cases']} ({summary['tie_pct']}%)")
+    typer.echo(f"  Loss: {summary['loss_count']}/{summary['n_cases']} ({summary['loss_pct']}%)")
+    typer.echo(f"  Do-no-harm: {summary['do_no_harm_pct']}%")
+    s.save_eval_run({"type": "counterfactual", "k": k, "project": project, "holdout": holdout}, summary)
+
+
 @app.command("bench-embed")
 def bench_embed(project: str = typer.Option(...), db: str = "memor.db",
                 k: int = 8, fake: bool = False):

{memor_cli-0.4.1 → memor_cli-0.5.0}/memor/dashboard/server.py

@@ -139,6 +139,19 @@ def create_app(db_path: str | None = None) -> FastAPI:
         store = _store()
         return store.get_token_roi(project=project)
 
+    @app.get("/api/roi-trend")
+    def roi_trend(project: str | None = Query(None)):
+        store = _store()
+        return store.get_roi_trend(project=project)
+
+    @app.get("/api/eval/latest")
+    def eval_latest(eval_type: str = Query("counterfactual")):
+        store = _store()
+        result = store.get_latest_eval(eval_type)
+        if not result:
+            return {"status": "no_runs"}
+        return result
+
     @app.get("/api/health")
     def health():
         store = _store()

{memor_cli-0.4.1 → memor_cli-0.5.0}/memor/dashboard/static/index.html

@@ -404,6 +404,10 @@
           </div>
         </div>
       </div>
+      <div style="margin-top:16px;border-top:1px solid var(--border-light);padding-top:12px;position:relative;">
+        <div id="roi-sparkline" style="width:100%;height:80px;"></div>
+        <div id="roi-spark-tooltip" style="display:none;position:absolute;top:0;background:var(--surface3);border:1px solid var(--border);border-radius:6px;padding:6px 10px;font-size:11px;color:var(--text);white-space:nowrap;pointer-events:none;box-shadow:0 4px 12px rgba(0,0,0,0.4);z-index:10;"></div>
+      </div>
     </div>
   </section>
 
@@ -835,6 +839,89 @@
     }
   }
 
+  function renderROITrend(data) {
+    var container = document.getElementById('roi-sparkline');
+    var tooltip = document.getElementById('roi-spark-tooltip');
+    if (!container || !data || !data.length) return;
+
+    var W = container.clientWidth || 400;
+    var H = 80;
+    var padL = 32, padR = 8, padT = 8, padB = 20;
+    var plotW = W - padL - padR;
+    var plotH = H - padT - padB;
+
+    var vals = data.map(function(d) { return d.reduction_pct; });
+    var minV = Math.min.apply(null, vals);
+    var maxV = Math.max.apply(null, vals);
+    var range = (maxV - minV) || 1;
+    var yFloor = Math.min(0, minV);
+    var yCeil = Math.max(maxV, 10);
+    range = (yCeil - yFloor) || 1;
+
+    function x(i) { return padL + (data.length > 1 ? (i / (data.length - 1)) * plotW : plotW / 2); }
+    function y(v) { return padT + plotH - ((v - yFloor) / range) * plotH; }
+
+    var lineColor = vals[vals.length - 1] >= 0 ? '#3dd68c' : '#e89320';
+    var gradId = 'roiGrad';
+
+    var pts = data.map(function(d, i) { return x(i) + ',' + y(d.reduction_pct); });
+    var linePath = 'M' + pts.join(' L');
+    var areaPath = linePath + ' L' + x(data.length - 1) + ',' + y(yFloor) + ' L' + x(0) + ',' + y(yFloor) + ' Z';
+
+    var gridLines = '';
+    var steps = [yFloor, Math.round((yFloor + yCeil) / 2), yCeil];
+    steps.forEach(function(v) {
+      var yy = y(v);
+      gridLines += '<line x1="' + padL + '" y1="' + yy + '" x2="' + (W - padR) + '" y2="' + yy + '" stroke="var(--border-light)" stroke-width="0.5" stroke-dasharray="3,3"/>';
+      gridLines += '<text x="' + (padL - 4) + '" y="' + (yy + 3) + '" fill="var(--text-muted)" font-size="8" text-anchor="end">' + v + '%</text>';
+    });
+
+    var xLabels = '';
+    data.forEach(function(d, i) {
+      xLabels += '<text x="' + x(i) + '" y="' + (H - 2) + '" fill="var(--text-muted)" font-size="8" text-anchor="middle">' + d.day.slice(5) + '</text>';
+    });
+
+    var dots = data.map(function(d, i) {
+      return '<circle cx="' + x(i) + '" cy="' + y(d.reduction_pct) + '" r="4" fill="' + lineColor + '" stroke="var(--surface)" stroke-width="2" style="opacity:0;" class="roi-dot" data-idx="' + i + '"/>';
+    }).join('');
+
+    var hitAreas = data.map(function(d, i) {
+      return '<rect x="' + (x(i) - plotW / data.length / 2) + '" y="' + padT + '" width="' + (plotW / data.length) + '" height="' + plotH + '" fill="transparent" class="roi-hit" data-idx="' + i + '"/>';
+    }).join('');
+
+    container.innerHTML = '<svg width="' + W + '" height="' + H + '" style="display:block;">' +
+      '<defs><linearGradient id="' + gradId + '" x1="0" y1="0" x2="0" y2="1">' +
+      '<stop offset="0%" stop-color="' + lineColor + '" stop-opacity="0.3"/>' +
+      '<stop offset="100%" stop-color="' + lineColor + '" stop-opacity="0.0"/>' +
+      '</linearGradient></defs>' +
+      gridLines + xLabels +
+      '<path d="' + areaPath + '" fill="url(#' + gradId + ')"/>' +
+      '<path d="' + linePath + '" fill="none" stroke="' + lineColor + '" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>' +
+      dots + hitAreas +
+      '</svg>';
+
+    container.querySelectorAll('.roi-hit').forEach(function(rect) {
+      rect.addEventListener('mouseenter', function() {
+        var i = parseInt(rect.getAttribute('data-idx'));
+        var d = data[i];
+        var dot = container.querySelectorAll('.roi-dot')[i];
+        if (dot) dot.style.opacity = '1';
+        tooltip.innerHTML = '<strong>' + d.day + '</strong><br>' +
+          d.reduction_pct + '% fewer tools<br>' +
+          d.turns_with + ' recall / ' + d.turns_without + ' no-recall turns';
+        tooltip.style.display = 'block';
+        var tx = Math.min(x(i), W - 160);
+        tooltip.style.left = tx + 'px';
+      });
+      rect.addEventListener('mouseleave', function() {
+        var i = parseInt(rect.getAttribute('data-idx'));
+        var dot = container.querySelectorAll('.roi-dot')[i];
+        if (dot) dot.style.opacity = '0';
+        tooltip.style.display = 'none';
+      });
+    });
+  }
+
   /* ── Data loaders ──────────────────────────────────────── */
   async function loadSummary()    { try { renderSummary(await api('/api/summary')); } catch(e) { console.warn('summary',e); } }
   async function loadProjects()   { try { renderProjects(await api('/api/projects')); } catch(e) { console.warn('projects',e); } }
@@ -843,6 +930,7 @@
   async function loadHealth()     { try { renderHealth(await api('/api/health')); } catch(e) { console.warn('health',e); } }
   async function loadTrend()      { try { renderTrend(await api('/api/recall-trend?days=30')); } catch(e) { console.warn('trend',e); } }
   async function loadROI()        { try { renderROI(await api('/api/roi')); } catch(e) { console.warn('roi',e); } }
+  async function loadROITrend()   { try { renderROITrend(await api('/api/roi-trend')); } catch(e) { console.warn('roi-trend',e); } }
   async function loadRecalls() {
     try {
       var url = '/api/recalls?limit=50' + (projectFilter ? '&project=' + encodeURIComponent(projectFilter) : '');
@@ -851,7 +939,7 @@
   }
 
   async function refresh() {
-    await Promise.allSettled([loadSummary(), loadProjects(), loadEfficiency(), loadSessionEfficiency(), loadHealth(), loadTrend(), loadRecalls(), loadROI()]);
+    await Promise.allSettled([loadSummary(), loadProjects(), loadEfficiency(), loadSessionEfficiency(), loadHealth(), loadTrend(), loadRecalls(), loadROI(), loadROITrend()]);
     document.getElementById('last-updated').textContent = new Date().toLocaleTimeString();
 
     renderMiniBars('mb-chunks', null);

memor_cli-0.5.0/memor/eval/counterfactual.py

@@ -0,0 +1,176 @@
+"""Counterfactual eval harness — win/tie/loss vs no-memory baseline.
+
+For each held-out session, uses the opening turn as query, retrieves context
+from prior sessions, and asks an LLM judge whether the recalled context would
+have helped (win), been irrelevant (tie), or misled the agent (loss).
+
+The key metric is do-no-harm rate = 1 - loss_rate."""
+from __future__ import annotations
+
+import enum
+import json
+import re
+import time
+from dataclasses import dataclass
+
+from memor.types import Artifact, Scope
+from memor.retrieve.retriever import Retriever
+
+
+class Outcome(enum.Enum):
+    WIN = "win"
+    TIE = "tie"
+    LOSS = "loss"
+
+
+@dataclass
+class CounterfactualCase:
+    query: str
+    holdout_texts: list[str]
+    scope_project: str
+    session_id: str
+
+
+@dataclass
+class CounterfactualVerdict:
+    outcome: Outcome
+    reasoning: str
+    confidence: float
+
+
+COUNTERFACTUAL_PROMPT = """You are evaluating whether recalled memory context helped or hurt a coding agent.
+
+The agent received this task:
+---
+{query}
+---
+
+Here is what actually happened next in the session (ground truth — the agent didn't see this):
+---
+{holdout}
+---
+
+Here is the context that was recalled from prior sessions and injected into the agent's prompt:
+---
+{recalled_context}
+---
+
+Compare the recalled context against what actually happened. Score as:
+
+- **win**: The recalled context directly anticipates what happened, would save the agent exploration/time, or would prevent repeating a past mistake. The agent would have been measurably better with this context.
+- **tie**: The recalled context is irrelevant to what happened. Neither helpful nor harmful — the agent would have arrived at the same outcome either way.
+- **loss**: The recalled context is stale, contradictory, or misleading. It would have sent the agent down the wrong path or caused confusion.
+
+When uncertain, default to **tie** — only score win when the context clearly helps, and loss when it clearly hurts.
+
+Return STRICT JSON: {{"outcome": "win"|"tie"|"loss", "reasoning": "<one sentence>", "confidence": <float 0-1>}}"""
+
+
+def build_cases_from_store(store, *, project: str,
+                           holdout_turns: int = 2,
+                           min_session_turns: int = 4,
+                           min_prior_sessions: int = 1) -> list[CounterfactualCase]:
+    rows = store.db.execute(
+        "SELECT * FROM artifacts WHERE project=? AND kind='session_chunk' AND active=1",
+        (project,)).fetchall()
+    artifacts = [store._row_to_artifact(r) for r in rows]
+
+    by_session: dict[str, list[Artifact]] = {}
+    for a in artifacts:
+        by_session.setdefault(a.meta.get("session_id", "?"), []).append(a)
+    for v in by_session.values():
+        v.sort(key=lambda a: a.meta.get("ord", 0))
+    sessions = sorted(by_session.items(), key=lambda kv: kv[1][0].created_at)
+
+    cases: list[CounterfactualCase] = []
+    for idx, (sid, chunks) in enumerate(sessions):
+        if idx < min_prior_sessions:
+            continue
+        if len(chunks) < min_session_turns:
+            continue
+        query = chunks[0].text
+        holdout = [c.text for c in chunks[-holdout_turns:]]
+        cases.append(CounterfactualCase(
+            query=query, holdout_texts=holdout,
+            scope_project=project, session_id=sid))
+    return cases
+
+
+def _extract_json(raw: str) -> dict:
+    m = re.search(r"\{.*\}", raw, re.DOTALL)
+    if m:
+        return json.loads(m.group())
+    return json.loads(raw)
+
+
+def parse_verdict_json(raw: str) -> CounterfactualVerdict:
+    data = _extract_json(raw)
+    outcome_str = data.get("outcome", "tie").lower().strip()
+    outcome = Outcome(outcome_str) if outcome_str in ("win", "tie", "loss") else Outcome.TIE
+    return CounterfactualVerdict(
+        outcome=outcome,
+        reasoning=data.get("reasoning", ""),
+        confidence=float(data.get("confidence", 0.5)),
+    )
+
+
+def run_case(case: CounterfactualCase, *, store, embedder, llm,
+             k: int = 8) -> CounterfactualVerdict:
+    scope = Scope(project=case.scope_project)
+    r = Retriever(store, embedder, k=k)
+    trace = r.query(case.query, scope)
+
+    if not trace.hits:
+        return CounterfactualVerdict(
+            outcome=Outcome.TIE,
+            reasoning="No context recalled — nothing to evaluate",
+            confidence=1.0)
+
+    recalled = "\n\n".join(
+        f"[{h.artifact.kind}] {h.artifact.text}" for h in trace.hits)
+    holdout = "\n".join(case.holdout_texts)
+
+    prompt = COUNTERFACTUAL_PROMPT.format(
+        query=case.query, holdout=holdout, recalled_context=recalled)
+    raw = llm.complete(prompt)
+    return parse_verdict_json(raw)
+
+
+def run_suite(cases: list[CounterfactualCase], *, store, embedder, llm,
+              k: int = 8) -> dict:
+    verdicts = []
+    for c in cases:
+        v = run_case(c, store=store, embedder=embedder, llm=llm, k=k)
+        verdicts.append((c, v))
+    verdict_list = [v for _, v in verdicts]
+    summary = summarize_verdicts(verdict_list)
+    summary["cases"] = [
+        {"session_id": c.session_id, "query_preview": c.query[:100],
+         "outcome": v.outcome.value, "reasoning": v.reasoning,
+         "confidence": v.confidence}
+        for c, v in verdicts
+    ]
+    return summary
+
+
+def summarize_verdicts(verdicts: list[CounterfactualVerdict]) -> dict:
+    n = len(verdicts)
+    if n == 0:
+        return {"n_cases": 0, "win_count": 0, "tie_count": 0, "loss_count": 0,
+                "win_pct": 0.0, "tie_pct": 0.0, "loss_pct": 0.0,
+                "do_no_harm_pct": 100.0}
+
+    wins = sum(1 for v in verdicts if v.outcome == Outcome.WIN)
+    ties = sum(1 for v in verdicts if v.outcome == Outcome.TIE)
+    losses = sum(1 for v in verdicts if v.outcome == Outcome.LOSS)
+
+    return {
+        "n_cases": n,
+        "win_count": wins,
+        "tie_count": ties,
+        "loss_count": losses,
+        "win_pct": round(wins / n * 100, 1),
+        "tie_pct": round(ties / n * 100, 1),
+        "loss_pct": round(losses / n * 100, 1),
+        "do_no_harm_pct": round((wins + ties) / n * 100, 1),
+    }

{memor_cli-0.4.1 → memor_cli-0.5.0}/memor/store/sqlite_store.py

@@ -234,6 +234,19 @@ class SqliteStore:
         self.db.commit()
         return cur.lastrowid
 
+    def get_latest_eval(self, eval_type: str = "counterfactual") -> dict | None:
+        row = self.db.execute(
+            "SELECT * FROM eval_runs WHERE json_extract(config, '$.type') = ? "
+            "ORDER BY created_at DESC LIMIT 1", (eval_type,)).fetchone()
+        if not row:
+            return None
+        return {
+            "id": row["id"],
+            "created_at": row["created_at"],
+            "config": json.loads(row["config"]),
+            "metrics": json.loads(row["metrics"]),
+        }
+
     def log_recall(self, project: str, query_preview: str, hits_count: int,
                    top_score: float, tokens_injected: int, latency_ms: float,
                    status: str, session_id: str = "") -> None:
@@ -571,6 +584,43 @@ class SqliteStore:
             "tool_call_reduction_pct": reduction,
         }
 
+    def get_roi_trend(self, project: str | None = None,
+                      days: int = 30) -> list[dict]:
+        where = "WHERE user_timestamp > 0"
+        params: list = []
+        if project:
+            where += " AND project = ?"
+            params.append(project)
+
+        rows = self.db.execute(f"""
+            SELECT date(user_timestamp, 'unixepoch', 'localtime') AS day,
+                   AVG(CASE WHEN had_recall = 1 THEN tool_call_count END) AS avg_with,
+                   AVG(CASE WHEN had_recall = 0 THEN tool_call_count END) AS avg_without,
+                   COUNT(CASE WHEN had_recall = 1 THEN 1 END) AS turns_with,
+                   COUNT(CASE WHEN had_recall = 0 THEN 1 END) AS turns_without,
+                   COUNT(*) AS turns_total
+            FROM turn_metrics {where}
+            GROUP BY day
+            HAVING turns_with > 0 AND turns_without > 0
+            ORDER BY day
+        """, params).fetchall()
+
+        result = []
+        for r in rows:
+            avg_w = round(r["avg_with"], 2)
+            avg_wo = round(r["avg_without"], 2)
+            reduction = round((1 - avg_w / avg_wo) * 100, 1) if avg_wo > 0 else 0
+            result.append({
+                "day": r["day"],
+                "avg_with": avg_w,
+                "avg_without": avg_wo,
+                "reduction_pct": reduction,
+                "turns_with": r["turns_with"],
+                "turns_without": r["turns_without"],
+                "turns_total": r["turns_total"],
+            })
+        return result
+
     def get_onboarding_status(self) -> str:
         chunks = self.db.execute(
             "SELECT COUNT(*) as c FROM artifacts WHERE kind='session_chunk' AND active=1"

{memor_cli-0.4.1 → memor_cli-0.5.0/memor_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memor-cli
-Version: 0.4.1
+Version: 0.5.0
 Summary: Measured memory for coding agents. Fire and forget — no API keys needed.
 Author-email: Nimit Bhandari <nimitbhandari17@gmail.com>
 License-Expression: MIT
@@ -46,7 +46,7 @@ Dynamic: license-file
 ```
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-215%20passing-brightgreen.svg)]()
+[![Tests](https://img.shields.io/badge/tests-227%20passing-brightgreen.svg)]()
 [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)]()
 [![PyPI](https://img.shields.io/pypi/v/memor-cli.svg)](https://pypi.org/project/memor-cli/)
 
@@ -200,6 +200,7 @@ memor ingest-project <dir>           Bulk ingest a project directory
 memor ingest-doc <file>              Ingest a markdown document
 memor distill --project <name>       Run distillation manually
 memor eval <cases.json>              Run eval suite
+memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
 memor bench-embed --project <name>   Compare embedding models
 ```
 

{memor_cli-0.4.1 → memor_cli-0.5.0}/memor_cli.egg-info/SOURCES.txt

@@ -28,6 +28,7 @@ memor/embed/api.py
 memor/embed/fake.py
 memor/embed/local.py
 memor/eval/__init__.py
+memor/eval/counterfactual.py
 memor/eval/dataset.py
 memor/eval/embed_benchmark.py
 memor/eval/judge.py
@@ -55,6 +56,7 @@ memor_cli.egg-info/entry_points.txt
 memor_cli.egg-info/requires.txt
 memor_cli.egg-info/top_level.txt
 tests/test_cli_smoke.py
+tests/test_counterfactual.py
 tests/test_daemon.py
 tests/test_dashboard.py
 tests/test_dataset_builder.py
@@ -83,6 +85,7 @@ tests/test_query_complexity.py
 tests/test_recall_core.py
 tests/test_redact.py
 tests/test_retriever.py
+tests/test_roi_trend.py
 tests/test_semantic_feedback.py
 tests/test_service.py
 tests/test_session_context.py

{memor_cli-0.4.1 → memor_cli-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "memor-cli"
-version = "0.4.1"
+version = "0.5.0"
 description = "Measured memory for coding agents. Fire and forget — no API keys needed."
 readme = "README.md"
 license = "MIT"

memor_cli-0.5.0/tests/test_counterfactual.py

@@ -0,0 +1,104 @@
+"""Tests for the counterfactual eval harness — win/tie/loss vs no-memory baseline."""
+import json
+import time
+
+from memor.types import Artifact
+from memor.store.sqlite_store import SqliteStore
+from memor.embed.fake import FakeEmbedder
+from memor.eval.counterfactual import (
+    build_cases_from_store, CounterfactualCase, CounterfactualVerdict,
+    Outcome, summarize_verdicts, parse_verdict_json,
+)
+
+
+def _seed_store(tmp_path, *, n_sessions=3, turns_per_session=5):
+    """Create a store with sessions that have recall logs and artifacts."""
+    db_path = str(tmp_path / "m.db")
+    e = FakeEmbedder(dim=16)
+    store = SqliteStore(db_path, dim=16)
+
+    now = time.time()
+    for s_idx in range(n_sessions):
+        sid = f"session-{s_idx}"
+        session_start = now - (n_sessions - s_idx) * 86400
+        artifacts = []
+        vectors = []
+        for t_idx in range(turns_per_session):
+            aid = f"a-{s_idx}-{t_idx}"
+            text = f"Turn {t_idx} of session {s_idx}: discussion about auth middleware and token refresh"
+            vec = e.embed([text])[0]
+            artifacts.append(Artifact(
+                id=aid, kind="session_chunk", project="testproj",
+                source="test", text=text, token_count=20,
+                created_at=session_start + t_idx,
+                meta={"session_id": sid, "ord": t_idx}))
+            vectors.append(vec)
+        store.add_artifacts(artifacts, vectors)
+
+    store.db.commit()
+    return store
+
+
+def test_build_cases_from_store(tmp_path):
+    store = _seed_store(tmp_path, n_sessions=3, turns_per_session=5)
+    cases = build_cases_from_store(store, project="testproj",
+                                   holdout_turns=2, min_session_turns=4)
+    assert len(cases) >= 1
+    for c in cases:
+        assert c.query
+        assert c.holdout_texts
+        assert c.scope_project == "testproj"
+
+
+def test_build_cases_needs_prior_sessions(tmp_path):
+    store = _seed_store(tmp_path, n_sessions=1, turns_per_session=5)
+    cases = build_cases_from_store(store, project="testproj")
+    assert len(cases) == 0
+
+
+def test_parse_verdict_json_win():
+    raw = '{"outcome": "win", "reasoning": "Context saved rework", "confidence": 0.9}'
+    v = parse_verdict_json(raw)
+    assert v.outcome == Outcome.WIN
+    assert v.confidence == 0.9
+
+
+def test_parse_verdict_json_loss():
+    raw = '{"outcome": "loss", "reasoning": "Stale info misled agent", "confidence": 0.8}'
+    v = parse_verdict_json(raw)
+    assert v.outcome == Outcome.LOSS
+
+
+def test_parse_verdict_json_tie():
+    raw = '{"outcome": "tie", "reasoning": "Irrelevant context", "confidence": 0.7}'
+    v = parse_verdict_json(raw)
+    assert v.outcome == Outcome.TIE
+
+
+def test_parse_verdict_embedded_json():
+    raw = 'Here is my assessment:\n```json\n{"outcome": "win", "reasoning": "Helped", "confidence": 0.85}\n```'
+    v = parse_verdict_json(raw)
+    assert v.outcome == Outcome.WIN
+
+
+def test_summarize_verdicts():
+    verdicts = [
+        CounterfactualVerdict(outcome=Outcome.WIN, reasoning="a", confidence=0.9),
+        CounterfactualVerdict(outcome=Outcome.WIN, reasoning="b", confidence=0.8),
+        CounterfactualVerdict(outcome=Outcome.TIE, reasoning="c", confidence=0.7),
+        CounterfactualVerdict(outcome=Outcome.LOSS, reasoning="d", confidence=0.6),
+    ]
+    s = summarize_verdicts(verdicts)
+    assert s["win_count"] == 2
+    assert s["tie_count"] == 1
+    assert s["loss_count"] == 1
+    assert s["n_cases"] == 4
+    assert s["win_pct"] == 50.0
+    assert s["loss_pct"] == 25.0
+    assert s["do_no_harm_pct"] == 75.0
+
+
+def test_summarize_empty():
+    s = summarize_verdicts([])
+    assert s["n_cases"] == 0
+    assert s["do_no_harm_pct"] == 100.0

memor_cli-0.5.0/tests/test_roi_trend.py

@@ -0,0 +1,87 @@
+"""Tests for daily ROI trend — tool call reduction over time."""
+from memor.store.sqlite_store import SqliteStore
+from memor.turn_metrics import TurnMetric
+
+
+def test_roi_trend_groups_by_day(tmp_path):
+    db_path = str(tmp_path / "m.db")
+    store = SqliteStore(db_path, dim=16)
+
+    day1_base = 1749427200.0  # 2025-06-09 00:00:00 UTC
+    day2_base = day1_base + 86400
+
+    store.save_turn_metrics("s1", "p", [
+        TurnMetric(turn_idx=0, user_timestamp=day1_base + 100, tool_call_count=1, had_recall=True),
+        TurnMetric(turn_idx=1, user_timestamp=day1_base + 200, tool_call_count=4, had_recall=False),
+    ])
+    store.save_turn_metrics("s2", "p", [
+        TurnMetric(turn_idx=0, user_timestamp=day2_base + 100, tool_call_count=2, had_recall=True),
+        TurnMetric(turn_idx=1, user_timestamp=day2_base + 200, tool_call_count=6, had_recall=False),
+    ])
+
+    trend = store.get_roi_trend()
+    assert len(trend) == 2
+    for day in trend:
+        assert "day" in day
+        assert "avg_with" in day
+        assert "avg_without" in day
+        assert "reduction_pct" in day
+        assert "turns_total" in day
+
+
+def test_roi_trend_calculates_reduction(tmp_path):
+    db_path = str(tmp_path / "m.db")
+    store = SqliteStore(db_path, dim=16)
+
+    base = 1749427200.0
+    store.save_turn_metrics("s1", "p", [
+        TurnMetric(turn_idx=0, user_timestamp=base + 100, tool_call_count=2, had_recall=True),
+        TurnMetric(turn_idx=1, user_timestamp=base + 200, tool_call_count=2, had_recall=True),
+        TurnMetric(turn_idx=2, user_timestamp=base + 300, tool_call_count=4, had_recall=False),
+        TurnMetric(turn_idx=3, user_timestamp=base + 400, tool_call_count=4, had_recall=False),
+    ])
+
+    trend = store.get_roi_trend()
+    assert len(trend) == 1
+    day = trend[0]
+    assert day["avg_with"] == 2.0
+    assert day["avg_without"] == 4.0
+    assert day["reduction_pct"] == 50.0
+
+
+def test_roi_trend_filters_by_project(tmp_path):
+    db_path = str(tmp_path / "m.db")
+    store = SqliteStore(db_path, dim=16)
+
+    base = 1749427200.0
+    store.save_turn_metrics("s1", "proj-a", [
+        TurnMetric(turn_idx=0, user_timestamp=base + 100, tool_call_count=1, had_recall=True),
+        TurnMetric(turn_idx=1, user_timestamp=base + 200, tool_call_count=5, had_recall=False),
+    ])
+    store.save_turn_metrics("s2", "proj-b", [
+        TurnMetric(turn_idx=0, user_timestamp=base + 100, tool_call_count=3, had_recall=True),
+        TurnMetric(turn_idx=1, user_timestamp=base + 200, tool_call_count=3, had_recall=False),
+    ])
+
+    trend_a = store.get_roi_trend(project="proj-a")
+    assert len(trend_a) == 1
+    assert trend_a[0]["avg_with"] == 1.0
+
+    trend_b = store.get_roi_trend(project="proj-b")
+    assert len(trend_b) == 1
+    assert trend_b[0]["avg_with"] == 3.0
+
+
+def test_roi_trend_skips_days_without_both_types(tmp_path):
+    """Days that have only recall or only non-recall turns can't compute reduction."""
+    db_path = str(tmp_path / "m.db")
+    store = SqliteStore(db_path, dim=16)
+
+    base = 1749427200.0
+    store.save_turn_metrics("s1", "p", [
+        TurnMetric(turn_idx=0, user_timestamp=base + 100, tool_call_count=2, had_recall=True),
+        TurnMetric(turn_idx=1, user_timestamp=base + 200, tool_call_count=3, had_recall=True),
+    ])
+
+    trend = store.get_roi_trend()
+    assert len(trend) == 0