memor-cli 0.4.0__tar.gz → 0.5.0__tar.gz

{memor_cli-0.4.0/memor_cli.egg-info → memor_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memor-cli
-Version: 0.4.0
+Version: 0.5.0
 Summary: Measured memory for coding agents. Fire and forget — no API keys needed.
 Author-email: Nimit Bhandari <nimitbhandari17@gmail.com>
 License-Expression: MIT
@@ -46,7 +46,7 @@ Dynamic: license-file
 ```
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-215%20passing-brightgreen.svg)]()
+[![Tests](https://img.shields.io/badge/tests-227%20passing-brightgreen.svg)]()
 [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)]()
 [![PyPI](https://img.shields.io/pypi/v/memor-cli.svg)](https://pypi.org/project/memor-cli/)
 
@@ -200,6 +200,7 @@ memor ingest-project <dir>           Bulk ingest a project directory
 memor ingest-doc <file>              Ingest a markdown document
 memor distill --project <name>       Run distillation manually
 memor eval <cases.json>              Run eval suite
+memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
 memor bench-embed --project <name>   Compare embedding models
 ```
 

{memor_cli-0.4.0 → memor_cli-0.5.0}/README.md

@@ -9,7 +9,7 @@
 ```
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
-[![Tests](https://img.shields.io/badge/tests-215%20passing-brightgreen.svg)]()
+[![Tests](https://img.shields.io/badge/tests-227%20passing-brightgreen.svg)]()
 [![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)]()
 [![PyPI](https://img.shields.io/pypi/v/memor-cli.svg)](https://pypi.org/project/memor-cli/)
 
@@ -163,6 +163,7 @@ memor ingest-project <dir>           Bulk ingest a project directory
 memor ingest-doc <file>              Ingest a markdown document
 memor distill --project <name>       Run distillation manually
 memor eval <cases.json>              Run eval suite
+memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
 memor bench-embed --project <name>   Compare embedding models
 ```
 

memor_cli-0.5.0/memor/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.1"

{memor_cli-0.4.0 → memor_cli-0.5.0}/memor/cli.py

@@ -78,9 +78,10 @@ MAINTENANCE
   memor version                    Print the installed version
 
 EVALUATION
-  memor eval <cases.json>          Run eval suite
-  memor eval-judge --project <name>  LLM-as-judge evaluation
-  memor bench-embed --project <name> Compare embedding models
+  memor eval <cases.json>              Run eval suite
+  memor eval-judge --project <name>    LLM-as-judge evaluation
+  memor eval-counterfactual --project  Win/tie/loss vs no-memory baseline
+  memor bench-embed --project <name>   Compare embedding models
 
 CONFIGURATION
   Everything works locally with zero API keys.
@@ -191,6 +192,38 @@ def eval_judge_cmd(project: str = typer.Option(...), db: str = "memor.db",
     typer.echo(f"Judge eval complete: mean relevance = {summary['mean_relevance']:.3f}")
 
 
+@app.command("eval-counterfactual")
+def eval_counterfactual_cmd(project: str = typer.Option(...), db: str = "memor.db",
+                            k: int = 8, fake: bool = False,
+                            llm_provider: str = "anthropic",
+                            llm_model: str = "claude-sonnet-4-6",
+                            holdout: int = 2):
+    """Counterfactual eval: win/tie/loss vs no-memory baseline. Requires an LLM API key."""
+    from memor.eval.counterfactual import build_cases_from_store, run_suite
+    e = _embedder(fake); s = SqliteStore(_db_path(db), dim=e.dim)
+    cases = build_cases_from_store(s, project=project, holdout_turns=holdout)
+    if not cases:
+        typer.echo("No cases — need at least 2 sessions with >= 4 turns each.")
+        raise typer.Exit(1)
+    typer.echo(f"Built {len(cases)} counterfactual cases. Running evaluation...")
+    if llm_provider == "anthropic":
+        from memor.llm.anthropic import AnthropicLLM
+        llm = AnthropicLLM(model=llm_model)
+    else:
+        from memor.llm.openai_compat import OpenAICompatLLM
+        import os
+        llm = OpenAICompatLLM(base_url=os.environ.get("OPENAI_BASE_URL", "http://localhost:11434/v1"),
+                              api_key=os.environ.get("OPENAI_API_KEY", ""), model=llm_model)
+    summary = run_suite(cases, store=s, embedder=e, llm=llm, k=k)
+    typer.echo(json.dumps({k: v for k, v in summary.items() if k != "cases"}, indent=2))
+    typer.echo("")
+    typer.echo(f"  Win:  {summary['win_count']}/{summary['n_cases']} ({summary['win_pct']}%)")
+    typer.echo(f"  Tie:  {summary['tie_count']}/{summary['n_cases']} ({summary['tie_pct']}%)")
+    typer.echo(f"  Loss: {summary['loss_count']}/{summary['n_cases']} ({summary['loss_pct']}%)")
+    typer.echo(f"  Do-no-harm: {summary['do_no_harm_pct']}%")
+    s.save_eval_run({"type": "counterfactual", "k": k, "project": project, "holdout": holdout}, summary)
+
+
 @app.command("bench-embed")
 def bench_embed(project: str = typer.Option(...), db: str = "memor.db",
                 k: int = 8, fake: bool = False):

{memor_cli-0.4.0 → memor_cli-0.5.0}/memor/dashboard/server.py

@@ -139,6 +139,19 @@ def create_app(db_path: str | None = None) -> FastAPI:
         store = _store()
         return store.get_token_roi(project=project)
 
+    @app.get("/api/roi-trend")
+    def roi_trend(project: str | None = Query(None)):
+        store = _store()
+        return store.get_roi_trend(project=project)
+
+    @app.get("/api/eval/latest")
+    def eval_latest(eval_type: str = Query("counterfactual")):
+        store = _store()
+        result = store.get_latest_eval(eval_type)
+        if not result:
+            return {"status": "no_runs"}
+        return result
+
     @app.get("/api/health")
     def health():
         store = _store()

{memor_cli-0.4.0 → memor_cli-0.5.0}/memor/dashboard/static/index.html

@@ -172,6 +172,7 @@
     background: var(--surface); border: 1px solid var(--border);
     border-radius: var(--radius); padding: 22px 24px;
     box-shadow: var(--card-shadow);
+    display: flex; flex-direction: column;
   }
   .chart-header {
     display: flex; align-items: center; justify-content: space-between;
@@ -199,7 +200,7 @@
 
   .bar-chart {
     display: flex; align-items: flex-end; gap: 4px;
-    height: 180px; padding-top: 8px;
+    flex: 1; min-height: 180px; padding-top: 8px;
     border-bottom: 1px solid var(--border-light);
   }
   .bar-group {
@@ -207,15 +208,16 @@
     align-items: center; gap: 0; height: 100%;
     justify-content: flex-end;
   }
-  .bar-stack { display: flex; flex-direction: column-reverse; gap: 1px; width: 100%; max-width: 22px; }
-  .bar-hits { background: var(--accent); border-radius: 2px 2px 0 0; min-height: 0; transition: height 0.3s; }
-  .bar-miss { background: var(--surface3); border-radius: 2px 2px 0 0; min-height: 0; transition: height 0.3s; }
+  .bar-stack { display: flex; flex-direction: column-reverse; gap: 2px; width: 100%; max-width: 28px; flex: 1; align-items: center; justify-content: flex-end; }
+  .bar-cell { width: 16px; height: 10px; border-radius: 2px; transition: opacity 0.3s; }
+  .bar-cell-hit { background: var(--accent); }
+  .bar-cell-miss { background: var(--surface3); }
   .bar-label {
     font-size: 9px; color: var(--text-muted); margin-top: 6px;
     text-align: center; white-space: nowrap;
   }
 
-  .bar-group:hover .bar-hits { background: #f0a030; }
+  .bar-group:hover .bar-cell-hit { background: #f0a030; }
   .bar-group { cursor: default; position: relative; }
   .bar-tooltip {
     display: none; position: absolute; bottom: calc(100% + 8px);
@@ -378,6 +380,37 @@
     </div>
   </section>
 
+  <!-- ── Token ROI hero ─────────────────────────────────── -->
+  <section>
+    <div class="chart-card" id="roi-card" style="display:none;">
+      <div style="display:flex;align-items:center;gap:20px;flex-wrap:wrap;">
+        <div>
+          <div class="chart-title" style="margin-bottom:4px;">Token ROI</div>
+          <div style="font-size:32px;font-weight:700;color:var(--ok);letter-spacing:-1px;" id="roi-value">&ndash;</div>
+          <div style="font-size:12px;color:var(--text-muted);margin-top:2px;" id="roi-desc">fewer tool calls when Memor injects context</div>
+        </div>
+        <div style="display:flex;gap:24px;flex:1;justify-content:flex-end;">
+          <div style="text-align:center;">
+            <div style="font-size:18px;font-weight:600;color:var(--text);" id="roi-tools-with">&ndash;</div>
+            <div style="font-size:10px;color:var(--text-muted);">avg tools/turn<br>with recall</div>
+          </div>
+          <div style="text-align:center;">
+            <div style="font-size:18px;font-weight:600;color:var(--text);" id="roi-tools-without">&ndash;</div>
+            <div style="font-size:10px;color:var(--text-muted);">avg tools/turn<br>without recall</div>
+          </div>
+          <div style="text-align:center;">
+            <div style="font-size:18px;font-weight:600;color:var(--text);" id="roi-turns">&ndash;</div>
+            <div style="font-size:10px;color:var(--text-muted);" id="roi-turns-sub">turns measured</div>
+          </div>
+        </div>
+      </div>
+      <div style="margin-top:16px;border-top:1px solid var(--border-light);padding-top:12px;position:relative;">
+        <div id="roi-sparkline" style="width:100%;height:80px;"></div>
+        <div id="roi-spark-tooltip" style="display:none;position:absolute;top:0;background:var(--surface3);border:1px solid var(--border);border-radius:6px;padding:6px 10px;font-size:11px;color:var(--text);white-space:nowrap;pointer-events:none;box-shadow:0 4px 12px rgba(0,0,0,0.4);z-index:10;"></div>
+      </div>
+    </div>
+  </section>
+
   <!-- ── Recall trend chart ─────────────────────────────── -->
   <section>
     <div class="chart-grid">
@@ -426,32 +459,6 @@
           </div>
         </div>
       </div>
-      <div class="chart-card">
-        <div class="chart-header">
-          <div class="chart-title">Token ROI</div>
-        </div>
-        <div id="roi-banner" style="display:none;background:var(--ok-dim);border:1px solid rgba(61,214,140,0.2);border-radius:var(--radius-sm);padding:12px 14px;margin-bottom:14px;">
-          <div style="font-size:22px;font-weight:700;color:var(--ok);letter-spacing:-0.5px;" id="roi-value">&ndash;</div>
-          <div style="font-size:11px;color:var(--text-muted);margin-top:2px;" id="roi-desc">fewer tool calls when Memor injects context</div>
-        </div>
-        <div class="side-stats" id="roi-side">
-          <div class="side-stat">
-            <div class="side-stat-label">Avg Tools / Turn (with recall)</div>
-            <div class="side-stat-value" id="roi-tools-with">&ndash;</div>
-            <div class="side-stat-sub">when Memor injected context</div>
-          </div>
-          <div class="side-stat">
-            <div class="side-stat-label">Avg Tools / Turn (without)</div>
-            <div class="side-stat-value" id="roi-tools-without">&ndash;</div>
-            <div class="side-stat-sub">when no context was injected</div>
-          </div>
-          <div class="side-stat">
-            <div class="side-stat-label">Turns Measured</div>
-            <div class="side-stat-value" id="roi-turns">&ndash;</div>
-            <div class="side-stat-sub" id="roi-turns-sub">with vs. without recall</div>
-          </div>
-        </div>
-      </div>
     </div>
   </section>
 
@@ -539,7 +546,7 @@
   function badge(status) {
     var map = {
       ok:              ['badge-ok',        'Success'],
-      extractive_only: ['badge-extractive','Extractive'],
+      extractive_only: ['badge-extractive','No Distill'],
       no_hits:         ['badge-no_hits',   'No Hits'],
     };
     var m = map[status] || ['badge-no_hits', status];
@@ -669,21 +676,20 @@
     var recallsByDay = data.map(function(d) { return d.recalls; });
     renderMiniBars('mb-recalls', recallsByDay);
 
+    var maxCells = 30;
     data.forEach(function(d) {
-      var pct = (d.recalls / maxRecalls) * 100;
-      var hitPct = d.hits ? (d.hits / d.recalls) * 100 : 0;
-      var missPct = 100 - hitPct;
-      var hitH = Math.max(0, pct * hitPct / 100);
-      var missH = Math.max(0, pct * missPct / 100);
+      var totalCells = Math.max(1, Math.round((d.recalls / maxRecalls) * maxCells));
+      var hitCells = Math.round((d.hits / d.recalls) * totalCells);
+      var missCells = totalCells - hitCells;
 
       var dayStr = d.day.slice(5);
       var group = el('div', {class: 'bar-group'});
+      var cells = '';
+      for (var i = 0; i < hitCells; i++) cells += '<div class="bar-cell bar-cell-hit"></div>';
+      for (var i = 0; i < missCells; i++) cells += '<div class="bar-cell bar-cell-miss"></div>';
       group.innerHTML =
         '<div class="bar-tooltip">' + esc(d.day) + '<br>' + d.recalls + ' recalls · ' + (d.hits||0) + ' hits</div>' +
-        '<div class="bar-stack">' +
-          '<div class="bar-hits" style="height:' + hitH + '%"></div>' +
-          '<div class="bar-miss" style="height:' + missH + '%"></div>' +
-        '</div>' +
+        '<div class="bar-stack">' + cells + '</div>' +
         '<div class="bar-label">' + dayStr + '</div>';
       container.appendChild(group);
     });
@@ -725,7 +731,7 @@
       document.getElementById('th-col3').textContent = 'Avg Score';
       document.getElementById('th-col4').textContent = 'OK';
       document.getElementById('th-col5').textContent = 'No Hits';
-      document.getElementById('th-col6').textContent = 'Extractive';
+      document.getElementById('th-col6').textContent = 'No Distill';
     }
 
     sorted.forEach(function(row) {
@@ -809,36 +815,113 @@
 
   /* ── ROI renderer ─────────────────────────────────────── */
   function renderROI(data) {
+    var card = document.getElementById('roi-card');
+    var roiValue = document.getElementById('roi-value');
+    var roiDesc = document.getElementById('roi-desc');
+
     document.getElementById('roi-tools-with').textContent = data.avg_tools_with_recall;
     document.getElementById('roi-tools-without').textContent = data.avg_tools_without_recall;
-    document.getElementById('roi-turns').textContent =
-      (data.turns_with_recall + data.turns_without_recall).toLocaleString();
+    var totalTurns = data.turns_with_recall + data.turns_without_recall;
+    document.getElementById('roi-turns').textContent = totalTurns.toLocaleString();
     document.getElementById('roi-turns-sub').textContent =
-      data.turns_with_recall + ' with recall · ' + data.turns_without_recall + ' without';
-
-    var banner = document.getElementById('roi-banner');
-    var roiValue = document.getElementById('roi-value');
-    var roiDesc = document.getElementById('roi-desc');
-    banner.style.display = 'none';
-    banner.style.background = 'var(--ok-dim)';
-    banner.style.borderColor = 'rgba(61,214,140,0.2)';
-    roiValue.style.color = 'var(--ok)';
-    roiValue.textContent = '–';
-    roiDesc.textContent = 'fewer tool calls when Memor injects context';
+      data.turns_with_recall + ' with · ' + data.turns_without_recall + ' without';
 
     if (data.tool_call_reduction_pct > 0 && data.turns_with_recall >= 5 && data.turns_without_recall >= 5) {
       roiValue.textContent = data.tool_call_reduction_pct + '% fewer';
-      banner.style.display = 'block';
+      roiValue.style.color = 'var(--ok)';
+      roiDesc.textContent = 'fewer tool calls when Memor injects context';
+      card.style.display = 'block';
     } else if (data.tool_call_reduction_pct < 0 && data.turns_with_recall >= 5) {
       roiValue.textContent = Math.abs(data.tool_call_reduction_pct) + '% more';
       roiValue.style.color = 'var(--warn)';
       roiDesc.textContent = 'tool calls with recall — investigating...';
-      banner.style.display = 'block';
-      banner.style.background = 'var(--warn-dim)';
-      banner.style.borderColor = 'rgba(232,147,32,0.2)';
+      card.style.display = 'block';
     }
   }
 
+  function renderROITrend(data) {
+    var container = document.getElementById('roi-sparkline');
+    var tooltip = document.getElementById('roi-spark-tooltip');
+    if (!container || !data || !data.length) return;
+
+    var W = container.clientWidth || 400;
+    var H = 80;
+    var padL = 32, padR = 8, padT = 8, padB = 20;
+    var plotW = W - padL - padR;
+    var plotH = H - padT - padB;
+
+    var vals = data.map(function(d) { return d.reduction_pct; });
+    var minV = Math.min.apply(null, vals);
+    var maxV = Math.max.apply(null, vals);
+    var range = (maxV - minV) || 1;
+    var yFloor = Math.min(0, minV);
+    var yCeil = Math.max(maxV, 10);
+    range = (yCeil - yFloor) || 1;
+
+    function x(i) { return padL + (data.length > 1 ? (i / (data.length - 1)) * plotW : plotW / 2); }
+    function y(v) { return padT + plotH - ((v - yFloor) / range) * plotH; }
+
+    var lineColor = vals[vals.length - 1] >= 0 ? '#3dd68c' : '#e89320';
+    var gradId = 'roiGrad';
+
+    var pts = data.map(function(d, i) { return x(i) + ',' + y(d.reduction_pct); });
+    var linePath = 'M' + pts.join(' L');
+    var areaPath = linePath + ' L' + x(data.length - 1) + ',' + y(yFloor) + ' L' + x(0) + ',' + y(yFloor) + ' Z';
+
+    var gridLines = '';
+    var steps = [yFloor, Math.round((yFloor + yCeil) / 2), yCeil];
+    steps.forEach(function(v) {
+      var yy = y(v);
+      gridLines += '<line x1="' + padL + '" y1="' + yy + '" x2="' + (W - padR) + '" y2="' + yy + '" stroke="var(--border-light)" stroke-width="0.5" stroke-dasharray="3,3"/>';
+      gridLines += '<text x="' + (padL - 4) + '" y="' + (yy + 3) + '" fill="var(--text-muted)" font-size="8" text-anchor="end">' + v + '%</text>';
+    });
+
+    var xLabels = '';
+    data.forEach(function(d, i) {
+      xLabels += '<text x="' + x(i) + '" y="' + (H - 2) + '" fill="var(--text-muted)" font-size="8" text-anchor="middle">' + d.day.slice(5) + '</text>';
+    });
+
+    var dots = data.map(function(d, i) {
+      return '<circle cx="' + x(i) + '" cy="' + y(d.reduction_pct) + '" r="4" fill="' + lineColor + '" stroke="var(--surface)" stroke-width="2" style="opacity:0;" class="roi-dot" data-idx="' + i + '"/>';
+    }).join('');
+
+    var hitAreas = data.map(function(d, i) {
+      return '<rect x="' + (x(i) - plotW / data.length / 2) + '" y="' + padT + '" width="' + (plotW / data.length) + '" height="' + plotH + '" fill="transparent" class="roi-hit" data-idx="' + i + '"/>';
+    }).join('');
+
+    container.innerHTML = '<svg width="' + W + '" height="' + H + '" style="display:block;">' +
+      '<defs><linearGradient id="' + gradId + '" x1="0" y1="0" x2="0" y2="1">' +
+      '<stop offset="0%" stop-color="' + lineColor + '" stop-opacity="0.3"/>' +
+      '<stop offset="100%" stop-color="' + lineColor + '" stop-opacity="0.0"/>' +
+      '</linearGradient></defs>' +
+      gridLines + xLabels +
+      '<path d="' + areaPath + '" fill="url(#' + gradId + ')"/>' +
+      '<path d="' + linePath + '" fill="none" stroke="' + lineColor + '" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>' +
+      dots + hitAreas +
+      '</svg>';
+
+    container.querySelectorAll('.roi-hit').forEach(function(rect) {
+      rect.addEventListener('mouseenter', function() {
+        var i = parseInt(rect.getAttribute('data-idx'));
+        var d = data[i];
+        var dot = container.querySelectorAll('.roi-dot')[i];
+        if (dot) dot.style.opacity = '1';
+        tooltip.innerHTML = '<strong>' + d.day + '</strong><br>' +
+          d.reduction_pct + '% fewer tools<br>' +
+          d.turns_with + ' recall / ' + d.turns_without + ' no-recall turns';
+        tooltip.style.display = 'block';
+        var tx = Math.min(x(i), W - 160);
+        tooltip.style.left = tx + 'px';
+      });
+      rect.addEventListener('mouseleave', function() {
+        var i = parseInt(rect.getAttribute('data-idx'));
+        var dot = container.querySelectorAll('.roi-dot')[i];
+        if (dot) dot.style.opacity = '0';
+        tooltip.style.display = 'none';
+      });
+    });
+  }
+
   /* ── Data loaders ──────────────────────────────────────── */
   async function loadSummary()    { try { renderSummary(await api('/api/summary')); } catch(e) { console.warn('summary',e); } }
   async function loadProjects()   { try { renderProjects(await api('/api/projects')); } catch(e) { console.warn('projects',e); } }
@@ -847,6 +930,7 @@
   async function loadHealth()     { try { renderHealth(await api('/api/health')); } catch(e) { console.warn('health',e); } }
   async function loadTrend()      { try { renderTrend(await api('/api/recall-trend?days=30')); } catch(e) { console.warn('trend',e); } }
   async function loadROI()        { try { renderROI(await api('/api/roi')); } catch(e) { console.warn('roi',e); } }
+  async function loadROITrend()   { try { renderROITrend(await api('/api/roi-trend')); } catch(e) { console.warn('roi-trend',e); } }
   async function loadRecalls() {
     try {
       var url = '/api/recalls?limit=50' + (projectFilter ? '&project=' + encodeURIComponent(projectFilter) : '');
@@ -855,7 +939,7 @@
   }
 
   async function refresh() {
-    await Promise.allSettled([loadSummary(), loadProjects(), loadEfficiency(), loadSessionEfficiency(), loadHealth(), loadTrend(), loadRecalls(), loadROI()]);
+    await Promise.allSettled([loadSummary(), loadProjects(), loadEfficiency(), loadSessionEfficiency(), loadHealth(), loadTrend(), loadRecalls(), loadROI(), loadROITrend()]);
     document.getElementById('last-updated').textContent = new Date().toLocaleTimeString();
 
     renderMiniBars('mb-chunks', null);

memor_cli-0.5.0/memor/eval/counterfactual.py

@@ -0,0 +1,176 @@
+"""Counterfactual eval harness — win/tie/loss vs no-memory baseline.
+
+For each held-out session, uses the opening turn as query, retrieves context
+from prior sessions, and asks an LLM judge whether the recalled context would
+have helped (win), been irrelevant (tie), or misled the agent (loss).
+
+The key metric is do-no-harm rate = 1 - loss_rate."""
+from __future__ import annotations
+
+import enum
+import json
+import re
+import time
+from dataclasses import dataclass
+
+from memor.types import Artifact, Scope
+from memor.retrieve.retriever import Retriever
+
+
+class Outcome(enum.Enum):
+    WIN = "win"
+    TIE = "tie"
+    LOSS = "loss"
+
+
+@dataclass
+class CounterfactualCase:
+    query: str
+    holdout_texts: list[str]
+    scope_project: str
+    session_id: str
+
+
+@dataclass
+class CounterfactualVerdict:
+    outcome: Outcome
+    reasoning: str
+    confidence: float
+
+
+COUNTERFACTUAL_PROMPT = """You are evaluating whether recalled memory context helped or hurt a coding agent.
+
+The agent received this task:
+---
+{query}
+---
+
+Here is what actually happened next in the session (ground truth — the agent didn't see this):
+---
+{holdout}
+---
+
+Here is the context that was recalled from prior sessions and injected into the agent's prompt:
+---
+{recalled_context}
+---
+
+Compare the recalled context against what actually happened. Score as:
+
+- **win**: The recalled context directly anticipates what happened, would save the agent exploration/time, or would prevent repeating a past mistake. The agent would have been measurably better with this context.
+- **tie**: The recalled context is irrelevant to what happened. Neither helpful nor harmful — the agent would have arrived at the same outcome either way.
+- **loss**: The recalled context is stale, contradictory, or misleading. It would have sent the agent down the wrong path or caused confusion.
+
+When uncertain, default to **tie** — only score win when the context clearly helps, and loss when it clearly hurts.
+
+Return STRICT JSON: {{"outcome": "win"|"tie"|"loss", "reasoning": "<one sentence>", "confidence": <float 0-1>}}"""
+
+
+def build_cases_from_store(store, *, project: str,
+                           holdout_turns: int = 2,
+                           min_session_turns: int = 4,
+                           min_prior_sessions: int = 1) -> list[CounterfactualCase]:
+    rows = store.db.execute(
+        "SELECT * FROM artifacts WHERE project=? AND kind='session_chunk' AND active=1",
+        (project,)).fetchall()
+    artifacts = [store._row_to_artifact(r) for r in rows]
+
+    by_session: dict[str, list[Artifact]] = {}
+    for a in artifacts:
+        by_session.setdefault(a.meta.get("session_id", "?"), []).append(a)
+    for v in by_session.values():
+        v.sort(key=lambda a: a.meta.get("ord", 0))
+    sessions = sorted(by_session.items(), key=lambda kv: kv[1][0].created_at)
+
+    cases: list[CounterfactualCase] = []
+    for idx, (sid, chunks) in enumerate(sessions):
+        if idx < min_prior_sessions:
+            continue
+        if len(chunks) < min_session_turns:
+            continue
+        query = chunks[0].text
+        holdout = [c.text for c in chunks[-holdout_turns:]]
+        cases.append(CounterfactualCase(
+            query=query, holdout_texts=holdout,
+            scope_project=project, session_id=sid))
+    return cases
+
+
+def _extract_json(raw: str) -> dict:
+    m = re.search(r"\{.*\}", raw, re.DOTALL)
+    if m:
+        return json.loads(m.group())
+    return json.loads(raw)
+
+
+def parse_verdict_json(raw: str) -> CounterfactualVerdict:
+    data = _extract_json(raw)
+    outcome_str = data.get("outcome", "tie").lower().strip()
+    outcome = Outcome(outcome_str) if outcome_str in ("win", "tie", "loss") else Outcome.TIE
+    return CounterfactualVerdict(
+        outcome=outcome,
+        reasoning=data.get("reasoning", ""),
+        confidence=float(data.get("confidence", 0.5)),
+    )
+
+
+def run_case(case: CounterfactualCase, *, store, embedder, llm,
+             k: int = 8) -> CounterfactualVerdict:
+    scope = Scope(project=case.scope_project)
+    r = Retriever(store, embedder, k=k)
+    trace = r.query(case.query, scope)
+
+    if not trace.hits:
+        return CounterfactualVerdict(
+            outcome=Outcome.TIE,
+            reasoning="No context recalled — nothing to evaluate",
+            confidence=1.0)
+
+    recalled = "\n\n".join(
+        f"[{h.artifact.kind}] {h.artifact.text}" for h in trace.hits)
+    holdout = "\n".join(case.holdout_texts)
+
+    prompt = COUNTERFACTUAL_PROMPT.format(
+        query=case.query, holdout=holdout, recalled_context=recalled)
+    raw = llm.complete(prompt)
+    return parse_verdict_json(raw)
+
+
+def run_suite(cases: list[CounterfactualCase], *, store, embedder, llm,
+              k: int = 8) -> dict:
+    verdicts = []
+    for c in cases:
+        v = run_case(c, store=store, embedder=embedder, llm=llm, k=k)
+        verdicts.append((c, v))
+    verdict_list = [v for _, v in verdicts]
+    summary = summarize_verdicts(verdict_list)
+    summary["cases"] = [
+        {"session_id": c.session_id, "query_preview": c.query[:100],
+         "outcome": v.outcome.value, "reasoning": v.reasoning,
+         "confidence": v.confidence}
+        for c, v in verdicts
+    ]
+    return summary
+
+
+def summarize_verdicts(verdicts: list[CounterfactualVerdict]) -> dict:
+    n = len(verdicts)
+    if n == 0:
+        return {"n_cases": 0, "win_count": 0, "tie_count": 0, "loss_count": 0,
+                "win_pct": 0.0, "tie_pct": 0.0, "loss_pct": 0.0,
+                "do_no_harm_pct": 100.0}
+
+    wins = sum(1 for v in verdicts if v.outcome == Outcome.WIN)
+    ties = sum(1 for v in verdicts if v.outcome == Outcome.TIE)
+    losses = sum(1 for v in verdicts if v.outcome == Outcome.LOSS)
+
+    return {
+        "n_cases": n,
+        "win_count": wins,
+        "tie_count": ties,
+        "loss_count": losses,
+        "win_pct": round(wins / n * 100, 1),
+        "tie_pct": round(ties / n * 100, 1),
+        "loss_pct": round(losses / n * 100, 1),
+        "do_no_harm_pct": round((wins + ties) / n * 100, 1),
+    }

{memor_cli-0.4.0 → memor_cli-0.5.0}/memor/store/sqlite_store.py

@@ -234,6 +234,19 @@ class SqliteStore:
         self.db.commit()
         return cur.lastrowid
 
+    def get_latest_eval(self, eval_type: str = "counterfactual") -> dict | None:
+        row = self.db.execute(
+            "SELECT * FROM eval_runs WHERE json_extract(config, '$.type') = ? "
+            "ORDER BY created_at DESC LIMIT 1", (eval_type,)).fetchone()
+        if not row:
+            return None
+        return {
+            "id": row["id"],
+            "created_at": row["created_at"],
+            "config": json.loads(row["config"]),
+            "metrics": json.loads(row["metrics"]),
+        }
+
     def log_recall(self, project: str, query_preview: str, hits_count: int,
                    top_score: float, tokens_injected: int, latency_ms: float,
                    status: str, session_id: str = "") -> None:
@@ -571,6 +584,43 @@ class SqliteStore:
             "tool_call_reduction_pct": reduction,
         }
 
+    def get_roi_trend(self, project: str | None = None,
+                      days: int = 30) -> list[dict]:
+        where = "WHERE user_timestamp > 0"
+        params: list = []
+        if project:
+            where += " AND project = ?"
+            params.append(project)
+
+        rows = self.db.execute(f"""
+            SELECT date(user_timestamp, 'unixepoch', 'localtime') AS day,
+                   AVG(CASE WHEN had_recall = 1 THEN tool_call_count END) AS avg_with,
+                   AVG(CASE WHEN had_recall = 0 THEN tool_call_count END) AS avg_without,
+                   COUNT(CASE WHEN had_recall = 1 THEN 1 END) AS turns_with,
+                   COUNT(CASE WHEN had_recall = 0 THEN 1 END) AS turns_without,
+                   COUNT(*) AS turns_total
+            FROM turn_metrics {where}
+            GROUP BY day
+            HAVING turns_with > 0 AND turns_without > 0
+            ORDER BY day
+        """, params).fetchall()
+
+        result = []
+        for r in rows:
+            avg_w = round(r["avg_with"], 2)
+            avg_wo = round(r["avg_without"], 2)
+            reduction = round((1 - avg_w / avg_wo) * 100, 1) if avg_wo > 0 else 0
+            result.append({
+                "day": r["day"],
+                "avg_with": avg_w,
+                "avg_without": avg_wo,
+                "reduction_pct": reduction,
+                "turns_with": r["turns_with"],
+                "turns_without": r["turns_without"],
+                "turns_total": r["turns_total"],
+            })
+        return result
+
     def get_onboarding_status(self) -> str:
         chunks = self.db.execute(
             "SELECT COUNT(*) as c FROM artifacts WHERE kind='session_chunk' AND active=1"