sigrank 0.0.194 → 0.0.196

package/adapters/index.mjs

@@ -747,8 +747,16 @@ export const hermesAdapter = {
 // Native 4-pillar: metrics.input_tokens is FRESH input (does NOT include cache
 // read or cache creation — verified against live data where input=3 alongside
 // cache_creation_tokens=23500). cache_read_tokens and cache_creation_tokens are
-// separate fields (nullable → treat null as 0). output_tokens includes any
-// reasoning tokens (no separate reasoning field in metrics). No estimation needed.
+// separate fields. output_tokens includes any reasoning tokens.
+//
+// CACHE_CREATION FALLBACK: the Devin CLI records cache_creation_tokens natively
+// for Claude models (claude-sonnet-4-6, claude-opus-4-6) but writes NULL for
+// GLM-5-2 and other non-Claude models. When the field is NULL, estimate
+// cacheCreate from cache_read using the construction ratio (cw/cr = 0.0635)
+// calibrated from the 8K Claude-era rows (50.3M cw / 792M cr). This preserves
+// the compounding signal — without it, cacheCreate=0 makes every window look
+// like a "stateless pipe" (no cache commits), which is a data gap, not reality.
+const DEVIN_CONSTRUCTION_RATIO = 0.0635;
 export const devinAdapter = {
   platform: "devin",
   defaultRoot: () => join(homedir(), ".local", "share", "devin", "cli"),
@@ -773,7 +781,12 @@ export const devinAdapter = {
         const input = Number(row.input_tokens || 0);
         const output = Number(row.output_tokens || 0);
         const cacheRead = Number(row.cache_read_tokens || 0);
-        const cacheCreate = Number(row.cache_creation_tokens || 0);
+        // Use native cache_creation_tokens when present; estimate from cache_read
+        // when NULL (GLM-5-2 era doesn't record the field).
+        const cacheCreate =
+          row.cache_creation_tokens != null
+            ? Number(row.cache_creation_tokens || 0)
+            : Math.round(cacheRead * DEVIN_CONSTRUCTION_RATIO);
         if (input + output + cacheRead + cacheCreate === 0) continue;
         yield {
           id: `devin:${row.row_id}`,
@@ -904,38 +917,104 @@ export const otherAdapter = {
 // File cap: a real omp tree runs to 20k+ transcripts, past walkFiles' 10_000 default,
 // which would silently drop half the operator's tokens — pass an explicit cap.
 const OMP_MAX_FILES = 500_000;
+
+// ── Stage 1: bounded-concurrency + line-guard optimization ───────────────────
+// Based on George-RD's measured findings (PR #32 review comment):
+//   - CPU-bound, not disk-bound: 90.5% of one thread, tree fully page-cached
+//   - c=8 is optimal: 45.06s → 34.92s (1.29x), regresses at c12/c16
+//   - Line guard MUST pass both "message" AND "session" lines, or it breaks
+//     the (session_id, entry_id) dedup key — session header lines carry the id
+//   - Two-term guard: 0 false negatives across 2.18M lines
+//   - All 504 duplicate keys carry byte-identical pillar tuples → order-safe
+//   - Only ~1-3% end-to-end win from the guard alone (omp's skippable lines are
+//     19.8% of lines but 15.3% of bytes); the guard costs 1.2s but saves parse
+//     on lines that can't contain usage. Folded into concurrency, not standalone.
+const OMP_CONCURRENCY = 8;
+
+/** Check if a line could contain an omp usage record or session header.
+ *  Conservative: false positives are fine, false negatives are not.
+ *  Must match both "message" (usage) and "session" (dedup key) lines. */
+function ompLineCouldMatter(line) {
+  // Fast substring check before JSON.parse — rejects ~20% of lines that are
+  // neither message nor session entries (tool calls, reasoning, etc.)
+  return line.includes('"message"') || line.includes('"session"');
+}
+
+/** Parse a single omp transcript file and yield usage records.
+ *  Isolated so it can be called concurrently from the bounded pool. */
+function* parseOmpFile(text, path) {
+  if (!text) return;
+  let sid = null;
+  for (const line of text.split("\n")) {
+    const s = line.trim();
+    if (!s) continue;
+    // Line guard: skip lines that can't be message or session entries.
+    // George-RD verified 0 false negatives across 2.18M lines with this guard.
+    if (!ompLineCouldMatter(s)) continue;
+    let ev;
+    try {
+      ev = JSON.parse(s);
+    } catch {
+      continue;
+    }
+    if (!ev) continue;
+    if (ev.type === "session") {
+      sid = ev.id || null;
+      continue;
+    }
+    if (ev.type !== "message") continue;
+    const u = ev.message && ev.message.usage;
+    if (!u) continue;
+    const input = Number(u.input || 0);
+    const output = Number(u.output || 0); // reasoningTokens ALREADY inside — TRAP 1
+    const cacheCreate = Number(u.cacheWrite || 0);
+    const cacheRead = Number(u.cacheRead || 0);
+    if (input + output + cacheCreate + cacheRead === 0) continue;
+    yield {
+      id: ev.id || null,
+      sid,
+      ts: ev.timestamp || null,
+      input,
+      output,
+      cacheCreate,
+      cacheRead,
+      file: path,
+    };
+  }
+}
+
 export const ompAdapter = {
   platform: "omp",
   defaultRoot: () => join(homedir(), ".omp", "agent", "sessions"),
   async *messages(root) {
     for (const r of roots("OMP_DATA_DIR", root)) {
+      // Collect all file paths first (readdir is 0.8% of scan time — negligible)
+      const paths = [];
       for await (const path of walkFiles(r, isJsonl, { n: 0 }, OMP_MAX_FILES)) {
-        const text = await readUtf8(path);
-        let sid = null;
-        for (const [ev] of parseJsonl(text, path)) {
-          if (!ev) continue;
-          if (ev.type === "session") {
-            sid = ev.id || null; // line-2 header: session half of the dedup key
-            continue;
-          }
-          if (ev.type !== "message") continue;
-          const u = ev.message && ev.message.usage;
-          if (!u) continue;
-          const input = Number(u.input || 0);
-          const output = Number(u.output || 0); // reasoningTokens ALREADY inside — TRAP 1
-          const cacheCreate = Number(u.cacheWrite || 0);
-          const cacheRead = Number(u.cacheRead || 0);
-          if (input + output + cacheCreate + cacheRead === 0) continue;
-          yield {
-            id: ev.id || null,
-            sid,
-            ts: ev.timestamp || null,
-            input,
-            output,
-            cacheCreate,
-            cacheRead,
-            file: path,
-          };
+        paths.push(path);
+      }
+
+      // Bounded-concurrency read+parse: George-RD measured c=8 as optimal
+      // (1.29x speedup, 45s → 35s). Order-safe: all 504 duplicate keys carry
+      // byte-identical pillar tuples, so completion order can't change totals.
+      const concurrency = OMP_CONCURRENCY;
+      const inflight = new Set();
+      const queue = [...paths];
+
+      while (queue.length > 0 || inflight.size > 0) {
+        while (inflight.size < concurrency && queue.length > 0) {
+          const path = queue.shift();
+          const p = (async () => {
+            const text = await readUtf8(path);
+            return { path, records: [...parseOmpFile(text, path)] };
+          })();
+          inflight.add(p);
+          p.finally(() => inflight.delete(p));
+        }
+        const done = await Promise.race(inflight);
+        inflight.delete(done);
+        for (const record of done.records) {
+          yield record;
         }
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sigrank",
-  "version": "0.0.194",
+  "version": "0.0.196",
   "author": "SunrisesIllNeverSee (https://github.com/SunrisesIllNeverSee)",
   "description": "SigRank MCP server — the yield cascade + live leaderboard as MCP tools any agent can call",
   "mcpName": "io.github.SunrisesIllNeverSee/sigrank-mcp",