polymath-society 0.2.4

package/dist/pipeline/coding-aggregate.js

@@ -0,0 +1,440 @@
+import{createRequire as __cr}from'module';const require=__cr(import.meta.url);
+
+// ../../scripts/coding-aggregate.mts
+import { promises as fs2 } from "fs";
+import path from "path";
+
+// ../../lib/agents/shared/cliAdapter.ts
+var CLAUDE_BIN = process.env.CLAUDE_BIN || "claude";
+
+// ../../lib/agents/shared/backends.ts
+import os from "os";
+var HOME = os.homedir();
+
+// ../../lib/agents/shared/tools.ts
+import { execFile } from "child_process";
+import { promisify } from "util";
+var exec = promisify(execFile);
+
+// ../../lib/agents/shared/limitGate.ts
+import { AsyncLocalStorage } from "async_hooks";
+var MIN = 6e4;
+var HOUR = 60 * MIN;
+var STALL_BACKOFFS_MS = [6e4, 3 * MIN, 10 * MIN, 20 * MIN];
+var SLEEP_CHUNK_MS = 5 * MIN;
+var CANONICAL = [
+  /claude (ai )?usage limit reached/i,
+  /\busage limit reached\b/i,
+  // "session" variant seen live 2026-07-06: "You've hit your session limit ·
+  // resets 4:10am (America/Los_Angeles)" — unrecognized, it failed EVERY
+  // post-wall stage of the overnight rehearsal instead of pausing. Keep the
+  // subject alternatives in sync with what the CLI actually emits.
+  /you'?ve (?:hit|reached) your (?:usage |session |weekly |daily )?limit/i,
+  /\bsession limit\b[^.]{0,40}resets?/i,
+  /reached your usage limit/i,
+  /5[\s-]?hour limit (?:reached|hit)/i,
+  /limit will reset/i
+];
+var LIBERAL = [
+  ...CANONICAL,
+  /usage limit/i,
+  /\bsession limit\b/i,
+  /\brate[\s-]?limit(?:ed|s|\b)/i,
+  /\btoo many requests\b/i,
+  /\b429\b/,
+  /quota (?:exceeded|reached|exhausted)/i,
+  /resets? (?:at|in)\b/i,
+  /resets? \d{1,2}(?::\d{2})?\s*(?:am|pm)/i
+  // "resets 4:10am" — no "at" (live form)
+];
+var ALS = new AsyncLocalStorage();
+
+// ../coding-core/dist/messages.js
+import { promises as fs } from "fs";
+
+// ../coding-core/dist/injected.js
+var SYSTEM_REMINDER_RE = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
+var INJECTED_RES = [
+  /<task-notification>[\s\S]*?<\/task-notification>/g,
+  /<local-command-stdout>[\s\S]*?<\/local-command-stdout>/g,
+  /<local-command-caveat>[\s\S]*?<\/local-command-caveat>/g,
+  /<ide_opened_file>[\s\S]*?<\/ide_opened_file>/g
+];
+var HARNESS_TURN_RES = [
+  /^Non user activity$/i,
+  /^<<autonomous-loop(-dynamic)?>>$/,
+  /^This session is being continued from a previous conversation/,
+  // the loop skill's expanded wake-up prompt — older CLI versions wrote it as a
+  // plain user turn with no isMeta flag (found counting 601 words, 2026-07-07)
+  /^(-{3,}\s*)?#\s*Autonomous loop (check|tick)/
+];
+function stripInjected(t) {
+  for (const re of INJECTED_RES)
+    t = t.replace(re, "");
+  return t.replace(SYSTEM_REMINDER_RE, "");
+}
+function isHarnessTurn(t) {
+  const s = t.trim();
+  return HARNESS_TURN_RES.some((re) => re.test(s));
+}
+function humanTypedText(t) {
+  const s = stripInjected(t).trim();
+  return s && !isHarnessTurn(s) ? s : "";
+}
+function isHarnessEntry(e) {
+  return e.isMeta === true || e.isCompactSummary === true;
+}
+
+// ../coding-core/dist/messages.js
+function extractText(content) {
+  if (typeof content === "string")
+    return content;
+  if (Array.isArray(content))
+    return content.filter((c) => c && typeof c === "object" && c.type === "text").map((c) => c.text || "").join("\n");
+  return "";
+}
+function isToolResultOnly(content) {
+  return Array.isArray(content) && content.length > 0 && content.every((c) => c && typeof c === "object" && c.type === "tool_result");
+}
+function createMessageDeduper() {
+  const seen = /* @__PURE__ */ new Set();
+  return (m) => {
+    const key = m.uuid ?? `${m.t}|${m.text}`;
+    if (seen.has(key))
+      return false;
+    seen.add(key);
+    return true;
+  };
+}
+async function extractMessages(file) {
+  let raw;
+  try {
+    raw = await fs.readFile(file, "utf-8");
+  } catch {
+    return [];
+  }
+  const out = [];
+  const userTexts = /* @__PURE__ */ new Set();
+  const seenUuids = /* @__PURE__ */ new Set();
+  let aiProse = [];
+  let aiTools = {};
+  let aiStart = 0;
+  const flushAI = () => {
+    if (!aiProse.length && Object.keys(aiTools).length === 0)
+      return;
+    const toolStr = Object.entries(aiTools).map(([n, c]) => c > 1 ? `${n}\xD7${c}` : n).join(", ");
+    let prose = aiProse.join(" ").replace(/\s+/g, " ").trim();
+    if (prose.length > 2200)
+      prose = prose.slice(0, 2200) + "\u2026";
+    let text = prose;
+    if (toolStr)
+      text = (text ? text + " " : "") + `[ran ${toolStr}]`;
+    if (text)
+      out.push({ t: aiStart, role: "ai", text });
+    aiProse = [];
+    aiTools = {};
+  };
+  for (const line of raw.split("\n")) {
+    const s = line.trim();
+    if (!s)
+      continue;
+    let e;
+    try {
+      e = JSON.parse(s);
+    } catch {
+      continue;
+    }
+    const t = typeof e.timestamp === "string" ? Date.parse(e.timestamp) : NaN;
+    if (!Number.isFinite(t))
+      continue;
+    if (e.type === "queue-operation" && e.operation === "enqueue" && typeof e.content === "string") {
+      const qt = humanTypedText(e.content);
+      if (qt) {
+        flushAI();
+        out.push({ t, role: "user", text: qt, q: true });
+      }
+      continue;
+    }
+    if (e.type === "user") {
+      if (isHarnessEntry(e))
+        continue;
+      const uuid = typeof e.uuid === "string" ? e.uuid : "";
+      if (uuid && seenUuids.has(uuid))
+        continue;
+      const content = e.message?.content;
+      if (isToolResultOnly(content))
+        continue;
+      const text = humanTypedText(extractText(content));
+      if (text) {
+        if (uuid)
+          seenUuids.add(uuid);
+        userTexts.add(text);
+        flushAI();
+        out.push({ t, role: "user", text });
+      }
+      continue;
+    }
+    if (e.type === "assistant") {
+      const msg = e.message ?? {};
+      if (!aiProse.length && Object.keys(aiTools).length === 0)
+        aiStart = t;
+      if (Array.isArray(msg.content)) {
+        for (const item of msg.content) {
+          const it = item;
+          if (it.type === "text" && it.text)
+            aiProse.push(it.text);
+          else if (it.type === "tool_use" && it.name)
+            aiTools[it.name] = (aiTools[it.name] ?? 0) + 1;
+        }
+      } else {
+        const p = extractText(msg.content);
+        if (p)
+          aiProse.push(p);
+      }
+    }
+  }
+  flushAI();
+  return out.filter((m) => !(m.q && userTexts.has(m.text))).map(({ q, ...m }) => m).sort((a, b) => a.t - b.t);
+}
+
+// ../coding-core/dist/words.js
+var CODE_START = /^\s*(import |export |const |let |var |function\b|class |def |async |await |return |if\s*\(|for\s*\(|while\s*\(|switch\s*\(|#include|<\/?[a-zA-Z][\w-]*[\s>]|[}{)\]]|@[A-Za-z]|\$\s|npm |npx |git |cd |sudo )/;
+var SEAM_RE = /\n\s*\n\s*\n+/;
+var SEGMENT_CEILING = 400;
+var NOTE = "((ran|created|edited|read|wrote|searched) (a|an|\\d+) [a-z]+|updated todos)";
+var TRANSCRIPT_LINE_RE = new RegExp(`^\\s*(${NOTE})((,| and) ${NOTE})*\\s*$|^\\s*thought for .{1,30}$`, "im");
+var EM_DASH_RE = / — /g;
+var MAX_WPM = 1e3;
+var DEFAULT_GAP_MIN = 5;
+function typedWords(text, gapMs) {
+  const typed = humanTypedText(text);
+  const msgDashes = (typed.match(EM_DASH_RE) || []).length;
+  const msgWords = proseWords(typed);
+  const draftedMsg = msgDashes >= 6 && msgWords > 0 && msgDashes / msgWords >= 1 / 200;
+  let counted = 0;
+  for (const seg of typed.split(SEAM_RE)) {
+    if (TRANSCRIPT_LINE_RE.test(seg))
+      continue;
+    const w = proseWords(seg);
+    if (!w)
+      continue;
+    const dashes = (seg.match(EM_DASH_RE) || []).length;
+    if (dashes >= 3 && dashes / w >= 1 / 200)
+      continue;
+    if (draftedMsg && dashes >= 1)
+      continue;
+    counted += Math.min(w, SEGMENT_CEILING);
+  }
+  const mins = gapMs / 6e4;
+  const window = Number.isFinite(mins) && mins > 0 ? mins : DEFAULT_GAP_MIN;
+  return Math.min(counted, Math.round(MAX_WPM * window));
+}
+function proseWords(text) {
+  let t = text;
+  const skillAt = t.indexOf("Base directory for this skill:");
+  if (skillAt >= 0)
+    t = t.slice(0, skillAt);
+  const contAt = t.indexOf("This session is being continued from a previous conversation");
+  if (contAt >= 0)
+    t = t.slice(0, contAt);
+  t = t.replace(/<command-[a-z-]+>[\s\S]*?<\/command-[a-z-]+>/gi, " ");
+  t = t.replace(/```[\s\S]*?```/g, " ");
+  t = t.replace(/`[^`\n]{2,}`/g, " ");
+  const kept = t.split("\n").filter((ln) => {
+    const s = ln.trim();
+    if (s.length < 8)
+      return true;
+    const letters = (s.match(/[A-Za-z]/g) || []).length;
+    const symbols = (s.match(/[{}()[\];=<>+\-*/%&|^~$@\\]/g) || []).length;
+    const codey = CODE_START.test(s) || symbols > 3 && symbols / s.length > 0.25 || // symbol-dense → code
+    letters / s.length < 0.45;
+    return !codey;
+  });
+  return (kept.join(" ").trim().match(/\S+/g) || []).length;
+}
+
+// ../coding-core/dist/throughput.js
+var THROUGHPUT_LADDER = [
+  { score: 11, min: 12e3, label: "superhuman \u2014 a day beyond the proven ceiling (unreached target)" },
+  { score: 10, min: 9e3, label: "world-class \u2014 sustained, very high directed output" },
+  { score: 9, min: 7e3, label: "exceptional output day" },
+  { score: 8, min: 6e3, label: "high-throughput day" },
+  { score: 7, min: 4500, label: "strong working day" },
+  { score: 6, min: 3e3, label: "solid focused day" },
+  { score: 5, min: 2e3, label: "real but moderate" },
+  { score: 4, min: 1200, label: "light" },
+  { score: 3, min: 600, label: "brief" },
+  { score: 2, min: 200, label: "incidental" },
+  { score: 1, min: 0, label: "barely touched it" }
+];
+function throughputScore(words) {
+  for (const r of THROUGHPUT_LADDER)
+    if (words >= r.min)
+      return r.score;
+  return 1;
+}
+function throughputCeiling(dayWords, proof = 3) {
+  const scores = dayWords.map(throughputScore).sort((a, b) => b - a);
+  if (scores.length === 0)
+    return 0;
+  return scores[Math.min(proof, scores.length) - 1];
+}
+
+// ../../scripts/coding-aggregate.mts
+var CODING = ".data/coding";
+var TZ = "America/Los_Angeles";
+var readJson = async (f, d) => {
+  try {
+    return JSON.parse(await fs2.readFile(path.join(CODING, f), "utf8"));
+  } catch {
+    return d;
+  }
+};
+async function throughputAvg(spanHourMin) {
+  const flow = await readJson("flow.json", { days: [] });
+  const spanH = {};
+  for (const d of flow.days || []) {
+    const slots = Object.keys(d.slots || {}).map(Number);
+    spanH[d.date] = slots.length ? (Math.max(...slots) - Math.min(...slots) + 1) / 6 : 0;
+  }
+  const sess = JSON.parse(await fs2.readFile(path.join(CODING, "sessions.json"), "utf8")).filter((s) => !s.duplicateOf && s.klass === "interactive");
+  const wordsByDay = {};
+  const firstSeen = createMessageDeduper();
+  for (const s of sess) {
+    let ms = [];
+    try {
+      ms = await extractMessages(s.file);
+    } catch {
+      continue;
+    }
+    for (const m of ms) if (m.role === "user" && firstSeen(m)) {
+      const date = new Date(m.t).toLocaleDateString("en-CA", { timeZone: TZ });
+      wordsByDay[date] = (wordsByDay[date] || 0) + typedWords(m.text || "", NaN);
+    }
+  }
+  const dates = Object.keys(wordsByDay).filter((d) => wordsByDay[d] > 0);
+  const mean = (a) => a.length ? Math.round(a.reduce((x, y) => x + y, 0) / a.length) : 0;
+  const substantial = dates.filter((d) => (spanH[d] || 0) >= spanHourMin);
+  const avgWords = mean(substantial.map((d) => wordsByDay[d]));
+  const sens = {};
+  for (const h of [2, 3, 4, 5, 6]) {
+    const ds = dates.filter((d) => (spanH[d] || 0) >= h);
+    const aw = mean(ds.map((d) => wordsByDay[d]));
+    sens["span\u2265" + h + "h"] = { days: ds.length, avgWords: aw, score: throughputScore(aw) };
+  }
+  const score = throughputScore(avgWords);
+  const label = THROUGHPUT_LADDER.find((r) => r.score === score)?.label ?? "";
+  return { spanHourMin, substantialDays: substantial.length, totalCodingDays: dates.length, avgWordsPerDay: avgWords, score, label, ceiling: throughputCeiling(dates.map((d) => wordsByDay[d])), daySensitivity: sens };
+}
+async function abilityCeiling() {
+  const dir = path.join(CODING, "grades");
+  const files = (await fs2.readdir(dir)).filter((f) => f.endsWith(".json"));
+  const collect = (key) => [];
+  const byKey = { generative: [], taste: [] };
+  for (const f of files) {
+    let j;
+    try {
+      j = JSON.parse(await fs2.readFile(path.join(dir, f), "utf8"));
+    } catch {
+      continue;
+    }
+    for (const key of ["generative", "taste"]) {
+      const c = (j.criteria || []).find((x) => x.key === key);
+      if (c && typeof c.score === "number") byKey[key].push({ date: (j.date || "").slice(0, 10), title: (j.title || "").slice(0, 48), score: c.score, instance: (c.instance || "").replace(/\s+/g, " ").slice(0, 600), quote: ((c.quotes || [])[0] || "").replace(/\s+/g, " ").slice(0, 400) });
+    }
+  }
+  const score1 = (rows, label) => {
+    const sorted = [...rows].sort((a, b) => b.score - a.score);
+    if (!sorted.length) return { label, score: null, peak: null, qualifying: 0, instances: [], consistency: "none", note: "no evidence yet" };
+    const peak = sorted[0].score;
+    const near = sorted.filter((r) => r.score >= Math.max(8, peak - 1));
+    const n = near.length;
+    const consistency = n >= 3 ? "consistent" : n === 2 ? "shown twice" : "single peak (thin)";
+    const score = n >= 2 ? peak : peak;
+    return { label, score, peak, qualifying: n, thin: n < 2, consistency, instances: near.slice(0, 10) };
+  };
+  return { abstraction: score1(byKey.generative, "Abstraction"), taste: score1(byKey.taste, "Taste") };
+}
+var slotHour = (slot) => Math.floor(slot / 6);
+var hm = (slot) => {
+  const h = Math.floor(slot / 6), m = slot % 6 * 10;
+  const ap = h < 12 ? "am" : "pm";
+  const h12 = h % 12 || 12;
+  return `${h12}:${String(m).padStart(2, "0")}${ap}`;
+};
+async function flowStats() {
+  const flow = await readJson("flow.json", { days: [], totals: {} });
+  const conc = await readJson("concurrency.json", { maxConcurrency: 0, perDay: {} });
+  const days = flow.days || [];
+  const starts = [], ends = [];
+  for (const d of days) {
+    const slots = Object.keys(d.slots || {}).map(Number);
+    if (!slots.length) continue;
+    starts.push(Math.min(...slots));
+    ends.push(Math.max(...slots));
+  }
+  const med = (a) => a.length ? [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)] : 0;
+  const hour = Array.from({ length: 24 }, (_, h) => ({ hour: h, F: 0, A: 0 }));
+  for (const d of days) for (const [slot, v] of Object.entries(d.slots || {})) {
+    const h = slotHour(Number(slot));
+    if (v === "F") hour[h].F++;
+    else hour[h].A++;
+  }
+  const flowByHour = hour.map((h) => ({ hour: h.hour, flowSlots: h.F, activeSlots: h.F + h.A, frac: h.F + h.A ? Math.round(h.F / (h.F + h.A) * 100) / 100 : 0 })).filter((h) => h.activeSlots > 0);
+  let bestStart = 0, bestSum = -1;
+  for (let h = 0; h <= 21; h++) {
+    const s = (hour[h]?.F || 0) + (hour[h + 1]?.F || 0) + (hour[h + 2]?.F || 0);
+    if (s > bestSum) {
+      bestSum = s;
+      bestStart = h;
+    }
+  }
+  const peakWindow = bestSum > 0 ? `${hm(bestStart * 6)}\u2013${hm((bestStart + 3) * 6)}` : null;
+  const perDay = conc.perDay || {};
+  const concOf = (date) => {
+    const v = perDay[date];
+    return typeof v === "number" ? v : v?.peak ?? v?.max ?? v?.maxConcurrency ?? 1;
+  };
+  const tiers = { "solo (\xD71)": { fracs: [] }, "light (\xD72\u20133)": { fracs: [] }, "heavy (\xD74+)": { fracs: [] } };
+  for (const d of days) {
+    const c = concOf(d.date);
+    const t = c <= 1 ? "solo (\xD71)" : c <= 3 ? "light (\xD72\u20133)" : "heavy (\xD74+)";
+    tiers[t].fracs.push(d.flowFraction || 0);
+  }
+  const byParallelism = Object.entries(tiers).map(([tier, { fracs }]) => ({ tier, days: fracs.length, avgFlowFraction: fracs.length ? Math.round(fracs.reduce((a, b) => a + b, 0) / fracs.length * 100) / 100 : 0 })).filter((t) => t.days > 0);
+  const totalFlowHours = Math.round(days.reduce((a, d) => a + (d.flowHours || 0), 0) * 10) / 10;
+  const flowHoursPerDay = days.length ? Math.round(totalFlowHours / days.length * 10) / 10 : 0;
+  const wdSlots = starts.length && ends.length ? Math.max(0, med(ends) - med(starts)) : 0;
+  const workDayHours = Math.round(wdSlots / 6 * 10) / 10;
+  const pctOfWorkDay = workDayHours > 0 ? Math.round(flowHoursPerDay / workDayHours * 100) / 100 : null;
+  const bestTier = [...byParallelism].sort((a, b) => b.avgFlowFraction - a.avgFlowFraction)[0];
+  const insight = peakWindow ? `Flow peaks ${peakWindow}; deepest on ${bestTier ? bestTier.tier : "\u2014"} days (${bestTier ? Math.round(bestTier.avgFlowFraction * 100) : 0}% in flow).` : "Not enough flow data yet.";
+  return {
+    workWindow: { typicalStart: starts.length ? hm(med(starts)) : null, typicalEnd: ends.length ? hm(med(ends)) : null, activeDays: days.length },
+    parallelism: { maxConcurrent: conc.maxConcurrency || 0 },
+    flow: { fractionOfActiveTime: flow.totals?.flowFraction ?? null, pctOfWorkDay, workDayHours, flowHoursPerDay, totalFlowHours },
+    peakWindow,
+    flowByHour,
+    byParallelism,
+    insight
+  };
+}
+async function main() {
+  const ability = await abilityCeiling();
+  const flow = await flowStats();
+  const throughput = await throughputAvg(4);
+  const prev = await readJson("aggregate.json", {});
+  const out = { generatedAt: (/* @__PURE__ */ new Date()).toISOString(), ability, flow, throughput, ...prev.bigPicture ? { bigPicture: prev.bigPicture } : {} };
+  await fs2.writeFile(path.join(CODING, "aggregate.json"), JSON.stringify(out, null, 2));
+  console.log("aggregate.json written" + (prev.bigPicture ? " (bigPicture preserved)" : " (no bigPicture present \u2014 run coding-nutshell.mts)"));
+  console.log(`  Throughput AVG (start\u2192finish span \u2265${throughput.spanHourMin}h): ${throughput.score}/11 (${throughput.label}) \xB7 ${throughput.avgWordsPerDay} words/day over ${throughput.substantialDays}/${throughput.totalCodingDays} days \xB7 [peak ceiling was ${throughput.ceiling}]`);
+  console.log(`  day sensitivity:`, throughput.daySensitivity);
+  console.log(`  Abstraction ceiling: ${ability.abstraction.score} (${ability.abstraction.consistency}, ${ability.abstraction.qualifying} qualifying)`);
+  console.log(`  Taste ceiling:       ${ability.taste.score} (${ability.taste.consistency}, ${ability.taste.qualifying} qualifying)`);
+  console.log(`  Work window: ${flow.workWindow.typicalStart}\u2013${flow.workWindow.typicalEnd} \xB7 ${flow.flow.fractionOfActiveTime != null ? Math.round(flow.flow.fractionOfActiveTime * 100) : "?"}% in flow`);
+  console.log(`  ${flow.insight}`);
+}
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});