slopie 1.8.0 → 1.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,7 +6,9 @@ import { Worker, parentPort } from "node:worker_threads";
6
6
 
7
7
  // packages/core/src/analytics/extract.ts
8
8
  import fs from "node:fs";
9
- import { createHash as createHash2 } from "node:crypto";
9
+ import zlib from "node:zlib";
10
+ import { execFileSync } from "node:child_process";
11
+ import { createHash as createHash3 } from "node:crypto";
10
12
  import path from "node:path";
11
13
 
12
14
  // packages/core/src/markers.ts
@@ -1106,33 +1108,6 @@ function topicWords(text) {
1106
1108
  return out;
1107
1109
  }
1108
1110
 
1109
- // packages/core/src/analytics/story-time.ts
1110
- var WEEKDAY = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 };
1111
- function localClock(timezone) {
1112
- const format = new Intl.DateTimeFormat("en-US", {
1113
- timeZone: timezone,
1114
- year: "numeric",
1115
- month: "2-digit",
1116
- day: "2-digit",
1117
- hour: "2-digit",
1118
- minute: "2-digit",
1119
- weekday: "short",
1120
- hourCycle: "h23"
1121
- });
1122
- const memo = /* @__PURE__ */ new Map();
1123
- return (at) => {
1124
- const slot = Math.floor(at / 9e5);
1125
- let hit = memo.get(slot);
1126
- if (hit) return { ...hit, minute: hit.minute + Math.floor((at - slot * 9e5) / 6e4) };
1127
- const parts = {};
1128
- for (const p of format.formatToParts(new Date(slot * 9e5))) parts[p.type] = p.value;
1129
- const hour = Number(parts.hour) % 24;
1130
- hit = { day: `${parts.year}-${parts.month}-${parts.day}`, month: `${parts.year}-${parts.month}`, hour, dow: WEEKDAY[parts.weekday] ?? 0, minute: Number(parts.minute) };
1131
- memo.set(slot, hit);
1132
- return { ...hit, minute: hit.minute + Math.floor((at - slot * 9e5) / 6e4) };
1133
- };
1134
- }
1135
-
1136
1111
  // packages/core/src/analytics/story-extract.ts
1137
1112
  var AGENT_TOOLS = /* @__PURE__ */ new Set(["Task", "Agent", "spawn_agent", "spawn_agents", "TaskCreate"]);
1138
1113
  var isAgentTool = (name) => AGENT_TOOLS.has(name) || name.startsWith("spawn");
@@ -1162,35 +1137,15 @@ function deriveStory(prompts, events, interrupts, options) {
1162
1137
  if (p.before) models.add(p.before);
1163
1138
  if (p.after) models.add(p.after);
1164
1139
  }
1165
- const vocab = options.counters ? options.previous?.vocab ?? /* @__PURE__ */ new Map() : null;
1166
- const topics = options.counters ? new Map((options.previous?.topics ?? []).map((t) => [t.month, t])) : null;
1167
- const clock = localClock(options.timezone);
1168
1140
  for (const p of human) {
1169
1141
  if (!fresh.has(p)) continue;
1170
1142
  p.f = featurize({ text: p.text, typed: options.typed });
1171
1143
  const lex = promptLex(p.text, models);
1172
1144
  if (lex) p.lex = lex;
1173
- if (vocab) for (const w of vocabularyWords(p.text)) vocab.set(w, (vocab.get(w) ?? 0) + 1);
1174
- if (topics) {
1175
- const { hour, month } = clock(p.at);
1176
- const slot = hour < 5 ? 0 : hour >= 9 && hour < 19 ? 1 : -1;
1177
- if (slot >= 0) {
1178
- let bag = topics.get(month);
1179
- if (!bag) {
1180
- bag = { month, night: 0, day: 0, words: /* @__PURE__ */ new Map() };
1181
- topics.set(month, bag);
1182
- }
1183
- if (slot === 0) bag.night++;
1184
- else bag.day++;
1185
- for (const w of topicWords(p.text)) {
1186
- let c = bag.words.get(w);
1187
- if (!c) {
1188
- c = [0, 0];
1189
- bag.words.set(w, c);
1190
- }
1191
- c[slot]++;
1192
- }
1193
- }
1145
+ if (options.counters) {
1146
+ const w = vocabularyWords(p.text), tw = [...topicWords(p.text)];
1147
+ if (w.length) p.w = w;
1148
+ if (tw.length) p.tw = tw;
1194
1149
  }
1195
1150
  }
1196
1151
  turnStats(human, events, interrupts);
@@ -1213,7 +1168,7 @@ function deriveStory(prompts, events, interrupts, options) {
1213
1168
  const q = safeQuote(p.text, 90);
1214
1169
  if (q) p.q = q;
1215
1170
  }
1216
- return { vocab, topics: topics ? [...topics.values()] : null };
1171
+ return { vocab: null, topics: null };
1217
1172
  }
1218
1173
  function turnStats(human, events, interrupts) {
1219
1174
  const ev = events.filter((e) => Number.isFinite(e.at)).sort((a, b) => a.at - b.at);
@@ -1252,19 +1207,61 @@ function turnWeights(human, events, out) {
1252
1207
  var round1 = (x) => Math.round(x * 10) / 10;
1253
1208
  var usageWeight = (input, output, cacheRead, cacheWrite) => input + 5 * output + 1.25 * cacheWrite + 0.1 * cacheRead;
1254
1209
 
1210
+ // packages/core/src/analytics/copy-prune.ts
1211
+ import { createHash as createHash2 } from "node:crypto";
1212
+ var HEAD = 4096;
1213
+ var TAIL = 4096;
1214
+ var SNIFF = 64 * 1024;
1215
+ var CLASSIFY_MAX_BYTES = 512 * 1024 * 1024;
1216
+ var sha1 = (b) => createHash2("sha1").update(b).digest("base64");
1217
+ var StreamHashes = class {
1218
+ headParts = [];
1219
+ headLength = 0;
1220
+ /** The last TAIL bytes seen. */
1221
+ window = Buffer.alloc(0);
1222
+ produced = 0;
1223
+ offsets;
1224
+ at = [];
1225
+ constructor(offsets) {
1226
+ this.offsets = [...offsets].sort((a, b) => a - b);
1227
+ }
1228
+ push(chunk) {
1229
+ if (!chunk.length) return;
1230
+ if (this.headLength < HEAD) {
1231
+ const part = chunk.subarray(0, HEAD - this.headLength);
1232
+ this.headParts.push(Buffer.from(part));
1233
+ this.headLength += part.length;
1234
+ }
1235
+ const from = this.produced, to = from + chunk.length;
1236
+ let joined = null;
1237
+ while (this.offsets.length && this.offsets[0] <= to) {
1238
+ const end = this.offsets.shift();
1239
+ if (end <= from) continue;
1240
+ joined ??= Buffer.concat([this.window, chunk]);
1241
+ const stop = end - (from - this.window.length);
1242
+ this.at.push([end, sha1(joined.subarray(Math.max(0, stop - TAIL), stop))]);
1243
+ }
1244
+ this.window = chunk.length >= TAIL ? Buffer.from(chunk.subarray(chunk.length - TAIL)) : Buffer.concat([this.window, chunk]).subarray(-TAIL);
1245
+ this.produced = to;
1246
+ }
1247
+ result() {
1248
+ return { head: sha1(Buffer.concat(this.headParts)), at: this.at };
1249
+ }
1250
+ };
1251
+
1255
1252
  // packages/core/src/analytics/extract.ts
1256
- var FACTS_VERSION = 5;
1253
+ var FACTS_VERSION = 8;
1257
1254
  var MAX_PROMPT_CHARS = 4e3;
1258
1255
  var isRec = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
1259
1256
  var str = (v) => typeof v === "string" && v.length > 0 ? v : null;
1260
1257
  var int = (v) => typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.floor(v) : 0;
1258
+ var MAX_DATE_MS = 864e13;
1261
1259
  function time(v) {
1262
- if (typeof v === "string") {
1263
- const t = Date.parse(v);
1264
- return Number.isFinite(t) ? t : null;
1265
- }
1266
- if (typeof v === "number" && Number.isFinite(v)) return v < 1e11 ? v * 1e3 : v;
1267
- return null;
1260
+ let t;
1261
+ if (typeof v === "string") t = Date.parse(v);
1262
+ else if (typeof v === "number" && Number.isFinite(v)) t = v < 1e11 ? v * 1e3 : v;
1263
+ else return null;
1264
+ return Number.isFinite(t) && Math.abs(t) <= MAX_DATE_MS && t > 0 ? t : null;
1268
1265
  }
1269
1266
  function textOf(content) {
1270
1267
  if (typeof content === "string") return content;
@@ -1321,7 +1318,7 @@ function makeBuilder(file, harness, size, mtimeMs) {
1321
1318
  file,
1322
1319
  size,
1323
1320
  mtimeMs,
1324
- sessionId: path.basename(file, ".jsonl"),
1321
+ sessionId: path.basename(file.replace(/\.zst$/, ""), ".jsonl"),
1325
1322
  parentId: null,
1326
1323
  child: false,
1327
1324
  project: null,
@@ -1385,7 +1382,8 @@ function addPrompt(b, key, at, project, kind2, reason, text) {
1385
1382
  touch(b, at);
1386
1383
  }
1387
1384
  var SUBAGENT_DIR = `${path.sep}subagents${path.sep}`;
1388
- function claudeLine(b, line, subagent, sessions) {
1385
+ function claudeLine(b, line, subagentPath, sessions, archive = false) {
1386
+ let subagent = subagentPath || !!b.sidechainFile;
1389
1387
  const assistant = line.includes('"type":"assistant"');
1390
1388
  if (assistant ? !line.includes('"usage"') : subagent || !line.includes('"type":"user"')) {
1391
1389
  if (line.length > 4096 || !line.includes('-title"') && !line.includes('"type":"summary"')) return;
@@ -1397,7 +1395,10 @@ function claudeLine(b, line, subagent, sessions) {
1397
1395
  b.facts.malformed++;
1398
1396
  return;
1399
1397
  }
1400
- if (!isRec(r)) return;
1398
+ if (!isRec(r)) {
1399
+ b.facts.malformed++;
1400
+ return;
1401
+ }
1401
1402
  const f = b.facts;
1402
1403
  if (r.type === "summary" || r.type === "custom-title" || r.type === "ai-title") {
1403
1404
  const title = str(r.customTitle) ?? str(r.aiTitle) ?? str(r.title) ?? str(r.summary);
@@ -1406,11 +1407,19 @@ function claudeLine(b, line, subagent, sessions) {
1406
1407
  }
1407
1408
  if (typeof r.cwd === "string" && !f.project) f.project = r.cwd;
1408
1409
  const sid = str(r.sessionId);
1410
+ if (archive && !subagent && sid && r.isSidechain === true && typeof r.agentId === "string" && !sessions.size && !f.prompts.length) {
1411
+ b.sidechainFile = subagent = true;
1412
+ f.parentId = sid;
1413
+ f.child = true;
1414
+ f.sessionId = `${sid}:agent-${r.agentId}`;
1415
+ }
1409
1416
  if (sid) {
1410
1417
  if (subagent) {
1411
- f.parentId = sid;
1412
- f.child = true;
1413
- f.sessionId = `${sid}:${path.basename(f.file, ".jsonl")}`;
1418
+ if (!b.sidechainFile) {
1419
+ f.parentId = sid;
1420
+ f.child = true;
1421
+ f.sessionId = `${sid}:${path.basename(f.file, ".jsonl")}`;
1422
+ }
1414
1423
  } else if (!sessions.size) f.sessionId = sid;
1415
1424
  }
1416
1425
  const m = isRec(r.message) ? r.message : {};
@@ -1474,6 +1483,8 @@ var newCodexState = () => ({
1474
1483
  previousTotal: null,
1475
1484
  tier: void 0,
1476
1485
  imported: false,
1486
+ inTurn: false,
1487
+ anyTurn: false,
1477
1488
  records: /* @__PURE__ */ new Map(),
1478
1489
  compacted: /* @__PURE__ */ new Set(),
1479
1490
  deltas: /* @__PURE__ */ new Set(),
@@ -1491,27 +1502,13 @@ var ResumeMismatch = class extends Error {
1491
1502
  };
1492
1503
  var U_KEYS = ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"];
1493
1504
  var tuple = (u) => isRec(u) ? U_KEYS.map((k) => int(u[k])) : null;
1494
- var AUTO_REVIEW_MODELS = [
1495
- ["2026-07-30", "gpt-5.6-luna"],
1496
- ["2026-03-05", "gpt-5.4"],
1497
- ["2026-02-05", "gpt-5.3-codex"],
1498
- ["2025-12-11", "gpt-5.2-codex"],
1499
- ["2025-11-13", "gpt-5.1-codex"],
1500
- ["2025-09-15", "gpt-5-codex"]
1501
- ];
1502
- function codexModel(model, timestamp) {
1503
- if (model === "codex-auto-review") {
1504
- const day = timestamp.slice(0, 10);
1505
- return { model: AUTO_REVIEW_MODELS.find(([since]) => day >= since)?.[1] ?? "gpt-5", inferred: true };
1506
- }
1507
- return model ? { model, inferred: false } : { model: "gpt-5", inferred: true };
1508
- }
1509
- function codexFact(b, s, at, timestamp, d, key) {
1505
+ var CODEX_NON_MODELS = /* @__PURE__ */ new Set(["codex-auto-review"]);
1506
+ var codexModel = (model) => model && !CODEX_NON_MODELS.has(model) ? model : null;
1507
+ function codexFact(model, tier, at, d, key) {
1510
1508
  const input = d[0], output = d[2], reasoning = d[3], cached = Math.min(d[1], input);
1511
- const { model, inferred } = codexModel(b.currentModel, timestamp);
1512
- const fact = { key, at, model, provider: "openai", input: input - cached, output, cacheRead: cached, cacheWrite: 0, cacheWrite1h: 0, reasoning };
1513
- if (inferred) fact.inferredModel = true;
1514
- if (s.tier) fact.tier = s.tier;
1509
+ const fact = { key, at, model: codexModel(model), provider: "openai", input: input - cached, output, cacheRead: cached, cacheWrite: 0, cacheWrite1h: 0, reasoning };
1510
+ if (model && !fact.model) fact.identity = model;
1511
+ if (tier) fact.tier = tier;
1515
1512
  return fact;
1516
1513
  }
1517
1514
  function codexWanted(head, lineNo) {
@@ -1523,11 +1520,12 @@ function codexLine(b, line, s) {
1523
1520
  const f = b.facts;
1524
1521
  const parse = () => {
1525
1522
  try {
1526
- return JSON.parse(line.toString("utf8"));
1523
+ const v = JSON.parse(line.toString("utf8"));
1524
+ if (isRec(v)) return v;
1527
1525
  } catch {
1528
- f.malformed++;
1529
- return null;
1530
1526
  }
1527
+ f.malformed++;
1528
+ return null;
1531
1529
  };
1532
1530
  if (n === 0 && head.includes("session_meta")) {
1533
1531
  const r = parse();
@@ -1543,6 +1541,7 @@ function codexLine(b, line, s) {
1543
1541
  const spawn = isRec(sub.thread_spawn) ? sub.thread_spawn : {};
1544
1542
  f.parentId = str(spawn.parent_thread_id) ?? str(p.parent_thread_id) ?? str(p.forked_from_id);
1545
1543
  }
1544
+ if (str(p.forked_from_id)) f.forkedFrom = str(p.forked_from_id);
1546
1545
  if (source === "exec" || p.originator === "codex_exec" || p.originator === "codex_sdk_ts") f.automated = true;
1547
1546
  touch(b, time(r.timestamp));
1548
1547
  return;
@@ -1564,8 +1563,8 @@ function codexLine(b, line, s) {
1564
1563
  if (totalSig) s.previousTotal = totalSig;
1565
1564
  if (!d || !(d[0] || d[1] || d[2] || d[3])) return;
1566
1565
  s.deltas.add(d.join(","));
1567
- const key = totalSig ? `x:${totalSig}|${last ? last.join(",") : ""}` : null;
1568
- const fact = codexFact(b, s, time(r.timestamp), String(r.timestamp ?? ""), d, key);
1566
+ const key = totalSig ? `x:${totalSig}|${last ? last.join(",") : ""}` : `y:${String(r.timestamp ?? "")}|${d.join(",")}`;
1567
+ const fact = codexFact(b.currentModel, s.tier, time(r.timestamp), d, key);
1569
1568
  addUsage(b, fact);
1570
1569
  if (fact.at !== null) b.events.push({ at: fact.at, tools: 0, subs: 0, weight: usageWeight(fact.input, fact.output, fact.cacheRead, fact.cacheWrite) });
1571
1570
  return;
@@ -1590,6 +1589,10 @@ function codexLine(b, line, s) {
1590
1589
  if (head.includes('"task_started"')) {
1591
1590
  const p = parse()?.payload;
1592
1591
  s.imported = isRec(p) && String(p.turn_id ?? "").startsWith("external-import");
1592
+ if (!s.imported) {
1593
+ s.inTurn = true;
1594
+ s.anyTurn = true;
1595
+ }
1593
1596
  return;
1594
1597
  }
1595
1598
  if (head.includes('"UserMessage"') && head.includes("item_completed")) {
@@ -1602,7 +1605,8 @@ function codexLine(b, line, s) {
1602
1605
  at: time(r.timestamp),
1603
1606
  text: textOf(item.content),
1604
1607
  imported: s.imported || String(p.turn_id ?? "").startsWith("external-import"),
1605
- project: f.project
1608
+ project: f.project,
1609
+ inTurn: s.inTurn
1606
1610
  });
1607
1611
  return;
1608
1612
  }
@@ -1610,14 +1614,14 @@ function codexLine(b, line, s) {
1610
1614
  const r = parse();
1611
1615
  if (!r) return;
1612
1616
  const p = isRec(r.payload) ? r.payload : {};
1613
- s.responsePrompts.push({ key: null, at: time(r.timestamp), text: textOf(p.content), imported: s.imported, project: f.project });
1617
+ s.responsePrompts.push({ key: null, at: time(r.timestamp), text: textOf(p.content), imported: s.imported, project: f.project, inTurn: s.inTurn });
1614
1618
  return;
1615
1619
  }
1616
1620
  if (head.includes('"event_msg"') && line.subarray(0, 400).includes('"type":"user_message"')) {
1617
1621
  const r = parse();
1618
1622
  if (!r) return;
1619
1623
  const p = isRec(r.payload) ? r.payload : {};
1620
- s.itemPrompts.push({ key: null, at: time(r.timestamp), text: typeof p.message === "string" ? p.message : "", imported: s.imported, project: f.project });
1624
+ s.itemPrompts.push({ key: null, at: time(r.timestamp), text: typeof p.message === "string" ? p.message : "", imported: s.imported, project: f.project, inTurn: s.inTurn });
1621
1625
  return;
1622
1626
  }
1623
1627
  if (codexToolOrAbort(b, head)) return;
@@ -1625,7 +1629,7 @@ function codexLine(b, line, s) {
1625
1629
  const r = parse();
1626
1630
  const p = r && isRec(r.payload) ? r.payload : null;
1627
1631
  const id = p ? str(p.response_id) : null;
1628
- if (r && p && id && isRec(p.usage)) s.records.set(id, { at: time(r.timestamp), usage: p.usage });
1632
+ if (r && p && id && isRec(p.usage) && !s.records.has(id)) s.records.set(id, { at: time(r.timestamp), usage: p.usage, model: b.currentModel, tier: s.tier });
1629
1633
  return;
1630
1634
  }
1631
1635
  if (head.includes('"type":"compacted"')) {
@@ -1661,7 +1665,7 @@ function finishCodex(b, s) {
1661
1665
  const d = record ? tuple(record.usage) : null;
1662
1666
  if (!record || !d || !(d[0] || d[1] || d[2] || d[3]) || s.deltas.has(d.join(","))) continue;
1663
1667
  s.compactionDone.add(id);
1664
- const fact = codexFact(b, s, record.at, record.at ? new Date(record.at).toISOString() : "", d, `codex-compaction:${id}`);
1668
+ const fact = codexFact(record.model, record.tier, record.at, d, `codex-compaction:${id}`);
1665
1669
  fact.compaction = true;
1666
1670
  f.usage.push(fact);
1667
1671
  touch(b, record.at);
@@ -1679,12 +1683,19 @@ function finishCodex(b, s) {
1679
1683
  if (seen.has(p.key)) return;
1680
1684
  seen.add(p.key);
1681
1685
  }
1682
- const key = `codex:${f.sessionId}:${p.key ?? i}`;
1686
+ const key = `codex:${p.key ?? `${i}@${p.at ?? ""}`}`;
1683
1687
  if (reason) return addPrompt(b, key, p.at, p.project, "excluded", reason, "");
1684
1688
  if (p.imported) return addPrompt(b, key, p.at, p.project, "excluded", "imported_from_claude_transcript", "");
1685
1689
  const c = classifyText(cleanCodexText(p.text));
1686
1690
  addPrompt(b, key, p.at, p.project, c.kind, c.reason, c.text);
1691
+ if (!p.inTurn) f.prompts[f.prompts.length - 1].imp = 1;
1687
1692
  });
1693
+ if (!s.anyTurn) {
1694
+ const firstUsage = f.usage.reduce((m, u) => u.at !== null && u.at < m ? u.at : m, Infinity);
1695
+ const before = f.prompts.filter((p) => (p.at ?? Infinity) < firstUsage).sort((x, y) => x.at - y.at);
1696
+ for (const p of f.prompts) delete p.imp;
1697
+ for (const p of before.slice(0, -1)) p.imp = 1;
1698
+ }
1688
1699
  const answers = f.usage.filter((u) => u.at !== null).sort((x, y) => x.at - y.at);
1689
1700
  let j = 0, previous = null;
1690
1701
  for (const p of [...f.prompts].sort((x, y) => (x.at ?? 0) - (y.at ?? 0))) {
@@ -1703,10 +1714,19 @@ function piLine(b, line) {
1703
1714
  b.facts.malformed++;
1704
1715
  return;
1705
1716
  }
1717
+ if (!isRec(r)) {
1718
+ b.facts.malformed++;
1719
+ return;
1720
+ }
1706
1721
  const f = b.facts, at = time(r.timestamp);
1707
1722
  if (r.type === "session") {
1708
1723
  f.sessionId = str(r.id) ?? f.sessionId;
1709
1724
  f.project = str(r.cwd);
1725
+ const parent = str(r.parentSession);
1726
+ if (parent) {
1727
+ const base = path.basename(parent.replace(/\\/g, "/")).replace(/\.jsonl$/, "");
1728
+ f.forkedFrom = base.includes("_") ? base.slice(base.lastIndexOf("_") + 1) : base;
1729
+ }
1710
1730
  touch(b, at);
1711
1731
  return;
1712
1732
  }
@@ -1720,7 +1740,7 @@ function piLine(b, line) {
1720
1740
  if (m.role === "assistant" && isRec(m.usage)) {
1721
1741
  const u = m.usage;
1722
1742
  const fact = {
1723
- key: `pi:${f.sessionId}:${String(r.id)}`,
1743
+ key: `pi:${String(r.id)}@${String(r.timestamp ?? "")}`,
1724
1744
  at,
1725
1745
  model: str(m.model) ?? b.currentModel,
1726
1746
  provider: str(m.provider) ?? b.currentProvider,
@@ -1741,18 +1761,24 @@ function piLine(b, line) {
1741
1761
  }
1742
1762
  if (m.role === "user") {
1743
1763
  const c = classifyText(textOf(m.content));
1744
- addPrompt(b, `pi:${f.sessionId}:${String(r.id)}`, at, f.project, c.kind, c.reason, c.text);
1764
+ addPrompt(b, `pi:${String(r.id)}@${String(r.timestamp ?? "")}`, at, f.project, c.kind, c.reason, c.text);
1745
1765
  }
1746
1766
  }
1747
1767
  var CHUNK = 4 * 1024 * 1024;
1748
1768
  var MAX_LINE = 256 * 1024 * 1024;
1749
1769
  var sharedBuffer = null;
1750
1770
  var RESUME_RECENT_MS = 48 * 36e5;
1751
- function extractFile(file, harness, options = {}, previous) {
1771
+ function extractFile(file, requested, options = {}, previous) {
1752
1772
  const fd = fs.openSync(file, "r");
1753
1773
  try {
1754
1774
  const stat = fs.fstatSync(fd);
1775
+ const hashes = options.checks?.length ? new StreamHashes(options.checks) : null;
1776
+ const zst = isZstd(fd) ? zstdStream(fd, file, hashes) : null;
1777
+ const harness = requested === "auto" ? sniffHarness(zst ? zst.peek(64 * 1024) : headBytes(fd, 64 * 1024)) : requested;
1778
+ if (!harness) throw new Error("unrecognised_archive_source");
1779
+ if (zst && previous) throw new ResumeMismatch();
1755
1780
  let b = makeBuilder(file, harness, stat.size, stat.mtimeMs);
1781
+ if (options.archive) b.facts.archived = true;
1756
1782
  let codex = newCodexState();
1757
1783
  let position = 0, firstNew = 0;
1758
1784
  const subagent = file.includes(SUBAGENT_DIR), sessions = /* @__PURE__ */ new Set();
@@ -1768,7 +1794,8 @@ function extractFile(file, harness, options = {}, previous) {
1768
1794
  position = r.offset;
1769
1795
  firstNew = b.facts.prompts.length;
1770
1796
  }
1771
- const onLine = harness === "claude" ? (line) => claudeLine(b, line, subagent, sessions) : harness === "codex" ? (line) => codexLine(b, line, codex) : (line) => piLine(b, line);
1797
+ const archive = !!options.archive;
1798
+ const onLine = harness === "claude" ? (line) => claudeLine(b, line, subagent, sessions, archive) : harness === "codex" ? (line) => codexLine(b, line, codex) : (line) => piLine(b, line);
1772
1799
  let buffer = sharedBuffer ??= Buffer.allocUnsafe(CHUNK);
1773
1800
  let filled = 0, skipping = false, base = position, endsWithNewline = true;
1774
1801
  for (; ; ) {
@@ -1791,7 +1818,7 @@ function extractFile(file, harness, options = {}, previous) {
1791
1818
  if (bigger.length <= 4 * CHUNK) sharedBuffer = bigger;
1792
1819
  }
1793
1820
  }
1794
- const n = fs.readSync(fd, buffer, filled, buffer.length - filled, position);
1821
+ const n = zst ? zst.read(buffer, filled, buffer.length - filled) : fs.readSync(fd, buffer, filled, buffer.length - filled, position);
1795
1822
  position += n;
1796
1823
  const end = filled + n;
1797
1824
  let start = 0;
@@ -1815,12 +1842,13 @@ function extractFile(file, harness, options = {}, previous) {
1815
1842
  filled = end - start;
1816
1843
  }
1817
1844
  if (harness === "codex") finishCodex(b, codex);
1845
+ if (hashes) b.facts.copyHashes = hashes.result();
1818
1846
  if (harness === "claude" && !subagent && sessions.size) b.facts.sessionIds = [...sessions];
1819
1847
  b.facts.size = stat.size;
1820
1848
  b.facts.mtimeMs = stat.mtimeMs;
1821
1849
  finishStory(b, harness, options, firstNew);
1822
1850
  delete b.facts.resume;
1823
- if (endsWithNewline && !b.facts.excluded && Date.now() - stat.mtimeMs < RESUME_RECENT_MS) {
1851
+ if (!zst && !options.archive && endsWithNewline && !b.facts.excluded && Date.now() - stat.mtimeMs < RESUME_RECENT_MS) {
1824
1852
  const events = new Float64Array(b.events.length * 4);
1825
1853
  b.events.forEach((e, i) => {
1826
1854
  events[i * 4] = e.at;
@@ -1847,6 +1875,144 @@ function extractFile(file, harness, options = {}, previous) {
1847
1875
  fs.closeSync(fd);
1848
1876
  }
1849
1877
  }
1878
+ var ZSTD_MAGIC = 4247762216;
1879
+ function headBytes(fd, max) {
1880
+ const buffer = Buffer.alloc(max);
1881
+ const n = fs.readSync(fd, buffer, 0, max, 0);
1882
+ return buffer.subarray(0, n);
1883
+ }
1884
+ function isZstd(fd) {
1885
+ const head = headBytes(fd, 4);
1886
+ return head.length === 4 && head.readUInt32LE(0) === ZSTD_MAGIC;
1887
+ }
1888
+ var ZSTD_INPUT = 1024 * 1024;
1889
+ var ZSTD_OUTPUT = 1024 * 1024;
1890
+ function chunkedZstd() {
1891
+ const Engine = zlib.ZstdDecompress;
1892
+ if (zstdMode !== "node" || !Engine) return null;
1893
+ let engine;
1894
+ try {
1895
+ engine = new Engine();
1896
+ } catch {
1897
+ return null;
1898
+ }
1899
+ const handle = engine._handle, state = engine._writeState;
1900
+ if (!handle || typeof handle.writeSync !== "function" || !(state instanceof Uint32Array)) {
1901
+ engine.close();
1902
+ return null;
1903
+ }
1904
+ engine.on("error", () => {
1905
+ });
1906
+ const flush = zlib.constants.ZSTD_e_continue;
1907
+ return {
1908
+ step(input) {
1909
+ const outs = [];
1910
+ let inOff = 0, availIn = input.length;
1911
+ for (; ; ) {
1912
+ const out = Buffer.allocUnsafe(ZSTD_OUTPUT);
1913
+ handle.writeSync(flush, input, inOff, availIn, out, 0, out.length);
1914
+ if (engine.destroyed) throw new Error("zstd_corrupt");
1915
+ const availOut = state[0], availInAfter = state[1];
1916
+ if (out.length - availOut > 0) outs.push(out.subarray(0, out.length - availOut));
1917
+ inOff += availIn - availInAfter;
1918
+ availIn = availInAfter;
1919
+ if (availOut !== 0) break;
1920
+ }
1921
+ return outs;
1922
+ },
1923
+ close: () => engine.close()
1924
+ };
1925
+ }
1926
+ function zstdStream(fd, file, hashes) {
1927
+ zstdMode ??= zstdSupport();
1928
+ const engine = chunkedZstd();
1929
+ let pending = [], inPos = 0, done = false;
1930
+ const input = engine ? Buffer.allocUnsafe(ZSTD_INPUT) : null;
1931
+ const emit = (out) => {
1932
+ if (out.length) {
1933
+ pending.push(out);
1934
+ hashes?.push(out);
1935
+ }
1936
+ };
1937
+ const fill = () => {
1938
+ if (!engine) {
1939
+ emit(decompressZstd(file));
1940
+ done = true;
1941
+ return;
1942
+ }
1943
+ const n = fs.readSync(fd, input, 0, ZSTD_INPUT, inPos);
1944
+ inPos += n;
1945
+ if (!n) {
1946
+ done = true;
1947
+ engine.close();
1948
+ return;
1949
+ }
1950
+ try {
1951
+ for (const out of engine.step(input.subarray(0, n))) emit(out);
1952
+ } catch (error) {
1953
+ done = true;
1954
+ engine.close();
1955
+ throw error;
1956
+ }
1957
+ };
1958
+ return {
1959
+ read(target, offset, length) {
1960
+ while (!pending.length && !done) fill();
1961
+ let written = 0;
1962
+ while (written < length && pending.length) {
1963
+ const head = pending[0], n = Math.min(head.length, length - written);
1964
+ head.copy(target, offset + written, 0, n);
1965
+ written += n;
1966
+ if (n === head.length) pending.shift();
1967
+ else pending[0] = head.subarray(n);
1968
+ }
1969
+ return written;
1970
+ },
1971
+ peek(max) {
1972
+ while (pending.reduce((x, p) => x + p.length, 0) < max && !done) fill();
1973
+ const all = Buffer.concat(pending);
1974
+ pending = all.length ? [all] : [];
1975
+ return all.subarray(0, max);
1976
+ }
1977
+ };
1978
+ }
1979
+ function zstdSupport() {
1980
+ if (typeof zlib.zstdDecompressSync === "function") return "node";
1981
+ try {
1982
+ execFileSync("zstd", ["--version"], { stdio: "ignore", timeout: 5e3 });
1983
+ return "cli";
1984
+ } catch {
1985
+ return null;
1986
+ }
1987
+ }
1988
+ var zstdMode;
1989
+ function decompressZstd(file) {
1990
+ zstdMode ??= zstdSupport();
1991
+ if (zstdMode === "node") {
1992
+ try {
1993
+ return zlib.zstdDecompressSync(fs.readFileSync(file));
1994
+ } catch {
1995
+ throw new Error("zstd_corrupt");
1996
+ }
1997
+ }
1998
+ if (zstdMode === "cli") {
1999
+ try {
2000
+ return execFileSync("zstd", ["-dcq", "--", file], { maxBuffer: 2 ** 32, stdio: ["ignore", "pipe", "ignore"] });
2001
+ } catch {
2002
+ throw new Error("zstd_corrupt");
2003
+ }
2004
+ }
2005
+ throw new Error("zstd_unsupported");
2006
+ }
2007
+ function sniffHarness(head) {
2008
+ const firstEnd = head.indexOf(10);
2009
+ const first = head.toString("utf8", 0, Math.min(firstEnd < 0 ? head.length : firstEnd, 4096));
2010
+ if (first.includes('"session_meta"')) return "codex";
2011
+ if (/"type"\s*:\s*"session"/.test(first) && !first.includes('"sessionId"')) return "pi";
2012
+ const text = head.toString("latin1");
2013
+ if (text.includes('"sessionId"') && /"type"\s*:\s*"(?:user|assistant|summary|system)"/.test(text)) return "claude";
2014
+ return null;
2015
+ }
1850
2016
  function resumeBuilder(previous, r, stat) {
1851
2017
  const facts = {
1852
2018
  ...previous,
@@ -1855,7 +2021,7 @@ function resumeBuilder(previous, r, stat) {
1855
2021
  usage: [...previous.usage],
1856
2022
  prompts: previous.prompts.map((p) => ({ ...p })),
1857
2023
  ...previous.sessionIds ? { sessionIds: [...previous.sessionIds] } : {},
1858
- ...previous.story ? { story: { vocab: previous.story.vocab ? new Map(previous.story.vocab) : null, topics: previous.story.topics ? previous.story.topics.map((t) => ({ ...t, words: new Map([...t.words].map(([w, c]) => [w, [c[0], c[1]]])) })) : null } } : {}
2024
+ ...previous.story ? { story: { vocab: null, topics: null } } : {}
1859
2025
  };
1860
2026
  delete facts.resume;
1861
2027
  const b = makeBuilder(previous.file, previous.harness, stat.size, stat.mtimeMs);
@@ -1893,19 +2059,19 @@ function stripText(facts) {
1893
2059
  return facts;
1894
2060
  }
1895
2061
  function tailHash(fd, offset) {
1896
- const start = Math.max(0, offset - TAIL), buffer = Buffer.alloc(offset - start);
2062
+ const start = Math.max(0, offset - TAIL2), buffer = Buffer.alloc(offset - start);
1897
2063
  let read = 0;
1898
2064
  while (read < buffer.length) {
1899
2065
  const n = fs.readSync(fd, buffer, read, buffer.length - read, start + read);
1900
2066
  if (!n) break;
1901
2067
  read += n;
1902
2068
  }
1903
- return createHash2("sha256").update(buffer.subarray(0, read)).digest("hex");
2069
+ return createHash3("sha256").update(buffer.subarray(0, read)).digest("hex");
1904
2070
  }
1905
- var TAIL = 4096;
2071
+ var TAIL2 = 4096;
1906
2072
 
1907
2073
  // packages/core/src/analytics/scan-pool.ts
1908
- var readFacts = (job) => stripText(extractFile(job.file, job.harness, { timezone: job.timezone }));
2074
+ var readFacts = (job) => stripText(extractFile(job.file, job.harness, { timezone: job.timezone, ...job.archive ? { archive: true } : {}, ...job.checks?.length ? { checks: job.checks } : {} }));
1909
2075
  var errorCode = (error) => error instanceof Error && /^[a-z][a-z0-9_]{0,63}$/.test(error.message) ? error.message : "source_unreadable";
1910
2076
  function runScanWorker() {
1911
2077
  if (!parentPort) throw new Error("scan_worker_requires_parent");