slopie 1.8.0 → 1.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -33
- package/bin/audit-splash.js +42 -82
- package/dist/audit-native-worker.js +274 -108
- package/dist/audit-native-worker.js.map +4 -4
- package/dist/potsherd.js +8814 -5673
- package/dist/potsherd.js.map +4 -4
- package/package.json +1 -1
|
@@ -6,7 +6,9 @@ import { Worker, parentPort } from "node:worker_threads";
|
|
|
6
6
|
|
|
7
7
|
// packages/core/src/analytics/extract.ts
|
|
8
8
|
import fs from "node:fs";
|
|
9
|
-
import
|
|
9
|
+
import zlib from "node:zlib";
|
|
10
|
+
import { execFileSync } from "node:child_process";
|
|
11
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
10
12
|
import path from "node:path";
|
|
11
13
|
|
|
12
14
|
// packages/core/src/markers.ts
|
|
@@ -1106,33 +1108,6 @@ function topicWords(text) {
|
|
|
1106
1108
|
return out;
|
|
1107
1109
|
}
|
|
1108
1110
|
|
|
1109
|
-
// packages/core/src/analytics/story-time.ts
|
|
1110
|
-
var WEEKDAY = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 };
|
|
1111
|
-
function localClock(timezone) {
|
|
1112
|
-
const format = new Intl.DateTimeFormat("en-US", {
|
|
1113
|
-
timeZone: timezone,
|
|
1114
|
-
year: "numeric",
|
|
1115
|
-
month: "2-digit",
|
|
1116
|
-
day: "2-digit",
|
|
1117
|
-
hour: "2-digit",
|
|
1118
|
-
minute: "2-digit",
|
|
1119
|
-
weekday: "short",
|
|
1120
|
-
hourCycle: "h23"
|
|
1121
|
-
});
|
|
1122
|
-
const memo = /* @__PURE__ */ new Map();
|
|
1123
|
-
return (at) => {
|
|
1124
|
-
const slot = Math.floor(at / 9e5);
|
|
1125
|
-
let hit = memo.get(slot);
|
|
1126
|
-
if (hit) return { ...hit, minute: hit.minute + Math.floor((at - slot * 9e5) / 6e4) };
|
|
1127
|
-
const parts = {};
|
|
1128
|
-
for (const p of format.formatToParts(new Date(slot * 9e5))) parts[p.type] = p.value;
|
|
1129
|
-
const hour = Number(parts.hour) % 24;
|
|
1130
|
-
hit = { day: `${parts.year}-${parts.month}-${parts.day}`, month: `${parts.year}-${parts.month}`, hour, dow: WEEKDAY[parts.weekday] ?? 0, minute: Number(parts.minute) };
|
|
1131
|
-
memo.set(slot, hit);
|
|
1132
|
-
return { ...hit, minute: hit.minute + Math.floor((at - slot * 9e5) / 6e4) };
|
|
1133
|
-
};
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
1111
|
// packages/core/src/analytics/story-extract.ts
|
|
1137
1112
|
var AGENT_TOOLS = /* @__PURE__ */ new Set(["Task", "Agent", "spawn_agent", "spawn_agents", "TaskCreate"]);
|
|
1138
1113
|
var isAgentTool = (name) => AGENT_TOOLS.has(name) || name.startsWith("spawn");
|
|
@@ -1162,35 +1137,15 @@ function deriveStory(prompts, events, interrupts, options) {
|
|
|
1162
1137
|
if (p.before) models.add(p.before);
|
|
1163
1138
|
if (p.after) models.add(p.after);
|
|
1164
1139
|
}
|
|
1165
|
-
const vocab = options.counters ? options.previous?.vocab ?? /* @__PURE__ */ new Map() : null;
|
|
1166
|
-
const topics = options.counters ? new Map((options.previous?.topics ?? []).map((t) => [t.month, t])) : null;
|
|
1167
|
-
const clock = localClock(options.timezone);
|
|
1168
1140
|
for (const p of human) {
|
|
1169
1141
|
if (!fresh.has(p)) continue;
|
|
1170
1142
|
p.f = featurize({ text: p.text, typed: options.typed });
|
|
1171
1143
|
const lex = promptLex(p.text, models);
|
|
1172
1144
|
if (lex) p.lex = lex;
|
|
1173
|
-
if (
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
if (slot >= 0) {
|
|
1178
|
-
let bag = topics.get(month);
|
|
1179
|
-
if (!bag) {
|
|
1180
|
-
bag = { month, night: 0, day: 0, words: /* @__PURE__ */ new Map() };
|
|
1181
|
-
topics.set(month, bag);
|
|
1182
|
-
}
|
|
1183
|
-
if (slot === 0) bag.night++;
|
|
1184
|
-
else bag.day++;
|
|
1185
|
-
for (const w of topicWords(p.text)) {
|
|
1186
|
-
let c = bag.words.get(w);
|
|
1187
|
-
if (!c) {
|
|
1188
|
-
c = [0, 0];
|
|
1189
|
-
bag.words.set(w, c);
|
|
1190
|
-
}
|
|
1191
|
-
c[slot]++;
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1145
|
+
if (options.counters) {
|
|
1146
|
+
const w = vocabularyWords(p.text), tw = [...topicWords(p.text)];
|
|
1147
|
+
if (w.length) p.w = w;
|
|
1148
|
+
if (tw.length) p.tw = tw;
|
|
1194
1149
|
}
|
|
1195
1150
|
}
|
|
1196
1151
|
turnStats(human, events, interrupts);
|
|
@@ -1213,7 +1168,7 @@ function deriveStory(prompts, events, interrupts, options) {
|
|
|
1213
1168
|
const q = safeQuote(p.text, 90);
|
|
1214
1169
|
if (q) p.q = q;
|
|
1215
1170
|
}
|
|
1216
|
-
return { vocab
|
|
1171
|
+
return { vocab: null, topics: null };
|
|
1217
1172
|
}
|
|
1218
1173
|
function turnStats(human, events, interrupts) {
|
|
1219
1174
|
const ev = events.filter((e) => Number.isFinite(e.at)).sort((a, b) => a.at - b.at);
|
|
@@ -1252,19 +1207,61 @@ function turnWeights(human, events, out) {
|
|
|
1252
1207
|
var round1 = (x) => Math.round(x * 10) / 10;
|
|
1253
1208
|
var usageWeight = (input, output, cacheRead, cacheWrite) => input + 5 * output + 1.25 * cacheWrite + 0.1 * cacheRead;
|
|
1254
1209
|
|
|
1210
|
+
// packages/core/src/analytics/copy-prune.ts
|
|
1211
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
1212
|
+
var HEAD = 4096;
|
|
1213
|
+
var TAIL = 4096;
|
|
1214
|
+
var SNIFF = 64 * 1024;
|
|
1215
|
+
var CLASSIFY_MAX_BYTES = 512 * 1024 * 1024;
|
|
1216
|
+
var sha1 = (b) => createHash2("sha1").update(b).digest("base64");
|
|
1217
|
+
var StreamHashes = class {
|
|
1218
|
+
headParts = [];
|
|
1219
|
+
headLength = 0;
|
|
1220
|
+
/** The last TAIL bytes seen. */
|
|
1221
|
+
window = Buffer.alloc(0);
|
|
1222
|
+
produced = 0;
|
|
1223
|
+
offsets;
|
|
1224
|
+
at = [];
|
|
1225
|
+
constructor(offsets) {
|
|
1226
|
+
this.offsets = [...offsets].sort((a, b) => a - b);
|
|
1227
|
+
}
|
|
1228
|
+
push(chunk) {
|
|
1229
|
+
if (!chunk.length) return;
|
|
1230
|
+
if (this.headLength < HEAD) {
|
|
1231
|
+
const part = chunk.subarray(0, HEAD - this.headLength);
|
|
1232
|
+
this.headParts.push(Buffer.from(part));
|
|
1233
|
+
this.headLength += part.length;
|
|
1234
|
+
}
|
|
1235
|
+
const from = this.produced, to = from + chunk.length;
|
|
1236
|
+
let joined = null;
|
|
1237
|
+
while (this.offsets.length && this.offsets[0] <= to) {
|
|
1238
|
+
const end = this.offsets.shift();
|
|
1239
|
+
if (end <= from) continue;
|
|
1240
|
+
joined ??= Buffer.concat([this.window, chunk]);
|
|
1241
|
+
const stop = end - (from - this.window.length);
|
|
1242
|
+
this.at.push([end, sha1(joined.subarray(Math.max(0, stop - TAIL), stop))]);
|
|
1243
|
+
}
|
|
1244
|
+
this.window = chunk.length >= TAIL ? Buffer.from(chunk.subarray(chunk.length - TAIL)) : Buffer.concat([this.window, chunk]).subarray(-TAIL);
|
|
1245
|
+
this.produced = to;
|
|
1246
|
+
}
|
|
1247
|
+
result() {
|
|
1248
|
+
return { head: sha1(Buffer.concat(this.headParts)), at: this.at };
|
|
1249
|
+
}
|
|
1250
|
+
};
|
|
1251
|
+
|
|
1255
1252
|
// packages/core/src/analytics/extract.ts
|
|
1256
|
-
var FACTS_VERSION =
|
|
1253
|
+
var FACTS_VERSION = 8;
|
|
1257
1254
|
var MAX_PROMPT_CHARS = 4e3;
|
|
1258
1255
|
var isRec = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
1259
1256
|
var str = (v) => typeof v === "string" && v.length > 0 ? v : null;
|
|
1260
1257
|
var int = (v) => typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.floor(v) : 0;
|
|
1258
|
+
var MAX_DATE_MS = 864e13;
|
|
1261
1259
|
function time(v) {
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
return null;
|
|
1260
|
+
let t;
|
|
1261
|
+
if (typeof v === "string") t = Date.parse(v);
|
|
1262
|
+
else if (typeof v === "number" && Number.isFinite(v)) t = v < 1e11 ? v * 1e3 : v;
|
|
1263
|
+
else return null;
|
|
1264
|
+
return Number.isFinite(t) && Math.abs(t) <= MAX_DATE_MS && t > 0 ? t : null;
|
|
1268
1265
|
}
|
|
1269
1266
|
function textOf(content) {
|
|
1270
1267
|
if (typeof content === "string") return content;
|
|
@@ -1321,7 +1318,7 @@ function makeBuilder(file, harness, size, mtimeMs) {
|
|
|
1321
1318
|
file,
|
|
1322
1319
|
size,
|
|
1323
1320
|
mtimeMs,
|
|
1324
|
-
sessionId: path.basename(file, ".jsonl"),
|
|
1321
|
+
sessionId: path.basename(file.replace(/\.zst$/, ""), ".jsonl"),
|
|
1325
1322
|
parentId: null,
|
|
1326
1323
|
child: false,
|
|
1327
1324
|
project: null,
|
|
@@ -1385,7 +1382,8 @@ function addPrompt(b, key, at, project, kind2, reason, text) {
|
|
|
1385
1382
|
touch(b, at);
|
|
1386
1383
|
}
|
|
1387
1384
|
var SUBAGENT_DIR = `${path.sep}subagents${path.sep}`;
|
|
1388
|
-
function claudeLine(b, line,
|
|
1385
|
+
function claudeLine(b, line, subagentPath, sessions, archive = false) {
|
|
1386
|
+
let subagent = subagentPath || !!b.sidechainFile;
|
|
1389
1387
|
const assistant = line.includes('"type":"assistant"');
|
|
1390
1388
|
if (assistant ? !line.includes('"usage"') : subagent || !line.includes('"type":"user"')) {
|
|
1391
1389
|
if (line.length > 4096 || !line.includes('-title"') && !line.includes('"type":"summary"')) return;
|
|
@@ -1397,7 +1395,10 @@ function claudeLine(b, line, subagent, sessions) {
|
|
|
1397
1395
|
b.facts.malformed++;
|
|
1398
1396
|
return;
|
|
1399
1397
|
}
|
|
1400
|
-
if (!isRec(r))
|
|
1398
|
+
if (!isRec(r)) {
|
|
1399
|
+
b.facts.malformed++;
|
|
1400
|
+
return;
|
|
1401
|
+
}
|
|
1401
1402
|
const f = b.facts;
|
|
1402
1403
|
if (r.type === "summary" || r.type === "custom-title" || r.type === "ai-title") {
|
|
1403
1404
|
const title = str(r.customTitle) ?? str(r.aiTitle) ?? str(r.title) ?? str(r.summary);
|
|
@@ -1406,11 +1407,19 @@ function claudeLine(b, line, subagent, sessions) {
|
|
|
1406
1407
|
}
|
|
1407
1408
|
if (typeof r.cwd === "string" && !f.project) f.project = r.cwd;
|
|
1408
1409
|
const sid = str(r.sessionId);
|
|
1410
|
+
if (archive && !subagent && sid && r.isSidechain === true && typeof r.agentId === "string" && !sessions.size && !f.prompts.length) {
|
|
1411
|
+
b.sidechainFile = subagent = true;
|
|
1412
|
+
f.parentId = sid;
|
|
1413
|
+
f.child = true;
|
|
1414
|
+
f.sessionId = `${sid}:agent-${r.agentId}`;
|
|
1415
|
+
}
|
|
1409
1416
|
if (sid) {
|
|
1410
1417
|
if (subagent) {
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1418
|
+
if (!b.sidechainFile) {
|
|
1419
|
+
f.parentId = sid;
|
|
1420
|
+
f.child = true;
|
|
1421
|
+
f.sessionId = `${sid}:${path.basename(f.file, ".jsonl")}`;
|
|
1422
|
+
}
|
|
1414
1423
|
} else if (!sessions.size) f.sessionId = sid;
|
|
1415
1424
|
}
|
|
1416
1425
|
const m = isRec(r.message) ? r.message : {};
|
|
@@ -1474,6 +1483,8 @@ var newCodexState = () => ({
|
|
|
1474
1483
|
previousTotal: null,
|
|
1475
1484
|
tier: void 0,
|
|
1476
1485
|
imported: false,
|
|
1486
|
+
inTurn: false,
|
|
1487
|
+
anyTurn: false,
|
|
1477
1488
|
records: /* @__PURE__ */ new Map(),
|
|
1478
1489
|
compacted: /* @__PURE__ */ new Set(),
|
|
1479
1490
|
deltas: /* @__PURE__ */ new Set(),
|
|
@@ -1491,27 +1502,13 @@ var ResumeMismatch = class extends Error {
|
|
|
1491
1502
|
};
|
|
1492
1503
|
var U_KEYS = ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"];
|
|
1493
1504
|
var tuple = (u) => isRec(u) ? U_KEYS.map((k) => int(u[k])) : null;
|
|
1494
|
-
var
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
["2026-02-05", "gpt-5.3-codex"],
|
|
1498
|
-
["2025-12-11", "gpt-5.2-codex"],
|
|
1499
|
-
["2025-11-13", "gpt-5.1-codex"],
|
|
1500
|
-
["2025-09-15", "gpt-5-codex"]
|
|
1501
|
-
];
|
|
1502
|
-
function codexModel(model, timestamp) {
|
|
1503
|
-
if (model === "codex-auto-review") {
|
|
1504
|
-
const day = timestamp.slice(0, 10);
|
|
1505
|
-
return { model: AUTO_REVIEW_MODELS.find(([since]) => day >= since)?.[1] ?? "gpt-5", inferred: true };
|
|
1506
|
-
}
|
|
1507
|
-
return model ? { model, inferred: false } : { model: "gpt-5", inferred: true };
|
|
1508
|
-
}
|
|
1509
|
-
function codexFact(b, s, at, timestamp, d, key) {
|
|
1505
|
+
var CODEX_NON_MODELS = /* @__PURE__ */ new Set(["codex-auto-review"]);
|
|
1506
|
+
var codexModel = (model) => model && !CODEX_NON_MODELS.has(model) ? model : null;
|
|
1507
|
+
function codexFact(model, tier, at, d, key) {
|
|
1510
1508
|
const input = d[0], output = d[2], reasoning = d[3], cached = Math.min(d[1], input);
|
|
1511
|
-
const {
|
|
1512
|
-
|
|
1513
|
-
if (
|
|
1514
|
-
if (s.tier) fact.tier = s.tier;
|
|
1509
|
+
const fact = { key, at, model: codexModel(model), provider: "openai", input: input - cached, output, cacheRead: cached, cacheWrite: 0, cacheWrite1h: 0, reasoning };
|
|
1510
|
+
if (model && !fact.model) fact.identity = model;
|
|
1511
|
+
if (tier) fact.tier = tier;
|
|
1515
1512
|
return fact;
|
|
1516
1513
|
}
|
|
1517
1514
|
function codexWanted(head, lineNo) {
|
|
@@ -1523,11 +1520,12 @@ function codexLine(b, line, s) {
|
|
|
1523
1520
|
const f = b.facts;
|
|
1524
1521
|
const parse = () => {
|
|
1525
1522
|
try {
|
|
1526
|
-
|
|
1523
|
+
const v = JSON.parse(line.toString("utf8"));
|
|
1524
|
+
if (isRec(v)) return v;
|
|
1527
1525
|
} catch {
|
|
1528
|
-
f.malformed++;
|
|
1529
|
-
return null;
|
|
1530
1526
|
}
|
|
1527
|
+
f.malformed++;
|
|
1528
|
+
return null;
|
|
1531
1529
|
};
|
|
1532
1530
|
if (n === 0 && head.includes("session_meta")) {
|
|
1533
1531
|
const r = parse();
|
|
@@ -1543,6 +1541,7 @@ function codexLine(b, line, s) {
|
|
|
1543
1541
|
const spawn = isRec(sub.thread_spawn) ? sub.thread_spawn : {};
|
|
1544
1542
|
f.parentId = str(spawn.parent_thread_id) ?? str(p.parent_thread_id) ?? str(p.forked_from_id);
|
|
1545
1543
|
}
|
|
1544
|
+
if (str(p.forked_from_id)) f.forkedFrom = str(p.forked_from_id);
|
|
1546
1545
|
if (source === "exec" || p.originator === "codex_exec" || p.originator === "codex_sdk_ts") f.automated = true;
|
|
1547
1546
|
touch(b, time(r.timestamp));
|
|
1548
1547
|
return;
|
|
@@ -1564,8 +1563,8 @@ function codexLine(b, line, s) {
|
|
|
1564
1563
|
if (totalSig) s.previousTotal = totalSig;
|
|
1565
1564
|
if (!d || !(d[0] || d[1] || d[2] || d[3])) return;
|
|
1566
1565
|
s.deltas.add(d.join(","));
|
|
1567
|
-
const key = totalSig ? `x:${totalSig}|${last ? last.join(",") : ""}` :
|
|
1568
|
-
const fact = codexFact(b, s, time(r.timestamp),
|
|
1566
|
+
const key = totalSig ? `x:${totalSig}|${last ? last.join(",") : ""}` : `y:${String(r.timestamp ?? "")}|${d.join(",")}`;
|
|
1567
|
+
const fact = codexFact(b.currentModel, s.tier, time(r.timestamp), d, key);
|
|
1569
1568
|
addUsage(b, fact);
|
|
1570
1569
|
if (fact.at !== null) b.events.push({ at: fact.at, tools: 0, subs: 0, weight: usageWeight(fact.input, fact.output, fact.cacheRead, fact.cacheWrite) });
|
|
1571
1570
|
return;
|
|
@@ -1590,6 +1589,10 @@ function codexLine(b, line, s) {
|
|
|
1590
1589
|
if (head.includes('"task_started"')) {
|
|
1591
1590
|
const p = parse()?.payload;
|
|
1592
1591
|
s.imported = isRec(p) && String(p.turn_id ?? "").startsWith("external-import");
|
|
1592
|
+
if (!s.imported) {
|
|
1593
|
+
s.inTurn = true;
|
|
1594
|
+
s.anyTurn = true;
|
|
1595
|
+
}
|
|
1593
1596
|
return;
|
|
1594
1597
|
}
|
|
1595
1598
|
if (head.includes('"UserMessage"') && head.includes("item_completed")) {
|
|
@@ -1602,7 +1605,8 @@ function codexLine(b, line, s) {
|
|
|
1602
1605
|
at: time(r.timestamp),
|
|
1603
1606
|
text: textOf(item.content),
|
|
1604
1607
|
imported: s.imported || String(p.turn_id ?? "").startsWith("external-import"),
|
|
1605
|
-
project: f.project
|
|
1608
|
+
project: f.project,
|
|
1609
|
+
inTurn: s.inTurn
|
|
1606
1610
|
});
|
|
1607
1611
|
return;
|
|
1608
1612
|
}
|
|
@@ -1610,14 +1614,14 @@ function codexLine(b, line, s) {
|
|
|
1610
1614
|
const r = parse();
|
|
1611
1615
|
if (!r) return;
|
|
1612
1616
|
const p = isRec(r.payload) ? r.payload : {};
|
|
1613
|
-
s.responsePrompts.push({ key: null, at: time(r.timestamp), text: textOf(p.content), imported: s.imported, project: f.project });
|
|
1617
|
+
s.responsePrompts.push({ key: null, at: time(r.timestamp), text: textOf(p.content), imported: s.imported, project: f.project, inTurn: s.inTurn });
|
|
1614
1618
|
return;
|
|
1615
1619
|
}
|
|
1616
1620
|
if (head.includes('"event_msg"') && line.subarray(0, 400).includes('"type":"user_message"')) {
|
|
1617
1621
|
const r = parse();
|
|
1618
1622
|
if (!r) return;
|
|
1619
1623
|
const p = isRec(r.payload) ? r.payload : {};
|
|
1620
|
-
s.itemPrompts.push({ key: null, at: time(r.timestamp), text: typeof p.message === "string" ? p.message : "", imported: s.imported, project: f.project });
|
|
1624
|
+
s.itemPrompts.push({ key: null, at: time(r.timestamp), text: typeof p.message === "string" ? p.message : "", imported: s.imported, project: f.project, inTurn: s.inTurn });
|
|
1621
1625
|
return;
|
|
1622
1626
|
}
|
|
1623
1627
|
if (codexToolOrAbort(b, head)) return;
|
|
@@ -1625,7 +1629,7 @@ function codexLine(b, line, s) {
|
|
|
1625
1629
|
const r = parse();
|
|
1626
1630
|
const p = r && isRec(r.payload) ? r.payload : null;
|
|
1627
1631
|
const id = p ? str(p.response_id) : null;
|
|
1628
|
-
if (r && p && id && isRec(p.usage)) s.records.set(id, { at: time(r.timestamp), usage: p.usage });
|
|
1632
|
+
if (r && p && id && isRec(p.usage) && !s.records.has(id)) s.records.set(id, { at: time(r.timestamp), usage: p.usage, model: b.currentModel, tier: s.tier });
|
|
1629
1633
|
return;
|
|
1630
1634
|
}
|
|
1631
1635
|
if (head.includes('"type":"compacted"')) {
|
|
@@ -1661,7 +1665,7 @@ function finishCodex(b, s) {
|
|
|
1661
1665
|
const d = record ? tuple(record.usage) : null;
|
|
1662
1666
|
if (!record || !d || !(d[0] || d[1] || d[2] || d[3]) || s.deltas.has(d.join(","))) continue;
|
|
1663
1667
|
s.compactionDone.add(id);
|
|
1664
|
-
const fact = codexFact(
|
|
1668
|
+
const fact = codexFact(record.model, record.tier, record.at, d, `codex-compaction:${id}`);
|
|
1665
1669
|
fact.compaction = true;
|
|
1666
1670
|
f.usage.push(fact);
|
|
1667
1671
|
touch(b, record.at);
|
|
@@ -1679,12 +1683,19 @@ function finishCodex(b, s) {
|
|
|
1679
1683
|
if (seen.has(p.key)) return;
|
|
1680
1684
|
seen.add(p.key);
|
|
1681
1685
|
}
|
|
1682
|
-
const key = `codex:${
|
|
1686
|
+
const key = `codex:${p.key ?? `${i}@${p.at ?? ""}`}`;
|
|
1683
1687
|
if (reason) return addPrompt(b, key, p.at, p.project, "excluded", reason, "");
|
|
1684
1688
|
if (p.imported) return addPrompt(b, key, p.at, p.project, "excluded", "imported_from_claude_transcript", "");
|
|
1685
1689
|
const c = classifyText(cleanCodexText(p.text));
|
|
1686
1690
|
addPrompt(b, key, p.at, p.project, c.kind, c.reason, c.text);
|
|
1691
|
+
if (!p.inTurn) f.prompts[f.prompts.length - 1].imp = 1;
|
|
1687
1692
|
});
|
|
1693
|
+
if (!s.anyTurn) {
|
|
1694
|
+
const firstUsage = f.usage.reduce((m, u) => u.at !== null && u.at < m ? u.at : m, Infinity);
|
|
1695
|
+
const before = f.prompts.filter((p) => (p.at ?? Infinity) < firstUsage).sort((x, y) => x.at - y.at);
|
|
1696
|
+
for (const p of f.prompts) delete p.imp;
|
|
1697
|
+
for (const p of before.slice(0, -1)) p.imp = 1;
|
|
1698
|
+
}
|
|
1688
1699
|
const answers = f.usage.filter((u) => u.at !== null).sort((x, y) => x.at - y.at);
|
|
1689
1700
|
let j = 0, previous = null;
|
|
1690
1701
|
for (const p of [...f.prompts].sort((x, y) => (x.at ?? 0) - (y.at ?? 0))) {
|
|
@@ -1703,10 +1714,19 @@ function piLine(b, line) {
|
|
|
1703
1714
|
b.facts.malformed++;
|
|
1704
1715
|
return;
|
|
1705
1716
|
}
|
|
1717
|
+
if (!isRec(r)) {
|
|
1718
|
+
b.facts.malformed++;
|
|
1719
|
+
return;
|
|
1720
|
+
}
|
|
1706
1721
|
const f = b.facts, at = time(r.timestamp);
|
|
1707
1722
|
if (r.type === "session") {
|
|
1708
1723
|
f.sessionId = str(r.id) ?? f.sessionId;
|
|
1709
1724
|
f.project = str(r.cwd);
|
|
1725
|
+
const parent = str(r.parentSession);
|
|
1726
|
+
if (parent) {
|
|
1727
|
+
const base = path.basename(parent.replace(/\\/g, "/")).replace(/\.jsonl$/, "");
|
|
1728
|
+
f.forkedFrom = base.includes("_") ? base.slice(base.lastIndexOf("_") + 1) : base;
|
|
1729
|
+
}
|
|
1710
1730
|
touch(b, at);
|
|
1711
1731
|
return;
|
|
1712
1732
|
}
|
|
@@ -1720,7 +1740,7 @@ function piLine(b, line) {
|
|
|
1720
1740
|
if (m.role === "assistant" && isRec(m.usage)) {
|
|
1721
1741
|
const u = m.usage;
|
|
1722
1742
|
const fact = {
|
|
1723
|
-
key: `pi:${
|
|
1743
|
+
key: `pi:${String(r.id)}@${String(r.timestamp ?? "")}`,
|
|
1724
1744
|
at,
|
|
1725
1745
|
model: str(m.model) ?? b.currentModel,
|
|
1726
1746
|
provider: str(m.provider) ?? b.currentProvider,
|
|
@@ -1741,18 +1761,24 @@ function piLine(b, line) {
|
|
|
1741
1761
|
}
|
|
1742
1762
|
if (m.role === "user") {
|
|
1743
1763
|
const c = classifyText(textOf(m.content));
|
|
1744
|
-
addPrompt(b, `pi:${
|
|
1764
|
+
addPrompt(b, `pi:${String(r.id)}@${String(r.timestamp ?? "")}`, at, f.project, c.kind, c.reason, c.text);
|
|
1745
1765
|
}
|
|
1746
1766
|
}
|
|
1747
1767
|
var CHUNK = 4 * 1024 * 1024;
|
|
1748
1768
|
var MAX_LINE = 256 * 1024 * 1024;
|
|
1749
1769
|
var sharedBuffer = null;
|
|
1750
1770
|
var RESUME_RECENT_MS = 48 * 36e5;
|
|
1751
|
-
function extractFile(file,
|
|
1771
|
+
function extractFile(file, requested, options = {}, previous) {
|
|
1752
1772
|
const fd = fs.openSync(file, "r");
|
|
1753
1773
|
try {
|
|
1754
1774
|
const stat = fs.fstatSync(fd);
|
|
1775
|
+
const hashes = options.checks?.length ? new StreamHashes(options.checks) : null;
|
|
1776
|
+
const zst = isZstd(fd) ? zstdStream(fd, file, hashes) : null;
|
|
1777
|
+
const harness = requested === "auto" ? sniffHarness(zst ? zst.peek(64 * 1024) : headBytes(fd, 64 * 1024)) : requested;
|
|
1778
|
+
if (!harness) throw new Error("unrecognised_archive_source");
|
|
1779
|
+
if (zst && previous) throw new ResumeMismatch();
|
|
1755
1780
|
let b = makeBuilder(file, harness, stat.size, stat.mtimeMs);
|
|
1781
|
+
if (options.archive) b.facts.archived = true;
|
|
1756
1782
|
let codex = newCodexState();
|
|
1757
1783
|
let position = 0, firstNew = 0;
|
|
1758
1784
|
const subagent = file.includes(SUBAGENT_DIR), sessions = /* @__PURE__ */ new Set();
|
|
@@ -1768,7 +1794,8 @@ function extractFile(file, harness, options = {}, previous) {
|
|
|
1768
1794
|
position = r.offset;
|
|
1769
1795
|
firstNew = b.facts.prompts.length;
|
|
1770
1796
|
}
|
|
1771
|
-
const
|
|
1797
|
+
const archive = !!options.archive;
|
|
1798
|
+
const onLine = harness === "claude" ? (line) => claudeLine(b, line, subagent, sessions, archive) : harness === "codex" ? (line) => codexLine(b, line, codex) : (line) => piLine(b, line);
|
|
1772
1799
|
let buffer = sharedBuffer ??= Buffer.allocUnsafe(CHUNK);
|
|
1773
1800
|
let filled = 0, skipping = false, base = position, endsWithNewline = true;
|
|
1774
1801
|
for (; ; ) {
|
|
@@ -1791,7 +1818,7 @@ function extractFile(file, harness, options = {}, previous) {
|
|
|
1791
1818
|
if (bigger.length <= 4 * CHUNK) sharedBuffer = bigger;
|
|
1792
1819
|
}
|
|
1793
1820
|
}
|
|
1794
|
-
const n = fs.readSync(fd, buffer, filled, buffer.length - filled, position);
|
|
1821
|
+
const n = zst ? zst.read(buffer, filled, buffer.length - filled) : fs.readSync(fd, buffer, filled, buffer.length - filled, position);
|
|
1795
1822
|
position += n;
|
|
1796
1823
|
const end = filled + n;
|
|
1797
1824
|
let start = 0;
|
|
@@ -1815,12 +1842,13 @@ function extractFile(file, harness, options = {}, previous) {
|
|
|
1815
1842
|
filled = end - start;
|
|
1816
1843
|
}
|
|
1817
1844
|
if (harness === "codex") finishCodex(b, codex);
|
|
1845
|
+
if (hashes) b.facts.copyHashes = hashes.result();
|
|
1818
1846
|
if (harness === "claude" && !subagent && sessions.size) b.facts.sessionIds = [...sessions];
|
|
1819
1847
|
b.facts.size = stat.size;
|
|
1820
1848
|
b.facts.mtimeMs = stat.mtimeMs;
|
|
1821
1849
|
finishStory(b, harness, options, firstNew);
|
|
1822
1850
|
delete b.facts.resume;
|
|
1823
|
-
if (endsWithNewline && !b.facts.excluded && Date.now() - stat.mtimeMs < RESUME_RECENT_MS) {
|
|
1851
|
+
if (!zst && !options.archive && endsWithNewline && !b.facts.excluded && Date.now() - stat.mtimeMs < RESUME_RECENT_MS) {
|
|
1824
1852
|
const events = new Float64Array(b.events.length * 4);
|
|
1825
1853
|
b.events.forEach((e, i) => {
|
|
1826
1854
|
events[i * 4] = e.at;
|
|
@@ -1847,6 +1875,144 @@ function extractFile(file, harness, options = {}, previous) {
|
|
|
1847
1875
|
fs.closeSync(fd);
|
|
1848
1876
|
}
|
|
1849
1877
|
}
|
|
1878
|
+
var ZSTD_MAGIC = 4247762216;
|
|
1879
|
+
function headBytes(fd, max) {
|
|
1880
|
+
const buffer = Buffer.alloc(max);
|
|
1881
|
+
const n = fs.readSync(fd, buffer, 0, max, 0);
|
|
1882
|
+
return buffer.subarray(0, n);
|
|
1883
|
+
}
|
|
1884
|
+
function isZstd(fd) {
|
|
1885
|
+
const head = headBytes(fd, 4);
|
|
1886
|
+
return head.length === 4 && head.readUInt32LE(0) === ZSTD_MAGIC;
|
|
1887
|
+
}
|
|
1888
|
+
var ZSTD_INPUT = 1024 * 1024;
|
|
1889
|
+
var ZSTD_OUTPUT = 1024 * 1024;
|
|
1890
|
+
function chunkedZstd() {
|
|
1891
|
+
const Engine = zlib.ZstdDecompress;
|
|
1892
|
+
if (zstdMode !== "node" || !Engine) return null;
|
|
1893
|
+
let engine;
|
|
1894
|
+
try {
|
|
1895
|
+
engine = new Engine();
|
|
1896
|
+
} catch {
|
|
1897
|
+
return null;
|
|
1898
|
+
}
|
|
1899
|
+
const handle = engine._handle, state = engine._writeState;
|
|
1900
|
+
if (!handle || typeof handle.writeSync !== "function" || !(state instanceof Uint32Array)) {
|
|
1901
|
+
engine.close();
|
|
1902
|
+
return null;
|
|
1903
|
+
}
|
|
1904
|
+
engine.on("error", () => {
|
|
1905
|
+
});
|
|
1906
|
+
const flush = zlib.constants.ZSTD_e_continue;
|
|
1907
|
+
return {
|
|
1908
|
+
step(input) {
|
|
1909
|
+
const outs = [];
|
|
1910
|
+
let inOff = 0, availIn = input.length;
|
|
1911
|
+
for (; ; ) {
|
|
1912
|
+
const out = Buffer.allocUnsafe(ZSTD_OUTPUT);
|
|
1913
|
+
handle.writeSync(flush, input, inOff, availIn, out, 0, out.length);
|
|
1914
|
+
if (engine.destroyed) throw new Error("zstd_corrupt");
|
|
1915
|
+
const availOut = state[0], availInAfter = state[1];
|
|
1916
|
+
if (out.length - availOut > 0) outs.push(out.subarray(0, out.length - availOut));
|
|
1917
|
+
inOff += availIn - availInAfter;
|
|
1918
|
+
availIn = availInAfter;
|
|
1919
|
+
if (availOut !== 0) break;
|
|
1920
|
+
}
|
|
1921
|
+
return outs;
|
|
1922
|
+
},
|
|
1923
|
+
close: () => engine.close()
|
|
1924
|
+
};
|
|
1925
|
+
}
|
|
1926
|
+
function zstdStream(fd, file, hashes) {
|
|
1927
|
+
zstdMode ??= zstdSupport();
|
|
1928
|
+
const engine = chunkedZstd();
|
|
1929
|
+
let pending = [], inPos = 0, done = false;
|
|
1930
|
+
const input = engine ? Buffer.allocUnsafe(ZSTD_INPUT) : null;
|
|
1931
|
+
const emit = (out) => {
|
|
1932
|
+
if (out.length) {
|
|
1933
|
+
pending.push(out);
|
|
1934
|
+
hashes?.push(out);
|
|
1935
|
+
}
|
|
1936
|
+
};
|
|
1937
|
+
const fill = () => {
|
|
1938
|
+
if (!engine) {
|
|
1939
|
+
emit(decompressZstd(file));
|
|
1940
|
+
done = true;
|
|
1941
|
+
return;
|
|
1942
|
+
}
|
|
1943
|
+
const n = fs.readSync(fd, input, 0, ZSTD_INPUT, inPos);
|
|
1944
|
+
inPos += n;
|
|
1945
|
+
if (!n) {
|
|
1946
|
+
done = true;
|
|
1947
|
+
engine.close();
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
try {
|
|
1951
|
+
for (const out of engine.step(input.subarray(0, n))) emit(out);
|
|
1952
|
+
} catch (error) {
|
|
1953
|
+
done = true;
|
|
1954
|
+
engine.close();
|
|
1955
|
+
throw error;
|
|
1956
|
+
}
|
|
1957
|
+
};
|
|
1958
|
+
return {
|
|
1959
|
+
read(target, offset, length) {
|
|
1960
|
+
while (!pending.length && !done) fill();
|
|
1961
|
+
let written = 0;
|
|
1962
|
+
while (written < length && pending.length) {
|
|
1963
|
+
const head = pending[0], n = Math.min(head.length, length - written);
|
|
1964
|
+
head.copy(target, offset + written, 0, n);
|
|
1965
|
+
written += n;
|
|
1966
|
+
if (n === head.length) pending.shift();
|
|
1967
|
+
else pending[0] = head.subarray(n);
|
|
1968
|
+
}
|
|
1969
|
+
return written;
|
|
1970
|
+
},
|
|
1971
|
+
peek(max) {
|
|
1972
|
+
while (pending.reduce((x, p) => x + p.length, 0) < max && !done) fill();
|
|
1973
|
+
const all = Buffer.concat(pending);
|
|
1974
|
+
pending = all.length ? [all] : [];
|
|
1975
|
+
return all.subarray(0, max);
|
|
1976
|
+
}
|
|
1977
|
+
};
|
|
1978
|
+
}
|
|
1979
|
+
function zstdSupport() {
|
|
1980
|
+
if (typeof zlib.zstdDecompressSync === "function") return "node";
|
|
1981
|
+
try {
|
|
1982
|
+
execFileSync("zstd", ["--version"], { stdio: "ignore", timeout: 5e3 });
|
|
1983
|
+
return "cli";
|
|
1984
|
+
} catch {
|
|
1985
|
+
return null;
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
var zstdMode;
|
|
1989
|
+
function decompressZstd(file) {
|
|
1990
|
+
zstdMode ??= zstdSupport();
|
|
1991
|
+
if (zstdMode === "node") {
|
|
1992
|
+
try {
|
|
1993
|
+
return zlib.zstdDecompressSync(fs.readFileSync(file));
|
|
1994
|
+
} catch {
|
|
1995
|
+
throw new Error("zstd_corrupt");
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
if (zstdMode === "cli") {
|
|
1999
|
+
try {
|
|
2000
|
+
return execFileSync("zstd", ["-dcq", "--", file], { maxBuffer: 2 ** 32, stdio: ["ignore", "pipe", "ignore"] });
|
|
2001
|
+
} catch {
|
|
2002
|
+
throw new Error("zstd_corrupt");
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
throw new Error("zstd_unsupported");
|
|
2006
|
+
}
|
|
2007
|
+
function sniffHarness(head) {
|
|
2008
|
+
const firstEnd = head.indexOf(10);
|
|
2009
|
+
const first = head.toString("utf8", 0, Math.min(firstEnd < 0 ? head.length : firstEnd, 4096));
|
|
2010
|
+
if (first.includes('"session_meta"')) return "codex";
|
|
2011
|
+
if (/"type"\s*:\s*"session"/.test(first) && !first.includes('"sessionId"')) return "pi";
|
|
2012
|
+
const text = head.toString("latin1");
|
|
2013
|
+
if (text.includes('"sessionId"') && /"type"\s*:\s*"(?:user|assistant|summary|system)"/.test(text)) return "claude";
|
|
2014
|
+
return null;
|
|
2015
|
+
}
|
|
1850
2016
|
function resumeBuilder(previous, r, stat) {
|
|
1851
2017
|
const facts = {
|
|
1852
2018
|
...previous,
|
|
@@ -1855,7 +2021,7 @@ function resumeBuilder(previous, r, stat) {
|
|
|
1855
2021
|
usage: [...previous.usage],
|
|
1856
2022
|
prompts: previous.prompts.map((p) => ({ ...p })),
|
|
1857
2023
|
...previous.sessionIds ? { sessionIds: [...previous.sessionIds] } : {},
|
|
1858
|
-
...previous.story ? { story: { vocab:
|
|
2024
|
+
...previous.story ? { story: { vocab: null, topics: null } } : {}
|
|
1859
2025
|
};
|
|
1860
2026
|
delete facts.resume;
|
|
1861
2027
|
const b = makeBuilder(previous.file, previous.harness, stat.size, stat.mtimeMs);
|
|
@@ -1893,19 +2059,19 @@ function stripText(facts) {
|
|
|
1893
2059
|
return facts;
|
|
1894
2060
|
}
|
|
1895
2061
|
function tailHash(fd, offset) {
|
|
1896
|
-
const start = Math.max(0, offset -
|
|
2062
|
+
const start = Math.max(0, offset - TAIL2), buffer = Buffer.alloc(offset - start);
|
|
1897
2063
|
let read = 0;
|
|
1898
2064
|
while (read < buffer.length) {
|
|
1899
2065
|
const n = fs.readSync(fd, buffer, read, buffer.length - read, start + read);
|
|
1900
2066
|
if (!n) break;
|
|
1901
2067
|
read += n;
|
|
1902
2068
|
}
|
|
1903
|
-
return
|
|
2069
|
+
return createHash3("sha256").update(buffer.subarray(0, read)).digest("hex");
|
|
1904
2070
|
}
|
|
1905
|
-
var
|
|
2071
|
+
var TAIL2 = 4096;
|
|
1906
2072
|
|
|
1907
2073
|
// packages/core/src/analytics/scan-pool.ts
|
|
1908
|
-
var readFacts = (job) => stripText(extractFile(job.file, job.harness, { timezone: job.timezone }));
|
|
2074
|
+
var readFacts = (job) => stripText(extractFile(job.file, job.harness, { timezone: job.timezone, ...job.archive ? { archive: true } : {}, ...job.checks?.length ? { checks: job.checks } : {} }));
|
|
1909
2075
|
var errorCode = (error) => error instanceof Error && /^[a-z][a-z0-9_]{0,63}$/.test(error.message) ? error.message : "source_unreadable";
|
|
1910
2076
|
function runScanWorker() {
|
|
1911
2077
|
if (!parentPort) throw new Error("scan_worker_requires_parent");
|