worktrust 0.10.17 → 0.10.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/chat-exports.mjs CHANGED
@@ -4,26 +4,35 @@
4
4
  * `conversations.json` in it. The counter reads it for signals; the session hook reads it as HISTORY, so a person's
5
5
  * years of browser chats appear on their record. One reader here, imported by both, so the two cannot drift.
6
6
  *
7
+ * READ AS A STREAM (0.10.18, owner 2026-10-17: a real 375 MB export was refused at the old 64 MB ceiling, and some
8
+ * exports run past 10 GB). The file is never read whole: its bytes pass through a small JSON walk of our own that keeps,
9
+ * per conversation, only the fields the two readers use (the conversation's id and clock; per message the role, the
10
+ * clock, ChatGPT's model name and, for ChatGPT, the tree's parent links to find the kept branch) and skips every other
11
+ * value as it passes, never building it. A zip entry is inflated as a stream the same way. One conversation lives in
12
+ * memory at a time; a conversation whose kept fields outgrow their budget is left out and counted, never the export.
13
+ * The counter alone asks for the messages' text (`text: true`), read on this computer for its rubric as it always was;
14
+ * the hook never does. A title is read for the --exclude test alone and dropped there.
15
+ *
7
16
  * WHAT IT NEVER DOES. It opens nothing of the zip but `conversations.json` (a vendor's export carries the account's
8
17
  * profile, memories and files beside it: those are never inflated). Claude's split export (2026-10-17: a manifest and one
9
18
  * zip per category) is read through its `conversations-NNN.zip` parts alone, in part order; the manifest only names them,
10
- * its links are never fetched, and every other category's zip is refused by name. It hands the text to the counter's rubric on this
11
- * computer, as it always did, and to the hook it hands clocks, roles and a model name only: the conversation's title
12
- * and every message stay here. Hours from an export are RECONSTRUCTED, never measured: the file can be edited before
13
- * it is read and no device key witnessed it, so the record shows them with their origin and never in verified hours.
19
+ * its links are never fetched, and every other category's zip is refused by name. To the hook it hands clocks, roles and
20
+ * a model name only: the conversation's title and every message stay here. Hours from an export are RECONSTRUCTED, never
21
+ * measured: the file can be edited before it is read and no device key witnessed it, so the record shows them with their
22
+ * origin and never in verified hours.
14
23
  *
15
- * readExport(path) { vendor, conversations, digest } a folder holding conversations.json or conversations
16
- * parts, the file, the zip, one conversations-NNN.zip part, or Claude's manifest json
17
- * exportSessions(path, …) one session per conversation, in the transcript's own shape (type, timestamp, message)
24
+ * readExport(path) Promise of { vendor, conversations, digest }: every conversation's kept fields, bounded
25
+ * exportSessions(path, …) async: one session per conversation as it streams, in the transcript's own shape
18
26
  * exportRef(vendor, id, day) the stretch's stable name: the same on any computer, so a second send is the same line
19
27
  * exportRunRef(vendor, id, day, index) one run's name, when the hook cuts a conversation-day into runs (0.10.17)
20
- * findExports(dir) the exports lying in one folder (its top level only), by path and vendor
28
+ * findExports(dir) Promise of the exports lying in one folder (its top level only), by path and vendor
21
29
  */
22
30
  import { createHash } from "node:crypto";
31
+ import { once } from "node:events";
23
32
  import { existsSync, openSync, readSync, closeSync, fstatSync, readdirSync, statSync, constants } from "node:fs";
24
33
  import { homedir } from "node:os";
25
34
  import { basename, dirname, join } from "node:path";
26
- import { inflateRawSync } from "node:zlib";
35
+ import { createInflateRaw } from "node:zlib";
27
36
 
28
37
  /** The two vendors, by the client name the door reads as their product (chatgpt → chatgpt_web, claude-web → claude_web). */
29
38
  export const EXPORT_CLIENTS = ["chatgpt", "claude-web"];
@@ -34,11 +43,14 @@ export const exportProductName = (vendor) => PRODUCT_NAMES[vendor] ?? vendor;
34
43
  const FILE = "conversations.json";
35
44
  const expand = (path) => String(path).replace(/^~(?=\/|$)/, homedir());
36
45
 
37
- // Bound allocations and parsing work before touching untrusted sizes. Large attachment-bearing ZIPs
38
- // need not fit in memory; only the directory and conversations entry are ever read. No limit override.
39
- export const EXPORT_LIMITS = Object.freeze({ fileBytes: 2 * 1024 ** 3, directoryBytes: 8 * 1024 ** 2,
40
- jsonBytes: 64 * 1024 ** 2, compressedBytes: 64 * 1024 ** 2, depth: 64, structure: 1_000_000, conversations: 50_000,
41
- parts: 64, manifestBytes: 1024 ** 2 });
46
+ // THE BOUNDS (0.10.18). A file, bare or a zip, up to 64 GB, read as a stream; a zip's directory 8 MB; up to 64 parts; a
47
+ // manifest 1 MB; nesting 64 deep; up to 2,000,000 conversations. Per conversation the KEPT fields (never the file's bytes)
48
+ // up to 8 MB (the counter's text, read here alone, 64 MB): past that the conversation is left out and counted. An export
49
+ // held whole (readExport) stops at 512 MB of kept fields. A zip64 archive (one over 4 GB) is refused by name: unpack it
50
+ // and give conversations.json itself. No limit override.
51
+ export const EXPORT_LIMITS = Object.freeze({ fileBytes: 64 * 1024 ** 3, directoryBytes: 8 * 1024 ** 2, depth: 64, conversations: 2_000_000,
52
+ conversationBytes: 8 * 1024 ** 2, conversationTextBytes: 64 * 1024 ** 2, keptBytes: 512 * 1024 ** 2, keyBytes: 256, parts: 64, manifestBytes: 1024 ** 2 });
53
+ const PROGRESS_BYTES = 64 * 1024 ** 2, CHUNK_BYTES = 1024 ** 2;
42
54
  const limit = (condition, label) => { if (!condition) throw Error(`export limit exceeded: ${label}`); };
43
55
  const damaged = (condition) => { if (!condition) throw Error("a damaged zip archive"); };
44
56
  /** One descriptor, bounded reads: reject special files/short reads and bound concurrent file growth. */
@@ -50,23 +62,48 @@ function openExport(path, maximum) {
50
62
  limit(stat.size <= maximum, "file bytes");
51
63
  const readAt = (at, length, end = stat.size) => {
52
64
  damaged(Number.isSafeInteger(at) && Number.isSafeInteger(length) && at >= 0 && length >= 0 && at + length <= end && end <= stat.size);
53
- const buffer = Buffer.alloc(length);
65
+ const buffer = Buffer.allocUnsafe(length);
54
66
  let got = 0;
55
67
  while (got < length) { const count = readSync(fd, buffer, got, length - got, at + got); damaged(count > 0); got += count; }
56
68
  return buffer;
57
69
  };
58
- return { fd, size: stat.size, readAt };
70
+ /** The bytes from `at`, a megabyte at a time into ONE buffer (the walk copies what it keeps, so the resident set stays flat), under the same checks: a file that shrinks while it is read is damaged. */
71
+ const chunksOf = function* (at, length, end) {
72
+ const buffer = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, length));
73
+ for (let done = 0; done < length; done += CHUNK_BYTES) {
74
+ const count = Math.min(CHUNK_BYTES, length - done);
75
+ damaged(Number.isSafeInteger(at + done) && at >= 0 && at + done + count <= end && end <= stat.size);
76
+ let got = 0;
77
+ while (got < count) { const read = readSync(fd, buffer, got, count - got, at + done + got); damaged(read > 0); got += read; }
78
+ yield buffer.subarray(0, count);
79
+ }
80
+ };
81
+ return { fd, size: stat.size, readAt, chunksOf, close: () => closeSync(fd) };
59
82
  } catch (error) { closeSync(fd); throw error; }
60
83
  }
84
+ /** A deflated entry, inflated as it is read: never more than a chunk of it in memory, its length held to the directory's word. */
85
+ async function* inflated(chunks, uncompressed) {
86
+ const inflate = createInflateRaw();
87
+ let total = 0;
88
+ const feed = (async () => { for (const chunk of chunks) { if (inflate.destroyed) return; if (!inflate.write(chunk)) await once(inflate, "drain"); } inflate.end(); })().catch(() => null);
89
+ try {
90
+ for await (const piece of inflate) { total += piece.length; damaged(total <= uncompressed); yield piece; }
91
+ } catch (error) {
92
+ if (/^a damaged zip|^export limit/.test(error?.message ?? "")) throw error;
93
+ throw Error("a damaged zip archive or export limit exceeded: expanded entry bytes");
94
+ } finally { inflate.destroy(); await feed; }
95
+ damaged(total === uncompressed);
96
+ }
61
97
 
62
98
  /**
63
99
  * A ZIP, READ FOR ITS CONVERSATIONS ENTRIES. No dependency: the end-of-central-directory record is found from the back, the
64
- * central directory walked for the names `accept` takes, and only those entries' bytes inflated (stored or deflated;
65
- * anything else, and a zip64 archive, is refused by name), in part order. Nothing else in the archive is touched. Two
66
- * entries of one name are ambiguous and refused.
100
+ * central directory walked for the names `accept` takes, and only those entries' bytes read (stored or deflated, as a
101
+ * stream; anything else, and a zip64 archive, is refused by name), in part order. Nothing else in the archive is touched.
102
+ * Two entries of one name are ambiguous and refused. The descriptor stays open until `close`, so the bytes read are the
103
+ * bytes the directory described.
67
104
  */
68
105
  function zipEntries(path, accept) {
69
- const { fd, size, readAt } = openExport(path, EXPORT_LIMITS.fileBytes);
106
+ const { size, readAt, chunksOf, close } = openExport(path, EXPORT_LIMITS.fileBytes);
70
107
  try {
71
108
  const tailLength = Math.min(size, 65_557);
72
109
  const tail = readAt(size - tailLength, tailLength);
@@ -74,7 +111,7 @@ function zipEntries(path, accept) {
74
111
  for (let at = tail.length - 22; at >= 0; at -= 1) if (tail.readUInt32LE(at) === 0x06054b50 && at + 22 + tail.readUInt16LE(at + 20) === tail.length) { eocd = at; break; }
75
112
  if (eocd < 0) throw Error(`not a zip archive: ${path}`);
76
113
  const entries = tail.readUInt16LE(eocd + 10), directorySize = tail.readUInt32LE(eocd + 12), directoryAt = tail.readUInt32LE(eocd + 16);
77
- if (entries === 0xffff || directorySize === 0xffffffff || directoryAt === 0xffffffff) throw Error(`a zip64 archive is not read: ${path}`);
114
+ if (entries === 0xffff || directorySize === 0xffffffff || directoryAt === 0xffffffff) throw Error(`a zip64 archive is not read: ${path} (unpack it and give its ${FILE})`);
78
115
  if (tail.readUInt16LE(eocd + 4) !== 0 || tail.readUInt16LE(eocd + 6) !== 0) throw Error("unsupported zip: multiple disks");
79
116
  damaged(tail.readUInt16LE(eocd + 8) === entries);
80
117
  limit(directorySize <= EXPORT_LIMITS.directoryBytes, "zip directory bytes");
@@ -91,35 +128,27 @@ function zipEntries(path, accept) {
91
128
  const name = directory.toString("utf8", at + 46, at + 46 + nameLength);
92
129
  at += 46 + nameLength + extraLength + commentLength;
93
130
  if (!accept(basename(name)) || name.endsWith("/")) continue;
94
- if (compressed === 0xffffffff || uncompressed === 0xffffffff || localAt === 0xffffffff) throw Error(`a zip64 entry is not read: ${path}`);
95
- limit(compressed <= EXPORT_LIMITS.compressedBytes, "compressed entry bytes");
96
- limit(uncompressed <= EXPORT_LIMITS.jsonBytes, "expanded entry bytes");
131
+ if (compressed === 0xffffffff || uncompressed === 0xffffffff || localAt === 0xffffffff) throw Error(`a zip64 entry is not read: ${path} (unpack it and give its ${FILE})`);
97
132
  if ((flags & (1 | 64)) !== 0 || disk !== 0) throw Error("unsupported zip: encrypted or multi-disk entry");
98
133
  damaged(!selected.has(basename(name)));
99
134
  selected.set(basename(name), { name, method, flags, compressed, uncompressed, localAt });
100
135
  }
101
136
  damaged(at === directory.length);
102
137
  limit(selected.size <= EXPORT_LIMITS.parts, "conversations parts");
103
- let total = 0;
104
- return [...selected.keys()].sort(byPart).map((key) => {
138
+ const parts = [...selected.keys()].sort(byPart).map((key) => {
105
139
  const { name, method, flags, compressed, uncompressed, localAt } = selected.get(key);
106
- total += uncompressed;
107
- limit(total <= EXPORT_LIMITS.jsonBytes, "expanded entry bytes");
108
140
  const local = readAt(localAt, 30, directoryAt);
109
141
  damaged(local.readUInt32LE(0) === 0x04034b50 && local.readUInt16LE(8) === method && local.readUInt16LE(6) === flags);
110
142
  const nameLength = local.readUInt16LE(26), dataAt = localAt + 30 + nameLength + local.readUInt16LE(28);
111
143
  damaged(readAt(localAt + 30, nameLength, directoryAt).toString("utf8") === name);
112
144
  if (!(flags & 8)) damaged(local.readUInt32LE(18) === compressed && local.readUInt32LE(22) === uncompressed);
113
- const bytes = readAt(dataAt, compressed, directoryAt);
114
- if (method === 0) { damaged(compressed === uncompressed); return bytes; }
115
- if (method !== 8) throw Error(`${basename(name)} in ${path} is compressed in a way this reader does not open (method ${method})`);
116
- let inflated;
117
- try { inflated = inflateRawSync(bytes, { maxOutputLength: Math.max(1, uncompressed) }); }
118
- catch { throw Error("a damaged zip archive or export limit exceeded: expanded entry bytes"); }
119
- damaged(inflated.length === uncompressed);
120
- return inflated;
145
+ damaged(dataAt + compressed <= directoryAt);
146
+ if (method === 0) damaged(compressed === uncompressed);
147
+ else if (method !== 8) throw Error(`${basename(name)} in ${path} is compressed in a way this reader does not open (method ${method})`);
148
+ return { path, bytes: uncompressed, chunks: () => (method === 0 ? chunksOf(dataAt, compressed, directoryAt) : inflated(chunksOf(dataAt, compressed, directoryAt), uncompressed)) };
121
149
  });
122
- } finally { closeSync(fd); }
150
+ return { parts, close };
151
+ } catch (error) { close(); throw error; }
123
152
  }
124
153
 
125
154
  /**
@@ -137,9 +166,9 @@ const refuseOther = (path) => { const other = OTHER_PART.exec(basename(path)); i
137
166
  /** One conversations zip: its conversations entries, or a refusal that names the zip. */
138
167
  function zipParts(path, given) {
139
168
  refuseOther(path);
140
- const parts = zipEntries(path, (name) => CONVERSATIONS_ENTRY.test(name));
141
- if (parts.length === 0) throw Error(`no ${FILE} in ${given}`);
142
- return parts.map((bytes) => ({ bytes, path }));
169
+ const { parts, close } = zipEntries(path, (name) => CONVERSATIONS_ENTRY.test(name));
170
+ if (parts.length === 0) { close(); throw Error(`no ${FILE} in ${given}`); }
171
+ return { parts, close };
143
172
  }
144
173
  /** A folder's conversations zips (conversations-000.zip, -001.zip …), in part order; nothing else of the folder is read. */
145
174
  const folderParts = (dir) => { let names = []; try { names = readdirSync(dir); } catch { return []; } return names.filter((name) => CONVERSATIONS_PART.test(name)).sort(byPart).map((name) => join(dir, name)); };
@@ -163,10 +192,19 @@ function manifestParts(manifest, path) {
163
192
  if (missing.length > 0) throw Error(`the manifest names conversations parts that are not beside it: ${missing.join(", ")} (download them into the same folder; the links in the manifest are never followed)`);
164
193
  return found;
165
194
  }
166
- /** Several parts together stay inside one export's budget. */
167
- const withinBudget = (parts) => { limit(parts.length <= EXPORT_LIMITS.parts && parts.reduce((sum, part) => sum + part.bytes.length, 0) <= EXPORT_LIMITS.jsonBytes * 4, "conversations parts"); return parts; };
195
+ /** Several zips' parts together: one export, inside one budget, every descriptor closed together. */
196
+ function zipsParts(zips, given) {
197
+ limit(zips.length <= EXPORT_LIMITS.parts, "conversations parts");
198
+ const opened = [];
199
+ try {
200
+ for (const zip of zips) opened.push(zipParts(zip, given));
201
+ const parts = opened.flatMap((one) => one.parts);
202
+ limit(parts.length <= EXPORT_LIMITS.parts && parts.reduce((sum, part) => sum + part.bytes, 0) <= EXPORT_LIMITS.fileBytes, "conversations parts");
203
+ return { parts, close: () => { for (const one of opened) one.close(); } };
204
+ } catch (error) { for (const one of opened) one.close(); throw error; }
205
+ }
168
206
 
169
- /** The bytes of every conversations part, wherever the given path keeps them: a folder, the file itself, a zip, a part, or the manifest. */
207
+ /** Every conversations part, wherever the given path keeps them (a folder, the file itself, a zip, a part, or the manifest), each a source of chunks. */
170
208
  function exportParts(given) {
171
209
  let path = expand(given);
172
210
  let stat; try { stat = statSync(path); } catch { throw Error(`export not found: ${given}`); }
@@ -175,117 +213,255 @@ function exportParts(given) {
175
213
  else {
176
214
  const zips = folderParts(path);
177
215
  if (zips.length === 0) throw Error(`no ${FILE} in ${given}`);
178
- limit(zips.length <= EXPORT_LIMITS.parts, "conversations parts");
179
- return withinBudget(zips.flatMap((zip) => zipParts(zip, given)));
216
+ return zipsParts(zips, given);
180
217
  }
181
218
  }
182
219
  if (/\.zip$/i.test(path)) return zipParts(path, given);
183
- const { fd, size, readAt } = openExport(path, EXPORT_LIMITS.jsonBytes);
184
- let bytes; try { bytes = readAt(0, size); } finally { closeSync(fd); }
185
- // A json holding an object, not an array of conversations, may be the vendor's manifest: read for its conversations parts alone.
186
- if (basename(path) !== FILE && size <= EXPORT_LIMITS.manifestBytes && /^\s*\{/.test(bytes.subarray(0, 64).toString("utf8"))) {
187
- boundJson(bytes);
188
- let manifest; try { manifest = JSON.parse(bytes.toString("utf8")); } catch { throw Error(`export unreadable: ${path}`); }
189
- const zips = manifestParts(manifest, path);
190
- limit(zips.length <= EXPORT_LIMITS.parts, "conversations parts");
191
- return withinBudget(zips.flatMap((zip) => zipParts(zip, given)));
192
- }
193
- return [{ bytes, path }];
194
- }
195
-
196
- /** Scan bytes before JSON.parse allocates an object graph; escaped quotes/brackets inside strings
197
- * are content, not structure. JSON.parse remains responsible for syntax and UTF-8 string handling. */
198
- function boundJson(bytes) {
199
- let quoted = false, escaped = false, depth = 0, structure = 0;
200
- for (const byte of bytes) {
201
- if (quoted) { if (escaped) escaped = false; else if (byte === 92) escaped = true; else if (byte === 34) quoted = false; continue; }
202
- if (byte === 34) quoted = true;
203
- else if (byte === 91 || byte === 123) { depth += 1; structure += 1; limit(depth <= EXPORT_LIMITS.depth, "JSON depth"); }
204
- else if (byte === 93 || byte === 125) depth -= 1;
205
- else if (byte === 44 || byte === 58) structure += 1;
206
- limit(structure <= EXPORT_LIMITS.structure, "JSON structure");
207
- }
208
- }
209
-
210
- /** Which vendor wrote these conversations, by their shape alone; mixed or unknown shapes are refused, never guessed. */
211
- export function vendorOf(conversations) {
212
- if (!Array.isArray(conversations) || conversations.length === 0) return null;
213
- const shapes = new Set(conversations.map((c) => (c && typeof c === "object" && c.mapping && typeof c.mapping === "object" ? "chatgpt" : c && typeof c === "object" && Array.isArray(c.chat_messages) ? "claude-web" : "unknown")));
214
- return shapes.size === 1 ? [...shapes][0] : null;
220
+ const file = openExport(path, EXPORT_LIMITS.fileBytes);
221
+ let manifest;
222
+ try {
223
+ // A json holding an object, not an array of conversations, may be the vendor's manifest: read for its conversations parts alone.
224
+ if (basename(path) === FILE || file.size > EXPORT_LIMITS.manifestBytes || !/^\s*\{/.test(file.readAt(0, Math.min(64, file.size)).toString("utf8"))) return { parts: [{ path, bytes: file.size, chunks: () => file.chunksOf(0, file.size, file.size) }], close: file.close };
225
+ try { manifest = JSON.parse(file.readAt(0, file.size).toString("utf8")); } catch { throw Error(`export unreadable: ${path}`); }
226
+ } catch (error) { file.close(); throw error; }
227
+ file.close();
228
+ return zipsParts(manifestParts(manifest, path), given);
215
229
  }
216
230
 
217
- /** One export, read: its vendor by shape, its conversations, and the digest of its bytes (what `sent` is remembered by). */
218
- export function readExport(given, expect = null) {
219
- const parts = exportParts(given), digest = createHash("sha256");
220
- const path = parts[0].path;
221
- const conversations = [];
222
- for (const { bytes, path: from } of parts) {
223
- boundJson(bytes);
224
- let some; try { some = JSON.parse(bytes.toString("utf8")); } catch { throw Error(`export unreadable: ${from}`); }
225
- if (!Array.isArray(some)) throw Error(`${from} must contain an array of conversations`);
226
- limit(conversations.length + some.length <= EXPORT_LIMITS.conversations, "conversation count");
227
- for (const conversation of some) conversations.push(conversation);
228
- digest.update(bytes); // one part: the digest of its bytes, as before; several: of their bytes in part order
229
- }
230
- const vendor = vendorOf(conversations);
231
- if (conversations.length > 0 && !vendor) throw Error(`${path} is neither a ChatGPT nor a Claude export (or mixes the two)`);
232
- if (expect && vendor && vendor !== expect) throw Error(`${path} is a ${exportProductName(vendor)} export, not ${exportProductName(expect)}`);
233
- return { vendor: vendor ?? expect, conversations, digest: digest.digest("hex"), path, parts: parts.length };
231
+ // ─── THE WALK: an incremental JSON tokenizer that keeps a plan's fields and skips the rest without building it ───
232
+ /** A kept field that held an object or array where a scalar was expected: String() and Number() of it read as JSON.parse's object would. */
233
+ const NONSCALAR = Object.freeze({});
234
+ /** An element whose kept fields outgrew their budget: handed over as this, counted, never built. */
235
+ const OVERSIZED = Symbol("oversized");
236
+ const S = "scalar", T = "text"; // a scalar kept as it is; a text kept as a string for the counter, as its byte length for the hook
237
+ const NODE = { keys: { parent: S, message: { keys: { author: { keys: { role: S } }, create_time: S, content: { keys: { parts: { items: T } } }, metadata: { keys: { model_slug: S, is_visually_hidden_from_conversation: S } } } } } };
238
+ const CONVERSATION = { keys: { title: S, name: S, conversation_id: S, id: S, uuid: S, create_time: S, created_at: S, current_node: S, mapping: { any: NODE },
239
+ chat_messages: { items: { keys: { sender: S, created_at: S, text: T, content: { items: { keys: { type: S, text: T } } } } } } } };
240
+ const malformed = () => { throw Object.assign(Error("malformed"), { malformed: true }); };
241
+ const NUMBER = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/;
242
+ const VALUE = 0, KEY = 1, COLON = 2, AFTER = 3, STRING = 4, TOKEN = 5, DONE = 6;
243
+ /**
244
+ * The walk over one part's bytes, chunk by chunk, in any chunking. The top level must be an array; each of its elements is
245
+ * handed to `onElement` as the fields of `plan` (a tree of `keys`, `any` for a dictionary, `items` for a list; `scalar`
246
+ * and `text` at the leaves), with what `budget` allows: past it the element's kept fields are dropped and it is handed over
247
+ * as OVERSIZED. Depth is bounded; a string's escapes and a number's shape are checked as JSON.parse would, kept or skipped.
248
+ */
249
+ function walker(plan, { text, budget, onElement }) {
250
+ const stack = []; // { out, plan, key, kind, fresh, root }: out is the kept container, null while skipping
251
+ let state = VALUE, used = 0, over = false, index = 0, started = false, vPlan = null, tok = "";
252
+ let sRaw = null, sLen = 0, sEsc = 0, sKey = false, sKeep = false, sEscaped = false;
253
+ const top = () => stack[stack.length - 1];
254
+ const oversize = () => { over = true; for (let i = 1; i < stack.length; i += 1) { stack[i].out = null; stack[i].plan = null; stack[i].key = null; } sRaw = null; sKeep = false; };
255
+ const spend = (cost) => { used += cost; if (used > budget) oversize(); };
256
+ const store = (value, cost) => {
257
+ const frame = top();
258
+ if (frame.root) { onElement(over ? OVERSIZED : value, index); index += 1; used = 0; over = false; }
259
+ else if (frame.out !== null) {
260
+ if (frame.out instanceof Map) { if (frame.key !== null) frame.out.set(frame.key, value); }
261
+ else if (Array.isArray(frame.out)) frame.out.push(value);
262
+ else if (frame.key !== null) frame.out[frame.key] = value;
263
+ spend(cost + 16);
264
+ }
265
+ frame.key = null; vPlan = null; state = AFTER;
266
+ };
267
+ const open = (kind) => {
268
+ limit(stack.length < EXPORT_LIMITS.depth, "JSON depth");
269
+ const root = stack.length === 0, wanted = root ? { items: plan } : vPlan && typeof vPlan === "object" ? vPlan : null;
270
+ const out = root ? null : kind === "{" && wanted?.keys ? {} : kind === "{" && wanted?.any ? new Map() : kind === "[" && wanted?.items ? [] : null;
271
+ stack.push({ out, plan: root || out !== null ? wanted : null, key: null, kind, fresh: true, root });
272
+ vPlan = kind === "[" ? top().plan?.items ?? null : null; state = kind === "{" ? KEY : VALUE;
273
+ };
274
+ const close = (kind) => { const frame = stack.pop(); if (frame.kind !== kind) malformed(); if (frame.root) state = DONE; else store(frame.out ?? NONSCALAR, 0); };
275
+ const beginString = (key) => { state = STRING; sKey = key; sKeep = key ? top().out !== null : vPlan === S || (vPlan === T && text); sRaw = sKeep ? [] : null; sLen = 0; sEsc = 0; sEscaped = false; };
276
+ /** Kept bytes of the string in flight: a key past its bound stops being one; a value past the budget takes the element with it. */
277
+ const take = (chunk, from, to) => {
278
+ sLen += to - from;
279
+ if (!sKeep) return;
280
+ if (sKey) { if (sLen > EXPORT_LIMITS.keyBytes) { sKeep = false; sRaw = null; return; } } else spend(to - from);
281
+ if (sKeep) sRaw.push(Buffer.from(chunk.subarray(from, to))); // a copy: the chunk's buffer is reused for the next read
282
+ };
283
+ const finishString = () => {
284
+ const raw = sRaw ? Buffer.concat(sRaw) : null;
285
+ sRaw = null;
286
+ const decode = () => (sEscaped ? JSON.parse(`"${raw.toString("utf8")}"`) : raw.toString("utf8"));
287
+ if (sKey) {
288
+ const frame = top(), key = raw ? decode() : null;
289
+ if (frame.plan?.any && key !== null) { frame.key = key; vPlan = frame.plan.any; spend(raw.length); }
290
+ else if (frame.plan?.keys && key !== null && Object.hasOwn(frame.plan.keys, key)) { frame.key = key; vPlan = frame.plan.keys[key]; }
291
+ else { frame.key = null; vPlan = null; }
292
+ state = COLON;
293
+ } else if (sKeep) store(decode(), 0);
294
+ else store(vPlan === T ? sLen : NONSCALAR, 8); // a text the hook never reads: its byte length says whether there was one
295
+ };
296
+ const finishToken = () => {
297
+ let value;
298
+ if (tok === "true") value = true; else if (tok === "false") value = false; else if (tok === "null") value = null;
299
+ else if (NUMBER.test(tok)) value = Number(tok); else malformed();
300
+ tok = ""; store(value, 8);
301
+ };
302
+ const push = (chunk) => {
303
+ const n = chunk.length;
304
+ for (let i = 0; i < n; i += 1) {
305
+ const b = chunk[i];
306
+ if (state === STRING) {
307
+ if (sEsc === 0) {
308
+ let j = i; // the common run: bytes that are neither a quote, a backslash nor a control character, taken at once
309
+ while (j < n && chunk[j] !== 34 && chunk[j] !== 92 && chunk[j] >= 32) j += 1;
310
+ if (j > i) { take(chunk, i, j); if (j === n) break; i = j; }
311
+ if (chunk[i] === 34) { finishString(); continue; }
312
+ if (chunk[i] !== 92) malformed();
313
+ sEsc = 1; sEscaped = true; take(chunk, i, i + 1); continue;
314
+ }
315
+ if (sEsc === 1) { if (b === 117) sEsc = 2; else if (b === 34 || b === 92 || b === 47 || b === 98 || b === 102 || b === 110 || b === 114 || b === 116) sEsc = 0; else malformed(); }
316
+ else { if (!((b >= 48 && b <= 57) || (b >= 65 && b <= 70) || (b >= 97 && b <= 102))) malformed(); sEsc = sEsc === 5 ? 0 : sEsc + 1; }
317
+ take(chunk, i, i + 1);
318
+ continue;
319
+ }
320
+ if (state === TOKEN) {
321
+ if ((b >= 48 && b <= 57) || b === 45 || b === 43 || b === 46 || (b >= 97 && b <= 122) || b === 69) { tok += String.fromCharCode(b); if (tok.length > 4096) malformed(); continue; }
322
+ finishToken();
323
+ }
324
+ if (b === 32 || b === 10 || b === 13 || b === 9) continue;
325
+ if (state === VALUE) {
326
+ if (!started) { started = true; if (b === 123) throw Object.assign(Error("not an array"), { notArray: true }); if (b !== 91) malformed(); }
327
+ const frame = top();
328
+ if (b === 93) { if (frame?.kind === "[" && frame.fresh) { close("["); continue; } malformed(); }
329
+ if (frame) frame.fresh = false;
330
+ if (b === 34) beginString(false);
331
+ else if (b === 123) open("{");
332
+ else if (b === 91) open("[");
333
+ else if ((b >= 48 && b <= 57) || b === 45 || b === 116 || b === 102 || b === 110) { state = TOKEN; tok = String.fromCharCode(b); }
334
+ else malformed();
335
+ } else if (state === KEY) {
336
+ const frame = top();
337
+ if (b === 34) { frame.fresh = false; beginString(true); }
338
+ else if (b === 125 && frame.fresh) close("{");
339
+ else malformed();
340
+ } else if (state === COLON) { if (b !== 58) malformed(); state = VALUE; }
341
+ else if (state === AFTER) {
342
+ const frame = top();
343
+ if (b === 44) { if (frame.kind === "{") state = KEY; else { state = VALUE; vPlan = frame.plan?.items ?? null; } }
344
+ else if (b === 93) close("[");
345
+ else if (b === 125) close("{");
346
+ else malformed();
347
+ } else malformed(); // DONE: nothing may follow the array
348
+ }
349
+ };
350
+ const end = () => { if (state === TOKEN) finishToken(); if (state !== DONE) malformed(); };
351
+ return { push, end };
234
352
  }
235
353
 
354
+ // ─── ONE CONVERSATION'S KEPT FIELDS → ITS LINES (the shapes the old whole-file reader produced, line for line) ───
236
355
  /** The kept branch of one ChatGPT conversation, oldest first: from `current_node` up through the parents. */
237
- function chatgptBranch(conversation) {
238
- const mapping = conversation?.mapping && typeof conversation.mapping === "object" ? conversation.mapping : {};
239
- const branch = [];
240
- const seen = new Set();
241
- for (let id = conversation?.current_node; id && mapping[id] && !seen.has(id); id = mapping[id].parent) { seen.add(id); branch.push(mapping[id]); }
356
+ function chatgptBranch(kept) {
357
+ const mapping = kept.mapping instanceof Map ? kept.mapping : new Map();
358
+ const branch = [], seen = new Set();
359
+ for (let id = kept.current_node; id && mapping.get(String(id)) && !seen.has(String(id)); id = mapping.get(String(id))?.parent) { seen.add(String(id)); branch.push(mapping.get(String(id))); }
242
360
  return branch.reverse();
243
361
  }
244
362
  /** A ChatGPT conversation's lines: the person and the model only; system prompts, tools and what the app hides are neither. */
245
- function chatgptLines(conversation) {
363
+ function chatgptLines(kept, text) {
246
364
  const lines = [];
247
- for (const node of chatgptBranch(conversation)) {
248
- const m = node?.message;
249
- const role = m?.author?.role;
365
+ for (const node of chatgptBranch(kept)) {
366
+ const m = node?.message, role = m?.author?.role;
250
367
  if ((role !== "user" && role !== "assistant") || m?.metadata?.is_visually_hidden_from_conversation) continue;
251
- const parts = Array.isArray(m?.content?.parts) ? m.content.parts.filter((part) => typeof part === "string") : [];
252
- const text = parts.join("\n");
253
- if (text.length === 0) continue;
254
- const seconds = Number(m?.create_time ?? conversation?.create_time);
368
+ const parts = Array.isArray(m?.content?.parts) ? m.content.parts : [];
369
+ // With the text: the string parts joined. Without: the parts' byte lengths say whether the join is empty.
370
+ let body;
371
+ if (text) { body = parts.filter((part) => typeof part === "string").join("\n"); if (body.length === 0) continue; }
372
+ else { const lengths = parts.filter((part) => typeof part === "number"); if (lengths.length === 0 || (lengths.length === 1 && lengths[0] === 0)) continue; }
373
+ const seconds = Number(m?.create_time ?? kept.create_time);
255
374
  const at = Number.isFinite(seconds) && seconds > 0 ? new Date(seconds * 1000).toISOString() : null;
256
375
  const model = typeof m?.metadata?.model_slug === "string" ? m.metadata.model_slug : undefined;
257
- lines.push({ type: role === "user" ? "user" : "assistant", timestamp: at, message: { ...(role === "assistant" && model ? { model } : {}), content: [{ type: "text", text }] } });
376
+ lines.push({ type: role === "user" ? "user" : "assistant", timestamp: at, ...(role === "assistant" && model ? { model } : {}), ...(text ? { text: body } : {}) });
258
377
  }
259
378
  return lines;
260
379
  }
261
- /** A claude.ai conversation's messages in the transcript's own shape; the export names no model. */
262
- function claudeLines(conversation) {
263
- const messages = Array.isArray(conversation?.chat_messages) ? conversation.chat_messages : [];
380
+ /** A claude.ai conversation's messages; the export names no model. */
381
+ function claudeLines(kept, text) {
382
+ const messages = Array.isArray(kept.chat_messages) ? kept.chat_messages : [];
264
383
  return messages.map((m) => {
265
- const text = typeof m?.text === "string" && m.text.length > 0 ? m.text : (Array.isArray(m?.content) ? m.content.filter((c) => c?.type === "text" && typeof c.text === "string").map((c) => c.text).join("\n") : "");
266
- const at = m?.created_at ?? conversation?.created_at ?? null;
267
- return { type: m?.sender === "human" ? "user" : "assistant", timestamp: at, message: { content: [{ type: "text", text }] } };
384
+ const body = !text ? undefined : typeof m?.text === "string" && m.text.length > 0 ? m.text : (Array.isArray(m?.content) ? m.content.filter((c) => c?.type === "text" && typeof c.text === "string").map((c) => c.text).join("\n") : "");
385
+ return { type: m?.sender === "human" ? "user" : "assistant", timestamp: m?.created_at ?? kept.created_at ?? null, ...(text ? { text: body } : {}) };
268
386
  });
269
387
  }
270
- const titleOf = (vendor, conversation) => String((vendor === "chatgpt" ? conversation?.title : conversation?.name) ?? "");
271
- const idOf = (vendor, conversation, index) => String((vendor === "chatgpt" ? conversation?.conversation_id ?? conversation?.id : conversation?.uuid) ?? `#${index}`);
388
+ /** A kept line in the transcript's own shape, as the counter and the hook read every client's lines. */
389
+ const transcriptLine = ({ type, timestamp, model, text }) => ({ type, timestamp, message: { ...(model ? { model } : {}), content: [{ type: "text", text: text ?? "" }] } });
390
+
391
+ /**
392
+ * THE EXPORT, ONE CONVERSATION AT A TIME: each part's bytes through the walk, the digest taken as they pass, the vendor
393
+ * told from the first conversation's shape and held to (a second shape, or an unknown one, refuses the export; `expect`
394
+ * refuses the other vendor by name), the title read for `excluded` and dropped there, a conversation whose kept fields
395
+ * outgrow their budget counted and left out. Yields one { vendor, id, lines } per conversation (an excluded or oversized
396
+ * one without an id, marked); `report` receives the path and vendor before the first conversation, the digest, the counts
397
+ * and the number of parts when the stream ends. Counts only on stderr.
398
+ */
399
+ async function* exportRecords(given, { expect = null, excluded = () => false, skipped = null, text = false, report = null } = {}) {
400
+ const { parts, close } = exportParts(given);
401
+ const path = parts[0].path, digest = createHash("sha256");
402
+ let vendor = null, count = 0, oversized = 0, bytes = 0, next = PROGRESS_BYTES, offset = 0;
403
+ if (report) { report.path = path; report.vendor = expect; report.digest = null; report.conversations = 0; report.oversized = 0; report.parts = parts.length; }
404
+ const one = (kept, index) => {
405
+ limit(count < EXPORT_LIMITS.conversations, "conversation count");
406
+ count += 1;
407
+ if (kept === OVERSIZED) { oversized += 1; return { vendor, oversized: true, lines: [] }; }
408
+ const shape = kept && typeof kept === "object" && kept !== NONSCALAR ? (kept.mapping instanceof Map || kept.mapping === NONSCALAR ? "chatgpt" : Array.isArray(kept.chat_messages) ? "claude-web" : "unknown") : "unknown";
409
+ if (vendor === null) vendor = shape;
410
+ if (shape === "unknown" || shape !== vendor) throw Error(`${path} is neither a ChatGPT nor a Claude export (or mixes the two)`);
411
+ if (expect && vendor !== expect) throw Error(`${path} is a ${exportProductName(vendor)} export, not ${exportProductName(expect)}`);
412
+ if (report) report.vendor = vendor;
413
+ const title = String((vendor === "chatgpt" ? kept.title : kept.name) ?? "");
414
+ if (excluded(title)) { skipped?.push(title.slice(0, 40)); return { vendor, excluded: true, lines: [] }; }
415
+ return { vendor, id: String((vendor === "chatgpt" ? kept.conversation_id ?? kept.id : kept.uuid) ?? `#${index}`), lines: vendor === "chatgpt" ? chatgptLines(kept, text) : claudeLines(kept, text) };
416
+ };
417
+ try {
418
+ for (const part of parts) {
419
+ let queue = [], seen = 0;
420
+ const walk = walker(CONVERSATION, { text, budget: text ? EXPORT_LIMITS.conversationTextBytes : EXPORT_LIMITS.conversationBytes, onElement: (kept, index) => queue.push([kept, offset + index]) });
421
+ const drain = function* () { const due = queue; queue = []; for (const [kept, index] of due) { seen = index + 1 - offset; yield one(kept, index); } };
422
+ try {
423
+ for await (const chunk of part.chunks()) {
424
+ digest.update(chunk); bytes += chunk.length; walk.push(chunk);
425
+ if (bytes >= next) { process.stderr.write(`export: ${Math.round(bytes / 1024 ** 2)} MB read, ${count} conversations\n`); next += PROGRESS_BYTES; }
426
+ yield* drain();
427
+ }
428
+ walk.end();
429
+ yield* drain();
430
+ } catch (error) {
431
+ if (error?.malformed) throw Error(`export unreadable: ${part.path}`);
432
+ if (error?.notArray) throw Error(`${part.path} must contain an array of conversations`);
433
+ throw error;
434
+ }
435
+ offset += seen;
436
+ }
437
+ } finally { close(); }
438
+ if (report) { report.vendor = vendor ?? expect; report.digest = digest.digest("hex"); report.conversations = count; report.oversized = oversized; }
439
+ }
440
+
441
+ /** One export, read whole but compact: its vendor by shape, one record per conversation (its kept lines; none for an excluded or oversized one), and the digest of its bytes (what `sent` is remembered by). */
442
+ export async function readExport(given, expect = null, { excluded = () => false, skipped = null, text = false } = {}) {
443
+ const report = {}, conversations = [];
444
+ let kept = 0;
445
+ for await (const record of exportRecords(given, { expect, excluded, skipped, text, report })) {
446
+ kept += 64 + (record.id?.length ?? 0) + record.lines.reduce((sum, line) => sum + 64 + (line.model?.length ?? 0) + (line.text?.length ?? 0), 0);
447
+ limit(kept <= EXPORT_LIMITS.keptBytes, "kept bytes");
448
+ conversations.push(record);
449
+ }
450
+ return { vendor: report.vendor, conversations, digest: report.digest, path: report.path, parts: report.parts, oversized: report.oversized };
451
+ }
272
452
 
273
453
  /**
274
- * Every conversation of an export as one session: `key` names the vendor and the conversation's id, `lines` carry
275
- * the transcript shape both readers measure. `excluded(title)` skips a conversation by its title before anything of
276
- * it is read (the same `--exclude` as a project directory); the title itself goes no further than that call.
277
- * `given` is a path, or an export `readExport` already returned (so a zip is inflated once).
454
+ * Every conversation of an export as one session, as it streams: `key` names the vendor and the conversation's id,
455
+ * `lines` carry the transcript shape both readers measure. `excluded(title)` skips a conversation by its title before
456
+ * anything else of it is kept (the same `--exclude` as a project directory); the title itself goes no further than that
457
+ * call. `text` hands the counter the messages' text (read here, for its rubric); the hook leaves it out and gets clocks,
458
+ * roles and a model name. `given` is a path, or an export `readExport` already returned; `report` is filled as above.
278
459
  */
279
- export function* exportSessions(given, { expect = null, excluded = () => false, skipped = null } = {}) {
280
- const { vendor, conversations } = typeof given === "string" ? readExport(given, expect) : given;
281
- for (let index = 0; index < conversations.length; index += 1) {
282
- const conversation = conversations[index];
283
- const title = titleOf(vendor, conversation);
284
- if (excluded(title)) { skipped?.push(title.slice(0, 40)); continue; }
285
- const lines = vendor === "chatgpt" ? chatgptLines(conversation) : claudeLines(conversation);
286
- if (lines.length === 0) continue;
287
- const id = idOf(vendor, conversation, index);
288
- yield { key: `${vendor === "chatgpt" ? "chatgpt" : "claude.ai"}:${id}`, projectDir: vendor === "chatgpt" ? "chatgpt" : "claude-ai", basename: vendor === "chatgpt" ? "chatgpt" : "claude.ai", vendor, id, lines };
460
+ export async function* exportSessions(given, { expect = null, excluded = () => false, skipped = null, text = false, report = null } = {}) {
461
+ const records = typeof given === "string" ? exportRecords(given, { expect, excluded, skipped, text, report }) : given.conversations;
462
+ for await (const { vendor, id, lines } of records) {
463
+ if (lines.length === 0) continue; // excluded, left out as too large, or without a message
464
+ yield { key: `${vendor === "chatgpt" ? "chatgpt" : "claude.ai"}:${id}`, projectDir: vendor === "chatgpt" ? "chatgpt" : "claude-ai", basename: vendor === "chatgpt" ? "chatgpt" : "claude.ai", vendor, id, lines: lines.map(transcriptLine) };
289
465
  }
290
466
  }
291
467
 
@@ -308,7 +484,7 @@ export const exportRunRef = (vendor, id, day, index) => createHash("sha256").upd
308
484
  * `conversations.json`, a folder holding one, or the file itself. Each is read to tell its vendor and digest; a file
309
485
  * that is none of these, or not an export, is passed over without a word.
310
486
  */
311
- export function findExports(dir = join(homedir(), "Downloads")) {
487
+ export async function findExports(dir = join(homedir(), "Downloads")) {
312
488
  const root = expand(dir);
313
489
  let names = []; try { names = readdirSync(root); } catch { return []; }
314
490
  const found = [];
@@ -318,7 +494,7 @@ export function findExports(dir = join(homedir(), "Downloads")) {
318
494
  const candidate = stat.isDirectory() ? existsSync(join(path, FILE)) || folderParts(path).length > 0 : (/\.zip$/i.test(name) && !OTHER_PART.test(name)) || name === FILE;
319
495
  if (!candidate) continue;
320
496
  try {
321
- const { vendor, conversations, digest } = readExport(path);
497
+ const { vendor, conversations, digest } = await readExport(path);
322
498
  if (vendor && conversations.length > 0) found.push({ path, vendor, digest, conversations: conversations.length });
323
499
  } catch { /* not an export, or not one this reader opens */ }
324
500
  }
package/claude-mod.mjs CHANGED
@@ -146,7 +146,7 @@ export function register(on) {
146
146
  }
147
147
  `;
148
148
 
149
- export const PLUGIN_JSON = { name: MOD_NAME, version: MOD_VERSION, description: "WorkTrust: counts how you work with Claude Code (permission answers, effort levels, context peaks, subagents), never a word of the session. Counts stay on this computer until the WorkTrust CLI reports a session.", author: { name: "WorkTrust", url: "https://worktrust.io" }, homepage: "https://worktrust.io", license: "MIT" };
149
+ export const PLUGIN_JSON = { name: MOD_NAME, version: MOD_VERSION, description: "WorkTrust: counts how you work with Claude Code (permission answers, effort levels, context peaks, subagents), never a word of the session. Counts stay on this computer until the WorkTrust CLI reports a session.", author: { name: "WorkTrust", url: "https://worktrust.io" }, homepage: "https://worktrust.io", license: "LicenseRef-WorkTrust-CLI" };
150
150
  export const HOOKS_JSON = { modules: ["./register.js"] };
151
151
 
152
152
  const dirs = (home) => ({ plugin: join(home, ".worktrust", "claude-mod"), counts: join(home, ".worktrust", "claude-mod-counts"), settings: join(home, ".claude", "settings.json") });