xtctx 0.11.1 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/src/cli/hook.d.ts.map +1 -1
- package/dist/src/cli/hook.js +25 -14
- package/dist/src/cli/hook.js.map +1 -1
- package/dist/src/cli/index.d.ts.map +1 -1
- package/dist/src/cli/index.js +16 -1
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/cli/setup.d.ts.map +1 -1
- package/dist/src/cli/setup.js +1 -2
- package/dist/src/cli/setup.js.map +1 -1
- package/dist/src/cli/status.d.ts.map +1 -1
- package/dist/src/cli/status.js +54 -5
- package/dist/src/cli/status.js.map +1 -1
- package/dist/src/config/disconnect.d.ts +2 -0
- package/dist/src/config/disconnect.d.ts.map +1 -1
- package/dist/src/config/disconnect.js +109 -27
- package/dist/src/config/disconnect.js.map +1 -1
- package/dist/src/config/managed-block.d.ts +29 -0
- package/dist/src/config/managed-block.d.ts.map +1 -0
- package/dist/src/config/managed-block.js +67 -0
- package/dist/src/config/managed-block.js.map +1 -0
- package/dist/src/config/mcp-config.d.ts +2 -0
- package/dist/src/config/mcp-config.d.ts.map +1 -1
- package/dist/src/config/mcp-config.js +101 -7
- package/dist/src/config/mcp-config.js.map +1 -1
- package/dist/src/config/setup.d.ts +2 -0
- package/dist/src/config/setup.d.ts.map +1 -1
- package/dist/src/config/setup.js +78 -37
- package/dist/src/config/setup.js.map +1 -1
- package/dist/src/config/skills.d.ts.map +1 -1
- package/dist/src/config/skills.js +4 -4
- package/dist/src/config/skills.js.map +1 -1
- package/dist/src/handoff/embeddings.d.ts +11 -0
- package/dist/src/handoff/embeddings.d.ts.map +1 -1
- package/dist/src/handoff/embeddings.js +74 -5
- package/dist/src/handoff/embeddings.js.map +1 -1
- package/dist/src/handoff/sqlite-index.d.ts +9 -0
- package/dist/src/handoff/sqlite-index.d.ts.map +1 -1
- package/dist/src/handoff/sqlite-index.js +285 -98
- package/dist/src/handoff/sqlite-index.js.map +1 -1
- package/dist/src/handoff/types.d.ts +6 -0
- package/dist/src/handoff/types.d.ts.map +1 -1
- package/dist/src/mcp/server.d.ts +1 -1
- package/dist/src/mcp/server.d.ts.map +1 -1
- package/dist/src/mcp/server.js +17 -8
- package/dist/src/mcp/server.js.map +1 -1
- package/dist/src/mcp/tools/continuity.d.ts +19 -1
- package/dist/src/mcp/tools/continuity.d.ts.map +1 -1
- package/dist/src/mcp/tools/continuity.js +27 -2
- package/dist/src/mcp/tools/continuity.js.map +1 -1
- package/dist/src/mcp/tools/manifest.d.ts.map +1 -1
- package/dist/src/mcp/tools/manifest.js +15 -6
- package/dist/src/mcp/tools/manifest.js.map +1 -1
- package/dist/src/mcp/tools/sessions.d.ts +9 -1
- package/dist/src/mcp/tools/sessions.d.ts.map +1 -1
- package/dist/src/mcp/tools/sessions.js +68 -9
- package/dist/src/mcp/tools/sessions.js.map +1 -1
- package/dist/src/runtime/services.d.ts.map +1 -1
- package/dist/src/runtime/services.js +22 -2
- package/dist/src/runtime/services.js.map +1 -1
- package/dist/src/scrapers/antigravity.d.ts +11 -0
- package/dist/src/scrapers/antigravity.d.ts.map +1 -1
- package/dist/src/scrapers/antigravity.js +47 -36
- package/dist/src/scrapers/antigravity.js.map +1 -1
- package/dist/src/scrapers/base.d.ts.map +1 -1
- package/dist/src/scrapers/base.js +4 -1
- package/dist/src/scrapers/base.js.map +1 -1
- package/dist/src/scrapers/claude-code.d.ts +17 -0
- package/dist/src/scrapers/claude-code.d.ts.map +1 -1
- package/dist/src/scrapers/claude-code.js +115 -59
- package/dist/src/scrapers/claude-code.js.map +1 -1
- package/dist/src/scrapers/codex.d.ts.map +1 -1
- package/dist/src/scrapers/codex.js +166 -153
- package/dist/src/scrapers/codex.js.map +1 -1
- package/dist/src/scrapers/copilot-cli.d.ts +2 -1
- package/dist/src/scrapers/copilot-cli.d.ts.map +1 -1
- package/dist/src/scrapers/copilot-cli.js +88 -3
- package/dist/src/scrapers/copilot-cli.js.map +1 -1
- package/dist/src/scrapers/cursor.d.ts.map +1 -1
- package/dist/src/scrapers/cursor.js +20 -9
- package/dist/src/scrapers/cursor.js.map +1 -1
- package/dist/src/scrapers/index.d.ts +7 -2
- package/dist/src/scrapers/index.d.ts.map +1 -1
- package/dist/src/scrapers/index.js +7 -2
- package/dist/src/scrapers/index.js.map +1 -1
- package/dist/src/scrapers/opencode.d.ts +2 -1
- package/dist/src/scrapers/opencode.d.ts.map +1 -1
- package/dist/src/scrapers/opencode.js +38 -9
- package/dist/src/scrapers/opencode.js.map +1 -1
- package/dist/src/tools/sources.d.ts.map +1 -1
- package/dist/src/tools/sources.js +2 -2
- package/dist/src/tools/sources.js.map +1 -1
- package/dist/src/utils/atomic-file.d.ts +8 -0
- package/dist/src/utils/atomic-file.d.ts.map +1 -0
- package/dist/src/utils/atomic-file.js +15 -0
- package/dist/src/utils/atomic-file.js.map +1 -0
- package/dist/src/utils/errors.d.ts +5 -0
- package/dist/src/utils/errors.d.ts.map +1 -1
- package/dist/src/utils/errors.js +9 -0
- package/dist/src/utils/errors.js.map +1 -1
- package/dist/src/utils/project-scope.d.ts +16 -0
- package/dist/src/utils/project-scope.d.ts.map +1 -1
- package/dist/src/utils/project-scope.js +18 -2
- package/dist/src/utils/project-scope.js.map +1 -1
- package/package.json +3 -3
|
@@ -3,7 +3,7 @@ import { join } from "node:path";
|
|
|
3
3
|
import { createReadStream } from "node:fs";
|
|
4
4
|
import { createInterface } from "node:readline";
|
|
5
5
|
import { AbstractScraper, estimateTokens } from "./base.js";
|
|
6
|
-
import { encodePathForToolDirectory } from "../utils/project-scope.js";
|
|
6
|
+
import { encodePathForToolDirectory, pathMatchesProject } from "../utils/project-scope.js";
|
|
7
7
|
const SCRAPER_NAME = "claude-code";
|
|
8
8
|
/**
|
|
9
9
|
* Mutation shapes the claude-code scraper tolerates silently (no warn, no
|
|
@@ -70,7 +70,7 @@ export class ClaudeCodeScraper extends AbstractScraper {
|
|
|
70
70
|
}
|
|
71
71
|
parseRaw(raw) {
|
|
72
72
|
const obj = raw;
|
|
73
|
-
const timestamp =
|
|
73
|
+
const timestamp = parseRecordTimestamp(obj.timestamp);
|
|
74
74
|
const content = extractContent(obj);
|
|
75
75
|
const type = obj.type || "unknown";
|
|
76
76
|
const role = extractRole(obj, type);
|
|
@@ -99,8 +99,12 @@ export class ClaudeCodeScraper extends AbstractScraper {
|
|
|
99
99
|
// ACCEPTED_DEGRADATIONS.missingProjectsDir
|
|
100
100
|
return;
|
|
101
101
|
}
|
|
102
|
+
const encodedProject = this.projectRoot
|
|
103
|
+
? encodePathForToolDirectory(this.projectRoot).toLowerCase()
|
|
104
|
+
: null;
|
|
102
105
|
for (const projectHash of this.filterProjectDirs(projectDirs)) {
|
|
103
106
|
const projectDir = join(this.claudeProjectsDir, projectHash);
|
|
107
|
+
const exactDirectory = encodedProject === null || projectHash.toLowerCase() === encodedProject;
|
|
104
108
|
let files;
|
|
105
109
|
try {
|
|
106
110
|
files = await readdir(projectDir);
|
|
@@ -112,67 +116,93 @@ export class ClaudeCodeScraper extends AbstractScraper {
|
|
|
112
116
|
if (!file.endsWith(".jsonl")) {
|
|
113
117
|
continue;
|
|
114
118
|
}
|
|
115
|
-
const sessionId = file.replace(
|
|
119
|
+
const sessionId = file.replace(/\.jsonl$/, "");
|
|
116
120
|
const filePath = join(projectDir, file);
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
crlfDelay: Infinity,
|
|
120
|
-
});
|
|
121
|
-
let messageIndex = 0;
|
|
122
|
-
let lineNo = 0;
|
|
123
|
-
for await (const line of reader) {
|
|
124
|
-
lineNo++;
|
|
125
|
-
if (!line.trim()) {
|
|
126
|
-
continue;
|
|
127
|
-
}
|
|
128
|
-
let obj;
|
|
129
|
-
try {
|
|
130
|
-
obj = JSON.parse(line);
|
|
131
|
-
}
|
|
132
|
-
catch (err) {
|
|
133
|
-
// ACCEPTED_DEGRADATIONS.malformedJsonlLine, but warn so the
|
|
134
|
-
// mutation test can see drift instead of data silently vanishing.
|
|
135
|
-
warnDrift(`${filePath}:${lineNo}`, `line is not valid JSON: ${err.message}`, 1);
|
|
136
|
-
continue;
|
|
137
|
-
}
|
|
138
|
-
if (NON_MESSAGE_TYPES.has(String(obj.type))) {
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
// Strict-mode schema check: warn (but still emit) on shape surprise.
|
|
142
|
-
if (!("type" in obj)) {
|
|
143
|
-
warnDrift(`${filePath}:${lineNo}`, "record is missing required 'type' field — likely renamed", 1);
|
|
144
|
-
}
|
|
145
|
-
else if (typeof obj.type !== "string" || !(obj.type in ROLE_MAP)) {
|
|
146
|
-
warnDrift(`${filePath}:${lineNo}`, `unknown 'type' value ${JSON.stringify(obj.type)}`, 1);
|
|
147
|
-
}
|
|
148
|
-
if ("content" in obj &&
|
|
149
|
-
obj.content !== undefined &&
|
|
150
|
-
typeof obj.content !== "string") {
|
|
151
|
-
warnDrift(`${filePath}:${lineNo}`, `expected 'content' to be a string, got ${describeType(obj.content)}`, 1);
|
|
152
|
-
}
|
|
153
|
-
if (!("timestamp" in obj)) {
|
|
154
|
-
warnDrift(`${filePath}:${lineNo}`, "record is missing 'timestamp' field", 1);
|
|
155
|
-
}
|
|
156
|
-
else if (typeof obj.timestamp !== "string") {
|
|
157
|
-
warnDrift(`${filePath}:${lineNo}`, `expected 'timestamp' string, got ${describeType(obj.timestamp)}`, 1);
|
|
158
|
-
}
|
|
159
|
-
const timestamp = new Date(obj.timestamp ?? 0);
|
|
160
|
-
if (timestamp <= since) {
|
|
161
|
-
messageIndex += 1;
|
|
162
|
-
continue;
|
|
163
|
-
}
|
|
164
|
-
obj.sessionId = sessionId;
|
|
165
|
-
obj.messageIndex = messageIndex;
|
|
166
|
-
const chunk = this.parseRaw(obj);
|
|
167
|
-
if (!chunk.content.trim()) {
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
messageIndex += 1;
|
|
171
|
-
yield chunk;
|
|
121
|
+
try {
|
|
122
|
+
yield* this.readSessionFile(filePath, sessionId, since, exactDirectory);
|
|
172
123
|
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
// One unreadable file must not abort the remaining files/projects.
|
|
126
|
+
warnDrift(filePath, `unreadable transcript file: ${err.message}`, 0);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
async *readSessionFile(filePath, sessionId, since, exactDirectory) {
|
|
132
|
+
const reader = createInterface({
|
|
133
|
+
input: createReadStream(filePath, { encoding: "utf8" }),
|
|
134
|
+
crlfDelay: Infinity,
|
|
135
|
+
});
|
|
136
|
+
let messageIndex = 0;
|
|
137
|
+
let lineNo = 0;
|
|
138
|
+
for await (const line of reader) {
|
|
139
|
+
lineNo++;
|
|
140
|
+
if (!line.trim()) {
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
let obj;
|
|
144
|
+
try {
|
|
145
|
+
obj = JSON.parse(line);
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
// ACCEPTED_DEGRADATIONS.malformedJsonlLine, but warn so the
|
|
149
|
+
// mutation test can see drift instead of data silently vanishing.
|
|
150
|
+
warnDrift(`${filePath}:${lineNo}`, `line is not valid JSON: ${err.message}`, 1);
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (NON_MESSAGE_TYPES.has(String(obj.type))) {
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
if (!this.recordMatchesProject(obj, exactDirectory)) {
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
// Strict-mode schema check: warn (but still emit) on shape surprise.
|
|
160
|
+
if (!("type" in obj)) {
|
|
161
|
+
warnDrift(`${filePath}:${lineNo}`, "record is missing required 'type' field — likely renamed", 1);
|
|
173
162
|
}
|
|
163
|
+
else if (typeof obj.type !== "string" || !(obj.type in ROLE_MAP)) {
|
|
164
|
+
warnDrift(`${filePath}:${lineNo}`, `unknown 'type' value ${JSON.stringify(obj.type)}`, 1);
|
|
165
|
+
}
|
|
166
|
+
if ("content" in obj &&
|
|
167
|
+
obj.content !== undefined &&
|
|
168
|
+
typeof obj.content !== "string") {
|
|
169
|
+
warnDrift(`${filePath}:${lineNo}`, `expected 'content' to be a string, got ${describeType(obj.content)}`, 1);
|
|
170
|
+
}
|
|
171
|
+
if (!("timestamp" in obj)) {
|
|
172
|
+
warnDrift(`${filePath}:${lineNo}`, "record is missing 'timestamp' field", 1);
|
|
173
|
+
}
|
|
174
|
+
else if (typeof obj.timestamp !== "string") {
|
|
175
|
+
warnDrift(`${filePath}:${lineNo}`, `expected 'timestamp' string, got ${describeType(obj.timestamp)}`, 1);
|
|
176
|
+
}
|
|
177
|
+
const timestamp = parseRecordTimestamp(obj.timestamp);
|
|
178
|
+
// Every message-shaped record consumes an index, whether or not it is
|
|
179
|
+
// emitted, so chunk identity is identical between full and incremental
|
|
180
|
+
// scrapes (chunk ids hash the index).
|
|
181
|
+
const currentIndex = messageIndex;
|
|
182
|
+
messageIndex += 1;
|
|
183
|
+
// A zero `since` means full sync: emit even epoch-sentinel timestamps.
|
|
184
|
+
if (since.getTime() > 0 && timestamp <= since) {
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
obj.sessionId = sessionId;
|
|
188
|
+
obj.messageIndex = currentIndex;
|
|
189
|
+
const chunk = this.parseRaw(obj);
|
|
190
|
+
if (!chunk.content.trim()) {
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
yield chunk;
|
|
174
194
|
}
|
|
175
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Coarse pre-filter over encoded store directory names.
|
|
198
|
+
*
|
|
199
|
+
* Encoded names are ambiguous — `-` stands for `:`, `\` and `/` alike — so
|
|
200
|
+
* this only decides which directories are worth opening. Each record's own
|
|
201
|
+
* `cwd` is what actually attributes it (see `recordMatchesProject`), which
|
|
202
|
+
* is both unambiguous and platform-independent. Keeping the prefix wide
|
|
203
|
+
* here lets sessions started from a subdirectory through; the per-record
|
|
204
|
+
* check then rejects a sibling like `<project>--secret`.
|
|
205
|
+
*/
|
|
176
206
|
filterProjectDirs(projectDirs) {
|
|
177
207
|
if (!this.projectRoot) {
|
|
178
208
|
return projectDirs;
|
|
@@ -180,9 +210,35 @@ export class ClaudeCodeScraper extends AbstractScraper {
|
|
|
180
210
|
const encodedProject = encodePathForToolDirectory(this.projectRoot).toLowerCase();
|
|
181
211
|
return projectDirs.filter((projectDir) => {
|
|
182
212
|
const normalized = projectDir.toLowerCase();
|
|
183
|
-
return normalized === encodedProject || normalized.startsWith(`${encodedProject}
|
|
213
|
+
return normalized === encodedProject || normalized.startsWith(`${encodedProject}-`);
|
|
184
214
|
});
|
|
185
215
|
}
|
|
216
|
+
/**
|
|
217
|
+
* Attribute a record to the project by the `cwd` Claude Code stamps on it.
|
|
218
|
+
* Records without a `cwd` fall back to the directory-name decision already
|
|
219
|
+
* made by `filterProjectDirs`.
|
|
220
|
+
*/
|
|
221
|
+
recordMatchesProject(obj, exactDirectory) {
|
|
222
|
+
if (!this.projectRoot) {
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
const cwd = obj.cwd;
|
|
226
|
+
if (typeof cwd !== "string" || cwd.length === 0) {
|
|
227
|
+
// No provenance on the record. An exact store-directory match is itself
|
|
228
|
+
// provenance, so keep it; a directory that only matched by prefix could
|
|
229
|
+
// be a plain sibling (`proj-v2` beside `proj`), so fail closed.
|
|
230
|
+
return exactDirectory;
|
|
231
|
+
}
|
|
232
|
+
return pathMatchesProject(cwd, this.projectRoot);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Missing or unparseable timestamps become the epoch sentinel rather than an
|
|
237
|
+
* Invalid Date, which would poison cutoff comparisons and toISOString().
|
|
238
|
+
*/
|
|
239
|
+
function parseRecordTimestamp(value) {
|
|
240
|
+
const parsed = new Date(value || 0);
|
|
241
|
+
return Number.isNaN(parsed.getTime()) ? new Date(0) : parsed;
|
|
186
242
|
}
|
|
187
243
|
function extractRole(obj, type) {
|
|
188
244
|
const message = isRecord(obj.message) ? obj.message : {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claude-code.js","sourceRoot":"","sources":["../../../src/scrapers/claude-code.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"claude-code.js","sourceRoot":"","sources":["../../../src/scrapers/claude-code.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE3F,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,wEAAwE;IACxE,kBAAkB,EAAE,gCAAgC;IACpD,oEAAoE;IACpE,kBAAkB,EAAE,kDAAkD;IACtE,sEAAsE;IACtE,sBAAsB,EAAE,mCAAmC;IAC3D,kCAAkC;IAClC,cAAc,EAAE,8BAA8B;CAC/C,CAAC;AAEF,MAAM,QAAQ,GAA4C;IACxD,KAAK,EAAE,MAAM;IACb,IAAI,EAAE,MAAM;IACZ,SAAS,EAAE,WAAW;IACtB,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,MAAM;CACpB,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,YAAY;IACZ,cAAc;IACd,aAAa;IACb,SAAS;IACT,UAAU;IACV,iBAAiB;CAClB,CAAC,CAAC;AAEH,SAAS,SAAS,CAAC,UAAkB,EAAE,QAAgB,EAAE,eAAuB;IAC9E,OAAO,CAAC,IAAI,CACV,IAAI,YAAY,8BAA8B,UAAU,KAAK,QAAQ,GAAG;QACtE,sBAAsB,eAAe,GAAG,CAC3C,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,iBAAkB,SAAQ,eAAgC;IAIlD;IAEA;IALV,IAAI,GAAG,aAAa,CAAC;IAE9B,YACmB,iBAAyB,EAC1C,QAAgB,EACC,WAAoB;QAErC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAJC,sBAAiB,GAAjB,iBAAiB,CAAQ;QAEzB,gBAAW,GAAX,WAAW,CAAS;IAGvC,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACnD,OAAO,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,KAAY;QACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC;QAC5C,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,QAAQ;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ,CAAC,GAAY;QACnB,MAAM,GAAG,GAAG,GAA8B,CAAC;QAC3C,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,IAAI,GAAI,GAAG,CAAC,IAAe,IAAI,SAAS,CAAC;QAC/C,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAEpC,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,SAAS,EAAG,GAAG,CAAC,SAAoB,IAAI,SAAS;YACjD,SAAS;YACT,IAAI;YACJ,OAAO;YACP,QAAQ,EAAE;gBACR,YAAY,EAAG,GAAG,CAAC,YAAuB,IAAI,CAAC;gBAC/C,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC;gBACtC,eAAe,EAAE,EAAE;gBACnB,SAAS,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAe,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;gBACzE,OAAO,EAAE,GAAG,CAAC,OAA6B;gBAC1C,WAAW,EAAE,aAAa;aACoB;SACjD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,KAAW;QACxC,IAAI,WAAqB,CAAC;QAE1B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,2CAA2C;YAC3C,OAAO;QACT,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW;YACrC,CAAC,CAAC,0BAA0B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE;YAC5D,CAAC,CAAC,IAAI,CAAC;QAET,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC;YAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;YAC7D,MAAM,cAAc,GAClB,cAAc,KAAK,IAAI,IAAI,WAAW,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC;YAC1E,IAAI,KAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACxC,IAAI,CAAC;oBACH,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;gBAC1E,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,mEAAmE;oBACnE,SAAS,CAAC,QAAQ,EAAE,+BAAgC,GAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;gBAClF,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAC5B,QAAgB,EAChB,SAAiB,EACjB,KAAW,EACX,cAAuB;QAEvB,MAAM,MAAM,GAAG,eAAe,CAAC;YAC7B,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YACvD,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,SAAS;YACX,CAAC;YAED,IAAI,GAA4B,CAAC;YACjC,IAAI,CAAC;gBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,4DAA4D;gBAC5D,kEAAkE;gBAClE,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,2BAA4B,GAAa,CAAC,OAAO,EAAE,EACnD,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,CAAC;gBACpD,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;gBACrB,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,0DAA0D,EAC1D,CAAC,CACF,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC;gBACnE,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,wBAAwB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAClD,CAAC,CACF,CAAC;YACJ,CAAC;YAED,IACE,SAAS,IAAI,GAAG;gBAChB,GAAG,CAAC,OAAO,KAAK,SAAS;gBACzB,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAC/B,CAAC;gBACD,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,0CAA0C,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EACrE,CAAC,CACF,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,qCAAqC,EACrC,CAAC,CACF,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC7C,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,oCAAoC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EACjE,CAAC,CACF,CAAC;YACJ,CAAC;YAED,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEtD,sEAAsE;YACtE,uEAAuE;YACvE,sCAAsC;YACtC,MAAM,YAAY,GAAG,YAAY,CAAC;YAClC,YAAY,IAAI,CAAC,CAAC;YAElB,uEAAuE;YACvE,IAAI,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;gBAC9C,SAAS;YACX,CAAC;YAED,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC;YAC1B,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;;OASG;IACK,iBAAiB,CAAC,WAAqB;QAC7C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,WAAW,CAAC;QACrB,CAAC;QAED,MAAM,cAAc,GAAG,0BAA0B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QAClF,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE;YACvC,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC5C,OAAO,UAAU,KAAK,cAAc,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,cAAc,GAAG,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,oBAAoB,CAAC,GAA4B,EAAE,cAAuB;QAChF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC;QACpB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChD,wEAAwE;YACxE,wEAAwE;YACxE,gEAAgE;YAChE,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,OAAO,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,KAAc;IAC1C,MAAM,MAAM,GAAG,IAAI,IAAI,CAAE,KAAgB,IAAI,CAAC,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/D,CAAC;AAED,SAAS,WAAW,CAClB,GAA4B,EAC5B,IAAY;IAEZ,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;AACtD,CAAC;AAED,SAAS,cAAc,CAAC,GAA4B;IAClD,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,GAAG,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAChE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,OAAO,OAAO,KAAK,CAAC;AACtB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codex.d.ts","sourceRoot":"","sources":["../../../src/scrapers/codex.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,eAAe,EAA0B,MAAM,WAAW,CAAC;AAKpE;;;GAGG;AACH,eAAO,MAAM,qBAAqB;IAChC,8DAA8D;;IAE9D,+DAA+D;;IAE/D,0EAA0E;;IAE1E,iFAAiF;;IAEjF,wDAAwD;;CAEzD,CAAC;AA2BF,qBAAa,eAAgB,SAAQ,eAAe,CAAC,UAAU,CAAC;IAI5D,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAElC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;IAL/B,QAAQ,CAAC,IAAI,WAAW;gBAGL,iBAAiB,EAAE,MAAM,EAC1C,QAAQ,EAAE,MAAM,EACC,WAAW,CAAC,EAAE,MAAM,YAAA;IAKjC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC;IAM/C,QAAQ,IAAI,aAAa,CAAC,UAAU,CAAC;IAI5C,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU;IAsBlC;;;;;;;;;OASG;YACY,eAAe;
|
|
1
|
+
{"version":3,"file":"codex.d.ts","sourceRoot":"","sources":["../../../src/scrapers/codex.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,eAAe,EAA0B,MAAM,WAAW,CAAC;AAKpE;;;GAGG;AACH,eAAO,MAAM,qBAAqB;IAChC,8DAA8D;;IAE9D,+DAA+D;;IAE/D,0EAA0E;;IAE1E,iFAAiF;;IAEjF,wDAAwD;;CAEzD,CAAC;AA2BF,qBAAa,eAAgB,SAAQ,eAAe,CAAC,UAAU,CAAC;IAI5D,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAElC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;IAL/B,QAAQ,CAAC,IAAI,WAAW;gBAGL,iBAAiB,EAAE,MAAM,EAC1C,QAAQ,EAAE,MAAM,EACC,WAAW,CAAC,EAAE,MAAM,YAAA;IAKjC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC;IAM/C,QAAQ,IAAI,aAAa,CAAC,UAAU,CAAC;IAI5C,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU;IAsBlC;;;;;;;;;OASG;YACY,eAAe;IAoP9B,OAAO,CAAC,cAAc;IAQtB;;;OAGG;YACW,iBAAiB;CAoBhC"}
|
|
@@ -83,7 +83,7 @@ export class CodexCliScraper extends AbstractScraper {
|
|
|
83
83
|
messageIndex: toMessageIndex(value.messageIndex),
|
|
84
84
|
tokenEstimate: estimateTokens(content),
|
|
85
85
|
referencedFiles: [],
|
|
86
|
-
approvalMode: normalizeApprovalMode(toStringValue(value.approvalMode)),
|
|
86
|
+
approvalMode: normalizeApprovalMode(toStringValue(value.approvalMode)) ?? "suggest",
|
|
87
87
|
sandboxed: toBoolean(value.sandboxed),
|
|
88
88
|
layer: toMessageIndex(value.layer),
|
|
89
89
|
},
|
|
@@ -102,100 +102,178 @@ export class CodexCliScraper extends AbstractScraper {
|
|
|
102
102
|
async *readAllSessions(since) {
|
|
103
103
|
const files = await this.resolveJsonlFiles();
|
|
104
104
|
for (const filePath of files) {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
105
|
+
try {
|
|
106
|
+
const sessionIdFromFile = inferSessionId(filePath);
|
|
107
|
+
const reader = createInterface({
|
|
108
|
+
input: createReadStream(filePath, { encoding: "utf8" }),
|
|
109
|
+
crlfDelay: Infinity,
|
|
110
|
+
});
|
|
111
|
+
// Session-level state, updated as we encounter meta/context events.
|
|
112
|
+
let sessionId = sessionIdFromFile;
|
|
113
|
+
let approvalMode = "suggest";
|
|
114
|
+
let sandboxed = false;
|
|
115
|
+
let messageIndex = 0;
|
|
116
|
+
let projectMatched = this.projectRoot ? false : true;
|
|
117
|
+
for await (const line of reader) {
|
|
118
|
+
if (!line.trim()) {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
let parsed;
|
|
122
|
+
try {
|
|
123
|
+
parsed = JSON.parse(line);
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
warnDrift(filePath, `JSONL line not parseable: ${err.message}`, 1);
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
const eventType = toStringValue(parsed.type);
|
|
130
|
+
// Strict-mode drift detection: an event with no readable 'type' is a
|
|
131
|
+
// surprise — Codex always stamps one. A null/missing/renamed 'type'
|
|
132
|
+
// means we cannot route the event and will silently lose data
|
|
133
|
+
// otherwise.
|
|
134
|
+
if (eventType === undefined) {
|
|
135
|
+
warnDrift(filePath, `event has no readable 'type' field (got ${describeType(parsed.type)})`, 1);
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (!KNOWN_EVENT_TYPES.has(eventType)) {
|
|
139
|
+
// Unknown event type — could be a new event kind added by a newer
|
|
140
|
+
// codex version. Warn so drift is observable; continue reading.
|
|
141
|
+
warnDrift(filePath, `unknown event type '${eventType}'`, 1);
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
// session_meta carries the canonical session UUID.
|
|
145
|
+
if (eventType === "session_meta") {
|
|
146
|
+
const payload = parsed.payload;
|
|
147
|
+
if (isRecord(payload)) {
|
|
148
|
+
sessionId = toStringValue(payload.id) ?? sessionIdFromFile;
|
|
149
|
+
const match = this.matchesProject(payload.cwd);
|
|
150
|
+
if (match === false) {
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
projectMatched = match ?? projectMatched;
|
|
151
154
|
}
|
|
152
|
-
|
|
155
|
+
continue;
|
|
153
156
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
157
|
+
// turn_context carries the approval policy and sandbox mode.
|
|
158
|
+
if (eventType === "turn_context") {
|
|
159
|
+
const payload = parsed.payload;
|
|
160
|
+
if (isRecord(payload)) {
|
|
161
|
+
const match = this.matchesProject(payload.cwd);
|
|
162
|
+
if (match === false) {
|
|
163
|
+
break;
|
|
164
|
+
}
|
|
165
|
+
projectMatched = match ?? projectMatched;
|
|
166
|
+
approvalMode =
|
|
167
|
+
normalizeApprovalMode(toStringValue(payload.approval_policy)) ?? approvalMode;
|
|
168
|
+
const sandboxPolicy = payload.sandbox_policy;
|
|
169
|
+
if (isRecord(sandboxPolicy)) {
|
|
170
|
+
sandboxed = toStringValue(sandboxPolicy.type) !== "none";
|
|
171
|
+
}
|
|
163
172
|
}
|
|
164
|
-
|
|
165
|
-
approvalMode =
|
|
166
|
-
normalizeApprovalMode(toStringValue(payload.approval_policy)) ?? approvalMode;
|
|
167
|
-
const sandboxPolicy = payload.sandbox_policy;
|
|
168
|
-
sandboxed =
|
|
169
|
-
isRecord(sandboxPolicy) && toStringValue(sandboxPolicy.type) !== "none";
|
|
173
|
+
continue;
|
|
170
174
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
175
|
+
if (!projectMatched) {
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
// Actual user messages arrive as event_msg with type "user_message".
|
|
179
|
+
// response_item events with role "user" are system-injected context
|
|
180
|
+
// (AGENTS.md, permissions, environment) and are intentionally skipped.
|
|
181
|
+
if (eventType === "event_msg") {
|
|
182
|
+
const payload = parsed.payload;
|
|
183
|
+
if (!isRecord(payload)) {
|
|
184
|
+
warnDrift(filePath, `event_msg payload is not an object (got ${describeType(payload)})`, 1);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
if (payload.type !== "user_message") {
|
|
188
|
+
// Could be a new message sub-type — warn so drift is visible.
|
|
189
|
+
if (!("type" in payload)) {
|
|
190
|
+
warnDrift(filePath, "event_msg payload missing 'type' key — likely renamed", 1);
|
|
191
|
+
}
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const content = toStringValue(payload.message);
|
|
195
|
+
if (!content) {
|
|
196
|
+
messageIndex++;
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
const timestamp = toDate(parsed.timestamp ?? parsed.created_at ?? parsed.createdAt);
|
|
200
|
+
if (since.getTime() > 0 && timestamp <= since) {
|
|
201
|
+
messageIndex++;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
yield this.parseRaw({
|
|
205
|
+
sessionId,
|
|
206
|
+
messageIndex,
|
|
207
|
+
timestamp,
|
|
208
|
+
role: "user",
|
|
209
|
+
content,
|
|
210
|
+
approvalMode,
|
|
211
|
+
sandboxed,
|
|
212
|
+
});
|
|
213
|
+
messageIndex++;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
// compacted events hold AI-generated summaries of prior turns that were
|
|
217
|
+
// removed from the active context window to save space. They represent
|
|
218
|
+
// a higher-abstraction view of the conversation (layer 1).
|
|
219
|
+
if (eventType === "compacted") {
|
|
220
|
+
const payload = parsed.payload;
|
|
221
|
+
const content = isRecord(payload)
|
|
222
|
+
? (toStringValue(payload.summary) ??
|
|
223
|
+
toStringValue(payload.content) ??
|
|
224
|
+
toStringValue(payload.text))
|
|
225
|
+
: undefined;
|
|
226
|
+
if (content) {
|
|
227
|
+
const timestamp = toDate(parsed.timestamp ?? parsed.created_at ?? parsed.createdAt);
|
|
228
|
+
if (since.getTime() === 0 || timestamp > since) {
|
|
229
|
+
yield this.parseRaw({
|
|
230
|
+
sessionId,
|
|
231
|
+
messageIndex,
|
|
232
|
+
timestamp,
|
|
233
|
+
role: "assistant",
|
|
234
|
+
content,
|
|
235
|
+
approvalMode,
|
|
236
|
+
sandboxed,
|
|
237
|
+
layer: 1,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
// Consume the index below the cutoff too, so chunk identity is
|
|
241
|
+
// stable between full and incremental scrapes.
|
|
242
|
+
messageIndex++;
|
|
243
|
+
}
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
// Only assistant messages from response_item events are conversation turns.
|
|
247
|
+
if (eventType !== "response_item")
|
|
248
|
+
continue;
|
|
180
249
|
const payload = parsed.payload;
|
|
181
250
|
if (!isRecord(payload)) {
|
|
182
|
-
warnDrift(filePath, `
|
|
251
|
+
warnDrift(filePath, `response_item payload is not an object (got ${describeType(payload)})`, 1);
|
|
183
252
|
continue;
|
|
184
253
|
}
|
|
185
|
-
if (payload.type !== "
|
|
186
|
-
// Could be a new message sub-type — warn so drift is visible.
|
|
254
|
+
if (payload.type !== "message") {
|
|
187
255
|
if (!("type" in payload)) {
|
|
188
|
-
warnDrift(filePath, "
|
|
256
|
+
warnDrift(filePath, "response_item payload missing 'type' key — likely renamed", 1);
|
|
189
257
|
}
|
|
190
258
|
continue;
|
|
191
259
|
}
|
|
192
|
-
|
|
260
|
+
// Skip system-injected context which Codex sends as "user" role items.
|
|
261
|
+
const role = toStringValue(payload.role);
|
|
262
|
+
if (role !== "assistant") {
|
|
263
|
+
if (!("role" in payload)) {
|
|
264
|
+
warnDrift(filePath, "response_item message payload missing 'role' key", 1);
|
|
265
|
+
}
|
|
266
|
+
messageIndex++;
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
// Extract text from the content-part array (or plain string fallback).
|
|
270
|
+
const content = extractContent(payload.content);
|
|
193
271
|
if (!content) {
|
|
194
272
|
messageIndex++;
|
|
195
273
|
continue;
|
|
196
274
|
}
|
|
197
275
|
const timestamp = toDate(parsed.timestamp ?? parsed.created_at ?? parsed.createdAt);
|
|
198
|
-
if (timestamp <= since) {
|
|
276
|
+
if (since.getTime() > 0 && timestamp <= since) {
|
|
199
277
|
messageIndex++;
|
|
200
278
|
continue;
|
|
201
279
|
}
|
|
@@ -203,86 +281,17 @@ export class CodexCliScraper extends AbstractScraper {
|
|
|
203
281
|
sessionId,
|
|
204
282
|
messageIndex,
|
|
205
283
|
timestamp,
|
|
206
|
-
role
|
|
284
|
+
role,
|
|
207
285
|
content,
|
|
208
286
|
approvalMode,
|
|
209
287
|
sandboxed,
|
|
210
288
|
});
|
|
211
289
|
messageIndex++;
|
|
212
|
-
continue;
|
|
213
|
-
}
|
|
214
|
-
// compacted events hold AI-generated summaries of prior turns that were
|
|
215
|
-
// removed from the active context window to save space. They represent
|
|
216
|
-
// a higher-abstraction view of the conversation (layer 1).
|
|
217
|
-
if (eventType === "compacted") {
|
|
218
|
-
const payload = parsed.payload;
|
|
219
|
-
const content = isRecord(payload)
|
|
220
|
-
? (toStringValue(payload.summary) ??
|
|
221
|
-
toStringValue(payload.content) ??
|
|
222
|
-
toStringValue(payload.text))
|
|
223
|
-
: undefined;
|
|
224
|
-
if (content) {
|
|
225
|
-
const timestamp = toDate(parsed.timestamp ?? parsed.created_at ?? parsed.createdAt);
|
|
226
|
-
if (timestamp > since) {
|
|
227
|
-
yield this.parseRaw({
|
|
228
|
-
sessionId,
|
|
229
|
-
messageIndex,
|
|
230
|
-
timestamp,
|
|
231
|
-
role: "assistant",
|
|
232
|
-
content,
|
|
233
|
-
approvalMode,
|
|
234
|
-
sandboxed,
|
|
235
|
-
layer: 1,
|
|
236
|
-
});
|
|
237
|
-
messageIndex++;
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
// Only assistant messages from response_item events are conversation turns.
|
|
243
|
-
if (eventType !== "response_item")
|
|
244
|
-
continue;
|
|
245
|
-
const payload = parsed.payload;
|
|
246
|
-
if (!isRecord(payload)) {
|
|
247
|
-
warnDrift(filePath, `response_item payload is not an object (got ${describeType(payload)})`, 1);
|
|
248
|
-
continue;
|
|
249
|
-
}
|
|
250
|
-
if (payload.type !== "message") {
|
|
251
|
-
if (!("type" in payload)) {
|
|
252
|
-
warnDrift(filePath, "response_item payload missing 'type' key — likely renamed", 1);
|
|
253
|
-
}
|
|
254
|
-
continue;
|
|
255
|
-
}
|
|
256
|
-
// Skip system-injected context which Codex sends as "user" role items.
|
|
257
|
-
const role = toStringValue(payload.role);
|
|
258
|
-
if (role !== "assistant") {
|
|
259
|
-
if (!("role" in payload)) {
|
|
260
|
-
warnDrift(filePath, "response_item message payload missing 'role' key", 1);
|
|
261
|
-
}
|
|
262
|
-
messageIndex++;
|
|
263
|
-
continue;
|
|
264
290
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
continue;
|
|
270
|
-
}
|
|
271
|
-
const timestamp = toDate(parsed.timestamp ?? parsed.created_at ?? parsed.createdAt);
|
|
272
|
-
if (timestamp <= since) {
|
|
273
|
-
messageIndex++;
|
|
274
|
-
continue;
|
|
275
|
-
}
|
|
276
|
-
yield this.parseRaw({
|
|
277
|
-
sessionId,
|
|
278
|
-
messageIndex,
|
|
279
|
-
timestamp,
|
|
280
|
-
role,
|
|
281
|
-
content,
|
|
282
|
-
approvalMode,
|
|
283
|
-
sandboxed,
|
|
284
|
-
});
|
|
285
|
-
messageIndex++;
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
// One unreadable file must not abort the remaining session files.
|
|
294
|
+
warnDrift(filePath, `unreadable transcript file: ${err.message}`, 0);
|
|
286
295
|
}
|
|
287
296
|
}
|
|
288
297
|
}
|
|
@@ -343,12 +352,16 @@ function normalizeRole(value) {
|
|
|
343
352
|
}
|
|
344
353
|
function normalizeApprovalMode(value) {
|
|
345
354
|
switch (value) {
|
|
355
|
+
case "suggest":
|
|
356
|
+
return "suggest";
|
|
346
357
|
case "auto-edit":
|
|
347
358
|
return "auto-edit";
|
|
348
359
|
case "full-auto":
|
|
349
360
|
return "full-auto";
|
|
350
361
|
default:
|
|
351
|
-
|
|
362
|
+
// Unknown/future policy values keep the previously observed mode
|
|
363
|
+
// rather than silently downgrading the record to "suggest".
|
|
364
|
+
return null;
|
|
352
365
|
}
|
|
353
366
|
}
|
|
354
367
|
function inferSessionId(filePath) {
|