slopie 1.6.2 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +2 -0
- package/README.md +9 -5
- package/dist/audit-native-worker.js +1402 -0
- package/dist/audit-native-worker.js.map +7 -0
- package/dist/potsherd.js +5250 -2483
- package/dist/potsherd.js.map +4 -4
- package/licenses/ccusage-MIT.txt +21 -0
- package/package.json +3 -1
|
@@ -0,0 +1,1402 @@
|
|
|
1
|
+
import { createRequire as __potsherdCreateRequire } from 'node:module';
|
|
2
|
+
const require = __potsherdCreateRequire(import.meta.url);
|
|
3
|
+
|
|
4
|
+
// packages/core/src/analytics/native-pool.ts
|
|
5
|
+
import { Worker, parentPort } from "node:worker_threads";
|
|
6
|
+
import { serialize, deserialize } from "node:v8";
|
|
7
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
8
|
+
|
|
9
|
+
// packages/core/src/analytics/native-stream.ts
|
|
10
|
+
import fs from "node:fs";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
13
|
+
|
|
14
|
+
// packages/core/src/parser/content.ts
|
|
15
|
+
function isRecord(value) {
|
|
16
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
17
|
+
}
|
|
18
|
+
function extractTypedText(content, blockType = "text") {
|
|
19
|
+
if (typeof content === "string") return content;
|
|
20
|
+
if (!Array.isArray(content)) return "";
|
|
21
|
+
return content.filter((b) => isRecord(b) && b.type === blockType && typeof b.text === "string").map((b) => b.text).join("\n");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// packages/core/src/markers.ts
|
|
25
|
+
var SUMMARIZER_CONTEXT_MARKER = "Context: This summary will be shown in a list to help users and Claude choose which conversations are relevant";
|
|
26
|
+
var POTSHERD_CARD_MARKER = "<INSTRUCTIONS-TO-POTSHERD>DO NOT INDEX THIS CHAT</INSTRUCTIONS-TO-POTSHERD>";
|
|
27
|
+
var EXCLUSION_MARKERS = [
|
|
28
|
+
"<INSTRUCTIONS-TO-EPISODIC-MEMORY>DO NOT INDEX THIS CHAT</INSTRUCTIONS-TO-EPISODIC-MEMORY>",
|
|
29
|
+
"Only use NO_INSIGHTS_FOUND",
|
|
30
|
+
SUMMARIZER_CONTEXT_MARKER,
|
|
31
|
+
POTSHERD_CARD_MARKER
|
|
32
|
+
];
|
|
33
|
+
function hasExclusionMarker(text) {
|
|
34
|
+
return EXCLUSION_MARKERS.some((marker) => text.includes(marker));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// packages/core/src/memory/source-identity.ts
|
|
38
|
+
import { createHash } from "node:crypto";
|
|
39
|
+
function sourceId(harness, nativeSessionId) {
|
|
40
|
+
return createHash("sha256").update(`source/v1\0${harness}\0${nativeSessionId}`).digest("hex");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// packages/core/src/analytics/source.ts
|
|
44
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
45
|
+
|
|
46
|
+
// packages/core/src/redact.ts
|
|
47
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
48
|
+
|
|
49
|
+
// packages/core/src/theme.ts
|
|
50
|
+
var ANSI_RE = new RegExp("\\u001b\\[[0-9;]*m", "g");
|
|
51
|
+
|
|
52
|
+
// packages/core/src/redact-rules.ts
|
|
53
|
+
function shannonEntropy(s) {
|
|
54
|
+
const n = s.length;
|
|
55
|
+
if (n === 0) return 0;
|
|
56
|
+
const counts = /* @__PURE__ */ new Map();
|
|
57
|
+
for (const ch of s) counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
|
58
|
+
let h = 0;
|
|
59
|
+
for (const c of counts.values()) {
|
|
60
|
+
const p = c / n;
|
|
61
|
+
h -= p * Math.log2(p);
|
|
62
|
+
}
|
|
63
|
+
return h;
|
|
64
|
+
}
|
|
65
|
+
var ENTROPY_MIN_LENGTH = 20;
|
|
66
|
+
var ENTROPY_THRESHOLD = 4.5;
|
|
67
|
+
var ALLOW_SPANS = [
|
|
68
|
+
// data: URIs — `data:image/png;base64,iVBORw0KGgo…`
|
|
69
|
+
/\bdata:[a-zA-Z0-9!#$&^_.+-]*(?:\/[a-zA-Z0-9!#$&^_.+-]*)?(?:;[a-zA-Z0-9-]+=[^;,\s]*)*;base64,[A-Za-z0-9+/=]{20,}(?:\s+[A-Za-z0-9+/=]{20,})*/g,
|
|
70
|
+
// Subresource Integrity — `integrity="sha384-oqVuAfXRKap7fdgcCY5uykM6+R9…"`
|
|
71
|
+
/\bsha(?:256|384|512)-[A-Za-z0-9+/]{20,}={0,3}/g
|
|
72
|
+
];
|
|
73
|
+
var UUID_RE = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
|
|
74
|
+
var HEX_RE = /^[0-9a-fA-F]+$/;
|
|
75
|
+
var NUMERIC_RE = /^[0-9_+-]+$/;
|
|
76
|
+
var MINTED_ID_RE = /^(?:toolu|srvtoolu|msg|req|resp|call|fc|run|step|thread|asst|evt|sess|session|exec|job|task|turn|conv|agent|snapshot|chatcmpl)[-_]/i;
|
|
77
|
+
var ULID_RE = /^[0-7][0-9ABCDEFGHJKMNPQRSTVWXYZ]{25}$/;
|
|
78
|
+
var SAMECASE_RUN_MIN = 7;
|
|
79
|
+
var SAMECASE_RUNS_NEEDED = 2;
|
|
80
|
+
var SAMECASE_RUN_ALONE = 12;
|
|
81
|
+
function looksLikeProse(token) {
|
|
82
|
+
let long = 0;
|
|
83
|
+
for (const run of token.match(/[a-z]+|[A-Z]+/g) ?? []) {
|
|
84
|
+
if (run.length >= SAMECASE_RUN_ALONE) return true;
|
|
85
|
+
if (run.length >= SAMECASE_RUN_MIN) long += 1;
|
|
86
|
+
}
|
|
87
|
+
return long >= SAMECASE_RUNS_NEEDED;
|
|
88
|
+
}
|
|
89
|
+
function looksLikeAlphabetConstant(token) {
|
|
90
|
+
return token.length >= 24 && new Set(token).size === token.length;
|
|
91
|
+
}
|
|
92
|
+
var PLACEHOLDER_RE = /(?:^|[^a-z0-9])(?:x{3,}|your|my[-_]?(?:key|token|secret|pass)|example|sample|dummy|fake|mock|placeholder|change[-_]?(?:me|it|this)|replace[-_]?me|insert|todo|fixme|redacted|hidden|omitted|elided|not[-_]?real|no[-_]?such|hunter2|password|passwd|secret|token|apikey|api[-_]key|abcdef|123456|s3cret|letmein|foobar|lorem|ipsum)(?:$|[^a-z0-9])/i;
|
|
93
|
+
var INTERPOLATION_RE = /\$\{|\{\{|%\(|<%|^\$[A-Za-z_(]|^<|^\{|\}$|^%[A-Za-z(]/;
|
|
94
|
+
var REFERENCE_RE = /^(?:process\.env|import\.meta\.env|os\.environ|System\.getenv|Deno\.env|env|ENV|config|conf|cfg|settings|opts|options|args|argv|params|props|state|data|input|payload|body|req|res|ctx|context|self|this|that|obj|row|item|user|account|creds|credentials|secrets|vault|store|keychain)\b\s*[.[(]/;
|
|
95
|
+
var WORDY_IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z_$]*$/;
|
|
96
|
+
var DOTTED_CHAIN_RE = /^[A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)+$/;
|
|
97
|
+
var PATHISH_RE = /^(?:[a-z][a-z0-9+.-]*:\/\/|\.{0,2}\/|~\/|[A-Za-z]:\\)/;
|
|
98
|
+
var CODEY_RE = /[<>{}[\]()|\\;,&?!*"'`]/;
|
|
99
|
+
var MASK_FRAGMENT_RE = /‹redacted:/;
|
|
100
|
+
var NAME_STOPWORDS = /* @__PURE__ */ new Set([
|
|
101
|
+
"monkey",
|
|
102
|
+
"donkey",
|
|
103
|
+
"turkey",
|
|
104
|
+
"whiskey",
|
|
105
|
+
"jockey",
|
|
106
|
+
"hockey",
|
|
107
|
+
"lackey",
|
|
108
|
+
"mickey",
|
|
109
|
+
"malarkey",
|
|
110
|
+
"hotkey",
|
|
111
|
+
"sortkey",
|
|
112
|
+
"oauth"
|
|
113
|
+
]);
|
|
114
|
+
function nameLooksLikeSecret(name) {
|
|
115
|
+
const segments = name.split(/[^A-Za-z0-9]+/).flatMap((part) => part.split(/(?<=[a-z0-9])(?=[A-Z])/)).filter(Boolean);
|
|
116
|
+
const last = segments[segments.length - 1];
|
|
117
|
+
if (!last) return false;
|
|
118
|
+
const lower = last.toLowerCase();
|
|
119
|
+
if (NAME_STOPWORDS.has(lower)) return false;
|
|
120
|
+
return /(?:key|token|secret|password|passwd|pwd|credential|auth)s?$/.test(lower);
|
|
121
|
+
}
|
|
122
|
+
function valueLooksLikeSecret(value) {
|
|
123
|
+
const v = value.trim();
|
|
124
|
+
if (v.length < 10 || v.length > 200) return false;
|
|
125
|
+
if (/\s/.test(v)) return false;
|
|
126
|
+
if (MASK_FRAGMENT_RE.test(v)) return false;
|
|
127
|
+
if (CODEY_RE.test(v)) return false;
|
|
128
|
+
if (INTERPOLATION_RE.test(v)) return false;
|
|
129
|
+
if (REFERENCE_RE.test(v)) return false;
|
|
130
|
+
if (DOTTED_CHAIN_RE.test(v)) return false;
|
|
131
|
+
if (PATHISH_RE.test(v)) return false;
|
|
132
|
+
if (NUMERIC_RE.test(v)) return false;
|
|
133
|
+
if (PLACEHOLDER_RE.test(v)) return false;
|
|
134
|
+
if (WORDY_IDENTIFIER_RE.test(v) && shannonEntropy(v) < 4) return false;
|
|
135
|
+
if (new Set(v).size <= 2) return false;
|
|
136
|
+
if (shannonEntropy(v) < 3.2) return false;
|
|
137
|
+
return true;
|
|
138
|
+
}
|
|
139
|
+
function entropyCandidateAllowed(token) {
|
|
140
|
+
if (token.length < ENTROPY_MIN_LENGTH) return false;
|
|
141
|
+
if (UUID_RE.test(token)) return false;
|
|
142
|
+
if (UUID_RE.test(token.replace(/^[A-Za-z][A-Za-z0-9]{0,15}[-_]/, ""))) return false;
|
|
143
|
+
if (HEX_RE.test(token)) return false;
|
|
144
|
+
if (/^(?:sha1|sha256|sha384|sha512|md5|blake3|crc32|xxh64)[-_:]/i.test(token)) return false;
|
|
145
|
+
if (NUMERIC_RE.test(token)) return false;
|
|
146
|
+
if (MASK_FRAGMENT_RE.test(token)) return false;
|
|
147
|
+
if (PLACEHOLDER_RE.test(token)) return false;
|
|
148
|
+
if (MINTED_ID_RE.test(token)) return false;
|
|
149
|
+
if (ULID_RE.test(token)) return false;
|
|
150
|
+
if (looksLikeProse(token)) return false;
|
|
151
|
+
if (looksLikeAlphabetConstant(token)) return false;
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
function regexRule(id, type, source, re, opts = {}) {
|
|
155
|
+
const group = opts.group ?? 0;
|
|
156
|
+
return {
|
|
157
|
+
id,
|
|
158
|
+
type,
|
|
159
|
+
source,
|
|
160
|
+
scan(text) {
|
|
161
|
+
const out = [];
|
|
162
|
+
const rx = new RegExp(re.source, re.flags.includes("g") ? re.flags : re.flags + "g");
|
|
163
|
+
let m;
|
|
164
|
+
while ((m = rx.exec(text)) !== null) {
|
|
165
|
+
if (m[0].length === 0) {
|
|
166
|
+
rx.lastIndex++;
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
const value = group === 0 ? m[0] : m[group];
|
|
170
|
+
if (value === void 0 || value.length === 0) continue;
|
|
171
|
+
if (opts.validate && !opts.validate(value, m)) continue;
|
|
172
|
+
const offset = group === 0 ? 0 : m[0].indexOf(value);
|
|
173
|
+
if (offset < 0) continue;
|
|
174
|
+
out.push({ start: m.index + offset, end: m.index + offset + value.length, value });
|
|
175
|
+
}
|
|
176
|
+
return out;
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
var GENERIC_ANCHOR = /(?:key|token|secret|password|passwd|pwd|credential|auth)s?["']?[ \t]*(?::=|=>|[:=])/gi;
|
|
181
|
+
var GENERIC_VALUE = (
|
|
182
|
+
// The bare alternative stops at a backslash on purpose: transcripts carry
|
|
183
|
+
// json-escaped text, where `TOKEN=abc\nNEXT=…` is one line and the `\n` is
|
|
184
|
+
// two literal characters. Without this the capture swallows the rest of the
|
|
185
|
+
// record and the value is thrown out as code.
|
|
186
|
+
/[ \t]*(?:"([^"\r\n]{10,200})"|'([^'\r\n]{10,200})'|`([^`\r\n]{10,200})`|([^\s"'`,;)\](}>\\]{10,200}))/y
|
|
187
|
+
);
|
|
188
|
+
var NAME_CHAR_RE = /[A-Za-z0-9_$.-]/;
|
|
189
|
+
var genericAssignmentRule = {
|
|
190
|
+
id: "generic-assignment",
|
|
191
|
+
type: "generic",
|
|
192
|
+
source: "03 \xA75, given the shape of gitleaks generic-api-key (MIT)",
|
|
193
|
+
scan(text) {
|
|
194
|
+
const out = [];
|
|
195
|
+
const anchor = new RegExp(GENERIC_ANCHOR.source, GENERIC_ANCHOR.flags);
|
|
196
|
+
let m;
|
|
197
|
+
while ((m = anchor.exec(text)) !== null) {
|
|
198
|
+
const opEnd = m.index + m[0].length;
|
|
199
|
+
const opLen = /(?::=|=>)$/.test(m[0]) ? 2 : 1;
|
|
200
|
+
const before = text[opEnd - opLen - 1] ?? "";
|
|
201
|
+
if (text[opEnd] === "=" || opLen === 1 && /[!<>+\-*/%&|^=]/.test(before)) continue;
|
|
202
|
+
const keywordEnd = m.index + m[0].replace(/["']?[ \t]*(?::=|=>|[:=])$/, "").length;
|
|
203
|
+
let nameStart = m.index;
|
|
204
|
+
while (nameStart > 0 && NAME_CHAR_RE.test(text[nameStart - 1] ?? "")) nameStart--;
|
|
205
|
+
const name = text.slice(nameStart, keywordEnd);
|
|
206
|
+
if (!nameLooksLikeSecret(name)) continue;
|
|
207
|
+
GENERIC_VALUE.lastIndex = opEnd;
|
|
208
|
+
const v = GENERIC_VALUE.exec(text);
|
|
209
|
+
if (!v) continue;
|
|
210
|
+
const value = v[1] ?? v[2] ?? v[3] ?? v[4];
|
|
211
|
+
if (value === void 0 || !valueLooksLikeSecret(value)) continue;
|
|
212
|
+
const start = v.index + v[0].lastIndexOf(value);
|
|
213
|
+
out.push({ start, end: start + value.length, value });
|
|
214
|
+
anchor.lastIndex = start + value.length;
|
|
215
|
+
}
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
var socketAuthRule = {
|
|
220
|
+
id: "socket-auth-argument",
|
|
221
|
+
type: "generic",
|
|
222
|
+
source: "03 \xA75 credential context",
|
|
223
|
+
scan(text) {
|
|
224
|
+
const out = [];
|
|
225
|
+
const anchor = /(?:^|[\s"'`])--socket-auth["']?(?:[ \t]*=[ \t]*|[ \t]+)/gi;
|
|
226
|
+
let match;
|
|
227
|
+
while ((match = anchor.exec(text)) !== null) {
|
|
228
|
+
GENERIC_VALUE.lastIndex = anchor.lastIndex;
|
|
229
|
+
const found = GENERIC_VALUE.exec(text);
|
|
230
|
+
if (!found) continue;
|
|
231
|
+
const value = found[1] ?? found[2] ?? found[3] ?? found[4];
|
|
232
|
+
if (value === void 0 || !valueLooksLikeSecret(value)) continue;
|
|
233
|
+
const start = found.index + found[0].lastIndexOf(value);
|
|
234
|
+
out.push({ start, end: start + value.length, value });
|
|
235
|
+
anchor.lastIndex = start + value.length;
|
|
236
|
+
}
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
var BEARER_RE = /(?:^|[^A-Za-z0-9_-])(?:Bearer|Token|ApiKey|Api-Key|Basic|DPoP)[ \t]+([A-Za-z0-9+/._~-]{16,512}={0,2})/g;
|
|
241
|
+
var bearerRule = regexRule(
|
|
242
|
+
"authorization-bearer",
|
|
243
|
+
"generic",
|
|
244
|
+
"03 \xA75 (credential context), given the shape of gitleaks generic-api-key (MIT)",
|
|
245
|
+
BEARER_RE,
|
|
246
|
+
{
|
|
247
|
+
group: 1,
|
|
248
|
+
// Same value screen as the `KEY=` rule: `Bearer ${token}`, `Bearer <TOKEN>`
|
|
249
|
+
// and `Bearer your-token-here` are documentation, not credentials.
|
|
250
|
+
validate: (value) => valueLooksLikeSecret(value)
|
|
251
|
+
}
|
|
252
|
+
);
|
|
253
|
+
var ENTROPY_TOKEN = /[A-Za-z0-9+_-]{20,}={0,2}/g;
|
|
254
|
+
var ESCAPE_LETTERS = "ntrbfv";
|
|
255
|
+
var highEntropyRule = {
|
|
256
|
+
id: "high-entropy-token",
|
|
257
|
+
type: "entropy",
|
|
258
|
+
source: "03 \xA75 (4.5 bits / 20 chars) + gitleaks entropy allowlists (MIT)",
|
|
259
|
+
scan(text) {
|
|
260
|
+
const out = [];
|
|
261
|
+
const rx = new RegExp(ENTROPY_TOKEN.source, ENTROPY_TOKEN.flags);
|
|
262
|
+
let m;
|
|
263
|
+
while ((m = rx.exec(text)) !== null) {
|
|
264
|
+
let start = m.index;
|
|
265
|
+
let value = m[0];
|
|
266
|
+
if (start > 0 && text[start - 1] === "\\" && ESCAPE_LETTERS.includes(value[0] ?? "")) {
|
|
267
|
+
start += 1;
|
|
268
|
+
value = value.slice(1);
|
|
269
|
+
}
|
|
270
|
+
if (start > 0 && text[start - 1] === "%" && /^[0-9A-Fa-f]{2}/.test(value)) {
|
|
271
|
+
start += 2;
|
|
272
|
+
value = value.slice(2);
|
|
273
|
+
}
|
|
274
|
+
const lead = value.length - value.replace(/^[-_+]+/, "").length;
|
|
275
|
+
start += lead;
|
|
276
|
+
value = value.slice(lead).replace(/[-_+]+$/, "");
|
|
277
|
+
if (value.length < ENTROPY_MIN_LENGTH) continue;
|
|
278
|
+
if (!entropyCandidateAllowed(value)) continue;
|
|
279
|
+
if (shannonEntropy(value) < ENTROPY_THRESHOLD) continue;
|
|
280
|
+
out.push({ start, end: start + value.length, value });
|
|
281
|
+
}
|
|
282
|
+
return out;
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
var RULES = [
|
|
286
|
+
// ---- key material ------------------------------------------------------
|
|
287
|
+
regexRule(
|
|
288
|
+
// gitleaks `private-key`; secretlint `@secretlint/secretlint-rule-privatekey`.
|
|
289
|
+
// The whole block is masked, header to footer: the base64 body alone would
|
|
290
|
+
// otherwise be shredded into a dozen separate entropy hits.
|
|
291
|
+
"private-key-block",
|
|
292
|
+
"private-key",
|
|
293
|
+
"gitleaks private-key / secretlint-rule-privatekey (MIT)",
|
|
294
|
+
/-----BEGIN[ A-Z0-9]{0,40}PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,200000}?-----END[ A-Z0-9]{0,40}PRIVATE KEY(?: BLOCK)?-----/g
|
|
295
|
+
),
|
|
296
|
+
regexRule(
|
|
297
|
+
// gitleaks `private-key`, unterminated variant: a transcript often quotes
|
|
298
|
+
// the header and the first body lines and then elides the rest.
|
|
299
|
+
//
|
|
300
|
+
// The body is taken line by line and only where a line is ≥ 20 characters
|
|
301
|
+
// of pure base64, so an unterminated header followed by prose masks the
|
|
302
|
+
// header alone instead of swallowing the sentence after it. The separator
|
|
303
|
+
// class carries the backslash so json-escaped `\n` works, and it is
|
|
304
|
+
// disjoint from the base64 class, which is what keeps this linear.
|
|
305
|
+
"private-key-header",
|
|
306
|
+
"private-key",
|
|
307
|
+
"gitleaks private-key (MIT)",
|
|
308
|
+
/-----BEGIN[ A-Z0-9]{0,40}PRIVATE KEY(?: BLOCK)?-----(?:[\r\n \t\\]+[A-Za-z0-9+/=]{20,})*/g
|
|
309
|
+
),
|
|
310
|
+
regexRule(
|
|
311
|
+
// gitleaks `jwt`: three base64url segments, the first two starting `ey`
|
|
312
|
+
// (the base64 of `{"`). Session transcripts are full of these.
|
|
313
|
+
"jwt",
|
|
314
|
+
"jwt",
|
|
315
|
+
"gitleaks jwt (MIT)",
|
|
316
|
+
/\bey[A-Za-z0-9_-]{17,}\.ey[A-Za-z0-9_-]{17,}\.[A-Za-z0-9_-]{10,}={0,2}/g
|
|
317
|
+
),
|
|
318
|
+
// ---- vendor tokens -----------------------------------------------------
|
|
319
|
+
regexRule(
|
|
320
|
+
// gitleaks `aws-access-token`; secretlint-rule-aws `AWSAccessKeyID`.
|
|
321
|
+
"aws-access-key-id",
|
|
322
|
+
"aws",
|
|
323
|
+
"gitleaks aws-access-token / secretlint-rule-aws (MIT)",
|
|
324
|
+
/\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\b/g
|
|
325
|
+
),
|
|
326
|
+
regexRule(
|
|
327
|
+
// secretlint-rule-aws `AWSSecretAccessKey`: a 40-character base64 value
|
|
328
|
+
// bound to an AWS-flavoured name. Typed `aws` and placed before the generic
|
|
329
|
+
// rule so `doctor` reports it as what it is.
|
|
330
|
+
"aws-secret-access-key",
|
|
331
|
+
"aws",
|
|
332
|
+
"secretlint-rule-aws AWSSecretAccessKey (MIT)",
|
|
333
|
+
/aws[_.-]?(?:secret|access)[_.-]?(?:access[_.-]?)?key(?:[_.-]?id)?["'\s]{0,4}[:=]["'\s]{0,4}([A-Za-z0-9/+=]{40})(?![A-Za-z0-9/+=])/gi,
|
|
334
|
+
{ group: 1 }
|
|
335
|
+
),
|
|
336
|
+
regexRule(
|
|
337
|
+
// gitleaks `gcp-api-key`; secretlint-rule-gcp `GCPApiKey`.
|
|
338
|
+
"gcp-api-key",
|
|
339
|
+
"gcp",
|
|
340
|
+
"gitleaks gcp-api-key / secretlint-rule-gcp (MIT)",
|
|
341
|
+
/\bAIza[0-9A-Za-z_-]{35}\b/g
|
|
342
|
+
),
|
|
343
|
+
regexRule(
|
|
344
|
+
// gitleaks `gcp-oauth-client-secret`.
|
|
345
|
+
"gcp-oauth-client-secret",
|
|
346
|
+
"gcp",
|
|
347
|
+
"gitleaks gcp-oauth-client-secret (MIT)",
|
|
348
|
+
/\bGOCSPX-[a-zA-Z0-9_-]{28}\b/g
|
|
349
|
+
),
|
|
350
|
+
regexRule(
|
|
351
|
+
// gitleaks `github-pat` / `-oauth` / `-app-token` / `-refresh-token`;
|
|
352
|
+
// secretlint-rule-github. ghp_ user, gho_ oauth, ghu_/ghs_ app, ghr_ refresh.
|
|
353
|
+
"github-token",
|
|
354
|
+
"github",
|
|
355
|
+
"gitleaks github-pat / secretlint-rule-github (MIT)",
|
|
356
|
+
/\bgh[pousr]_[A-Za-z0-9]{36,255}\b/g
|
|
357
|
+
),
|
|
358
|
+
regexRule(
|
|
359
|
+
// gitleaks `github-fine-grained-pat`.
|
|
360
|
+
"github-fine-grained-pat",
|
|
361
|
+
"github",
|
|
362
|
+
"gitleaks github-fine-grained-pat (MIT)",
|
|
363
|
+
/\bgithub_pat_[0-9a-zA-Z_]{82}\b/g
|
|
364
|
+
),
|
|
365
|
+
regexRule(
|
|
366
|
+
// gitleaks `slack-bot-token` / `-user-token` / `-app-token`;
|
|
367
|
+
// secretlint-rule-slack.
|
|
368
|
+
"slack-token",
|
|
369
|
+
"slack",
|
|
370
|
+
"gitleaks slack-bot-token / secretlint-rule-slack (MIT)",
|
|
371
|
+
/\bxox[abprs]-[0-9a-zA-Z-]{10,72}\b/g
|
|
372
|
+
),
|
|
373
|
+
regexRule(
|
|
374
|
+
// gitleaks `slack-webhook-url`; secretlint-rule-slack `SlackWebhook`.
|
|
375
|
+
"slack-webhook",
|
|
376
|
+
"slack",
|
|
377
|
+
"gitleaks slack-webhook-url (MIT)",
|
|
378
|
+
/https:\/\/hooks\.slack\.com\/(?:services|workflows|triggers)\/[A-Za-z0-9+/]{6,}\/[A-Za-z0-9+/]{6,}\/[A-Za-z0-9+/]{6,}/g
|
|
379
|
+
),
|
|
380
|
+
regexRule(
|
|
381
|
+
// gitleaks `stripe-access-token`. `pk_` publishable keys are public by
|
|
382
|
+
// design and are deliberately not matched.
|
|
383
|
+
"stripe-key",
|
|
384
|
+
"stripe",
|
|
385
|
+
"gitleaks stripe-access-token (MIT)",
|
|
386
|
+
/\b(?:sk|rk)_(?:test|live|prod)_[A-Za-z0-9]{10,99}\b/g
|
|
387
|
+
),
|
|
388
|
+
regexRule(
|
|
389
|
+
// gitleaks `anthropic-api-key`. Must precede the openai rules: both start
|
|
390
|
+
// `sk-`, and whichever rule claims the span first wins.
|
|
391
|
+
"anthropic-api-key",
|
|
392
|
+
"anthropic",
|
|
393
|
+
"gitleaks anthropic-api-key (MIT)",
|
|
394
|
+
/\bsk-ant-(?:api|admin)[0-9]{2}-[A-Za-z0-9_-]{80,120}\b/g
|
|
395
|
+
),
|
|
396
|
+
regexRule(
|
|
397
|
+
// gitleaks `openai-api-key`: the `T3BlbkFJ` infix is the base64 of
|
|
398
|
+
// "OpenAI" and is what makes this rule safe to run over prose.
|
|
399
|
+
"openai-api-key-project",
|
|
400
|
+
"openai",
|
|
401
|
+
"gitleaks openai-api-key (MIT)",
|
|
402
|
+
/\bsk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}T3BlbkFJ[A-Za-z0-9_-]{20,}\b/g
|
|
403
|
+
),
|
|
404
|
+
regexRule(
|
|
405
|
+
// gitleaks `openai-api-key`, legacy 48-character form with no infix.
|
|
406
|
+
"openai-api-key-legacy",
|
|
407
|
+
"openai",
|
|
408
|
+
"gitleaks openai-api-key (MIT)",
|
|
409
|
+
/\bsk-[A-Za-z0-9]{48}\b/g
|
|
410
|
+
),
|
|
411
|
+
regexRule(
|
|
412
|
+
// gitleaks `npm-access-token`; secretlint-rule-npm.
|
|
413
|
+
"npm-access-token",
|
|
414
|
+
"npm",
|
|
415
|
+
"gitleaks npm-access-token / secretlint-rule-npm (MIT)",
|
|
416
|
+
/\bnpm_[A-Za-z0-9]{36}\b/g
|
|
417
|
+
),
|
|
418
|
+
// ---- credentials in urls -----------------------------------------------
|
|
419
|
+
regexRule(
|
|
420
|
+
// gitleaks `authenticated-url`; secretlint `-rule-basicauth`. Only the
|
|
421
|
+
// password is masked: `postgres://app:‹redacted:basic-auth:…›@db:5432/x`
|
|
422
|
+
// still says which host and which user, which is the point of an index that
|
|
423
|
+
// stays searchable by shape.
|
|
424
|
+
"basic-auth-url",
|
|
425
|
+
"basic-auth",
|
|
426
|
+
"gitleaks authenticated-url / secretlint-rule-basicauth (MIT)",
|
|
427
|
+
// The user half is `{0,64}`: `redis://:hunter2@host` has an empty user and
|
|
428
|
+
// is exactly as leaked as the two-part form.
|
|
429
|
+
/\b[a-zA-Z][a-zA-Z0-9+.-]{1,20}:\/\/[^\s:@/]{0,64}:([^\s:@/]{1,128})@/g,
|
|
430
|
+
{
|
|
431
|
+
group: 1,
|
|
432
|
+
validate: (value) => {
|
|
433
|
+
if (INTERPOLATION_RE.test(value)) return false;
|
|
434
|
+
if (PLACEHOLDER_RE.test(value)) return false;
|
|
435
|
+
if (MASK_FRAGMENT_RE.test(value)) return false;
|
|
436
|
+
if (/^(?:pass|pw|user|admin|root|test|guest|\*+|x+|\d{1,4})$/i.test(value)) return false;
|
|
437
|
+
return true;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
),
|
|
441
|
+
// ---- credential context (03 §5) ----------------------------------------
|
|
442
|
+
bearerRule,
|
|
443
|
+
genericAssignmentRule,
|
|
444
|
+
socketAuthRule,
|
|
445
|
+
// ---- entropy (03 §5) ---------------------------------------------------
|
|
446
|
+
// Last, so that anything a named rule understands is reported under its real
|
|
447
|
+
// type and a bare token is the fallback rather than the default.
|
|
448
|
+
highEntropyRule
|
|
449
|
+
];
|
|
450
|
+
|
|
451
|
+
// packages/core/src/redact-elide.ts
|
|
452
|
+
function emptyElisions() {
|
|
453
|
+
return { binaryParts: 0, charsElided: 0 };
|
|
454
|
+
}
|
|
455
|
+
var MIN_PAYLOAD = 512;
|
|
456
|
+
var BASE64_MAGIC = [
|
|
457
|
+
["iVBORw0KGgo", "image/png", "89 50 4E 47 0D 0A 1A 0A"],
|
|
458
|
+
["/9j/", "image/jpeg", "FF D8 FF (JFIF/Exif SOI)"],
|
|
459
|
+
["R0lGODdh", "image/gif", "GIF87a"],
|
|
460
|
+
["R0lGODlh", "image/gif", "GIF89a"],
|
|
461
|
+
["Qk0", "image/bmp", '42 4D ("BM")'],
|
|
462
|
+
["SUkqAA", "image/tiff", "49 49 2A 00 (little-endian TIFF)"],
|
|
463
|
+
["TU0AK", "image/tiff", "4D 4D 00 2A (big-endian TIFF)"],
|
|
464
|
+
["AAABAA", "image/x-icon", "00 00 01 00 (ICO)"],
|
|
465
|
+
["UklGR", "application/octet-stream", '52 49 46 46 ("RIFF" \u2014 webp/wav/avi)'],
|
|
466
|
+
["JVBERi0", "application/pdf", '25 50 44 46 2D ("%PDF-")'],
|
|
467
|
+
["UEsDB", "application/zip", '50 4B 03 04 ("PK\\x03\\x04" \u2014 zip/docx/xlsx)'],
|
|
468
|
+
["H4sI", "application/gzip", "1F 8B 08"],
|
|
469
|
+
["SUQz", "audio/mpeg", '49 44 33 ("ID3")'],
|
|
470
|
+
["T2dnU", "application/ogg", '4F 67 67 53 ("OggS")'],
|
|
471
|
+
["d09GRg", "font/woff", '77 4F 46 46 ("wOFF")'],
|
|
472
|
+
["d09GMg", "font/woff2", '77 4F 46 32 ("wOF2")'],
|
|
473
|
+
["AAEAAA", "font/ttf", "00 01 00 00 (TrueType)"],
|
|
474
|
+
["T1RUTw", "font/otf", '4F 54 54 4F ("OTTO")']
|
|
475
|
+
];
|
|
476
|
+
var B64_RUN = `(?:[A-Za-z0-9+/=]|\\\\/)`;
|
|
477
|
+
var B64_BODY = `${B64_RUN}{40,}(?:[\\r\\n \\t]+${B64_RUN}{40,})*`;
|
|
478
|
+
var DATA_URI = new RegExp(
|
|
479
|
+
`data:([a-zA-Z0-9.+-]+/[a-zA-Z0-9.+-]+)?(?:;[a-zA-Z0-9.+=-]+)*;base64,${B64_BODY}`,
|
|
480
|
+
"g"
|
|
481
|
+
);
|
|
482
|
+
var CONTENT_BLOCK = new RegExp(
|
|
483
|
+
`"(?:data|image_data|b64_json|base64)"\\s*:\\s*"(${B64_BODY})"`,
|
|
484
|
+
"g"
|
|
485
|
+
);
|
|
486
|
+
var MEDIA_HINT = /"(?:media_?type|mime_?type|mimeType)"\s*:\s*"([a-zA-Z0-9.+-]+\/[a-zA-Z0-9.+-]+)"|"type"\s*:\s*"base64"/gi;
|
|
487
|
+
var HINT_WINDOW = 200;
|
|
488
|
+
var MAGIC_PREFIXES = BASE64_MAGIC.map(([p]) => p.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")).join("|");
|
|
489
|
+
var BARE_RUN = new RegExp(
|
|
490
|
+
`(?<![A-Za-z0-9+/=])(?:${MAGIC_PREFIXES})${B64_BODY}`,
|
|
491
|
+
"g"
|
|
492
|
+
);
|
|
493
|
+
function magicOf(payload) {
|
|
494
|
+
const head = payload.slice(0, 32).replace(/\\\//g, "/").replace(/\s+/g, "");
|
|
495
|
+
for (const [prefix, mime] of BASE64_MAGIC) if (head.startsWith(prefix)) return mime;
|
|
496
|
+
return void 0;
|
|
497
|
+
}
|
|
498
|
+
function mimeNear(text, start, end) {
|
|
499
|
+
const from = Math.max(0, start - HINT_WINDOW);
|
|
500
|
+
const window = text.slice(from, start) + text.slice(end, Math.min(text.length, end + HINT_WINDOW));
|
|
501
|
+
const rx = new RegExp(MEDIA_HINT.source, MEDIA_HINT.flags);
|
|
502
|
+
let m;
|
|
503
|
+
let sawBase64 = false;
|
|
504
|
+
while ((m = rx.exec(window)) !== null) {
|
|
505
|
+
if (m[1]) return m[1];
|
|
506
|
+
sawBase64 = true;
|
|
507
|
+
}
|
|
508
|
+
return sawBase64 ? "application/octet-stream" : void 0;
|
|
509
|
+
}
|
|
510
|
+
function collect(re, text, group, decide, out) {
|
|
511
|
+
const rx = new RegExp(re.source, re.flags);
|
|
512
|
+
let m;
|
|
513
|
+
while ((m = rx.exec(text)) !== null) {
|
|
514
|
+
const payload = group === 0 ? m[0] : m[group];
|
|
515
|
+
if (payload === void 0 || payload.length === 0) {
|
|
516
|
+
rx.lastIndex += 1;
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
const start = m.index + (group === 0 ? 0 : m[0].indexOf(payload));
|
|
520
|
+
const end = start + payload.length;
|
|
521
|
+
const mime = decide(payload, start, end);
|
|
522
|
+
if (mime === void 0) continue;
|
|
523
|
+
out.push({ start, end, mime });
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
function elideBinary(text, tally = emptyElisions()) {
|
|
527
|
+
if (typeof text !== "string" || text.length < MIN_PAYLOAD) return text ?? "";
|
|
528
|
+
if (!/[A-Za-z0-9+/]{64}/.test(text)) return text;
|
|
529
|
+
const spans = [];
|
|
530
|
+
const big = (payload) => payload.length >= MIN_PAYLOAD;
|
|
531
|
+
collect(DATA_URI, text, 0, (payload) => {
|
|
532
|
+
if (!big(payload)) return void 0;
|
|
533
|
+
const m = /^data:([a-zA-Z0-9.+-]+\/[a-zA-Z0-9.+-]+)?/.exec(payload);
|
|
534
|
+
return m?.[1] ?? "application/octet-stream";
|
|
535
|
+
}, spans);
|
|
536
|
+
collect(CONTENT_BLOCK, text, 1, (payload, start, end) => {
|
|
537
|
+
if (!big(payload)) return void 0;
|
|
538
|
+
const hint = mimeNear(text, start, end);
|
|
539
|
+
if (hint === void 0) return void 0;
|
|
540
|
+
return magicOf(payload) ?? hint;
|
|
541
|
+
}, spans);
|
|
542
|
+
collect(BARE_RUN, text, 0, (payload) => big(payload) ? magicOf(payload) : void 0, spans);
|
|
543
|
+
if (spans.length === 0) return text;
|
|
544
|
+
spans.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
545
|
+
const out = [];
|
|
546
|
+
let cursor = 0;
|
|
547
|
+
for (const s of spans) {
|
|
548
|
+
if (s.start < cursor) continue;
|
|
549
|
+
out.push(text.slice(cursor, s.start));
|
|
550
|
+
const n = s.end - s.start;
|
|
551
|
+
out.push(`\u2039elided:${s.mime}:${n} bytes\u203A`);
|
|
552
|
+
tally.binaryParts += 1;
|
|
553
|
+
tally.charsElided += n;
|
|
554
|
+
cursor = s.end;
|
|
555
|
+
}
|
|
556
|
+
out.push(text.slice(cursor));
|
|
557
|
+
return out.join("");
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// packages/core/src/redact.ts
|
|
561
|
+
var OPEN = "\u2039";
|
|
562
|
+
var CLOSE = "\u203A";
|
|
563
|
+
var MASK_RE = new RegExp(`${OPEN}redacted:[a-z-]+:[0-9a-f]{8}${CLOSE}`, "g");
|
|
564
|
+
function secretDigest(secret) {
|
|
565
|
+
return createHash2("sha256").update(secret, "utf8").digest("hex").slice(0, 8);
|
|
566
|
+
}
|
|
567
|
+
function redact(text) {
|
|
568
|
+
if (typeof text !== "string" || text.length === 0) return { text: text ?? "", hits: [] };
|
|
569
|
+
const claimed = new Uint8Array(text.length);
|
|
570
|
+
const spans = [];
|
|
571
|
+
for (const span of protectedSpans(text)) claim(claimed, span);
|
|
572
|
+
for (const rule of RULES) {
|
|
573
|
+
for (const m of rule.scan(text)) {
|
|
574
|
+
if (m.start < 0 || m.end > text.length || m.end <= m.start) continue;
|
|
575
|
+
if (isClaimed(claimed, m)) continue;
|
|
576
|
+
claim(claimed, m);
|
|
577
|
+
spans.push({ start: m.start, end: m.end, rule, value: m.value });
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (spans.length === 0) return { text, hits: [] };
|
|
581
|
+
spans.sort((a, b) => a.start - b.start);
|
|
582
|
+
const out = [];
|
|
583
|
+
const hits = [];
|
|
584
|
+
let cursor = 0;
|
|
585
|
+
for (const s of spans) {
|
|
586
|
+
out.push(text.slice(cursor, s.start));
|
|
587
|
+
const sha8 = secretDigest(s.value);
|
|
588
|
+
out.push(`${OPEN}redacted:${s.rule.type}:${sha8}${CLOSE}`);
|
|
589
|
+
hits.push({ type: s.rule.type, rule: s.rule.id, sha8, start: s.start, length: s.end - s.start });
|
|
590
|
+
cursor = s.end;
|
|
591
|
+
}
|
|
592
|
+
out.push(text.slice(cursor));
|
|
593
|
+
return { text: out.join(""), hits };
|
|
594
|
+
}
|
|
595
|
+
function protectedSpans(text) {
|
|
596
|
+
const spans = [];
|
|
597
|
+
const patterns = [MASK_RE, ...ALLOW_SPANS];
|
|
598
|
+
for (const p of patterns) {
|
|
599
|
+
const rx = new RegExp(p.source, p.flags.includes("g") ? p.flags : p.flags + "g");
|
|
600
|
+
let m;
|
|
601
|
+
while ((m = rx.exec(text)) !== null) {
|
|
602
|
+
if (m[0].length === 0) {
|
|
603
|
+
rx.lastIndex++;
|
|
604
|
+
continue;
|
|
605
|
+
}
|
|
606
|
+
spans.push({ start: m.index, end: m.index + m[0].length });
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return spans;
|
|
610
|
+
}
|
|
611
|
+
function isClaimed(claimed, s) {
|
|
612
|
+
for (let i = s.start; i < s.end; i++) if (claimed[i]) return true;
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
function claim(claimed, s) {
|
|
616
|
+
const end = Math.min(s.end, claimed.length);
|
|
617
|
+
for (let i = Math.max(0, s.start); i < end; i++) claimed[i] = 1;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
// packages/core/src/analytics/source.ts
|
|
621
|
+
var digest = (value) => createHash3("sha256").update(value).digest("hex");
|
|
622
|
+
var clean = (text) => redact(elideBinary(text)).text;
|
|
623
|
+
var clock = (value) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null;
|
|
624
|
+
|
|
625
|
+
// packages/core/src/analytics/native-metadata-projection.ts
|
|
626
|
+
import { StringDecoder } from "node:string_decoder";
|
|
627
|
+
var NativeMetadataProjection = class {
|
|
628
|
+
constructor(maxBytes = 8 * 1024 * 1024, harness = null) {
|
|
629
|
+
this.maxBytes = maxBytes;
|
|
630
|
+
this.harness = harness;
|
|
631
|
+
}
|
|
632
|
+
decoder = new StringDecoder("utf8");
|
|
633
|
+
frames = [];
|
|
634
|
+
root;
|
|
635
|
+
done = false;
|
|
636
|
+
mode = "idle";
|
|
637
|
+
token = "";
|
|
638
|
+
isKey = false;
|
|
639
|
+
keep = false;
|
|
640
|
+
escape = false;
|
|
641
|
+
unicode = "";
|
|
642
|
+
markerTail = "";
|
|
643
|
+
retained = 0;
|
|
644
|
+
error = null;
|
|
645
|
+
marker = false;
|
|
646
|
+
scanString = false;
|
|
647
|
+
scanEscape = false;
|
|
648
|
+
scanUnicode = "";
|
|
649
|
+
scanTail = "";
|
|
650
|
+
// Defer the producer check until the whole record is known, retaining unknown values only within the shared metadata budget.
|
|
651
|
+
imageBodies = [];
|
|
652
|
+
imageCandidate = null;
|
|
653
|
+
imageBytes = 0;
|
|
654
|
+
trimImages() {
|
|
655
|
+
for (let i = this.imageBodies.length - 1; i >= 0 && this.retained + this.imageBytes > this.maxBytes; i--) {
|
|
656
|
+
const body = this.imageBodies[i];
|
|
657
|
+
this.imageBytes -= body.bytes;
|
|
658
|
+
body.chunks = [];
|
|
659
|
+
body.bytes = 0;
|
|
660
|
+
body.overflow = true;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
fail(code2 = "native_metadata_invalid") {
|
|
664
|
+
this.error ??= code2;
|
|
665
|
+
}
|
|
666
|
+
add(c) {
|
|
667
|
+
if (this.keep || this.isKey) {
|
|
668
|
+
this.retained += Buffer.byteLength(JSON.stringify(c)) - 2;
|
|
669
|
+
if (this.retained > this.maxBytes || this.isKey && this.token.length > 32768) {
|
|
670
|
+
this.fail("native_metadata_bytes_limit");
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
this.token += c;
|
|
674
|
+
this.trimImages();
|
|
675
|
+
}
|
|
676
|
+
if (this.imageCandidate && !this.imageCandidate.overflow) {
|
|
677
|
+
const bytes = Buffer.byteLength(JSON.stringify(c)) - 2;
|
|
678
|
+
this.imageCandidate.bytes += bytes;
|
|
679
|
+
this.imageBytes += bytes;
|
|
680
|
+
this.imageCandidate.chunks.push(c);
|
|
681
|
+
this.trimImages();
|
|
682
|
+
}
|
|
683
|
+
const searchable = this.markerTail + c;
|
|
684
|
+
if (EXCLUSION_MARKERS.some((m) => searchable.includes(m))) this.marker = true;
|
|
685
|
+
this.markerTail = searchable.slice(-256);
|
|
686
|
+
}
|
|
687
|
+
expectedValue() {
|
|
688
|
+
const f = this.frames.at(-1);
|
|
689
|
+
return f ? f.state === "value" || f.state === "valueRequired" : !this.done;
|
|
690
|
+
}
|
|
691
|
+
imageBody(f) {
|
|
692
|
+
return this.harness === "codex" && f.key === "image_url" && f.path.length === 3 && f.path[0] === "payload" && f.path[1] === "output" && f.path[2] === "[]";
|
|
693
|
+
}
|
|
694
|
+
retainValue() {
|
|
695
|
+
const f = this.frames.at(-1);
|
|
696
|
+
return !f || f.keep && !(f.key === "content" || f.key === "text" || f.omitImage);
|
|
697
|
+
}
|
|
698
|
+
accept(value) {
|
|
699
|
+
if (this.retainValue()) {
|
|
700
|
+
this.retained += 16;
|
|
701
|
+
this.trimImages();
|
|
702
|
+
if (this.retained > this.maxBytes) {
|
|
703
|
+
this.fail("native_metadata_bytes_limit");
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
const f = this.frames.at(-1);
|
|
708
|
+
if (!f) {
|
|
709
|
+
if (this.done) {
|
|
710
|
+
this.fail();
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
this.root = value;
|
|
714
|
+
this.done = true;
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
if (f.state !== "value" && f.state !== "valueRequired") {
|
|
718
|
+
this.fail();
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
if (f.keep) {
|
|
722
|
+
if (f.array) f.value.push(value);
|
|
723
|
+
else if (f.key !== null && this.retainValue()) Object.defineProperty(f.value, f.key, { value, writable: true, enumerable: true, configurable: true });
|
|
724
|
+
}
|
|
725
|
+
f.key = null;
|
|
726
|
+
f.omitImage = false;
|
|
727
|
+
f.state = "comma";
|
|
728
|
+
}
|
|
729
|
+
scalarEnd() {
|
|
730
|
+
if (this.mode === "number") {
|
|
731
|
+
if (!/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(this.token)) {
|
|
732
|
+
this.fail();
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
this.accept(this.keep ? Number(this.token) : null);
|
|
736
|
+
} else if (this.mode === "literal") {
|
|
737
|
+
if (!["true", "false", "null"].includes(this.token)) {
|
|
738
|
+
this.fail();
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
this.accept(this.keep ? JSON.parse(this.token) : null);
|
|
742
|
+
}
|
|
743
|
+
this.mode = "idle";
|
|
744
|
+
this.token = "";
|
|
745
|
+
}
|
|
746
|
+
char(c) {
|
|
747
|
+
if (this.error) return;
|
|
748
|
+
if (this.mode === "string") {
|
|
749
|
+
if (this.unicode) {
|
|
750
|
+
if (!/[0-9a-f]/i.test(c)) {
|
|
751
|
+
this.fail();
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
this.unicode += c;
|
|
755
|
+
if (this.unicode.length === 5) {
|
|
756
|
+
this.add(String.fromCharCode(parseInt(this.unicode.slice(1), 16)));
|
|
757
|
+
this.unicode = "";
|
|
758
|
+
}
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
if (this.escape) {
|
|
762
|
+
this.escape = false;
|
|
763
|
+
if (c === "u") {
|
|
764
|
+
this.unicode = "u";
|
|
765
|
+
return;
|
|
766
|
+
}
|
|
767
|
+
const decoded = { '"': '"', "\\": "\\", "/": "/", "b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": " " };
|
|
768
|
+
if (!(c in decoded)) {
|
|
769
|
+
this.fail();
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
this.add(decoded[c]);
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
if (c === "\\") {
|
|
776
|
+
this.escape = true;
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
if (c === '"') {
|
|
780
|
+
if (this.isKey) {
|
|
781
|
+
const f2 = this.frames.at(-1);
|
|
782
|
+
f2.key = this.token;
|
|
783
|
+
f2.state = "colon";
|
|
784
|
+
} else this.accept(this.keep ? this.token : null);
|
|
785
|
+
this.mode = "idle";
|
|
786
|
+
this.token = "";
|
|
787
|
+
this.imageCandidate = null;
|
|
788
|
+
this.markerTail = "";
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
if (c.charCodeAt(0) < 32) {
|
|
792
|
+
this.fail();
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
this.add(c);
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
if (this.mode === "number" || this.mode === "literal") {
|
|
799
|
+
const allowed = this.mode === "number" ? /[0-9eE+.-]/ : /[a-z]/;
|
|
800
|
+
if (allowed.test(c)) {
|
|
801
|
+
if (this.token.length >= 256) {
|
|
802
|
+
this.fail("native_metadata_scalar_limit");
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
this.token += c;
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
this.scalarEnd();
|
|
809
|
+
if (this.error) return;
|
|
810
|
+
}
|
|
811
|
+
if (/\s/.test(c)) {
|
|
812
|
+
if (![" ", " ", "\r", "\n"].includes(c)) this.fail();
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
const f = this.frames.at(-1);
|
|
816
|
+
if (c === '"') {
|
|
817
|
+
this.isKey = !!f && !f.array && (f.state === "key" || f.state === "keyRequired");
|
|
818
|
+
if (!this.isKey && !this.expectedValue()) {
|
|
819
|
+
this.fail();
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
if (!this.isKey && f && this.imageBody(f)) {
|
|
823
|
+
f.omitImage = true;
|
|
824
|
+
this.imageCandidate = { frame: f, chunks: [], bytes: 0, overflow: false };
|
|
825
|
+
this.imageBodies.push(this.imageCandidate);
|
|
826
|
+
}
|
|
827
|
+
this.keep = this.isKey || this.retainValue() && f?.key !== "message";
|
|
828
|
+
this.token = "";
|
|
829
|
+
this.markerTail = "";
|
|
830
|
+
this.mode = "string";
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
if (c === "{" || c === "[") {
|
|
834
|
+
if (!this.expectedValue() || this.frames.length >= 128) {
|
|
835
|
+
this.fail("native_metadata_depth_limit");
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
const keep = this.retainValue();
|
|
839
|
+
this.frames.push({ path: f ? [...f.path, f.array ? "[]" : f.key] : [], omitImage: false, array: c === "[", state: c === "[" ? "value" : "key", key: null, keep, value: keep ? c === "[" ? [] : {} : null });
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
if (c === "}" || c === "]") {
|
|
843
|
+
if (!f || f.array !== (c === "]") || !["comma", f.array ? "value" : "key"].includes(f.state)) {
|
|
844
|
+
this.fail();
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
this.frames.pop();
|
|
848
|
+
this.accept(f.value);
|
|
849
|
+
return;
|
|
850
|
+
}
|
|
851
|
+
if (c === ":") {
|
|
852
|
+
if (!f || f.state !== "colon") {
|
|
853
|
+
this.fail();
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
f.state = "valueRequired";
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
if (c === ",") {
|
|
860
|
+
if (!f || f.state !== "comma") {
|
|
861
|
+
this.fail();
|
|
862
|
+
return;
|
|
863
|
+
}
|
|
864
|
+
f.state = f.array ? "valueRequired" : "keyRequired";
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
if (!this.expectedValue()) {
|
|
868
|
+
this.fail();
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
871
|
+
this.keep = this.retainValue();
|
|
872
|
+
this.token = c;
|
|
873
|
+
if (c === "-" || /[0-9]/.test(c)) this.mode = "number";
|
|
874
|
+
else if (/[tfn]/.test(c)) this.mode = "literal";
|
|
875
|
+
else this.fail();
|
|
876
|
+
}
|
|
877
|
+
scanMarkers(text) {
|
|
878
|
+
const special = /["\\]/g;
|
|
879
|
+
let at2 = 0;
|
|
880
|
+
const add = (value) => {
|
|
881
|
+
const combined = this.scanTail + value;
|
|
882
|
+
if (EXCLUSION_MARKERS.some((m) => combined.includes(m))) this.marker = true;
|
|
883
|
+
this.scanTail = combined.slice(-256);
|
|
884
|
+
};
|
|
885
|
+
while (at2 < text.length) {
|
|
886
|
+
if (this.scanString && !this.scanEscape && !this.scanUnicode) {
|
|
887
|
+
special.lastIndex = at2;
|
|
888
|
+
const hit = special.exec(text), end = hit?.index ?? text.length;
|
|
889
|
+
add(text.slice(at2, end));
|
|
890
|
+
at2 = end;
|
|
891
|
+
if (at2 >= text.length) break;
|
|
892
|
+
}
|
|
893
|
+
const c = text[at2++];
|
|
894
|
+
if (this.scanUnicode) {
|
|
895
|
+
this.scanUnicode += c;
|
|
896
|
+
if (this.scanUnicode.length === 5) {
|
|
897
|
+
if (/^u[0-9a-f]{4}$/i.test(this.scanUnicode)) add(String.fromCharCode(parseInt(this.scanUnicode.slice(1), 16)));
|
|
898
|
+
this.scanUnicode = "";
|
|
899
|
+
}
|
|
900
|
+
continue;
|
|
901
|
+
}
|
|
902
|
+
if (this.scanEscape) {
|
|
903
|
+
this.scanEscape = false;
|
|
904
|
+
if (c === "u") this.scanUnicode = "u";
|
|
905
|
+
else add({ n: "\n", r: "\r", t: " " }[c] ?? c);
|
|
906
|
+
continue;
|
|
907
|
+
}
|
|
908
|
+
if (c === '"') {
|
|
909
|
+
this.scanString = !this.scanString;
|
|
910
|
+
this.scanTail = "";
|
|
911
|
+
} else if (this.scanString && c === "\\") this.scanEscape = true;
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
push(bytes) {
|
|
915
|
+
const text = this.decoder.write(bytes);
|
|
916
|
+
this.scanMarkers(text);
|
|
917
|
+
const special = /["\\\x00-\x1f]/g;
|
|
918
|
+
let at2 = 0;
|
|
919
|
+
while (at2 < text.length && !this.error) {
|
|
920
|
+
if (this.mode === "string" && !this.escape && !this.unicode) {
|
|
921
|
+
special.lastIndex = at2;
|
|
922
|
+
const hit = special.exec(text), end = hit?.index ?? text.length;
|
|
923
|
+
if (end > at2) this.add(text.slice(at2, end));
|
|
924
|
+
at2 = end;
|
|
925
|
+
if (at2 >= text.length) break;
|
|
926
|
+
}
|
|
927
|
+
this.char(text[at2++]);
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
finish() {
|
|
931
|
+
for (const c of this.decoder.end()) this.char(c);
|
|
932
|
+
if (this.mode === "number" || this.mode === "literal") this.scalarEnd();
|
|
933
|
+
if (this.mode !== "idle" || this.frames.length || !this.done) this.fail();
|
|
934
|
+
if (!this.error && (!this.root || typeof this.root !== "object" || Array.isArray(this.root))) this.fail();
|
|
935
|
+
if (!this.error && this.imageBodies.length) {
|
|
936
|
+
const root = this.root, payload = root.payload;
|
|
937
|
+
for (const body of this.imageBodies) {
|
|
938
|
+
if (root.type === "response_item" && (payload?.type === "custom_tool_call_output" || payload?.type === "function_call_output") && body.frame.value?.type === "input_image") continue;
|
|
939
|
+
if (body.overflow) this.fail("native_metadata_bytes_limit");
|
|
940
|
+
else Object.defineProperty(body.frame.value, "image_url", { value: body.chunks.join(""), writable: true, enumerable: true, configurable: true });
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
if (!this.error && Buffer.byteLength(JSON.stringify(this.root)) > this.maxBytes) this.fail("native_metadata_bytes_limit");
|
|
944
|
+
return { record: this.error ? null : this.root, code: this.error, exclusionMarker: this.marker };
|
|
945
|
+
}
|
|
946
|
+
};
|
|
947
|
+
|
|
948
|
+
// packages/core/src/analytics/native-usage.ts
|
|
949
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
950
|
+
var obj = (v) => v !== null && typeof v === "object" && !Array.isArray(v) ? v : {};
|
|
951
|
+
var str = (...v) => v.find((x) => typeof x === "string" && x.length > 0) ?? null;
|
|
952
|
+
var num2 = (v) => typeof v === "number" && Number.isSafeInteger(v) && v >= 0 ? v : null;
|
|
953
|
+
var cost = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
|
|
954
|
+
var hash = (v) => createHash4("sha256").update(v).digest("hex");
|
|
955
|
+
var at = (v) => typeof v === "number" && Number.isFinite(v) && Math.abs(v) < 864e13 ? new Date(v).toISOString() : typeof v === "string" && Number.isFinite(Date.parse(v)) ? new Date(v).toISOString() : null;
|
|
956
|
+
function createNativeUsageAccumulator(harness, conversationId, options = {}) {
|
|
957
|
+
let observed = 0, excluded = 0;
|
|
958
|
+
const records = /* @__PURE__ */ new Map();
|
|
959
|
+
let project = options.project ?? null, model = null, provider = null, session = conversationId, offset = 0, seq = 0;
|
|
960
|
+
let previous = null;
|
|
961
|
+
const seenTotals = /* @__PURE__ */ new Set();
|
|
962
|
+
const identities = /* @__PURE__ */ new Map();
|
|
963
|
+
const score = (v) => (v.inputTokens ?? 0) + (v.outputTokens ?? 0) + (v.cacheReadTokens ?? 0) + Math.max(v.cacheWriteTokens ?? 0, (v.cacheWrite5mTokens ?? 0) + (v.cacheWrite1hTokens ?? 0)) + (v.reasoningTokens ?? 0);
|
|
964
|
+
const put = (r) => {
|
|
965
|
+
const old = records.get(r.id), identity = identities.get(r.id) ?? { models: /* @__PURE__ */ new Set(), providers: /* @__PURE__ */ new Set() };
|
|
966
|
+
if (r.model !== null) identity.models.add(r.model);
|
|
967
|
+
if (r.provider !== null) identity.providers.add(r.provider);
|
|
968
|
+
identities.set(r.id, identity);
|
|
969
|
+
const chosen = !old || score(r) > score(old) ? r : old;
|
|
970
|
+
const conflict = identity.models.size > 1 || identity.providers.size > 1;
|
|
971
|
+
records.set(r.id, conflict ? { ...chosen, model: null, provider: null, gaps: [.../* @__PURE__ */ new Set([...chosen.gaps, "response_identity_model_conflict"])] } : chosen);
|
|
972
|
+
};
|
|
973
|
+
const pushRecord = (record, position) => {
|
|
974
|
+
const start = position?.rawStart ?? offset;
|
|
975
|
+
if (position) offset = position.rawEnd;
|
|
976
|
+
if (++seq > (options.maxRecords ?? 1e5)) return;
|
|
977
|
+
let r = record;
|
|
978
|
+
if (harness === "opencode") {
|
|
979
|
+
r = { ...r, ...obj(r.message) };
|
|
980
|
+
const wrapped = r.data ?? r.content;
|
|
981
|
+
if (typeof wrapped === "string") {
|
|
982
|
+
try {
|
|
983
|
+
r = { ...r, ...obj(JSON.parse(wrapped)) };
|
|
984
|
+
} catch {
|
|
985
|
+
}
|
|
986
|
+
} else if (wrapped && typeof wrapped === "object") r = { ...r, ...obj(wrapped) };
|
|
987
|
+
}
|
|
988
|
+
const p = obj(r.payload), m = obj(r.message);
|
|
989
|
+
project = str(r.cwd, p.cwd, r.directory) ?? project;
|
|
990
|
+
if (harness === "claude") session = str(r.sessionId) ?? session;
|
|
991
|
+
if (harness === "pi" && r.type === "session") session = str(r.id) ?? session;
|
|
992
|
+
if (harness === "codex" && (r.type === "session_meta" || r.type === "turn_context")) {
|
|
993
|
+
session = str(p.session_id, p.id) ?? session;
|
|
994
|
+
model = str(p.model) ?? model;
|
|
995
|
+
provider = str(p.model_provider, p.provider) ?? provider;
|
|
996
|
+
return;
|
|
997
|
+
}
|
|
998
|
+
const eventAt = at(r.timestamp) ?? at(m.timestamp) ?? at(obj(r.time).created);
|
|
999
|
+
const key = str(r.uuid, r.id, p.id, m.id) ?? `record:${seq}`;
|
|
1000
|
+
const accepted = options.acceptRecord?.(r, { key, rawStart: start, rawEnd: offset, eventAt, project }) ?? true;
|
|
1001
|
+
let input = null, output = null, read = null, write = null, reasoning = null, reported = null;
|
|
1002
|
+
let observedModel = null, observedProvider = null, id, basis, includesCache = false, includesReasoning = true;
|
|
1003
|
+
const gaps = [];
|
|
1004
|
+
if (harness === "codex") {
|
|
1005
|
+
if (r.type !== "event_msg" || p.type !== "token_count") return;
|
|
1006
|
+
const info = obj(p.info), last = obj(info.last_token_usage), total = obj(info.total_token_usage);
|
|
1007
|
+
if (!Object.keys(last).length && !Object.keys(total).length) return;
|
|
1008
|
+
observed++;
|
|
1009
|
+
const previousTotal = previous;
|
|
1010
|
+
previous = Object.keys(total).length ? total : previous;
|
|
1011
|
+
const signature = Object.keys(total).length ? JSON.stringify([session, total.input_tokens, total.output_tokens, total.cached_input_tokens, total.reasoning_output_tokens]) : null;
|
|
1012
|
+
if (signature && seenTotals.has(signature)) return;
|
|
1013
|
+
if (signature) seenTotals.add(signature);
|
|
1014
|
+
const delta = (field) => {
|
|
1015
|
+
const current = num2(total[field]), prev = num2(previousTotal?.[field]);
|
|
1016
|
+
return current === null || prev === null || current < prev ? null : current - prev;
|
|
1017
|
+
};
|
|
1018
|
+
const firstComplete = previousTotal === null && num2(last.input_tokens) !== null && last.input_tokens === total.input_tokens && last.output_tokens === total.output_tokens;
|
|
1019
|
+
const bucket = (field) => num2(last[field]) ?? delta(field) ?? (firstComplete ? num2(total[field]) : null);
|
|
1020
|
+
input = bucket("input_tokens");
|
|
1021
|
+
output = bucket("output_tokens");
|
|
1022
|
+
read = bucket("cached_input_tokens");
|
|
1023
|
+
write = bucket("cache_write_input_tokens");
|
|
1024
|
+
reasoning = bucket("reasoning_output_tokens");
|
|
1025
|
+
if (!Object.keys(last).length && previousTotal === null) gaps.push("cumulative_baseline_unknown");
|
|
1026
|
+
if (previousTotal && Object.keys(total).length && ["input_tokens", "output_tokens"].some((k) => num2(total[k]) !== null && num2(previousTotal[k]) !== null && num2(total[k]) < num2(previousTotal[k]))) gaps.push("cumulative_counter_reset");
|
|
1027
|
+
observedModel = str(p.model, info.model) ?? model;
|
|
1028
|
+
observedProvider = str(p.model_provider, p.provider, info.provider) ?? provider;
|
|
1029
|
+
includesCache = true;
|
|
1030
|
+
basis = "codex_last_response_with_cumulative_dedup";
|
|
1031
|
+
id = `codex:${hash(JSON.stringify([session, str(p.response_id, info.response_id) ?? signature ?? [eventAt, last]]))}`;
|
|
1032
|
+
write = write ?? 0;
|
|
1033
|
+
} else if (harness === "claude") {
|
|
1034
|
+
if (r.type !== "assistant" || m.role !== "assistant") return;
|
|
1035
|
+
const u = obj(m.usage);
|
|
1036
|
+
input = num2(u.input_tokens);
|
|
1037
|
+
output = num2(u.output_tokens);
|
|
1038
|
+
read = num2(u.cache_read_input_tokens) ?? (input !== null ? 0 : null);
|
|
1039
|
+
write = num2(u.cache_creation_input_tokens) ?? (input !== null ? 0 : null);
|
|
1040
|
+
reasoning = num2(u.reasoning_tokens);
|
|
1041
|
+
const duration = obj(u.cache_creation);
|
|
1042
|
+
const five = num2(duration.ephemeral_5m_input_tokens), hour = num2(duration.ephemeral_1h_input_tokens);
|
|
1043
|
+
if (five !== null && hour !== null && write !== null && five + hour !== write) gaps.push("cache_write_duration_conflict");
|
|
1044
|
+
if (hour !== null && hour > 0 && (five === null || write === null || five + hour !== write)) gaps.push("cache_write_duration_unknown");
|
|
1045
|
+
observedModel = str(m.model, r.model);
|
|
1046
|
+
observedProvider = str(m.provider, r.provider, r.modelProvider);
|
|
1047
|
+
reported = cost(r.costUSD) ?? cost(m.costUSD);
|
|
1048
|
+
basis = "claude_response_usage";
|
|
1049
|
+
id = `claude:${hash(JSON.stringify([str(m.id, r.uuid) ?? key, str(r.requestId) ?? [session, eventAt]]))}`;
|
|
1050
|
+
} else if (harness === "pi") {
|
|
1051
|
+
if (r.type !== "message" || m.role !== "assistant") return;
|
|
1052
|
+
const u = obj(m.usage);
|
|
1053
|
+
input = num2(u.input);
|
|
1054
|
+
output = num2(u.output);
|
|
1055
|
+
read = num2(u.cacheRead) ?? (input !== null ? 0 : null);
|
|
1056
|
+
write = num2(u.cacheWrite) ?? (input !== null ? 0 : null);
|
|
1057
|
+
reasoning = num2(u.reasoning);
|
|
1058
|
+
reported = cost(obj(u.cost).total);
|
|
1059
|
+
observedModel = str(m.model);
|
|
1060
|
+
observedProvider = str(m.provider);
|
|
1061
|
+
basis = "pi_response_usage";
|
|
1062
|
+
id = `pi:${hash(JSON.stringify([session, str(r.id) ?? key, eventAt]))}`;
|
|
1063
|
+
} else {
|
|
1064
|
+
if (r.role !== "assistant") return;
|
|
1065
|
+
const u = obj(r.tokens), cache = obj(u.cache);
|
|
1066
|
+
input = num2(u.input);
|
|
1067
|
+
output = num2(u.output);
|
|
1068
|
+
read = num2(cache.read) ?? (input !== null ? 0 : null);
|
|
1069
|
+
write = num2(cache.write) ?? (input !== null ? 0 : null);
|
|
1070
|
+
reasoning = num2(u.reasoning);
|
|
1071
|
+
reported = cost(r.cost);
|
|
1072
|
+
observedModel = str(r.modelID, obj(r.model).modelID, obj(r.model).id, r.model);
|
|
1073
|
+
observedProvider = str(r.providerID, obj(r.model).providerID, r.provider);
|
|
1074
|
+
basis = "opencode_response_tokens";
|
|
1075
|
+
includesReasoning = false;
|
|
1076
|
+
id = `opencode:${str(r.id) ?? hash(JSON.stringify([conversationId, key, eventAt]))}`;
|
|
1077
|
+
}
|
|
1078
|
+
if (observedModel === "codex-auto-review") {
|
|
1079
|
+
observedModel = null;
|
|
1080
|
+
gaps.push("workflow_model_identity_unavailable");
|
|
1081
|
+
}
|
|
1082
|
+
if (harness !== "codex") observed++;
|
|
1083
|
+
if (r.isSynthetic === true || r.isMeta === true || m.isSynthetic === true || observedModel === "<synthetic>") {
|
|
1084
|
+
excluded++;
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
if (!accepted) {
|
|
1088
|
+
excluded++;
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
if (!observedModel) gaps.push("model_unrecorded");
|
|
1092
|
+
if (!observedProvider) gaps.push("provider_unrecorded");
|
|
1093
|
+
if (input === null || output === null) gaps.push("usage_partial");
|
|
1094
|
+
if (includesCache && read === null) gaps.push("cache_inclusion_unknown");
|
|
1095
|
+
if (includesCache && input !== null && read !== null && write !== null && read + write > input) gaps.push("cache_exceeds_input");
|
|
1096
|
+
if (includesReasoning && output !== null && reasoning !== null && reasoning > output) gaps.push("reasoning_exceeds_output");
|
|
1097
|
+
put({ id, conversationId, harness, eventAt, project, provider: observedProvider, model: observedModel, canonicalModel: null, inputTokens: input, outputTokens: output, cacheReadTokens: read, cacheWriteTokens: write, reasoningTokens: reasoning, inputIncludesCache: includesCache, outputIncludesReasoning: includesReasoning, reportedCostUsd: reported, basis, gaps, ...harness === "claude" ? { cacheWrite5mTokens: num2(obj(obj(m.usage).cache_creation).ephemeral_5m_input_tokens), cacheWrite1hTokens: num2(obj(obj(m.usage).cache_creation).ephemeral_1h_input_tokens) } : {} });
|
|
1098
|
+
};
|
|
1099
|
+
const pushLine = (line) => {
|
|
1100
|
+
const start = offset;
|
|
1101
|
+
offset += Buffer.byteLength(line) + 1;
|
|
1102
|
+
if (!line.trim()) return;
|
|
1103
|
+
let r;
|
|
1104
|
+
try {
|
|
1105
|
+
r = obj(JSON.parse(line));
|
|
1106
|
+
} catch {
|
|
1107
|
+
seq++;
|
|
1108
|
+
return;
|
|
1109
|
+
}
|
|
1110
|
+
pushRecord(r, { rawStart: start, rawEnd: offset });
|
|
1111
|
+
};
|
|
1112
|
+
return { pushRecord, invalidateContext: () => {
|
|
1113
|
+
project = null;
|
|
1114
|
+
model = null;
|
|
1115
|
+
provider = null;
|
|
1116
|
+
previous = null;
|
|
1117
|
+
}, pushLine, records: () => [...records.values()], counts: () => ({ observed, excluded, deduplicated: Math.max(0, observed - excluded - records.size) }) };
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// packages/core/src/analytics/native-stream.ts
|
|
1121
|
+
async function streamNativeFacts(file, harness, options) {
|
|
1122
|
+
const before = fs.statSync(file, { bigint: true }), captureSize = options.captureFileBytes ?? Number(before.size);
|
|
1123
|
+
if (!before.isFile() || !Number.isSafeInteger(captureSize) || captureSize < 0 || captureSize > options.maxBytes) throw new Error("native_usage_bytes_limit");
|
|
1124
|
+
if (before.size < BigInt(captureSize) || options.expectedIdentity && (String(before.dev) !== options.expectedIdentity.dev || String(before.ino) !== options.expectedIdentity.ino)) throw new Error("source_changed");
|
|
1125
|
+
let nativeId = path.basename(file, ".jsonl"), project = null, parent = null, records = 0, offset = 0, promptBytes = 0, maintenance = false, programmatic = false, currentModel = null, currentProvider = null;
|
|
1126
|
+
const events = [], language = [], gaps = /* @__PURE__ */ new Set(), seen = /* @__PURE__ */ new Set(), hash2 = createHash5("sha256");
|
|
1127
|
+
let usage = null;
|
|
1128
|
+
let held = [], heldBytes = 0, projection = null, projected = false;
|
|
1129
|
+
const turnLimit = options.captureBytes ?? 0, turns = [];
|
|
1130
|
+
let turnLines = [], turnBytes = 0, turnPrefix = "", keptBytes = 0, capturedRecords = 0, scopeUncertain = false;
|
|
1131
|
+
const finishTurn = () => {
|
|
1132
|
+
if (!turnLines.length) {
|
|
1133
|
+
if (turnLimit > 0 && turnBytes > 0) gaps.add("context_turn_oversize");
|
|
1134
|
+
turnBytes = 0;
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
if (turnBytes + Buffer.byteLength(turnPrefix) <= turnLimit) {
|
|
1138
|
+
turns.push({ lines: turnLines, bytes: turnBytes, prefix: turnPrefix });
|
|
1139
|
+
keptBytes += turnBytes;
|
|
1140
|
+
while (turns.length > 1 && keptBytes + Buffer.byteLength(turns[0].prefix) > turnLimit) keptBytes -= turns.shift().bytes;
|
|
1141
|
+
} else gaps.add("context_turn_oversize");
|
|
1142
|
+
turnLines = [];
|
|
1143
|
+
turnBytes = 0;
|
|
1144
|
+
};
|
|
1145
|
+
const process2 = (raw, end) => {
|
|
1146
|
+
if (++records > (options.maxRecords ?? 1e6)) {
|
|
1147
|
+
gaps.add("native_usage_records_limit");
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
const line = raw.toString("utf8");
|
|
1151
|
+
let r;
|
|
1152
|
+
try {
|
|
1153
|
+
const parsed = JSON.parse(line);
|
|
1154
|
+
if (!isRecord(parsed)) return;
|
|
1155
|
+
r = parsed;
|
|
1156
|
+
} catch {
|
|
1157
|
+
gaps.add("malformed_record");
|
|
1158
|
+
return;
|
|
1159
|
+
}
|
|
1160
|
+
if (r.entrypoint === "sdk-ts" || r.originator === "codex_exec") programmatic = true;
|
|
1161
|
+
const p = isRecord(r.payload) ? r.payload : r;
|
|
1162
|
+
if (p.originator === "codex_exec" || p.source === "exec") programmatic = true;
|
|
1163
|
+
const m = isRecord(r.message) ? r.message : {};
|
|
1164
|
+
if (typeof r.cwd === "string") project = r.cwd;
|
|
1165
|
+
if (typeof p.cwd === "string") project = p.cwd;
|
|
1166
|
+
if (harness === "claude" && typeof r.sessionId === "string") {
|
|
1167
|
+
if (file.includes(`${path.sep}subagents${path.sep}`) || r.isSidechain === true) {
|
|
1168
|
+
parent = r.sessionId;
|
|
1169
|
+
nativeId = `${parent}:${path.basename(file, ".jsonl")}`;
|
|
1170
|
+
} else nativeId = r.sessionId;
|
|
1171
|
+
}
|
|
1172
|
+
if (harness === "codex" && r.type === "session_meta") {
|
|
1173
|
+
nativeId = typeof p.id === "string" ? p.id : typeof p.session_id === "string" ? p.session_id : nativeId;
|
|
1174
|
+
const source = isRecord(p.source) ? p.source : {}, sub = isRecord(source.subagent) ? source.subagent : {}, spawn = isRecord(sub.thread_spawn) ? sub.thread_spawn : {};
|
|
1175
|
+
if (typeof spawn.parent_thread_id === "string") parent = spawn.parent_thread_id;
|
|
1176
|
+
}
|
|
1177
|
+
if (harness === "pi" && r.type === "session") {
|
|
1178
|
+
if (typeof r.id === "string") nativeId = r.id;
|
|
1179
|
+
if (typeof r.parentSessionId === "string") parent = r.parentSessionId;
|
|
1180
|
+
}
|
|
1181
|
+
if (r.type === "turn_context" || r.type === "session_meta" || r.type === "model_change") {
|
|
1182
|
+
currentModel = typeof p.model === "string" ? p.model : typeof r.modelId === "string" ? r.modelId : currentModel;
|
|
1183
|
+
currentProvider = typeof p.model_provider === "string" ? p.model_provider : typeof r.provider === "string" ? r.provider : currentProvider;
|
|
1184
|
+
}
|
|
1185
|
+
if (scopeUncertain && project !== null && (harness === "claude" && typeof r.sessionId === "string" || harness === "codex" && r.type === "session_meta" && (typeof p.id === "string" || typeof p.session_id === "string") || harness === "pi" && r.type === "session" && typeof r.id === "string")) scopeUncertain = false;
|
|
1186
|
+
if (options.sourceAllowed && !options.sourceAllowed(nativeId, project)) throw new Error("native_source_policy_excluded");
|
|
1187
|
+
if (hasExclusionMarker(extractTypedText(m.content)) || typeof p.message === "string" && hasExclusionMarker(p.message)) maintenance = true;
|
|
1188
|
+
const isInput = harness === "claude" && r.type === "user" && m.role === "user" && !(Array.isArray(m.content) && m.content.some((b) => isRecord(b) && b.type === "tool_result")) || harness === "codex" && r.type === "event_msg" && p.type === "user_message" || harness === "pi" && r.type === "message" && m.role === "user";
|
|
1189
|
+
if (isInput) {
|
|
1190
|
+
finishTurn();
|
|
1191
|
+
turnPrefix = JSON.stringify({ type: "turn_context", payload: { cwd: project, model: currentModel, model_provider: currentProvider } }) + "\n";
|
|
1192
|
+
}
|
|
1193
|
+
if (projected && turnLimit > 0) {
|
|
1194
|
+
turnLines = [];
|
|
1195
|
+
turnBytes = turnLimit + 1;
|
|
1196
|
+
gaps.add("context_turn_oversize");
|
|
1197
|
+
}
|
|
1198
|
+
if (turnLimit > 0 && !projected && turnBytes <= turnLimit) {
|
|
1199
|
+
const recordKey = typeof r.uuid === "string" ? r.uuid : typeof r.id === "string" ? r.id : typeof p.id === "string" ? p.id : `record:${records}`;
|
|
1200
|
+
const encoded = JSON.stringify({ ...r, _auditRecordKey: recordKey }) + "\n";
|
|
1201
|
+
turnBytes += Buffer.byteLength(encoded);
|
|
1202
|
+
if (turnBytes <= turnLimit) turnLines.push(encoded);
|
|
1203
|
+
else if (turnLines.length) turnLines = [];
|
|
1204
|
+
}
|
|
1205
|
+
if (!usage) usage = createNativeUsageAccumulator(harness, sourceId(harness, nativeId), { maxRecords: 1e6, acceptRecord: (_r, scope) => !maintenance && !scopeUncertain && options.accept(scope.project, scope.eventAt) });
|
|
1206
|
+
usage.pushRecord(r, { rawStart: offset, rawEnd: end });
|
|
1207
|
+
if (scopeUncertain) return;
|
|
1208
|
+
let text = "", identity = "", origin = "unknown", eligible = false, excluded = null;
|
|
1209
|
+
const time = clock(r.timestamp), key = typeof r.uuid === "string" ? r.uuid : typeof r.id === "string" ? r.id : typeof p.id === "string" ? p.id : `record:${records}`;
|
|
1210
|
+
if (harness === "claude" && r.type === "user" && m.role === "user") {
|
|
1211
|
+
if (Array.isArray(m.content) && m.content.some((b) => isRecord(b) && b.type === "tool_result")) return;
|
|
1212
|
+
text = extractTypedText(m.content);
|
|
1213
|
+
identity = typeof r.promptId === "string" ? `prompt:${r.promptId}` : `record:${key}`;
|
|
1214
|
+
origin = typeof r.promptId === "string" ? "claude_prompt_id" : "unknown";
|
|
1215
|
+
eligible = typeof r.promptId === "string" && !programmatic;
|
|
1216
|
+
} else if (harness === "codex" && r.type === "event_msg" && p.type === "user_message") {
|
|
1217
|
+
text = typeof p.message === "string" ? p.message : "";
|
|
1218
|
+
identity = `marker:${offset}`;
|
|
1219
|
+
origin = programmatic ? "unknown" : "codex_human_marker";
|
|
1220
|
+
eligible = !programmatic;
|
|
1221
|
+
} else if (harness === "pi" && r.type === "message" && m.role === "user") {
|
|
1222
|
+
text = extractTypedText(m.content);
|
|
1223
|
+
identity = `node:${key}`;
|
|
1224
|
+
origin = "pi_user_projection";
|
|
1225
|
+
} else {
|
|
1226
|
+
const info = isRecord(p.info) ? p.info : {};
|
|
1227
|
+
const observedModel = typeof m.model === "string" ? m.model : typeof p.model === "string" ? p.model : typeof r.model === "string" ? r.model : typeof info.model === "string" ? info.model : currentModel, observedProvider = typeof m.provider === "string" ? m.provider : typeof p.model_provider === "string" ? p.model_provider : typeof p.provider === "string" ? p.provider : typeof r.provider === "string" ? r.provider : currentProvider;
|
|
1228
|
+
if ((m.role === "assistant" || harness === "codex" && (r.type === "event_msg" && p.type === "token_count" || r.type === "response_item" && p.type === "message" && p.role === "assistant")) && observedModel !== "<synthetic>" && r.isSynthetic !== true && options.accept(project, time)) language.push({ id: `metadata:${records}`, conversationId: sourceId(harness, nativeId), parentId: parent, harness, eventAt: time, project, role: "assistant", text: "", model: observedModel === "codex-auto-review" ? null : observedModel, provider: observedProvider, directUser: false, route: null });
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
if (!text.trim() && !projected) return;
|
|
1232
|
+
if (hasExclusionMarker(text)) {
|
|
1233
|
+
maintenance = true;
|
|
1234
|
+
excluded = "maintenance_exclusion_marker";
|
|
1235
|
+
} else if (r.isMeta === true || r.isSynthetic === true || m.isSynthetic === true) excluded = "declared_meta_or_synthetic_input";
|
|
1236
|
+
else if (parent && events.length === 0) excluded = "child_initialization";
|
|
1237
|
+
else if (typeof r.session_id === "string" && r.session_id !== nativeId) excluded = "inherited_identity_unverified";
|
|
1238
|
+
if (seen.has(identity)) return;
|
|
1239
|
+
seen.add(identity);
|
|
1240
|
+
if (!options.accept(project, time)) return;
|
|
1241
|
+
const redacted = clean(text);
|
|
1242
|
+
if (promptBytes + Buffer.byteLength(redacted) > (options.maxPromptBytes ?? 64 * 1024 * 1024)) {
|
|
1243
|
+
gaps.add("native_input_text_limit");
|
|
1244
|
+
return;
|
|
1245
|
+
}
|
|
1246
|
+
promptBytes += Buffer.byteLength(redacted);
|
|
1247
|
+
language.push({ id: digest(`${sourceId(harness, nativeId)}:${identity}`).slice(0, 32), conversationId: sourceId(harness, nativeId), parentId: parent, harness, eventAt: time, project, role: "user", text: "", model: null, provider: null, directUser: excluded === null && parent === null, route: null });
|
|
1248
|
+
const proofRecord = { ...r };
|
|
1249
|
+
delete proofRecord.sessionId;
|
|
1250
|
+
delete proofRecord.session_id;
|
|
1251
|
+
delete proofRecord.promptId;
|
|
1252
|
+
const stable = (value) => Array.isArray(value) ? value.map(stable) : isRecord(value) ? Object.fromEntries(Object.keys(value).sort().map((k) => [k, stable(value[k])])) : value;
|
|
1253
|
+
events.push({ key, rawStart: offset, rawEnd: end, role: "user", text: redacted, eventAt: time, project, origin, eligible: eligible && excluded === null && parent === null, languageEligible: parent === null && !projected, excluded, identity, observed: excluded === null, ...harness === "claude" && typeof r.uuid === "string" ? { nativeRecordId: r.uuid, recordCommitment: digest(JSON.stringify(stable(proofRecord))) } : {}, ...harness === "claude" && typeof r.session_id === "string" && r.session_id !== nativeId ? { declaredOrigin: r.session_id } : {} });
|
|
1254
|
+
};
|
|
1255
|
+
const finishLine = (end) => {
|
|
1256
|
+
if (projection) {
|
|
1257
|
+
const result = projection.finish();
|
|
1258
|
+
projection = null;
|
|
1259
|
+
gaps.add("native_record_body_omitted");
|
|
1260
|
+
if (result.exclusionMarker) maintenance = true;
|
|
1261
|
+
if (result.record) {
|
|
1262
|
+
projected = true;
|
|
1263
|
+
process2(Buffer.from(JSON.stringify(result.record)), end);
|
|
1264
|
+
projected = false;
|
|
1265
|
+
} else {
|
|
1266
|
+
turnLines = [];
|
|
1267
|
+
turnBytes = turnLimit + 1;
|
|
1268
|
+
records++;
|
|
1269
|
+
gaps.add(result.code ?? "native_metadata_invalid");
|
|
1270
|
+
gaps.add("native_scope_record_unavailable");
|
|
1271
|
+
usage?.invalidateContext();
|
|
1272
|
+
scopeUncertain = true;
|
|
1273
|
+
project = null;
|
|
1274
|
+
currentModel = null;
|
|
1275
|
+
currentProvider = null;
|
|
1276
|
+
}
|
|
1277
|
+
} else process2(held.length === 1 ? held[0] : Buffer.concat(held, heldBytes), end);
|
|
1278
|
+
held = [];
|
|
1279
|
+
heldBytes = 0;
|
|
1280
|
+
offset = end;
|
|
1281
|
+
};
|
|
1282
|
+
const stream = fs.createReadStream(file, { highWaterMark: 64 * 1024, ...captureSize > 0 ? { end: captureSize - 1 } : {} });
|
|
1283
|
+
try {
|
|
1284
|
+
let consumed = 0;
|
|
1285
|
+
for await (const chunk of captureSize === 0 ? [] : stream) {
|
|
1286
|
+
if (options.signal.aborted) {
|
|
1287
|
+
stream.destroy();
|
|
1288
|
+
throw new Error("cancelled");
|
|
1289
|
+
}
|
|
1290
|
+
const bytes = chunk;
|
|
1291
|
+
hash2.update(bytes);
|
|
1292
|
+
let at2 = 0;
|
|
1293
|
+
while (at2 < bytes.length) {
|
|
1294
|
+
const newline = bytes.indexOf(10, at2), end = newline < 0 ? bytes.length : newline, part = bytes.subarray(at2, end);
|
|
1295
|
+
if (projection) projection.push(part);
|
|
1296
|
+
else if (heldBytes + part.length > 8 * 1024 * 1024) {
|
|
1297
|
+
projection = new NativeMetadataProjection(void 0, harness);
|
|
1298
|
+
for (const piece of held) projection.push(piece);
|
|
1299
|
+
projection.push(part);
|
|
1300
|
+
held = [];
|
|
1301
|
+
heldBytes = 0;
|
|
1302
|
+
} else {
|
|
1303
|
+
held.push(part);
|
|
1304
|
+
heldBytes += part.length;
|
|
1305
|
+
}
|
|
1306
|
+
if (newline < 0) break;
|
|
1307
|
+
finishLine(consumed + newline + 1);
|
|
1308
|
+
at2 = newline + 1;
|
|
1309
|
+
}
|
|
1310
|
+
consumed += bytes.length;
|
|
1311
|
+
}
|
|
1312
|
+
} finally {
|
|
1313
|
+
stream.destroy();
|
|
1314
|
+
}
|
|
1315
|
+
if (heldBytes || projection) gaps.add("unfinished_tail");
|
|
1316
|
+
const after = fs.statSync(file, { bigint: true });
|
|
1317
|
+
if (before.dev !== after.dev || before.ino !== after.ino || after.size < BigInt(captureSize)) throw new Error("source_changed");
|
|
1318
|
+
if (BigInt(captureSize) !== after.size || before.mtimeNs !== after.mtimeNs || before.ctimeNs !== after.ctimeNs) {
|
|
1319
|
+
const verify = createHash5("sha256"), fd = fs.openSync(file, "r"), buffer = Buffer.allocUnsafe(65536);
|
|
1320
|
+
try {
|
|
1321
|
+
let at2 = 0;
|
|
1322
|
+
while (at2 < captureSize) {
|
|
1323
|
+
const n = fs.readSync(fd, buffer, 0, Math.min(buffer.length, captureSize - at2), at2);
|
|
1324
|
+
if (!n) throw new Error("source_changed");
|
|
1325
|
+
verify.update(buffer.subarray(0, n));
|
|
1326
|
+
at2 += n;
|
|
1327
|
+
}
|
|
1328
|
+
} finally {
|
|
1329
|
+
fs.closeSync(fd);
|
|
1330
|
+
}
|
|
1331
|
+
if (verify.digest("hex") !== hash2.copy().digest("hex")) throw new Error("source_changed");
|
|
1332
|
+
gaps.add("native_appended_after_capture");
|
|
1333
|
+
}
|
|
1334
|
+
if (maintenance) {
|
|
1335
|
+
gaps.add("maintenance_source_excluded");
|
|
1336
|
+
for (const event of events) {
|
|
1337
|
+
event.observed = false;
|
|
1338
|
+
event.eligible = false;
|
|
1339
|
+
event.excluded = "maintenance_exclusion_marker";
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
const artifactHash = hash2.digest("hex"), acc = usage, all = acc?.records() ?? [];
|
|
1343
|
+
finishTurn();
|
|
1344
|
+
const captured = Buffer.from((turns[0]?.prefix ?? "") + turns.flatMap((t) => t.lines).join(""));
|
|
1345
|
+
capturedRecords = turns.reduce((n, t) => n + t.lines.length, 0);
|
|
1346
|
+
if (turnLimit > 0 && capturedRecords < records) gaps.add("context_capture_partial");
|
|
1347
|
+
return { facts: { nativeId, project, parent, child: parent !== null, title: null, events, gaps: [...gaps], hash: artifactHash, consumed: offset, bytes: captured }, usage: maintenance ? [] : all, language: maintenance ? [] : language, hash: artifactHash, bytes: captureSize, stamp: { dev: String(after.dev), ino: String(after.ino), size: Number(after.size), mtimeNs: String(after.mtimeNs), ctimeNs: String(after.ctimeNs) }, counts: maintenance ? { observed: acc?.counts().observed ?? 0, excluded: acc?.counts().observed ?? 0, deduplicated: 0 } : acc?.counts() ?? { observed: 0, excluded: 0, deduplicated: 0 } };
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
// packages/core/src/ignore.ts
|
|
1351
|
+
function fold(value) {
|
|
1352
|
+
return value.replace(/\\/g, "/").replace(/\/+$/, "").toLowerCase();
|
|
1353
|
+
}
|
|
1354
|
+
function matchesIgnoreEntry(project, entry) {
|
|
1355
|
+
if (!project) return false;
|
|
1356
|
+
const p = fold(project);
|
|
1357
|
+
const e = fold(entry);
|
|
1358
|
+
if (!p || !e) return false;
|
|
1359
|
+
if (p === e) return true;
|
|
1360
|
+
if (e.includes("/")) return p.startsWith(`${e}/`);
|
|
1361
|
+
return p.split("/").filter(Boolean).includes(e);
|
|
1362
|
+
}
|
|
1363
|
+
function isIgnoredProject(project, entries) {
|
|
1364
|
+
return entries.some((entry) => matchesIgnoreEntry(project, entry));
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// packages/core/src/analytics/native-pool.ts
|
|
1368
|
+
var PROTOCOL = "audit-native-v1";
|
|
1369
|
+
var FRAME = 1024 * 1024;
|
|
1370
|
+
var code = (error) => error instanceof Error && /^(?:[a-z][a-z0-9_]{0,95})$/.test(error.message) ? error.message : "native_worker_failed";
|
|
1371
|
+
function runNativeScanWorker() {
|
|
1372
|
+
if (!parentPort) throw new Error("native_worker_requires_port");
|
|
1373
|
+
const port = parentPort;
|
|
1374
|
+
let busy = false;
|
|
1375
|
+
port.on("message", async (message) => {
|
|
1376
|
+
if (message.protocol !== PROTOCOL || message.kind !== "scan" || !message.job || busy) return;
|
|
1377
|
+
busy = true;
|
|
1378
|
+
const job = message.job;
|
|
1379
|
+
try {
|
|
1380
|
+
const policy = job.policy, forgotten = new Set(policy.forgottenSourceIds), accept = (project, time) => !(project && isIgnoredProject(project, policy.ignoredProjects)) && (!policy.project || project === policy.project) && (!policy.eventFrom || time !== null && time >= policy.eventFrom) && (!policy.asOf || time !== null && time <= policy.asOf);
|
|
1381
|
+
const snapshot = await streamNativeFacts(job.file, job.harness, { signal: new AbortController().signal, maxBytes: job.maxBytes, maxRecords: job.maxRecords, maxPromptBytes: job.maxPromptBytes, captureBytes: 0, captureFileBytes: job.bytes, expectedIdentity: { dev: job.stamp.dev, ino: job.stamp.ino }, sourceAllowed: (id, project) => !forgotten.has(sourceId(job.harness, id)) && !(project && isIgnoredProject(project, policy.ignoredProjects)), accept });
|
|
1382
|
+
const buffer = serialize(snapshot);
|
|
1383
|
+
if (buffer.length > (message.maxResultBytes ?? 256 * 1024 * 1024)) throw new Error("native_pool_result_bytes_limit");
|
|
1384
|
+
const hash2 = createHash6("sha256").update(buffer).digest("hex");
|
|
1385
|
+
let sequence = 0;
|
|
1386
|
+
for (let at2 = 0; at2 < buffer.length; at2 += FRAME) {
|
|
1387
|
+
const chunk = Uint8Array.from(buffer.subarray(at2, at2 + FRAME));
|
|
1388
|
+
port.postMessage({ protocol: PROTOCOL, kind: "frame", jobId: job.jobId, sequence: sequence++, data: chunk }, [chunk.buffer]);
|
|
1389
|
+
}
|
|
1390
|
+
port.postMessage({ protocol: PROTOCOL, kind: "result", jobId: job.jobId, authorityCommitment: policy.authorityCommitment, bytes: buffer.length, hash: hash2 });
|
|
1391
|
+
} catch (error) {
|
|
1392
|
+
port.postMessage({ protocol: PROTOCOL, kind: "error", jobId: job.jobId, code: code(error) });
|
|
1393
|
+
} finally {
|
|
1394
|
+
busy = false;
|
|
1395
|
+
}
|
|
1396
|
+
});
|
|
1397
|
+
port.postMessage({ protocol: PROTOCOL, kind: "ready" });
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
// packages/cli/src/audit-native-worker.ts
|
|
1401
|
+
runNativeScanWorker();
|
|
1402
|
+
//# sourceMappingURL=audit-native-worker.js.map
|