slopie 1.7.0 → 1.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/audit-splash.js +71 -0
- package/bin/potsherd.js +2 -0
- package/bin/slopie.js +2 -0
- package/dist/audit-native-worker.js +477 -767
- package/dist/audit-native-worker.js.map +4 -4
- package/dist/potsherd.js +18928 -20414
- package/dist/potsherd.js.map +4 -4
- package/package.json +2 -6
|
@@ -1,25 +1,12 @@
|
|
|
1
1
|
import { createRequire as __potsherdCreateRequire } from 'node:module';
|
|
2
2
|
const require = __potsherdCreateRequire(import.meta.url);
|
|
3
3
|
|
|
4
|
-
// packages/core/src/analytics/
|
|
4
|
+
// packages/core/src/analytics/scan-pool.ts
|
|
5
5
|
import { Worker, parentPort } from "node:worker_threads";
|
|
6
|
-
import { serialize, deserialize } from "node:v8";
|
|
7
|
-
import { createHash as createHash6 } from "node:crypto";
|
|
8
6
|
|
|
9
|
-
// packages/core/src/analytics/
|
|
7
|
+
// packages/core/src/analytics/extract.ts
|
|
10
8
|
import fs from "node:fs";
|
|
11
9
|
import path from "node:path";
|
|
12
|
-
import { createHash as createHash5 } from "node:crypto";
|
|
13
|
-
|
|
14
|
-
// packages/core/src/parser/content.ts
|
|
15
|
-
function isRecord(value) {
|
|
16
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
17
|
-
}
|
|
18
|
-
function extractTypedText(content, blockType = "text") {
|
|
19
|
-
if (typeof content === "string") return content;
|
|
20
|
-
if (!Array.isArray(content)) return "";
|
|
21
|
-
return content.filter((b) => isRecord(b) && b.type === blockType && typeof b.text === "string").map((b) => b.text).join("\n");
|
|
22
|
-
}
|
|
23
10
|
|
|
24
11
|
// packages/core/src/markers.ts
|
|
25
12
|
var SUMMARIZER_CONTEXT_MARKER = "Context: This summary will be shown in a list to help users and Claude choose which conversations are relevant";
|
|
@@ -34,17 +21,8 @@ function hasExclusionMarker(text) {
|
|
|
34
21
|
return EXCLUSION_MARKERS.some((marker) => text.includes(marker));
|
|
35
22
|
}
|
|
36
23
|
|
|
37
|
-
// packages/core/src/memory/source-identity.ts
|
|
38
|
-
import { createHash } from "node:crypto";
|
|
39
|
-
function sourceId(harness, nativeSessionId) {
|
|
40
|
-
return createHash("sha256").update(`source/v1\0${harness}\0${nativeSessionId}`).digest("hex");
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// packages/core/src/analytics/source.ts
|
|
44
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
45
|
-
|
|
46
24
|
// packages/core/src/redact.ts
|
|
47
|
-
import { createHash
|
|
25
|
+
import { createHash } from "node:crypto";
|
|
48
26
|
|
|
49
27
|
// packages/core/src/theme.ts
|
|
50
28
|
var ANSI_RE = new RegExp("\\u001b\\[[0-9;]*m", "g");
|
|
@@ -562,7 +540,7 @@ var OPEN = "\u2039";
|
|
|
562
540
|
var CLOSE = "\u203A";
|
|
563
541
|
var MASK_RE = new RegExp(`${OPEN}redacted:[a-z-]+:[0-9a-f]{8}${CLOSE}`, "g");
|
|
564
542
|
function secretDigest(secret) {
|
|
565
|
-
return
|
|
543
|
+
return createHash("sha256").update(secret, "utf8").digest("hex").slice(0, 8);
|
|
566
544
|
}
|
|
567
545
|
function redact(text) {
|
|
568
546
|
if (typeof text !== "string" || text.length === 0) return { text: text ?? "", hits: [] };
|
|
@@ -618,785 +596,517 @@ function claim(claimed, s) {
|
|
|
618
596
|
}
|
|
619
597
|
|
|
620
598
|
// packages/core/src/analytics/source.ts
|
|
621
|
-
var digest = (value) => createHash3("sha256").update(value).digest("hex");
|
|
622
599
|
var clean = (text) => redact(elideBinary(text)).text;
|
|
623
|
-
var clock = (value) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null;
|
|
624
600
|
|
|
625
|
-
// packages/core/src/analytics/
|
|
626
|
-
|
|
627
|
-
var
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
601
|
+
// packages/core/src/analytics/extract.ts
|
|
602
|
+
var FACTS_VERSION = 4;
|
|
603
|
+
var MAX_PROMPT_CHARS = 4e3;
|
|
604
|
+
var isRec = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
605
|
+
var str = (v) => typeof v === "string" && v.length > 0 ? v : null;
|
|
606
|
+
var int = (v) => typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.floor(v) : 0;
|
|
607
|
+
function time(v) {
|
|
608
|
+
if (typeof v === "string") {
|
|
609
|
+
const t = Date.parse(v);
|
|
610
|
+
return Number.isFinite(t) ? t : null;
|
|
631
611
|
}
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
612
|
+
if (typeof v === "number" && Number.isFinite(v)) return v < 1e11 ? v * 1e3 : v;
|
|
613
|
+
return null;
|
|
614
|
+
}
|
|
615
|
+
function textOf(content) {
|
|
616
|
+
if (typeof content === "string") return content;
|
|
617
|
+
if (!Array.isArray(content)) return "";
|
|
618
|
+
const parts = [];
|
|
619
|
+
for (const block of content) if (isRec(block) && typeof block.text === "string") parts.push(block.text);
|
|
620
|
+
return parts.join("\n");
|
|
621
|
+
}
|
|
622
|
+
var INJECTED_PREFIXES = [
|
|
623
|
+
["<task-notification>", "task_notification"],
|
|
624
|
+
["<local-command-stdout>", "local_command_output"],
|
|
625
|
+
["<local-command-stderr>", "local_command_output"],
|
|
626
|
+
["<local-command-caveat>", "local_command_output"],
|
|
627
|
+
["[Request interrupted", "interrupt_marker"],
|
|
628
|
+
["<turn_aborted>", "interrupt_marker"],
|
|
629
|
+
["<environment_context>", "environment_context"],
|
|
630
|
+
["# AGENTS.md instructions", "agents_md"],
|
|
631
|
+
["<user_instructions>", "agents_md"],
|
|
632
|
+
["<external_codex_apps", "app_state"],
|
|
633
|
+
["<recommended_plugins>", "app_state"],
|
|
634
|
+
["<heartbeat>", "automation"],
|
|
635
|
+
["The following is the Codex agent history", "guardian_review"],
|
|
636
|
+
["You have new hive inbox message", "automation"],
|
|
637
|
+
["Another Claude session sent a message", "cross_session_message"],
|
|
638
|
+
["This session is being continued from a previous conversation", "compaction_summary"],
|
|
639
|
+
["Your claude.ai usage limit has reset", "auto_continuation"],
|
|
640
|
+
["Base directory for this skill:", "skill_expansion"]
|
|
641
|
+
];
|
|
642
|
+
var SYSTEM_REMINDER = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
643
|
+
function classifyText(raw) {
|
|
644
|
+
const text = (raw.includes("<system-reminder>") ? raw.replace(SYSTEM_REMINDER, "") : raw).trim();
|
|
645
|
+
if (!text) return { kind: "excluded", reason: "empty", text };
|
|
646
|
+
for (const [prefix, reason] of INJECTED_PREFIXES) if (text.startsWith(prefix)) return { kind: "excluded", reason, text };
|
|
647
|
+
if (text.startsWith("<command-name>") || text.startsWith("<command-message>")) return { kind: "command", reason: "slash_command", text };
|
|
648
|
+
if (text.startsWith("<send_user_message_question_reply>")) return { kind: "answer", reason: "question_answer", text };
|
|
649
|
+
return { kind: "human", reason: null, text };
|
|
650
|
+
}
|
|
651
|
+
var CODEX_STRIP = [/<in-app-browser-context[\s\S]*?<\/in-app-browser-context>/g, /<image name=[^>]*>\s*<\/image>/g, /<\/?image[^>]*>/g];
|
|
652
|
+
function cleanCodexText(raw) {
|
|
653
|
+
let t = raw;
|
|
654
|
+
if (t.includes("<")) for (const rx of CODEX_STRIP) t = t.replace(rx, "");
|
|
655
|
+
const at = t.indexOf("## My request");
|
|
656
|
+
if (at >= 0) {
|
|
657
|
+
const rest = t.slice(at + "## My request".length), colon = rest.indexOf(":");
|
|
658
|
+
t = colon >= 0 ? rest.slice(colon + 1) : rest;
|
|
659
|
+
} else if (t.trimStart().startsWith("# Files mentioned by the user")) t = "";
|
|
660
|
+
return t.trim();
|
|
661
|
+
}
|
|
662
|
+
function makeBuilder(file, harness, size, mtimeMs) {
|
|
663
|
+
return {
|
|
664
|
+
facts: {
|
|
665
|
+
v: FACTS_VERSION,
|
|
666
|
+
harness,
|
|
667
|
+
file,
|
|
668
|
+
size,
|
|
669
|
+
mtimeMs,
|
|
670
|
+
sessionId: path.basename(file, ".jsonl"),
|
|
671
|
+
parentId: null,
|
|
672
|
+
child: false,
|
|
673
|
+
project: null,
|
|
674
|
+
title: null,
|
|
675
|
+
automated: false,
|
|
676
|
+
excluded: false,
|
|
677
|
+
startedAt: null,
|
|
678
|
+
firstAt: null,
|
|
679
|
+
lastAt: null,
|
|
680
|
+
usage: [],
|
|
681
|
+
prompts: [],
|
|
682
|
+
malformed: 0
|
|
683
|
+
},
|
|
684
|
+
currentModel: null,
|
|
685
|
+
currentProvider: null,
|
|
686
|
+
pending: [],
|
|
687
|
+
seenPrompts: /* @__PURE__ */ new Set()
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
function touch(b, at) {
|
|
691
|
+
if (at === null) return;
|
|
692
|
+
if (b.facts.firstAt === null || at < b.facts.firstAt) b.facts.firstAt = at;
|
|
693
|
+
if (b.facts.lastAt === null || at > b.facts.lastAt) b.facts.lastAt = at;
|
|
694
|
+
}
|
|
695
|
+
function answered(b, model, provider) {
|
|
696
|
+
if (!model) return;
|
|
697
|
+
for (const p of b.pending) {
|
|
698
|
+
p.after = model;
|
|
699
|
+
p.afterProvider = provider;
|
|
662
700
|
}
|
|
663
|
-
|
|
664
|
-
|
|
701
|
+
b.pending.length = 0;
|
|
702
|
+
b.currentModel = model;
|
|
703
|
+
b.currentProvider = provider;
|
|
704
|
+
}
|
|
705
|
+
function addUsage(b, u) {
|
|
706
|
+
b.facts.usage.push(u);
|
|
707
|
+
touch(b, u.at);
|
|
708
|
+
answered(b, u.model, u.provider);
|
|
709
|
+
}
|
|
710
|
+
function addPrompt(b, key, at, project, kind, reason, text) {
|
|
711
|
+
if (b.seenPrompts.has(key)) return;
|
|
712
|
+
b.seenPrompts.add(key);
|
|
713
|
+
if (kind === "human" && hasExclusionMarker(text)) b.facts.excluded = true;
|
|
714
|
+
const prompt = {
|
|
715
|
+
key,
|
|
716
|
+
at,
|
|
717
|
+
project,
|
|
718
|
+
kind,
|
|
719
|
+
reason,
|
|
720
|
+
text: kind === "human" ? clean(text.length > MAX_PROMPT_CHARS ? text.slice(0, MAX_PROMPT_CHARS) : text) : "",
|
|
721
|
+
before: b.currentModel,
|
|
722
|
+
beforeProvider: b.currentProvider,
|
|
723
|
+
after: null,
|
|
724
|
+
afterProvider: null
|
|
725
|
+
};
|
|
726
|
+
b.facts.prompts.push(prompt);
|
|
727
|
+
if (kind === "human") b.pending.push(prompt);
|
|
728
|
+
touch(b, at);
|
|
729
|
+
}
|
|
730
|
+
var SUBAGENT_DIR = `${path.sep}subagents${path.sep}`;
|
|
731
|
+
function claudeLine(b, line, subagent, sessions) {
|
|
732
|
+
const assistant = line.includes('"type":"assistant"');
|
|
733
|
+
if (assistant ? !line.includes('"usage"') : subagent || !line.includes('"type":"user"')) {
|
|
734
|
+
if (line.length > 4096 || !line.includes('-title"') && !line.includes('"type":"summary"')) return;
|
|
665
735
|
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
}
|
|
673
|
-
this.token += c;
|
|
674
|
-
this.trimImages();
|
|
675
|
-
}
|
|
676
|
-
if (this.imageCandidate && !this.imageCandidate.overflow) {
|
|
677
|
-
const bytes = Buffer.byteLength(JSON.stringify(c)) - 2;
|
|
678
|
-
this.imageCandidate.bytes += bytes;
|
|
679
|
-
this.imageBytes += bytes;
|
|
680
|
-
this.imageCandidate.chunks.push(c);
|
|
681
|
-
this.trimImages();
|
|
682
|
-
}
|
|
683
|
-
const searchable = this.markerTail + c;
|
|
684
|
-
if (EXCLUSION_MARKERS.some((m) => searchable.includes(m))) this.marker = true;
|
|
685
|
-
this.markerTail = searchable.slice(-256);
|
|
736
|
+
let r;
|
|
737
|
+
try {
|
|
738
|
+
r = JSON.parse(line.toString("utf8"));
|
|
739
|
+
} catch {
|
|
740
|
+
b.facts.malformed++;
|
|
741
|
+
return;
|
|
686
742
|
}
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
743
|
+
if (!isRec(r)) return;
|
|
744
|
+
const f = b.facts;
|
|
745
|
+
if (r.type === "summary" || r.type === "custom-title" || r.type === "ai-title") {
|
|
746
|
+
const title = str(r.customTitle) ?? str(r.aiTitle) ?? str(r.title) ?? str(r.summary);
|
|
747
|
+
if (title && (r.type !== "summary" || !f.title)) f.title = title;
|
|
748
|
+
return;
|
|
690
749
|
}
|
|
691
|
-
|
|
692
|
-
|
|
750
|
+
if (typeof r.cwd === "string" && !f.project) f.project = r.cwd;
|
|
751
|
+
const sid = str(r.sessionId);
|
|
752
|
+
if (sid) {
|
|
753
|
+
if (subagent) {
|
|
754
|
+
f.parentId = sid;
|
|
755
|
+
f.child = true;
|
|
756
|
+
f.sessionId = `${sid}:${path.basename(f.file, ".jsonl")}`;
|
|
757
|
+
} else if (!sessions.size) f.sessionId = sid;
|
|
693
758
|
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
759
|
+
const m = isRec(r.message) ? r.message : {};
|
|
760
|
+
const at = time(r.timestamp);
|
|
761
|
+
if (r.type === "assistant") {
|
|
762
|
+
const u = m.usage;
|
|
763
|
+
if (!isRec(u)) return;
|
|
764
|
+
const model = str(m.model);
|
|
765
|
+
if (model === "<synthetic>") return;
|
|
766
|
+
const cc = isRec(u.cache_creation) ? u.cache_creation : null;
|
|
767
|
+
const write = cc ? int(cc.ephemeral_5m_input_tokens) + int(cc.ephemeral_1h_input_tokens) : int(u.cache_creation_input_tokens);
|
|
768
|
+
const id = str(m.id), request = str(r.requestId), session = sid ?? f.file;
|
|
769
|
+
const fact = {
|
|
770
|
+
key: id ? request ? `c:${id}:${request}` : `c:${id}::${session}:${String(r.timestamp)}` : null,
|
|
771
|
+
at,
|
|
772
|
+
model,
|
|
773
|
+
provider: null,
|
|
774
|
+
input: int(u.input_tokens),
|
|
775
|
+
output: int(u.output_tokens),
|
|
776
|
+
cacheRead: int(u.cache_read_input_tokens),
|
|
777
|
+
cacheWrite: write,
|
|
778
|
+
cacheWrite1h: cc ? int(cc.ephemeral_1h_input_tokens) : 0,
|
|
779
|
+
reasoning: 0
|
|
780
|
+
};
|
|
781
|
+
if (r.isSidechain === true) fact.sidechain = true;
|
|
782
|
+
if (!request) fact.requestless = true;
|
|
783
|
+
if (id) fact.replayKey = `${id}:${session}`;
|
|
784
|
+
if (u.speed === "fast") fact.tier = "fast";
|
|
785
|
+
addUsage(b, fact);
|
|
786
|
+
return;
|
|
697
787
|
}
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
}
|
|
713
|
-
this.root = value;
|
|
714
|
-
this.done = true;
|
|
715
|
-
return;
|
|
716
|
-
}
|
|
717
|
-
if (f.state !== "value" && f.state !== "valueRequired") {
|
|
718
|
-
this.fail();
|
|
719
|
-
return;
|
|
720
|
-
}
|
|
721
|
-
if (f.keep) {
|
|
722
|
-
if (f.array) f.value.push(value);
|
|
723
|
-
else if (f.key !== null && this.retainValue()) Object.defineProperty(f.value, f.key, { value, writable: true, enumerable: true, configurable: true });
|
|
724
|
-
}
|
|
725
|
-
f.key = null;
|
|
726
|
-
f.omitImage = false;
|
|
727
|
-
f.state = "comma";
|
|
788
|
+
if (r.type !== "user" || r.isSidechain === true || subagent) return;
|
|
789
|
+
const content = m.content;
|
|
790
|
+
if (Array.isArray(content) && content.some((block) => isRec(block) && block.type === "tool_result")) return;
|
|
791
|
+
if (sid) sessions.add(sid);
|
|
792
|
+
const origin = isRec(r.origin) ? str(r.origin.kind) : null;
|
|
793
|
+
const flagged = r.isMeta === true ? "is_meta" : r.isCompactSummary === true ? "compaction_summary" : origin && origin !== "human" ? `origin_${origin}` : r.entrypoint === "sdk-cli" ? "headless_claude_p" : r.promptSource === "system" ? "prompt_source_system" : null;
|
|
794
|
+
let { kind, reason, text } = classifyText(textOf(content));
|
|
795
|
+
if (flagged) {
|
|
796
|
+
kind = "excluded";
|
|
797
|
+
reason = flagged;
|
|
798
|
+
} else if (reason === "empty" && origin === "human" && Array.isArray(content) && content.some((block) => isRec(block) && block.type === "image")) {
|
|
799
|
+
kind = "human";
|
|
800
|
+
reason = null;
|
|
801
|
+
text = "[image]";
|
|
728
802
|
}
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
803
|
+
addPrompt(b, str(r.uuid) ?? `${f.sessionId}:${String(r.timestamp)}`, at, str(r.cwd) ?? f.project, kind, reason, text);
|
|
804
|
+
}
|
|
805
|
+
var U_KEYS = ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"];
|
|
806
|
+
var tuple = (u) => isRec(u) ? U_KEYS.map((k) => int(u[k])) : null;
|
|
807
|
+
var AUTO_REVIEW_MODELS = [
|
|
808
|
+
["2026-07-30", "gpt-5.6-luna"],
|
|
809
|
+
["2026-03-05", "gpt-5.4"],
|
|
810
|
+
["2026-02-05", "gpt-5.3-codex"],
|
|
811
|
+
["2025-12-11", "gpt-5.2-codex"],
|
|
812
|
+
["2025-11-13", "gpt-5.1-codex"],
|
|
813
|
+
["2025-09-15", "gpt-5-codex"]
|
|
814
|
+
];
|
|
815
|
+
function codexModel(model, timestamp) {
|
|
816
|
+
if (model === "codex-auto-review") {
|
|
817
|
+
const day = timestamp.slice(0, 10);
|
|
818
|
+
return { model: AUTO_REVIEW_MODELS.find(([since]) => day >= since)?.[1] ?? "gpt-5", inferred: true };
|
|
745
819
|
}
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
this.fail();
|
|
770
|
-
return;
|
|
771
|
-
}
|
|
772
|
-
this.add(decoded[c]);
|
|
773
|
-
return;
|
|
774
|
-
}
|
|
775
|
-
if (c === "\\") {
|
|
776
|
-
this.escape = true;
|
|
777
|
-
return;
|
|
778
|
-
}
|
|
779
|
-
if (c === '"') {
|
|
780
|
-
if (this.isKey) {
|
|
781
|
-
const f2 = this.frames.at(-1);
|
|
782
|
-
f2.key = this.token;
|
|
783
|
-
f2.state = "colon";
|
|
784
|
-
} else this.accept(this.keep ? this.token : null);
|
|
785
|
-
this.mode = "idle";
|
|
786
|
-
this.token = "";
|
|
787
|
-
this.imageCandidate = null;
|
|
788
|
-
this.markerTail = "";
|
|
789
|
-
return;
|
|
790
|
-
}
|
|
791
|
-
if (c.charCodeAt(0) < 32) {
|
|
792
|
-
this.fail();
|
|
793
|
-
return;
|
|
794
|
-
}
|
|
795
|
-
this.add(c);
|
|
796
|
-
return;
|
|
797
|
-
}
|
|
798
|
-
if (this.mode === "number" || this.mode === "literal") {
|
|
799
|
-
const allowed = this.mode === "number" ? /[0-9eE+.-]/ : /[a-z]/;
|
|
800
|
-
if (allowed.test(c)) {
|
|
801
|
-
if (this.token.length >= 256) {
|
|
802
|
-
this.fail("native_metadata_scalar_limit");
|
|
803
|
-
return;
|
|
804
|
-
}
|
|
805
|
-
this.token += c;
|
|
806
|
-
return;
|
|
807
|
-
}
|
|
808
|
-
this.scalarEnd();
|
|
809
|
-
if (this.error) return;
|
|
810
|
-
}
|
|
811
|
-
if (/\s/.test(c)) {
|
|
812
|
-
if (![" ", " ", "\r", "\n"].includes(c)) this.fail();
|
|
813
|
-
return;
|
|
814
|
-
}
|
|
815
|
-
const f = this.frames.at(-1);
|
|
816
|
-
if (c === '"') {
|
|
817
|
-
this.isKey = !!f && !f.array && (f.state === "key" || f.state === "keyRequired");
|
|
818
|
-
if (!this.isKey && !this.expectedValue()) {
|
|
819
|
-
this.fail();
|
|
820
|
-
return;
|
|
821
|
-
}
|
|
822
|
-
if (!this.isKey && f && this.imageBody(f)) {
|
|
823
|
-
f.omitImage = true;
|
|
824
|
-
this.imageCandidate = { frame: f, chunks: [], bytes: 0, overflow: false };
|
|
825
|
-
this.imageBodies.push(this.imageCandidate);
|
|
826
|
-
}
|
|
827
|
-
this.keep = this.isKey || this.retainValue() && f?.key !== "message";
|
|
828
|
-
this.token = "";
|
|
829
|
-
this.markerTail = "";
|
|
830
|
-
this.mode = "string";
|
|
831
|
-
return;
|
|
832
|
-
}
|
|
833
|
-
if (c === "{" || c === "[") {
|
|
834
|
-
if (!this.expectedValue() || this.frames.length >= 128) {
|
|
835
|
-
this.fail("native_metadata_depth_limit");
|
|
836
|
-
return;
|
|
837
|
-
}
|
|
838
|
-
const keep = this.retainValue();
|
|
839
|
-
this.frames.push({ path: f ? [...f.path, f.array ? "[]" : f.key] : [], omitImage: false, array: c === "[", state: c === "[" ? "value" : "key", key: null, keep, value: keep ? c === "[" ? [] : {} : null });
|
|
840
|
-
return;
|
|
841
|
-
}
|
|
842
|
-
if (c === "}" || c === "]") {
|
|
843
|
-
if (!f || f.array !== (c === "]") || !["comma", f.array ? "value" : "key"].includes(f.state)) {
|
|
844
|
-
this.fail();
|
|
845
|
-
return;
|
|
846
|
-
}
|
|
847
|
-
this.frames.pop();
|
|
848
|
-
this.accept(f.value);
|
|
849
|
-
return;
|
|
850
|
-
}
|
|
851
|
-
if (c === ":") {
|
|
852
|
-
if (!f || f.state !== "colon") {
|
|
853
|
-
this.fail();
|
|
854
|
-
return;
|
|
855
|
-
}
|
|
856
|
-
f.state = "valueRequired";
|
|
857
|
-
return;
|
|
858
|
-
}
|
|
859
|
-
if (c === ",") {
|
|
860
|
-
if (!f || f.state !== "comma") {
|
|
861
|
-
this.fail();
|
|
862
|
-
return;
|
|
863
|
-
}
|
|
864
|
-
f.state = f.array ? "valueRequired" : "keyRequired";
|
|
865
|
-
return;
|
|
820
|
+
return model ? { model, inferred: false } : { model: "gpt-5", inferred: true };
|
|
821
|
+
}
|
|
822
|
+
function codexFact(b, s, at, timestamp, d, key) {
|
|
823
|
+
const input = d[0], output = d[2], reasoning = d[3], cached = Math.min(d[1], input);
|
|
824
|
+
const { model, inferred } = codexModel(b.currentModel, timestamp);
|
|
825
|
+
const fact = { key, at, model, provider: "openai", input: input - cached, output, cacheRead: cached, cacheWrite: 0, cacheWrite1h: 0, reasoning };
|
|
826
|
+
if (inferred) fact.inferredModel = true;
|
|
827
|
+
if (s.tier) fact.tier = s.tier;
|
|
828
|
+
return fact;
|
|
829
|
+
}
|
|
830
|
+
function codexWanted(head, lineNo) {
|
|
831
|
+
return lineNo === 0 && head.includes("session_meta") || head.includes('"token_count"') || head.includes('"turn_context"') || head.includes("thread_settings_applied") || head.includes('"task_started"') || head.includes('"token_usage_record"') || head.includes('"type":"compacted"') || head.includes('"UserMessage"') && head.includes("item_completed") || head.includes('"role":"user"') && head.includes('"response_item"') || head.includes('"event_msg"') && head.includes('"type":"user_message"');
|
|
832
|
+
}
|
|
833
|
+
function codexLine(b, line, s) {
|
|
834
|
+
const n = s.lineNo++;
|
|
835
|
+
const head = line.toString("latin1", 0, Math.min(line.length, 260));
|
|
836
|
+
const f = b.facts;
|
|
837
|
+
const parse = () => {
|
|
838
|
+
try {
|
|
839
|
+
return JSON.parse(line.toString("utf8"));
|
|
840
|
+
} catch {
|
|
841
|
+
f.malformed++;
|
|
842
|
+
return null;
|
|
866
843
|
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
844
|
+
};
|
|
845
|
+
if (n === 0 && head.includes("session_meta")) {
|
|
846
|
+
const r = parse();
|
|
847
|
+
if (!r) return;
|
|
848
|
+
const p = isRec(r.payload) ? r.payload : {};
|
|
849
|
+
f.sessionId = str(p.id) ?? str(p.session_id) ?? f.sessionId;
|
|
850
|
+
f.project = str(p.cwd);
|
|
851
|
+
f.startedAt = str(p.timestamp) ?? str(r.timestamp);
|
|
852
|
+
const source = p.source;
|
|
853
|
+
if (isRec(source) && "subagent" in source) {
|
|
854
|
+
f.child = true;
|
|
855
|
+
const sub = isRec(source.subagent) ? source.subagent : {};
|
|
856
|
+
const spawn = isRec(sub.thread_spawn) ? sub.thread_spawn : {};
|
|
857
|
+
f.parentId = str(spawn.parent_thread_id) ?? str(p.parent_thread_id) ?? str(p.forked_from_id);
|
|
870
858
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
else if (/[tfn]/.test(c)) this.mode = "literal";
|
|
875
|
-
else this.fail();
|
|
859
|
+
if (source === "exec" || p.originator === "codex_exec" || p.originator === "codex_sdk_ts") f.automated = true;
|
|
860
|
+
touch(b, time(r.timestamp));
|
|
861
|
+
return;
|
|
876
862
|
}
|
|
877
|
-
|
|
878
|
-
const
|
|
879
|
-
|
|
880
|
-
const
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
at2 = end;
|
|
891
|
-
if (at2 >= text.length) break;
|
|
892
|
-
}
|
|
893
|
-
const c = text[at2++];
|
|
894
|
-
if (this.scanUnicode) {
|
|
895
|
-
this.scanUnicode += c;
|
|
896
|
-
if (this.scanUnicode.length === 5) {
|
|
897
|
-
if (/^u[0-9a-f]{4}$/i.test(this.scanUnicode)) add(String.fromCharCode(parseInt(this.scanUnicode.slice(1), 16)));
|
|
898
|
-
this.scanUnicode = "";
|
|
899
|
-
}
|
|
900
|
-
continue;
|
|
901
|
-
}
|
|
902
|
-
if (this.scanEscape) {
|
|
903
|
-
this.scanEscape = false;
|
|
904
|
-
if (c === "u") this.scanUnicode = "u";
|
|
905
|
-
else add({ n: "\n", r: "\r", t: " " }[c] ?? c);
|
|
906
|
-
continue;
|
|
907
|
-
}
|
|
908
|
-
if (c === '"') {
|
|
909
|
-
this.scanString = !this.scanString;
|
|
910
|
-
this.scanTail = "";
|
|
911
|
-
} else if (this.scanString && c === "\\") this.scanEscape = true;
|
|
863
|
+
if (head.includes('"token_count"')) {
|
|
864
|
+
const r = parse();
|
|
865
|
+
const p = r && isRec(r.payload) ? r.payload : null;
|
|
866
|
+
const info = p && isRec(p.info) ? p.info : null;
|
|
867
|
+
if (!r || !info) return;
|
|
868
|
+
const total = tuple(info.total_token_usage), last = tuple(info.last_token_usage);
|
|
869
|
+
const totalSig = total ? total.join(",") : null;
|
|
870
|
+
const advanced = totalSig === null || totalSig !== s.previousTotal;
|
|
871
|
+
let d = null;
|
|
872
|
+
if (last && advanced) d = last;
|
|
873
|
+
else if (total) {
|
|
874
|
+
const prev = s.previousTotal ? s.previousTotal.split(",").map(Number) : [0, 0, 0, 0, 0];
|
|
875
|
+
d = total.map((v, i) => Math.max(0, v - prev[i]));
|
|
912
876
|
}
|
|
877
|
+
if (totalSig) s.previousTotal = totalSig;
|
|
878
|
+
if (!d || !(d[0] || d[1] || d[2] || d[3])) return;
|
|
879
|
+
s.deltas.add(d.join(","));
|
|
880
|
+
const key = totalSig ? `x:${totalSig}|${last ? last.join(",") : ""}` : null;
|
|
881
|
+
addUsage(b, codexFact(b, s, time(r.timestamp), String(r.timestamp ?? ""), d, key));
|
|
882
|
+
return;
|
|
913
883
|
}
|
|
914
|
-
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
while (at2 < text.length && !this.error) {
|
|
920
|
-
if (this.mode === "string" && !this.escape && !this.unicode) {
|
|
921
|
-
special.lastIndex = at2;
|
|
922
|
-
const hit = special.exec(text), end = hit?.index ?? text.length;
|
|
923
|
-
if (end > at2) this.add(text.slice(at2, end));
|
|
924
|
-
at2 = end;
|
|
925
|
-
if (at2 >= text.length) break;
|
|
926
|
-
}
|
|
927
|
-
this.char(text[at2++]);
|
|
884
|
+
if (head.includes('"turn_context"')) {
|
|
885
|
+
const p = parse()?.payload;
|
|
886
|
+
if (isRec(p)) {
|
|
887
|
+
b.currentModel = str(p.model) ?? b.currentModel;
|
|
888
|
+
if (typeof p.cwd === "string") f.project = p.cwd;
|
|
928
889
|
}
|
|
890
|
+
return;
|
|
929
891
|
}
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
if (
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
const root = this.root, payload = root.payload;
|
|
937
|
-
for (const body of this.imageBodies) {
|
|
938
|
-
if (root.type === "response_item" && (payload?.type === "custom_tool_call_output" || payload?.type === "function_call_output") && body.frame.value?.type === "input_image") continue;
|
|
939
|
-
if (body.overflow) this.fail("native_metadata_bytes_limit");
|
|
940
|
-
else Object.defineProperty(body.frame.value, "image_url", { value: body.chunks.join(""), writable: true, enumerable: true, configurable: true });
|
|
941
|
-
}
|
|
892
|
+
if (head.includes("thread_settings_applied")) {
|
|
893
|
+
const p = parse()?.payload;
|
|
894
|
+
const settings = isRec(p) && isRec(p.thread_settings) ? p.thread_settings : null;
|
|
895
|
+
if (settings && "service_tier" in settings) {
|
|
896
|
+
const v = settings.service_tier;
|
|
897
|
+
s.tier = v === "priority" || v === "fast" ? "fast" : v === "default" || v === "standard" ? "standard" : void 0;
|
|
942
898
|
}
|
|
943
|
-
|
|
944
|
-
return { record: this.error ? null : this.root, code: this.error, exclusionMarker: this.marker };
|
|
899
|
+
return;
|
|
945
900
|
}
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
if (r
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
if (harness === "pi" && r.type === "session") session = str(r.id) ?? session;
|
|
992
|
-
if (harness === "codex" && (r.type === "session_meta" || r.type === "turn_context")) {
|
|
993
|
-
session = str(p.session_id, p.id) ?? session;
|
|
994
|
-
model = str(p.model) ?? model;
|
|
995
|
-
provider = str(p.model_provider, p.provider) ?? provider;
|
|
996
|
-
return;
|
|
997
|
-
}
|
|
998
|
-
const eventAt = at(r.timestamp) ?? at(m.timestamp) ?? at(obj(r.time).created);
|
|
999
|
-
const key = str(r.uuid, r.id, p.id, m.id) ?? `record:${seq}`;
|
|
1000
|
-
const accepted = options.acceptRecord?.(r, { key, rawStart: start, rawEnd: offset, eventAt, project }) ?? true;
|
|
1001
|
-
let input = null, output = null, read = null, write = null, reasoning = null, reported = null;
|
|
1002
|
-
let observedModel = null, observedProvider = null, id, basis, includesCache = false, includesReasoning = true;
|
|
1003
|
-
const gaps = [];
|
|
1004
|
-
if (harness === "codex") {
|
|
1005
|
-
if (r.type !== "event_msg" || p.type !== "token_count") return;
|
|
1006
|
-
const info = obj(p.info), last = obj(info.last_token_usage), total = obj(info.total_token_usage);
|
|
1007
|
-
if (!Object.keys(last).length && !Object.keys(total).length) return;
|
|
1008
|
-
observed++;
|
|
1009
|
-
const previousTotal = previous;
|
|
1010
|
-
previous = Object.keys(total).length ? total : previous;
|
|
1011
|
-
const signature = Object.keys(total).length ? JSON.stringify([session, total.input_tokens, total.output_tokens, total.cached_input_tokens, total.reasoning_output_tokens]) : null;
|
|
1012
|
-
if (signature && seenTotals.has(signature)) return;
|
|
1013
|
-
if (signature) seenTotals.add(signature);
|
|
1014
|
-
const delta = (field) => {
|
|
1015
|
-
const current = num2(total[field]), prev = num2(previousTotal?.[field]);
|
|
1016
|
-
return current === null || prev === null || current < prev ? null : current - prev;
|
|
1017
|
-
};
|
|
1018
|
-
const firstComplete = previousTotal === null && num2(last.input_tokens) !== null && last.input_tokens === total.input_tokens && last.output_tokens === total.output_tokens;
|
|
1019
|
-
const bucket = (field) => num2(last[field]) ?? delta(field) ?? (firstComplete ? num2(total[field]) : null);
|
|
1020
|
-
input = bucket("input_tokens");
|
|
1021
|
-
output = bucket("output_tokens");
|
|
1022
|
-
read = bucket("cached_input_tokens");
|
|
1023
|
-
write = bucket("cache_write_input_tokens");
|
|
1024
|
-
reasoning = bucket("reasoning_output_tokens");
|
|
1025
|
-
if (!Object.keys(last).length && previousTotal === null) gaps.push("cumulative_baseline_unknown");
|
|
1026
|
-
if (previousTotal && Object.keys(total).length && ["input_tokens", "output_tokens"].some((k) => num2(total[k]) !== null && num2(previousTotal[k]) !== null && num2(total[k]) < num2(previousTotal[k]))) gaps.push("cumulative_counter_reset");
|
|
1027
|
-
observedModel = str(p.model, info.model) ?? model;
|
|
1028
|
-
observedProvider = str(p.model_provider, p.provider, info.provider) ?? provider;
|
|
1029
|
-
includesCache = true;
|
|
1030
|
-
basis = "codex_last_response_with_cumulative_dedup";
|
|
1031
|
-
id = `codex:${hash(JSON.stringify([session, str(p.response_id, info.response_id) ?? signature ?? [eventAt, last]]))}`;
|
|
1032
|
-
write = write ?? 0;
|
|
1033
|
-
} else if (harness === "claude") {
|
|
1034
|
-
if (r.type !== "assistant" || m.role !== "assistant") return;
|
|
1035
|
-
const u = obj(m.usage);
|
|
1036
|
-
input = num2(u.input_tokens);
|
|
1037
|
-
output = num2(u.output_tokens);
|
|
1038
|
-
read = num2(u.cache_read_input_tokens) ?? (input !== null ? 0 : null);
|
|
1039
|
-
write = num2(u.cache_creation_input_tokens) ?? (input !== null ? 0 : null);
|
|
1040
|
-
reasoning = num2(u.reasoning_tokens);
|
|
1041
|
-
const duration = obj(u.cache_creation);
|
|
1042
|
-
const five = num2(duration.ephemeral_5m_input_tokens), hour = num2(duration.ephemeral_1h_input_tokens);
|
|
1043
|
-
if (five !== null && hour !== null && write !== null && five + hour !== write) gaps.push("cache_write_duration_conflict");
|
|
1044
|
-
if (hour !== null && hour > 0 && (five === null || write === null || five + hour !== write)) gaps.push("cache_write_duration_unknown");
|
|
1045
|
-
observedModel = str(m.model, r.model);
|
|
1046
|
-
observedProvider = str(m.provider, r.provider, r.modelProvider);
|
|
1047
|
-
reported = cost(r.costUSD) ?? cost(m.costUSD);
|
|
1048
|
-
basis = "claude_response_usage";
|
|
1049
|
-
id = `claude:${hash(JSON.stringify([str(m.id, r.uuid) ?? key, str(r.requestId) ?? [session, eventAt]]))}`;
|
|
1050
|
-
} else if (harness === "pi") {
|
|
1051
|
-
if (r.type !== "message" || m.role !== "assistant") return;
|
|
1052
|
-
const u = obj(m.usage);
|
|
1053
|
-
input = num2(u.input);
|
|
1054
|
-
output = num2(u.output);
|
|
1055
|
-
read = num2(u.cacheRead) ?? (input !== null ? 0 : null);
|
|
1056
|
-
write = num2(u.cacheWrite) ?? (input !== null ? 0 : null);
|
|
1057
|
-
reasoning = num2(u.reasoning);
|
|
1058
|
-
reported = cost(obj(u.cost).total);
|
|
1059
|
-
observedModel = str(m.model);
|
|
1060
|
-
observedProvider = str(m.provider);
|
|
1061
|
-
basis = "pi_response_usage";
|
|
1062
|
-
id = `pi:${hash(JSON.stringify([session, str(r.id) ?? key, eventAt]))}`;
|
|
1063
|
-
} else {
|
|
1064
|
-
if (r.role !== "assistant") return;
|
|
1065
|
-
const u = obj(r.tokens), cache = obj(u.cache);
|
|
1066
|
-
input = num2(u.input);
|
|
1067
|
-
output = num2(u.output);
|
|
1068
|
-
read = num2(cache.read) ?? (input !== null ? 0 : null);
|
|
1069
|
-
write = num2(cache.write) ?? (input !== null ? 0 : null);
|
|
1070
|
-
reasoning = num2(u.reasoning);
|
|
1071
|
-
reported = cost(r.cost);
|
|
1072
|
-
observedModel = str(r.modelID, obj(r.model).modelID, obj(r.model).id, r.model);
|
|
1073
|
-
observedProvider = str(r.providerID, obj(r.model).providerID, r.provider);
|
|
1074
|
-
basis = "opencode_response_tokens";
|
|
1075
|
-
includesReasoning = false;
|
|
1076
|
-
id = `opencode:${str(r.id) ?? hash(JSON.stringify([conversationId, key, eventAt]))}`;
|
|
1077
|
-
}
|
|
1078
|
-
if (observedModel === "codex-auto-review") {
|
|
1079
|
-
observedModel = null;
|
|
1080
|
-
gaps.push("workflow_model_identity_unavailable");
|
|
1081
|
-
}
|
|
1082
|
-
if (harness !== "codex") observed++;
|
|
1083
|
-
if (r.isSynthetic === true || r.isMeta === true || m.isSynthetic === true || observedModel === "<synthetic>") {
|
|
1084
|
-
excluded++;
|
|
1085
|
-
return;
|
|
1086
|
-
}
|
|
1087
|
-
if (!accepted) {
|
|
1088
|
-
excluded++;
|
|
1089
|
-
return;
|
|
1090
|
-
}
|
|
1091
|
-
if (!observedModel) gaps.push("model_unrecorded");
|
|
1092
|
-
if (!observedProvider) gaps.push("provider_unrecorded");
|
|
1093
|
-
if (input === null || output === null) gaps.push("usage_partial");
|
|
1094
|
-
if (includesCache && read === null) gaps.push("cache_inclusion_unknown");
|
|
1095
|
-
if (includesCache && input !== null && read !== null && write !== null && read + write > input) gaps.push("cache_exceeds_input");
|
|
1096
|
-
if (includesReasoning && output !== null && reasoning !== null && reasoning > output) gaps.push("reasoning_exceeds_output");
|
|
1097
|
-
put({ id, conversationId, harness, eventAt, project, provider: observedProvider, model: observedModel, canonicalModel: null, inputTokens: input, outputTokens: output, cacheReadTokens: read, cacheWriteTokens: write, reasoningTokens: reasoning, inputIncludesCache: includesCache, outputIncludesReasoning: includesReasoning, reportedCostUsd: reported, basis, gaps, ...harness === "claude" ? { cacheWrite5mTokens: num2(obj(obj(m.usage).cache_creation).ephemeral_5m_input_tokens), cacheWrite1hTokens: num2(obj(obj(m.usage).cache_creation).ephemeral_1h_input_tokens) } : {} });
|
|
1098
|
-
};
|
|
1099
|
-
const pushLine = (line) => {
|
|
1100
|
-
const start = offset;
|
|
1101
|
-
offset += Buffer.byteLength(line) + 1;
|
|
1102
|
-
if (!line.trim()) return;
|
|
1103
|
-
let r;
|
|
1104
|
-
try {
|
|
1105
|
-
r = obj(JSON.parse(line));
|
|
1106
|
-
} catch {
|
|
1107
|
-
seq++;
|
|
1108
|
-
return;
|
|
901
|
+
if (head.includes('"task_started"')) {
|
|
902
|
+
const p = parse()?.payload;
|
|
903
|
+
s.imported = isRec(p) && String(p.turn_id ?? "").startsWith("external-import");
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
if (head.includes('"UserMessage"') && head.includes("item_completed")) {
|
|
907
|
+
const r = parse();
|
|
908
|
+
const p = r && isRec(r.payload) ? r.payload : null;
|
|
909
|
+
if (!r || !p) return;
|
|
910
|
+
const item = isRec(p.item) ? p.item : {};
|
|
911
|
+
s.itemPrompts.push({
|
|
912
|
+
key: str(item.id),
|
|
913
|
+
at: time(r.timestamp),
|
|
914
|
+
text: textOf(item.content),
|
|
915
|
+
imported: s.imported || String(p.turn_id ?? "").startsWith("external-import"),
|
|
916
|
+
project: f.project
|
|
917
|
+
});
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
if (head.includes('"role":"user"') && head.includes('"response_item"')) {
|
|
921
|
+
const r = parse();
|
|
922
|
+
if (!r) return;
|
|
923
|
+
const p = isRec(r.payload) ? r.payload : {};
|
|
924
|
+
s.responsePrompts.push({ key: null, at: time(r.timestamp), text: textOf(p.content), imported: s.imported, project: f.project });
|
|
925
|
+
return;
|
|
926
|
+
}
|
|
927
|
+
if (head.includes('"event_msg"') && line.subarray(0, 400).includes('"type":"user_message"')) {
|
|
928
|
+
const r = parse();
|
|
929
|
+
if (!r) return;
|
|
930
|
+
const p = isRec(r.payload) ? r.payload : {};
|
|
931
|
+
s.itemPrompts.push({ key: null, at: time(r.timestamp), text: typeof p.message === "string" ? p.message : "", imported: s.imported, project: f.project });
|
|
932
|
+
return;
|
|
933
|
+
}
|
|
934
|
+
if (head.includes('"token_usage_record"')) {
|
|
935
|
+
const r = parse();
|
|
936
|
+
const p = r && isRec(r.payload) ? r.payload : null;
|
|
937
|
+
const id = p ? str(p.response_id) : null;
|
|
938
|
+
if (r && p && id && isRec(p.usage)) s.records.set(id, { at: time(r.timestamp), usage: p.usage });
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
if (head.includes('"type":"compacted"')) {
|
|
942
|
+
const at = line.lastIndexOf('"compaction_response_id":"');
|
|
943
|
+
if (at >= 0) {
|
|
944
|
+
const start = at + 26, end = line.indexOf(34, start);
|
|
945
|
+
if (end > start) s.compacted.add(line.toString("latin1", start, end));
|
|
1109
946
|
}
|
|
1110
|
-
|
|
1111
|
-
};
|
|
1112
|
-
return { pushRecord, invalidateContext: () => {
|
|
1113
|
-
project = null;
|
|
1114
|
-
model = null;
|
|
1115
|
-
provider = null;
|
|
1116
|
-
previous = null;
|
|
1117
|
-
}, pushLine, records: () => [...records.values()], counts: () => ({ observed, excluded, deduplicated: Math.max(0, observed - excluded - records.size) }) };
|
|
947
|
+
}
|
|
1118
948
|
}
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
const
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
return;
|
|
1136
|
-
|
|
1137
|
-
if (turnBytes + Buffer.byteLength(turnPrefix) <= turnLimit) {
|
|
1138
|
-
turns.push({ lines: turnLines, bytes: turnBytes, prefix: turnPrefix });
|
|
1139
|
-
keptBytes += turnBytes;
|
|
1140
|
-
while (turns.length > 1 && keptBytes + Buffer.byteLength(turns[0].prefix) > turnLimit) keptBytes -= turns.shift().bytes;
|
|
1141
|
-
} else gaps.add("context_turn_oversize");
|
|
1142
|
-
turnLines = [];
|
|
1143
|
-
turnBytes = 0;
|
|
1144
|
-
};
|
|
1145
|
-
const process2 = (raw, end) => {
|
|
1146
|
-
if (++records > (options.maxRecords ?? 1e6)) {
|
|
1147
|
-
gaps.add("native_usage_records_limit");
|
|
1148
|
-
return;
|
|
1149
|
-
}
|
|
1150
|
-
const line = raw.toString("utf8");
|
|
1151
|
-
let r;
|
|
1152
|
-
try {
|
|
1153
|
-
const parsed = JSON.parse(line);
|
|
1154
|
-
if (!isRecord(parsed)) return;
|
|
1155
|
-
r = parsed;
|
|
1156
|
-
} catch {
|
|
1157
|
-
gaps.add("malformed_record");
|
|
1158
|
-
return;
|
|
1159
|
-
}
|
|
1160
|
-
if (r.entrypoint === "sdk-ts" || r.originator === "codex_exec") programmatic = true;
|
|
1161
|
-
const p = isRecord(r.payload) ? r.payload : r;
|
|
1162
|
-
if (p.originator === "codex_exec" || p.source === "exec") programmatic = true;
|
|
1163
|
-
const m = isRecord(r.message) ? r.message : {};
|
|
1164
|
-
if (typeof r.cwd === "string") project = r.cwd;
|
|
1165
|
-
if (typeof p.cwd === "string") project = p.cwd;
|
|
1166
|
-
if (harness === "claude" && typeof r.sessionId === "string") {
|
|
1167
|
-
if (file.includes(`${path.sep}subagents${path.sep}`) || r.isSidechain === true) {
|
|
1168
|
-
parent = r.sessionId;
|
|
1169
|
-
nativeId = `${parent}:${path.basename(file, ".jsonl")}`;
|
|
1170
|
-
} else nativeId = r.sessionId;
|
|
1171
|
-
}
|
|
1172
|
-
if (harness === "codex" && r.type === "session_meta") {
|
|
1173
|
-
nativeId = typeof p.id === "string" ? p.id : typeof p.session_id === "string" ? p.session_id : nativeId;
|
|
1174
|
-
const source = isRecord(p.source) ? p.source : {}, sub = isRecord(source.subagent) ? source.subagent : {}, spawn = isRecord(sub.thread_spawn) ? sub.thread_spawn : {};
|
|
1175
|
-
if (typeof spawn.parent_thread_id === "string") parent = spawn.parent_thread_id;
|
|
1176
|
-
}
|
|
1177
|
-
if (harness === "pi" && r.type === "session") {
|
|
1178
|
-
if (typeof r.id === "string") nativeId = r.id;
|
|
1179
|
-
if (typeof r.parentSessionId === "string") parent = r.parentSessionId;
|
|
1180
|
-
}
|
|
1181
|
-
if (r.type === "turn_context" || r.type === "session_meta" || r.type === "model_change") {
|
|
1182
|
-
currentModel = typeof p.model === "string" ? p.model : typeof r.modelId === "string" ? r.modelId : currentModel;
|
|
1183
|
-
currentProvider = typeof p.model_provider === "string" ? p.model_provider : typeof r.provider === "string" ? r.provider : currentProvider;
|
|
1184
|
-
}
|
|
1185
|
-
if (scopeUncertain && project !== null && (harness === "claude" && typeof r.sessionId === "string" || harness === "codex" && r.type === "session_meta" && (typeof p.id === "string" || typeof p.session_id === "string") || harness === "pi" && r.type === "session" && typeof r.id === "string")) scopeUncertain = false;
|
|
1186
|
-
if (options.sourceAllowed && !options.sourceAllowed(nativeId, project)) throw new Error("native_source_policy_excluded");
|
|
1187
|
-
if (hasExclusionMarker(extractTypedText(m.content)) || typeof p.message === "string" && hasExclusionMarker(p.message)) maintenance = true;
|
|
1188
|
-
const isInput = harness === "claude" && r.type === "user" && m.role === "user" && !(Array.isArray(m.content) && m.content.some((b) => isRecord(b) && b.type === "tool_result")) || harness === "codex" && r.type === "event_msg" && p.type === "user_message" || harness === "pi" && r.type === "message" && m.role === "user";
|
|
1189
|
-
if (isInput) {
|
|
1190
|
-
finishTurn();
|
|
1191
|
-
turnPrefix = JSON.stringify({ type: "turn_context", payload: { cwd: project, model: currentModel, model_provider: currentProvider } }) + "\n";
|
|
1192
|
-
}
|
|
1193
|
-
if (projected && turnLimit > 0) {
|
|
1194
|
-
turnLines = [];
|
|
1195
|
-
turnBytes = turnLimit + 1;
|
|
1196
|
-
gaps.add("context_turn_oversize");
|
|
1197
|
-
}
|
|
1198
|
-
if (turnLimit > 0 && !projected && turnBytes <= turnLimit) {
|
|
1199
|
-
const recordKey = typeof r.uuid === "string" ? r.uuid : typeof r.id === "string" ? r.id : typeof p.id === "string" ? p.id : `record:${records}`;
|
|
1200
|
-
const encoded = JSON.stringify({ ...r, _auditRecordKey: recordKey }) + "\n";
|
|
1201
|
-
turnBytes += Buffer.byteLength(encoded);
|
|
1202
|
-
if (turnBytes <= turnLimit) turnLines.push(encoded);
|
|
1203
|
-
else if (turnLines.length) turnLines = [];
|
|
1204
|
-
}
|
|
1205
|
-
if (!usage) usage = createNativeUsageAccumulator(harness, sourceId(harness, nativeId), { maxRecords: 1e6, acceptRecord: (_r, scope) => !maintenance && !scopeUncertain && options.accept(scope.project, scope.eventAt) });
|
|
1206
|
-
usage.pushRecord(r, { rawStart: offset, rawEnd: end });
|
|
1207
|
-
if (scopeUncertain) return;
|
|
1208
|
-
let text = "", identity = "", origin = "unknown", eligible = false, excluded = null;
|
|
1209
|
-
const time = clock(r.timestamp), key = typeof r.uuid === "string" ? r.uuid : typeof r.id === "string" ? r.id : typeof p.id === "string" ? p.id : `record:${records}`;
|
|
1210
|
-
if (harness === "claude" && r.type === "user" && m.role === "user") {
|
|
1211
|
-
if (Array.isArray(m.content) && m.content.some((b) => isRecord(b) && b.type === "tool_result")) return;
|
|
1212
|
-
text = extractTypedText(m.content);
|
|
1213
|
-
identity = typeof r.promptId === "string" ? `prompt:${r.promptId}` : `record:${key}`;
|
|
1214
|
-
origin = typeof r.promptId === "string" ? "claude_prompt_id" : "unknown";
|
|
1215
|
-
eligible = typeof r.promptId === "string" && !programmatic;
|
|
1216
|
-
} else if (harness === "codex" && r.type === "event_msg" && p.type === "user_message") {
|
|
1217
|
-
text = typeof p.message === "string" ? p.message : "";
|
|
1218
|
-
identity = `marker:${offset}`;
|
|
1219
|
-
origin = programmatic ? "unknown" : "codex_human_marker";
|
|
1220
|
-
eligible = !programmatic;
|
|
1221
|
-
} else if (harness === "pi" && r.type === "message" && m.role === "user") {
|
|
1222
|
-
text = extractTypedText(m.content);
|
|
1223
|
-
identity = `node:${key}`;
|
|
1224
|
-
origin = "pi_user_projection";
|
|
1225
|
-
} else {
|
|
1226
|
-
const info = isRecord(p.info) ? p.info : {};
|
|
1227
|
-
const observedModel = typeof m.model === "string" ? m.model : typeof p.model === "string" ? p.model : typeof r.model === "string" ? r.model : typeof info.model === "string" ? info.model : currentModel, observedProvider = typeof m.provider === "string" ? m.provider : typeof p.model_provider === "string" ? p.model_provider : typeof p.provider === "string" ? p.provider : typeof r.provider === "string" ? r.provider : currentProvider;
|
|
1228
|
-
if ((m.role === "assistant" || harness === "codex" && (r.type === "event_msg" && p.type === "token_count" || r.type === "response_item" && p.type === "message" && p.role === "assistant")) && observedModel !== "<synthetic>" && r.isSynthetic !== true && options.accept(project, time)) language.push({ id: `metadata:${records}`, conversationId: sourceId(harness, nativeId), parentId: parent, harness, eventAt: time, project, role: "assistant", text: "", model: observedModel === "codex-auto-review" ? null : observedModel, provider: observedProvider, directUser: false, route: null });
|
|
1229
|
-
return;
|
|
1230
|
-
}
|
|
1231
|
-
if (!text.trim() && !projected) return;
|
|
1232
|
-
if (hasExclusionMarker(text)) {
|
|
1233
|
-
maintenance = true;
|
|
1234
|
-
excluded = "maintenance_exclusion_marker";
|
|
1235
|
-
} else if (r.isMeta === true || r.isSynthetic === true || m.isSynthetic === true) excluded = "declared_meta_or_synthetic_input";
|
|
1236
|
-
else if (parent && events.length === 0) excluded = "child_initialization";
|
|
1237
|
-
else if (typeof r.session_id === "string" && r.session_id !== nativeId) excluded = "inherited_identity_unverified";
|
|
1238
|
-
if (seen.has(identity)) return;
|
|
1239
|
-
seen.add(identity);
|
|
1240
|
-
if (!options.accept(project, time)) return;
|
|
1241
|
-
const redacted = clean(text);
|
|
1242
|
-
if (promptBytes + Buffer.byteLength(redacted) > (options.maxPromptBytes ?? 64 * 1024 * 1024)) {
|
|
1243
|
-
gaps.add("native_input_text_limit");
|
|
1244
|
-
return;
|
|
949
|
+
function finishCodex(b, s) {
|
|
950
|
+
const f = b.facts;
|
|
951
|
+
for (const id of s.compacted) {
|
|
952
|
+
const record = s.records.get(id);
|
|
953
|
+
const d = record ? tuple(record.usage) : null;
|
|
954
|
+
if (!record || !d || !(d[0] || d[1] || d[2] || d[3]) || s.deltas.has(d.join(","))) continue;
|
|
955
|
+
const fact = codexFact(b, s, record.at, record.at ? new Date(record.at).toISOString() : "", d, `codex-compaction:${id}`);
|
|
956
|
+
fact.compaction = true;
|
|
957
|
+
f.usage.push(fact);
|
|
958
|
+
touch(b, record.at);
|
|
959
|
+
}
|
|
960
|
+
const list = s.itemPrompts.length ? s.itemPrompts : s.responsePrompts;
|
|
961
|
+
const reason = f.child ? "subagent_parent_written" : f.automated ? "programmatic_exec_session" : null;
|
|
962
|
+
const seen = /* @__PURE__ */ new Set();
|
|
963
|
+
list.forEach((p, i) => {
|
|
964
|
+
if (p.key) {
|
|
965
|
+
if (seen.has(p.key)) return;
|
|
966
|
+
seen.add(p.key);
|
|
1245
967
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
projected = false;
|
|
1265
|
-
} else {
|
|
1266
|
-
turnLines = [];
|
|
1267
|
-
turnBytes = turnLimit + 1;
|
|
1268
|
-
records++;
|
|
1269
|
-
gaps.add(result.code ?? "native_metadata_invalid");
|
|
1270
|
-
gaps.add("native_scope_record_unavailable");
|
|
1271
|
-
usage?.invalidateContext();
|
|
1272
|
-
scopeUncertain = true;
|
|
1273
|
-
project = null;
|
|
1274
|
-
currentModel = null;
|
|
1275
|
-
currentProvider = null;
|
|
1276
|
-
}
|
|
1277
|
-
} else process2(held.length === 1 ? held[0] : Buffer.concat(held, heldBytes), end);
|
|
1278
|
-
held = [];
|
|
1279
|
-
heldBytes = 0;
|
|
1280
|
-
offset = end;
|
|
1281
|
-
};
|
|
1282
|
-
const stream = fs.createReadStream(file, { highWaterMark: 64 * 1024, ...captureSize > 0 ? { end: captureSize - 1 } : {} });
|
|
968
|
+
const key = `codex:${f.sessionId}:${p.key ?? i}`;
|
|
969
|
+
if (reason) return addPrompt(b, key, p.at, p.project, "excluded", reason, "");
|
|
970
|
+
if (p.imported) return addPrompt(b, key, p.at, p.project, "excluded", "imported_from_claude_transcript", "");
|
|
971
|
+
const c = classifyText(cleanCodexText(p.text));
|
|
972
|
+
addPrompt(b, key, p.at, p.project, c.kind, c.reason, c.text);
|
|
973
|
+
});
|
|
974
|
+
const answers = f.usage.filter((u) => u.at !== null).sort((x, y) => x.at - y.at);
|
|
975
|
+
let j = 0, previous = null;
|
|
976
|
+
for (const p of [...f.prompts].sort((x, y) => (x.at ?? 0) - (y.at ?? 0))) {
|
|
977
|
+
while (j < answers.length && answers[j].at <= (p.at ?? 0)) previous = answers[j++].model;
|
|
978
|
+
p.before = previous;
|
|
979
|
+
p.beforeProvider = previous ? "openai" : null;
|
|
980
|
+
p.after = answers[j]?.model ?? null;
|
|
981
|
+
p.afterProvider = p.after ? "openai" : null;
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
function piLine(b, line) {
|
|
985
|
+
let r;
|
|
1283
986
|
try {
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
987
|
+
r = JSON.parse(line.toString("utf8"));
|
|
988
|
+
} catch {
|
|
989
|
+
b.facts.malformed++;
|
|
990
|
+
return;
|
|
991
|
+
}
|
|
992
|
+
const f = b.facts, at = time(r.timestamp);
|
|
993
|
+
if (r.type === "session") {
|
|
994
|
+
f.sessionId = str(r.id) ?? f.sessionId;
|
|
995
|
+
f.project = str(r.cwd);
|
|
996
|
+
touch(b, at);
|
|
997
|
+
return;
|
|
998
|
+
}
|
|
999
|
+
if (r.type === "model_change") {
|
|
1000
|
+
b.currentModel = str(r.modelId) ?? b.currentModel;
|
|
1001
|
+
b.currentProvider = str(r.provider) ?? b.currentProvider;
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
if (r.type !== "message" || !isRec(r.message)) return;
|
|
1005
|
+
const m = r.message;
|
|
1006
|
+
if (m.role === "assistant" && isRec(m.usage)) {
|
|
1007
|
+
const u = m.usage;
|
|
1008
|
+
const fact = {
|
|
1009
|
+
key: `pi:${f.sessionId}:${String(r.id)}`,
|
|
1010
|
+
at,
|
|
1011
|
+
model: str(m.model) ?? b.currentModel,
|
|
1012
|
+
provider: str(m.provider) ?? b.currentProvider,
|
|
1013
|
+
input: int(u.input),
|
|
1014
|
+
output: int(u.output),
|
|
1015
|
+
cacheRead: int(u.cacheRead),
|
|
1016
|
+
cacheWrite: int(u.cacheWrite),
|
|
1017
|
+
cacheWrite1h: 0,
|
|
1018
|
+
reasoning: 0
|
|
1019
|
+
};
|
|
1020
|
+
if (fact.input + fact.output + fact.cacheRead + fact.cacheWrite > 0) addUsage(b, fact);
|
|
1021
|
+
else answered(b, fact.model, fact.provider);
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
if (m.role === "user") {
|
|
1025
|
+
const c = classifyText(textOf(m.content));
|
|
1026
|
+
addPrompt(b, `pi:${f.sessionId}:${String(r.id)}`, at, f.project, c.kind, c.reason, c.text);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
var CHUNK = 4 * 1024 * 1024;
|
|
1030
|
+
var MAX_LINE = 256 * 1024 * 1024;
|
|
1031
|
+
var sharedBuffer = null;
|
|
1032
|
+
function extractFile(file, harness) {
|
|
1033
|
+
const fd = fs.openSync(file, "r");
|
|
1034
|
+
try {
|
|
1035
|
+
const stat = fs.fstatSync(fd);
|
|
1036
|
+
const b = makeBuilder(file, harness, stat.size, stat.mtimeMs);
|
|
1037
|
+
const codex = {
|
|
1038
|
+
lineNo: 0,
|
|
1039
|
+
previousTotal: null,
|
|
1040
|
+
tier: void 0,
|
|
1041
|
+
imported: false,
|
|
1042
|
+
records: /* @__PURE__ */ new Map(),
|
|
1043
|
+
compacted: /* @__PURE__ */ new Set(),
|
|
1044
|
+
deltas: /* @__PURE__ */ new Set(),
|
|
1045
|
+
itemPrompts: [],
|
|
1046
|
+
responsePrompts: []
|
|
1047
|
+
};
|
|
1048
|
+
const subagent = file.includes(SUBAGENT_DIR), sessions = /* @__PURE__ */ new Set();
|
|
1049
|
+
const onLine = harness === "claude" ? (line) => claudeLine(b, line, subagent, sessions) : harness === "codex" ? (line) => codexLine(b, line, codex) : (line) => piLine(b, line);
|
|
1050
|
+
let buffer = sharedBuffer ??= Buffer.allocUnsafe(CHUNK);
|
|
1051
|
+
let filled = 0, position = 0, skipping = false;
|
|
1052
|
+
for (; ; ) {
|
|
1053
|
+
if (filled === buffer.length) {
|
|
1054
|
+
if (harness === "codex" && !codexWanted(buffer.toString("latin1", 0, Math.min(filled, 400)), codex.lineNo)) {
|
|
1055
|
+
skipping = true;
|
|
1056
|
+
filled = 0;
|
|
1057
|
+
codex.lineNo++;
|
|
1058
|
+
} else if (buffer.length >= MAX_LINE) {
|
|
1059
|
+
skipping = true;
|
|
1060
|
+
filled = 0;
|
|
1061
|
+
b.facts.malformed++;
|
|
1302
1062
|
} else {
|
|
1303
|
-
|
|
1304
|
-
|
|
1063
|
+
const bigger = Buffer.allocUnsafe(buffer.length * 2);
|
|
1064
|
+
buffer.copy(bigger, 0, 0, filled);
|
|
1065
|
+
buffer = bigger;
|
|
1066
|
+
if (bigger.length <= 4 * CHUNK) sharedBuffer = bigger;
|
|
1305
1067
|
}
|
|
1306
|
-
if (newline < 0) break;
|
|
1307
|
-
finishLine(consumed + newline + 1);
|
|
1308
|
-
at2 = newline + 1;
|
|
1309
1068
|
}
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
try {
|
|
1321
|
-
let at2 = 0;
|
|
1322
|
-
while (at2 < captureSize) {
|
|
1323
|
-
const n = fs.readSync(fd, buffer, 0, Math.min(buffer.length, captureSize - at2), at2);
|
|
1324
|
-
if (!n) throw new Error("source_changed");
|
|
1325
|
-
verify.update(buffer.subarray(0, n));
|
|
1326
|
-
at2 += n;
|
|
1069
|
+
const n = fs.readSync(fd, buffer, filled, buffer.length - filled, position);
|
|
1070
|
+
position += n;
|
|
1071
|
+
const end = filled + n;
|
|
1072
|
+
let start = 0;
|
|
1073
|
+
for (; ; ) {
|
|
1074
|
+
const nl = buffer.indexOf(10, start);
|
|
1075
|
+
if (nl < 0 || nl >= end) break;
|
|
1076
|
+
if (skipping) skipping = false;
|
|
1077
|
+
else if (nl > start) onLine(buffer.subarray(start, nl));
|
|
1078
|
+
start = nl + 1;
|
|
1327
1079
|
}
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
if (maintenance) {
|
|
1335
|
-
gaps.add("maintenance_source_excluded");
|
|
1336
|
-
for (const event of events) {
|
|
1337
|
-
event.observed = false;
|
|
1338
|
-
event.eligible = false;
|
|
1339
|
-
event.excluded = "maintenance_exclusion_marker";
|
|
1080
|
+
if (n === 0) {
|
|
1081
|
+
if (!skipping && start < end) onLine(buffer.subarray(start, end));
|
|
1082
|
+
break;
|
|
1083
|
+
}
|
|
1084
|
+
buffer.copyWithin(0, start, end);
|
|
1085
|
+
filled = end - start;
|
|
1340
1086
|
}
|
|
1087
|
+
if (harness === "codex") finishCodex(b, codex);
|
|
1088
|
+
if (harness === "claude" && !subagent && sessions.size) b.facts.sessionIds = [...sessions];
|
|
1089
|
+
if (b.facts.excluded) for (const p of b.facts.prompts) p.text = "";
|
|
1090
|
+
return b.facts;
|
|
1091
|
+
} finally {
|
|
1092
|
+
fs.closeSync(fd);
|
|
1341
1093
|
}
|
|
1342
|
-
const artifactHash = hash2.digest("hex"), acc = usage, all = acc?.records() ?? [];
|
|
1343
|
-
finishTurn();
|
|
1344
|
-
const captured = Buffer.from((turns[0]?.prefix ?? "") + turns.flatMap((t) => t.lines).join(""));
|
|
1345
|
-
capturedRecords = turns.reduce((n, t) => n + t.lines.length, 0);
|
|
1346
|
-
if (turnLimit > 0 && capturedRecords < records) gaps.add("context_capture_partial");
|
|
1347
|
-
return { facts: { nativeId, project, parent, child: parent !== null, title: null, events, gaps: [...gaps], hash: artifactHash, consumed: offset, bytes: captured }, usage: maintenance ? [] : all, language: maintenance ? [] : language, hash: artifactHash, bytes: captureSize, stamp: { dev: String(after.dev), ino: String(after.ino), size: Number(after.size), mtimeNs: String(after.mtimeNs), ctimeNs: String(after.ctimeNs) }, counts: maintenance ? { observed: acc?.counts().observed ?? 0, excluded: acc?.counts().observed ?? 0, deduplicated: 0 } : acc?.counts() ?? { observed: 0, excluded: 0, deduplicated: 0 } };
|
|
1348
1094
|
}
|
|
1349
1095
|
|
|
1350
|
-
// packages/core/src/
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
function matchesIgnoreEntry(project, entry) {
|
|
1355
|
-
if (!project) return false;
|
|
1356
|
-
const p = fold(project);
|
|
1357
|
-
const e = fold(entry);
|
|
1358
|
-
if (!p || !e) return false;
|
|
1359
|
-
if (p === e) return true;
|
|
1360
|
-
if (e.includes("/")) return p.startsWith(`${e}/`);
|
|
1361
|
-
return p.split("/").filter(Boolean).includes(e);
|
|
1362
|
-
}
|
|
1363
|
-
function isIgnoredProject(project, entries) {
|
|
1364
|
-
return entries.some((entry) => matchesIgnoreEntry(project, entry));
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
// packages/core/src/analytics/native-pool.ts
|
|
1368
|
-
var PROTOCOL = "audit-native-v1";
|
|
1369
|
-
var FRAME = 1024 * 1024;
|
|
1370
|
-
var code = (error) => error instanceof Error && /^(?:[a-z][a-z0-9_]{0,95})$/.test(error.message) ? error.message : "native_worker_failed";
|
|
1371
|
-
function runNativeScanWorker() {
|
|
1372
|
-
if (!parentPort) throw new Error("native_worker_requires_port");
|
|
1096
|
+
// packages/core/src/analytics/scan-pool.ts
|
|
1097
|
+
var errorCode = (error) => error instanceof Error && /^[a-z][a-z0-9_]{0,63}$/.test(error.message) ? error.message : "source_unreadable";
|
|
1098
|
+
function runScanWorker() {
|
|
1099
|
+
if (!parentPort) throw new Error("scan_worker_requires_parent");
|
|
1373
1100
|
const port = parentPort;
|
|
1374
|
-
|
|
1375
|
-
port.on("message", async (message) => {
|
|
1376
|
-
if (message.protocol !== PROTOCOL || message.kind !== "scan" || !message.job || busy) return;
|
|
1377
|
-
busy = true;
|
|
1378
|
-
const job = message.job;
|
|
1101
|
+
port.on("message", (job) => {
|
|
1379
1102
|
try {
|
|
1380
|
-
|
|
1381
|
-
const snapshot = await streamNativeFacts(job.file, job.harness, { signal: new AbortController().signal, maxBytes: job.maxBytes, maxRecords: job.maxRecords, maxPromptBytes: job.maxPromptBytes, captureBytes: 0, captureFileBytes: job.bytes, expectedIdentity: { dev: job.stamp.dev, ino: job.stamp.ino }, sourceAllowed: (id, project) => !forgotten.has(sourceId(job.harness, id)) && !(project && isIgnoredProject(project, policy.ignoredProjects)), accept });
|
|
1382
|
-
const buffer = serialize(snapshot);
|
|
1383
|
-
if (buffer.length > (message.maxResultBytes ?? 256 * 1024 * 1024)) throw new Error("native_pool_result_bytes_limit");
|
|
1384
|
-
const hash2 = createHash6("sha256").update(buffer).digest("hex");
|
|
1385
|
-
let sequence = 0;
|
|
1386
|
-
for (let at2 = 0; at2 < buffer.length; at2 += FRAME) {
|
|
1387
|
-
const chunk = Uint8Array.from(buffer.subarray(at2, at2 + FRAME));
|
|
1388
|
-
port.postMessage({ protocol: PROTOCOL, kind: "frame", jobId: job.jobId, sequence: sequence++, data: chunk }, [chunk.buffer]);
|
|
1389
|
-
}
|
|
1390
|
-
port.postMessage({ protocol: PROTOCOL, kind: "result", jobId: job.jobId, authorityCommitment: policy.authorityCommitment, bytes: buffer.length, hash: hash2 });
|
|
1103
|
+
port.postMessage({ facts: extractFile(job.file, job.harness) });
|
|
1391
1104
|
} catch (error) {
|
|
1392
|
-
port.postMessage({
|
|
1393
|
-
} finally {
|
|
1394
|
-
busy = false;
|
|
1105
|
+
port.postMessage({ error: errorCode(error) });
|
|
1395
1106
|
}
|
|
1396
1107
|
});
|
|
1397
|
-
port.postMessage({ protocol: PROTOCOL, kind: "ready" });
|
|
1398
1108
|
}
|
|
1399
1109
|
|
|
1400
1110
|
// packages/cli/src/audit-native-worker.ts
|
|
1401
|
-
|
|
1111
|
+
runScanWorker();
|
|
1402
1112
|
//# sourceMappingURL=audit-native-worker.js.map
|