slopie 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/audit-splash.js +147 -0
- package/bin/potsherd.js +2 -0
- package/bin/slopie.js +2 -0
- package/dist/audit-native-worker.js +1265 -743
- package/dist/audit-native-worker.js.map +4 -4
- package/dist/potsherd.js +23658 -16750
- package/dist/potsherd.js.map +4 -4
- package/package.json +2 -6
|
@@ -1,25 +1,13 @@
|
|
|
1
1
|
import { createRequire as __potsherdCreateRequire } from 'node:module';
|
|
2
2
|
const require = __potsherdCreateRequire(import.meta.url);
|
|
3
3
|
|
|
4
|
-
// packages/core/src/analytics/
|
|
4
|
+
// packages/core/src/analytics/scan-pool.ts
|
|
5
5
|
import { Worker, parentPort } from "node:worker_threads";
|
|
6
|
-
import { serialize, deserialize } from "node:v8";
|
|
7
|
-
import { createHash as createHash6 } from "node:crypto";
|
|
8
6
|
|
|
9
|
-
// packages/core/src/analytics/
|
|
7
|
+
// packages/core/src/analytics/extract.ts
|
|
10
8
|
import fs from "node:fs";
|
|
9
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
11
10
|
import path from "node:path";
|
|
12
|
-
import { createHash as createHash5 } from "node:crypto";
|
|
13
|
-
|
|
14
|
-
// packages/core/src/parser/content.ts
|
|
15
|
-
function isRecord(value) {
|
|
16
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
17
|
-
}
|
|
18
|
-
function extractTypedText(content, blockType = "text") {
|
|
19
|
-
if (typeof content === "string") return content;
|
|
20
|
-
if (!Array.isArray(content)) return "";
|
|
21
|
-
return content.filter((b) => isRecord(b) && b.type === blockType && typeof b.text === "string").map((b) => b.text).join("\n");
|
|
22
|
-
}
|
|
23
11
|
|
|
24
12
|
// packages/core/src/markers.ts
|
|
25
13
|
var SUMMARIZER_CONTEXT_MARKER = "Context: This summary will be shown in a list to help users and Claude choose which conversations are relevant";
|
|
@@ -34,17 +22,8 @@ function hasExclusionMarker(text) {
|
|
|
34
22
|
return EXCLUSION_MARKERS.some((marker) => text.includes(marker));
|
|
35
23
|
}
|
|
36
24
|
|
|
37
|
-
// packages/core/src/memory/source-identity.ts
|
|
38
|
-
import { createHash } from "node:crypto";
|
|
39
|
-
function sourceId(harness, nativeSessionId) {
|
|
40
|
-
return createHash("sha256").update(`source/v1\0${harness}\0${nativeSessionId}`).digest("hex");
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// packages/core/src/analytics/source.ts
|
|
44
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
45
|
-
|
|
46
25
|
// packages/core/src/redact.ts
|
|
47
|
-
import { createHash
|
|
26
|
+
import { createHash } from "node:crypto";
|
|
48
27
|
|
|
49
28
|
// packages/core/src/theme.ts
|
|
50
29
|
var ANSI_RE = new RegExp("\\u001b\\[[0-9;]*m", "g");
|
|
@@ -562,7 +541,7 @@ var OPEN = "\u2039";
|
|
|
562
541
|
var CLOSE = "\u203A";
|
|
563
542
|
var MASK_RE = new RegExp(`${OPEN}redacted:[a-z-]+:[0-9a-f]{8}${CLOSE}`, "g");
|
|
564
543
|
function secretDigest(secret) {
|
|
565
|
-
return
|
|
544
|
+
return createHash("sha256").update(secret, "utf8").digest("hex").slice(0, 8);
|
|
566
545
|
}
|
|
567
546
|
function redact(text) {
|
|
568
547
|
if (typeof text !== "string" || text.length === 0) return { text: text ?? "", hits: [] };
|
|
@@ -618,785 +597,1328 @@ function claim(claimed, s) {
|
|
|
618
597
|
}
|
|
619
598
|
|
|
620
599
|
// packages/core/src/analytics/source.ts
|
|
621
|
-
var digest = (value) => createHash3("sha256").update(value).digest("hex");
|
|
622
600
|
var clean = (text) => redact(elideBinary(text)).text;
|
|
623
|
-
var clock = (value) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null;
|
|
624
601
|
|
|
625
|
-
// packages/core/src/analytics/
|
|
626
|
-
|
|
627
|
-
var
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
602
|
+
// packages/core/src/analytics/profanity.ts
|
|
603
|
+
var ENGLISH_EXPLICIT_LEXICON = Object.freeze(["fuck", "fucked", "fucking", "shit", "shitty", "bullshit", "asshole", "bastard"]);
|
|
604
|
+
var PROFANITY_LEXICON_VERSION = "en-explicit-8-v1";
|
|
605
|
+
var words = new Set(ENGLISH_EXPLICIT_LEXICON);
|
|
606
|
+
var LEXICON_HINT = /fuck|shit|asshole|bastard/i;
|
|
607
|
+
var CODE = 2;
|
|
608
|
+
var QUOTE = 1;
|
|
609
|
+
var UNKNOWN = 3;
|
|
610
|
+
var kind = (label) => label === CODE ? "code" : label === QUOTE ? "quoted" : label === UNKNOWN ? "unknown" : "direct_prose";
|
|
611
|
+
var escaped = (text, index) => {
|
|
612
|
+
let count2 = 0;
|
|
613
|
+
for (let i = index - 1; i >= 0 && text[i] === "\\"; i--) count2++;
|
|
614
|
+
return count2 % 2 === 1;
|
|
615
|
+
};
|
|
616
|
+
var wordChar = (char) => char !== void 0 && /[\p{L}\p{N}\p{M}_\u200c\u200d]/u.test(char);
|
|
617
|
+
var labelMemo = /* @__PURE__ */ new Map();
|
|
618
|
+
function labels(text) {
|
|
619
|
+
let hit = labelMemo.get(text);
|
|
620
|
+
if (hit) return hit;
|
|
621
|
+
if (labelMemo.size > 2e4) labelMemo.clear();
|
|
622
|
+
hit = computeLabels(text);
|
|
623
|
+
labelMemo.set(text, hit);
|
|
624
|
+
return hit;
|
|
625
|
+
}
|
|
626
|
+
function computeLabels(text) {
|
|
627
|
+
const map = new Uint8Array(text.length);
|
|
628
|
+
let ambiguous = false, offset = 0, fence = null;
|
|
629
|
+
for (const line of text.split(/(?<=\n)/u)) {
|
|
630
|
+
const marker = line.match(/^ {0,3}(`{3,}|~{3,})/u);
|
|
631
|
+
if (fence) {
|
|
632
|
+
if (marker && marker[1][0] === fence.char && marker[1].length >= fence.length && line.slice(marker[0].length).trim() === "") {
|
|
633
|
+
map.fill(CODE, fence.start, offset + line.length);
|
|
634
|
+
fence = null;
|
|
635
|
+
}
|
|
636
|
+
} else if (marker) fence = { char: marker[1][0], length: marker[1].length, start: offset };
|
|
637
|
+
else if (/^ {0,3}>/u.test(line)) map.fill(QUOTE, offset, offset + line.length);
|
|
638
|
+
else if (/^(?: {4}|\t)\S/u.test(line)) {
|
|
639
|
+
map.fill(UNKNOWN, offset, offset + line.length);
|
|
640
|
+
ambiguous = true;
|
|
661
641
|
}
|
|
642
|
+
offset += line.length;
|
|
662
643
|
}
|
|
663
|
-
|
|
664
|
-
|
|
644
|
+
if (fence) {
|
|
645
|
+
map.fill(UNKNOWN, fence.start);
|
|
646
|
+
ambiguous = true;
|
|
665
647
|
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
648
|
+
const sourceRows = text.split(/(?<=\n)/u), timestamp = /^\s*(?:(?:[-*#]+|\*\*)\s*)?\[?\d{1,2}:\d{2}(?::\d{2})?(?:\.\d+)?\]?/u;
|
|
649
|
+
if (/(?:transcript|captions|youtube|youtu\.be|video transcription)/iu.test(text) && sourceRows.filter((line) => timestamp.test(line)).length >= 3) {
|
|
650
|
+
let at = 0;
|
|
651
|
+
for (const line of sourceRows) {
|
|
652
|
+
if (timestamp.test(line)) map.fill(QUOTE, at, at + line.length);
|
|
653
|
+
at += line.length;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
for (let i = 0; i < text.length; i++) {
|
|
657
|
+
if (map[i] || escaped(text, i)) continue;
|
|
658
|
+
const char = text[i];
|
|
659
|
+
if (char === "`") {
|
|
660
|
+
let count2 = 1;
|
|
661
|
+
while (text[i + count2] === "`") count2++;
|
|
662
|
+
const token = "`".repeat(count2);
|
|
663
|
+
let end2 = text.indexOf(token, i + count2);
|
|
664
|
+
while (end2 >= 0 && (map[end2] !== 0 || text[end2 - 1] === "`" || text[end2 + count2] === "`")) end2 = text.indexOf(token, end2 + count2);
|
|
665
|
+
if (end2 < 0) {
|
|
666
|
+
map.fill(UNKNOWN, i);
|
|
667
|
+
ambiguous = true;
|
|
668
|
+
break;
|
|
672
669
|
}
|
|
673
|
-
|
|
674
|
-
|
|
670
|
+
map.fill(CODE, i, end2 + count2);
|
|
671
|
+
i = end2 + count2 - 1;
|
|
672
|
+
continue;
|
|
675
673
|
}
|
|
676
|
-
if (
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
674
|
+
if (!['"', "'", "\u201C", "\u2018"].includes(char)) continue;
|
|
675
|
+
if ((char === "'" || char === "\u2018") && wordChar(text[i - 1]) && wordChar(text[i + 1])) continue;
|
|
676
|
+
const closing = char === "\u201C" ? "\u201D" : char === "\u2018" ? "\u2019" : char;
|
|
677
|
+
let end = i + 1;
|
|
678
|
+
while (end < text.length) {
|
|
679
|
+
if (text[end] === closing && !escaped(text, end) && map[end] === 0 && !((closing === "'" || closing === "\u2019") && wordChar(text[end - 1]) && wordChar(text[end + 1]))) break;
|
|
680
|
+
end++;
|
|
682
681
|
}
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
682
|
+
if (end >= text.length) {
|
|
683
|
+
map.fill(UNKNOWN, i);
|
|
684
|
+
ambiguous = true;
|
|
685
|
+
break;
|
|
686
|
+
}
|
|
687
|
+
map.fill(QUOTE, i, end + 1);
|
|
688
|
+
i = end;
|
|
686
689
|
}
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
+
return { labels: map, ambiguous };
|
|
691
|
+
}
|
|
692
|
+
function proseLabels(text) {
|
|
693
|
+
return labels(text).labels;
|
|
694
|
+
}
|
|
695
|
+
var TOKEN = /[\p{L}\p{N}\p{M}_\u200c\u200d]+(?:['’][\p{L}\p{N}\p{M}_\u200c\u200d]+)*/gu;
|
|
696
|
+
function profanityHits(text) {
|
|
697
|
+
if (!LEXICON_HINT.test(text)) return null;
|
|
698
|
+
const spans = labels(text), hits = [];
|
|
699
|
+
TOKEN.lastIndex = 0;
|
|
700
|
+
let token;
|
|
701
|
+
while ((token = TOKEN.exec(text)) !== null) {
|
|
702
|
+
if (!words.has(token[0].toLowerCase())) continue;
|
|
703
|
+
hits.push([token[0], spans.labels[token.index] ?? 0, token.index, token.index + token[0].length]);
|
|
690
704
|
}
|
|
691
|
-
|
|
692
|
-
|
|
705
|
+
return { hits, ambiguous: spans.ambiguous };
|
|
706
|
+
}
|
|
707
|
+
var knownExcluded = /* @__PURE__ */ new Set(["child_initialization", "maintenance_exclusion_marker", "tool_result", "inherited_native_event", "declared_meta_or_synthetic_input", "declared_synthetic_input"]);
|
|
708
|
+
var languageEligibleInput = (p) => p.languageEligible !== false && (p.eligibleNativeInput === true && !knownExcluded.has(p.excludedReason ?? "") || p.eligibleNativeInput === void 0 && p.eligibleHuman && p.excludedReason === null && ["claude_prompt_id", "codex_human_marker"].includes(p.originBasis));
|
|
709
|
+
function hitsOf(p) {
|
|
710
|
+
if (p.lex) return p.lex.pt ? { hits: p.lex.pt, ambiguous: !!p.lex.pa } : null;
|
|
711
|
+
return profanityHits(p.text);
|
|
712
|
+
}
|
|
713
|
+
function auditProfanity(prompts, partial = false) {
|
|
714
|
+
const eligible = prompts.filter(languageEligibleInput), bucketCounts = new Map(["direct_prose", "quoted", "code", "unknown"].map((k) => [k, { occurrences: 0, prompts: /* @__PURE__ */ new Set() }])), containing = /* @__PURE__ */ new Set(), matches = [];
|
|
715
|
+
const terms = /* @__PURE__ */ new Map();
|
|
716
|
+
let occurrences = 0, ambiguous = false;
|
|
717
|
+
for (const prompt of eligible) {
|
|
718
|
+
const found = hitsOf(prompt);
|
|
719
|
+
if (!found) continue;
|
|
720
|
+
ambiguous ||= found.ambiguous;
|
|
721
|
+
for (const [raw, label, start, end] of found.hits) {
|
|
722
|
+
const bucket = kind(label), counter = bucketCounts.get(bucket);
|
|
723
|
+
occurrences++;
|
|
724
|
+
containing.add(prompt.id);
|
|
725
|
+
counter.occurrences++;
|
|
726
|
+
counter.prompts.add(prompt.id);
|
|
727
|
+
const normalized = raw.toLowerCase(), key = normalized + "\0" + bucket;
|
|
728
|
+
const term = terms.get(key) ?? { term: normalized, kind: bucket, occurrences: 0, ids: /* @__PURE__ */ new Set(), samples: [] };
|
|
729
|
+
term.occurrences++;
|
|
730
|
+
term.ids.add(prompt.id);
|
|
731
|
+
if (term.samples.length < 3) term.samples.push({ promptId: prompt.id, conversationId: prompt.conversationId, startUtf16: start, endUtf16: end, route: prompt.route });
|
|
732
|
+
terms.set(key, term);
|
|
733
|
+
if (matches.length < 8) matches.push({ term: raw, kind: bucket, promptId: prompt.id, conversationId: prompt.conversationId, startUtf16: start, endUtf16: end, route: prompt.route });
|
|
734
|
+
}
|
|
693
735
|
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
736
|
+
const measurementBasis = "literal_english_lexicon_observed_native_input_utf16_v2", definition = "Literal occurrences of the documented eight-word English lexicon in eligible observed native user-role inputs after existing redaction/elision; this does not attest universal human authorship. Unicode whole-word tokens; containing inputs counted once. No emotion, intent or universal-language inference. Ranges are relative to the immutable redacted containing input; routes identify that input.";
|
|
737
|
+
return { lexiconVersion: PROFANITY_LEXICON_VERSION, language: "en-explicit-lexicon", measurementBasis, eligiblePrompts: eligible.length, occurrences: { value: occurrences, numerator: occurrences, denominator: eligible.length, unit: "english_lexicon_occurrence", measurementBasis, state: partial ? "partial" : "observed", definition }, containingPrompts: { value: containing.size, numerator: containing.size, denominator: eligible.length, unit: "eligible_input_containing_english_lexicon_match", measurementBasis, state: partial ? "partial" : "observed", definition }, buckets: [...bucketCounts].map(([kind2, value]) => ({ kind: kind2, occurrences: value.occurrences, containingPrompts: value.prompts.size })), coverageGaps: ["outside_english_lexicon_unassessed", "hindi_hinglish_and_other_languages_unassessed", ...ambiguous ? ["ambiguous_quote_or_code_span"] : []], terms: [...terms.values()].map((t) => ({ term: t.term, kind: t.kind, occurrences: t.occurrences, containingPrompts: t.ids.size, samples: t.samples })).sort((a, b) => b.occurrences - a.occurrences || a.term.localeCompare(b.term) || a.kind.localeCompare(b.kind)), matches };
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// packages/core/src/analytics/language-feedback.ts
|
|
741
|
+
var POSITIVE_HINT = /good (?:work|job)|well done|you understood|you (?:got|nailed)|you(?:['’]re| are)/i;
|
|
742
|
+
var POSITIVE = /\b(?:good (?:work|job)|well done|you understood (?:me|it)|you (?:got|nailed) (?:it|this)|you(?:['’]re| are) (?:fucking )?(?:great|awesome))\b/iu;
|
|
743
|
+
var NEGATED = /\b(?:not|never|don['’]?t|didn['’]?t|no)\b.{0,35}\b(?:good|well done|understood|nailed|great|awesome)\b/iu;
|
|
744
|
+
var NEGATIVE = /\b(?:wrong|broken|stupid|missed|failed|why|mess|didn['’]?t|not|bullshit|fuck(?:ed|ing)? (?:up|off))\b/iu;
|
|
745
|
+
var YOU = /\b(?:you|your|you['’]re)\b/iu;
|
|
746
|
+
var words2 = (s) => s.match(/[\p{L}\p{N}]+/gu)?.length ?? 0;
|
|
747
|
+
function namedModels(text, models) {
|
|
748
|
+
const out = [];
|
|
749
|
+
let labels2 = null;
|
|
750
|
+
for (const model of models) {
|
|
751
|
+
const at = text.indexOf(model);
|
|
752
|
+
if (at < 0) continue;
|
|
753
|
+
labels2 ??= proseLabels(text);
|
|
754
|
+
if (labels2[at] !== 0 || /[\p{L}\p{N}_/-]/u.test(text[at - 1] ?? "") || /[\p{L}\p{N}_/-]/u.test(text[at + model.length] ?? "")) continue;
|
|
755
|
+
out.push(model);
|
|
697
756
|
}
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
757
|
+
return out;
|
|
758
|
+
}
|
|
759
|
+
function languageLines(text) {
|
|
760
|
+
const out = [];
|
|
761
|
+
let labelCache = null;
|
|
762
|
+
const L = (i) => (labelCache ??= proseLabels(text))[i];
|
|
763
|
+
let offset = 0;
|
|
764
|
+
for (const line of text.split("\n")) {
|
|
765
|
+
const start = offset;
|
|
766
|
+
offset += line.length + 1;
|
|
767
|
+
const trimmed = line.trim();
|
|
768
|
+
if (!trimmed) continue;
|
|
769
|
+
const at = start + line.indexOf(trimmed), end = at + trimmed.length;
|
|
770
|
+
if (!LEXICON_HINT.test(trimmed) && !POSITIVE_HINT.test(trimmed)) continue;
|
|
771
|
+
const probe = { id: "line", conversationId: "", role: "user", originBasis: "claude_prompt_id", identityBasis: "", eligibleHuman: true, excludedReason: null, eventAt: null, route: { basis: "canonical", refs: [], scope: {} } };
|
|
772
|
+
const local = auditProfanity([{ ...probe, text: line }]);
|
|
773
|
+
const direct = local.terms?.some((t) => t.kind === "direct_prose" && t.samples.some((sample) => L(start + sample.startUtf16) === 0)) ?? false;
|
|
774
|
+
const nonprose = local.terms?.some((t) => t.kind !== "direct_prose") ?? false;
|
|
775
|
+
const positiveMatch = trimmed.match(POSITIVE), positive = !!positiveMatch && L(at + (positiveMatch.index ?? 0)) === 0, negated = NEGATED.test(trimmed);
|
|
776
|
+
const quoted = L(at) !== 0 || /^\s*(?:>|[`~]{3}|["“'])/u.test(line) || /^ {4}|^\t/u.test(line);
|
|
777
|
+
const row = {};
|
|
778
|
+
if (direct) row.d = 1;
|
|
779
|
+
if (nonprose) row.np = 1;
|
|
780
|
+
if (YOU.test(trimmed)) row.y = 1;
|
|
781
|
+
if (positive) row.p = 1;
|
|
782
|
+
if (negated) row.ng = 1;
|
|
783
|
+
if (quoted) row.q = 1;
|
|
784
|
+
if (NEGATIVE.test(trimmed)) row.n = 1;
|
|
785
|
+
if (direct && !quoted && words2(trimmed) >= 2) {
|
|
786
|
+
const spans = trimmed.length <= 160 ? [{ text: trimmed, start: at, end }] : [...trimmed.matchAll(/[^.!?;\n]+[.!?;]?/gu)].map((match) => {
|
|
787
|
+
const value = match[0].trim(), begin = at + match.index + match[0].indexOf(value);
|
|
788
|
+
return { text: value, start: begin, end: begin + value.length };
|
|
789
|
+
}).filter((span) => span.text.length <= 160 && words2(span.text) >= 2 && auditProfanity([{ ...probe, text: span.text }]).terms?.some((t) => t.kind === "direct_prose" && L(span.start + t.samples[0].startUtf16) === 0));
|
|
790
|
+
if (spans.length) row.sp = spans.map((s) => [s.text, s.start, s.end]);
|
|
791
|
+
else row.ns = 1;
|
|
720
792
|
}
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
793
|
+
out.push(row);
|
|
794
|
+
}
|
|
795
|
+
return out;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// packages/core/src/analytics/project-focus.ts
|
|
799
|
+
var categories = [
|
|
800
|
+
["Documentation", /\b(?:write|update|fix|add|edit|draft|revise)\b.{0,64}\b(?:docs?|documentation|readme|guide|changelog)\b/iu],
|
|
801
|
+
["Review", /\b(?:review|audit|inspect)\b.{0,64}\b(?:code|diff|pr|pull request|implementation|tests?|changes|repo|repository)\b/iu],
|
|
802
|
+
["Debugging", /\b(?:fix|debug|repair|resolve|reproduce)\b.{0,64}\b(?:bug|error|crash|failure|issue|test|broken|timeout|memory|hang|problem)\b/iu],
|
|
803
|
+
["Development", /\b(?:build|implement|add|create|develop|refactor|ship|integrate)\b.{0,64}\b(?:feature|app|api|cli|function|component|command|module|tool|parser|service|code|support)\b/iu],
|
|
804
|
+
["Research", /\b(?:research|look up|investigate|compare|verify)\b.{0,64}\b(?:docs?|source|pricing|model|library|approach|behavior|format|implementation)\b/iu]
|
|
805
|
+
];
|
|
806
|
+
var VERBS = /\b(?:write|update|fix|add|edit|draft|revise|review|audit|inspect|debug|repair|resolve|reproduce|build|implement|create|develop|refactor|ship|integrate|research|look up|investigate|compare|verify)\b/iu;
|
|
807
|
+
function focusLabels(raw) {
|
|
808
|
+
if (!VERBS.test(raw) || !categories.some(([, pattern]) => pattern.test(raw))) return [];
|
|
809
|
+
const labels2 = proseLabels(raw), text = Array.from({ length: raw.length }, (_, i) => labels2[i] === 0 ? raw[i] : " ").join("");
|
|
810
|
+
return categories.filter(([, pattern]) => pattern.test(text)).map(([label]) => label);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
// packages/core/src/analytics/findings.ts
|
|
814
|
+
var EXACT_REPEAT_LIMITS = Object.freeze({ rows: 20, supports: 8, maxRowBytes: 8192 });
|
|
815
|
+
var PLACEHOLDER = /\[(?:Pasted text|Image) #\d+[^\]]*\]/gi;
|
|
816
|
+
function normalizedLine(text) {
|
|
817
|
+
const n = text.replace(PLACEHOLDER, "").toLowerCase().replace(/\s+/g, " ").trim().replace(/^[ .!?,]+|[ .!?,]+$/g, "");
|
|
818
|
+
return n.length > 0 && n.length <= 60 ? n : null;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// packages/core/src/analytics/story-lexicon.ts
|
|
822
|
+
var FLAG = {
|
|
823
|
+
paste: 1,
|
|
824
|
+
image: 2,
|
|
825
|
+
code: 4,
|
|
826
|
+
url: 8,
|
|
827
|
+
pasteSuspect: 16,
|
|
828
|
+
slur: 32,
|
|
829
|
+
polite: 64,
|
|
830
|
+
thanks: 128,
|
|
831
|
+
sorry: 256,
|
|
832
|
+
multiPunct: 512,
|
|
833
|
+
question: 1024,
|
|
834
|
+
delegate: 2048,
|
|
835
|
+
lets: 4096,
|
|
836
|
+
please: 8192,
|
|
837
|
+
history: 16384
|
|
838
|
+
};
|
|
839
|
+
var PLACEHOLDER2 = /\[(?:Pasted text|Image) #\d+[^\]]*\]/gi;
|
|
840
|
+
var URL_RX = /https?:\/\/\S+/g;
|
|
841
|
+
var CODEISH = /```[\s\S]*?```/g;
|
|
842
|
+
var WORD = /[a-z][a-z']*/g;
|
|
843
|
+
var SWEAR_STEMS = [
|
|
844
|
+
"fuck",
|
|
845
|
+
"shit",
|
|
846
|
+
"bullshit",
|
|
847
|
+
"wtf",
|
|
848
|
+
"damn",
|
|
849
|
+
"goddamn",
|
|
850
|
+
"crap",
|
|
851
|
+
"bitch",
|
|
852
|
+
"asshole",
|
|
853
|
+
"dumbass",
|
|
854
|
+
"stfu",
|
|
855
|
+
"ffs",
|
|
856
|
+
"piss",
|
|
857
|
+
"bastard",
|
|
858
|
+
"motherf",
|
|
859
|
+
"bloody",
|
|
860
|
+
"bsdk",
|
|
861
|
+
"chutiya",
|
|
862
|
+
"chutiye",
|
|
863
|
+
"madarchod",
|
|
864
|
+
"bhenchod",
|
|
865
|
+
"behenchod",
|
|
866
|
+
"gandu"
|
|
867
|
+
];
|
|
868
|
+
var SWEAR_EXACT = /* @__PURE__ */ new Set(["ffs", "wtf", "stfu"]);
|
|
869
|
+
var SWEAR_PREFIX = SWEAR_STEMS.filter((s) => !SWEAR_EXACT.has(s));
|
|
870
|
+
var INSULT = ["stupid", "idiot", "dumb", "moron", "useless", "pathetic", "lazy", "incompetent", "garbage", "trash", "retard"];
|
|
871
|
+
var SLUR_STEMS = ["retard"];
|
|
872
|
+
var POLITE = /\b(please|pls|plz|thanks|thank you|thank u|thx|ty|appreciate|great job|good job|well done|awesome|amazing|love it|love this|nice work|brilliant|you're the best|perfect)\b/;
|
|
873
|
+
var THANKS = /\b(thanks|thank you|thank u|thx|ty|appreciate it|great job|good job|well done|nice work)\b/;
|
|
874
|
+
var SORRY = /\b(sorry|my bad|apologi[sz]e)\b/;
|
|
875
|
+
var FRUSTRATION = /\b(again|still not|still doesn't|still isn't|i told you|i said|i already|not working|doesn't work|didn't work|isn't working|why (?:the|did|are|is|would) |what the|seriously|come on|for the last time|how many times|are you (?:kidding|serious|dumb|stupid))\b/g;
|
|
876
|
+
var DELEGATE = /\b(sub-?agents?|agents? in parallel|parallel agents?|spawn|launch (?:\w+ )?agents?|team of agents|multiple agents|\d+ agents)\b/;
|
|
877
|
+
var FILLER = /\b(basically|kind of|sort of|i guess|i mean|you know|like,|um+|uh+|okay so|so yeah|whatever)\b/g;
|
|
878
|
+
var PERFECTION = /\b(best possible|perfect(?:ly)?|exactly|properly|every single|completely|100%|flawless|world[- ]class|insanely|extremely)\b/g;
|
|
879
|
+
var QWORDS = /* @__PURE__ */ new Set(["what", "why", "how", "is", "are", "can", "could", "should", "does", "do", "did", "where", "when", "which", "who", "will", "would"]);
|
|
880
|
+
var WE = /\b(we|we're|we've|let's|us|our)\b/g;
|
|
881
|
+
var I_ = /\b(i|i'm|i've|me|my)\b/g;
|
|
882
|
+
var HINGLISH = /\b(bhai|yaar|kya|hai|nahi|kar|karo|acha|accha|theek|matlab|abhi|bas|chal|haan)\b/g;
|
|
883
|
+
var CAPS = /\b[A-Z][A-Z']{2,}\b/g;
|
|
884
|
+
var CAPS_OK = /* @__PURE__ */ new Set(["API", "URL", "HTML", "CSS", "JSON", "CLI", "PR", "UI", "UX", "MCP", "LLM", "SDK", "README", "TODO", "IST", "PDF", "NPM", "AWS", "GPT", "MD", "SEO", "JS", "TS", "ID", "OK", "AI", "CEO", "USA", "SQL", "DB", "SSH", "HTTP", "HTTPS", "PNG", "SVG", "CTA", "FAQ"]);
|
|
885
|
+
function isSwearWord(w) {
|
|
886
|
+
if (SWEAR_EXACT.has(w)) return true;
|
|
887
|
+
for (const s of SWEAR_PREFIX) if (w.startsWith(s)) return true;
|
|
888
|
+
return false;
|
|
889
|
+
}
|
|
890
|
+
var isInsult = (w) => INSULT.some((s) => w.startsWith(s));
|
|
891
|
+
var isSlur = (w) => SLUR_STEMS.some((s) => w.startsWith(s));
|
|
892
|
+
function mask(w) {
|
|
893
|
+
return w.length <= 2 ? w : w[0] + "*".repeat(w.length - 2) + w[w.length - 1];
|
|
894
|
+
}
|
|
895
|
+
function cleanForTone(text) {
|
|
896
|
+
return text.replace(PLACEHOLDER2, " ").replace(/[‘’]/g, "'").replace(CODEISH, " ").replace(URL_RX, " ");
|
|
897
|
+
}
|
|
898
|
+
var count = (rx, s) => {
|
|
899
|
+
rx.lastIndex = 0;
|
|
900
|
+
let n = 0;
|
|
901
|
+
while (rx.exec(s)) n++;
|
|
902
|
+
rx.lastIndex = 0;
|
|
903
|
+
return n;
|
|
904
|
+
};
|
|
905
|
+
var words3 = (s) => s.match(WORD) ?? [];
|
|
906
|
+
var TICS = [
|
|
907
|
+
"one more thing",
|
|
908
|
+
"each and every",
|
|
909
|
+
"do one thing",
|
|
910
|
+
"every single thing",
|
|
911
|
+
"kind of a thing",
|
|
912
|
+
"and all",
|
|
913
|
+
"and stuff",
|
|
914
|
+
"or something",
|
|
915
|
+
"at the end of the day",
|
|
916
|
+
"to be honest",
|
|
917
|
+
"basically",
|
|
918
|
+
"you know what",
|
|
919
|
+
"let me know",
|
|
920
|
+
"best possible",
|
|
921
|
+
"world class",
|
|
922
|
+
"insanely",
|
|
923
|
+
"make it pop",
|
|
924
|
+
"help me out",
|
|
925
|
+
"what do you think",
|
|
926
|
+
"i don't know",
|
|
927
|
+
"just do it",
|
|
928
|
+
"ship it",
|
|
929
|
+
"go ahead",
|
|
930
|
+
"lfg",
|
|
931
|
+
"let's go",
|
|
932
|
+
"no worries",
|
|
933
|
+
"makes sense",
|
|
934
|
+
"figure it out",
|
|
935
|
+
"deep research",
|
|
936
|
+
"ultra think",
|
|
937
|
+
"ultrathink",
|
|
938
|
+
"step by step",
|
|
939
|
+
"one shot",
|
|
940
|
+
"whatever it takes",
|
|
941
|
+
"end to end",
|
|
942
|
+
"from scratch",
|
|
943
|
+
"like a pro",
|
|
944
|
+
"production ready",
|
|
945
|
+
"properly",
|
|
946
|
+
"exactly",
|
|
947
|
+
"and everything",
|
|
948
|
+
"all of that",
|
|
949
|
+
"the whole thing",
|
|
950
|
+
"you know",
|
|
951
|
+
"i mean",
|
|
952
|
+
"sort of",
|
|
953
|
+
"kind of",
|
|
954
|
+
"so yeah",
|
|
955
|
+
"okay so"
|
|
956
|
+
];
|
|
957
|
+
var CLAUSE_FINAL = /* @__PURE__ */ new Set(["and all", "and stuff", "or something", "and everything", "all of that", "and all that"]);
|
|
958
|
+
var esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
959
|
+
var TIC_BY_FIRST = /* @__PURE__ */ new Map();
|
|
960
|
+
TICS.forEach((phrase, index) => {
|
|
961
|
+
const first = phrase.split(" ")[0];
|
|
962
|
+
const list = TIC_BY_FIRST.get(first) ?? [];
|
|
963
|
+
list.push({ phrase, index, final: CLAUSE_FINAL.has(phrase) });
|
|
964
|
+
TIC_BY_FIRST.set(first, list);
|
|
965
|
+
});
|
|
966
|
+
var CLAUSE_END = /^\s*(?:[.,!?;:)\n]|$|and\b|so\b|but\b)/;
|
|
967
|
+
var isWordChar = (c) => c !== void 0 && /[A-Za-z0-9_]/.test(c);
|
|
968
|
+
function ticsIn(low) {
|
|
969
|
+
const found = /* @__PURE__ */ new Set();
|
|
970
|
+
const rx = /[a-z']+/g;
|
|
971
|
+
let m;
|
|
972
|
+
while ((m = rx.exec(low)) !== null) {
|
|
973
|
+
const list = TIC_BY_FIRST.get(m[0]);
|
|
974
|
+
if (!list || isWordChar(low[m.index - 1])) continue;
|
|
975
|
+
for (const t of list) {
|
|
976
|
+
if (found.has(t.index) || !low.startsWith(t.phrase, m.index)) continue;
|
|
977
|
+
const end = m.index + t.phrase.length;
|
|
978
|
+
if (t.final ? CLAUSE_END.test(low.slice(end, end + 12)) : !isWordChar(low[end]) || !isWordChar(low[end - 1])) found.add(t.index);
|
|
724
979
|
}
|
|
725
|
-
f.key = null;
|
|
726
|
-
f.omitImage = false;
|
|
727
|
-
f.state = "comma";
|
|
728
980
|
}
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
981
|
+
return [...found].sort((a, b) => a - b);
|
|
982
|
+
}
|
|
983
|
+
var ADDRESS = ["dude", "bro", "buddy", "babe", "baby", "my friend", "man", "mate", "boss", "sir", "brother", "bhai", "yaar", "chief", "champ", "my guy", "darling", "love"];
|
|
984
|
+
var ADDRESS_GATE = new RegExp("\\b(" + ADDRESS.map(esc).join("|") + ")\\b", "g");
|
|
985
|
+
var ADDRESS_INDEX = new Map(ADDRESS.map((a, i) => [a, i]));
|
|
986
|
+
var VOCATIVE_BEFORE = /(?:^|[,.!?]\s*|\b(?:hey|yo|ok|okay|thanks|no|come on)\s+)$/;
|
|
987
|
+
function addressIn(low) {
|
|
988
|
+
const found = /* @__PURE__ */ new Set();
|
|
989
|
+
ADDRESS_GATE.lastIndex = 0;
|
|
990
|
+
let m;
|
|
991
|
+
while ((m = ADDRESS_GATE.exec(low)) !== null) {
|
|
992
|
+
const i = ADDRESS_INDEX.get(m[1]);
|
|
993
|
+
if (found.has(i)) continue;
|
|
994
|
+
const end = m.index + m[0].length;
|
|
995
|
+
if (VOCATIVE_BEFORE.test(low.slice(Math.max(0, m.index - 12), m.index)) || /^\s*[,.!?]/.test(low.slice(end, end + 8)) || end === low.length) found.add(i);
|
|
996
|
+
}
|
|
997
|
+
return [...found].sort((a, b) => a - b);
|
|
998
|
+
}
|
|
999
|
+
function featurize(input) {
|
|
1000
|
+
const raw = input.text;
|
|
1001
|
+
let t = cleanForTone(raw);
|
|
1002
|
+
const all = words3(t.toLowerCase());
|
|
1003
|
+
const pasteSuspect = all.length > 1500 || all.length > 600 && !input.typed;
|
|
1004
|
+
if (pasteSuspect) {
|
|
1005
|
+
const toks = t.split(/\s+/).filter(Boolean);
|
|
1006
|
+
t = toks.slice(0, 150).concat("\u2026", toks.slice(-150)).join(" ");
|
|
1007
|
+
}
|
|
1008
|
+
const low = t.toLowerCase();
|
|
1009
|
+
const tone = pasteSuspect ? words3(low) : all;
|
|
1010
|
+
let letters = 0, upper = 0;
|
|
1011
|
+
for (let k = 0; k < t.length; k++) {
|
|
1012
|
+
const c = t.charCodeAt(k);
|
|
1013
|
+
if (c >= 65 && c <= 90) {
|
|
1014
|
+
letters++;
|
|
1015
|
+
upper++;
|
|
1016
|
+
} else if (c >= 97 && c <= 122) letters++;
|
|
1017
|
+
else if (c > 127 && new RegExp("\\p{L}", "u").test(t[k])) {
|
|
1018
|
+
letters++;
|
|
1019
|
+
if (t[k] !== t[k].toLowerCase()) upper++;
|
|
742
1020
|
}
|
|
743
|
-
this.mode = "idle";
|
|
744
|
-
this.token = "";
|
|
745
1021
|
}
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
1022
|
+
let caps = 0;
|
|
1023
|
+
for (const m of t.match(CAPS) ?? []) if (!CAPS_OK.has(m)) caps++;
|
|
1024
|
+
const swears = [];
|
|
1025
|
+
let insults = 0, slur = false;
|
|
1026
|
+
for (const w of tone) {
|
|
1027
|
+
if (isSwearWord(w)) swears.push(w);
|
|
1028
|
+
if (isInsult(w)) insults++;
|
|
1029
|
+
if (!slur && isSlur(w)) slur = true;
|
|
1030
|
+
}
|
|
1031
|
+
const first = all[0] ?? "";
|
|
1032
|
+
const rawLow = raw.toLowerCase();
|
|
1033
|
+
const trimmed = raw.trim();
|
|
1034
|
+
let fl = 0;
|
|
1035
|
+
if (/\[Pasted text #/.test(raw) || input.hasPaste) fl |= FLAG.paste;
|
|
1036
|
+
if (input.hasImage || raw.includes("[Image #")) fl |= FLAG.image;
|
|
1037
|
+
if (raw.includes("```")) fl |= FLAG.code;
|
|
1038
|
+
URL_RX.lastIndex = 0;
|
|
1039
|
+
if (URL_RX.test(raw)) fl |= FLAG.url;
|
|
1040
|
+
URL_RX.lastIndex = 0;
|
|
1041
|
+
if (pasteSuspect) fl |= FLAG.pasteSuspect;
|
|
1042
|
+
if (slur) fl |= FLAG.slur;
|
|
1043
|
+
if (POLITE.test(low)) fl |= FLAG.polite;
|
|
1044
|
+
if (THANKS.test(low)) fl |= FLAG.thanks;
|
|
1045
|
+
if (SORRY.test(low)) fl |= FLAG.sorry;
|
|
1046
|
+
if (/[!?]{3,}/.test(raw)) fl |= FLAG.multiPunct;
|
|
1047
|
+
if (trimmed.endsWith("?") || QWORDS.has(first) && all.length < 40) fl |= FLAG.question;
|
|
1048
|
+
if (DELEGATE.test(low)) fl |= FLAG.delegate;
|
|
1049
|
+
if (/\blet'?s\b/.test(rawLow)) fl |= FLAG.lets;
|
|
1050
|
+
if (/\b(please|pls|plz)\b/.test(rawLow)) fl |= FLAG.please;
|
|
1051
|
+
if (input.typed) fl |= FLAG.history;
|
|
1052
|
+
let excl = 0;
|
|
1053
|
+
for (let k = raw.indexOf("!"); k >= 0; k = raw.indexOf("!", k + 1)) excl++;
|
|
1054
|
+
let lines = 1;
|
|
1055
|
+
for (let k = raw.indexOf("\n"); k >= 0; k = raw.indexOf("\n", k + 1)) lines++;
|
|
1056
|
+
const f = {
|
|
1057
|
+
ch: raw.length,
|
|
1058
|
+
w: all.length,
|
|
1059
|
+
ln: lines,
|
|
1060
|
+
fl,
|
|
1061
|
+
sw: swears.length,
|
|
1062
|
+
ins: insults,
|
|
1063
|
+
fr: count(FRUSTRATION, low),
|
|
1064
|
+
cw: caps,
|
|
1065
|
+
cr: letters >= 12 ? Math.round(upper / letters * 1e3) / 1e3 : 0,
|
|
1066
|
+
ex: excl,
|
|
1067
|
+
we: count(WE, low),
|
|
1068
|
+
i: count(I_, low),
|
|
1069
|
+
fi: count(FILLER, low),
|
|
1070
|
+
pf: count(PERFECTION, low),
|
|
1071
|
+
hi: count(HINGLISH, low),
|
|
1072
|
+
fw: first.slice(0, 24)
|
|
1073
|
+
};
|
|
1074
|
+
if (swears.length) f.sws = swears.slice(0, 6);
|
|
1075
|
+
if (raw.length <= 80) {
|
|
1076
|
+
const sl = raw.replace(PLACEHOLDER2, "").toLowerCase().replace(/\s+/g, " ").trim().slice(0, 60);
|
|
1077
|
+
if (sl) f.sl = sl;
|
|
1078
|
+
}
|
|
1079
|
+
const tc = ticsIn(rawLow);
|
|
1080
|
+
if (tc.length) f.tc = tc;
|
|
1081
|
+
const ad = addressIn(rawLow);
|
|
1082
|
+
if (ad.length) f.ad = ad;
|
|
1083
|
+
return f;
|
|
1084
|
+
}
|
|
1085
|
+
var SENSITIVE = /\b(die|dying|suicid\w*|kill (?:my|me)\w*|depress\w*|demons?|deserve|anxiety|therap\w*|panic|lonely|worthless|hate myself|cry(?:ing)?|breakup|girlfriend|boyfriend|wife|husband|sex\w*|porn|nude|drunk|password|token|secret|api[_ -]?key|ssn|salary|loan|debt|visa|lawyer|doctor|hospital|meds?)\b/i;
|
|
1086
|
+
var SECRETISH = /(sk-[A-Za-z0-9]|tvly-|AKIA|ghp_|xox[bp]-|@[\w.-]+\.\w{2,}|[A-Za-z0-9_-]{32,}|\/Users\/|\/home\/|~\/|https?:\/\/|\[REDACTED|<redacted)/;
|
|
1087
|
+
function safeQuote(text, maxLength = 90) {
|
|
1088
|
+
let t = text.replace(PLACEHOLDER2, "").replace(/\s+/g, " ").trim();
|
|
1089
|
+
if (!t || SENSITIVE.test(t) || SECRETISH.test(t)) return null;
|
|
1090
|
+
t = t.slice(0, maxLength + 40).replace(/[A-Za-z']+/g, (w) => {
|
|
1091
|
+
const l = w.toLowerCase();
|
|
1092
|
+
return isSwearWord(l) || isSlur(l) ? mask(w) : w;
|
|
1093
|
+
}) + t.slice(maxLength + 40);
|
|
1094
|
+
if (t.length <= maxLength) return t;
|
|
1095
|
+
const cut = t.slice(0, maxLength - 1);
|
|
1096
|
+
const space = cut.lastIndexOf(" ");
|
|
1097
|
+
return (space > 0 ? cut.slice(0, space) : cut) + "\u2026";
|
|
1098
|
+
}
|
|
1099
|
+
var TOPIC_STOP = new Set("the a an and or of to in on for is it that this i you we be are was with as at by from so if but not do can me my your our just what how all also now like then there they them these those one some any more much very really thing things something everything get got make made want need see know think going go let lets let's okay ok yes no up out into about will would should could have has had been being he she his her him its it's i'm don't can't isn't doesn't which who when where why here same other only even still well right good way time new use using done try give take put come look".split(" "));
|
|
1100
|
+
function vocabularyWords(text) {
|
|
1101
|
+
return words3(cleanForTone(text).toLowerCase()).filter((w) => w.length >= 3 && w.length <= 24);
|
|
1102
|
+
}
|
|
1103
|
+
function topicWords(text) {
|
|
1104
|
+
const out = /* @__PURE__ */ new Set();
|
|
1105
|
+
for (const w of words3(cleanForTone(text).toLowerCase())) if (w.length >= 3 && w.length <= 24 && !TOPIC_STOP.has(w) && !isSwearWord(w)) out.add(w);
|
|
1106
|
+
return out;
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
// packages/core/src/analytics/story-time.ts
|
|
1110
|
+
var WEEKDAY = { Mon: 0, Tue: 1, Wed: 2, Thu: 3, Fri: 4, Sat: 5, Sun: 6 };
|
|
1111
|
+
function localClock(timezone) {
|
|
1112
|
+
const format = new Intl.DateTimeFormat("en-US", {
|
|
1113
|
+
timeZone: timezone,
|
|
1114
|
+
year: "numeric",
|
|
1115
|
+
month: "2-digit",
|
|
1116
|
+
day: "2-digit",
|
|
1117
|
+
hour: "2-digit",
|
|
1118
|
+
minute: "2-digit",
|
|
1119
|
+
weekday: "short",
|
|
1120
|
+
hourCycle: "h23"
|
|
1121
|
+
});
|
|
1122
|
+
const memo = /* @__PURE__ */ new Map();
|
|
1123
|
+
return (at) => {
|
|
1124
|
+
const slot = Math.floor(at / 9e5);
|
|
1125
|
+
let hit = memo.get(slot);
|
|
1126
|
+
if (hit) return { ...hit, minute: hit.minute + Math.floor((at - slot * 9e5) / 6e4) };
|
|
1127
|
+
const parts = {};
|
|
1128
|
+
for (const p of format.formatToParts(new Date(slot * 9e5))) parts[p.type] = p.value;
|
|
1129
|
+
const hour = Number(parts.hour) % 24;
|
|
1130
|
+
hit = { day: `${parts.year}-${parts.month}-${parts.day}`, month: `${parts.year}-${parts.month}`, hour, dow: WEEKDAY[parts.weekday] ?? 0, minute: Number(parts.minute) };
|
|
1131
|
+
memo.set(slot, hit);
|
|
1132
|
+
return { ...hit, minute: hit.minute + Math.floor((at - slot * 9e5) / 6e4) };
|
|
1133
|
+
};
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
// packages/core/src/analytics/story-extract.ts
|
|
1137
|
+
var AGENT_TOOLS = /* @__PURE__ */ new Set(["Task", "Agent", "spawn_agent", "spawn_agents", "TaskCreate"]);
|
|
1138
|
+
var isAgentTool = (name) => AGENT_TOOLS.has(name) || name.startsWith("spawn");
|
|
1139
|
+
function promptLex(text, models) {
|
|
1140
|
+
const lex = {};
|
|
1141
|
+
const hits = profanityHits(text);
|
|
1142
|
+
if (hits?.hits.length) lex.pt = hits.hits;
|
|
1143
|
+
if (hits?.ambiguous) lex.pa = 1;
|
|
1144
|
+
const nl = normalizedLine(text);
|
|
1145
|
+
if (nl !== null) lex.nl = nl;
|
|
1146
|
+
const fc = focusLabels(text);
|
|
1147
|
+
if (fc.length) lex.fc = fc;
|
|
1148
|
+
const nm = namedModels(text, models);
|
|
1149
|
+
if (nm.length) lex.nm = nm;
|
|
1150
|
+
const ll = languageLines(text);
|
|
1151
|
+
if (ll.length) lex.ll = ll;
|
|
1152
|
+
return Object.keys(lex).length ? lex : void 0;
|
|
1153
|
+
}
|
|
1154
|
+
var QUOTE_TOP = 8;
|
|
1155
|
+
var QUOTE_SUBAGENT_TOP = 5;
|
|
1156
|
+
var QUOTE_LAST_PER_PROJECT = 3;
|
|
1157
|
+
function deriveStory(prompts, events, interrupts, options) {
|
|
1158
|
+
const human = prompts.filter((p) => p.kind === "human" && p.at !== null).sort((a, b) => a.at - b.at);
|
|
1159
|
+
const fresh = new Set(prompts.slice(options.from ?? 0));
|
|
1160
|
+
const models = /* @__PURE__ */ new Set();
|
|
1161
|
+
for (const p of prompts) {
|
|
1162
|
+
if (p.before) models.add(p.before);
|
|
1163
|
+
if (p.after) models.add(p.after);
|
|
1164
|
+
}
|
|
1165
|
+
const vocab = options.counters ? options.previous?.vocab ?? /* @__PURE__ */ new Map() : null;
|
|
1166
|
+
const topics = options.counters ? new Map((options.previous?.topics ?? []).map((t) => [t.month, t])) : null;
|
|
1167
|
+
const clock = localClock(options.timezone);
|
|
1168
|
+
for (const p of human) {
|
|
1169
|
+
if (!fresh.has(p)) continue;
|
|
1170
|
+
p.f = featurize({ text: p.text, typed: options.typed });
|
|
1171
|
+
const lex = promptLex(p.text, models);
|
|
1172
|
+
if (lex) p.lex = lex;
|
|
1173
|
+
if (vocab) for (const w of vocabularyWords(p.text)) vocab.set(w, (vocab.get(w) ?? 0) + 1);
|
|
1174
|
+
if (topics) {
|
|
1175
|
+
const { hour, month } = clock(p.at);
|
|
1176
|
+
const slot = hour < 5 ? 0 : hour >= 9 && hour < 19 ? 1 : -1;
|
|
1177
|
+
if (slot >= 0) {
|
|
1178
|
+
let bag = topics.get(month);
|
|
1179
|
+
if (!bag) {
|
|
1180
|
+
bag = { month, night: 0, day: 0, words: /* @__PURE__ */ new Map() };
|
|
1181
|
+
topics.set(month, bag);
|
|
766
1182
|
}
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1183
|
+
if (slot === 0) bag.night++;
|
|
1184
|
+
else bag.day++;
|
|
1185
|
+
for (const w of topicWords(p.text)) {
|
|
1186
|
+
let c = bag.words.get(w);
|
|
1187
|
+
if (!c) {
|
|
1188
|
+
c = [0, 0];
|
|
1189
|
+
bag.words.set(w, c);
|
|
1190
|
+
}
|
|
1191
|
+
c[slot]++;
|
|
771
1192
|
}
|
|
772
|
-
this.add(decoded[c]);
|
|
773
|
-
return;
|
|
774
|
-
}
|
|
775
|
-
if (c === "\\") {
|
|
776
|
-
this.escape = true;
|
|
777
|
-
return;
|
|
778
1193
|
}
|
|
779
|
-
if (c === '"') {
|
|
780
|
-
if (this.isKey) {
|
|
781
|
-
const f2 = this.frames.at(-1);
|
|
782
|
-
f2.key = this.token;
|
|
783
|
-
f2.state = "colon";
|
|
784
|
-
} else this.accept(this.keep ? this.token : null);
|
|
785
|
-
this.mode = "idle";
|
|
786
|
-
this.token = "";
|
|
787
|
-
this.imageCandidate = null;
|
|
788
|
-
this.markerTail = "";
|
|
789
|
-
return;
|
|
790
|
-
}
|
|
791
|
-
if (c.charCodeAt(0) < 32) {
|
|
792
|
-
this.fail();
|
|
793
|
-
return;
|
|
794
|
-
}
|
|
795
|
-
this.add(c);
|
|
796
|
-
return;
|
|
797
|
-
}
|
|
798
|
-
if (this.mode === "number" || this.mode === "literal") {
|
|
799
|
-
const allowed = this.mode === "number" ? /[0-9eE+.-]/ : /[a-z]/;
|
|
800
|
-
if (allowed.test(c)) {
|
|
801
|
-
if (this.token.length >= 256) {
|
|
802
|
-
this.fail("native_metadata_scalar_limit");
|
|
803
|
-
return;
|
|
804
|
-
}
|
|
805
|
-
this.token += c;
|
|
806
|
-
return;
|
|
807
|
-
}
|
|
808
|
-
this.scalarEnd();
|
|
809
|
-
if (this.error) return;
|
|
810
|
-
}
|
|
811
|
-
if (/\s/.test(c)) {
|
|
812
|
-
if (![" ", " ", "\r", "\n"].includes(c)) this.fail();
|
|
813
|
-
return;
|
|
814
|
-
}
|
|
815
|
-
const f = this.frames.at(-1);
|
|
816
|
-
if (c === '"') {
|
|
817
|
-
this.isKey = !!f && !f.array && (f.state === "key" || f.state === "keyRequired");
|
|
818
|
-
if (!this.isKey && !this.expectedValue()) {
|
|
819
|
-
this.fail();
|
|
820
|
-
return;
|
|
821
|
-
}
|
|
822
|
-
if (!this.isKey && f && this.imageBody(f)) {
|
|
823
|
-
f.omitImage = true;
|
|
824
|
-
this.imageCandidate = { frame: f, chunks: [], bytes: 0, overflow: false };
|
|
825
|
-
this.imageBodies.push(this.imageCandidate);
|
|
826
|
-
}
|
|
827
|
-
this.keep = this.isKey || this.retainValue() && f?.key !== "message";
|
|
828
|
-
this.token = "";
|
|
829
|
-
this.markerTail = "";
|
|
830
|
-
this.mode = "string";
|
|
831
|
-
return;
|
|
832
|
-
}
|
|
833
|
-
if (c === "{" || c === "[") {
|
|
834
|
-
if (!this.expectedValue() || this.frames.length >= 128) {
|
|
835
|
-
this.fail("native_metadata_depth_limit");
|
|
836
|
-
return;
|
|
837
|
-
}
|
|
838
|
-
const keep = this.retainValue();
|
|
839
|
-
this.frames.push({ path: f ? [...f.path, f.array ? "[]" : f.key] : [], omitImage: false, array: c === "[", state: c === "[" ? "value" : "key", key: null, keep, value: keep ? c === "[" ? [] : {} : null });
|
|
840
|
-
return;
|
|
841
|
-
}
|
|
842
|
-
if (c === "}" || c === "]") {
|
|
843
|
-
if (!f || f.array !== (c === "]") || !["comma", f.array ? "value" : "key"].includes(f.state)) {
|
|
844
|
-
this.fail();
|
|
845
|
-
return;
|
|
846
|
-
}
|
|
847
|
-
this.frames.pop();
|
|
848
|
-
this.accept(f.value);
|
|
849
|
-
return;
|
|
850
1194
|
}
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
1195
|
+
}
|
|
1196
|
+
turnStats(human, events, interrupts);
|
|
1197
|
+
const weight = /* @__PURE__ */ new Map();
|
|
1198
|
+
const slots = /* @__PURE__ */ new Set();
|
|
1199
|
+
if (human.length) slots.add(human[0]);
|
|
1200
|
+
const byProject = /* @__PURE__ */ new Map();
|
|
1201
|
+
for (const p of human) {
|
|
1202
|
+
const k = p.project ?? "";
|
|
1203
|
+
const list = byProject.get(k) ?? [];
|
|
1204
|
+
list.push(p);
|
|
1205
|
+
byProject.set(k, list);
|
|
1206
|
+
}
|
|
1207
|
+
for (const list of byProject.values()) for (const p of list.slice(-QUOTE_LAST_PER_PROJECT)) slots.add(p);
|
|
1208
|
+
turnWeights(human, events, weight);
|
|
1209
|
+
for (const p of [...human].sort((a, b) => (weight.get(b) ?? 0) - (weight.get(a) ?? 0)).slice(0, QUOTE_TOP)) slots.add(p);
|
|
1210
|
+
for (const p of human.filter((p2) => (p2.t?.sa ?? 0) > 0).sort((a, b) => b.t.sa - a.t.sa).slice(0, QUOTE_SUBAGENT_TOP)) slots.add(p);
|
|
1211
|
+
for (const p of slots) {
|
|
1212
|
+
if (!p.text) continue;
|
|
1213
|
+
const q = safeQuote(p.text, 90);
|
|
1214
|
+
if (q) p.q = q;
|
|
1215
|
+
}
|
|
1216
|
+
return { vocab, topics: topics ? [...topics.values()] : null };
|
|
1217
|
+
}
|
|
1218
|
+
function turnStats(human, events, interrupts) {
|
|
1219
|
+
const ev = events.filter((e) => Number.isFinite(e.at)).sort((a, b) => a.at - b.at);
|
|
1220
|
+
const intr = interrupts.filter(Number.isFinite).sort((a, b) => a - b);
|
|
1221
|
+
let j = 0, k = 0, lastAt = null;
|
|
1222
|
+
human.forEach((p, i) => {
|
|
1223
|
+
const start = p.at, next = i + 1 < human.length ? human[i + 1].at : Infinity;
|
|
1224
|
+
while (j < ev.length && ev[j].at < start) lastAt = ev[j++].at;
|
|
1225
|
+
const t = { m: 0, tl: 0, sa: 0, d: null, x: 0, g: lastAt === null ? null : round1((start - lastAt) / 1e3) };
|
|
1226
|
+
let end = null;
|
|
1227
|
+
while (j < ev.length && ev[j].at < next) {
|
|
1228
|
+
const e = ev[j++];
|
|
1229
|
+
t.m++;
|
|
1230
|
+
t.tl += e.tools;
|
|
1231
|
+
t.sa += e.subs;
|
|
1232
|
+
end = e.at;
|
|
1233
|
+
lastAt = e.at;
|
|
870
1234
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
if (
|
|
874
|
-
|
|
875
|
-
|
|
1235
|
+
if (end !== null) t.d = round1((end - start) / 1e3);
|
|
1236
|
+
while (k < intr.length && intr[k] < start) k++;
|
|
1237
|
+
if (k < intr.length && intr[k] < next) t.x = 1;
|
|
1238
|
+
p.t = t;
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
function turnWeights(human, events, out) {
|
|
1242
|
+
const ev = events.filter((e) => e.weight > 0).sort((a, b) => a.at - b.at);
|
|
1243
|
+
let j = 0;
|
|
1244
|
+
human.forEach((p, i) => {
|
|
1245
|
+
const next = i + 1 < human.length ? human[i + 1].at : Infinity;
|
|
1246
|
+
while (j < ev.length && ev[j].at < p.at) j++;
|
|
1247
|
+
let w = 0;
|
|
1248
|
+
while (j < ev.length && ev[j].at < next) w += ev[j++].weight;
|
|
1249
|
+
out.set(p, w);
|
|
1250
|
+
});
|
|
1251
|
+
}
|
|
1252
|
+
var round1 = (x) => Math.round(x * 10) / 10;
|
|
1253
|
+
var usageWeight = (input, output, cacheRead, cacheWrite) => input + 5 * output + 1.25 * cacheWrite + 0.1 * cacheRead;
|
|
1254
|
+
|
|
1255
|
+
// packages/core/src/analytics/extract.ts
|
|
1256
|
+
var FACTS_VERSION = 5;
|
|
1257
|
+
var MAX_PROMPT_CHARS = 4e3;
|
|
1258
|
+
var isRec = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
1259
|
+
var str = (v) => typeof v === "string" && v.length > 0 ? v : null;
|
|
1260
|
+
var int = (v) => typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.floor(v) : 0;
|
|
1261
|
+
function time(v) {
|
|
1262
|
+
if (typeof v === "string") {
|
|
1263
|
+
const t = Date.parse(v);
|
|
1264
|
+
return Number.isFinite(t) ? t : null;
|
|
1265
|
+
}
|
|
1266
|
+
if (typeof v === "number" && Number.isFinite(v)) return v < 1e11 ? v * 1e3 : v;
|
|
1267
|
+
return null;
|
|
1268
|
+
}
|
|
1269
|
+
function textOf(content) {
|
|
1270
|
+
if (typeof content === "string") return content;
|
|
1271
|
+
if (!Array.isArray(content)) return "";
|
|
1272
|
+
const parts = [];
|
|
1273
|
+
for (const block of content) if (isRec(block) && typeof block.text === "string") parts.push(block.text);
|
|
1274
|
+
return parts.join("\n");
|
|
1275
|
+
}
|
|
1276
|
+
var INJECTED_PREFIXES = [
|
|
1277
|
+
["<task-notification>", "task_notification"],
|
|
1278
|
+
["<local-command-stdout>", "local_command_output"],
|
|
1279
|
+
["<local-command-stderr>", "local_command_output"],
|
|
1280
|
+
["<local-command-caveat>", "local_command_output"],
|
|
1281
|
+
["[Request interrupted", "interrupt_marker"],
|
|
1282
|
+
["<turn_aborted>", "interrupt_marker"],
|
|
1283
|
+
["<environment_context>", "environment_context"],
|
|
1284
|
+
["# AGENTS.md instructions", "agents_md"],
|
|
1285
|
+
["<user_instructions>", "agents_md"],
|
|
1286
|
+
["<external_codex_apps", "app_state"],
|
|
1287
|
+
["<recommended_plugins>", "app_state"],
|
|
1288
|
+
["<heartbeat>", "automation"],
|
|
1289
|
+
["The following is the Codex agent history", "guardian_review"],
|
|
1290
|
+
["You have new hive inbox message", "automation"],
|
|
1291
|
+
["Another Claude session sent a message", "cross_session_message"],
|
|
1292
|
+
["This session is being continued from a previous conversation", "compaction_summary"],
|
|
1293
|
+
["Your claude.ai usage limit has reset", "auto_continuation"],
|
|
1294
|
+
["Base directory for this skill:", "skill_expansion"]
|
|
1295
|
+
];
|
|
1296
|
+
var SYSTEM_REMINDER = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
1297
|
+
function classifyText(raw) {
|
|
1298
|
+
const text = (raw.includes("<system-reminder>") ? raw.replace(SYSTEM_REMINDER, "") : raw).trim();
|
|
1299
|
+
if (!text) return { kind: "excluded", reason: "empty", text };
|
|
1300
|
+
for (const [prefix, reason] of INJECTED_PREFIXES) if (text.startsWith(prefix)) return { kind: "excluded", reason, text };
|
|
1301
|
+
if (text.startsWith("<command-name>") || text.startsWith("<command-message>")) return { kind: "command", reason: "slash_command", text };
|
|
1302
|
+
if (text.startsWith("<send_user_message_question_reply>")) return { kind: "answer", reason: "question_answer", text };
|
|
1303
|
+
return { kind: "human", reason: null, text };
|
|
1304
|
+
}
|
|
1305
|
+
var CODEX_STRIP = [/<in-app-browser-context[\s\S]*?<\/in-app-browser-context>/g, /<image name=[^>]*>\s*<\/image>/g, /<\/?image[^>]*>/g];
|
|
1306
|
+
function cleanCodexText(raw) {
|
|
1307
|
+
let t = raw;
|
|
1308
|
+
if (t.includes("<")) for (const rx of CODEX_STRIP) t = t.replace(rx, "");
|
|
1309
|
+
const at = t.indexOf("## My request");
|
|
1310
|
+
if (at >= 0) {
|
|
1311
|
+
const rest = t.slice(at + "## My request".length), colon = rest.indexOf(":");
|
|
1312
|
+
t = colon >= 0 ? rest.slice(colon + 1) : rest;
|
|
1313
|
+
} else if (t.trimStart().startsWith("# Files mentioned by the user")) t = "";
|
|
1314
|
+
return t.trim();
|
|
1315
|
+
}
|
|
1316
|
+
function makeBuilder(file, harness, size, mtimeMs) {
|
|
1317
|
+
return {
|
|
1318
|
+
facts: {
|
|
1319
|
+
v: FACTS_VERSION,
|
|
1320
|
+
harness,
|
|
1321
|
+
file,
|
|
1322
|
+
size,
|
|
1323
|
+
mtimeMs,
|
|
1324
|
+
sessionId: path.basename(file, ".jsonl"),
|
|
1325
|
+
parentId: null,
|
|
1326
|
+
child: false,
|
|
1327
|
+
project: null,
|
|
1328
|
+
title: null,
|
|
1329
|
+
automated: false,
|
|
1330
|
+
excluded: false,
|
|
1331
|
+
startedAt: null,
|
|
1332
|
+
firstAt: null,
|
|
1333
|
+
lastAt: null,
|
|
1334
|
+
usage: [],
|
|
1335
|
+
prompts: [],
|
|
1336
|
+
malformed: 0
|
|
1337
|
+
},
|
|
1338
|
+
currentModel: null,
|
|
1339
|
+
currentProvider: null,
|
|
1340
|
+
pending: [],
|
|
1341
|
+
seenPrompts: /* @__PURE__ */ new Set(),
|
|
1342
|
+
events: [],
|
|
1343
|
+
interrupts: []
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
function touch(b, at) {
|
|
1347
|
+
if (at === null) return;
|
|
1348
|
+
if (b.facts.firstAt === null || at < b.facts.firstAt) b.facts.firstAt = at;
|
|
1349
|
+
if (b.facts.lastAt === null || at > b.facts.lastAt) b.facts.lastAt = at;
|
|
1350
|
+
}
|
|
1351
|
+
function answered(b, model, provider) {
|
|
1352
|
+
if (!model) return;
|
|
1353
|
+
for (const p of b.pending) {
|
|
1354
|
+
p.after = model;
|
|
1355
|
+
p.afterProvider = provider;
|
|
876
1356
|
}
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
1357
|
+
b.pending.length = 0;
|
|
1358
|
+
b.currentModel = model;
|
|
1359
|
+
b.currentProvider = provider;
|
|
1360
|
+
}
|
|
1361
|
+
function addUsage(b, u) {
|
|
1362
|
+
b.facts.usage.push(u);
|
|
1363
|
+
touch(b, u.at);
|
|
1364
|
+
answered(b, u.model, u.provider);
|
|
1365
|
+
}
|
|
1366
|
+
function addPrompt(b, key, at, project, kind2, reason, text) {
|
|
1367
|
+
if (b.seenPrompts.has(key)) return;
|
|
1368
|
+
b.seenPrompts.add(key);
|
|
1369
|
+
if (kind2 === "human" && hasExclusionMarker(text)) b.facts.excluded = true;
|
|
1370
|
+
if (reason === "interrupt_marker" && at !== null) b.interrupts.push(at);
|
|
1371
|
+
const prompt = {
|
|
1372
|
+
key,
|
|
1373
|
+
at,
|
|
1374
|
+
project,
|
|
1375
|
+
kind: kind2,
|
|
1376
|
+
reason,
|
|
1377
|
+
text: kind2 === "human" ? clean(text.length > MAX_PROMPT_CHARS ? text.slice(0, MAX_PROMPT_CHARS) : text) : "",
|
|
1378
|
+
before: b.currentModel,
|
|
1379
|
+
beforeProvider: b.currentProvider,
|
|
1380
|
+
after: null,
|
|
1381
|
+
afterProvider: null
|
|
1382
|
+
};
|
|
1383
|
+
b.facts.prompts.push(prompt);
|
|
1384
|
+
if (kind2 === "human") b.pending.push(prompt);
|
|
1385
|
+
touch(b, at);
|
|
1386
|
+
}
|
|
1387
|
+
var SUBAGENT_DIR = `${path.sep}subagents${path.sep}`;
|
|
1388
|
+
function claudeLine(b, line, subagent, sessions) {
|
|
1389
|
+
const assistant = line.includes('"type":"assistant"');
|
|
1390
|
+
if (assistant ? !line.includes('"usage"') : subagent || !line.includes('"type":"user"')) {
|
|
1391
|
+
if (line.length > 4096 || !line.includes('-title"') && !line.includes('"type":"summary"')) return;
|
|
1392
|
+
}
|
|
1393
|
+
let r;
|
|
1394
|
+
try {
|
|
1395
|
+
r = JSON.parse(line.toString("utf8"));
|
|
1396
|
+
} catch {
|
|
1397
|
+
b.facts.malformed++;
|
|
1398
|
+
return;
|
|
1399
|
+
}
|
|
1400
|
+
if (!isRec(r)) return;
|
|
1401
|
+
const f = b.facts;
|
|
1402
|
+
if (r.type === "summary" || r.type === "custom-title" || r.type === "ai-title") {
|
|
1403
|
+
const title = str(r.customTitle) ?? str(r.aiTitle) ?? str(r.title) ?? str(r.summary);
|
|
1404
|
+
if (title && (r.type !== "summary" || !f.title)) f.title = title;
|
|
1405
|
+
return;
|
|
1406
|
+
}
|
|
1407
|
+
if (typeof r.cwd === "string" && !f.project) f.project = r.cwd;
|
|
1408
|
+
const sid = str(r.sessionId);
|
|
1409
|
+
if (sid) {
|
|
1410
|
+
if (subagent) {
|
|
1411
|
+
f.parentId = sid;
|
|
1412
|
+
f.child = true;
|
|
1413
|
+
f.sessionId = `${sid}:${path.basename(f.file, ".jsonl")}`;
|
|
1414
|
+
} else if (!sessions.size) f.sessionId = sid;
|
|
1415
|
+
}
|
|
1416
|
+
const m = isRec(r.message) ? r.message : {};
|
|
1417
|
+
const at = time(r.timestamp);
|
|
1418
|
+
if (r.type === "assistant") {
|
|
1419
|
+
const u = m.usage;
|
|
1420
|
+
if (!isRec(u)) return;
|
|
1421
|
+
const model = str(m.model);
|
|
1422
|
+
if (model === "<synthetic>") return;
|
|
1423
|
+
const cc = isRec(u.cache_creation) ? u.cache_creation : null;
|
|
1424
|
+
const write = cc ? int(cc.ephemeral_5m_input_tokens) + int(cc.ephemeral_1h_input_tokens) : int(u.cache_creation_input_tokens);
|
|
1425
|
+
const id = str(m.id), request = str(r.requestId), session = sid ?? f.file;
|
|
1426
|
+
const fact = {
|
|
1427
|
+
key: id ? request ? `c:${id}:${request}` : `c:${id}::${session}:${String(r.timestamp)}` : null,
|
|
1428
|
+
at,
|
|
1429
|
+
model,
|
|
1430
|
+
provider: null,
|
|
1431
|
+
input: int(u.input_tokens),
|
|
1432
|
+
output: int(u.output_tokens),
|
|
1433
|
+
cacheRead: int(u.cache_read_input_tokens),
|
|
1434
|
+
cacheWrite: write,
|
|
1435
|
+
cacheWrite1h: cc ? int(cc.ephemeral_1h_input_tokens) : 0,
|
|
1436
|
+
reasoning: 0
|
|
884
1437
|
};
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
if (this.scanUnicode.length === 5) {
|
|
897
|
-
if (/^u[0-9a-f]{4}$/i.test(this.scanUnicode)) add(String.fromCharCode(parseInt(this.scanUnicode.slice(1), 16)));
|
|
898
|
-
this.scanUnicode = "";
|
|
1438
|
+
if (r.isSidechain === true) fact.sidechain = true;
|
|
1439
|
+
if (!request) fact.requestless = true;
|
|
1440
|
+
if (id) fact.replayKey = `${id}:${session}`;
|
|
1441
|
+
if (u.speed === "fast") fact.tier = "fast";
|
|
1442
|
+
addUsage(b, fact);
|
|
1443
|
+
if (at !== null) {
|
|
1444
|
+
let tools = 0, subs = 0;
|
|
1445
|
+
if (Array.isArray(m.content)) {
|
|
1446
|
+
for (const block of m.content) if (isRec(block) && block.type === "tool_use") {
|
|
1447
|
+
tools++;
|
|
1448
|
+
if (typeof block.name === "string" && isAgentTool(block.name)) subs++;
|
|
899
1449
|
}
|
|
900
|
-
continue;
|
|
901
|
-
}
|
|
902
|
-
if (this.scanEscape) {
|
|
903
|
-
this.scanEscape = false;
|
|
904
|
-
if (c === "u") this.scanUnicode = "u";
|
|
905
|
-
else add({ n: "\n", r: "\r", t: " " }[c] ?? c);
|
|
906
|
-
continue;
|
|
907
1450
|
}
|
|
908
|
-
|
|
909
|
-
this.scanString = !this.scanString;
|
|
910
|
-
this.scanTail = "";
|
|
911
|
-
} else if (this.scanString && c === "\\") this.scanEscape = true;
|
|
1451
|
+
b.events.push({ at, tools, subs, weight: usageWeight(fact.input, fact.output, fact.cacheRead, fact.cacheWrite) });
|
|
912
1452
|
}
|
|
1453
|
+
return;
|
|
913
1454
|
}
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
}
|
|
1455
|
+
if (r.type !== "user" || r.isSidechain === true || subagent) return;
|
|
1456
|
+
const content = m.content;
|
|
1457
|
+
if (Array.isArray(content) && content.some((block) => isRec(block) && block.type === "tool_result")) return;
|
|
1458
|
+
if (sid) sessions.add(sid);
|
|
1459
|
+
const origin = isRec(r.origin) ? str(r.origin.kind) : null;
|
|
1460
|
+
const flagged = r.isMeta === true ? "is_meta" : r.isCompactSummary === true ? "compaction_summary" : origin && origin !== "human" ? `origin_${origin}` : r.entrypoint === "sdk-cli" ? "headless_claude_p" : r.promptSource === "system" ? "prompt_source_system" : null;
|
|
1461
|
+
let { kind: kind2, reason, text } = classifyText(textOf(content));
|
|
1462
|
+
if (flagged) {
|
|
1463
|
+
kind2 = "excluded";
|
|
1464
|
+
reason = flagged;
|
|
1465
|
+
} else if (reason === "empty" && origin === "human" && Array.isArray(content) && content.some((block) => isRec(block) && block.type === "image")) {
|
|
1466
|
+
kind2 = "human";
|
|
1467
|
+
reason = null;
|
|
1468
|
+
text = "[image]";
|
|
929
1469
|
}
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
1470
|
+
addPrompt(b, str(r.uuid) ?? `${f.sessionId}:${String(r.timestamp)}`, at, str(r.cwd) ?? f.project, kind2, reason, text);
|
|
1471
|
+
}
|
|
1472
|
+
var newCodexState = () => ({
|
|
1473
|
+
lineNo: 0,
|
|
1474
|
+
previousTotal: null,
|
|
1475
|
+
tier: void 0,
|
|
1476
|
+
imported: false,
|
|
1477
|
+
records: /* @__PURE__ */ new Map(),
|
|
1478
|
+
compacted: /* @__PURE__ */ new Set(),
|
|
1479
|
+
deltas: /* @__PURE__ */ new Set(),
|
|
1480
|
+
itemPrompts: [],
|
|
1481
|
+
responsePrompts: [],
|
|
1482
|
+
mode: null,
|
|
1483
|
+
converted: 0,
|
|
1484
|
+
seen: /* @__PURE__ */ new Set(),
|
|
1485
|
+
compactionDone: /* @__PURE__ */ new Set()
|
|
1486
|
+
});
|
|
1487
|
+
var ResumeMismatch = class extends Error {
|
|
1488
|
+
constructor() {
|
|
1489
|
+
super("resume_mismatch");
|
|
945
1490
|
}
|
|
946
1491
|
};
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
const score = (v) => (v.inputTokens ?? 0) + (v.outputTokens ?? 0) + (v.cacheReadTokens ?? 0) + Math.max(v.cacheWriteTokens ?? 0, (v.cacheWrite5mTokens ?? 0) + (v.cacheWrite1hTokens ?? 0)) + (v.reasoningTokens ?? 0);
|
|
964
|
-
const put = (r) => {
|
|
965
|
-
const old = records.get(r.id), identity = identities.get(r.id) ?? { models: /* @__PURE__ */ new Set(), providers: /* @__PURE__ */ new Set() };
|
|
966
|
-
if (r.model !== null) identity.models.add(r.model);
|
|
967
|
-
if (r.provider !== null) identity.providers.add(r.provider);
|
|
968
|
-
identities.set(r.id, identity);
|
|
969
|
-
const chosen = !old || score(r) > score(old) ? r : old;
|
|
970
|
-
const conflict = identity.models.size > 1 || identity.providers.size > 1;
|
|
971
|
-
records.set(r.id, conflict ? { ...chosen, model: null, provider: null, gaps: [.../* @__PURE__ */ new Set([...chosen.gaps, "response_identity_model_conflict"])] } : chosen);
|
|
972
|
-
};
|
|
973
|
-
const pushRecord = (record, position) => {
|
|
974
|
-
const start = position?.rawStart ?? offset;
|
|
975
|
-
if (position) offset = position.rawEnd;
|
|
976
|
-
if (++seq > (options.maxRecords ?? 1e5)) return;
|
|
977
|
-
let r = record;
|
|
978
|
-
if (harness === "opencode") {
|
|
979
|
-
r = { ...r, ...obj(r.message) };
|
|
980
|
-
const wrapped = r.data ?? r.content;
|
|
981
|
-
if (typeof wrapped === "string") {
|
|
982
|
-
try {
|
|
983
|
-
r = { ...r, ...obj(JSON.parse(wrapped)) };
|
|
984
|
-
} catch {
|
|
985
|
-
}
|
|
986
|
-
} else if (wrapped && typeof wrapped === "object") r = { ...r, ...obj(wrapped) };
|
|
987
|
-
}
|
|
988
|
-
const p = obj(r.payload), m = obj(r.message);
|
|
989
|
-
project = str(r.cwd, p.cwd, r.directory) ?? project;
|
|
990
|
-
if (harness === "claude") session = str(r.sessionId) ?? session;
|
|
991
|
-
if (harness === "pi" && r.type === "session") session = str(r.id) ?? session;
|
|
992
|
-
if (harness === "codex" && (r.type === "session_meta" || r.type === "turn_context")) {
|
|
993
|
-
session = str(p.session_id, p.id) ?? session;
|
|
994
|
-
model = str(p.model) ?? model;
|
|
995
|
-
provider = str(p.model_provider, p.provider) ?? provider;
|
|
996
|
-
return;
|
|
997
|
-
}
|
|
998
|
-
const eventAt = at(r.timestamp) ?? at(m.timestamp) ?? at(obj(r.time).created);
|
|
999
|
-
const key = str(r.uuid, r.id, p.id, m.id) ?? `record:${seq}`;
|
|
1000
|
-
const accepted = options.acceptRecord?.(r, { key, rawStart: start, rawEnd: offset, eventAt, project }) ?? true;
|
|
1001
|
-
let input = null, output = null, read = null, write = null, reasoning = null, reported = null;
|
|
1002
|
-
let observedModel = null, observedProvider = null, id, basis, includesCache = false, includesReasoning = true;
|
|
1003
|
-
const gaps = [];
|
|
1004
|
-
if (harness === "codex") {
|
|
1005
|
-
if (r.type !== "event_msg" || p.type !== "token_count") return;
|
|
1006
|
-
const info = obj(p.info), last = obj(info.last_token_usage), total = obj(info.total_token_usage);
|
|
1007
|
-
if (!Object.keys(last).length && !Object.keys(total).length) return;
|
|
1008
|
-
observed++;
|
|
1009
|
-
const previousTotal = previous;
|
|
1010
|
-
previous = Object.keys(total).length ? total : previous;
|
|
1011
|
-
const signature = Object.keys(total).length ? JSON.stringify([session, total.input_tokens, total.output_tokens, total.cached_input_tokens, total.reasoning_output_tokens]) : null;
|
|
1012
|
-
if (signature && seenTotals.has(signature)) return;
|
|
1013
|
-
if (signature) seenTotals.add(signature);
|
|
1014
|
-
const delta = (field) => {
|
|
1015
|
-
const current = num2(total[field]), prev = num2(previousTotal?.[field]);
|
|
1016
|
-
return current === null || prev === null || current < prev ? null : current - prev;
|
|
1017
|
-
};
|
|
1018
|
-
const firstComplete = previousTotal === null && num2(last.input_tokens) !== null && last.input_tokens === total.input_tokens && last.output_tokens === total.output_tokens;
|
|
1019
|
-
const bucket = (field) => num2(last[field]) ?? delta(field) ?? (firstComplete ? num2(total[field]) : null);
|
|
1020
|
-
input = bucket("input_tokens");
|
|
1021
|
-
output = bucket("output_tokens");
|
|
1022
|
-
read = bucket("cached_input_tokens");
|
|
1023
|
-
write = bucket("cache_write_input_tokens");
|
|
1024
|
-
reasoning = bucket("reasoning_output_tokens");
|
|
1025
|
-
if (!Object.keys(last).length && previousTotal === null) gaps.push("cumulative_baseline_unknown");
|
|
1026
|
-
if (previousTotal && Object.keys(total).length && ["input_tokens", "output_tokens"].some((k) => num2(total[k]) !== null && num2(previousTotal[k]) !== null && num2(total[k]) < num2(previousTotal[k]))) gaps.push("cumulative_counter_reset");
|
|
1027
|
-
observedModel = str(p.model, info.model) ?? model;
|
|
1028
|
-
observedProvider = str(p.model_provider, p.provider, info.provider) ?? provider;
|
|
1029
|
-
includesCache = true;
|
|
1030
|
-
basis = "codex_last_response_with_cumulative_dedup";
|
|
1031
|
-
id = `codex:${hash(JSON.stringify([session, str(p.response_id, info.response_id) ?? signature ?? [eventAt, last]]))}`;
|
|
1032
|
-
write = write ?? 0;
|
|
1033
|
-
} else if (harness === "claude") {
|
|
1034
|
-
if (r.type !== "assistant" || m.role !== "assistant") return;
|
|
1035
|
-
const u = obj(m.usage);
|
|
1036
|
-
input = num2(u.input_tokens);
|
|
1037
|
-
output = num2(u.output_tokens);
|
|
1038
|
-
read = num2(u.cache_read_input_tokens) ?? (input !== null ? 0 : null);
|
|
1039
|
-
write = num2(u.cache_creation_input_tokens) ?? (input !== null ? 0 : null);
|
|
1040
|
-
reasoning = num2(u.reasoning_tokens);
|
|
1041
|
-
const duration = obj(u.cache_creation);
|
|
1042
|
-
const five = num2(duration.ephemeral_5m_input_tokens), hour = num2(duration.ephemeral_1h_input_tokens);
|
|
1043
|
-
if (five !== null && hour !== null && write !== null && five + hour !== write) gaps.push("cache_write_duration_conflict");
|
|
1044
|
-
if (hour !== null && hour > 0 && (five === null || write === null || five + hour !== write)) gaps.push("cache_write_duration_unknown");
|
|
1045
|
-
observedModel = str(m.model, r.model);
|
|
1046
|
-
observedProvider = str(m.provider, r.provider, r.modelProvider);
|
|
1047
|
-
reported = cost(r.costUSD) ?? cost(m.costUSD);
|
|
1048
|
-
basis = "claude_response_usage";
|
|
1049
|
-
id = `claude:${hash(JSON.stringify([str(m.id, r.uuid) ?? key, str(r.requestId) ?? [session, eventAt]]))}`;
|
|
1050
|
-
} else if (harness === "pi") {
|
|
1051
|
-
if (r.type !== "message" || m.role !== "assistant") return;
|
|
1052
|
-
const u = obj(m.usage);
|
|
1053
|
-
input = num2(u.input);
|
|
1054
|
-
output = num2(u.output);
|
|
1055
|
-
read = num2(u.cacheRead) ?? (input !== null ? 0 : null);
|
|
1056
|
-
write = num2(u.cacheWrite) ?? (input !== null ? 0 : null);
|
|
1057
|
-
reasoning = num2(u.reasoning);
|
|
1058
|
-
reported = cost(obj(u.cost).total);
|
|
1059
|
-
observedModel = str(m.model);
|
|
1060
|
-
observedProvider = str(m.provider);
|
|
1061
|
-
basis = "pi_response_usage";
|
|
1062
|
-
id = `pi:${hash(JSON.stringify([session, str(r.id) ?? key, eventAt]))}`;
|
|
1063
|
-
} else {
|
|
1064
|
-
if (r.role !== "assistant") return;
|
|
1065
|
-
const u = obj(r.tokens), cache = obj(u.cache);
|
|
1066
|
-
input = num2(u.input);
|
|
1067
|
-
output = num2(u.output);
|
|
1068
|
-
read = num2(cache.read) ?? (input !== null ? 0 : null);
|
|
1069
|
-
write = num2(cache.write) ?? (input !== null ? 0 : null);
|
|
1070
|
-
reasoning = num2(u.reasoning);
|
|
1071
|
-
reported = cost(r.cost);
|
|
1072
|
-
observedModel = str(r.modelID, obj(r.model).modelID, obj(r.model).id, r.model);
|
|
1073
|
-
observedProvider = str(r.providerID, obj(r.model).providerID, r.provider);
|
|
1074
|
-
basis = "opencode_response_tokens";
|
|
1075
|
-
includesReasoning = false;
|
|
1076
|
-
id = `opencode:${str(r.id) ?? hash(JSON.stringify([conversationId, key, eventAt]))}`;
|
|
1077
|
-
}
|
|
1078
|
-
if (observedModel === "codex-auto-review") {
|
|
1079
|
-
observedModel = null;
|
|
1080
|
-
gaps.push("workflow_model_identity_unavailable");
|
|
1081
|
-
}
|
|
1082
|
-
if (harness !== "codex") observed++;
|
|
1083
|
-
if (r.isSynthetic === true || r.isMeta === true || m.isSynthetic === true || observedModel === "<synthetic>") {
|
|
1084
|
-
excluded++;
|
|
1085
|
-
return;
|
|
1086
|
-
}
|
|
1087
|
-
if (!accepted) {
|
|
1088
|
-
excluded++;
|
|
1089
|
-
return;
|
|
1090
|
-
}
|
|
1091
|
-
if (!observedModel) gaps.push("model_unrecorded");
|
|
1092
|
-
if (!observedProvider) gaps.push("provider_unrecorded");
|
|
1093
|
-
if (input === null || output === null) gaps.push("usage_partial");
|
|
1094
|
-
if (includesCache && read === null) gaps.push("cache_inclusion_unknown");
|
|
1095
|
-
if (includesCache && input !== null && read !== null && write !== null && read + write > input) gaps.push("cache_exceeds_input");
|
|
1096
|
-
if (includesReasoning && output !== null && reasoning !== null && reasoning > output) gaps.push("reasoning_exceeds_output");
|
|
1097
|
-
put({ id, conversationId, harness, eventAt, project, provider: observedProvider, model: observedModel, canonicalModel: null, inputTokens: input, outputTokens: output, cacheReadTokens: read, cacheWriteTokens: write, reasoningTokens: reasoning, inputIncludesCache: includesCache, outputIncludesReasoning: includesReasoning, reportedCostUsd: reported, basis, gaps, ...harness === "claude" ? { cacheWrite5mTokens: num2(obj(obj(m.usage).cache_creation).ephemeral_5m_input_tokens), cacheWrite1hTokens: num2(obj(obj(m.usage).cache_creation).ephemeral_1h_input_tokens) } : {} });
|
|
1098
|
-
};
|
|
1099
|
-
const pushLine = (line) => {
|
|
1100
|
-
const start = offset;
|
|
1101
|
-
offset += Buffer.byteLength(line) + 1;
|
|
1102
|
-
if (!line.trim()) return;
|
|
1103
|
-
let r;
|
|
1104
|
-
try {
|
|
1105
|
-
r = obj(JSON.parse(line));
|
|
1106
|
-
} catch {
|
|
1107
|
-
seq++;
|
|
1108
|
-
return;
|
|
1109
|
-
}
|
|
1110
|
-
pushRecord(r, { rawStart: start, rawEnd: offset });
|
|
1111
|
-
};
|
|
1112
|
-
return { pushRecord, invalidateContext: () => {
|
|
1113
|
-
project = null;
|
|
1114
|
-
model = null;
|
|
1115
|
-
provider = null;
|
|
1116
|
-
previous = null;
|
|
1117
|
-
}, pushLine, records: () => [...records.values()], counts: () => ({ observed, excluded, deduplicated: Math.max(0, observed - excluded - records.size) }) };
|
|
1492
|
+
var U_KEYS = ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens"];
|
|
1493
|
+
var tuple = (u) => isRec(u) ? U_KEYS.map((k) => int(u[k])) : null;
|
|
1494
|
+
var AUTO_REVIEW_MODELS = [
|
|
1495
|
+
["2026-07-30", "gpt-5.6-luna"],
|
|
1496
|
+
["2026-03-05", "gpt-5.4"],
|
|
1497
|
+
["2026-02-05", "gpt-5.3-codex"],
|
|
1498
|
+
["2025-12-11", "gpt-5.2-codex"],
|
|
1499
|
+
["2025-11-13", "gpt-5.1-codex"],
|
|
1500
|
+
["2025-09-15", "gpt-5-codex"]
|
|
1501
|
+
];
|
|
1502
|
+
function codexModel(model, timestamp) {
|
|
1503
|
+
if (model === "codex-auto-review") {
|
|
1504
|
+
const day = timestamp.slice(0, 10);
|
|
1505
|
+
return { model: AUTO_REVIEW_MODELS.find(([since]) => day >= since)?.[1] ?? "gpt-5", inferred: true };
|
|
1506
|
+
}
|
|
1507
|
+
return model ? { model, inferred: false } : { model: "gpt-5", inferred: true };
|
|
1118
1508
|
}
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
const
|
|
1123
|
-
if (
|
|
1124
|
-
if (
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
const
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
return;
|
|
1136
|
-
}
|
|
1137
|
-
if (turnBytes + Buffer.byteLength(turnPrefix) <= turnLimit) {
|
|
1138
|
-
turns.push({ lines: turnLines, bytes: turnBytes, prefix: turnPrefix });
|
|
1139
|
-
keptBytes += turnBytes;
|
|
1140
|
-
while (turns.length > 1 && keptBytes + Buffer.byteLength(turns[0].prefix) > turnLimit) keptBytes -= turns.shift().bytes;
|
|
1141
|
-
} else gaps.add("context_turn_oversize");
|
|
1142
|
-
turnLines = [];
|
|
1143
|
-
turnBytes = 0;
|
|
1144
|
-
};
|
|
1145
|
-
const process2 = (raw, end) => {
|
|
1146
|
-
if (++records > (options.maxRecords ?? 1e6)) {
|
|
1147
|
-
gaps.add("native_usage_records_limit");
|
|
1148
|
-
return;
|
|
1149
|
-
}
|
|
1150
|
-
const line = raw.toString("utf8");
|
|
1151
|
-
let r;
|
|
1509
|
+
function codexFact(b, s, at, timestamp, d, key) {
|
|
1510
|
+
const input = d[0], output = d[2], reasoning = d[3], cached = Math.min(d[1], input);
|
|
1511
|
+
const { model, inferred } = codexModel(b.currentModel, timestamp);
|
|
1512
|
+
const fact = { key, at, model, provider: "openai", input: input - cached, output, cacheRead: cached, cacheWrite: 0, cacheWrite1h: 0, reasoning };
|
|
1513
|
+
if (inferred) fact.inferredModel = true;
|
|
1514
|
+
if (s.tier) fact.tier = s.tier;
|
|
1515
|
+
return fact;
|
|
1516
|
+
}
|
|
1517
|
+
function codexWanted(head, lineNo) {
|
|
1518
|
+
return lineNo === 0 && head.includes("session_meta") || head.includes('"token_count"') || head.includes('"turn_context"') || head.includes("thread_settings_applied") || head.includes('"task_started"') || head.includes('"token_usage_record"') || head.includes('"type":"compacted"') || head.includes('"UserMessage"') && head.includes("item_completed") || head.includes('"role":"user"') && head.includes('"response_item"') || head.includes('"event_msg"') && head.includes('"type":"user_message"');
|
|
1519
|
+
}
|
|
1520
|
+
function codexLine(b, line, s) {
|
|
1521
|
+
const n = s.lineNo++;
|
|
1522
|
+
const head = line.toString("latin1", 0, Math.min(line.length, 260));
|
|
1523
|
+
const f = b.facts;
|
|
1524
|
+
const parse = () => {
|
|
1152
1525
|
try {
|
|
1153
|
-
|
|
1154
|
-
if (!isRecord(parsed)) return;
|
|
1155
|
-
r = parsed;
|
|
1526
|
+
return JSON.parse(line.toString("utf8"));
|
|
1156
1527
|
} catch {
|
|
1157
|
-
|
|
1158
|
-
return;
|
|
1159
|
-
}
|
|
1160
|
-
if (r.entrypoint === "sdk-ts" || r.originator === "codex_exec") programmatic = true;
|
|
1161
|
-
const p = isRecord(r.payload) ? r.payload : r;
|
|
1162
|
-
if (p.originator === "codex_exec" || p.source === "exec") programmatic = true;
|
|
1163
|
-
const m = isRecord(r.message) ? r.message : {};
|
|
1164
|
-
if (typeof r.cwd === "string") project = r.cwd;
|
|
1165
|
-
if (typeof p.cwd === "string") project = p.cwd;
|
|
1166
|
-
if (harness === "claude" && typeof r.sessionId === "string") {
|
|
1167
|
-
if (file.includes(`${path.sep}subagents${path.sep}`) || r.isSidechain === true) {
|
|
1168
|
-
parent = r.sessionId;
|
|
1169
|
-
nativeId = `${parent}:${path.basename(file, ".jsonl")}`;
|
|
1170
|
-
} else nativeId = r.sessionId;
|
|
1528
|
+
f.malformed++;
|
|
1529
|
+
return null;
|
|
1171
1530
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1531
|
+
};
|
|
1532
|
+
if (n === 0 && head.includes("session_meta")) {
|
|
1533
|
+
const r = parse();
|
|
1534
|
+
if (!r) return;
|
|
1535
|
+
const p = isRec(r.payload) ? r.payload : {};
|
|
1536
|
+
f.sessionId = str(p.id) ?? str(p.session_id) ?? f.sessionId;
|
|
1537
|
+
f.project = str(p.cwd);
|
|
1538
|
+
f.startedAt = str(p.timestamp) ?? str(r.timestamp);
|
|
1539
|
+
const source = p.source;
|
|
1540
|
+
if (isRec(source) && "subagent" in source) {
|
|
1541
|
+
f.child = true;
|
|
1542
|
+
const sub = isRec(source.subagent) ? source.subagent : {};
|
|
1543
|
+
const spawn = isRec(sub.thread_spawn) ? sub.thread_spawn : {};
|
|
1544
|
+
f.parentId = str(spawn.parent_thread_id) ?? str(p.parent_thread_id) ?? str(p.forked_from_id);
|
|
1176
1545
|
}
|
|
1177
|
-
if (
|
|
1178
|
-
|
|
1179
|
-
|
|
1546
|
+
if (source === "exec" || p.originator === "codex_exec" || p.originator === "codex_sdk_ts") f.automated = true;
|
|
1547
|
+
touch(b, time(r.timestamp));
|
|
1548
|
+
return;
|
|
1549
|
+
}
|
|
1550
|
+
if (head.includes('"token_count"')) {
|
|
1551
|
+
const r = parse();
|
|
1552
|
+
const p = r && isRec(r.payload) ? r.payload : null;
|
|
1553
|
+
const info = p && isRec(p.info) ? p.info : null;
|
|
1554
|
+
if (!r || !info) return;
|
|
1555
|
+
const total = tuple(info.total_token_usage), last = tuple(info.last_token_usage);
|
|
1556
|
+
const totalSig = total ? total.join(",") : null;
|
|
1557
|
+
const advanced = totalSig === null || totalSig !== s.previousTotal;
|
|
1558
|
+
let d = null;
|
|
1559
|
+
if (last && advanced) d = last;
|
|
1560
|
+
else if (total) {
|
|
1561
|
+
const prev = s.previousTotal ? s.previousTotal.split(",").map(Number) : [0, 0, 0, 0, 0];
|
|
1562
|
+
d = total.map((v, i) => Math.max(0, v - prev[i]));
|
|
1180
1563
|
}
|
|
1181
|
-
if (
|
|
1182
|
-
|
|
1183
|
-
|
|
1564
|
+
if (totalSig) s.previousTotal = totalSig;
|
|
1565
|
+
if (!d || !(d[0] || d[1] || d[2] || d[3])) return;
|
|
1566
|
+
s.deltas.add(d.join(","));
|
|
1567
|
+
const key = totalSig ? `x:${totalSig}|${last ? last.join(",") : ""}` : null;
|
|
1568
|
+
const fact = codexFact(b, s, time(r.timestamp), String(r.timestamp ?? ""), d, key);
|
|
1569
|
+
addUsage(b, fact);
|
|
1570
|
+
if (fact.at !== null) b.events.push({ at: fact.at, tools: 0, subs: 0, weight: usageWeight(fact.input, fact.output, fact.cacheRead, fact.cacheWrite) });
|
|
1571
|
+
return;
|
|
1572
|
+
}
|
|
1573
|
+
if (head.includes('"turn_context"')) {
|
|
1574
|
+
const p = parse()?.payload;
|
|
1575
|
+
if (isRec(p)) {
|
|
1576
|
+
b.currentModel = str(p.model) ?? b.currentModel;
|
|
1577
|
+
if (typeof p.cwd === "string") f.project = p.cwd;
|
|
1184
1578
|
}
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
const
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
if (head.includes("thread_settings_applied")) {
|
|
1582
|
+
const p = parse()?.payload;
|
|
1583
|
+
const settings = isRec(p) && isRec(p.thread_settings) ? p.thread_settings : null;
|
|
1584
|
+
if (settings && "service_tier" in settings) {
|
|
1585
|
+
const v = settings.service_tier;
|
|
1586
|
+
s.tier = v === "priority" || v === "fast" ? "fast" : v === "default" || v === "standard" ? "standard" : void 0;
|
|
1192
1587
|
}
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1588
|
+
return;
|
|
1589
|
+
}
|
|
1590
|
+
if (head.includes('"task_started"')) {
|
|
1591
|
+
const p = parse()?.payload;
|
|
1592
|
+
s.imported = isRec(p) && String(p.turn_id ?? "").startsWith("external-import");
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
if (head.includes('"UserMessage"') && head.includes("item_completed")) {
|
|
1596
|
+
const r = parse();
|
|
1597
|
+
const p = r && isRec(r.payload) ? r.payload : null;
|
|
1598
|
+
if (!r || !p) return;
|
|
1599
|
+
const item = isRec(p.item) ? p.item : {};
|
|
1600
|
+
s.itemPrompts.push({
|
|
1601
|
+
key: str(item.id),
|
|
1602
|
+
at: time(r.timestamp),
|
|
1603
|
+
text: textOf(item.content),
|
|
1604
|
+
imported: s.imported || String(p.turn_id ?? "").startsWith("external-import"),
|
|
1605
|
+
project: f.project
|
|
1606
|
+
});
|
|
1607
|
+
return;
|
|
1608
|
+
}
|
|
1609
|
+
if (head.includes('"role":"user"') && head.includes('"response_item"')) {
|
|
1610
|
+
const r = parse();
|
|
1611
|
+
if (!r) return;
|
|
1612
|
+
const p = isRec(r.payload) ? r.payload : {};
|
|
1613
|
+
s.responsePrompts.push({ key: null, at: time(r.timestamp), text: textOf(p.content), imported: s.imported, project: f.project });
|
|
1614
|
+
return;
|
|
1615
|
+
}
|
|
1616
|
+
if (head.includes('"event_msg"') && line.subarray(0, 400).includes('"type":"user_message"')) {
|
|
1617
|
+
const r = parse();
|
|
1618
|
+
if (!r) return;
|
|
1619
|
+
const p = isRec(r.payload) ? r.payload : {};
|
|
1620
|
+
s.itemPrompts.push({ key: null, at: time(r.timestamp), text: typeof p.message === "string" ? p.message : "", imported: s.imported, project: f.project });
|
|
1621
|
+
return;
|
|
1622
|
+
}
|
|
1623
|
+
if (codexToolOrAbort(b, head)) return;
|
|
1624
|
+
if (head.includes('"token_usage_record"')) {
|
|
1625
|
+
const r = parse();
|
|
1626
|
+
const p = r && isRec(r.payload) ? r.payload : null;
|
|
1627
|
+
const id = p ? str(p.response_id) : null;
|
|
1628
|
+
if (r && p && id && isRec(p.usage)) s.records.set(id, { at: time(r.timestamp), usage: p.usage });
|
|
1629
|
+
return;
|
|
1630
|
+
}
|
|
1631
|
+
if (head.includes('"type":"compacted"')) {
|
|
1632
|
+
const at = line.lastIndexOf('"compaction_response_id":"');
|
|
1633
|
+
if (at >= 0) {
|
|
1634
|
+
const start = at + 26, end = line.indexOf(34, start);
|
|
1635
|
+
if (end > start) s.compacted.add(line.toString("latin1", start, end));
|
|
1197
1636
|
}
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
var TOOL_CALL = /"type":"(?:function_call|custom_tool_call)"/;
|
|
1640
|
+
var NAME = /"name":"([^"]{1,80})"/;
|
|
1641
|
+
var STAMP = /"timestamp":"([^"]{10,40})"/;
|
|
1642
|
+
function codexToolOrAbort(b, head) {
|
|
1643
|
+
if (TOOL_CALL.test(head) && head.includes('"response_item"')) {
|
|
1644
|
+
const at = time(STAMP.exec(head)?.[1]);
|
|
1645
|
+
const name = NAME.exec(head)?.[1] ?? "";
|
|
1646
|
+
if (at !== null) b.events.push({ at, tools: 1, subs: isAgentTool(name) ? 1 : 0, weight: 0 });
|
|
1647
|
+
return true;
|
|
1648
|
+
}
|
|
1649
|
+
if (head.includes('"turn_aborted"') && head.includes('"event_msg"')) {
|
|
1650
|
+
const at = time(STAMP.exec(head)?.[1]);
|
|
1651
|
+
if (at !== null) b.interrupts.push(at);
|
|
1652
|
+
return true;
|
|
1653
|
+
}
|
|
1654
|
+
return false;
|
|
1655
|
+
}
|
|
1656
|
+
function finishCodex(b, s) {
|
|
1657
|
+
const f = b.facts;
|
|
1658
|
+
for (const id of s.compacted) {
|
|
1659
|
+
if (s.compactionDone.has(id)) continue;
|
|
1660
|
+
const record = s.records.get(id);
|
|
1661
|
+
const d = record ? tuple(record.usage) : null;
|
|
1662
|
+
if (!record || !d || !(d[0] || d[1] || d[2] || d[3]) || s.deltas.has(d.join(","))) continue;
|
|
1663
|
+
s.compactionDone.add(id);
|
|
1664
|
+
const fact = codexFact(b, s, record.at, record.at ? new Date(record.at).toISOString() : "", d, `codex-compaction:${id}`);
|
|
1665
|
+
fact.compaction = true;
|
|
1666
|
+
f.usage.push(fact);
|
|
1667
|
+
touch(b, record.at);
|
|
1668
|
+
}
|
|
1669
|
+
if (s.mode === "responses" && s.itemPrompts.length) throw new ResumeMismatch();
|
|
1670
|
+
const mode = s.mode ?? (s.itemPrompts.length ? "items" : s.responsePrompts.length ? "responses" : null);
|
|
1671
|
+
const list = mode === "items" ? s.itemPrompts : mode === "responses" ? s.responsePrompts : [];
|
|
1672
|
+
s.mode = mode;
|
|
1673
|
+
const reason = f.child ? "subagent_parent_written" : f.automated ? "programmatic_exec_session" : null;
|
|
1674
|
+
const seen = s.seen, base = s.converted;
|
|
1675
|
+
s.converted += list.length;
|
|
1676
|
+
list.forEach((p, n) => {
|
|
1677
|
+
const i = base + n;
|
|
1678
|
+
if (p.key) {
|
|
1679
|
+
if (seen.has(p.key)) return;
|
|
1680
|
+
seen.add(p.key);
|
|
1204
1681
|
}
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
if (
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1682
|
+
const key = `codex:${f.sessionId}:${p.key ?? i}`;
|
|
1683
|
+
if (reason) return addPrompt(b, key, p.at, p.project, "excluded", reason, "");
|
|
1684
|
+
if (p.imported) return addPrompt(b, key, p.at, p.project, "excluded", "imported_from_claude_transcript", "");
|
|
1685
|
+
const c = classifyText(cleanCodexText(p.text));
|
|
1686
|
+
addPrompt(b, key, p.at, p.project, c.kind, c.reason, c.text);
|
|
1687
|
+
});
|
|
1688
|
+
const answers = f.usage.filter((u) => u.at !== null).sort((x, y) => x.at - y.at);
|
|
1689
|
+
let j = 0, previous = null;
|
|
1690
|
+
for (const p of [...f.prompts].sort((x, y) => (x.at ?? 0) - (y.at ?? 0))) {
|
|
1691
|
+
while (j < answers.length && answers[j].at <= (p.at ?? 0)) previous = answers[j++].model;
|
|
1692
|
+
p.before = previous;
|
|
1693
|
+
p.beforeProvider = previous ? "openai" : null;
|
|
1694
|
+
p.after = answers[j]?.model ?? null;
|
|
1695
|
+
p.afterProvider = p.after ? "openai" : null;
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
function piLine(b, line) {
|
|
1699
|
+
let r;
|
|
1700
|
+
try {
|
|
1701
|
+
r = JSON.parse(line.toString("utf8"));
|
|
1702
|
+
} catch {
|
|
1703
|
+
b.facts.malformed++;
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
const f = b.facts, at = time(r.timestamp);
|
|
1707
|
+
if (r.type === "session") {
|
|
1708
|
+
f.sessionId = str(r.id) ?? f.sessionId;
|
|
1709
|
+
f.project = str(r.cwd);
|
|
1710
|
+
touch(b, at);
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1713
|
+
if (r.type === "model_change") {
|
|
1714
|
+
b.currentModel = str(r.modelId) ?? b.currentModel;
|
|
1715
|
+
b.currentProvider = str(r.provider) ?? b.currentProvider;
|
|
1716
|
+
return;
|
|
1717
|
+
}
|
|
1718
|
+
if (r.type !== "message" || !isRec(r.message)) return;
|
|
1719
|
+
const m = r.message;
|
|
1720
|
+
if (m.role === "assistant" && isRec(m.usage)) {
|
|
1721
|
+
const u = m.usage;
|
|
1722
|
+
const fact = {
|
|
1723
|
+
key: `pi:${f.sessionId}:${String(r.id)}`,
|
|
1724
|
+
at,
|
|
1725
|
+
model: str(m.model) ?? b.currentModel,
|
|
1726
|
+
provider: str(m.provider) ?? b.currentProvider,
|
|
1727
|
+
input: int(u.input),
|
|
1728
|
+
output: int(u.output),
|
|
1729
|
+
cacheRead: int(u.cacheRead),
|
|
1730
|
+
cacheWrite: int(u.cacheWrite),
|
|
1731
|
+
cacheWrite1h: 0,
|
|
1732
|
+
reasoning: 0
|
|
1733
|
+
};
|
|
1734
|
+
if (fact.input + fact.output + fact.cacheRead + fact.cacheWrite > 0) addUsage(b, fact);
|
|
1735
|
+
else answered(b, fact.model, fact.provider);
|
|
1736
|
+
if (at !== null) {
|
|
1737
|
+
const tools = Array.isArray(m.content) ? m.content.filter((c) => isRec(c) && typeof c.type === "string" && c.type.toLowerCase().includes("tool")).length : 0;
|
|
1738
|
+
b.events.push({ at, tools, subs: 0, weight: usageWeight(fact.input, fact.output, fact.cacheRead, fact.cacheWrite) });
|
|
1230
1739
|
}
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
}
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1740
|
+
return;
|
|
1741
|
+
}
|
|
1742
|
+
if (m.role === "user") {
|
|
1743
|
+
const c = classifyText(textOf(m.content));
|
|
1744
|
+
addPrompt(b, `pi:${f.sessionId}:${String(r.id)}`, at, f.project, c.kind, c.reason, c.text);
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
1747
|
+
var CHUNK = 4 * 1024 * 1024;
|
|
1748
|
+
var MAX_LINE = 256 * 1024 * 1024;
|
|
1749
|
+
var sharedBuffer = null;
|
|
1750
|
+
var RESUME_RECENT_MS = 48 * 36e5;
|
|
1751
|
+
function extractFile(file, harness, options = {}, previous) {
|
|
1752
|
+
const fd = fs.openSync(file, "r");
|
|
1753
|
+
try {
|
|
1754
|
+
const stat = fs.fstatSync(fd);
|
|
1755
|
+
let b = makeBuilder(file, harness, stat.size, stat.mtimeMs);
|
|
1756
|
+
let codex = newCodexState();
|
|
1757
|
+
let position = 0, firstNew = 0;
|
|
1758
|
+
const subagent = file.includes(SUBAGENT_DIR), sessions = /* @__PURE__ */ new Set();
|
|
1759
|
+
if (previous) {
|
|
1760
|
+
const r = previous.resume;
|
|
1761
|
+
if (!r || previous.excluded || r.dev !== stat.dev || r.ino !== stat.ino || r.offset > stat.size || r.offset < 1 || tailHash(fd, r.offset) !== r.tail) throw new ResumeMismatch();
|
|
1762
|
+
const nl = Buffer.alloc(1);
|
|
1763
|
+
fs.readSync(fd, nl, 0, 1, r.offset - 1);
|
|
1764
|
+
if (nl[0] !== 10) throw new ResumeMismatch();
|
|
1765
|
+
b = resumeBuilder(previous, r, stat);
|
|
1766
|
+
if (r.codex) codex = { ...structuredClone(r.codex), itemPrompts: [], responsePrompts: [] };
|
|
1767
|
+
for (const sid of r.sessions ?? []) sessions.add(sid);
|
|
1768
|
+
position = r.offset;
|
|
1769
|
+
firstNew = b.facts.prompts.length;
|
|
1245
1770
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
records++;
|
|
1269
|
-
gaps.add(result.code ?? "native_metadata_invalid");
|
|
1270
|
-
gaps.add("native_scope_record_unavailable");
|
|
1271
|
-
usage?.invalidateContext();
|
|
1272
|
-
scopeUncertain = true;
|
|
1273
|
-
project = null;
|
|
1274
|
-
currentModel = null;
|
|
1275
|
-
currentProvider = null;
|
|
1771
|
+
const onLine = harness === "claude" ? (line) => claudeLine(b, line, subagent, sessions) : harness === "codex" ? (line) => codexLine(b, line, codex) : (line) => piLine(b, line);
|
|
1772
|
+
let buffer = sharedBuffer ??= Buffer.allocUnsafe(CHUNK);
|
|
1773
|
+
let filled = 0, skipping = false, base = position, endsWithNewline = true;
|
|
1774
|
+
for (; ; ) {
|
|
1775
|
+
if (filled === buffer.length) {
|
|
1776
|
+
if (harness === "codex" && !codexWanted(buffer.toString("latin1", 0, Math.min(filled, 400)), codex.lineNo)) {
|
|
1777
|
+
codexToolOrAbort(b, buffer.toString("latin1", 0, Math.min(filled, 400)));
|
|
1778
|
+
skipping = true;
|
|
1779
|
+
base += filled;
|
|
1780
|
+
filled = 0;
|
|
1781
|
+
codex.lineNo++;
|
|
1782
|
+
} else if (buffer.length >= MAX_LINE) {
|
|
1783
|
+
skipping = true;
|
|
1784
|
+
base += filled;
|
|
1785
|
+
filled = 0;
|
|
1786
|
+
b.facts.malformed++;
|
|
1787
|
+
} else {
|
|
1788
|
+
const bigger = Buffer.allocUnsafe(buffer.length * 2);
|
|
1789
|
+
buffer.copy(bigger, 0, 0, filled);
|
|
1790
|
+
buffer = bigger;
|
|
1791
|
+
if (bigger.length <= 4 * CHUNK) sharedBuffer = bigger;
|
|
1792
|
+
}
|
|
1276
1793
|
}
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
stream.destroy();
|
|
1288
|
-
throw new Error("cancelled");
|
|
1794
|
+
const n = fs.readSync(fd, buffer, filled, buffer.length - filled, position);
|
|
1795
|
+
position += n;
|
|
1796
|
+
const end = filled + n;
|
|
1797
|
+
let start = 0;
|
|
1798
|
+
for (; ; ) {
|
|
1799
|
+
const nl = buffer.indexOf(10, start);
|
|
1800
|
+
if (nl < 0 || nl >= end) break;
|
|
1801
|
+
if (skipping) skipping = false;
|
|
1802
|
+
else if (nl > start) onLine(buffer.subarray(start, nl));
|
|
1803
|
+
start = nl + 1;
|
|
1289
1804
|
}
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
const newline = bytes.indexOf(10, at2), end = newline < 0 ? bytes.length : newline, part = bytes.subarray(at2, end);
|
|
1295
|
-
if (projection) projection.push(part);
|
|
1296
|
-
else if (heldBytes + part.length > 8 * 1024 * 1024) {
|
|
1297
|
-
projection = new NativeMetadataProjection(void 0, harness);
|
|
1298
|
-
for (const piece of held) projection.push(piece);
|
|
1299
|
-
projection.push(part);
|
|
1300
|
-
held = [];
|
|
1301
|
-
heldBytes = 0;
|
|
1302
|
-
} else {
|
|
1303
|
-
held.push(part);
|
|
1304
|
-
heldBytes += part.length;
|
|
1805
|
+
if (n === 0) {
|
|
1806
|
+
if (start < end) {
|
|
1807
|
+
endsWithNewline = false;
|
|
1808
|
+
if (!skipping) onLine(buffer.subarray(start, end));
|
|
1305
1809
|
}
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
at2 = newline + 1;
|
|
1810
|
+
base += start;
|
|
1811
|
+
break;
|
|
1309
1812
|
}
|
|
1310
|
-
|
|
1813
|
+
buffer.copyWithin(0, start, end);
|
|
1814
|
+
base += start;
|
|
1815
|
+
filled = end - start;
|
|
1311
1816
|
}
|
|
1817
|
+
if (harness === "codex") finishCodex(b, codex);
|
|
1818
|
+
if (harness === "claude" && !subagent && sessions.size) b.facts.sessionIds = [...sessions];
|
|
1819
|
+
b.facts.size = stat.size;
|
|
1820
|
+
b.facts.mtimeMs = stat.mtimeMs;
|
|
1821
|
+
finishStory(b, harness, options, firstNew);
|
|
1822
|
+
delete b.facts.resume;
|
|
1823
|
+
if (endsWithNewline && !b.facts.excluded && Date.now() - stat.mtimeMs < RESUME_RECENT_MS) {
|
|
1824
|
+
const events = new Float64Array(b.events.length * 4);
|
|
1825
|
+
b.events.forEach((e, i) => {
|
|
1826
|
+
events[i * 4] = e.at;
|
|
1827
|
+
events[i * 4 + 1] = e.tools;
|
|
1828
|
+
events[i * 4 + 2] = e.subs;
|
|
1829
|
+
events[i * 4 + 3] = e.weight;
|
|
1830
|
+
});
|
|
1831
|
+
const { itemPrompts: _items, responsePrompts: _responses, ...codexState } = codex;
|
|
1832
|
+
b.facts.resume = {
|
|
1833
|
+
offset: base,
|
|
1834
|
+
tail: tailHash(fd, base),
|
|
1835
|
+
dev: stat.dev,
|
|
1836
|
+
ino: stat.ino,
|
|
1837
|
+
model: b.currentModel,
|
|
1838
|
+
provider: b.currentProvider,
|
|
1839
|
+
events,
|
|
1840
|
+
interrupts: Float64Array.from(b.interrupts),
|
|
1841
|
+
...sessions.size ? { sessions: [...sessions] } : {},
|
|
1842
|
+
...harness === "codex" ? { codex: codexState } : {}
|
|
1843
|
+
};
|
|
1844
|
+
}
|
|
1845
|
+
return b.facts;
|
|
1312
1846
|
} finally {
|
|
1313
|
-
|
|
1847
|
+
fs.closeSync(fd);
|
|
1314
1848
|
}
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1849
|
+
}
|
|
1850
|
+
function resumeBuilder(previous, r, stat) {
|
|
1851
|
+
const facts = {
|
|
1852
|
+
...previous,
|
|
1853
|
+
size: stat.size,
|
|
1854
|
+
mtimeMs: stat.mtimeMs,
|
|
1855
|
+
usage: [...previous.usage],
|
|
1856
|
+
prompts: previous.prompts.map((p) => ({ ...p })),
|
|
1857
|
+
...previous.sessionIds ? { sessionIds: [...previous.sessionIds] } : {},
|
|
1858
|
+
...previous.story ? { story: { vocab: previous.story.vocab ? new Map(previous.story.vocab) : null, topics: previous.story.topics ? previous.story.topics.map((t) => ({ ...t, words: new Map([...t.words].map(([w, c]) => [w, [c[0], c[1]]])) })) : null } } : {}
|
|
1859
|
+
};
|
|
1860
|
+
delete facts.resume;
|
|
1861
|
+
const b = makeBuilder(previous.file, previous.harness, stat.size, stat.mtimeMs);
|
|
1862
|
+
b.facts = facts;
|
|
1863
|
+
b.currentModel = r.model;
|
|
1864
|
+
b.currentProvider = r.provider;
|
|
1865
|
+
for (const p of facts.prompts) b.seenPrompts.add(p.key);
|
|
1866
|
+
for (let i = facts.prompts.length - 1; i >= 0; i--) {
|
|
1867
|
+
const p = facts.prompts[i];
|
|
1868
|
+
if (p.kind !== "human") continue;
|
|
1869
|
+
if (p.after !== null) break;
|
|
1870
|
+
b.pending.unshift(p);
|
|
1333
1871
|
}
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1872
|
+
for (let i = 0; i < r.events.length; i += 4) b.events.push({ at: r.events[i], tools: r.events[i + 1], subs: r.events[i + 2], weight: r.events[i + 3] });
|
|
1873
|
+
b.interrupts.push(...r.interrupts);
|
|
1874
|
+
return b;
|
|
1875
|
+
}
|
|
1876
|
+
function finishStory(b, harness, options, from = 0) {
|
|
1877
|
+
const f = b.facts;
|
|
1878
|
+
f.tz = options.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
1879
|
+
if (f.excluded) {
|
|
1880
|
+
for (const p of f.prompts) {
|
|
1881
|
+
p.text = "";
|
|
1882
|
+
delete p.f;
|
|
1883
|
+
delete p.lex;
|
|
1884
|
+
delete p.q;
|
|
1340
1885
|
}
|
|
1886
|
+
delete f.story;
|
|
1887
|
+
return;
|
|
1341
1888
|
}
|
|
1342
|
-
|
|
1343
|
-
finishTurn();
|
|
1344
|
-
const captured = Buffer.from((turns[0]?.prefix ?? "") + turns.flatMap((t) => t.lines).join(""));
|
|
1345
|
-
capturedRecords = turns.reduce((n, t) => n + t.lines.length, 0);
|
|
1346
|
-
if (turnLimit > 0 && capturedRecords < records) gaps.add("context_capture_partial");
|
|
1347
|
-
return { facts: { nativeId, project, parent, child: parent !== null, title: null, events, gaps: [...gaps], hash: artifactHash, consumed: offset, bytes: captured }, usage: maintenance ? [] : all, language: maintenance ? [] : language, hash: artifactHash, bytes: captureSize, stamp: { dev: String(after.dev), ino: String(after.ino), size: Number(after.size), mtimeNs: String(after.mtimeNs), ctimeNs: String(after.ctimeNs) }, counts: maintenance ? { observed: acc?.counts().observed ?? 0, excluded: acc?.counts().observed ?? 0, deduplicated: 0 } : acc?.counts() ?? { observed: 0, excluded: 0, deduplicated: 0 } };
|
|
1889
|
+
f.story = deriveStory(f.prompts, b.events, b.interrupts, { typed: false, counters: harness !== "claude" && !f.child, timezone: f.tz, from, previous: f.story });
|
|
1348
1890
|
}
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
return value.replace(/\\/g, "/").replace(/\/+$/, "").toLowerCase();
|
|
1353
|
-
}
|
|
1354
|
-
function matchesIgnoreEntry(project, entry) {
|
|
1355
|
-
if (!project) return false;
|
|
1356
|
-
const p = fold(project);
|
|
1357
|
-
const e = fold(entry);
|
|
1358
|
-
if (!p || !e) return false;
|
|
1359
|
-
if (p === e) return true;
|
|
1360
|
-
if (e.includes("/")) return p.startsWith(`${e}/`);
|
|
1361
|
-
return p.split("/").filter(Boolean).includes(e);
|
|
1362
|
-
}
|
|
1363
|
-
function isIgnoredProject(project, entries) {
|
|
1364
|
-
return entries.some((entry) => matchesIgnoreEntry(project, entry));
|
|
1891
|
+
function stripText(facts) {
|
|
1892
|
+
for (const p of facts.prompts) p.text = "";
|
|
1893
|
+
return facts;
|
|
1365
1894
|
}
|
|
1895
|
+
function tailHash(fd, offset) {
|
|
1896
|
+
const start = Math.max(0, offset - TAIL), buffer = Buffer.alloc(offset - start);
|
|
1897
|
+
let read = 0;
|
|
1898
|
+
while (read < buffer.length) {
|
|
1899
|
+
const n = fs.readSync(fd, buffer, read, buffer.length - read, start + read);
|
|
1900
|
+
if (!n) break;
|
|
1901
|
+
read += n;
|
|
1902
|
+
}
|
|
1903
|
+
return createHash2("sha256").update(buffer.subarray(0, read)).digest("hex");
|
|
1904
|
+
}
|
|
1905
|
+
var TAIL = 4096;
|
|
1366
1906
|
|
|
1367
|
-
// packages/core/src/analytics/
|
|
1368
|
-
var
|
|
1369
|
-
var
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
if (!parentPort) throw new Error("native_worker_requires_port");
|
|
1907
|
+
// packages/core/src/analytics/scan-pool.ts
|
|
1908
|
+
var readFacts = (job) => stripText(extractFile(job.file, job.harness, { timezone: job.timezone }));
|
|
1909
|
+
var errorCode = (error) => error instanceof Error && /^[a-z][a-z0-9_]{0,63}$/.test(error.message) ? error.message : "source_unreadable";
|
|
1910
|
+
function runScanWorker() {
|
|
1911
|
+
if (!parentPort) throw new Error("scan_worker_requires_parent");
|
|
1373
1912
|
const port = parentPort;
|
|
1374
|
-
|
|
1375
|
-
port.on("message", async (message) => {
|
|
1376
|
-
if (message.protocol !== PROTOCOL || message.kind !== "scan" || !message.job || busy) return;
|
|
1377
|
-
busy = true;
|
|
1378
|
-
const job = message.job;
|
|
1913
|
+
port.on("message", (job) => {
|
|
1379
1914
|
try {
|
|
1380
|
-
|
|
1381
|
-
const snapshot = await streamNativeFacts(job.file, job.harness, { signal: new AbortController().signal, maxBytes: job.maxBytes, maxRecords: job.maxRecords, maxPromptBytes: job.maxPromptBytes, captureBytes: 0, captureFileBytes: job.bytes, expectedIdentity: { dev: job.stamp.dev, ino: job.stamp.ino }, sourceAllowed: (id, project) => !forgotten.has(sourceId(job.harness, id)) && !(project && isIgnoredProject(project, policy.ignoredProjects)), accept });
|
|
1382
|
-
const buffer = serialize(snapshot);
|
|
1383
|
-
if (buffer.length > (message.maxResultBytes ?? 256 * 1024 * 1024)) throw new Error("native_pool_result_bytes_limit");
|
|
1384
|
-
const hash2 = createHash6("sha256").update(buffer).digest("hex");
|
|
1385
|
-
let sequence = 0;
|
|
1386
|
-
for (let at2 = 0; at2 < buffer.length; at2 += FRAME) {
|
|
1387
|
-
const chunk = Uint8Array.from(buffer.subarray(at2, at2 + FRAME));
|
|
1388
|
-
port.postMessage({ protocol: PROTOCOL, kind: "frame", jobId: job.jobId, sequence: sequence++, data: chunk }, [chunk.buffer]);
|
|
1389
|
-
}
|
|
1390
|
-
port.postMessage({ protocol: PROTOCOL, kind: "result", jobId: job.jobId, authorityCommitment: policy.authorityCommitment, bytes: buffer.length, hash: hash2 });
|
|
1915
|
+
port.postMessage({ facts: readFacts(job) });
|
|
1391
1916
|
} catch (error) {
|
|
1392
|
-
port.postMessage({
|
|
1393
|
-
} finally {
|
|
1394
|
-
busy = false;
|
|
1917
|
+
port.postMessage({ error: errorCode(error) });
|
|
1395
1918
|
}
|
|
1396
1919
|
});
|
|
1397
|
-
port.postMessage({ protocol: PROTOCOL, kind: "ready" });
|
|
1398
1920
|
}
|
|
1399
1921
|
|
|
1400
1922
|
// packages/cli/src/audit-native-worker.ts
|
|
1401
|
-
|
|
1923
|
+
runScanWorker();
|
|
1402
1924
|
//# sourceMappingURL=audit-native-worker.js.map
|