polymath-society 0.2.19 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +420 -146
- package/dist/index.js +352 -136
- package/dist/pipeline/FOCUS-EXAMPLES.md +127 -0
- package/dist/pipeline/FOCUS-RUBRIC.md +418 -0
- package/dist/pipeline/coding-agglomerate.js +126 -2
- package/dist/pipeline/coding-aggregate.js +136 -0
- package/dist/pipeline/coding-build.js +118 -4
- package/dist/pipeline/coding-coaching.js +173 -14
- package/dist/pipeline/coding-day-digest.js +109 -0
- package/dist/pipeline/coding-delegation.js +189 -17
- package/dist/pipeline/coding-expertise.js +126 -2
- package/dist/pipeline/coding-flow.js +136 -0
- package/dist/pipeline/coding-focus.js +136 -0
- package/dist/pipeline/coding-frontier-detail.js +126 -2
- package/dist/pipeline/coding-gap-dist.js +136 -0
- package/dist/pipeline/coding-gap.js +173 -14
- package/dist/pipeline/coding-grade.js +132 -2
- package/dist/pipeline/coding-nutshell.js +159 -150
- package/dist/pipeline/coding-walkthrough.js +144 -0
- package/dist/web/app.js +9 -10
- package/package.json +1 -1
|
@@ -85,6 +85,97 @@ function isHarnessEntry(e) {
|
|
|
85
85
|
return e.isMeta === true || e.isCompactSummary === true;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
// ../coding-core/dist/codex.js
|
|
89
|
+
var SYSTEM_REMINDER_RE2 = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
90
|
+
var CODEX_INJECTED_TEXT_RE = /^\s*(<(environment_context|user_instructions|ENVIRONMENT_CONTEXT|turn_aborted)\b|#\s*AGENTS\.md instructions\b)/;
|
|
91
|
+
function isCodexInjectedUserText(text) {
|
|
92
|
+
return CODEX_INJECTED_TEXT_RE.test(text);
|
|
93
|
+
}
|
|
94
|
+
function codexContentText(content) {
|
|
95
|
+
if (typeof content === "string")
|
|
96
|
+
return content;
|
|
97
|
+
if (!Array.isArray(content))
|
|
98
|
+
return "";
|
|
99
|
+
const parts = [];
|
|
100
|
+
for (const item of content) {
|
|
101
|
+
if (typeof item === "string") {
|
|
102
|
+
parts.push(item);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (item && typeof item === "object") {
|
|
106
|
+
const it = item;
|
|
107
|
+
if (it.type === "input_text" || it.type === "output_text" || it.type === "text")
|
|
108
|
+
parts.push(it.text || "");
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return parts.join("\n");
|
|
112
|
+
}
|
|
113
|
+
function sniffCodexRaw(raw) {
|
|
114
|
+
let checked = 0;
|
|
115
|
+
for (const lineRaw of raw.split("\n")) {
|
|
116
|
+
const line = lineRaw.trim();
|
|
117
|
+
if (!line)
|
|
118
|
+
continue;
|
|
119
|
+
if (checked++ >= 5)
|
|
120
|
+
break;
|
|
121
|
+
try {
|
|
122
|
+
const e = JSON.parse(line);
|
|
123
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "turn_context")
|
|
124
|
+
return true;
|
|
125
|
+
if (e.type === "user" || e.type === "assistant" || e.type === "summary" || e.type === "queue-operation")
|
|
126
|
+
return false;
|
|
127
|
+
} catch {
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
var EXIT_CODE_RE = /(?:Process exited with code|Exit code:)\s+(\d+)/;
|
|
133
|
+
function extractCodexEvents(raw) {
|
|
134
|
+
const out = [];
|
|
135
|
+
for (const lineRaw of raw.split("\n")) {
|
|
136
|
+
const line = lineRaw.trim();
|
|
137
|
+
if (!line)
|
|
138
|
+
continue;
|
|
139
|
+
let e;
|
|
140
|
+
try {
|
|
141
|
+
e = JSON.parse(line);
|
|
142
|
+
} catch {
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (e.type !== "response_item")
|
|
146
|
+
continue;
|
|
147
|
+
const p = e.payload ?? {};
|
|
148
|
+
const t = typeof e.timestamp === "string" ? Date.parse(e.timestamp) : NaN;
|
|
149
|
+
const ptype = p.type;
|
|
150
|
+
if (ptype === "message") {
|
|
151
|
+
const role = p.role;
|
|
152
|
+
const text = codexContentText(p.content).replace(SYSTEM_REMINDER_RE2, "").trim();
|
|
153
|
+
if (!text)
|
|
154
|
+
continue;
|
|
155
|
+
if (role === "user") {
|
|
156
|
+
if (isCodexInjectedUserText(text))
|
|
157
|
+
continue;
|
|
158
|
+
out.push({ kind: "user", t, text });
|
|
159
|
+
} else if (role === "assistant") {
|
|
160
|
+
out.push({ kind: "assistant", t, text });
|
|
161
|
+
}
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
if (ptype === "function_call" || ptype === "custom_tool_call" || ptype === "local_shell_call") {
|
|
165
|
+
const name = typeof p.name === "string" && p.name ? p.name : ptype === "local_shell_call" ? "shell" : String(ptype);
|
|
166
|
+
const input = typeof p.arguments === "string" ? p.arguments : typeof p.input === "string" ? p.input : "";
|
|
167
|
+
out.push({ kind: "tool", t, name, input });
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
170
|
+
if (ptype === "function_call_output" || ptype === "custom_tool_call_output") {
|
|
171
|
+
const output = typeof p.output === "string" ? p.output : "";
|
|
172
|
+
const m = EXIT_CODE_RE.exec(output.slice(0, 200));
|
|
173
|
+
out.push({ kind: "tool_output", t, output, isError: !!m && m[1] !== "0" });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return out;
|
|
177
|
+
}
|
|
178
|
+
|
|
88
179
|
// ../coding-core/dist/messages.js
|
|
89
180
|
function extractText(content) {
|
|
90
181
|
if (typeof content === "string")
|
|
@@ -96,6 +187,49 @@ function extractText(content) {
|
|
|
96
187
|
function isToolResultOnly(content) {
|
|
97
188
|
return Array.isArray(content) && content.length > 0 && content.every((c) => c && typeof c === "object" && c.type === "tool_result");
|
|
98
189
|
}
|
|
190
|
+
function extractMessagesCodex(raw) {
|
|
191
|
+
const out = [];
|
|
192
|
+
let aiProse = [];
|
|
193
|
+
let aiTools = {};
|
|
194
|
+
let aiStart = 0;
|
|
195
|
+
const flushAI = () => {
|
|
196
|
+
if (!aiProse.length && Object.keys(aiTools).length === 0)
|
|
197
|
+
return;
|
|
198
|
+
const toolStr = Object.entries(aiTools).map(([n, c]) => c > 1 ? `${n}\xD7${c}` : n).join(", ");
|
|
199
|
+
let prose = aiProse.join(" ").replace(/\s+/g, " ").trim();
|
|
200
|
+
if (prose.length > 2200)
|
|
201
|
+
prose = prose.slice(0, 2200) + "\u2026";
|
|
202
|
+
let text = prose;
|
|
203
|
+
if (toolStr)
|
|
204
|
+
text = (text ? text + " " : "") + `[ran ${toolStr}]`;
|
|
205
|
+
if (text)
|
|
206
|
+
out.push({ t: aiStart, role: "ai", text });
|
|
207
|
+
aiProse = [];
|
|
208
|
+
aiTools = {};
|
|
209
|
+
};
|
|
210
|
+
for (const ev of extractCodexEvents(raw)) {
|
|
211
|
+
if (!Number.isFinite(ev.t))
|
|
212
|
+
continue;
|
|
213
|
+
if (ev.kind === "user") {
|
|
214
|
+
const text = humanTypedText(ev.text);
|
|
215
|
+
if (text) {
|
|
216
|
+
flushAI();
|
|
217
|
+
out.push({ t: ev.t, role: "user", text });
|
|
218
|
+
}
|
|
219
|
+
} else if (ev.kind === "assistant") {
|
|
220
|
+
if (!aiProse.length && Object.keys(aiTools).length === 0)
|
|
221
|
+
aiStart = ev.t;
|
|
222
|
+
if (ev.text)
|
|
223
|
+
aiProse.push(ev.text);
|
|
224
|
+
} else if (ev.kind === "tool") {
|
|
225
|
+
if (!aiProse.length && Object.keys(aiTools).length === 0)
|
|
226
|
+
aiStart = ev.t;
|
|
227
|
+
aiTools[ev.name] = (aiTools[ev.name] ?? 0) + 1;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
flushAI();
|
|
231
|
+
return out.sort((a, b) => a.t - b.t);
|
|
232
|
+
}
|
|
99
233
|
async function extractMessages(file) {
|
|
100
234
|
let raw;
|
|
101
235
|
try {
|
|
@@ -103,6 +237,8 @@ async function extractMessages(file) {
|
|
|
103
237
|
} catch {
|
|
104
238
|
return [];
|
|
105
239
|
}
|
|
240
|
+
if (sniffCodexRaw(raw))
|
|
241
|
+
return extractMessagesCodex(raw);
|
|
106
242
|
const out = [];
|
|
107
243
|
const userTexts = /* @__PURE__ */ new Set();
|
|
108
244
|
const seenUuids = /* @__PURE__ */ new Set();
|
|
@@ -15811,6 +15811,97 @@ function isHarnessEntry(e) {
|
|
|
15811
15811
|
return e.isMeta === true || e.isCompactSummary === true;
|
|
15812
15812
|
}
|
|
15813
15813
|
|
|
15814
|
+
// ../coding-core/dist/codex.js
|
|
15815
|
+
var SYSTEM_REMINDER_RE2 = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
15816
|
+
var CODEX_INJECTED_TEXT_RE = /^\s*(<(environment_context|user_instructions|ENVIRONMENT_CONTEXT|turn_aborted)\b|#\s*AGENTS\.md instructions\b)/;
|
|
15817
|
+
function isCodexInjectedUserText(text2) {
|
|
15818
|
+
return CODEX_INJECTED_TEXT_RE.test(text2);
|
|
15819
|
+
}
|
|
15820
|
+
function codexContentText(content) {
|
|
15821
|
+
if (typeof content === "string")
|
|
15822
|
+
return content;
|
|
15823
|
+
if (!Array.isArray(content))
|
|
15824
|
+
return "";
|
|
15825
|
+
const parts = [];
|
|
15826
|
+
for (const item of content) {
|
|
15827
|
+
if (typeof item === "string") {
|
|
15828
|
+
parts.push(item);
|
|
15829
|
+
continue;
|
|
15830
|
+
}
|
|
15831
|
+
if (item && typeof item === "object") {
|
|
15832
|
+
const it = item;
|
|
15833
|
+
if (it.type === "input_text" || it.type === "output_text" || it.type === "text")
|
|
15834
|
+
parts.push(it.text || "");
|
|
15835
|
+
}
|
|
15836
|
+
}
|
|
15837
|
+
return parts.join("\n");
|
|
15838
|
+
}
|
|
15839
|
+
function sniffCodexRaw(raw) {
|
|
15840
|
+
let checked = 0;
|
|
15841
|
+
for (const lineRaw of raw.split("\n")) {
|
|
15842
|
+
const line = lineRaw.trim();
|
|
15843
|
+
if (!line)
|
|
15844
|
+
continue;
|
|
15845
|
+
if (checked++ >= 5)
|
|
15846
|
+
break;
|
|
15847
|
+
try {
|
|
15848
|
+
const e = JSON.parse(line);
|
|
15849
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "turn_context")
|
|
15850
|
+
return true;
|
|
15851
|
+
if (e.type === "user" || e.type === "assistant" || e.type === "summary" || e.type === "queue-operation")
|
|
15852
|
+
return false;
|
|
15853
|
+
} catch {
|
|
15854
|
+
}
|
|
15855
|
+
}
|
|
15856
|
+
return false;
|
|
15857
|
+
}
|
|
15858
|
+
var EXIT_CODE_RE = /(?:Process exited with code|Exit code:)\s+(\d+)/;
|
|
15859
|
+
function extractCodexEvents(raw) {
|
|
15860
|
+
const out = [];
|
|
15861
|
+
for (const lineRaw of raw.split("\n")) {
|
|
15862
|
+
const line = lineRaw.trim();
|
|
15863
|
+
if (!line)
|
|
15864
|
+
continue;
|
|
15865
|
+
let e;
|
|
15866
|
+
try {
|
|
15867
|
+
e = JSON.parse(line);
|
|
15868
|
+
} catch {
|
|
15869
|
+
continue;
|
|
15870
|
+
}
|
|
15871
|
+
if (e.type !== "response_item")
|
|
15872
|
+
continue;
|
|
15873
|
+
const p = e.payload ?? {};
|
|
15874
|
+
const t = typeof e.timestamp === "string" ? Date.parse(e.timestamp) : NaN;
|
|
15875
|
+
const ptype = p.type;
|
|
15876
|
+
if (ptype === "message") {
|
|
15877
|
+
const role = p.role;
|
|
15878
|
+
const text2 = codexContentText(p.content).replace(SYSTEM_REMINDER_RE2, "").trim();
|
|
15879
|
+
if (!text2)
|
|
15880
|
+
continue;
|
|
15881
|
+
if (role === "user") {
|
|
15882
|
+
if (isCodexInjectedUserText(text2))
|
|
15883
|
+
continue;
|
|
15884
|
+
out.push({ kind: "user", t, text: text2 });
|
|
15885
|
+
} else if (role === "assistant") {
|
|
15886
|
+
out.push({ kind: "assistant", t, text: text2 });
|
|
15887
|
+
}
|
|
15888
|
+
continue;
|
|
15889
|
+
}
|
|
15890
|
+
if (ptype === "function_call" || ptype === "custom_tool_call" || ptype === "local_shell_call") {
|
|
15891
|
+
const name17 = typeof p.name === "string" && p.name ? p.name : ptype === "local_shell_call" ? "shell" : String(ptype);
|
|
15892
|
+
const input = typeof p.arguments === "string" ? p.arguments : typeof p.input === "string" ? p.input : "";
|
|
15893
|
+
out.push({ kind: "tool", t, name: name17, input });
|
|
15894
|
+
continue;
|
|
15895
|
+
}
|
|
15896
|
+
if (ptype === "function_call_output" || ptype === "custom_tool_call_output") {
|
|
15897
|
+
const output = typeof p.output === "string" ? p.output : "";
|
|
15898
|
+
const m = EXIT_CODE_RE.exec(output.slice(0, 200));
|
|
15899
|
+
out.push({ kind: "tool_output", t, output, isError: !!m && m[1] !== "0" });
|
|
15900
|
+
}
|
|
15901
|
+
}
|
|
15902
|
+
return out;
|
|
15903
|
+
}
|
|
15904
|
+
|
|
15814
15905
|
// ../coding-core/dist/messages.js
|
|
15815
15906
|
function extractText(content) {
|
|
15816
15907
|
if (typeof content === "string")
|
|
@@ -15822,6 +15913,22 @@ function extractText(content) {
|
|
|
15822
15913
|
function isToolResultOnly(content) {
|
|
15823
15914
|
return Array.isArray(content) && content.length > 0 && content.every((c) => c && typeof c === "object" && c.type === "tool_result");
|
|
15824
15915
|
}
|
|
15916
|
+
function extractUserMessagesCodex(raw) {
|
|
15917
|
+
const out = [];
|
|
15918
|
+
let lastTs = null;
|
|
15919
|
+
for (const ev of extractCodexEvents(raw)) {
|
|
15920
|
+
const t = ev.t;
|
|
15921
|
+
const gapMs = Number.isFinite(t) && lastTs != null ? t - lastTs : NaN;
|
|
15922
|
+
if (Number.isFinite(t))
|
|
15923
|
+
lastTs = t;
|
|
15924
|
+
if (ev.kind !== "user")
|
|
15925
|
+
continue;
|
|
15926
|
+
const text2 = humanTypedText(ev.text);
|
|
15927
|
+
if (text2 && Number.isFinite(t))
|
|
15928
|
+
out.push({ t, text: text2, gapMs });
|
|
15929
|
+
}
|
|
15930
|
+
return out.sort((a, b) => a.t - b.t);
|
|
15931
|
+
}
|
|
15825
15932
|
async function extractUserMessages(file2) {
|
|
15826
15933
|
let raw;
|
|
15827
15934
|
try {
|
|
@@ -15829,6 +15936,8 @@ async function extractUserMessages(file2) {
|
|
|
15829
15936
|
} catch {
|
|
15830
15937
|
return [];
|
|
15831
15938
|
}
|
|
15939
|
+
if (sniffCodexRaw(raw))
|
|
15940
|
+
return extractUserMessagesCodex(raw);
|
|
15832
15941
|
const out = [];
|
|
15833
15942
|
const userTexts = /* @__PURE__ */ new Set();
|
|
15834
15943
|
const seenUuids = /* @__PURE__ */ new Set();
|
|
@@ -16488,6 +16597,18 @@ function distOf(takes) {
|
|
|
16488
16597
|
const hist = [...h.keys()].sort((a, b) => b - a).map((level) => ({ level, count: h.get(level) }));
|
|
16489
16598
|
return { n, observed, mean, median: median2, hist };
|
|
16490
16599
|
}
|
|
16600
|
+
var phi = (z) => {
|
|
16601
|
+
const t = 1 / (1 + 0.2316419 * Math.abs(z));
|
|
16602
|
+
const d = 0.3989423 * Math.exp(-z * z / 2);
|
|
16603
|
+
let pr = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))));
|
|
16604
|
+
if (z > 0) pr = 1 - pr;
|
|
16605
|
+
return pr;
|
|
16606
|
+
};
|
|
16607
|
+
var lognPct = (x, c) => x == null || !c || x <= 0 ? null : Math.min(99.9, Math.max(0.1, Math.round(phi(Math.log(x / c.median) / c.sigma) * 1e3) / 10));
|
|
16608
|
+
var NO_GRADES_GAP = "no sessions graded for this yet \u2014 this fills in once grading has run";
|
|
16609
|
+
function estimateRowParts(score, nTakes, evidenceGap) {
|
|
16610
|
+
return nTakes === 0 && score == null ? { score: null, gap: NO_GRADES_GAP } : { score, gap: evidenceGap };
|
|
16611
|
+
}
|
|
16491
16612
|
async function compileCodingProfile() {
|
|
16492
16613
|
const allRecords = await readJson(path10.join(CODING_DIR, "sessions.json"), []);
|
|
16493
16614
|
const dupIds = new Set(allRecords.filter((s) => s.duplicateOf).map((s) => s.sessionId));
|
|
@@ -16742,15 +16863,11 @@ async function compileCodingProfile() {
|
|
|
16742
16863
|
const unb = ceilOf("unblocking");
|
|
16743
16864
|
const agency = out2 != null && unb != null ? Math.round((out2 + unb) / 2) : out2 ?? unb;
|
|
16744
16865
|
const sRow = (label, score, gap2) => ({ label, score, percentile: pctOf(score), gap: gap2 });
|
|
16745
|
-
const
|
|
16746
|
-
|
|
16747
|
-
|
|
16748
|
-
const d = 0.3989423 * Math.exp(-z * z / 2);
|
|
16749
|
-
let pr = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))));
|
|
16750
|
-
if (z > 0) pr = 1 - pr;
|
|
16751
|
-
return pr;
|
|
16866
|
+
const est = (label, score, key, evidenceGap) => {
|
|
16867
|
+
const e = estimateRowParts(score, nTk(key), evidenceGap);
|
|
16868
|
+
return sRow(label, e.score, e.gap);
|
|
16752
16869
|
};
|
|
16753
|
-
const
|
|
16870
|
+
const tiers = DEFAULT_CALIBRATION.codingTiers;
|
|
16754
16871
|
const avgWords = aggregate?.throughput?.avgWordsPerDay ?? null;
|
|
16755
16872
|
const pushPctl = lognPct(avgWords, tiers?.push);
|
|
16756
16873
|
const focusPctl = lognPct(flowPctDay, tiers?.focus);
|
|
@@ -16769,10 +16886,16 @@ async function compileCodingProfile() {
|
|
|
16769
16886
|
null,
|
|
16770
16887
|
workDayHrs != null ? `~${workDayHrs}h of active building on a typical work day \u2014 engaged builders average ~3h, and sustaining 7h+ day after day, week after week, is top-1% territory` : "not enough dated activity yet to measure a typical work day"
|
|
16771
16888
|
),
|
|
16889
|
+
// The copy FOLLOWS the person's actual orchestration band (orchPctl above) —
|
|
16890
|
+
// a hardcoded "single biggest speed-up still on the table" once shipped to a
|
|
16891
|
+
// user at orchPctl 99.5 (18 agents peak, 48% of hours at 2+), flatly
|
|
16892
|
+
// contradicting the gap section's "already strong" on the same page
|
|
16893
|
+
// (2026-07-17). The still-on-the-table claim is reserved for genuinely low
|
|
16894
|
+
// orchestration.
|
|
16772
16895
|
sRow(
|
|
16773
16896
|
"Your AI parallelism",
|
|
16774
16897
|
ceilOf("delegation"),
|
|
16775
|
-
soloPct != null ? `~${soloPct}% of your hours you drive a single agent (${Math.round(soloMin / 60)}h solo vs ${Math.round(parMin / 60)}h with 2+ live) \u2014 the Claude Code lead keeps 10\u201315 going at once, so this is the single biggest speed-up still on the table` : "you brief the AI sharply, but mostly drive one agent at a time \u2014 the top run 10\u201315 in parallel, a several-fold speed-up"
|
|
16898
|
+
soloPct != null ? orchPctl >= 97 ? `you already run parallel at the frontier \u2014 ${Math.round(parMin / 60)}h of your hours with 2+ agents live vs ${Math.round(soloMin / 60)}h solo; the Claude Code lead keeps 10\u201315 going at once, and you work the same way` : `~${soloPct}% of your hours you drive a single agent (${Math.round(soloMin / 60)}h solo vs ${Math.round(parMin / 60)}h with 2+ live) \u2014 the Claude Code lead keeps 10\u201315 going at once, so this is the single biggest speed-up still on the table` : "you brief the AI sharply, but mostly drive one agent at a time \u2014 the top run 10\u201315 in parallel, a several-fold speed-up"
|
|
16776
16899
|
),
|
|
16777
16900
|
// The evidence line cites the RANKING metric (flow as a share of the work
|
|
16778
16901
|
// day), never absolute hours — a friend with MORE flow hours over a longer
|
|
@@ -16788,22 +16911,25 @@ async function compileCodingProfile() {
|
|
|
16788
16911
|
// The four estimates LEAD with a concrete moment from their own sessions, then an
|
|
16789
16912
|
// HONEST caveat — how many graded sessions back the read. No invented weakness, no
|
|
16790
16913
|
// claim about what they "didn't" do (false specifics churn a real user); it's an estimate.
|
|
16791
|
-
|
|
16914
|
+
est(
|
|
16792
16915
|
"Agency",
|
|
16793
16916
|
agency,
|
|
16917
|
+
"outcomes",
|
|
16794
16918
|
`${proud("outcomes", "you ship working systems and unblock yourself")} \u2014 one of ${nTk("outcomes")} sessions behind this read; a strong, consistent signal, still an estimate from your logs`
|
|
16795
16919
|
),
|
|
16796
16920
|
// Judgement/Taste use the SAME numbers their ability cards prove (aggregate
|
|
16797
16921
|
// peak-with-consistency over near-peak instances) — a chip saying "top 1%"
|
|
16798
16922
|
// above a card proving 10/Top 0.1% with ten receipts reads as a bug.
|
|
16799
|
-
|
|
16923
|
+
est(
|
|
16800
16924
|
"Judgement",
|
|
16801
16925
|
aggregate?.ability?.abstraction?.score ?? ceilOf("generative"),
|
|
16926
|
+
"generative",
|
|
16802
16927
|
`${proud("generative", "you make sharp, non-obvious calls on what actually matters")} \u2014 one of ${nTk("generative")} sessions we've graded for this; clear so far, not yet a final verdict`
|
|
16803
16928
|
),
|
|
16804
|
-
|
|
16929
|
+
est(
|
|
16805
16930
|
"Taste",
|
|
16806
16931
|
aggregate?.ability?.taste?.score ?? ceilOf("taste"),
|
|
16932
|
+
"taste",
|
|
16807
16933
|
`${proud("taste", "you hold a high, opinionated quality bar")} \u2014 one of ${nTk("taste")} sessions behind this read; a real signal, still an estimate`
|
|
16808
16934
|
)
|
|
16809
16935
|
// Expertise: NO chip — the scored criterion is deprecated; the real judge is
|
|
@@ -16860,7 +16986,7 @@ import path12 from "path";
|
|
|
16860
16986
|
|
|
16861
16987
|
// ../../lib/agents/coding/materialize.ts
|
|
16862
16988
|
import { promises as fs8 } from "fs";
|
|
16863
|
-
var
|
|
16989
|
+
var SYSTEM_REMINDER_RE3 = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
16864
16990
|
var INJECT_GIST = 120;
|
|
16865
16991
|
var INJECTED = [
|
|
16866
16992
|
{ re: /<task-notification>[\s\S]*?<\/task-notification>/g, label: "bg task", gistRe: /<summary>([\s\S]*?)<\/summary>/ },
|
|
@@ -16880,7 +17006,7 @@ function collapseInjected(input) {
|
|
|
16880
17006
|
return "";
|
|
16881
17007
|
});
|
|
16882
17008
|
}
|
|
16883
|
-
return { text: text2.replace(
|
|
17009
|
+
return { text: text2.replace(SYSTEM_REMINDER_RE3, "").trim(), markers };
|
|
16884
17010
|
}
|
|
16885
17011
|
function extractText2(content) {
|
|
16886
17012
|
if (typeof content === "string") return content;
|
|
@@ -16897,6 +17023,38 @@ function isToolResultOnly2(content) {
|
|
|
16897
17023
|
return Array.isArray(content) && content.length > 0 && content.every((c) => c && typeof c === "object" && c.type === "tool_result");
|
|
16898
17024
|
}
|
|
16899
17025
|
var AI_GIST = 160;
|
|
17026
|
+
function materializeCodex(raw) {
|
|
17027
|
+
const lines = [];
|
|
17028
|
+
let humanTurns = 0, humanChars = 0;
|
|
17029
|
+
let aiLastProse = "";
|
|
17030
|
+
const aiTools = /* @__PURE__ */ new Map();
|
|
17031
|
+
const flushAI = () => {
|
|
17032
|
+
if (!aiLastProse && aiTools.size === 0) return;
|
|
17033
|
+
const tools = [...aiTools.entries()].map(([n, c]) => c > 1 ? `${n}\xD7${c}` : n).join(", ");
|
|
17034
|
+
const gist = aiLastProse ? aiLastProse.length > AI_GIST ? aiLastProse.slice(0, AI_GIST) + "\u2026" : aiLastProse : "";
|
|
17035
|
+
const bits = [];
|
|
17036
|
+
if (gist) bits.push(gist);
|
|
17037
|
+
if (tools) bits.push(`ran ${tools}`);
|
|
17038
|
+
lines.push(`[AI: ${bits.join(" \xB7 ")}]`);
|
|
17039
|
+
aiLastProse = "";
|
|
17040
|
+
aiTools.clear();
|
|
17041
|
+
};
|
|
17042
|
+
for (const ev of extractCodexEvents(raw)) {
|
|
17043
|
+
if (ev.kind === "user") {
|
|
17044
|
+
flushAI();
|
|
17045
|
+
humanTurns++;
|
|
17046
|
+
humanChars += ev.text.length;
|
|
17047
|
+
lines.push(`>> ${ev.text}`);
|
|
17048
|
+
} else if (ev.kind === "assistant") {
|
|
17049
|
+
const prose = ev.text.replace(/\s+/g, " ").trim();
|
|
17050
|
+
if (prose) aiLastProse = prose;
|
|
17051
|
+
} else if (ev.kind === "tool") {
|
|
17052
|
+
aiTools.set(ev.name, (aiTools.get(ev.name) ?? 0) + 1);
|
|
17053
|
+
}
|
|
17054
|
+
}
|
|
17055
|
+
flushAI();
|
|
17056
|
+
return { text: lines.join("\n"), humanTurns, humanChars };
|
|
17057
|
+
}
|
|
16900
17058
|
async function materializeSession(file2) {
|
|
16901
17059
|
let raw;
|
|
16902
17060
|
try {
|
|
@@ -16904,6 +17062,7 @@ async function materializeSession(file2) {
|
|
|
16904
17062
|
} catch {
|
|
16905
17063
|
return null;
|
|
16906
17064
|
}
|
|
17065
|
+
if (sniffCodexRaw(raw)) return materializeCodex(raw);
|
|
16907
17066
|
const lines = [];
|
|
16908
17067
|
let humanTurns = 0, humanChars = 0;
|
|
16909
17068
|
const pendingQueued = [];
|
|
@@ -15863,8 +15863,99 @@ function isHarnessEntry(e) {
|
|
|
15863
15863
|
return e.isMeta === true || e.isCompactSummary === true;
|
|
15864
15864
|
}
|
|
15865
15865
|
|
|
15866
|
-
//
|
|
15866
|
+
// ../coding-core/dist/codex.js
|
|
15867
15867
|
var SYSTEM_REMINDER_RE = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
15868
|
+
var CODEX_INJECTED_TEXT_RE = /^\s*(<(environment_context|user_instructions|ENVIRONMENT_CONTEXT|turn_aborted)\b|#\s*AGENTS\.md instructions\b)/;
|
|
15869
|
+
function isCodexInjectedUserText(text2) {
|
|
15870
|
+
return CODEX_INJECTED_TEXT_RE.test(text2);
|
|
15871
|
+
}
|
|
15872
|
+
function codexContentText(content) {
|
|
15873
|
+
if (typeof content === "string")
|
|
15874
|
+
return content;
|
|
15875
|
+
if (!Array.isArray(content))
|
|
15876
|
+
return "";
|
|
15877
|
+
const parts = [];
|
|
15878
|
+
for (const item of content) {
|
|
15879
|
+
if (typeof item === "string") {
|
|
15880
|
+
parts.push(item);
|
|
15881
|
+
continue;
|
|
15882
|
+
}
|
|
15883
|
+
if (item && typeof item === "object") {
|
|
15884
|
+
const it = item;
|
|
15885
|
+
if (it.type === "input_text" || it.type === "output_text" || it.type === "text")
|
|
15886
|
+
parts.push(it.text || "");
|
|
15887
|
+
}
|
|
15888
|
+
}
|
|
15889
|
+
return parts.join("\n");
|
|
15890
|
+
}
|
|
15891
|
+
function sniffCodexRaw(raw) {
|
|
15892
|
+
let checked = 0;
|
|
15893
|
+
for (const lineRaw of raw.split("\n")) {
|
|
15894
|
+
const line = lineRaw.trim();
|
|
15895
|
+
if (!line)
|
|
15896
|
+
continue;
|
|
15897
|
+
if (checked++ >= 5)
|
|
15898
|
+
break;
|
|
15899
|
+
try {
|
|
15900
|
+
const e = JSON.parse(line);
|
|
15901
|
+
if (e.type === "session_meta" || e.type === "response_item" || e.type === "turn_context")
|
|
15902
|
+
return true;
|
|
15903
|
+
if (e.type === "user" || e.type === "assistant" || e.type === "summary" || e.type === "queue-operation")
|
|
15904
|
+
return false;
|
|
15905
|
+
} catch {
|
|
15906
|
+
}
|
|
15907
|
+
}
|
|
15908
|
+
return false;
|
|
15909
|
+
}
|
|
15910
|
+
var EXIT_CODE_RE = /(?:Process exited with code|Exit code:)\s+(\d+)/;
|
|
15911
|
+
function extractCodexEvents(raw) {
|
|
15912
|
+
const out = [];
|
|
15913
|
+
for (const lineRaw of raw.split("\n")) {
|
|
15914
|
+
const line = lineRaw.trim();
|
|
15915
|
+
if (!line)
|
|
15916
|
+
continue;
|
|
15917
|
+
let e;
|
|
15918
|
+
try {
|
|
15919
|
+
e = JSON.parse(line);
|
|
15920
|
+
} catch {
|
|
15921
|
+
continue;
|
|
15922
|
+
}
|
|
15923
|
+
if (e.type !== "response_item")
|
|
15924
|
+
continue;
|
|
15925
|
+
const p = e.payload ?? {};
|
|
15926
|
+
const t = typeof e.timestamp === "string" ? Date.parse(e.timestamp) : NaN;
|
|
15927
|
+
const ptype = p.type;
|
|
15928
|
+
if (ptype === "message") {
|
|
15929
|
+
const role = p.role;
|
|
15930
|
+
const text2 = codexContentText(p.content).replace(SYSTEM_REMINDER_RE, "").trim();
|
|
15931
|
+
if (!text2)
|
|
15932
|
+
continue;
|
|
15933
|
+
if (role === "user") {
|
|
15934
|
+
if (isCodexInjectedUserText(text2))
|
|
15935
|
+
continue;
|
|
15936
|
+
out.push({ kind: "user", t, text: text2 });
|
|
15937
|
+
} else if (role === "assistant") {
|
|
15938
|
+
out.push({ kind: "assistant", t, text: text2 });
|
|
15939
|
+
}
|
|
15940
|
+
continue;
|
|
15941
|
+
}
|
|
15942
|
+
if (ptype === "function_call" || ptype === "custom_tool_call" || ptype === "local_shell_call") {
|
|
15943
|
+
const name17 = typeof p.name === "string" && p.name ? p.name : ptype === "local_shell_call" ? "shell" : String(ptype);
|
|
15944
|
+
const input = typeof p.arguments === "string" ? p.arguments : typeof p.input === "string" ? p.input : "";
|
|
15945
|
+
out.push({ kind: "tool", t, name: name17, input });
|
|
15946
|
+
continue;
|
|
15947
|
+
}
|
|
15948
|
+
if (ptype === "function_call_output" || ptype === "custom_tool_call_output") {
|
|
15949
|
+
const output = typeof p.output === "string" ? p.output : "";
|
|
15950
|
+
const m = EXIT_CODE_RE.exec(output.slice(0, 200));
|
|
15951
|
+
out.push({ kind: "tool_output", t, output, isError: !!m && m[1] !== "0" });
|
|
15952
|
+
}
|
|
15953
|
+
}
|
|
15954
|
+
return out;
|
|
15955
|
+
}
|
|
15956
|
+
|
|
15957
|
+
// ../../lib/agents/coding/materialize.ts
|
|
15958
|
+
var SYSTEM_REMINDER_RE2 = /<system-reminder>[\s\S]*?<\/system-reminder>/g;
|
|
15868
15959
|
var INJECT_GIST = 120;
|
|
15869
15960
|
var INJECTED = [
|
|
15870
15961
|
{ re: /<task-notification>[\s\S]*?<\/task-notification>/g, label: "bg task", gistRe: /<summary>([\s\S]*?)<\/summary>/ },
|
|
@@ -15884,7 +15975,7 @@ function collapseInjected(input) {
|
|
|
15884
15975
|
return "";
|
|
15885
15976
|
});
|
|
15886
15977
|
}
|
|
15887
|
-
return { text: text2.replace(
|
|
15978
|
+
return { text: text2.replace(SYSTEM_REMINDER_RE2, "").trim(), markers };
|
|
15888
15979
|
}
|
|
15889
15980
|
function extractText(content) {
|
|
15890
15981
|
if (typeof content === "string") return content;
|
|
@@ -15901,6 +15992,38 @@ function isToolResultOnly(content) {
|
|
|
15901
15992
|
return Array.isArray(content) && content.length > 0 && content.every((c) => c && typeof c === "object" && c.type === "tool_result");
|
|
15902
15993
|
}
|
|
15903
15994
|
var AI_GIST = 160;
|
|
15995
|
+
function materializeCodex(raw) {
|
|
15996
|
+
const lines = [];
|
|
15997
|
+
let humanTurns = 0, humanChars = 0;
|
|
15998
|
+
let aiLastProse = "";
|
|
15999
|
+
const aiTools = /* @__PURE__ */ new Map();
|
|
16000
|
+
const flushAI = () => {
|
|
16001
|
+
if (!aiLastProse && aiTools.size === 0) return;
|
|
16002
|
+
const tools = [...aiTools.entries()].map(([n, c]) => c > 1 ? `${n}\xD7${c}` : n).join(", ");
|
|
16003
|
+
const gist = aiLastProse ? aiLastProse.length > AI_GIST ? aiLastProse.slice(0, AI_GIST) + "\u2026" : aiLastProse : "";
|
|
16004
|
+
const bits = [];
|
|
16005
|
+
if (gist) bits.push(gist);
|
|
16006
|
+
if (tools) bits.push(`ran ${tools}`);
|
|
16007
|
+
lines.push(`[AI: ${bits.join(" \xB7 ")}]`);
|
|
16008
|
+
aiLastProse = "";
|
|
16009
|
+
aiTools.clear();
|
|
16010
|
+
};
|
|
16011
|
+
for (const ev of extractCodexEvents(raw)) {
|
|
16012
|
+
if (ev.kind === "user") {
|
|
16013
|
+
flushAI();
|
|
16014
|
+
humanTurns++;
|
|
16015
|
+
humanChars += ev.text.length;
|
|
16016
|
+
lines.push(`>> ${ev.text}`);
|
|
16017
|
+
} else if (ev.kind === "assistant") {
|
|
16018
|
+
const prose = ev.text.replace(/\s+/g, " ").trim();
|
|
16019
|
+
if (prose) aiLastProse = prose;
|
|
16020
|
+
} else if (ev.kind === "tool") {
|
|
16021
|
+
aiTools.set(ev.name, (aiTools.get(ev.name) ?? 0) + 1);
|
|
16022
|
+
}
|
|
16023
|
+
}
|
|
16024
|
+
flushAI();
|
|
16025
|
+
return { text: lines.join("\n"), humanTurns, humanChars };
|
|
16026
|
+
}
|
|
15904
16027
|
async function materializeSession(file2) {
|
|
15905
16028
|
let raw;
|
|
15906
16029
|
try {
|
|
@@ -15908,6 +16031,7 @@ async function materializeSession(file2) {
|
|
|
15908
16031
|
} catch {
|
|
15909
16032
|
return null;
|
|
15910
16033
|
}
|
|
16034
|
+
if (sniffCodexRaw(raw)) return materializeCodex(raw);
|
|
15911
16035
|
const lines = [];
|
|
15912
16036
|
let humanTurns = 0, humanChars = 0;
|
|
15913
16037
|
const pendingQueued = [];
|
|
@@ -16440,6 +16564,12 @@ async function main() {
|
|
|
16440
16564
|
}
|
|
16441
16565
|
})));
|
|
16442
16566
|
endRun();
|
|
16567
|
+
if (picked.length > 0 && doneN === 0) {
|
|
16568
|
+
console.error(`
|
|
16569
|
+
[coding-grade] FATAL: 0 of ${picked.length} attempted sessions produced a grade \u2014 every attempt failed or returned no usable grade.`);
|
|
16570
|
+
console.error(`[coding-grade] The report must NOT pretend to be graded. Likely cause: unreadable or unsupported transcript files (a log format the reader cannot parse). Fix ingestion, then re-run this stage.`);
|
|
16571
|
+
process.exit(1);
|
|
16572
|
+
}
|
|
16443
16573
|
console.log(`[coding-grade] wrote \u2192 .data/coding/grades/`);
|
|
16444
16574
|
}
|
|
16445
16575
|
main().catch((e) => {
|