polymath-society 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3132 -1694
- package/dist/engine/ingest-export.js +14 -8
- package/dist/engine/public-report.js +14 -8
- package/dist/index.js +1506 -57
- package/dist/pipeline/coding-agglomerate.js +439 -26
- package/dist/pipeline/coding-aggregate.js +313 -9
- package/dist/pipeline/coding-build.js +361 -8
- package/dist/pipeline/coding-coaching.js +274 -22
- package/dist/pipeline/coding-day-digest.js +351 -33
- package/dist/pipeline/coding-delegation.js +369 -22
- package/dist/pipeline/coding-expertise.js +456 -49
- package/dist/pipeline/coding-flow.js +308 -4
- package/dist/pipeline/coding-focus.js +16306 -0
- package/dist/pipeline/coding-frontier-detail.js +444 -26
- package/dist/pipeline/coding-frontier.js +356 -1
- package/dist/pipeline/coding-gap-dist.js +308 -4
- package/dist/pipeline/coding-gap.js +141 -40
- package/dist/pipeline/coding-grade.js +216 -26
- package/dist/pipeline/coding-nutshell.js +347 -17
- package/dist/pipeline/coding-projects.js +387 -7
- package/dist/pipeline/coding-walkthrough.js +324 -19
- package/dist/web/app.js +539 -113
- package/dist/web/styles.css +1 -1
- package/package.json +2 -2
|
@@ -135,7 +135,7 @@ var require_secure_json_parse = __commonJS({
|
|
|
135
135
|
|
|
136
136
|
// ../../scripts/coding-grade.mts
|
|
137
137
|
import { promises as fs6 } from "fs";
|
|
138
|
-
import
|
|
138
|
+
import path11 from "path";
|
|
139
139
|
|
|
140
140
|
// ../../lib/agents/coding/grade.ts
|
|
141
141
|
import { promises as fs5 } from "fs";
|
|
@@ -2155,8 +2155,8 @@ function getErrorMap() {
|
|
|
2155
2155
|
|
|
2156
2156
|
// ../../node_modules/zod/v3/helpers/parseUtil.js
|
|
2157
2157
|
var makeIssue = (params) => {
|
|
2158
|
-
const { data, path:
|
|
2159
|
-
const fullPath = [...
|
|
2158
|
+
const { data, path: path12, errorMaps, issueData } = params;
|
|
2159
|
+
const fullPath = [...path12, ...issueData.path || []];
|
|
2160
2160
|
const fullIssue = {
|
|
2161
2161
|
...issueData,
|
|
2162
2162
|
path: fullPath
|
|
@@ -2272,11 +2272,11 @@ var errorUtil;
|
|
|
2272
2272
|
|
|
2273
2273
|
// ../../node_modules/zod/v3/types.js
|
|
2274
2274
|
var ParseInputLazyPath = class {
|
|
2275
|
-
constructor(parent, value,
|
|
2275
|
+
constructor(parent, value, path12, key) {
|
|
2276
2276
|
this._cachedPath = [];
|
|
2277
2277
|
this.parent = parent;
|
|
2278
2278
|
this.data = value;
|
|
2279
|
-
this._path =
|
|
2279
|
+
this._path = path12;
|
|
2280
2280
|
this._key = key;
|
|
2281
2281
|
}
|
|
2282
2282
|
get path() {
|
|
@@ -15063,39 +15063,39 @@ function createOpenAI(options = {}) {
|
|
|
15063
15063
|
});
|
|
15064
15064
|
const createChatModel = (modelId, settings = {}) => new OpenAIChatLanguageModel(modelId, settings, {
|
|
15065
15065
|
provider: `${providerName}.chat`,
|
|
15066
|
-
url: ({ path:
|
|
15066
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15067
15067
|
headers: getHeaders,
|
|
15068
15068
|
compatibility,
|
|
15069
15069
|
fetch: options.fetch
|
|
15070
15070
|
});
|
|
15071
15071
|
const createCompletionModel = (modelId, settings = {}) => new OpenAICompletionLanguageModel(modelId, settings, {
|
|
15072
15072
|
provider: `${providerName}.completion`,
|
|
15073
|
-
url: ({ path:
|
|
15073
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15074
15074
|
headers: getHeaders,
|
|
15075
15075
|
compatibility,
|
|
15076
15076
|
fetch: options.fetch
|
|
15077
15077
|
});
|
|
15078
15078
|
const createEmbeddingModel = (modelId, settings = {}) => new OpenAIEmbeddingModel(modelId, settings, {
|
|
15079
15079
|
provider: `${providerName}.embedding`,
|
|
15080
|
-
url: ({ path:
|
|
15080
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15081
15081
|
headers: getHeaders,
|
|
15082
15082
|
fetch: options.fetch
|
|
15083
15083
|
});
|
|
15084
15084
|
const createImageModel = (modelId, settings = {}) => new OpenAIImageModel(modelId, settings, {
|
|
15085
15085
|
provider: `${providerName}.image`,
|
|
15086
|
-
url: ({ path:
|
|
15086
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15087
15087
|
headers: getHeaders,
|
|
15088
15088
|
fetch: options.fetch
|
|
15089
15089
|
});
|
|
15090
15090
|
const createTranscriptionModel = (modelId) => new OpenAITranscriptionModel(modelId, {
|
|
15091
15091
|
provider: `${providerName}.transcription`,
|
|
15092
|
-
url: ({ path:
|
|
15092
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15093
15093
|
headers: getHeaders,
|
|
15094
15094
|
fetch: options.fetch
|
|
15095
15095
|
});
|
|
15096
15096
|
const createSpeechModel = (modelId) => new OpenAISpeechModel(modelId, {
|
|
15097
15097
|
provider: `${providerName}.speech`,
|
|
15098
|
-
url: ({ path:
|
|
15098
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15099
15099
|
headers: getHeaders,
|
|
15100
15100
|
fetch: options.fetch
|
|
15101
15101
|
});
|
|
@@ -15116,7 +15116,7 @@ function createOpenAI(options = {}) {
|
|
|
15116
15116
|
const createResponsesModel = (modelId) => {
|
|
15117
15117
|
return new OpenAIResponsesLanguageModel(modelId, {
|
|
15118
15118
|
provider: `${providerName}.responses`,
|
|
15119
|
-
url: ({ path:
|
|
15119
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15120
15120
|
headers: getHeaders,
|
|
15121
15121
|
fetch: options.fetch
|
|
15122
15122
|
});
|
|
@@ -15637,6 +15637,9 @@ function invokeAgent(inv, opts = {}) {
|
|
|
15637
15637
|
return withLimitRetry(once, { label });
|
|
15638
15638
|
}
|
|
15639
15639
|
|
|
15640
|
+
// ../../lib/agents/coding/grade-calibration.ts
|
|
15641
|
+
var GRADE_CALIBRATION = "\n\nSCORE CALIBRATION \u2014 graders systematically COMPRESS to the middle: real peaks get hedged down to 6-7, and routine sessions get polite 5-6s instead of null. Both are failures; fight them explicitly. Work each criterion in EXACTLY this order:\nSTEP 1 \xB7 EVIDENCE GATE \u2014 find the 1-3 verbatim '>> ' quotes that bear on the criterion. INCLUDE a criterion whenever genuine evidence is present (do not MISS quiet evidence \u2014 check every thread of the session, not just the dominant one). But with NO quotable moment the score is null, full stop: a session can be long, competent, and productive while being null on most criteria. Routine hand-holding earns null, never a 5 \u2014 and null is never a deduction.\nSTEP 2 \xB7 LADDER MATCH \u2014 with quotes in hand, read the criterion's rungs and pick the ONE rung whose marker best describes the quoted moment; name that rung in 'why'. AWARD THE RUNG THE EVIDENCE MATCHES: if your own 'why' describes rung N's criteria, give rung N \u2014 never deny it for lacking the NEXT rung's criteria.\nSTEP 3 \xB7 THE SINGLE BEST MOMENT IS THE SCORE \u2014 set the score by the strongest quoted moment, as if it were the whole session. NEVER average a peak down because the surrounding session is routine (the high-water rule), and never raise a score for volume, effort, or topic prestige without a matching moment.\nSTEP 4 \xB7 ANTI-HEDGE \u2014 if the moment genuinely matches an L8-10 anchor (an override the AI conceded, a shipped-and-verified capability, an 'oh shit, they're right' call, a first-principles redefinition that held), the score IS 8-10. Hedging a real L9 to a 7 is exactly as wrong as inventing a 6 for nothing. The middle is NOT a safe default: commit high on anchor-matching evidence, or go null.\nSTEP 5 \xB7 'CONSERVATIVE' \u2260 SUBTRACT \u2014 when a criterion's ladder says to grade conservatively, that means DO NOT EXCEED the rung whose marker the evidence matches; it NEVER means subtracting rungs from matched evidence. Evidence that matches a rung's marker gets that rung under a conservative reading too.";
|
|
15642
|
+
|
|
15640
15643
|
// ../../lib/agents/shared/chunk.ts
|
|
15641
15644
|
function chunkText(text2, max) {
|
|
15642
15645
|
if (text2.length <= max) return [text2];
|
|
@@ -15985,14 +15988,28 @@ function rubric() {
|
|
|
15985
15988
|
${(c.rungs ?? []).map((r) => ` ${r.level} \u2014 ${r.marker}`).join("\n")}`
|
|
15986
15989
|
).join("\n\n");
|
|
15987
15990
|
}
|
|
15991
|
+
function rubricCompact() {
|
|
15992
|
+
return GRADED_CRITERIA.map((c) => {
|
|
15993
|
+
const anchors = /ANCHORS?:/.exec(c.read) ? c.read.slice(c.read.search(/ANCHORS?:/)) : "";
|
|
15994
|
+
return `CRITERION "${c.key}" \u2014 ${c.label} [score 1\u201311, or null if not shown]
|
|
15995
|
+
${c.definition}
|
|
15996
|
+
${anchors ? ` ${anchors}
|
|
15997
|
+
` : ""} LADDER:
|
|
15998
|
+
${(c.rungs ?? []).map((r) => ` ${r.level} \u2014 ${r.marker}`).join("\n")}`;
|
|
15999
|
+
}).join("\n\n");
|
|
16000
|
+
}
|
|
16001
|
+
var HAIKU_SCORE_MAP_ID = "haiku+1@2026-07-14";
|
|
16002
|
+
var mapSmallScore = (v) => v != null && v >= 4 ? Math.min(10, v + 1) : v;
|
|
16003
|
+
var HAIKU_CALIBRATION = "\n\nWork each criterion in TWO STEPS, in order:\nSTEP A \u2014 QUOTE the 1-3 strongest verbatim '>> ' lines bearing on the criterion, checking every thread of the session, not just the dominant one. No quotable moment = null (routine hand-holding is null, never a 5; null is never a deduction).\nSTEP B \u2014 score by the SINGLE BEST quoted moment as if it were the whole session: pick the rung whose marker it matches and AWARD THAT RUNG \u2014 never deny rung N for lacking rung N+1, never average a peak down because the rest is routine, never raise for volume or topic prestige. Evidence matching an L8-10 anchor scores 8-10; the middle is not a safe default. Include every criterion genuinely present (don't MISS quiet evidence) \u2014 but rate strength strictly: weak-but-present is low, not mid.";
|
|
15988
16004
|
var SYSTEM = "You grade how a person WORKS from one of their coding-agent sessions (Claude Code). You read mostly THEIR words; the model's code and output are compressed to context and you ignore the mechanical scaffolding. You never flatter: most sessions are routine hand-holding and should score low or null on most criteria. A score is a placement on the ladder backed by VERBATIM quotes of the user's own lines \u2014 never a vibe.\n\nSTANDING RULES (apply to every criterion):\n1. ABSENCE IS NEVER A DEDUCTION. If a criterion had no chance to show, its score is null \u2014 NOT a low. Don't invent a number for routine work.\n2. HIGH-WATER MARK. Taste and abstraction are PEAK measures \u2014 one genuinely brilliant move in a session counts, even surrounded by routine work. Never grade the person at their worst; a lone weak moment among strong ones is a slip, not the score.\n3. LOW-EFFORT-UNTIL-NEEDED IS THE IDEAL. A terse or vague prompt on routine work is correct economy \u2014 never penalize a vague ask, never reward word-count.\n4. DISCOUNT THE LITERAL HARSHNESS OF SELF-TALK. 'you idiot', 'you stupid cunt never again', 'this is retarded' \u2014 aimed at self or AI \u2014 are DRIVE and high standards, not defects. Grade the demonstrated behavior.\n5. PER-CONVERSATION CEILING IS 10. Never output 11 \u2014 that is the aggregator's job (it needs cross-conversation consistency / a stacked day this single session can't show). Scores run 2\u201310.\n6. LOG THE SPIKES IN HIGH DETAIL. When a moment is genuinely good (a flare \u2014 a sharp taste call, an out-reasons-the-tool insight, a real systems override), name it CONCRETELY and quote it. The 'instance' + 'why' are the high-detail record of exactly what was cool and why it placed where it did.";
|
|
15989
|
-
function gradeSystemPrompt() {
|
|
15990
|
-
|
|
16005
|
+
function gradeSystemPrompt(model) {
|
|
16006
|
+
const small = /haiku/i.test(model ?? "");
|
|
16007
|
+
return `${SYSTEM}${small ? HAIKU_CALIBRATION : GRADE_CALIBRATION}
|
|
15991
16008
|
|
|
15992
16009
|
Grade ONE coding session (provided in the user message) along the criteria below. ">> " lines are the USER's own words (the signal). ALL bracketed lines are machine context, never the user: "[AI: \u2026]" is the model working, "[bg task: \u2026]" / "[cmd output: \u2026]" are harness notifications, "[mode \u2192 \u2026]" marks a mode switch (context only \u2014 do not grade or quote them). ">> (queued \u2026)" is the user queuing work while the AI runs.
|
|
15993
16010
|
|
|
15994
16011
|
CRITERIA:
|
|
15995
|
-
${rubric()}
|
|
16012
|
+
${small ? rubricCompact() : rubric()}
|
|
15996
16013
|
|
|
15997
16014
|
RULES:
|
|
15998
16015
|
- Score ONLY what the user's moves genuinely show. Null is the right answer for most criteria in most sessions \u2014 do not force a number. Absence is never a deduction.
|
|
@@ -16058,7 +16075,7 @@ async function gradeSession(rec, opts = { outDir: "" }) {
|
|
|
16058
16075
|
const seg = chunks.length > 1 ? ` This is SEGMENT ${ci + 1} of ${chunks.length} of one longer session \u2014 grade only what THIS segment shows; null for criteria not shown here (segments are merged afterward).` : "";
|
|
16059
16076
|
const run2 = await invokeAgent({
|
|
16060
16077
|
prompt: buildPrompt(rec, chunks[ci], seg),
|
|
16061
|
-
systemPrompt: gradeSystemPrompt(),
|
|
16078
|
+
systemPrompt: gradeSystemPrompt(opts.model),
|
|
16062
16079
|
addDir: sandbox,
|
|
16063
16080
|
allowedTools: ["Read"],
|
|
16064
16081
|
maxTurns: 6,
|
|
@@ -16074,6 +16091,13 @@ async function gradeSession(rec, opts = { outDir: "" }) {
|
|
|
16074
16091
|
if (perChunk.length === 0) return null;
|
|
16075
16092
|
const criteria = perChunk.length > 1 ? peakMergeByKey(perChunk, "key", "quotes") : perChunk[0];
|
|
16076
16093
|
if (criteria.length === 0) return null;
|
|
16094
|
+
const small = /haiku/i.test(opts.model ?? "");
|
|
16095
|
+
if (small) for (const c of criteria) {
|
|
16096
|
+
c.score = mapSmallScore(c.score);
|
|
16097
|
+
c.low = mapSmallScore(c.low);
|
|
16098
|
+
c.high = mapSmallScore(c.high);
|
|
16099
|
+
c.bestEstimate = mapSmallScore(c.bestEstimate);
|
|
16100
|
+
}
|
|
16077
16101
|
const grade = {
|
|
16078
16102
|
sessionId: rec.sessionId,
|
|
16079
16103
|
title: rec.title,
|
|
@@ -16082,7 +16106,7 @@ async function gradeSession(rec, opts = { outDir: "" }) {
|
|
|
16082
16106
|
gradedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
16083
16107
|
rubricVersion: RUBRIC_FINGERPRINT,
|
|
16084
16108
|
criteria,
|
|
16085
|
-
meta: { costUsd, durationMs, calls: chunks.length }
|
|
16109
|
+
meta: { costUsd, durationMs, calls: chunks.length, ...small ? { scoreMap: HAIKU_SCORE_MAP_ID } : {} }
|
|
16086
16110
|
};
|
|
16087
16111
|
await fs5.mkdir(opts.outDir, { recursive: true });
|
|
16088
16112
|
await fs5.writeFile(path9.join(opts.outDir, `${rec.sessionId}.json`), JSON.stringify(grade, null, 2));
|
|
@@ -16116,6 +16140,171 @@ function partition(sessions) {
|
|
|
16116
16140
|
return { gradable, thin, trivial };
|
|
16117
16141
|
}
|
|
16118
16142
|
|
|
16143
|
+
// ../../lib/agents/coding/profile.ts
|
|
16144
|
+
import path10 from "path";
|
|
16145
|
+
|
|
16146
|
+
// ../../lib/agents/coding/walkthrough.ts
|
|
16147
|
+
var IDLE_CAP_MS = 12 * 6e4;
|
|
16148
|
+
|
|
16149
|
+
// ../coding-core/dist/words.js
|
|
16150
|
+
var NOTE = "((ran|created|edited|read|wrote|searched) (a|an|\\d+) [a-z]+|updated todos)";
|
|
16151
|
+
var TRANSCRIPT_LINE_RE = new RegExp(`^\\s*(${NOTE})((,| and) ${NOTE})*\\s*$|^\\s*thought for .{1,30}$`, "im");
|
|
16152
|
+
|
|
16153
|
+
// ../../lib/calibration/index.ts
|
|
16154
|
+
var DEFAULT_CALIBRATION = {
|
|
16155
|
+
version: "2026-07-14.2",
|
|
16156
|
+
// = RUBRIC_FINGERPRINT of the shipped npm rubric (packages/coding-analyzer/
|
|
16157
|
+
// src/grade/criteria.ts). A unit test fails loud when the rubric changes
|
|
16158
|
+
// without this being updated (and re-pushed to the server).
|
|
16159
|
+
rubricVersion: "f51b3a93a3",
|
|
16160
|
+
// Canonical scale = the one the product owner set for the public report
|
|
16161
|
+
// (11=0.01% … 8=2%), extended downward from the old report ladder. The old
|
|
16162
|
+
// CodeAnalysisView / npm-viewer maps (7→10%/15%, 6→25%/30%) were drift, not
|
|
16163
|
+
// intent.
|
|
16164
|
+
scoreBands: [
|
|
16165
|
+
{ score: 11, label: "Top 0.01%" },
|
|
16166
|
+
{ score: 10, label: "Top 0.1%" },
|
|
16167
|
+
{ score: 9, label: "Top 0.5%" },
|
|
16168
|
+
{ score: 8, label: "Top 2%" },
|
|
16169
|
+
{ score: 7, label: "Top 5%" },
|
|
16170
|
+
{ score: 6, label: "Top 15%" },
|
|
16171
|
+
{ score: 5, label: "Top 50%" },
|
|
16172
|
+
{ score: 4, label: "Top 65%" },
|
|
16173
|
+
{ score: 3, label: "Top 80%" },
|
|
16174
|
+
{ score: 2, label: "Top 90%" },
|
|
16175
|
+
{ score: 1, label: "Top 97%" }
|
|
16176
|
+
],
|
|
16177
|
+
minRankedScore: 6,
|
|
16178
|
+
benchmarkBands: [
|
|
16179
|
+
{
|
|
16180
|
+
level: "11",
|
|
16181
|
+
name: "Superhuman",
|
|
16182
|
+
rank: "Top 0.01%",
|
|
16183
|
+
oneIn: "1 in 10,000+",
|
|
16184
|
+
meaning: "Beyond the normal human ceiling for the trait \u2014 the genuine power-law outlier. Almost no one earns this; when the evidence shows it, it is scored, not rounded down.",
|
|
16185
|
+
foundIn: [
|
|
16186
|
+
"Fields Medal and Nobel-track researchers",
|
|
16187
|
+
"Founders of generational companies",
|
|
16188
|
+
"The handful of people a frontier field is named after"
|
|
16189
|
+
],
|
|
16190
|
+
maxTopPct: 0.01
|
|
16191
|
+
},
|
|
16192
|
+
{
|
|
16193
|
+
level: "10",
|
|
16194
|
+
name: "World-class",
|
|
16195
|
+
rank: "Top 0.1%",
|
|
16196
|
+
oneIn: "1 in 1,000",
|
|
16197
|
+
meaning: "Among the best alive at this trait \u2014 the level entire institutions are built to find.",
|
|
16198
|
+
foundIn: [
|
|
16199
|
+
"Researchers at frontier AI labs",
|
|
16200
|
+
"Faculty at top-5 research universities",
|
|
16201
|
+
"IMO / IOI medalists",
|
|
16202
|
+
"Founders backed by the top decile of venture firms"
|
|
16203
|
+
],
|
|
16204
|
+
maxTopPct: 0.1
|
|
16205
|
+
},
|
|
16206
|
+
{
|
|
16207
|
+
level: "9",
|
|
16208
|
+
name: "Exceptional",
|
|
16209
|
+
rank: "Top 1%",
|
|
16210
|
+
oneIn: "1 in 100",
|
|
16211
|
+
meaning: "Clearly exceptional \u2014 the strongest person on most strong teams.",
|
|
16212
|
+
foundIn: [
|
|
16213
|
+
"PhD students at top programs",
|
|
16214
|
+
"Early engineers at breakout startups",
|
|
16215
|
+
"YC-class founders",
|
|
16216
|
+
"National olympiad finalists"
|
|
16217
|
+
],
|
|
16218
|
+
maxTopPct: 1
|
|
16219
|
+
},
|
|
16220
|
+
{
|
|
16221
|
+
level: "8",
|
|
16222
|
+
name: "Strong",
|
|
16223
|
+
rank: "Top 10%",
|
|
16224
|
+
oneIn: "1 in 10",
|
|
16225
|
+
meaning: "Strong against the whole population \u2014 the best person in most rooms, not every room.",
|
|
16226
|
+
foundIn: [
|
|
16227
|
+
"Senior engineers at selective tech companies",
|
|
16228
|
+
"Graduates of demanding technical programs",
|
|
16229
|
+
"Operators who get promoted everywhere they go"
|
|
16230
|
+
],
|
|
16231
|
+
maxTopPct: 10
|
|
16232
|
+
},
|
|
16233
|
+
{
|
|
16234
|
+
level: "5",
|
|
16235
|
+
name: "Median",
|
|
16236
|
+
rank: "50th percentile",
|
|
16237
|
+
oneIn: "1 in 2",
|
|
16238
|
+
meaning: 'The middle of the general population. The reference class is all ~8 billion humans \u2014 so even "strong" above already means top 10% of everyone.',
|
|
16239
|
+
foundIn: ["The general population \u2014 most people, most places"],
|
|
16240
|
+
maxTopPct: 100
|
|
16241
|
+
}
|
|
16242
|
+
],
|
|
16243
|
+
codingTiers: {
|
|
16244
|
+
push: { median: 300, sigma: 0.91, tag: "estimated \u2014 log-normal fit", source: "Anthropic 2026 Claude Code study (400k sessions): median engaged user ~8 prompts/day at ~35 words \u2014 ~300 directed words/day; tail: a 1-in-1,000 user AVERAGES ~5k directed words/day (near-daily heavy dictation, clearly below the ~10k single-day record). AVERAGE over substantial days, cumulative not spiky." },
|
|
16245
|
+
focus: { median: 0.08, sigma: 0.559, tag: "estimated \u2014 proxy-anchored", source: "median: Anthropic 20h/week engaged-user figure implies ~1h/day of true rapid exchange against an 8h working day; ceiling anchored to the ~4h/day deep-work limit (45% share = 1-in-1,000). AVERAGE share, cumulative not spiky." },
|
|
16246
|
+
orchestration: { tag: "estimated \u2014 behavior bands", source: "no published concurrency distribution exists; ceiling anchored to the Claude Code lead's documented 10-15 parallel sessions" }
|
|
16247
|
+
},
|
|
16248
|
+
codingBenchmarks: {
|
|
16249
|
+
flow: {
|
|
16250
|
+
// typical knowledge worker ≈ 35% of an 8h day focused; ~4h/day is the
|
|
16251
|
+
// recognized deep-work ceiling (Newport) ≈ 50% — most sit far below it.
|
|
16252
|
+
typicalPctOfDay: 0.35,
|
|
16253
|
+
topPctOfDay: 0.5,
|
|
16254
|
+
tag: "measured",
|
|
16255
|
+
source: "RescueTime 2019 (~2h48m focused/day) \xB7 Cal Newport, Deep Work (~4h/day deep-work ceiling)",
|
|
16256
|
+
line: "the best spend ~half the workday in true deep flow; ~4h/day is the human ceiling"
|
|
16257
|
+
},
|
|
16258
|
+
longestRun: {
|
|
16259
|
+
typicalHours: 0.67,
|
|
16260
|
+
// ~40 min before the average worker is interrupted
|
|
16261
|
+
topHours: 4,
|
|
16262
|
+
tag: "measured",
|
|
16263
|
+
source: "RescueTime (avg max ~40 min focus before interruption) \xB7 deep-work single-block ceiling ~3\u20134h",
|
|
16264
|
+
line: "a top performer can hold a single ~3\u20134h uninterrupted block; the average worker breaks at ~40 min"
|
|
16265
|
+
},
|
|
16266
|
+
parallelism: {
|
|
16267
|
+
topMaxConcurrent: 12,
|
|
16268
|
+
// Boris Cherny ~10–15 interactive sessions
|
|
16269
|
+
topAvgConcurrent: 6,
|
|
16270
|
+
// time-weighted average while active (anecdotal, same source)
|
|
16271
|
+
worktreesTip: "3\u20135 git worktrees at once",
|
|
16272
|
+
tag: "anecdotal",
|
|
16273
|
+
source: "Boris Cherny (Claude Code lead, Anthropic): ~10\u201315 concurrent sessions, 'dozens of Claudes running at all times'; 3\u20135 worktrees = 'the single biggest productivity unlock'",
|
|
16274
|
+
line: "the Claude Code lead runs 10\u201315 sessions at once; his #1 tip is 3\u20135 git worktrees in parallel"
|
|
16275
|
+
},
|
|
16276
|
+
throughput: {
|
|
16277
|
+
// Two DIFFERENT kinds of line (2026-07-14.2): topAvg is a PERCENTILE
|
|
16278
|
+
// claim — what a 1-in-1,000 user AVERAGES over substantial days (5k/day:
|
|
16279
|
+
// near-daily heavy dictation; a measured top dictating power user
|
|
16280
|
+
// averages ~3.9k). topPeak is NOT a percentile — it is the CEILING
|
|
16281
|
+
// reference, the wildest single days on record (~10k: a 14-16h
|
|
16282
|
+
// launch-day marathon), rendered as "wildest single days", never
|
|
16283
|
+
// "top X% peak". An avg-percentile and a peak-percentile can't both be
|
|
16284
|
+
// shown at a same-ish value (avg≈peak reads broken — user call), and a
|
|
16285
|
+
// peak PERCENTILE line is unknowable anyway; the record framing is
|
|
16286
|
+
// honest and keeps the visual gap.
|
|
16287
|
+
topAvgWordsPerDay: 5e3,
|
|
16288
|
+
topPeakWordsPerDay: 1e4,
|
|
16289
|
+
benchLabel: "top 0.1%",
|
|
16290
|
+
// No published "words typed/day" for heavy users — the real story is OUTPUT.
|
|
16291
|
+
loopsHeadline: ">1,000,000 lines of Rust at 99.8% tests passing",
|
|
16292
|
+
loopsDetail: "the Bun runtime was ported Zig\u2192Rust largely autonomously \u2014 hundreds of parallel subagents, two AI reviewers per file",
|
|
16293
|
+
tag: "measured",
|
|
16294
|
+
source: "The Register, May 2026 (Bun Zig\u2192Rust rewrite)",
|
|
16295
|
+
line: "autonomous fleets ported >1M lines of Rust at 99.8% tests passing; a while-loop agent delivered a $50k contract for $297 of compute"
|
|
16296
|
+
}
|
|
16297
|
+
}
|
|
16298
|
+
};
|
|
16299
|
+
|
|
16300
|
+
// ../../lib/agents/coding/benchmarks.ts
|
|
16301
|
+
var BEST = DEFAULT_CALIBRATION.codingBenchmarks;
|
|
16302
|
+
|
|
16303
|
+
// ../../lib/agents/coding/profile.ts
|
|
16304
|
+
var ROOT = process.cwd();
|
|
16305
|
+
var CODING_DIR = process.env.POLYMATH_DATA_DIR ? path10.join(process.env.POLYMATH_DATA_DIR, "coding") : path10.join(ROOT, ".data", "coding");
|
|
16306
|
+
var GRADES = path10.join(CODING_DIR, "grades");
|
|
16307
|
+
|
|
16119
16308
|
// ../../scripts/coding-grade.mts
|
|
16120
16309
|
var argv = process.argv.slice(2);
|
|
16121
16310
|
var flag = (n, d) => {
|
|
@@ -16141,12 +16330,12 @@ function makeLimiter(max) {
|
|
|
16141
16330
|
}
|
|
16142
16331
|
};
|
|
16143
16332
|
}
|
|
16144
|
-
var CODING =
|
|
16145
|
-
var
|
|
16146
|
-
var LOGS =
|
|
16333
|
+
var CODING = CODING_DIR;
|
|
16334
|
+
var GRADES2 = path11.join(CODING, "grades");
|
|
16335
|
+
var LOGS = path11.join(GRADES2, "logs");
|
|
16147
16336
|
async function main() {
|
|
16148
|
-
const sessions = JSON.parse(await fs6.readFile(
|
|
16149
|
-
const { gradable, thin, trivial } = partition(sessions.filter((s) => !s.duplicateOf));
|
|
16337
|
+
const sessions = JSON.parse(await fs6.readFile(path11.join(CODING, "sessions.json"), "utf8"));
|
|
16338
|
+
const { gradable, thin, trivial } = partition(sessions.filter((s) => s.klass === "interactive" && !s.duplicateOf));
|
|
16150
16339
|
const ranked = gradable.slice().sort((a, b) => b.humanChars * Math.log1p(b.humanTurns) - a.humanChars * Math.log1p(a.humanTurns));
|
|
16151
16340
|
console.log(`[coding-grade] ${sessions.length} sessions \u2192 ${gradable.length} gradable \xB7 ${thin.length} thin \xB7 ${trivial.length} trivial (skipped)`);
|
|
16152
16341
|
await fs6.mkdir(LOGS, { recursive: true });
|
|
@@ -16155,12 +16344,13 @@ async function main() {
|
|
|
16155
16344
|
for (const s of ranked) {
|
|
16156
16345
|
if (picked.length >= limit) break;
|
|
16157
16346
|
if (!REGRADE) {
|
|
16158
|
-
const done = await fs6.readFile(
|
|
16347
|
+
const done = await fs6.readFile(path11.join(GRADES2, `${s.sessionId}.json`), "utf8").catch(() => null);
|
|
16159
16348
|
if (done) continue;
|
|
16160
16349
|
}
|
|
16161
16350
|
picked.push(s);
|
|
16162
16351
|
}
|
|
16163
16352
|
console.log(`[coding-grade] grading ${picked.length}${ALL ? " (ALL gradable, resuming)" : ""} on ${MODEL}, conc ${CONC}\u2026`);
|
|
16353
|
+
console.log(` (each session below is a single small ${MODEL} call \u2014 a long run is many tiny calls, not a huge one)`);
|
|
16164
16354
|
const lim = makeLimiter(CONC);
|
|
16165
16355
|
startRun("coding-grade", picked.length);
|
|
16166
16356
|
let doneN = 0, sumUsd = 0, sumCalls = 0;
|
|
@@ -16168,19 +16358,19 @@ async function main() {
|
|
|
16168
16358
|
await Promise.all(picked.map((s) => lim(async () => {
|
|
16169
16359
|
const t0 = Date.now();
|
|
16170
16360
|
try {
|
|
16171
|
-
const g = await gradeSession(s, { model: MODEL, outDir:
|
|
16361
|
+
const g = await gradeSession(s, { model: MODEL, outDir: GRADES2, logDir: LOGS });
|
|
16172
16362
|
tick(!!g);
|
|
16173
16363
|
if (!g) {
|
|
16174
16364
|
console.log(` \u2022 "${s.title.slice(0, 48)}" \u2014 skipped (no usable grade)`);
|
|
16175
16365
|
return;
|
|
16176
16366
|
}
|
|
16177
16367
|
const scored = g.criteria.filter((c) => c.score != null).map((c) => `${c.key}=${c.score}`).join(" ");
|
|
16178
|
-
console.log(` \u2022 "${s.title.slice(0, 48)}" done ${(Date.now() - t0) / 1e3 | 0}s
|
|
16368
|
+
console.log(` \u2022 "${s.title.slice(0, 48)}" done ${(Date.now() - t0) / 1e3 | 0}s \u2192 ${scored || "(all null)"}`);
|
|
16179
16369
|
doneN++;
|
|
16180
16370
|
sumUsd += g.meta.costUsd ?? 0;
|
|
16181
16371
|
sumCalls += g.meta.calls ?? 0;
|
|
16182
16372
|
const elapsedMin = (Date.now() - runT0) / 6e4;
|
|
16183
|
-
console.log(` \u03A3 ${doneN}/${picked.length} \xB7
|
|
16373
|
+
console.log(` \u03A3 ${doneN}/${picked.length} \xB7 ${elapsedMin.toFixed(0)}m elapsed \u2192 ~${(elapsedMin / doneN * picked.length / 60).toFixed(1)}h total for this run (small ${MODEL} calls)`);
|
|
16184
16374
|
} catch (e) {
|
|
16185
16375
|
tick(false);
|
|
16186
16376
|
console.log(` \u2022 "${s.title.slice(0, 48)}" ERROR ${e.message.slice(0, 60)}`);
|