polymath-society 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -135,7 +135,7 @@ var require_secure_json_parse = __commonJS({
135
135
 
136
136
  // ../../scripts/coding-grade.mts
137
137
  import { promises as fs6 } from "fs";
138
- import path10 from "path";
138
+ import path11 from "path";
139
139
 
140
140
  // ../../lib/agents/coding/grade.ts
141
141
  import { promises as fs5 } from "fs";
@@ -2155,8 +2155,8 @@ function getErrorMap() {
2155
2155
 
2156
2156
  // ../../node_modules/zod/v3/helpers/parseUtil.js
2157
2157
  var makeIssue = (params) => {
2158
- const { data, path: path11, errorMaps, issueData } = params;
2159
- const fullPath = [...path11, ...issueData.path || []];
2158
+ const { data, path: path12, errorMaps, issueData } = params;
2159
+ const fullPath = [...path12, ...issueData.path || []];
2160
2160
  const fullIssue = {
2161
2161
  ...issueData,
2162
2162
  path: fullPath
@@ -2272,11 +2272,11 @@ var errorUtil;
2272
2272
 
2273
2273
  // ../../node_modules/zod/v3/types.js
2274
2274
  var ParseInputLazyPath = class {
2275
- constructor(parent, value, path11, key) {
2275
+ constructor(parent, value, path12, key) {
2276
2276
  this._cachedPath = [];
2277
2277
  this.parent = parent;
2278
2278
  this.data = value;
2279
- this._path = path11;
2279
+ this._path = path12;
2280
2280
  this._key = key;
2281
2281
  }
2282
2282
  get path() {
@@ -15063,39 +15063,39 @@ function createOpenAI(options = {}) {
15063
15063
  });
15064
15064
  const createChatModel = (modelId, settings = {}) => new OpenAIChatLanguageModel(modelId, settings, {
15065
15065
  provider: `${providerName}.chat`,
15066
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15066
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15067
15067
  headers: getHeaders,
15068
15068
  compatibility,
15069
15069
  fetch: options.fetch
15070
15070
  });
15071
15071
  const createCompletionModel = (modelId, settings = {}) => new OpenAICompletionLanguageModel(modelId, settings, {
15072
15072
  provider: `${providerName}.completion`,
15073
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15073
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15074
15074
  headers: getHeaders,
15075
15075
  compatibility,
15076
15076
  fetch: options.fetch
15077
15077
  });
15078
15078
  const createEmbeddingModel = (modelId, settings = {}) => new OpenAIEmbeddingModel(modelId, settings, {
15079
15079
  provider: `${providerName}.embedding`,
15080
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15080
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15081
15081
  headers: getHeaders,
15082
15082
  fetch: options.fetch
15083
15083
  });
15084
15084
  const createImageModel = (modelId, settings = {}) => new OpenAIImageModel(modelId, settings, {
15085
15085
  provider: `${providerName}.image`,
15086
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15086
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15087
15087
  headers: getHeaders,
15088
15088
  fetch: options.fetch
15089
15089
  });
15090
15090
  const createTranscriptionModel = (modelId) => new OpenAITranscriptionModel(modelId, {
15091
15091
  provider: `${providerName}.transcription`,
15092
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15092
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15093
15093
  headers: getHeaders,
15094
15094
  fetch: options.fetch
15095
15095
  });
15096
15096
  const createSpeechModel = (modelId) => new OpenAISpeechModel(modelId, {
15097
15097
  provider: `${providerName}.speech`,
15098
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15098
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15099
15099
  headers: getHeaders,
15100
15100
  fetch: options.fetch
15101
15101
  });
@@ -15116,7 +15116,7 @@ function createOpenAI(options = {}) {
15116
15116
  const createResponsesModel = (modelId) => {
15117
15117
  return new OpenAIResponsesLanguageModel(modelId, {
15118
15118
  provider: `${providerName}.responses`,
15119
- url: ({ path: path11 }) => `${baseURL}${path11}`,
15119
+ url: ({ path: path12 }) => `${baseURL}${path12}`,
15120
15120
  headers: getHeaders,
15121
15121
  fetch: options.fetch
15122
15122
  });
@@ -15637,6 +15637,9 @@ function invokeAgent(inv, opts = {}) {
15637
15637
  return withLimitRetry(once, { label });
15638
15638
  }
15639
15639
 
15640
+ // ../../lib/agents/coding/grade-calibration.ts
15641
+ var GRADE_CALIBRATION = "\n\nSCORE CALIBRATION \u2014 graders systematically COMPRESS to the middle: real peaks get hedged down to 6-7, and routine sessions get polite 5-6s instead of null. Both are failures; fight them explicitly. Work each criterion in EXACTLY this order:\nSTEP 1 \xB7 EVIDENCE GATE \u2014 find the 1-3 verbatim '>> ' quotes that bear on the criterion. INCLUDE a criterion whenever genuine evidence is present (do not MISS quiet evidence \u2014 check every thread of the session, not just the dominant one). But with NO quotable moment the score is null, full stop: a session can be long, competent, and productive while being null on most criteria. Routine hand-holding earns null, never a 5 \u2014 and null is never a deduction.\nSTEP 2 \xB7 LADDER MATCH \u2014 with quotes in hand, read the criterion's rungs and pick the ONE rung whose marker best describes the quoted moment; name that rung in 'why'. AWARD THE RUNG THE EVIDENCE MATCHES: if your own 'why' describes rung N's criteria, give rung N \u2014 never deny it for lacking the NEXT rung's criteria.\nSTEP 3 \xB7 THE SINGLE BEST MOMENT IS THE SCORE \u2014 set the score by the strongest quoted moment, as if it were the whole session. NEVER average a peak down because the surrounding session is routine (the high-water rule), and never raise a score for volume, effort, or topic prestige without a matching moment.\nSTEP 4 \xB7 ANTI-HEDGE \u2014 if the moment genuinely matches an L8-10 anchor (an override the AI conceded, a shipped-and-verified capability, an 'oh shit, they're right' call, a first-principles redefinition that held), the score IS 8-10. Hedging a real L9 to a 7 is exactly as wrong as inventing a 6 for nothing. The middle is NOT a safe default: commit high on anchor-matching evidence, or go null.\nSTEP 5 \xB7 'CONSERVATIVE' \u2260 SUBTRACT \u2014 when a criterion's ladder says to grade conservatively, that means DO NOT EXCEED the rung whose marker the evidence matches; it NEVER means subtracting rungs from matched evidence. Evidence that matches a rung's marker gets that rung under a conservative reading too.";
15642
+
15640
15643
  // ../../lib/agents/shared/chunk.ts
15641
15644
  function chunkText(text2, max) {
15642
15645
  if (text2.length <= max) return [text2];
@@ -15985,14 +15988,28 @@ function rubric() {
15985
15988
  ${(c.rungs ?? []).map((r) => ` ${r.level} \u2014 ${r.marker}`).join("\n")}`
15986
15989
  ).join("\n\n");
15987
15990
  }
15991
+ function rubricCompact() {
15992
+ return GRADED_CRITERIA.map((c) => {
15993
+ const anchors = /ANCHORS?:/.exec(c.read) ? c.read.slice(c.read.search(/ANCHORS?:/)) : "";
15994
+ return `CRITERION "${c.key}" \u2014 ${c.label} [score 1\u201311, or null if not shown]
15995
+ ${c.definition}
15996
+ ${anchors ? ` ${anchors}
15997
+ ` : ""} LADDER:
15998
+ ${(c.rungs ?? []).map((r) => ` ${r.level} \u2014 ${r.marker}`).join("\n")}`;
15999
+ }).join("\n\n");
16000
+ }
16001
+ var HAIKU_SCORE_MAP_ID = "haiku+1@2026-07-14";
16002
+ var mapSmallScore = (v) => v != null && v >= 4 ? Math.min(10, v + 1) : v;
16003
+ var HAIKU_CALIBRATION = "\n\nWork each criterion in TWO STEPS, in order:\nSTEP A \u2014 QUOTE the 1-3 strongest verbatim '>> ' lines bearing on the criterion, checking every thread of the session, not just the dominant one. No quotable moment = null (routine hand-holding is null, never a 5; null is never a deduction).\nSTEP B \u2014 score by the SINGLE BEST quoted moment as if it were the whole session: pick the rung whose marker it matches and AWARD THAT RUNG \u2014 never deny rung N for lacking rung N+1, never average a peak down because the rest is routine, never raise for volume or topic prestige. Evidence matching an L8-10 anchor scores 8-10; the middle is not a safe default. Include every criterion genuinely present (don't MISS quiet evidence) \u2014 but rate strength strictly: weak-but-present is low, not mid.";
15988
16004
  var SYSTEM = "You grade how a person WORKS from one of their coding-agent sessions (Claude Code). You read mostly THEIR words; the model's code and output are compressed to context and you ignore the mechanical scaffolding. You never flatter: most sessions are routine hand-holding and should score low or null on most criteria. A score is a placement on the ladder backed by VERBATIM quotes of the user's own lines \u2014 never a vibe.\n\nSTANDING RULES (apply to every criterion):\n1. ABSENCE IS NEVER A DEDUCTION. If a criterion had no chance to show, its score is null \u2014 NOT a low. Don't invent a number for routine work.\n2. HIGH-WATER MARK. Taste and abstraction are PEAK measures \u2014 one genuinely brilliant move in a session counts, even surrounded by routine work. Never grade the person at their worst; a lone weak moment among strong ones is a slip, not the score.\n3. LOW-EFFORT-UNTIL-NEEDED IS THE IDEAL. A terse or vague prompt on routine work is correct economy \u2014 never penalize a vague ask, never reward word-count.\n4. DISCOUNT THE LITERAL HARSHNESS OF SELF-TALK. 'you idiot', 'you stupid cunt never again', 'this is retarded' \u2014 aimed at self or AI \u2014 are DRIVE and high standards, not defects. Grade the demonstrated behavior.\n5. PER-CONVERSATION CEILING IS 10. Never output 11 \u2014 that is the aggregator's job (it needs cross-conversation consistency / a stacked day this single session can't show). Scores run 2\u201310.\n6. LOG THE SPIKES IN HIGH DETAIL. When a moment is genuinely good (a flare \u2014 a sharp taste call, an out-reasons-the-tool insight, a real systems override), name it CONCRETELY and quote it. The 'instance' + 'why' are the high-detail record of exactly what was cool and why it placed where it did.";
15989
- function gradeSystemPrompt() {
15990
- return `${SYSTEM}
16005
+ function gradeSystemPrompt(model) {
16006
+ const small = /haiku/i.test(model ?? "");
16007
+ return `${SYSTEM}${small ? HAIKU_CALIBRATION : GRADE_CALIBRATION}
15991
16008
 
15992
16009
  Grade ONE coding session (provided in the user message) along the criteria below. ">> " lines are the USER's own words (the signal). ALL bracketed lines are machine context, never the user: "[AI: \u2026]" is the model working, "[bg task: \u2026]" / "[cmd output: \u2026]" are harness notifications, "[mode \u2192 \u2026]" marks a mode switch (context only \u2014 do not grade or quote them). ">> (queued \u2026)" is the user queuing work while the AI runs.
15993
16010
 
15994
16011
  CRITERIA:
15995
- ${rubric()}
16012
+ ${small ? rubricCompact() : rubric()}
15996
16013
 
15997
16014
  RULES:
15998
16015
  - Score ONLY what the user's moves genuinely show. Null is the right answer for most criteria in most sessions \u2014 do not force a number. Absence is never a deduction.
@@ -16058,7 +16075,7 @@ async function gradeSession(rec, opts = { outDir: "" }) {
16058
16075
  const seg = chunks.length > 1 ? ` This is SEGMENT ${ci + 1} of ${chunks.length} of one longer session \u2014 grade only what THIS segment shows; null for criteria not shown here (segments are merged afterward).` : "";
16059
16076
  const run2 = await invokeAgent({
16060
16077
  prompt: buildPrompt(rec, chunks[ci], seg),
16061
- systemPrompt: gradeSystemPrompt(),
16078
+ systemPrompt: gradeSystemPrompt(opts.model),
16062
16079
  addDir: sandbox,
16063
16080
  allowedTools: ["Read"],
16064
16081
  maxTurns: 6,
@@ -16074,6 +16091,13 @@ async function gradeSession(rec, opts = { outDir: "" }) {
16074
16091
  if (perChunk.length === 0) return null;
16075
16092
  const criteria = perChunk.length > 1 ? peakMergeByKey(perChunk, "key", "quotes") : perChunk[0];
16076
16093
  if (criteria.length === 0) return null;
16094
+ const small = /haiku/i.test(opts.model ?? "");
16095
+ if (small) for (const c of criteria) {
16096
+ c.score = mapSmallScore(c.score);
16097
+ c.low = mapSmallScore(c.low);
16098
+ c.high = mapSmallScore(c.high);
16099
+ c.bestEstimate = mapSmallScore(c.bestEstimate);
16100
+ }
16077
16101
  const grade = {
16078
16102
  sessionId: rec.sessionId,
16079
16103
  title: rec.title,
@@ -16082,7 +16106,7 @@ async function gradeSession(rec, opts = { outDir: "" }) {
16082
16106
  gradedAt: (/* @__PURE__ */ new Date()).toISOString(),
16083
16107
  rubricVersion: RUBRIC_FINGERPRINT,
16084
16108
  criteria,
16085
- meta: { costUsd, durationMs, calls: chunks.length }
16109
+ meta: { costUsd, durationMs, calls: chunks.length, ...small ? { scoreMap: HAIKU_SCORE_MAP_ID } : {} }
16086
16110
  };
16087
16111
  await fs5.mkdir(opts.outDir, { recursive: true });
16088
16112
  await fs5.writeFile(path9.join(opts.outDir, `${rec.sessionId}.json`), JSON.stringify(grade, null, 2));
@@ -16116,6 +16140,171 @@ function partition(sessions) {
16116
16140
  return { gradable, thin, trivial };
16117
16141
  }
16118
16142
 
16143
+ // ../../lib/agents/coding/profile.ts
16144
+ import path10 from "path";
16145
+
16146
+ // ../../lib/agents/coding/walkthrough.ts
16147
+ var IDLE_CAP_MS = 12 * 6e4;
16148
+
16149
+ // ../coding-core/dist/words.js
16150
+ var NOTE = "((ran|created|edited|read|wrote|searched) (a|an|\\d+) [a-z]+|updated todos)";
16151
+ var TRANSCRIPT_LINE_RE = new RegExp(`^\\s*(${NOTE})((,| and) ${NOTE})*\\s*$|^\\s*thought for .{1,30}$`, "im");
16152
+
16153
+ // ../../lib/calibration/index.ts
16154
+ var DEFAULT_CALIBRATION = {
16155
+ version: "2026-07-14.2",
16156
+ // = RUBRIC_FINGERPRINT of the shipped npm rubric (packages/coding-analyzer/
16157
+ // src/grade/criteria.ts). A unit test fails loud when the rubric changes
16158
+ // without this being updated (and re-pushed to the server).
16159
+ rubricVersion: "f51b3a93a3",
16160
+ // Canonical scale = the one the product owner set for the public report
16161
+ // (11=0.01% … 8=2%), extended downward from the old report ladder. The old
16162
+ // CodeAnalysisView / npm-viewer maps (7→10%/15%, 6→25%/30%) were drift, not
16163
+ // intent.
16164
+ scoreBands: [
16165
+ { score: 11, label: "Top 0.01%" },
16166
+ { score: 10, label: "Top 0.1%" },
16167
+ { score: 9, label: "Top 0.5%" },
16168
+ { score: 8, label: "Top 2%" },
16169
+ { score: 7, label: "Top 5%" },
16170
+ { score: 6, label: "Top 15%" },
16171
+ { score: 5, label: "Top 50%" },
16172
+ { score: 4, label: "Top 65%" },
16173
+ { score: 3, label: "Top 80%" },
16174
+ { score: 2, label: "Top 90%" },
16175
+ { score: 1, label: "Top 97%" }
16176
+ ],
16177
+ minRankedScore: 6,
16178
+ benchmarkBands: [
16179
+ {
16180
+ level: "11",
16181
+ name: "Superhuman",
16182
+ rank: "Top 0.01%",
16183
+ oneIn: "1 in 10,000+",
16184
+ meaning: "Beyond the normal human ceiling for the trait \u2014 the genuine power-law outlier. Almost no one earns this; when the evidence shows it, it is scored, not rounded down.",
16185
+ foundIn: [
16186
+ "Fields Medal and Nobel-track researchers",
16187
+ "Founders of generational companies",
16188
+ "The handful of people a frontier field is named after"
16189
+ ],
16190
+ maxTopPct: 0.01
16191
+ },
16192
+ {
16193
+ level: "10",
16194
+ name: "World-class",
16195
+ rank: "Top 0.1%",
16196
+ oneIn: "1 in 1,000",
16197
+ meaning: "Among the best alive at this trait \u2014 the level entire institutions are built to find.",
16198
+ foundIn: [
16199
+ "Researchers at frontier AI labs",
16200
+ "Faculty at top-5 research universities",
16201
+ "IMO / IOI medalists",
16202
+ "Founders backed by the top decile of venture firms"
16203
+ ],
16204
+ maxTopPct: 0.1
16205
+ },
16206
+ {
16207
+ level: "9",
16208
+ name: "Exceptional",
16209
+ rank: "Top 1%",
16210
+ oneIn: "1 in 100",
16211
+ meaning: "Clearly exceptional \u2014 the strongest person on most strong teams.",
16212
+ foundIn: [
16213
+ "PhD students at top programs",
16214
+ "Early engineers at breakout startups",
16215
+ "YC-class founders",
16216
+ "National olympiad finalists"
16217
+ ],
16218
+ maxTopPct: 1
16219
+ },
16220
+ {
16221
+ level: "8",
16222
+ name: "Strong",
16223
+ rank: "Top 10%",
16224
+ oneIn: "1 in 10",
16225
+ meaning: "Strong against the whole population \u2014 the best person in most rooms, not every room.",
16226
+ foundIn: [
16227
+ "Senior engineers at selective tech companies",
16228
+ "Graduates of demanding technical programs",
16229
+ "Operators who get promoted everywhere they go"
16230
+ ],
16231
+ maxTopPct: 10
16232
+ },
16233
+ {
16234
+ level: "5",
16235
+ name: "Median",
16236
+ rank: "50th percentile",
16237
+ oneIn: "1 in 2",
16238
+ meaning: 'The middle of the general population. The reference class is all ~8 billion humans \u2014 so even "strong" above already means top 10% of everyone.',
16239
+ foundIn: ["The general population \u2014 most people, most places"],
16240
+ maxTopPct: 100
16241
+ }
16242
+ ],
16243
+ codingTiers: {
16244
+ push: { median: 300, sigma: 0.91, tag: "estimated \u2014 log-normal fit", source: "Anthropic 2026 Claude Code study (400k sessions): median engaged user ~8 prompts/day at ~35 words \u2014 ~300 directed words/day; tail: a 1-in-1,000 user AVERAGES ~5k directed words/day (near-daily heavy dictation, clearly below the ~10k single-day record). AVERAGE over substantial days, cumulative not spiky." },
16245
+ focus: { median: 0.08, sigma: 0.559, tag: "estimated \u2014 proxy-anchored", source: "median: Anthropic 20h/week engaged-user figure implies ~1h/day of true rapid exchange against an 8h working day; ceiling anchored to the ~4h/day deep-work limit (45% share = 1-in-1,000). AVERAGE share, cumulative not spiky." },
16246
+ orchestration: { tag: "estimated \u2014 behavior bands", source: "no published concurrency distribution exists; ceiling anchored to the Claude Code lead's documented 10-15 parallel sessions" }
16247
+ },
16248
+ codingBenchmarks: {
16249
+ flow: {
16250
+ // typical knowledge worker ≈ 35% of an 8h day focused; ~4h/day is the
16251
+ // recognized deep-work ceiling (Newport) ≈ 50% — most sit far below it.
16252
+ typicalPctOfDay: 0.35,
16253
+ topPctOfDay: 0.5,
16254
+ tag: "measured",
16255
+ source: "RescueTime 2019 (~2h48m focused/day) \xB7 Cal Newport, Deep Work (~4h/day deep-work ceiling)",
16256
+ line: "the best spend ~half the workday in true deep flow; ~4h/day is the human ceiling"
16257
+ },
16258
+ longestRun: {
16259
+ typicalHours: 0.67,
16260
+ // ~40 min before the average worker is interrupted
16261
+ topHours: 4,
16262
+ tag: "measured",
16263
+ source: "RescueTime (avg max ~40 min focus before interruption) \xB7 deep-work single-block ceiling ~3\u20134h",
16264
+ line: "a top performer can hold a single ~3\u20134h uninterrupted block; the average worker breaks at ~40 min"
16265
+ },
16266
+ parallelism: {
16267
+ topMaxConcurrent: 12,
16268
+ // Boris Cherny ~10–15 interactive sessions
16269
+ topAvgConcurrent: 6,
16270
+ // time-weighted average while active (anecdotal, same source)
16271
+ worktreesTip: "3\u20135 git worktrees at once",
16272
+ tag: "anecdotal",
16273
+ source: "Boris Cherny (Claude Code lead, Anthropic): ~10\u201315 concurrent sessions, 'dozens of Claudes running at all times'; 3\u20135 worktrees = 'the single biggest productivity unlock'",
16274
+ line: "the Claude Code lead runs 10\u201315 sessions at once; his #1 tip is 3\u20135 git worktrees in parallel"
16275
+ },
16276
+ throughput: {
16277
+ // Two DIFFERENT kinds of line (2026-07-14.2): topAvg is a PERCENTILE
16278
+ // claim — what a 1-in-1,000 user AVERAGES over substantial days (5k/day:
16279
+ // near-daily heavy dictation; a measured top dictating power user
16280
+ // averages ~3.9k). topPeak is NOT a percentile — it is the CEILING
16281
+ // reference, the wildest single days on record (~10k: a 14-16h
16282
+ // launch-day marathon), rendered as "wildest single days", never
16283
+ // "top X% peak". An avg-percentile and a peak-percentile can't both be
16284
+ // shown at a same-ish value (avg≈peak reads broken — user call), and a
16285
+ // peak PERCENTILE line is unknowable anyway; the record framing is
16286
+ // honest and keeps the visual gap.
16287
+ topAvgWordsPerDay: 5e3,
16288
+ topPeakWordsPerDay: 1e4,
16289
+ benchLabel: "top 0.1%",
16290
+ // No published "words typed/day" for heavy users — the real story is OUTPUT.
16291
+ loopsHeadline: ">1,000,000 lines of Rust at 99.8% tests passing",
16292
+ loopsDetail: "the Bun runtime was ported Zig\u2192Rust largely autonomously \u2014 hundreds of parallel subagents, two AI reviewers per file",
16293
+ tag: "measured",
16294
+ source: "The Register, May 2026 (Bun Zig\u2192Rust rewrite)",
16295
+ line: "autonomous fleets ported >1M lines of Rust at 99.8% tests passing; a while-loop agent delivered a $50k contract for $297 of compute"
16296
+ }
16297
+ }
16298
+ };
16299
+
16300
+ // ../../lib/agents/coding/benchmarks.ts
16301
+ var BEST = DEFAULT_CALIBRATION.codingBenchmarks;
16302
+
16303
+ // ../../lib/agents/coding/profile.ts
16304
+ var ROOT = process.cwd();
16305
+ var CODING_DIR = process.env.POLYMATH_DATA_DIR ? path10.join(process.env.POLYMATH_DATA_DIR, "coding") : path10.join(ROOT, ".data", "coding");
16306
+ var GRADES = path10.join(CODING_DIR, "grades");
16307
+
16119
16308
  // ../../scripts/coding-grade.mts
16120
16309
  var argv = process.argv.slice(2);
16121
16310
  var flag = (n, d) => {
@@ -16141,12 +16330,12 @@ function makeLimiter(max) {
16141
16330
  }
16142
16331
  };
16143
16332
  }
16144
- var CODING = path10.join(process.cwd(), ".data", "coding");
16145
- var GRADES = path10.join(CODING, "grades");
16146
- var LOGS = path10.join(GRADES, "logs");
16333
+ var CODING = CODING_DIR;
16334
+ var GRADES2 = path11.join(CODING, "grades");
16335
+ var LOGS = path11.join(GRADES2, "logs");
16147
16336
  async function main() {
16148
- const sessions = JSON.parse(await fs6.readFile(path10.join(CODING, "sessions.json"), "utf8"));
16149
- const { gradable, thin, trivial } = partition(sessions.filter((s) => !s.duplicateOf));
16337
+ const sessions = JSON.parse(await fs6.readFile(path11.join(CODING, "sessions.json"), "utf8"));
16338
+ const { gradable, thin, trivial } = partition(sessions.filter((s) => s.klass === "interactive" && !s.duplicateOf));
16150
16339
  const ranked = gradable.slice().sort((a, b) => b.humanChars * Math.log1p(b.humanTurns) - a.humanChars * Math.log1p(a.humanTurns));
16151
16340
  console.log(`[coding-grade] ${sessions.length} sessions \u2192 ${gradable.length} gradable \xB7 ${thin.length} thin \xB7 ${trivial.length} trivial (skipped)`);
16152
16341
  await fs6.mkdir(LOGS, { recursive: true });
@@ -16155,12 +16344,13 @@ async function main() {
16155
16344
  for (const s of ranked) {
16156
16345
  if (picked.length >= limit) break;
16157
16346
  if (!REGRADE) {
16158
- const done = await fs6.readFile(path10.join(GRADES, `${s.sessionId}.json`), "utf8").catch(() => null);
16347
+ const done = await fs6.readFile(path11.join(GRADES2, `${s.sessionId}.json`), "utf8").catch(() => null);
16159
16348
  if (done) continue;
16160
16349
  }
16161
16350
  picked.push(s);
16162
16351
  }
16163
16352
  console.log(`[coding-grade] grading ${picked.length}${ALL ? " (ALL gradable, resuming)" : ""} on ${MODEL}, conc ${CONC}\u2026`);
16353
+ console.log(` (each session below is a single small ${MODEL} call \u2014 a long run is many tiny calls, not a huge one)`);
16164
16354
  const lim = makeLimiter(CONC);
16165
16355
  startRun("coding-grade", picked.length);
16166
16356
  let doneN = 0, sumUsd = 0, sumCalls = 0;
@@ -16168,19 +16358,19 @@ async function main() {
16168
16358
  await Promise.all(picked.map((s) => lim(async () => {
16169
16359
  const t0 = Date.now();
16170
16360
  try {
16171
- const g = await gradeSession(s, { model: MODEL, outDir: GRADES, logDir: LOGS });
16361
+ const g = await gradeSession(s, { model: MODEL, outDir: GRADES2, logDir: LOGS });
16172
16362
  tick(!!g);
16173
16363
  if (!g) {
16174
16364
  console.log(` \u2022 "${s.title.slice(0, 48)}" \u2014 skipped (no usable grade)`);
16175
16365
  return;
16176
16366
  }
16177
16367
  const scored = g.criteria.filter((c) => c.score != null).map((c) => `${c.key}=${c.score}`).join(" ");
16178
- console.log(` \u2022 "${s.title.slice(0, 48)}" done ${(Date.now() - t0) / 1e3 | 0}s $${g.meta.costUsd?.toFixed(2) ?? "?"} \xB7 ${g.meta.calls ?? "?"} call(s) \u2192 ${scored || "(all null)"}`);
16368
+ console.log(` \u2022 "${s.title.slice(0, 48)}" done ${(Date.now() - t0) / 1e3 | 0}s \u2192 ${scored || "(all null)"}`);
16179
16369
  doneN++;
16180
16370
  sumUsd += g.meta.costUsd ?? 0;
16181
16371
  sumCalls += g.meta.calls ?? 0;
16182
16372
  const elapsedMin = (Date.now() - runT0) / 6e4;
16183
- console.log(` \u03A3 ${doneN}/${picked.length} \xB7 $${sumUsd.toFixed(2)} \xB7 ${sumCalls} calls \xB7 ${elapsedMin.toFixed(0)}m \u2192 projected $${(sumUsd / doneN * picked.length).toFixed(0)} / ${(elapsedMin / doneN * picked.length / 60).toFixed(1)}h for this run`);
16373
+ console.log(` \u03A3 ${doneN}/${picked.length} \xB7 ${elapsedMin.toFixed(0)}m elapsed \u2192 ~${(elapsedMin / doneN * picked.length / 60).toFixed(1)}h total for this run (small ${MODEL} calls)`);
16184
16374
  } catch (e) {
16185
16375
  tick(false);
16186
16376
  console.log(` \u2022 "${s.title.slice(0, 48)}" ERROR ${e.message.slice(0, 60)}`);