polymath-society 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2420 -1573
- package/dist/engine/ingest-export.js +14 -8
- package/dist/engine/public-report.js +14 -8
- package/dist/index.js +1475 -60
- package/dist/pipeline/coding-agglomerate.js +289 -35
- package/dist/pipeline/coding-aggregate.js +15 -9
- package/dist/pipeline/coding-build.js +14 -8
- package/dist/pipeline/coding-coaching.js +273 -21
- package/dist/pipeline/coding-day-digest.js +17 -10
- package/dist/pipeline/coding-delegation.js +62 -17
- package/dist/pipeline/coding-expertise.js +166 -61
- package/dist/pipeline/coding-flow.js +14 -8
- package/dist/pipeline/coding-frontier-detail.js +143 -27
- package/dist/pipeline/coding-frontier.js +14 -8
- package/dist/pipeline/coding-gap-dist.js +14 -8
- package/dist/pipeline/coding-gap.js +140 -39
- package/dist/pipeline/coding-grade.js +44 -14
- package/dist/pipeline/coding-nutshell.js +39 -11
- package/dist/pipeline/coding-projects.js +41 -10
- package/dist/pipeline/coding-walkthrough.js +15 -9
- package/dist/web/app.js +493 -117
- package/dist/web/styles.css +1 -1
- package/package.json +1 -1
|
@@ -134,9 +134,9 @@ var require_secure_json_parse = __commonJS({
|
|
|
134
134
|
});
|
|
135
135
|
|
|
136
136
|
// ../../scripts/coding-expertise.mts
|
|
137
|
-
import { promises as
|
|
137
|
+
import { promises as fs7 } from "fs";
|
|
138
138
|
import os2 from "os";
|
|
139
|
-
import
|
|
139
|
+
import path11 from "path";
|
|
140
140
|
|
|
141
141
|
// ../../lib/agents/shared/agent.ts
|
|
142
142
|
import { readFileSync } from "fs";
|
|
@@ -2151,8 +2151,8 @@ function getErrorMap() {
|
|
|
2151
2151
|
|
|
2152
2152
|
// ../../node_modules/zod/v3/helpers/parseUtil.js
|
|
2153
2153
|
var makeIssue = (params) => {
|
|
2154
|
-
const { data, path:
|
|
2155
|
-
const fullPath = [...
|
|
2154
|
+
const { data, path: path12, errorMaps, issueData } = params;
|
|
2155
|
+
const fullPath = [...path12, ...issueData.path || []];
|
|
2156
2156
|
const fullIssue = {
|
|
2157
2157
|
...issueData,
|
|
2158
2158
|
path: fullPath
|
|
@@ -2268,11 +2268,11 @@ var errorUtil;
|
|
|
2268
2268
|
|
|
2269
2269
|
// ../../node_modules/zod/v3/types.js
|
|
2270
2270
|
var ParseInputLazyPath = class {
|
|
2271
|
-
constructor(parent, value,
|
|
2271
|
+
constructor(parent, value, path12, key) {
|
|
2272
2272
|
this._cachedPath = [];
|
|
2273
2273
|
this.parent = parent;
|
|
2274
2274
|
this.data = value;
|
|
2275
|
-
this._path =
|
|
2275
|
+
this._path = path12;
|
|
2276
2276
|
this._key = key;
|
|
2277
2277
|
}
|
|
2278
2278
|
get path() {
|
|
@@ -15059,39 +15059,39 @@ function createOpenAI(options = {}) {
|
|
|
15059
15059
|
});
|
|
15060
15060
|
const createChatModel = (modelId, settings = {}) => new OpenAIChatLanguageModel(modelId, settings, {
|
|
15061
15061
|
provider: `${providerName}.chat`,
|
|
15062
|
-
url: ({ path:
|
|
15062
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15063
15063
|
headers: getHeaders,
|
|
15064
15064
|
compatibility,
|
|
15065
15065
|
fetch: options.fetch
|
|
15066
15066
|
});
|
|
15067
15067
|
const createCompletionModel = (modelId, settings = {}) => new OpenAICompletionLanguageModel(modelId, settings, {
|
|
15068
15068
|
provider: `${providerName}.completion`,
|
|
15069
|
-
url: ({ path:
|
|
15069
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15070
15070
|
headers: getHeaders,
|
|
15071
15071
|
compatibility,
|
|
15072
15072
|
fetch: options.fetch
|
|
15073
15073
|
});
|
|
15074
15074
|
const createEmbeddingModel = (modelId, settings = {}) => new OpenAIEmbeddingModel(modelId, settings, {
|
|
15075
15075
|
provider: `${providerName}.embedding`,
|
|
15076
|
-
url: ({ path:
|
|
15076
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15077
15077
|
headers: getHeaders,
|
|
15078
15078
|
fetch: options.fetch
|
|
15079
15079
|
});
|
|
15080
15080
|
const createImageModel = (modelId, settings = {}) => new OpenAIImageModel(modelId, settings, {
|
|
15081
15081
|
provider: `${providerName}.image`,
|
|
15082
|
-
url: ({ path:
|
|
15082
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15083
15083
|
headers: getHeaders,
|
|
15084
15084
|
fetch: options.fetch
|
|
15085
15085
|
});
|
|
15086
15086
|
const createTranscriptionModel = (modelId) => new OpenAITranscriptionModel(modelId, {
|
|
15087
15087
|
provider: `${providerName}.transcription`,
|
|
15088
|
-
url: ({ path:
|
|
15088
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15089
15089
|
headers: getHeaders,
|
|
15090
15090
|
fetch: options.fetch
|
|
15091
15091
|
});
|
|
15092
15092
|
const createSpeechModel = (modelId) => new OpenAISpeechModel(modelId, {
|
|
15093
15093
|
provider: `${providerName}.speech`,
|
|
15094
|
-
url: ({ path:
|
|
15094
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15095
15095
|
headers: getHeaders,
|
|
15096
15096
|
fetch: options.fetch
|
|
15097
15097
|
});
|
|
@@ -15112,7 +15112,7 @@ function createOpenAI(options = {}) {
|
|
|
15112
15112
|
const createResponsesModel = (modelId) => {
|
|
15113
15113
|
return new OpenAIResponsesLanguageModel(modelId, {
|
|
15114
15114
|
provider: `${providerName}.responses`,
|
|
15115
|
-
url: ({ path:
|
|
15115
|
+
url: ({ path: path12 }) => `${baseURL}${path12}`,
|
|
15116
15116
|
headers: getHeaders,
|
|
15117
15117
|
fetch: options.fetch
|
|
15118
15118
|
});
|
|
@@ -15625,20 +15625,9 @@ function chunkText(text2, max) {
|
|
|
15625
15625
|
return out;
|
|
15626
15626
|
}
|
|
15627
15627
|
|
|
15628
|
-
// ../../lib/agents/coding/
|
|
15629
|
-
|
|
15630
|
-
|
|
15631
|
-
const kept = [];
|
|
15632
|
-
const dropped = [];
|
|
15633
|
-
for (const d of domains) {
|
|
15634
|
-
const dates = new Set(
|
|
15635
|
-
(d.evidence ?? []).map((e) => (e?.date ?? "").slice(0, 10)).filter(Boolean)
|
|
15636
|
-
);
|
|
15637
|
-
if (dates.size >= min && (d.sessions ?? 0) >= min) kept.push(d);
|
|
15638
|
-
else dropped.push({ domain: d.domain ?? "?", distinctDates: dates.size, sessions: d.sessions ?? 0 });
|
|
15639
|
-
}
|
|
15640
|
-
return { kept, dropped };
|
|
15641
|
-
}
|
|
15628
|
+
// ../../lib/agents/coding/docsDir.ts
|
|
15629
|
+
import { promises as fs5 } from "fs";
|
|
15630
|
+
import path9 from "path";
|
|
15642
15631
|
|
|
15643
15632
|
// ../../lib/agents/coding/materialize.ts
|
|
15644
15633
|
import { promises as fs4 } from "fs";
|
|
@@ -15783,13 +15772,115 @@ async function materializeSession(file2) {
|
|
|
15783
15772
|
return { text: lines.join("\n"), humanTurns, humanChars };
|
|
15784
15773
|
}
|
|
15785
15774
|
|
|
15775
|
+
// ../../lib/agents/coding/docsDir.ts
|
|
15776
|
+
async function writeScoreboard(codingDir, docs) {
|
|
15777
|
+
const gradesDir = path9.join(codingDir, "grades");
|
|
15778
|
+
let files = [];
|
|
15779
|
+
try {
|
|
15780
|
+
files = (await fs5.readdir(gradesDir)).filter((f) => f.endsWith(".json")).sort();
|
|
15781
|
+
} catch {
|
|
15782
|
+
return;
|
|
15783
|
+
}
|
|
15784
|
+
const byCriterion = /* @__PURE__ */ new Map();
|
|
15785
|
+
for (const f of files) {
|
|
15786
|
+
let g;
|
|
15787
|
+
try {
|
|
15788
|
+
g = JSON.parse(await fs5.readFile(path9.join(gradesDir, f), "utf8"));
|
|
15789
|
+
} catch {
|
|
15790
|
+
continue;
|
|
15791
|
+
}
|
|
15792
|
+
for (const c of g.criteria ?? []) {
|
|
15793
|
+
const s = c.score ?? c.bestEstimate;
|
|
15794
|
+
if (s == null) continue;
|
|
15795
|
+
const line = `L${s}${c.score == null ? "~" : ""} \xB7 ${(g.date ?? "").slice(0, 10)} \xB7 ${(g.title ?? g.sessionId ?? "").replace(/\n.*/s, "").slice(0, 60)} \xB7 ${g.sessionId} \u2014 ${(c.instance ?? "").slice(0, 110)}`;
|
|
15796
|
+
(byCriterion.get(c.key) ?? byCriterion.set(c.key, []).get(c.key)).push({ s, line });
|
|
15797
|
+
}
|
|
15798
|
+
}
|
|
15799
|
+
if (!byCriterion.size) return;
|
|
15800
|
+
const blocks = [...byCriterion.entries()].sort(([a], [b]) => a < b ? -1 : 1).map(
|
|
15801
|
+
([key, rows]) => `## ${key}
|
|
15802
|
+
${rows.sort((a, b) => b.s - a.s).map((r) => r.line).join("\n")}`
|
|
15803
|
+
);
|
|
15804
|
+
await fs5.writeFile(
|
|
15805
|
+
path9.join(docs, "_scoreboard.md"),
|
|
15806
|
+
`# Scoreboard \u2014 every graded session per criterion, best first.
|
|
15807
|
+
# Lx~ = estimate. Format: level \xB7 date \xB7 title \xB7 sessionId \u2014 graded instance.
|
|
15808
|
+
|
|
15809
|
+
${blocks.join("\n\n")}
|
|
15810
|
+
`
|
|
15811
|
+
);
|
|
15812
|
+
}
|
|
15813
|
+
async function ensureCodingDocs(codingDir, sessions) {
|
|
15814
|
+
const docs = path9.join(codingDir, "docs");
|
|
15815
|
+
await fs5.mkdir(docs, { recursive: true });
|
|
15816
|
+
const user = sessions.filter((s) => s.sessionId && !s.duplicateOf && (!("klass" in s) || s.klass === "interactive"));
|
|
15817
|
+
let written = 0;
|
|
15818
|
+
for (const s of user) {
|
|
15819
|
+
const out = path9.join(docs, `${s.sessionId}.txt`);
|
|
15820
|
+
try {
|
|
15821
|
+
await fs5.access(out);
|
|
15822
|
+
continue;
|
|
15823
|
+
} catch {
|
|
15824
|
+
}
|
|
15825
|
+
const mat = await materializeSession(s.file).catch(() => null);
|
|
15826
|
+
if (!mat?.text) continue;
|
|
15827
|
+
await fs5.writeFile(out, mat.text);
|
|
15828
|
+
written++;
|
|
15829
|
+
}
|
|
15830
|
+
const rows = user.sort((a, b) => (a.start ?? "") < (b.start ?? "") ? -1 : 1).map((s) => `${(s.start ?? "").slice(0, 10)} \xB7 ${(s.title ?? "").replace(/\n.*/s, "").slice(0, 80)} \xB7 ${s.sessionId}`);
|
|
15831
|
+
await fs5.writeFile(path9.join(docs, "_map.md"), rows.join("\n") + "\n");
|
|
15832
|
+
await writeScoreboard(codingDir, docs);
|
|
15833
|
+
if (written) console.log(`[docs] materialized ${written} new transcript(s) \u2192 ${docs}`);
|
|
15834
|
+
return docs;
|
|
15835
|
+
}
|
|
15836
|
+
function sourceLookupNote() {
|
|
15837
|
+
return `
|
|
15838
|
+
|
|
15839
|
+
SOURCE LOOKUP \u2014 you have Read/Grep/Glob over the working directory, which holds EVERYTHING known about this person's coding work. MAPS FIRST, grep second (the engine rule): docs/_scoreboard.md ranks every graded session per criterion, best first \u2014 one glance shows where the spikes/flair are, so start there when hunting peaks or checking whether a high mark is consistent or a one-off; day-digest.json is the day-by-day story of what they did in order \u2014 read the relevant days to get the arc before searching; docs/_map.md lists every session (date \xB7 title \xB7 sessionId). Then drill down: walkthroughs/<sessionId>.json is that session's phase-by-phase account (cheap, already distilled), grades/<sessionId>.json its graded findings with quotes, and docs/<sessionId>.txt the FULL transcript (">> " = the person, "[AI]" = the model) \u2014 the ground truth, read it when the distilled layers can't settle it. Other artifacts (flow.json, delegation/, expertise/, aggregate.json) are there too if a claim depends on them. The digest you were given is the spine \u2014 use lookups when a CLAIM-MOVING question cannot be settled from it (a moment you are about to headline, a promotion/depth call, an ambiguous engagement, whether detail was vision or boilerplate). VERIFY every verbatim quote you cite by finding it in the transcript first; if you cannot find it, do not use it. Look up what changes the output \u2014 do not re-read the whole corpus.`;
|
|
15840
|
+
}
|
|
15841
|
+
|
|
15842
|
+
// ../../lib/agents/coding/expertise-floor.ts
|
|
15843
|
+
var MIN_EXPERTISE_SESSIONS = 3;
|
|
15844
|
+
function applyExpertiseFloor(domains, min = MIN_EXPERTISE_SESSIONS) {
|
|
15845
|
+
const kept = [];
|
|
15846
|
+
const dropped = [];
|
|
15847
|
+
for (const d of domains) {
|
|
15848
|
+
const dates = new Set(
|
|
15849
|
+
(d.evidence ?? []).map((e) => (e?.date ?? "").slice(0, 10)).filter(Boolean)
|
|
15850
|
+
);
|
|
15851
|
+
if (dates.size >= min && (d.sessions ?? 0) >= min) kept.push(d);
|
|
15852
|
+
else dropped.push({ domain: d.domain ?? "?", distinctDates: dates.size, sessions: d.sessions ?? 0 });
|
|
15853
|
+
}
|
|
15854
|
+
return { kept, dropped };
|
|
15855
|
+
}
|
|
15856
|
+
|
|
15857
|
+
// ../../lib/agents/coding/promptCache.ts
|
|
15858
|
+
import { createHash } from "crypto";
|
|
15859
|
+
import { promises as fs6 } from "fs";
|
|
15860
|
+
function promptSha(parts) {
|
|
15861
|
+
const h = createHash("sha256");
|
|
15862
|
+
for (const p of parts) {
|
|
15863
|
+
h.update(p ?? "");
|
|
15864
|
+
h.update("\0");
|
|
15865
|
+
}
|
|
15866
|
+
return h.digest("hex").slice(0, 16);
|
|
15867
|
+
}
|
|
15868
|
+
async function reusableOutput(outPath, sha, valid, key = "promptSha") {
|
|
15869
|
+
try {
|
|
15870
|
+
const o = JSON.parse(await fs6.readFile(outPath, "utf8"));
|
|
15871
|
+
if (o[key] === sha && valid(o)) return o;
|
|
15872
|
+
} catch {
|
|
15873
|
+
}
|
|
15874
|
+
return null;
|
|
15875
|
+
}
|
|
15876
|
+
|
|
15786
15877
|
// ../../lib/agents/coding/profile.ts
|
|
15787
|
-
import
|
|
15878
|
+
import path10 from "path";
|
|
15788
15879
|
|
|
15789
15880
|
// ../../lib/calibration/fingerprint.ts
|
|
15790
|
-
import { createHash } from "node:crypto";
|
|
15881
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
15791
15882
|
function rubricFingerprint(criteria) {
|
|
15792
|
-
return
|
|
15883
|
+
return createHash2("sha256").update(JSON.stringify(criteria)).digest("hex").slice(0, 10);
|
|
15793
15884
|
}
|
|
15794
15885
|
|
|
15795
15886
|
// ../../lib/agents/coding/criteria.ts
|
|
@@ -15938,7 +16029,7 @@ var TRANSCRIPT_LINE_RE = new RegExp(`^\\s*(${NOTE})((,| and) ${NOTE})*\\s*$|^\\s
|
|
|
15938
16029
|
|
|
15939
16030
|
// ../../lib/calibration/index.ts
|
|
15940
16031
|
var DEFAULT_CALIBRATION = {
|
|
15941
|
-
version: "2026-07-
|
|
16032
|
+
version: "2026-07-14.2",
|
|
15942
16033
|
// = RUBRIC_FINGERPRINT of the shipped npm rubric (packages/coding-analyzer/
|
|
15943
16034
|
// src/grade/criteria.ts). A unit test fails loud when the rubric changes
|
|
15944
16035
|
// without this being updated (and re-pushed to the server).
|
|
@@ -16027,7 +16118,7 @@ var DEFAULT_CALIBRATION = {
|
|
|
16027
16118
|
}
|
|
16028
16119
|
],
|
|
16029
16120
|
codingTiers: {
|
|
16030
|
-
push: { median: 300, sigma:
|
|
16121
|
+
push: { median: 300, sigma: 0.91, tag: "estimated \u2014 log-normal fit", source: "Anthropic 2026 Claude Code study (400k sessions): median engaged user ~8 prompts/day at ~35 words \u2014 ~300 directed words/day; tail: a 1-in-1,000 user AVERAGES ~5k directed words/day (near-daily heavy dictation, clearly below the ~10k single-day record). AVERAGE over substantial days, cumulative not spiky." },
|
|
16031
16122
|
focus: { median: 0.08, sigma: 0.559, tag: "estimated \u2014 proxy-anchored", source: "median: Anthropic 20h/week engaged-user figure implies ~1h/day of true rapid exchange against an 8h working day; ceiling anchored to the ~4h/day deep-work limit (45% share = 1-in-1,000). AVERAGE share, cumulative not spiky." },
|
|
16032
16123
|
orchestration: { tag: "estimated \u2014 behavior bands", source: "no published concurrency distribution exists; ceiling anchored to the Claude Code lead's documented 10-15 parallel sessions" }
|
|
16033
16124
|
},
|
|
@@ -16060,12 +16151,18 @@ var DEFAULT_CALIBRATION = {
|
|
|
16060
16151
|
line: "the Claude Code lead runs 10\u201315 sessions at once; his #1 tip is 3\u20135 git worktrees in parallel"
|
|
16061
16152
|
},
|
|
16062
16153
|
throughput: {
|
|
16063
|
-
//
|
|
16064
|
-
//
|
|
16065
|
-
//
|
|
16066
|
-
//
|
|
16067
|
-
|
|
16068
|
-
|
|
16154
|
+
// Two DIFFERENT kinds of line (2026-07-14.2): topAvg is a PERCENTILE
|
|
16155
|
+
// claim — what a 1-in-1,000 user AVERAGES over substantial days (5k/day:
|
|
16156
|
+
// near-daily heavy dictation; a measured top dictating power user
|
|
16157
|
+
// averages ~3.9k). topPeak is NOT a percentile — it is the CEILING
|
|
16158
|
+
// reference, the wildest single days on record (~10k: a 14-16h
|
|
16159
|
+
// launch-day marathon), rendered as "wildest single days", never
|
|
16160
|
+
// "top X% peak". An avg-percentile and a peak-percentile can't both be
|
|
16161
|
+
// shown at a same-ish value (avg≈peak reads broken — user call), and a
|
|
16162
|
+
// peak PERCENTILE line is unknowable anyway; the record framing is
|
|
16163
|
+
// honest and keeps the visual gap.
|
|
16164
|
+
topAvgWordsPerDay: 5e3,
|
|
16165
|
+
topPeakWordsPerDay: 1e4,
|
|
16069
16166
|
benchLabel: "top 0.1%",
|
|
16070
16167
|
// No published "words typed/day" for heavy users — the real story is OUTPUT.
|
|
16071
16168
|
loopsHeadline: ">1,000,000 lines of Rust at 99.8% tests passing",
|
|
@@ -16082,14 +16179,15 @@ var BEST = DEFAULT_CALIBRATION.codingBenchmarks;
|
|
|
16082
16179
|
|
|
16083
16180
|
// ../../lib/agents/coding/profile.ts
|
|
16084
16181
|
var ROOT = process.cwd();
|
|
16085
|
-
var CODING_DIR = process.env.POLYMATH_DATA_DIR ?
|
|
16086
|
-
var GRADES =
|
|
16182
|
+
var CODING_DIR = process.env.POLYMATH_DATA_DIR ? path10.join(process.env.POLYMATH_DATA_DIR, "coding") : path10.join(ROOT, ".data", "coding");
|
|
16183
|
+
var GRADES = path10.join(CODING_DIR, "grades");
|
|
16087
16184
|
|
|
16088
16185
|
// ../../scripts/coding-expertise.mts
|
|
16089
16186
|
var CODING = CODING_DIR;
|
|
16090
|
-
var OUTDIR =
|
|
16187
|
+
var OUTDIR = path11.join(CODING, "expertise");
|
|
16091
16188
|
var CHUNK = 46e3;
|
|
16092
|
-
var
|
|
16189
|
+
var concFlag = process.argv.indexOf("--conc");
|
|
16190
|
+
var CONC = concFlag >= 0 ? Math.max(1, Number(process.argv[concFlag + 1]) || 5) : 5;
|
|
16093
16191
|
var EXTRACT_SYS = `You extract HARD TECHNICAL / SYSTEMS domain expertise from one coding session \u2014 ONLY genuine, specialized technical knowledge a very smart generalist who never worked in that domain could NOT produce. WHAT EXPERTISE IS \u2014 accumulated FIELD knowledge. An expert has internalized a domain's empirical landscape: what its practitioners collectively learned by trial and error \u2014 which approaches win in practice, named techniques / benchmarks / known results, the field's failure taxonomies, tool trade-offs and their folklore. It is knowledge that exists OUTSIDE the system in this session and could only come from having read, built, and experimented deeply IN the field itself. THE BAR is HARD systems/infra domains: memory architectures, vector databases, graph databases, RAG / retrieval systems, agent harnesses, distributed systems, database internals, kernels / OS, networking, GPU / CUDA, compilers, concurrency, ML training / serving infrastructure, large-scale data engineering. HIRING-BAR TEST \u2014 apply to EVERY candidate signal: would this line convince a team hiring a cutting-edge specialist in that exact domain that this person already operates at that level? If it merely shows a very smart person reasoning well about the system in front of them, it fails the bar \u2192 exclude. DERIVED-ON-THE-SPOT \u2260 EXPERTISE \u2014 the most common false positive. A correct, even mechanism-level, call that a smart generalist could DERIVE in the moment by reasoning about the system in front of them \u2014 cost/latency arithmetic, quality-vs-price trade-offs between components, 'the bottleneck will be X', 'cache that, it's called in a loop', 'shard by the hot key' read straight off the schema at hand \u2014 is GENERAL REASONING, not expertise, EVEN WHEN the AI hadn't thought of it and the person turned out right. Real expertise REFERENCES the field: named methods, known empirical results, failure modes a non-practitioner would not know existed. EXCLUDE ENTIRELY \u2014 these do NOT count here: the person's own PRODUCT / APPLICATION area (evaluating humans, personality / psychometrics / Big Five / OCEAN, talent assessment, rubric design for their own scoring product, analyzing chat-log corpora), generic Claude Code / CLI usage, common sense, general intelligence, and building a product that merely USES a domain's concepts. Building an evaluator FOR X is NOT expertise IN X. NARROW-SLICE RULE \u2014 fluently handling the narrow slice of a domain that the person's OWN product consumes is USING the domain, not expertise in it (tuning your own app's queries does not make you a database-internals expert; wiring a vector search into your app does not make you a vector-DB expert). Capture TWO distinct KINDS of signal and TAG each with "type": (1) "built" \u2014 the person HANDS-ON built, optimized, or deeply experimented with a hard technical SYSTEM in this session: implemented internals, tried multiple approaches, debugged it at the mechanism level, studied how it works. Real sustained engagement with the system itself. (2) "directs" \u2014 the person DIRECTS the AI with OPINIONATED, MECHANISM-LEVEL technical authority in a hard domain, even without building it here: e.g. 'use HNSW not IVF-flat, recall collapses past ~1M vectors', 'wrong index for this access pattern', 'that isolation level lets the anomaly through'. The knowledge must be SPECIALIZED and NON-INTUITIVE \u2014 something a very smart generalist could NOT reach by good judgment alone. EXCLUDE good architectural / engineering JUDGMENT (fan-out in parallel, explore-before-building, staging, caching, don't-use-a-critic, cheap-pass-first-expensive-pass-when-warranted) \u2014 that is GENERAL INTELLIGENCE, not domain expertise. For BOTH, the knowledge must be PREDICTIVE / constraint-level and clearly require accumulated specialized experience. TRANSFERABILITY TEST: could a very smart generalist produce this from common sense + on-the-spot reasoning about the system at hand? YES \u2192 exclude. When in doubt, exclude. Most sessions show NO hard-technical-domain expertise \u2014 say so plainly. Never flatter, never pad, never invent.`;
|
|
16094
16192
|
function extractPrompt(rec, transcript, seg = "") {
|
|
16095
16193
|
return `SESSION: "${rec.title}" \xB7 ${rec.start?.slice(0, 10)}${seg}
|
|
@@ -16128,17 +16226,17 @@ async function pool(items, n, fn) {
|
|
|
16128
16226
|
}
|
|
16129
16227
|
async function main() {
|
|
16130
16228
|
const aggOnly = process.argv.includes("--agg");
|
|
16131
|
-
await
|
|
16132
|
-
const sessions = JSON.parse(await
|
|
16229
|
+
await fs7.mkdir(OUTDIR, { recursive: true });
|
|
16230
|
+
const sessions = JSON.parse(await fs7.readFile(path11.join(CODING, "sessions.json"), "utf8"));
|
|
16133
16231
|
const byId = new Map(sessions.map((s) => [s.sessionId, s]));
|
|
16134
|
-
const gradeFiles = (await
|
|
16232
|
+
const gradeFiles = (await fs7.readdir(path11.join(CODING, "grades"))).filter((f) => f.endsWith(".json")).sort();
|
|
16135
16233
|
const ids = gradeFiles.map((f) => f.replace(".json", ""));
|
|
16136
16234
|
if (!aggOnly) {
|
|
16137
16235
|
let done = 0;
|
|
16138
16236
|
await pool(ids, CONC, async (id) => {
|
|
16139
|
-
const out =
|
|
16237
|
+
const out = path11.join(OUTDIR, id + ".json");
|
|
16140
16238
|
try {
|
|
16141
|
-
await
|
|
16239
|
+
await fs7.access(out);
|
|
16142
16240
|
done++;
|
|
16143
16241
|
return;
|
|
16144
16242
|
} catch {
|
|
@@ -16147,7 +16245,7 @@ async function main() {
|
|
|
16147
16245
|
if (!rec) return;
|
|
16148
16246
|
const mat = await materializeSession(rec.file);
|
|
16149
16247
|
if (!mat) return;
|
|
16150
|
-
const
|
|
16248
|
+
const sandbox = await fs7.mkdtemp(path11.join(os2.tmpdir(), "exp-"));
|
|
16151
16249
|
try {
|
|
16152
16250
|
const chunks = chunkText(mat.text, CHUNK);
|
|
16153
16251
|
const realExpertise = [];
|
|
@@ -16156,49 +16254,56 @@ async function main() {
|
|
|
16156
16254
|
const summaries = [];
|
|
16157
16255
|
for (let ci = 0; ci < chunks.length; ci++) {
|
|
16158
16256
|
const seg = chunks.length > 1 ? ` \xB7 segment ${ci + 1}/${chunks.length}` : "";
|
|
16159
|
-
const run2 = await invokeAgent({ prompt: extractPrompt(rec, chunks[ci], seg), systemPrompt: EXTRACT_SYS, addDir:
|
|
16257
|
+
const run2 = await invokeAgent({ prompt: extractPrompt(rec, chunks[ci], seg), systemPrompt: EXTRACT_SYS, addDir: sandbox, allowedTools: ["Read"], maxTurns: 5, timeoutMs: 24e4, model: "sonnet" });
|
|
16160
16258
|
const j = run2.json || {};
|
|
16161
16259
|
realExpertise.push(...j.realExpertise || []);
|
|
16162
16260
|
excluded.push(...j.excluded || []);
|
|
16163
16261
|
for (const d of j.domains || []) if (!domains.includes(d)) domains.push(d);
|
|
16164
16262
|
if (j.summary) summaries.push(j.summary);
|
|
16165
16263
|
}
|
|
16166
|
-
await
|
|
16264
|
+
await fs7.writeFile(out, JSON.stringify({ sessionId: id, date: rec.start?.slice(0, 10), title: rec.title, realExpertise, excluded, domains, summary: summaries.join(" ") }, null, 1));
|
|
16167
16265
|
done++;
|
|
16168
16266
|
console.log(`[${done}/${ids.length}] ${rec.title} \u2014 ${realExpertise.length} real`);
|
|
16169
16267
|
} catch (e) {
|
|
16170
16268
|
console.log(`ERR ${rec.title}: ${e.message.slice(0, 80)}`);
|
|
16171
16269
|
} finally {
|
|
16172
|
-
await
|
|
16270
|
+
await fs7.rm(sandbox, { recursive: true, force: true }).catch(() => {
|
|
16173
16271
|
});
|
|
16174
16272
|
}
|
|
16175
16273
|
});
|
|
16176
16274
|
}
|
|
16177
16275
|
const perSession = [];
|
|
16178
|
-
for (const f of (await
|
|
16276
|
+
for (const f of (await fs7.readdir(OUTDIR)).filter((f2) => f2.endsWith(".json")).sort()) {
|
|
16179
16277
|
try {
|
|
16180
|
-
perSession.push(JSON.parse(await
|
|
16278
|
+
perSession.push(JSON.parse(await fs7.readFile(path11.join(OUTDIR, f), "utf8")));
|
|
16181
16279
|
} catch {
|
|
16182
16280
|
}
|
|
16183
16281
|
}
|
|
16184
16282
|
const items = perSession.flatMap((s) => (s.realExpertise || []).map((e) => ({ ...e, date: s.date, title: s.title })));
|
|
16185
16283
|
const titles = perSession.map((s) => ({ date: s.date, title: s.title })).filter((t) => t.title);
|
|
16186
16284
|
const stats = { sessionsProcessed: perSession.length, sessionsWithExpertise: perSession.filter((s) => (s.realExpertise || []).length).length, sessionsZero: perSession.filter((s) => !(s.realExpertise || []).length).length, totalSignals: items.length };
|
|
16285
|
+
const AGG_SYSX = AGG_SYS + sourceLookupNote();
|
|
16286
|
+
const prompt2 = aggPrompt(items, stats, titles);
|
|
16287
|
+
const sha = promptSha([AGG_SYSX, prompt2, "sonnet"]);
|
|
16288
|
+
if (!process.argv.includes("--refresh")) {
|
|
16289
|
+
const prev = await reusableOutput(path11.join(CODING, "expertise.json"), sha, (o) => Array.isArray(o.domains));
|
|
16290
|
+
if (prev) {
|
|
16291
|
+
console.log(`[expertise] inputs unchanged \u2014 reusing aggregate from ${prev.generatedAt} (--refresh to regenerate)`);
|
|
16292
|
+
return;
|
|
16293
|
+
}
|
|
16294
|
+
}
|
|
16295
|
+
await ensureCodingDocs(CODING, sessions);
|
|
16187
16296
|
console.log("\nAGGREGATING", stats);
|
|
16188
|
-
const sandbox = await fs5.mkdtemp(path10.join(os2.tmpdir(), "expagg-"));
|
|
16189
16297
|
let map = { domains: [], building: [], summary: "", directiveQuality: "", seeingAroundCorners: "" };
|
|
16190
16298
|
let aggOk = false;
|
|
16191
16299
|
try {
|
|
16192
|
-
const run2 = await invokeAgent({ prompt:
|
|
16300
|
+
const run2 = await invokeAgent({ prompt: prompt2, systemPrompt: AGG_SYSX, addDir: CODING, allowedTools: ["Read", "Grep", "Glob"], maxTurns: 10, timeoutMs: 54e4, model: "sonnet" });
|
|
16193
16301
|
if (run2.json) {
|
|
16194
16302
|
map = run2.json;
|
|
16195
16303
|
aggOk = true;
|
|
16196
16304
|
}
|
|
16197
16305
|
} catch (e) {
|
|
16198
16306
|
console.log("AGG ERR", e.message.slice(0, 120));
|
|
16199
|
-
} finally {
|
|
16200
|
-
await fs5.rm(sandbox, { recursive: true, force: true }).catch(() => {
|
|
16201
|
-
});
|
|
16202
16307
|
}
|
|
16203
16308
|
const REJECT_DOMAIN = /psychometr|big.?five|\bocean\b|personality|\btalent\b|performance assessment|agent.?harness|harness (design|architect)/i;
|
|
16204
16309
|
if (Array.isArray(map.domains)) {
|
|
@@ -16210,15 +16315,15 @@ async function main() {
|
|
|
16210
16315
|
map.domains = kept;
|
|
16211
16316
|
}
|
|
16212
16317
|
if (!aggOk && !(map.domains || []).length) {
|
|
16213
|
-
const existing = await
|
|
16318
|
+
const existing = await fs7.readFile(path11.join(CODING, "expertise.json"), "utf8").then(JSON.parse).catch(() => null);
|
|
16214
16319
|
if (existing && (existing.domains || []).length) {
|
|
16215
16320
|
console.log("AGG FAILED (0 domains) \u2014 keeping existing", existing.domains.length, "domains");
|
|
16216
16321
|
return;
|
|
16217
16322
|
}
|
|
16218
16323
|
console.log("AGG failed and no good existing file \u2014 writing empty");
|
|
16219
16324
|
}
|
|
16220
|
-
const result = { generatedAt: (/* @__PURE__ */ new Date()).toISOString(), ...stats, ...map };
|
|
16221
|
-
await
|
|
16325
|
+
const result = { generatedAt: (/* @__PURE__ */ new Date()).toISOString(), ...aggOk ? { promptSha: sha } : {}, ...stats, ...map };
|
|
16326
|
+
await fs7.writeFile(path11.join(CODING, "expertise.json"), JSON.stringify(result, null, 2));
|
|
16222
16327
|
console.log("\nWROTE .data/coding/expertise.json \u2014", (map.domains || []).length, "domains");
|
|
16223
16328
|
}
|
|
16224
16329
|
main().catch((e) => {
|
|
@@ -394,7 +394,7 @@ var IDLE_CAP_MS = 12 * 6e4;
|
|
|
394
394
|
|
|
395
395
|
// ../../lib/calibration/index.ts
|
|
396
396
|
var DEFAULT_CALIBRATION = {
|
|
397
|
-
version: "2026-07-
|
|
397
|
+
version: "2026-07-14.2",
|
|
398
398
|
// = RUBRIC_FINGERPRINT of the shipped npm rubric (packages/coding-analyzer/
|
|
399
399
|
// src/grade/criteria.ts). A unit test fails loud when the rubric changes
|
|
400
400
|
// without this being updated (and re-pushed to the server).
|
|
@@ -483,7 +483,7 @@ var DEFAULT_CALIBRATION = {
|
|
|
483
483
|
}
|
|
484
484
|
],
|
|
485
485
|
codingTiers: {
|
|
486
|
-
push: { median: 300, sigma:
|
|
486
|
+
push: { median: 300, sigma: 0.91, tag: "estimated \u2014 log-normal fit", source: "Anthropic 2026 Claude Code study (400k sessions): median engaged user ~8 prompts/day at ~35 words \u2014 ~300 directed words/day; tail: a 1-in-1,000 user AVERAGES ~5k directed words/day (near-daily heavy dictation, clearly below the ~10k single-day record). AVERAGE over substantial days, cumulative not spiky." },
|
|
487
487
|
focus: { median: 0.08, sigma: 0.559, tag: "estimated \u2014 proxy-anchored", source: "median: Anthropic 20h/week engaged-user figure implies ~1h/day of true rapid exchange against an 8h working day; ceiling anchored to the ~4h/day deep-work limit (45% share = 1-in-1,000). AVERAGE share, cumulative not spiky." },
|
|
488
488
|
orchestration: { tag: "estimated \u2014 behavior bands", source: "no published concurrency distribution exists; ceiling anchored to the Claude Code lead's documented 10-15 parallel sessions" }
|
|
489
489
|
},
|
|
@@ -516,12 +516,18 @@ var DEFAULT_CALIBRATION = {
|
|
|
516
516
|
line: "the Claude Code lead runs 10\u201315 sessions at once; his #1 tip is 3\u20135 git worktrees in parallel"
|
|
517
517
|
},
|
|
518
518
|
throughput: {
|
|
519
|
-
//
|
|
520
|
-
//
|
|
521
|
-
//
|
|
522
|
-
//
|
|
523
|
-
|
|
524
|
-
|
|
519
|
+
// Two DIFFERENT kinds of line (2026-07-14.2): topAvg is a PERCENTILE
|
|
520
|
+
// claim — what a 1-in-1,000 user AVERAGES over substantial days (5k/day:
|
|
521
|
+
// near-daily heavy dictation; a measured top dictating power user
|
|
522
|
+
// averages ~3.9k). topPeak is NOT a percentile — it is the CEILING
|
|
523
|
+
// reference, the wildest single days on record (~10k: a 14-16h
|
|
524
|
+
// launch-day marathon), rendered as "wildest single days", never
|
|
525
|
+
// "top X% peak". An avg-percentile and a peak-percentile can't both be
|
|
526
|
+
// shown at a same-ish value (avg≈peak reads broken — user call), and a
|
|
527
|
+
// peak PERCENTILE line is unknowable anyway; the record framing is
|
|
528
|
+
// honest and keeps the visual gap.
|
|
529
|
+
topAvgWordsPerDay: 5e3,
|
|
530
|
+
topPeakWordsPerDay: 1e4,
|
|
525
531
|
benchLabel: "top 0.1%",
|
|
526
532
|
// No published "words typed/day" for heavy users — the real story is OUTPUT.
|
|
527
533
|
loopsHeadline: ">1,000,000 lines of Rust at 99.8% tests passing",
|