sdkproof 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +192 -0
  3. package/dist/cli.d.ts +2 -0
  4. package/dist/cli.js +201 -0
  5. package/dist/cli.js.map +1 -0
  6. package/dist/core/classify.d.ts +4 -0
  7. package/dist/core/classify.js +72 -0
  8. package/dist/core/classify.js.map +1 -0
  9. package/dist/core/model.d.ts +30 -0
  10. package/dist/core/model.js +27 -0
  11. package/dist/core/model.js.map +1 -0
  12. package/dist/core/prompt.d.ts +4 -0
  13. package/dist/core/prompt.js +43 -0
  14. package/dist/core/prompt.js.map +1 -0
  15. package/dist/core/score.d.ts +3 -0
  16. package/dist/core/score.js +37 -0
  17. package/dist/core/score.js.map +1 -0
  18. package/dist/core/stats.d.ts +99 -0
  19. package/dist/core/stats.js +121 -0
  20. package/dist/core/stats.js.map +1 -0
  21. package/dist/core/types.d.ts +162 -0
  22. package/dist/core/types.js +11 -0
  23. package/dist/core/types.js.map +1 -0
  24. package/dist/core/verify.d.ts +33 -0
  25. package/dist/core/verify.js +191 -0
  26. package/dist/core/verify.js.map +1 -0
  27. package/dist/drift.d.ts +87 -0
  28. package/dist/drift.js +105 -0
  29. package/dist/drift.js.map +1 -0
  30. package/dist/index.d.ts +19 -0
  31. package/dist/index.js +19 -0
  32. package/dist/index.js.map +1 -0
  33. package/dist/models.d.ts +33 -0
  34. package/dist/models.js +186 -0
  35. package/dist/models.js.map +1 -0
  36. package/dist/registry.d.ts +65 -0
  37. package/dist/registry.js +98 -0
  38. package/dist/registry.js.map +1 -0
  39. package/dist/report.d.ts +26 -0
  40. package/dist/report.js +240 -0
  41. package/dist/report.js.map +1 -0
  42. package/dist/run.d.ts +36 -0
  43. package/dist/run.js +188 -0
  44. package/dist/run.js.map +1 -0
  45. package/dist/surface.d.ts +95 -0
  46. package/dist/surface.js +281 -0
  47. package/dist/surface.js.map +1 -0
  48. package/dist/tarball.d.ts +7 -0
  49. package/dist/tarball.js +66 -0
  50. package/dist/tarball.js.map +1 -0
  51. package/dist/tasks.d.ts +41 -0
  52. package/dist/tasks.js +199 -0
  53. package/dist/tasks.js.map +1 -0
  54. package/dist/workspace.d.ts +42 -0
  55. package/dist/workspace.js +251 -0
  56. package/dist/workspace.js.map +1 -0
  57. package/package.json +58 -0
package/dist/models.js ADDED
@@ -0,0 +1,186 @@
1
+ import { FatalApiError, RefusalError } from "./core/model.js";
2
+ /** Stochastic refusals on benign SDK tasks; re-sample the same prompt this many times. */
3
+ const REFUSAL_ATTEMPTS = 4;
4
+ /** Transient server errors (529 overloaded, 429 rate limit) — retried with backoff. */
5
+ const TRANSIENT_ATTEMPTS = 5;
6
+ /**
7
+ * Set once a fatal error is seen, so in-flight siblings fail instantly instead
8
+ * of each making its own doomed request.
9
+ */
10
+ let fatal = null;
11
+ /** Reset between runs; only the test suite and long-lived embeddings need this. */
12
+ export function clearFatal() {
13
+ fatal = null;
14
+ }
15
+ export class ApiError extends Error {
16
+ status;
17
+ constructor(status, message) {
18
+ super(message);
19
+ this.status = status;
20
+ this.name = "ApiError";
21
+ }
22
+ }
23
+ async function post(url, headers, body, timeoutMs) {
24
+ const ac = new AbortController();
25
+ const timer = setTimeout(() => ac.abort(), timeoutMs);
26
+ try {
27
+ const res = await fetch(url, {
28
+ method: "POST",
29
+ headers: { "content-type": "application/json", ...headers },
30
+ body: JSON.stringify(body),
31
+ signal: ac.signal,
32
+ });
33
+ const text = await res.text();
34
+ if (!res.ok)
35
+ throw new ApiError(res.status, `${res.status} ${text.slice(0, 400)}`);
36
+ return JSON.parse(text);
37
+ }
38
+ finally {
39
+ clearTimeout(timer);
40
+ }
41
+ }
42
+ /**
43
+ * Retry the transient failures and stop dead on the terminal ones.
44
+ *
45
+ * A 529/429 is the API being busy, not the model saying anything. Left
46
+ * unretried it drops a task from the run — and a dropped task silently shrinks
47
+ * the denominator, which on 2026-08-05 scored a context arm HIGHER than bare
48
+ * simply because overload took its hardest tasks away. A 401/403, or a 400
49
+ * about billing, is terminal for every remaining request: retrying it just
50
+ * spends the rest of the run failing.
51
+ */
52
+ async function withRetry(fn) {
53
+ for (let t = 1;; t++) {
54
+ if (fatal)
55
+ throw new FatalApiError(fatal);
56
+ try {
57
+ return await fn();
58
+ }
59
+ catch (e) {
60
+ const status = e instanceof ApiError ? e.status : 0;
61
+ const msg = String(e.message ?? "");
62
+ if (status === 401 || status === 403 || (status === 400 && /credit balance|billing|quota/i.test(msg))) {
63
+ fatal = msg.slice(0, 200);
64
+ throw new FatalApiError(fatal);
65
+ }
66
+ const transient = status === 429 || status >= 500 || /aborted|ETIMEDOUT|ECONNRESET|fetch failed/i.test(msg);
67
+ if (!transient || t === TRANSIENT_ATTEMPTS)
68
+ throw e;
69
+ await new Promise((r) => setTimeout(r, 1500 * 2 ** (t - 1)));
70
+ }
71
+ }
72
+ }
73
+ /**
74
+ * Claude adapter over the raw Messages API — no SDK, so `npx sdkproof` installs
75
+ * one dependency (typescript) instead of a tree.
76
+ *
77
+ * ANTHROPIC_AUTH_TOKEN is accepted alongside ANTHROPIC_API_KEY because an OAuth
78
+ * token from a Claude subscription is the key most people already have.
79
+ */
80
+ export function anthropicAdapter(model) {
81
+ const base = process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com";
82
+ const key = process.env.ANTHROPIC_API_KEY;
83
+ const token = process.env.ANTHROPIC_AUTH_TOKEN;
84
+ return {
85
+ id: model,
86
+ async generate({ system, user, maxTokens = 16000 }) {
87
+ if (fatal)
88
+ throw new FatalApiError(fatal);
89
+ // Both guards below exist because either failure used to arrive as an
90
+ // EMPTY candidate, and an empty file compiles clean — so a generation
91
+ // failure was scored as a perfect answer. Found 2026-08-04 on the first
92
+ // Stripe run, which reported 100/100 with four of fifteen candidates blank.
93
+ for (let attempt = 1; attempt <= REFUSAL_ATTEMPTS; attempt++) {
94
+ const res = await withRetry(() => post(`${base}/v1/messages`, {
95
+ "anthropic-version": "2023-06-01",
96
+ ...(key ? { "x-api-key": key } : {}),
97
+ ...(token ? { authorization: `Bearer ${token}` } : {}),
98
+ }, { model, max_tokens: maxTokens, system, messages: [{ role: "user", content: user }] }, 300_000));
99
+ // A truncated completion loses its closing fence, so extraction returns
100
+ // a fragment or nothing. That is a harness problem, not model drift.
101
+ if (res.stop_reason === "max_tokens") {
102
+ throw new Error(`generation truncated at max_tokens=${maxTokens} — raise it with --max-tokens or shorten the task`);
103
+ }
104
+ // Refusals are stochastic on ordinary SDK tasks: measured 2026-08-04,
105
+ // the same Stripe prompts came back refusal/end_turn/end_turn and
106
+ // end_turn/refusal/end_turn across three trials, including a task that
107
+ // just lists customers. Re-sampling the identical prompt is the honest
108
+ // response — the prompt is not reworded to avoid the classifier, and a
109
+ // task that refuses every time still fails rather than scoring.
110
+ if (res.stop_reason === "refusal") {
111
+ if (attempt === REFUSAL_ATTEMPTS)
112
+ throw new RefusalError(REFUSAL_ATTEMPTS);
113
+ continue;
114
+ }
115
+ const blocks = (res.content ?? []);
116
+ return blocks.filter((b) => b.type === "text").map((b) => b.text ?? "").join("\n");
117
+ }
118
+ throw new Error("unreachable");
119
+ },
120
+ };
121
+ }
122
+ /** GPT adapter over the raw Chat Completions API. */
123
+ export function openaiAdapter(model) {
124
+ const base = process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
125
+ return {
126
+ id: model,
127
+ async generate({ system, user, maxTokens = 16000 }) {
128
+ if (fatal)
129
+ throw new FatalApiError(fatal);
130
+ const res = await withRetry(() => post(`${base}/chat/completions`, { authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ""}` }, {
131
+ model,
132
+ max_completion_tokens: maxTokens,
133
+ messages: [
134
+ { role: "system", content: system },
135
+ { role: "user", content: user },
136
+ ],
137
+ }, 300_000));
138
+ const choice = res.choices?.[0];
139
+ // Same guard as Anthropic's max_tokens: a cut-off completion is a harness
140
+ // failure, and it must not reach verify() as an empty candidate.
141
+ if (choice?.finish_reason === "length") {
142
+ throw new Error(`generation truncated at max_completion_tokens=${maxTokens} — raise it with --max-tokens or shorten the task`);
143
+ }
144
+ if (choice?.message?.refusal) {
145
+ throw new RefusalError(1);
146
+ }
147
+ return choice?.message?.content ?? "";
148
+ },
149
+ };
150
+ }
151
+ export const DEFAULT_ANTHROPIC_MODEL = process.env.SDKPROOF_ANTHROPIC_MODEL ?? "claude-opus-5";
152
+ export const DEFAULT_OPENAI_MODEL = process.env.SDKPROOF_OPENAI_MODEL ?? "gpt-5";
153
+ /**
154
+ * Turn a `--model` value into a provider + model id. Accepts an explicit
155
+ * `anthropic:<id>` / `openai:<id>`, or a bare id whose provider is inferred
156
+ * from its prefix — so `--model claude-sonnet-5` and `--model gpt-5` both work.
157
+ */
158
+ export function parseModelRef(value) {
159
+ const [head, ...rest] = value.split(":");
160
+ if (rest.length && (head === "anthropic" || head === "openai")) {
161
+ return { provider: head, model: rest.join(":") };
162
+ }
163
+ if (/^claude/i.test(value))
164
+ return { provider: "anthropic", model: value };
165
+ if (/^(gpt|o\d)/i.test(value))
166
+ return { provider: "openai", model: value };
167
+ throw new Error(`cannot tell which provider "${value}" belongs to — write it as anthropic:<id> or openai:<id>`);
168
+ }
169
+ export function adapterFor(ref) {
170
+ return ref.provider === "anthropic" ? anthropicAdapter(ref.model) : openaiAdapter(ref.model);
171
+ }
172
+ export function hasKeyFor(provider) {
173
+ return provider === "anthropic"
174
+ ? Boolean(process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_AUTH_TOKEN)
175
+ : Boolean(process.env.OPENAI_API_KEY);
176
+ }
177
+ /** Every model the environment holds a key for, when --model was not given. */
178
+ export function defaultAdapters() {
179
+ const out = [];
180
+ if (hasKeyFor("anthropic"))
181
+ out.push(anthropicAdapter(DEFAULT_ANTHROPIC_MODEL));
182
+ if (hasKeyFor("openai"))
183
+ out.push(openaiAdapter(DEFAULT_OPENAI_MODEL));
184
+ return out;
185
+ }
186
+ //# sourceMappingURL=models.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE9D,0FAA0F;AAC1F,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,uFAAuF;AACvF,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B;;;GAGG;AACH,IAAI,KAAK,GAAkB,IAAI,CAAC;AAEhC,mFAAmF;AACnF,MAAM,UAAU,UAAU;IACxB,KAAK,GAAG,IAAI,CAAC;AACf,CAAC;AAED,MAAM,OAAO,QAAS,SAAQ,KAAK;IACZ,MAAM;IAA3B,YAAqB,MAAc,EAAE,OAAe;QAClD,KAAK,CAAC,OAAO,CAAC,CAAC;sBADI,MAAM;QAEzB,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;IACzB,CAAC;CACF;AAED,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,OAA+B,EAAE,IAAa,EAAE,SAAiB;IAChG,MAAM,EAAE,GAAG,IAAI,eAAe,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,EAAE;YAC3D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,EAAE,CAAC,MAAM;SAClB,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAwB,CAAC;IACjD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,SAAS,CAAI,EAAoB;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAI,CAAC,EAAE,EAAE,CAAC;QACtB,IAAI,KAAK;YAAE,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,MAAM,GAAG,CAAC,YAAY,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,GAAG,GAAG,MAAM,CAAE,CAAW,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,+BAA+B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACtG,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1B,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,IAAI,4CAA4C,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5G,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,kBAAkB;gBAAE,MAAM,CAAC,CAAC;YACpD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,2BAA2B,CAAC;IAC3E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IAC/C,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAmB;YACjE,IAAI,KAAK;gBAAE,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;YAC1C,sEAAsE;YACtE,sEAAsE;YACtE,wEAAwE;YACxE,4EAA4E;YAC5E,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,gBAAgB,EAAE,OAAO,EAAE,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAC/B,IAAI,CACF,GAAG,IAAI,cAAc,EACrB;oBACE,mBAAmB,EAAE,YAAY;oBACjC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvD,EACD,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,EACrF,OAAO,CACR,CACF,CAAC;gBAEF,wEAAwE;gBACxE,qEAAqE;gBACrE,IAAI,GAAG,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CACb,sCAAsC,SAAS,mDAAmD,CACnG,CAAC;gBACJ,CAAC;gBAED,sEAAsE;gBACtE,kEAAkE;gBAClE,uEAAuE;gBACvE,uEAAuE;gBACvE,uEAAuE;gBACvE,gEAAgE;gBAChE,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;oBAClC,IAAI,OAAO,KAAK,gBAAgB;wBAAE,MAAM,IAAI,YAAY,CAAC,gBAAgB,CAAC,CAAC;oBAC3E,SAAS;gBACX,CAAC;gBAED,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAA2C,CAAC;gBAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;QACjC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,qDAAqD;AACrD,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,2BAA2B,CAAC;IACxE,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAmB;YACjE,IAAI,KAAK;gBAAE,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;YAC1C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAC/B,IAAI,CACF,GAAG,IAAI,mBAAmB,EAC1B,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,EAAE,EAAE,EAC/D;gBACE,KAAK;gBACL,qBAAqB,EAAE,SAAS;gBAChC,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;oBACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;iBAChC;aACF,EACD,OAAO,CACR,CACF,CAAC;YACF,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YAChC,0EAA0E;YAC1E,iEAAiE;YACjE,IAAI,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,iDAAiD,SAAS,mDAAmD,CAC9G,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;YAC5B,CAAC;YACD,OAAO,MAAM,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QACxC,CAAC;KACF,CAAC;AACJ,CAAC;AAOD,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,eAAe,CAAC;AAC/F,MAAM,CAAC,MAAM,oBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC;AAEjF;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;QAC/D,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACnD,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3E,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3E,MAAM,IAAI,KAAK,CACb,+BAA+B,KAAK,0DAA0D,CAC/F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAa;IACtC,OAAO,GAAG,CAAC,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,QAA8B;IACtD,OAAO,QAAQ,KAAK,WAAW;QAC7B,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QAC5E,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;AAC1C,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,eAAe;IAC7B,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,IAAI,SAAS,CAAC,WAAW,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC,CAAC;IAChF,IAAI,SAAS,CAAC,QAAQ,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACvE,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,65 @@
1
+ /**
2
+ * npm registry access, with no dependencies. Everything here is one GET against
3
+ * registry.npmjs.org and some arithmetic on the packument — no `npm` process,
4
+ * no semver package.
5
+ */
6
+ export interface Packument {
7
+ name: string;
8
+ "dist-tags": Record<string, string>;
9
+ versions: Record<string, VersionMeta>;
10
+ time: Record<string, string>;
11
+ readme?: string;
12
+ description?: string;
13
+ repository?: {
14
+ url?: string;
15
+ } | string;
16
+ homepage?: string;
17
+ }
18
+ export interface VersionMeta {
19
+ name: string;
20
+ version: string;
21
+ description?: string;
22
+ types?: string;
23
+ typings?: string;
24
+ readme?: string;
25
+ dependencies?: Record<string, string>;
26
+ peerDependencies?: Record<string, string>;
27
+ peerDependenciesMeta?: Record<string, {
28
+ optional?: boolean;
29
+ }>;
30
+ deprecated?: string;
31
+ dist?: {
32
+ tarball: string;
33
+ };
34
+ exports?: unknown;
35
+ }
36
+ export declare function fetchPackument(name: string): Promise<Packument>;
37
+ /** Stable releases only — a prerelease is not what a user installs. */
38
+ export declare function stableVersions(p: Packument): string[];
39
+ export declare function compareVersions(a: string, b: string): number;
40
+ /**
41
+ * Resolve what the user typed after the `@`. Accepts an exact version, a
42
+ * dist-tag (`latest`, `next`), a major (`4`) or a major.minor (`4.2`), and
43
+ * nothing at all — which means the `latest` tag, i.e. what `npm i pkg` gives.
44
+ */
45
+ export declare function resolveVersion(p: Packument, spec?: string): string;
46
+ export interface MajorLine {
47
+ major: number;
48
+ /** ISO date the first release in this major line was published */
49
+ first: string;
50
+ /** highest stable version in the line */
51
+ latest: string;
52
+ }
53
+ /** Every stable major line, oldest first, with the date the major landed. */
54
+ export declare function majorLines(p: Packument): MajorLine[];
55
+ export declare function monthsSince(iso: string): number;
56
+ /**
57
+ * The README as published for a specific version. The packument carries the
58
+ * README of the latest release at the top level, which is the wrong document
59
+ * when an older version is being scored — so the per-version copy wins when the
60
+ * registry has one.
61
+ */
62
+ export declare function readmeFor(p: Packument, version: string): string;
63
+ /** The peers a consumer is expected to install themselves, optional ones dropped. */
64
+ export declare function requiredPeers(meta: VersionMeta): string[];
65
+ export declare function repoUrl(p: Packument): string | undefined;
@@ -0,0 +1,98 @@
1
+ /**
2
+ * npm registry access, with no dependencies. Everything here is one GET against
3
+ * registry.npmjs.org and some arithmetic on the packument — no `npm` process,
4
+ * no semver package.
5
+ */
6
+ const REGISTRY = process.env.SDKPROOF_REGISTRY ?? "https://registry.npmjs.org";
7
+ export async function fetchPackument(name) {
8
+ // The scope slash must survive; only the name itself is escaped.
9
+ const url = `${REGISTRY}/${name.replace(/\//g, "%2f")}`;
10
+ const res = await fetch(url, { headers: { accept: "application/json" } });
11
+ if (res.status === 404)
12
+ throw new Error(`no such package on npm: ${name}`);
13
+ if (!res.ok)
14
+ throw new Error(`registry error for ${name}: ${res.status} ${res.statusText}`);
15
+ return (await res.json());
16
+ }
17
+ /** Stable releases only — a prerelease is not what a user installs. */
18
+ export function stableVersions(p) {
19
+ return Object.keys(p.versions)
20
+ .filter((v) => /^\d+\.\d+\.\d+$/.test(v))
21
+ .sort(compareVersions);
22
+ }
23
+ export function compareVersions(a, b) {
24
+ const pa = a.split(".").map(Number);
25
+ const pb = b.split(".").map(Number);
26
+ for (let i = 0; i < 3; i++)
27
+ if (pa[i] !== pb[i])
28
+ return pa[i] - pb[i];
29
+ return 0;
30
+ }
31
+ /**
32
+ * Resolve what the user typed after the `@`. Accepts an exact version, a
33
+ * dist-tag (`latest`, `next`), a major (`4`) or a major.minor (`4.2`), and
34
+ * nothing at all — which means the `latest` tag, i.e. what `npm i pkg` gives.
35
+ */
36
+ export function resolveVersion(p, spec) {
37
+ const stable = stableVersions(p);
38
+ if (!spec || spec === "latest") {
39
+ const tag = p["dist-tags"]?.latest;
40
+ if (tag)
41
+ return tag;
42
+ if (!stable.length)
43
+ throw new Error(`${p.name} has no stable release`);
44
+ return stable[stable.length - 1];
45
+ }
46
+ if (p["dist-tags"]?.[spec])
47
+ return p["dist-tags"][spec];
48
+ if (p.versions[spec])
49
+ return spec;
50
+ if (/^\d+(\.\d+)?$/.test(spec)) {
51
+ const matches = stable.filter((v) => v === spec || v.startsWith(`${spec}.`));
52
+ if (matches.length)
53
+ return matches[matches.length - 1];
54
+ }
55
+ throw new Error(`${p.name} has no version matching "${spec}"`);
56
+ }
57
+ /** Every stable major line, oldest first, with the date the major landed. */
58
+ export function majorLines(p) {
59
+ const byMajor = new Map();
60
+ for (const v of stableVersions(p)) {
61
+ const major = Number(v.split(".")[0]);
62
+ const published = p.time?.[v];
63
+ const cur = byMajor.get(major);
64
+ if (!cur) {
65
+ byMajor.set(major, { major, first: published ?? "", latest: v });
66
+ continue;
67
+ }
68
+ if (compareVersions(v, cur.latest) > 0)
69
+ cur.latest = v;
70
+ if (published && (!cur.first || new Date(published) < new Date(cur.first)))
71
+ cur.first = published;
72
+ }
73
+ return [...byMajor.values()].sort((a, b) => a.major - b.major);
74
+ }
75
+ export function monthsSince(iso) {
76
+ if (!iso)
77
+ return Infinity;
78
+ return (Date.now() - new Date(iso).getTime()) / (1000 * 60 * 60 * 24 * 30.44);
79
+ }
80
+ /**
81
+ * The README as published for a specific version. The packument carries the
82
+ * README of the latest release at the top level, which is the wrong document
83
+ * when an older version is being scored — so the per-version copy wins when the
84
+ * registry has one.
85
+ */
86
+ export function readmeFor(p, version) {
87
+ return p.versions[version]?.readme ?? (version === p["dist-tags"]?.latest ? p.readme ?? "" : "");
88
+ }
89
+ /** The peers a consumer is expected to install themselves, optional ones dropped. */
90
+ export function requiredPeers(meta) {
91
+ const peers = meta.peerDependencies ?? {};
92
+ return Object.keys(peers).filter((name) => !meta.peerDependenciesMeta?.[name]?.optional);
93
+ }
94
+ export function repoUrl(p) {
95
+ const r = typeof p.repository === "string" ? p.repository : p.repository?.url;
96
+ return r?.replace(/^git\+/, "").replace(/\.git$/, "");
97
+ }
98
+ //# sourceMappingURL=registry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,4BAA4B,CAAC;AA4B/E,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY;IAC/C,iEAAiE;IACjE,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;IACxD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAC1E,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IAC3E,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,KAAK,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5F,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAc,CAAC;AACzC,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,cAAc,CAAC,CAAY;IACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACxC,IAAI,CAAC,eAAe,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,CAAS,EAAE,CAAS;IAClD,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,CAAY,EAAE,IAAa;IACxD,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QACnC,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,wBAAwB,CAAC,CAAC;QACvE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC;IACxD,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;QAC7E,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,6BAA6B,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAUD,6EAA6E;AAC7E,MAAM,UAAU,UAAU,CAAC,CAAY;IACrC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QACD,IAAI,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QACvD,IAAI,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAAE,GAAG,CAAC,KAAK,GAAG,SAAS,CAAC;IACpG,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;AAChF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,CAAY,EAAE,OAAe;IACrD,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACnG,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAC1C,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,CAAY;IAClC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,GAAG,CAAC;IAC9E,OAAO,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACxD,CAAC"}
@@ -0,0 +1,26 @@
1
+ import type { Result } from "./core/types.ts";
2
+ import type { TaskSet } from "./tasks.ts";
3
+ import { type DriftReport } from "./drift.ts";
4
+ export interface RunContext {
5
+ packageName: string;
6
+ version: string;
7
+ taskSet: TaskSet;
8
+ drift?: DriftReport;
9
+ /** modules installed on top of the target to make it type-check */
10
+ extraInstalls: string[];
11
+ }
12
+ /**
13
+ * What the run found, for a terminal.
14
+ *
15
+ * Findings lead and the score follows, because the score is the part nobody
16
+ * acts on. Across this project's own distribution, findings pages drew ~10x the
17
+ * traffic of scorecards and no one has ever written in about a number.
18
+ */
19
+ export declare function renderTerminal(r: Result, ctx: RunContext): string;
20
+ /** The same run as a markdown report, for pasting into an issue or a thread. */
21
+ export declare function renderMarkdown(r: Result, ctx: RunContext): string;
22
+ /** Drift mode's own output — no model was called, so there is no score. */
23
+ export declare function renderDrift(d: DriftReport, verdict: {
24
+ worth: boolean;
25
+ reason: string;
26
+ }): string;
package/dist/report.js ADDED
@@ -0,0 +1,240 @@
1
+ import { fmtInterval, rates } from "./core/stats.js";
2
+ import { headlineRemovals, headlineSource } from "./drift.js";
3
+ const BAR = "─".repeat(60);
4
+ function firstLibraryError(v) {
5
+ return v.errors.find((e) => e.libraryRelated) ?? v.errors[0];
6
+ }
7
+ /**
8
+ * What the run found, for a terminal.
9
+ *
10
+ * Findings lead and the score follows, because the score is the part nobody
11
+ * acts on. Across this project's own distribution, findings pages drew ~10x the
12
+ * traffic of scorecards and no one has ever written in about a number.
13
+ */
14
+ export function renderTerminal(r, ctx) {
15
+ const out = [];
16
+ const failed = r.verdicts.filter((v) => !v.passed);
17
+ const passed = r.verdicts.filter((v) => v.passed).length;
18
+ out.push("");
19
+ out.push(BAR);
20
+ // Counts, not a percentage. "0 of 3 compiled" is a claim a reader can check;
21
+ // "0/100" reads as a grade the tool handed out, and a bare percentage hides
22
+ // how small the denominator is.
23
+ out.push(` ${ctx.packageName}@${ctx.version} — ${passed} of ${r.verdicts.length} answers compiled`);
24
+ out.push(BAR);
25
+ if (r.lost?.length) {
26
+ const ids = [...new Set(r.lost.map((l) => l.taskId))].join(", ");
27
+ out.push("");
28
+ out.push(` INCOMPLETE RUN — ${r.lost.length} task(s) never generated: ${ids}`);
29
+ out.push(" These are not refusals; the request never landed. The denominator");
30
+ out.push(" below is smaller than the task set, so do not publish this number.");
31
+ }
32
+ out.push("");
33
+ for (const m of r.perModel) {
34
+ const refused = r.refusals.filter((x) => x.model === m.model).length;
35
+ const s = rates({ passed: m.passed, scored: m.total, refused });
36
+ out.push(` ${m.model.padEnd(24)} ${String(m.passed).padStart(2)}/${String(m.total).padEnd(3)} ` +
37
+ `compiled (${s.conditional.pct}%, 95% CI ${fmtInterval(s.conditional.ci)})`);
38
+ if (refused) {
39
+ out.push(` ${" ".repeat(24)} ${refused} refused — unmeasured, not drift; ` +
40
+ `${s.unconditional.pct}% of everything asked`);
41
+ }
42
+ }
43
+ if (failed.length) {
44
+ out.push("");
45
+ out.push(" WHAT BROKE");
46
+ out.push("");
47
+ for (const v of failed) {
48
+ const e = firstLibraryError(v);
49
+ const task = ctx.taskSet.tasks.find((t) => t.id === v.taskId);
50
+ out.push(` ${v.taskId}${task ? ` (${task.area}, ${task.difficulty})` : ""}`);
51
+ if (e)
52
+ out.push(` ${e.code}: ${truncate(e.message, 140)}`);
53
+ const extra = v.errors.length - 1;
54
+ if (extra > 0)
55
+ out.push(` +${extra} more diagnostic${extra === 1 ? "" : "s"}`);
56
+ out.push("");
57
+ }
58
+ }
59
+ else {
60
+ out.push("");
61
+ out.push(" Every candidate compiled. Nothing drifted on this task set.");
62
+ out.push("");
63
+ }
64
+ if (r.failurePatterns.length) {
65
+ out.push(" FAILURE PATTERNS");
66
+ for (const p of r.failurePatterns) {
67
+ out.push(` ${p.category.padEnd(24)} ${p.count}x`);
68
+ }
69
+ out.push("");
70
+ }
71
+ const headline = ctx.drift ? headlineRemovals(ctx.drift) : [];
72
+ if (ctx.drift && headline.length) {
73
+ const d = ctx.drift;
74
+ out.push(` DRIFT SURFACE v${d.from.major} -> v${d.to.major}: ` +
75
+ `${headline.length} ${headlineSource(d)}`);
76
+ out.push(` ${headline.slice(0, 12).map(symbolLabel).join(", ")}${headline.length > 12 ? " ..." : ""}`);
77
+ out.push("");
78
+ }
79
+ if (ctx.taskSet.rejected.length) {
80
+ out.push(` ${ctx.taskSet.rejected.length} synthesized task(s) rejected before the run:`);
81
+ for (const x of ctx.taskSet.rejected.slice(0, 5))
82
+ out.push(` ${x.id}: ${x.reason}`);
83
+ out.push("");
84
+ }
85
+ out.push(" Pass means it compiled against the real installed package.");
86
+ out.push(" No model judged another model; tsc decided.");
87
+ out.push("");
88
+ return out.join("\n");
89
+ }
90
+ /** `default` is a real removed export (zustand v5 dropped it) but reads as a typo. */
91
+ function symbolLabel(s) {
92
+ return s === "default" ? "default export" : s;
93
+ }
94
+ function truncate(s, n) {
95
+ return s.length <= n ? s : `${s.slice(0, n - 1)}…`;
96
+ }
97
+ /** The same run as a markdown report, for pasting into an issue or a thread. */
98
+ export function renderMarkdown(r, ctx) {
99
+ const lines = [];
100
+ const failed = r.verdicts.filter((v) => !v.passed);
101
+ lines.push(`# ${ctx.packageName} v${ctx.version} — SDKProof run`);
102
+ lines.push("");
103
+ lines.push(`**Generated:** ${r.generatedAt} `);
104
+ lines.push(`**Method:** every answer is type-checked against the real installed package with \`tsc\`. Pass = it compiles. No model judges another model. `);
105
+ lines.push(`**Tasks:** ${ctx.taskSet.tasks.length} (${ctx.taskSet.source === "synthesized" ? "written from the package README at this version" : ctx.taskSet.source})`);
106
+ lines.push("");
107
+ if (r.lost?.length) {
108
+ lines.push(`> **Incomplete run — ${r.lost.length} task(s) never generated** (${[...new Set(r.lost.map((l) => l.taskId))].join(", ")}). ` +
109
+ `Not refusals: generation errored before any code existed, so the denominator is smaller than the task set. ` +
110
+ `A partial run that loses the hardest tasks scores higher than the real one — do not publish this number.`);
111
+ lines.push("");
112
+ }
113
+ lines.push(`## ${r.verdicts.filter((v) => v.passed).length} of ${r.verdicts.length} answers compiled`);
114
+ lines.push("");
115
+ lines.push("**Conditional API correctness** = passes / completions that produced code. " +
116
+ "**Unconditional task success** = passes / every task asked, refusals included. " +
117
+ "Ranges are Wilson 95% intervals.");
118
+ lines.push("");
119
+ lines.push("| Model | Conditional | Unconditional | Passed | Scored | Refused |");
120
+ lines.push("|---|---:|---:|---:|---:|---:|");
121
+ for (const m of r.perModel) {
122
+ const refused = r.refusals.filter((x) => x.model === m.model).length;
123
+ const s = rates({ passed: m.passed, scored: m.total, refused });
124
+ lines.push(`| ${m.model} | ${s.conditional.pct}% (${fmtInterval(s.conditional.ci)}) | ` +
125
+ `${s.unconditional.pct}% (${fmtInterval(s.unconditional.ci)}) | ${m.passed} | ${m.total} | ${refused || "—"} |`);
126
+ }
127
+ lines.push("");
128
+ if (failed.length) {
129
+ lines.push("## What broke");
130
+ lines.push("");
131
+ for (const v of failed) {
132
+ const task = ctx.taskSet.tasks.find((t) => t.id === v.taskId);
133
+ lines.push(`### \`${v.taskId}\`${task ? ` — ${task.area}, ${task.difficulty}` : ""}`);
134
+ lines.push("");
135
+ if (task) {
136
+ lines.push(`> ${task.prompt}`);
137
+ lines.push("");
138
+ }
139
+ lines.push("```");
140
+ for (const e of v.errors.slice(0, 8)) {
141
+ lines.push(`${e.line}:${e.column} error ${e.code}: ${e.message}`);
142
+ }
143
+ if (v.errors.length > 8)
144
+ lines.push(`… ${v.errors.length - 8} more`);
145
+ lines.push("```");
146
+ lines.push("");
147
+ }
148
+ }
149
+ else {
150
+ lines.push("Every candidate compiled. Nothing on this task set drifted.");
151
+ lines.push("");
152
+ }
153
+ if (r.failurePatterns.length) {
154
+ lines.push("## Failure patterns");
155
+ lines.push("");
156
+ lines.push("| Category | Count | Example |");
157
+ lines.push("|---|---:|---|");
158
+ for (const p of r.failurePatterns) {
159
+ lines.push(`| ${p.category} | ${p.count} | \`${p.example.taskId}\`: ${p.example.message.replace(/\|/g, "\\|").slice(0, 120)} |`);
160
+ }
161
+ lines.push("");
162
+ }
163
+ if (ctx.drift) {
164
+ const d = ctx.drift;
165
+ lines.push(`## Drift surface — v${d.from.major} → v${d.to.major}`);
166
+ lines.push("");
167
+ const shown = headlineRemovals(d);
168
+ lines.push(`Comparing \`${d.from.version}\` with \`${d.to.version}\`: ${shown.length} ${headlineSource(d)}. ` +
169
+ `(${d.removedFromEntry.length} left the entrypoint in all; ${d.removed.length} removed under the wider \`${d.mode}\` diff.)`);
170
+ lines.push("");
171
+ if (shown.length) {
172
+ lines.push("```");
173
+ lines.push(shown.map(symbolLabel).join("\n"));
174
+ lines.push("```");
175
+ lines.push("");
176
+ }
177
+ }
178
+ if (ctx.extraInstalls.length) {
179
+ lines.push(`_Sandbox: \`${ctx.packageName}@${ctx.version}\` plus ${ctx.extraInstalls.join(", ")}, ` +
180
+ `type-checked under \`strict\` with \`skipLibCheck\`._`);
181
+ lines.push("");
182
+ }
183
+ lines.push("---");
184
+ lines.push("");
185
+ lines.push("Generated by [SDKProof](https://sdkproof.dev) — `npx sdkproof " + ctx.packageName + "`");
186
+ return lines.join("\n");
187
+ }
188
+ /** Drift mode's own output — no model was called, so there is no score. */
189
+ export function renderDrift(d, verdict) {
190
+ const out = [];
191
+ const headline = headlineRemovals(d);
192
+ out.push("");
193
+ out.push(BAR);
194
+ out.push(` ${d.package} v${d.from.version} -> v${d.to.version}`);
195
+ out.push(BAR);
196
+ out.push("");
197
+ out.push(` v${d.to.major} landed ${d.majorAgeMonths.toFixed(0)} months ago`);
198
+ out.push(` ${d.fromCount} exported symbols -> ${d.toCount} (${d.mode} diff)`);
199
+ out.push("");
200
+ if (headline.length) {
201
+ out.push(` WHAT LEFT (${headline.length}) — ${headlineSource(d)}`);
202
+ out.push(` This is what a model trained on v${d.from.major} will still write.`);
203
+ out.push("");
204
+ for (const s of headline.slice(0, 40))
205
+ out.push(` ${symbolLabel(s)}`);
206
+ if (headline.length > 40)
207
+ out.push(` ... ${headline.length - 40} more`);
208
+ out.push("");
209
+ }
210
+ const deprecatedFirst = d.removed.filter((s) => !d.withoutRunway.includes(s));
211
+ if (deprecatedFirst.length) {
212
+ out.push(` Deprecated first, then removed (${deprecatedFirst.length}) — these rarely produce drift:`);
213
+ out.push(` ${deprecatedFirst.slice(0, 20).map(symbolLabel).join(", ")}${deprecatedFirst.length > 20 ? " ..." : ""}`);
214
+ out.push("");
215
+ }
216
+ const rest = d.removedFromEntry.length - headline.length;
217
+ if (d.documentedRemovals.length && rest > 0) {
218
+ out.push(` (${d.removedFromEntry.length} exports left the entrypoint in total. The ` +
219
+ `${d.documentedRemovals.length} above ${d.documentedRemovals.length === 1 ? "is the one" : "are the ones"} ` +
220
+ `v${d.from.major}'s own README taught people to write; the rest are mostly types.)`);
221
+ out.push("");
222
+ }
223
+ else if (d.valueRemovals.length && rest > 0) {
224
+ out.push(` (${rest} type-only export(s) also left the entrypoint. They are listed in --json; ` +
225
+ `a model writes a hook far more often than it writes a type name.)`);
226
+ out.push("");
227
+ }
228
+ else if (d.mode === "all-dts" && d.removedFromEntry.length < d.withoutRunway.length) {
229
+ out.push(` (${d.withoutRunway.length} symbols vanished across every .d.ts in the package, but only the ` +
230
+ `${d.removedFromEntry.length} above were reachable from the entrypoint. The rest are internals.)`);
231
+ out.push("");
232
+ }
233
+ out.push(` ${verdict.worth ? "WORTH SCORING" : "NOT WORTH SCORING"} — ${verdict.reason}`);
234
+ out.push("");
235
+ if (verdict.worth)
236
+ out.push(` Next: npx sdkproof ${d.package}`);
237
+ out.push("");
238
+ return out.join("\n");
239
+ }
240
+ //# sourceMappingURL=report.js.map