toolnexus 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/acp.d.ts +10 -2
- package/dist/acp.js +36 -5
- package/dist/classifier.d.ts +199 -31
- package/dist/classifier.js +639 -31
- package/dist/client.d.ts +6 -0
- package/dist/judge.d.ts +6 -3
- package/dist/judge.js +13 -1
- package/dist/mcp.js +16 -6
- package/dist/mcp2026/source.d.ts +5 -0
- package/dist/mcp2026/source.js +11 -2
- package/dist/mcp2026/stdio.d.ts +43 -0
- package/dist/mcp2026/stdio.js +144 -20
- package/dist/mcphandler.js +2 -4
- package/dist/mcpserve.js +2 -1
- package/dist/serve.js +12 -1
- package/dist/skill.d.ts +14 -0
- package/dist/skill.js +33 -2
- package/package.json +2 -2
- package/src/acp.ts +59 -7
- package/src/classifier.ts +723 -47
- package/src/client.ts +6 -0
- package/src/judge.ts +19 -4
- package/src/mcp.ts +18 -6
- package/src/mcp2026/source.ts +14 -4
- package/src/mcp2026/stdio.ts +148 -23
- package/src/mcphandler.ts +2 -3
- package/src/mcpserve.ts +2 -1
- package/src/serve.ts +13 -2
- package/src/skill.ts +51 -2
package/dist/classifier.js
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Classifier (SPEC.md §8B) — the contract for a JUDGMENT, as `Tool` is the contract for an
|
|
3
|
+
* ACTION. A System One model takes a state plus pre-declared, typed questions and returns
|
|
4
|
+
* calibrated answers with no free text. It has no messages, no tool calling and no streaming,
|
|
5
|
+
* so it never enters the §8 client loop.
|
|
6
|
+
*
|
|
7
|
+
* A classifier INTERPRETS; it never AUTHORISES. Schema validity is not correctness: a decision
|
|
8
|
+
* can be confidently wrong, and "cannot hallucinate" means only that the returned value is in
|
|
9
|
+
* the declared schema. Numeric limits, permission checks and allowlists stay in code. Nothing
|
|
10
|
+
* here is a security control.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
1
13
|
import { redactErrorBody, capErrorBody } from "./client.js";
|
|
2
14
|
import { retryAfterMs, isRetryableStatus } from "./retry.js";
|
|
15
|
+
import { isContentPart } from "./content.js";
|
|
3
16
|
// ---------------------------------------------------------------- constants
|
|
4
17
|
/** The System One endpoint base. */
|
|
5
18
|
export const DEFAULT_CLASSIFIER_BASE_URL = "https://api.typesafe.ai/v1";
|
|
@@ -11,11 +24,99 @@ export const DEFAULT_CLASSIFIER_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
|
11
24
|
export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 10_000;
|
|
12
25
|
/** `evaluateBatch`: at most this many states in flight by default. */
|
|
13
26
|
export const DEFAULT_BATCH_CONCURRENCY = 16;
|
|
14
|
-
/**
|
|
15
|
-
export const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
27
|
+
/** TypeSafe's required opt-in header for image ("danger zone") input. */
|
|
28
|
+
export const CLASSIFIER_IMAGES_HEADER = "TypeSafe-Danger-Zone-Images";
|
|
29
|
+
/** Image limits (docs.typesafe.ai/images): checked client-side, never resized. */
|
|
30
|
+
export const CLASSIFIER_IMAGE_MIN_SIDE = 280;
|
|
31
|
+
export const CLASSIFIER_IMAGE_MAX_SIDE = 1568;
|
|
32
|
+
export const CLASSIFIER_IMAGE_MAX_BYTES = 16 * 1024 * 1024;
|
|
33
|
+
export const CLASSIFIER_REQUEST_IMAGE_BYTES = 32 * 1024 * 1024;
|
|
34
|
+
/**
|
|
35
|
+
* The backend table (PROPOSED defaults, owner to confirm — design.md PD-1..PD-4). Vercel and
|
|
36
|
+
* Cloudflare-Jev images stay false until a live probe proves them.
|
|
37
|
+
*/
|
|
38
|
+
export const CLASSIFIER_BACKENDS = Object.freeze({
|
|
39
|
+
typesafe: {
|
|
40
|
+
baseUrl: DEFAULT_CLASSIFIER_BASE_URL,
|
|
41
|
+
model: DEFAULT_CLASSIFIER_MODEL,
|
|
42
|
+
apiKeyEnv: DEFAULT_CLASSIFIER_API_KEY_ENV,
|
|
43
|
+
wire: "systemone",
|
|
44
|
+
imagesHeader: true,
|
|
45
|
+
supports: () => ({ images: true }),
|
|
46
|
+
},
|
|
47
|
+
openrouter: {
|
|
48
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
49
|
+
model: "typesafe/jev-1.14",
|
|
50
|
+
apiKeyEnv: "OPENROUTER_API_KEY",
|
|
51
|
+
wire: "systemone",
|
|
52
|
+
imagesHeader: true,
|
|
53
|
+
supports: () => ({ images: true }),
|
|
54
|
+
},
|
|
55
|
+
cloudflare: {
|
|
56
|
+
baseUrl: "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/ai/run",
|
|
57
|
+
model: "typesafe/jev",
|
|
58
|
+
apiKeyEnv: "CLOUDFLARE_JEV_API_KEY",
|
|
59
|
+
wire: "cloudflare",
|
|
60
|
+
imagesHeader: false,
|
|
61
|
+
supports: (model) => ({ images: model.includes("clef") }),
|
|
62
|
+
},
|
|
63
|
+
vercel: {
|
|
64
|
+
baseUrl: "https://ai-gateway.vercel.sh/typesafe/v1",
|
|
65
|
+
model: "typesafe-ai/jev",
|
|
66
|
+
apiKeyEnv: "AI_GATEWAY_API_KEY",
|
|
67
|
+
wire: "systemone",
|
|
68
|
+
imagesHeader: true,
|
|
69
|
+
supports: () => ({ images: false }),
|
|
70
|
+
},
|
|
71
|
+
custom: {
|
|
72
|
+
baseUrl: "",
|
|
73
|
+
model: DEFAULT_CLASSIFIER_MODEL,
|
|
74
|
+
apiKeyEnv: "",
|
|
75
|
+
wire: "systemone",
|
|
76
|
+
imagesHeader: true,
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
/**
|
|
80
|
+
* The env var whose key may be sent to `baseUrl` (already `${NAME}`-expanded): the backend's own
|
|
81
|
+
* variable, and only when `baseUrl` is that backend's own host over https (OpenRouter also its
|
|
82
|
+
* subdomains). `""` when none may — `custom` has no preset key at all. A key is never sent to a
|
|
83
|
+
* host the caller did not name (#155, SPEC §8B).
|
|
84
|
+
*/
|
|
85
|
+
export function classifierKeyEnv(backend, baseUrl) {
|
|
86
|
+
const p = Object.prototype.hasOwnProperty.call(CLASSIFIER_BACKENDS, backend) ? CLASSIFIER_BACKENDS[backend] : undefined;
|
|
87
|
+
if (!p || !p.apiKeyEnv || !p.baseUrl)
|
|
88
|
+
return "";
|
|
89
|
+
const hostOf = (u) => {
|
|
90
|
+
try {
|
|
91
|
+
const url = new URL(u);
|
|
92
|
+
return url.protocol === "https:" ? url.hostname.toLowerCase() : "";
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return "";
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
const h = hostOf(baseUrl);
|
|
99
|
+
const own = hostOf(p.baseUrl);
|
|
100
|
+
if (!h || !own)
|
|
101
|
+
return "";
|
|
102
|
+
return h === own || (backend === "openrouter" && h.endsWith(".openrouter.ai")) ? p.apiKeyEnv : "";
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Raised BEFORE any network call when the backend table says the backend cannot take what the
|
|
106
|
+
* request carries. Message, exact: `classifier: <backend> (<model>) does not support <feature>`.
|
|
107
|
+
*/
|
|
108
|
+
export class ClassifierUnsupported extends Error {
|
|
109
|
+
backend;
|
|
110
|
+
model;
|
|
111
|
+
feature;
|
|
112
|
+
constructor(backend, model, feature) {
|
|
113
|
+
super(`classifier: ${backend} (${model}) does not support ${feature}`);
|
|
114
|
+
this.backend = backend;
|
|
115
|
+
this.model = model;
|
|
116
|
+
this.feature = feature;
|
|
117
|
+
this.name = "ClassifierUnsupported";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
19
120
|
/** Client-side cap on a choice's named options (§8B). */
|
|
20
121
|
export const MAX_CHOICE_OPTIONS = 255;
|
|
21
122
|
/** Bounds of a score rubric (§8B). */
|
|
@@ -26,6 +127,16 @@ export const MAX_SCORE_LEVELS = 10;
|
|
|
26
127
|
* port; `examples/judge/near-uniform.json` pins both sides at 0.0499 / 0.0501.
|
|
27
128
|
*/
|
|
28
129
|
export const NEAR_UNIFORM_TOLERANCE = 0.05;
|
|
130
|
+
const MODELLED_TYPES = new Set(["noul", "choice", "score"]);
|
|
131
|
+
/** True when `q` is a raw question (its type is not noul/choice/score). Not a type guard: every
|
|
132
|
+
* modelled question is structurally a `RawQuestion` too. */
|
|
133
|
+
export function isRawQuestion(q) {
|
|
134
|
+
return !MODELLED_TYPES.has(q?.type);
|
|
135
|
+
}
|
|
136
|
+
/** A raw question, sent as written. */
|
|
137
|
+
export function raw(type, instructions, criteria) {
|
|
138
|
+
return criteria === undefined ? { type, instructions } : { type, instructions, criteria };
|
|
139
|
+
}
|
|
29
140
|
/** A noul question. Pass `criteria` only when you mean it — absent ≠ empty on the wire. */
|
|
30
141
|
export function noul(instructions, criteria) {
|
|
31
142
|
return criteria === undefined ? { type: "noul", instructions } : { type: "noul", instructions, criteria };
|
|
@@ -47,7 +158,14 @@ export function choiceOver(instructions, items) {
|
|
|
47
158
|
return choice(instructions, items);
|
|
48
159
|
}
|
|
49
160
|
/** The wire form of one question — the object itself, minus an absent `criteria`. */
|
|
50
|
-
export function questionWire(
|
|
161
|
+
export function questionWire(question) {
|
|
162
|
+
if (isRawQuestion(question)) {
|
|
163
|
+
const r = question;
|
|
164
|
+
return r.criteria === undefined
|
|
165
|
+
? { type: r.type, instructions: r.instructions }
|
|
166
|
+
: { type: r.type, instructions: r.instructions, criteria: r.criteria };
|
|
167
|
+
}
|
|
168
|
+
const q = question;
|
|
51
169
|
switch (q.type) {
|
|
52
170
|
case "noul":
|
|
53
171
|
return q.criteria === undefined
|
|
@@ -60,7 +178,13 @@ export function questionWire(q) {
|
|
|
60
178
|
}
|
|
61
179
|
}
|
|
62
180
|
/** Client-side limits, enforced BEFORE the request. The error names the key and the limit. */
|
|
63
|
-
function validateQuestion(key,
|
|
181
|
+
function validateQuestion(key, question) {
|
|
182
|
+
const q = question;
|
|
183
|
+
if (isRawQuestion(question)) {
|
|
184
|
+
if (!question?.type)
|
|
185
|
+
throw new Error(`classifier: question "${key}": a raw question needs a type`);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
64
188
|
if (q.type === "choice") {
|
|
65
189
|
const n = Object.keys(q.criteria ?? {}).length;
|
|
66
190
|
if (n < 1 || n > MAX_CHOICE_OPTIONS) {
|
|
@@ -119,6 +243,10 @@ export function canonicalRequest(model, questions) {
|
|
|
119
243
|
}
|
|
120
244
|
return canonicalJson({ model, questions: qs });
|
|
121
245
|
}
|
|
246
|
+
/** True when an answer is a raw (unmodelled-type) answer. */
|
|
247
|
+
export function isRawAnswer(a) {
|
|
248
|
+
return !!a && typeof a === "object" && !MODELLED_TYPES.has(a.type) && "value" in a;
|
|
249
|
+
}
|
|
122
250
|
/** The legend in LEVEL order, which the map itself loses ("2" sorts before "10" as a level). */
|
|
123
251
|
export function levels(answer) {
|
|
124
252
|
return Object.keys(answer.legend)
|
|
@@ -134,6 +262,7 @@ export class Decision {
|
|
|
134
262
|
answers;
|
|
135
263
|
usage;
|
|
136
264
|
calibrated;
|
|
265
|
+
cached;
|
|
137
266
|
constructor(
|
|
138
267
|
/** What actually answered, which may be more specific than the model that was asked for. */
|
|
139
268
|
model, answers, usage,
|
|
@@ -142,11 +271,23 @@ export class Decision {
|
|
|
142
271
|
* unless it derived them from provider token probabilities. A THRESHOLD TUNED AGAINST ONE
|
|
143
272
|
* BACKEND DOES NOT TRANSFER TO ANOTHER.
|
|
144
273
|
*/
|
|
145
|
-
calibrated
|
|
274
|
+
calibrated,
|
|
275
|
+
/** True when the decision came from `ClassifierOptions.cache` and no request was sent. */
|
|
276
|
+
cached = false) {
|
|
146
277
|
this.model = model;
|
|
147
278
|
this.answers = answers;
|
|
148
279
|
this.usage = usage;
|
|
149
280
|
this.calibrated = calibrated;
|
|
281
|
+
this.cached = cached;
|
|
282
|
+
}
|
|
283
|
+
/** Typed read of a raw (unmodelled-type) answer. Throws on an absent key or a modelled type. */
|
|
284
|
+
raw(key) {
|
|
285
|
+
const a = this.answers[key];
|
|
286
|
+
if (a === undefined)
|
|
287
|
+
throw new Error(`classifier: no answer "${key}" in this decision`);
|
|
288
|
+
if (!isRawAnswer(a))
|
|
289
|
+
throw new Error(`classifier: answer "${key}" is a ${a.type} answer, not raw`);
|
|
290
|
+
return a;
|
|
150
291
|
}
|
|
151
292
|
/** Typed read. Throws (never returns undefined) on a wrong-type or absent key. */
|
|
152
293
|
noul(key) {
|
|
@@ -223,7 +364,10 @@ export function decodeDecision(raw, fallbackModel) {
|
|
|
223
364
|
};
|
|
224
365
|
break;
|
|
225
366
|
default:
|
|
226
|
-
|
|
367
|
+
// The union is OPEN (§8B): an unmodelled type (e.g. Jev 1.14 "box") decodes to a raw
|
|
368
|
+
// answer instead of failing the whole decision.
|
|
369
|
+
;
|
|
370
|
+
answers[key] = { type: String(a.type ?? ""), value: { ...a } };
|
|
227
371
|
}
|
|
228
372
|
}
|
|
229
373
|
const usage = {
|
|
@@ -261,7 +405,13 @@ export class Classifier {
|
|
|
261
405
|
baseUrl;
|
|
262
406
|
model;
|
|
263
407
|
apiKeyEnv;
|
|
408
|
+
/** The caller NAMED apiKeyEnv: sent wherever baseUrl points. Otherwise the preset's own key,
|
|
409
|
+
* and only to its own host (classifierKeyEnv, #155). */
|
|
410
|
+
explicitKeyEnv;
|
|
264
411
|
timeoutMs;
|
|
412
|
+
backend;
|
|
413
|
+
preset;
|
|
414
|
+
supports;
|
|
265
415
|
corpus = new Map();
|
|
266
416
|
/** The once-per-question-key set for the degenerate warning, so a per-turn judge does not
|
|
267
417
|
* flood the sink. */
|
|
@@ -269,14 +419,24 @@ export class Classifier {
|
|
|
269
419
|
constructor(opts) {
|
|
270
420
|
this.opts = opts;
|
|
271
421
|
this.style = opts.style ?? "systemone";
|
|
272
|
-
|
|
273
|
-
|
|
422
|
+
this.backend = opts.backend ?? "typesafe";
|
|
423
|
+
if (!Object.prototype.hasOwnProperty.call(CLASSIFIER_BACKENDS, this.backend)) {
|
|
424
|
+
throw new Error(`classifier: unknown backend "${this.backend}" (want "typesafe", "openrouter", "cloudflare", "vercel" or "custom")`);
|
|
425
|
+
}
|
|
426
|
+
const preset = CLASSIFIER_BACKENDS[this.backend];
|
|
427
|
+
this.preset = preset;
|
|
428
|
+
if (this.backend === "custom" && this.style === "systemone" && (!opts.baseUrl || !opts.supports)) {
|
|
429
|
+
throw new Error('classifier: backend "custom" requires baseUrl and supports');
|
|
430
|
+
}
|
|
431
|
+
this.baseUrl = opts.baseUrl || preset.baseUrl;
|
|
274
432
|
this.model = opts.model ?? preset.model;
|
|
275
433
|
this.apiKeyEnv = opts.apiKeyEnv ?? preset.apiKeyEnv;
|
|
434
|
+
this.explicitKeyEnv = opts.apiKeyEnv !== undefined;
|
|
276
435
|
const mismatch = backendMismatch(this.baseUrl, this.model);
|
|
277
436
|
if (mismatch)
|
|
278
437
|
throw new Error(mismatch);
|
|
279
438
|
this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS;
|
|
439
|
+
this.supports = opts.supports ? { ...opts.supports } : (preset.supports?.(this.model) ?? { images: false });
|
|
280
440
|
switch (this.style) {
|
|
281
441
|
case "systemone":
|
|
282
442
|
break;
|
|
@@ -309,11 +469,60 @@ export class Classifier {
|
|
|
309
469
|
* INDEPENDENT — one answer is never context for another.
|
|
310
470
|
*/
|
|
311
471
|
async evaluate(state, questions, signal) {
|
|
472
|
+
return this.run(state, questions, signal, false);
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Binds one state (a VALUE, not a session: nothing is sent until `evaluate`). Its `evaluate`
|
|
476
|
+
* runs the same path as `evaluate` PLUS fail-closed validation (`validateDecision`).
|
|
477
|
+
*/
|
|
478
|
+
state(value, opts = {}) {
|
|
479
|
+
return {
|
|
480
|
+
evaluate: async (...qs) => this.run(value, namedQuestions(qs), opts.signal, true),
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* Binds a batch of states. Its `evaluate` runs `state(v).evaluate` per state, at most 16 in
|
|
485
|
+
* flight, and returns one `{decision | error}` per state IN ORDER. One state's failure never
|
|
486
|
+
* fails the batch; only a question-set error (duplicate name) throws, before any request.
|
|
487
|
+
*/
|
|
488
|
+
states(...values) {
|
|
489
|
+
return {
|
|
490
|
+
evaluate: async (...qs) => {
|
|
491
|
+
const questions = namedQuestions(qs);
|
|
492
|
+
const out = new Array(values.length);
|
|
493
|
+
let next = 0;
|
|
494
|
+
const worker = async () => {
|
|
495
|
+
while (next < values.length) {
|
|
496
|
+
const i = next++;
|
|
497
|
+
try {
|
|
498
|
+
out[i] = { decision: await this.run(values[i], questions, undefined, true) };
|
|
499
|
+
}
|
|
500
|
+
catch (e) {
|
|
501
|
+
out[i] = { error: e instanceof Error ? e : new Error(String(e)) };
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
};
|
|
505
|
+
await Promise.all(Array.from({ length: Math.min(DEFAULT_BATCH_CONCURRENCY, values.length) }, worker));
|
|
506
|
+
return out;
|
|
507
|
+
},
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
/** The one path. `validate` = fail closed on `validateDecision` (the value builders). */
|
|
511
|
+
async run(state, questions, signal, validate) {
|
|
312
512
|
const start = Date.now();
|
|
313
513
|
const keys = Object.keys(questions ?? {}).sort();
|
|
514
|
+
// L1: count + hash of the canonical request on every evaluate event ("" when unencodable).
|
|
515
|
+
let qHash = "";
|
|
516
|
+
try {
|
|
517
|
+
qHash = sha256Hex(canonicalRequest(this.model, questions ?? {}));
|
|
518
|
+
}
|
|
519
|
+
catch {
|
|
520
|
+
qHash = "";
|
|
521
|
+
}
|
|
522
|
+
const l1 = { questions: keys.length, questionsHash: qHash };
|
|
314
523
|
if (keys.length === 0) {
|
|
315
524
|
const err = new Error("classifier: no questions to evaluate");
|
|
316
|
-
this.emitError(start, err);
|
|
525
|
+
this.emitError(start, err, l1);
|
|
317
526
|
throw err;
|
|
318
527
|
}
|
|
319
528
|
try {
|
|
@@ -328,7 +537,7 @@ export class Classifier {
|
|
|
328
537
|
}
|
|
329
538
|
}
|
|
330
539
|
catch (e) {
|
|
331
|
-
this.emitError(start, e);
|
|
540
|
+
this.emitError(start, e, l1);
|
|
332
541
|
throw e;
|
|
333
542
|
}
|
|
334
543
|
// Detection, never repair (ADR 0020/0021): the request goes out BYTE-UNCHANGED and the
|
|
@@ -347,11 +556,16 @@ export class Classifier {
|
|
|
347
556
|
decision = await this.evaluateLLM(state, questions, signal);
|
|
348
557
|
break;
|
|
349
558
|
default:
|
|
350
|
-
decision = await this.
|
|
559
|
+
decision = await this.evaluateWire(state, questions, signal, validate);
|
|
560
|
+
}
|
|
561
|
+
if (validate && this.style !== "systemone") {
|
|
562
|
+
const verr = validateDecision(questions, decision);
|
|
563
|
+
if (verr)
|
|
564
|
+
throw verr;
|
|
351
565
|
}
|
|
352
566
|
}
|
|
353
567
|
catch (e) {
|
|
354
|
-
this.emitError(start, e);
|
|
568
|
+
this.emitError(start, e, l1);
|
|
355
569
|
throw e;
|
|
356
570
|
}
|
|
357
571
|
this.emit({
|
|
@@ -361,6 +575,8 @@ export class Classifier {
|
|
|
361
575
|
ms: Date.now() - start,
|
|
362
576
|
promptTokens: decision.usage.inputTokens,
|
|
363
577
|
completionTokens: decision.usage.outputTokens,
|
|
578
|
+
...l1,
|
|
579
|
+
cached: decision.cached === true,
|
|
364
580
|
});
|
|
365
581
|
return decision;
|
|
366
582
|
}
|
|
@@ -421,15 +637,26 @@ export class Classifier {
|
|
|
421
637
|
}
|
|
422
638
|
// ---------------------------------------------------------------- backends
|
|
423
639
|
/**
|
|
424
|
-
* Assembles the request: the canonical model + questions, plus
|
|
425
|
-
*
|
|
426
|
-
* bodyTransform → marshal).
|
|
640
|
+
* Assembles the request for the backend's wire: the canonical model + questions, plus the state
|
|
641
|
+
* (VERBATIM unless it carried content parts) and, on systemone, `assets`; then the §8 Gap 1
|
|
642
|
+
* pipeline in §8 order (base → requestParams merge → bodyTransform → marshal).
|
|
427
643
|
*/
|
|
428
|
-
body(
|
|
644
|
+
body(p, questions) {
|
|
429
645
|
const qs = {};
|
|
430
646
|
for (const [key, q] of Object.entries(questions))
|
|
431
647
|
qs[key] = questionWire(q);
|
|
432
|
-
let body
|
|
648
|
+
let body;
|
|
649
|
+
if (this.preset.wire === "cloudflare") {
|
|
650
|
+
const input = { questions: qs, state: p.state };
|
|
651
|
+
if (p.images.length > 0)
|
|
652
|
+
input.images = p.images.map((im) => `data:image/${im.format};base64,${im.data}`);
|
|
653
|
+
body = { model: this.model, input };
|
|
654
|
+
}
|
|
655
|
+
else {
|
|
656
|
+
body = { model: this.model, questions: qs, state: p.state };
|
|
657
|
+
if (p.assets)
|
|
658
|
+
body.assets = p.assets;
|
|
659
|
+
}
|
|
433
660
|
// A requestParams key WINS on collision.
|
|
434
661
|
if (this.opts.requestParams)
|
|
435
662
|
body = { ...body, ...this.opts.requestParams };
|
|
@@ -447,8 +674,44 @@ export class Classifier {
|
|
|
447
674
|
throw new Error("classifier: static: no recorded decision for this request+state");
|
|
448
675
|
return decodeDecision(raw, this.model);
|
|
449
676
|
}
|
|
450
|
-
|
|
451
|
-
|
|
677
|
+
/** Lift images, apply the support table and the limits — all before any request. */
|
|
678
|
+
prepare(state) {
|
|
679
|
+
const { state: sent, images } = liftImages(state);
|
|
680
|
+
if (images.length > 0 && !this.supports.images)
|
|
681
|
+
throw new ClassifierUnsupported(this.backend, this.model, "images");
|
|
682
|
+
checkImages(images);
|
|
683
|
+
let assets;
|
|
684
|
+
if (images.length > 0) {
|
|
685
|
+
assets = {};
|
|
686
|
+
for (const im of images)
|
|
687
|
+
assets[im.ref] = { type: "image", format: im.format, data_base64: im.data };
|
|
688
|
+
}
|
|
689
|
+
return { state: sent, images, assets };
|
|
690
|
+
}
|
|
691
|
+
/** The systemone style on every backend: lift, check, cache, post, decode. */
|
|
692
|
+
async evaluateWire(state, questions, signal, validate) {
|
|
693
|
+
const p = this.prepare(state);
|
|
694
|
+
const cache = this.opts.cache;
|
|
695
|
+
let modelKey = "";
|
|
696
|
+
if (cache) {
|
|
697
|
+
modelKey = classifierModelCacheKey(this.baseUrl, this.model);
|
|
698
|
+
const served = await cache.get(modelKey);
|
|
699
|
+
if (typeof served === "string") {
|
|
700
|
+
const hit = await cache.get(classifierDecisionCacheKey(this.baseUrl, served, p.state, p.assets, questions));
|
|
701
|
+
if (typeof hit === "string") {
|
|
702
|
+
try {
|
|
703
|
+
const d = decodeDecision(JSON.parse(hit), served);
|
|
704
|
+
if (!validateDecision(questions, d))
|
|
705
|
+
return new Decision(d.model, d.answers, d.usage, d.calibrated, true);
|
|
706
|
+
}
|
|
707
|
+
catch {
|
|
708
|
+
// a corrupt entry is a miss
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
const extra = p.images.length > 0 && this.preset.imagesHeader ? { [CLASSIFIER_IMAGES_HEADER]: "true" } : {};
|
|
714
|
+
let text = await this.post(this.body(p, questions), signal, extra);
|
|
452
715
|
let parsed;
|
|
453
716
|
try {
|
|
454
717
|
parsed = JSON.parse(text);
|
|
@@ -456,7 +719,20 @@ export class Classifier {
|
|
|
456
719
|
catch (e) {
|
|
457
720
|
throw new Error(`classifier: response was not JSON: ${e.message}`);
|
|
458
721
|
}
|
|
459
|
-
|
|
722
|
+
if (this.preset.wire === "cloudflare") {
|
|
723
|
+
parsed = unwrapCloudflare(parsed);
|
|
724
|
+
text = canonicalJson(parsed);
|
|
725
|
+
}
|
|
726
|
+
const d = decodeDecision(parsed, this.model);
|
|
727
|
+
const verr = validateDecision(questions, d);
|
|
728
|
+
if (validate && verr)
|
|
729
|
+
throw verr;
|
|
730
|
+
if (cache && !verr) {
|
|
731
|
+
const served = d.model || this.model;
|
|
732
|
+
await cache.set(modelKey, served);
|
|
733
|
+
await cache.set(classifierDecisionCacheKey(this.baseUrl, served, p.state, p.assets, questions), text);
|
|
734
|
+
}
|
|
735
|
+
return d;
|
|
460
736
|
}
|
|
461
737
|
/**
|
|
462
738
|
* The one POST, with the §8 retry budget, reusing the client's `ErrorInfo`/`ErrorTier`
|
|
@@ -467,8 +743,13 @@ export class Classifier {
|
|
|
467
743
|
* is never echoed back (a gateway happily reflects a bad Authorization header into its own
|
|
468
744
|
* 401 text).
|
|
469
745
|
*/
|
|
470
|
-
async post(payload, signal) {
|
|
471
|
-
|
|
746
|
+
async post(payload, signal, extraHeaders = {}) {
|
|
747
|
+
// `endpoint` is what errors name: the CONFIGURED base, ${ENV} unexpanded, so an account id in
|
|
748
|
+
// a templated URL never reaches an error. `url` is what is dialled, expanded at call time.
|
|
749
|
+
let endpoint = this.baseUrl.replace(/\/+$/, "");
|
|
750
|
+
if (this.preset.wire !== "cloudflare")
|
|
751
|
+
endpoint += "/systemone";
|
|
752
|
+
const url = endpoint.includes("${") ? expandEnv(endpoint) : endpoint;
|
|
472
753
|
const retries = this.opts.retries ?? 2;
|
|
473
754
|
const base = this.opts.retryBaseMs ?? 500;
|
|
474
755
|
const classify = this.opts.onError ?? ((info) => (info.retryable ? "retry" : "fail"));
|
|
@@ -491,9 +772,9 @@ export class Classifier {
|
|
|
491
772
|
let res;
|
|
492
773
|
let thrown;
|
|
493
774
|
try {
|
|
494
|
-
res = await doFetch(
|
|
775
|
+
res = await doFetch(url, {
|
|
495
776
|
method: "POST",
|
|
496
|
-
headers: this.requestHeaders(),
|
|
777
|
+
headers: this.requestHeaders(extraHeaders, url),
|
|
497
778
|
body: payload,
|
|
498
779
|
signal: timer.signal,
|
|
499
780
|
});
|
|
@@ -516,7 +797,7 @@ export class Classifier {
|
|
|
516
797
|
lastErr = new Error(`classifier: POST ${endpoint}: HTTP ${res.status}${cause(res.status, text)}`);
|
|
517
798
|
}
|
|
518
799
|
else {
|
|
519
|
-
lastErr = new Error(`classifier: POST ${endpoint}: ${errText(thrown)}`);
|
|
800
|
+
lastErr = new Error(`classifier: POST ${endpoint}: ${errText(thrown).split(url).join(endpoint)}`);
|
|
520
801
|
}
|
|
521
802
|
const retryable = res ? res.status === 408 || isRetryableStatus(res.status, this.opts.retryableStatuses) : true;
|
|
522
803
|
if (attempt >= retries)
|
|
@@ -529,11 +810,15 @@ export class Classifier {
|
|
|
529
810
|
}
|
|
530
811
|
}
|
|
531
812
|
/** Credential and `${ENV_VAR}` headers resolve AT CALL TIME. Never logged, never returned. */
|
|
532
|
-
requestHeaders() {
|
|
813
|
+
requestHeaders(extra, url) {
|
|
533
814
|
const headers = { "Content-Type": "application/json" };
|
|
534
|
-
const
|
|
815
|
+
const keyEnv = this.explicitKeyEnv ? this.apiKeyEnv : classifierKeyEnv(this.backend, url);
|
|
816
|
+
const key = keyEnv ? process.env[keyEnv] : undefined;
|
|
535
817
|
if (key)
|
|
536
818
|
headers["Authorization"] = `Bearer ${key}`;
|
|
819
|
+
// The image opt-in header goes BEFORE host headers, which win.
|
|
820
|
+
for (const [k, v] of Object.entries(extra))
|
|
821
|
+
headers[k] = v;
|
|
537
822
|
for (const [k, v] of Object.entries(this.opts.headers ?? {}))
|
|
538
823
|
headers[k] = expandEnv(v);
|
|
539
824
|
return headers;
|
|
@@ -577,7 +862,7 @@ export class Classifier {
|
|
|
577
862
|
emit(ev) {
|
|
578
863
|
this.opts.onMetric?.(ev);
|
|
579
864
|
}
|
|
580
|
-
emitError(start, e) {
|
|
865
|
+
emitError(start, e, l1) {
|
|
581
866
|
this.emit({
|
|
582
867
|
event: "classifier.evaluate",
|
|
583
868
|
model: this.model,
|
|
@@ -586,6 +871,8 @@ export class Classifier {
|
|
|
586
871
|
promptTokens: 0,
|
|
587
872
|
completionTokens: 0,
|
|
588
873
|
error: errText(e),
|
|
874
|
+
...l1,
|
|
875
|
+
cached: false,
|
|
589
876
|
});
|
|
590
877
|
}
|
|
591
878
|
}
|
|
@@ -669,3 +956,324 @@ function sleep(ms) {
|
|
|
669
956
|
setTimeout(resolve, ms);
|
|
670
957
|
});
|
|
671
958
|
}
|
|
959
|
+
// ---------------------------------------------------------------- backends, images, cache, validation
|
|
960
|
+
// SPEC.md §8B "Backends, images, cache and validation" (change add-classifier-backends, ADR 0038).
|
|
961
|
+
// Every default here is a PROPOSED default, owner to confirm (design.md PD-1..PD-15).
|
|
962
|
+
/** Ordered named questions -> the questions map. A duplicate name is an error naming it. */
|
|
963
|
+
function namedQuestions(qs) {
|
|
964
|
+
const out = {};
|
|
965
|
+
for (const { name, question } of qs) {
|
|
966
|
+
if (Object.prototype.hasOwnProperty.call(out, name))
|
|
967
|
+
throw new Error(`duplicate question name ${JSON.stringify(name)}`);
|
|
968
|
+
out[name] = question;
|
|
969
|
+
}
|
|
970
|
+
return out;
|
|
971
|
+
}
|
|
972
|
+
function sha256Hex(s) {
|
|
973
|
+
return createHash("sha256").update(s, "utf8").digest("hex");
|
|
974
|
+
}
|
|
975
|
+
function containsPart(v) {
|
|
976
|
+
if (Array.isArray(v))
|
|
977
|
+
return v.some(containsPart);
|
|
978
|
+
if (v !== null && typeof v === "object") {
|
|
979
|
+
if (isContentPart(v))
|
|
980
|
+
return true;
|
|
981
|
+
return Object.values(v).some(containsPart);
|
|
982
|
+
}
|
|
983
|
+
return false;
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* The §8B lifting walk. A state with no content part is returned VERBATIM (the same value), so a
|
|
987
|
+
* text-only state is byte-identical to before. Otherwise depth-first, object keys in ASCII order,
|
|
988
|
+
* arrays in order: an image part with inline data becomes a reference (its key when unused, else
|
|
989
|
+
* the smallest unused `image<n>`), a text part becomes its text, any other part is an error.
|
|
990
|
+
*/
|
|
991
|
+
function liftImages(state) {
|
|
992
|
+
if (!containsPart(state))
|
|
993
|
+
return { state, images: [] };
|
|
994
|
+
const used = new Set();
|
|
995
|
+
const images = [];
|
|
996
|
+
const nextRef = () => {
|
|
997
|
+
for (let n = 1;; n++)
|
|
998
|
+
if (!used.has(`image${n}`))
|
|
999
|
+
return `image${n}`;
|
|
1000
|
+
};
|
|
1001
|
+
const walk = (v, key) => {
|
|
1002
|
+
if (Array.isArray(v))
|
|
1003
|
+
return v.map((e) => walk(e, ""));
|
|
1004
|
+
if (v === null || typeof v !== "object")
|
|
1005
|
+
return v;
|
|
1006
|
+
if (isContentPart(v)) {
|
|
1007
|
+
const part = v;
|
|
1008
|
+
if (part.type === "text")
|
|
1009
|
+
return part.text;
|
|
1010
|
+
if (part.type === "image") {
|
|
1011
|
+
if (!part.data)
|
|
1012
|
+
throw new Error("classifier: state: an image part needs inline data (a URL image cannot be sent)");
|
|
1013
|
+
const ref = key === "" || used.has(key) ? nextRef() : key;
|
|
1014
|
+
used.add(ref);
|
|
1015
|
+
images.push({ ref, format: part.mimeType, data: part.data });
|
|
1016
|
+
return ref;
|
|
1017
|
+
}
|
|
1018
|
+
throw new Error(`classifier: state: ${part.type} parts are not supported`);
|
|
1019
|
+
}
|
|
1020
|
+
const obj = v;
|
|
1021
|
+
const out = {};
|
|
1022
|
+
for (const k of Object.keys(obj).sort())
|
|
1023
|
+
out[k] = walk(obj[k], k);
|
|
1024
|
+
return out;
|
|
1025
|
+
};
|
|
1026
|
+
return { state: walk(state, ""), images };
|
|
1027
|
+
}
|
|
1028
|
+
const IMAGE_FORMATS = { "image/png": "png", "image/jpeg": "jpeg", "image/jpg": "jpeg" };
|
|
1029
|
+
/** Format, per-image and per-request limits, in collection order; normalises `format`. */
|
|
1030
|
+
function checkImages(images) {
|
|
1031
|
+
let total = 0;
|
|
1032
|
+
for (const im of images) {
|
|
1033
|
+
const format = IMAGE_FORMATS[im.format];
|
|
1034
|
+
if (!format)
|
|
1035
|
+
throw new Error(`classifier: image "${im.ref}": type "${im.format}" is not supported (PNG or JPEG only)`);
|
|
1036
|
+
im.format = format;
|
|
1037
|
+
if (im.data.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(im.data)) {
|
|
1038
|
+
throw new Error(`classifier: image "${im.ref}": data is not base64`);
|
|
1039
|
+
}
|
|
1040
|
+
const raw = Buffer.from(im.data, "base64");
|
|
1041
|
+
if (raw.length > CLASSIFIER_IMAGE_MAX_BYTES) {
|
|
1042
|
+
throw new Error(`classifier: image "${im.ref}": ${raw.length} bytes, over 16 MiB`);
|
|
1043
|
+
}
|
|
1044
|
+
const size = imageSize(format, raw);
|
|
1045
|
+
if (!size)
|
|
1046
|
+
throw new Error(`classifier: image "${im.ref}": not a readable ${format.toUpperCase()}`);
|
|
1047
|
+
const [w, h] = size;
|
|
1048
|
+
const ok = (n) => n >= CLASSIFIER_IMAGE_MIN_SIDE && n <= CLASSIFIER_IMAGE_MAX_SIDE;
|
|
1049
|
+
if (!ok(w) || !ok(h)) {
|
|
1050
|
+
throw new Error(`classifier: image "${im.ref}": ${w}x${h}; each side must be ${CLASSIFIER_IMAGE_MIN_SIDE}..${CLASSIFIER_IMAGE_MAX_SIDE} px`);
|
|
1051
|
+
}
|
|
1052
|
+
total += raw.length;
|
|
1053
|
+
}
|
|
1054
|
+
if (total > CLASSIFIER_REQUEST_IMAGE_BYTES) {
|
|
1055
|
+
throw new Error(`classifier: image "${images[images.length - 1].ref}": request images total ${total} bytes, over 32 MiB`);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
/** Width/height from a PNG IHDR or the first JPEG SOF marker; undefined when unreadable. */
|
|
1059
|
+
function imageSize(format, b) {
|
|
1060
|
+
const be16 = (i) => (b[i] << 8) | b[i + 1];
|
|
1061
|
+
const be32 = (i) => ((b[i] << 24) >>> 0) + (b[i + 1] << 16) + (b[i + 2] << 8) + b[i + 3];
|
|
1062
|
+
if (format === "png") {
|
|
1063
|
+
const sig = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
|
1064
|
+
if (b.length < 24 || sig.some((x, i) => b[i] !== x))
|
|
1065
|
+
return undefined;
|
|
1066
|
+
if (String.fromCharCode(b[12], b[13], b[14], b[15]) !== "IHDR")
|
|
1067
|
+
return undefined;
|
|
1068
|
+
return [be32(16), be32(20)];
|
|
1069
|
+
}
|
|
1070
|
+
if (b.length < 4 || b[0] !== 0xff || b[1] !== 0xd8)
|
|
1071
|
+
return undefined;
|
|
1072
|
+
let i = 2;
|
|
1073
|
+
while (i + 3 < b.length) {
|
|
1074
|
+
if (b[i] !== 0xff)
|
|
1075
|
+
return undefined;
|
|
1076
|
+
const m = b[i + 1];
|
|
1077
|
+
if (m === 0xff) {
|
|
1078
|
+
i++;
|
|
1079
|
+
continue;
|
|
1080
|
+
}
|
|
1081
|
+
if (m >= 0xc0 && m <= 0xcf && m !== 0xc4 && m !== 0xc8 && m !== 0xcc) {
|
|
1082
|
+
if (i + 8 >= b.length)
|
|
1083
|
+
return undefined;
|
|
1084
|
+
return [be16(i + 7), be16(i + 5)];
|
|
1085
|
+
}
|
|
1086
|
+
i += 2 + be16(i + 2);
|
|
1087
|
+
}
|
|
1088
|
+
return undefined;
|
|
1089
|
+
}
|
|
1090
|
+
/**
|
|
1091
|
+
* `: <code> <message>` from errors[0] of a failed envelope (SPEC §8B), through the same redaction
|
|
1092
|
+
* and cap as cause(). Cloudflare reports a missing balance, a bad model id and a malformed
|
|
1093
|
+
* request alike as HTTP 200 + success:false, so this is the only thing that tells them apart.
|
|
1094
|
+
* `""` when there is nothing to show.
|
|
1095
|
+
*/
|
|
1096
|
+
function cloudflareCause(env) {
|
|
1097
|
+
const errs = env !== null && typeof env === "object" ? env.errors : undefined;
|
|
1098
|
+
const e0 = Array.isArray(errs) ? errs[0] : undefined;
|
|
1099
|
+
if (e0 === null || typeof e0 !== "object")
|
|
1100
|
+
return "";
|
|
1101
|
+
const parts = [];
|
|
1102
|
+
if ((typeof e0.code === "number" || typeof e0.code === "string") && String(e0.code) !== "")
|
|
1103
|
+
parts.push(String(e0.code));
|
|
1104
|
+
if (typeof e0.message === "string" && e0.message !== "")
|
|
1105
|
+
parts.push(e0.message);
|
|
1106
|
+
const s = capErrorBody(redactErrorBody(parts.join(" ")));
|
|
1107
|
+
return s ? ": " + s : "";
|
|
1108
|
+
}
|
|
1109
|
+
/** The System One reply inside a Cloudflare `/ai/run` envelope. Only errors[0] is ever echoed. */
|
|
1110
|
+
function unwrapCloudflare(env) {
|
|
1111
|
+
if (env === null || typeof env !== "object" || Array.isArray(env) || env.success !== true) {
|
|
1112
|
+
throw new Error("classifier: cloudflare: envelope not successful" + cloudflareCause(env));
|
|
1113
|
+
}
|
|
1114
|
+
let result = env.result;
|
|
1115
|
+
if (result !== null && typeof result === "object" && !Array.isArray(result) && typeof result.state === "string") {
|
|
1116
|
+
if (result.state !== "Completed")
|
|
1117
|
+
throw new Error(`classifier: cloudflare: run state "${result.state}"`);
|
|
1118
|
+
result = result.result;
|
|
1119
|
+
}
|
|
1120
|
+
return result;
|
|
1121
|
+
}
|
|
1122
|
+
/** §8B model key: which model an endpoint + requested model (alias) served last. */
|
|
1123
|
+
export function classifierModelCacheKey(endpoint, model) {
|
|
1124
|
+
return sha256Hex(canonicalJson({ endpoint, kind: "model", model }));
|
|
1125
|
+
}
|
|
1126
|
+
/** §8B decision key, on the SERVED model. No assets ⇒ the `assets` member is omitted. */
|
|
1127
|
+
export function classifierDecisionCacheKey(endpoint, servedModel, state, assets, questions) {
|
|
1128
|
+
return sha256Hex(classifierDecisionCacheKeyBytes(endpoint, servedModel, state, assets, questions));
|
|
1129
|
+
}
|
|
1130
|
+
/** The canonical JSON a decision key hashes (exported for conformance tests). */
|
|
1131
|
+
export function classifierDecisionCacheKeyBytes(endpoint, servedModel, state, assets, questions) {
|
|
1132
|
+
const qs = {};
|
|
1133
|
+
for (const [k, q] of Object.entries(questions))
|
|
1134
|
+
qs[k] = questionWire(q);
|
|
1135
|
+
const m = { endpoint, kind: "decision", model: servedModel, questions: qs, state };
|
|
1136
|
+
if (assets && Object.keys(assets).length > 0)
|
|
1137
|
+
m.assets = assets;
|
|
1138
|
+
return canonicalJson(m);
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* In-process TTL cache with a size cap (oldest-inserted out first). `ttlMs` 0 = no expiry; `max`
|
|
1142
|
+
* 0 = unbounded. toolnexus persists nothing itself: implement `ClassifierCache` to persist.
|
|
1143
|
+
*/
|
|
1144
|
+
export class MemoryCache {
|
|
1145
|
+
ttlMs;
|
|
1146
|
+
max;
|
|
1147
|
+
now;
|
|
1148
|
+
m = new Map();
|
|
1149
|
+
constructor(ttlMs = 0, max = 0, now = Date.now) {
|
|
1150
|
+
this.ttlMs = ttlMs;
|
|
1151
|
+
this.max = max;
|
|
1152
|
+
this.now = now;
|
|
1153
|
+
}
|
|
1154
|
+
get(key) {
|
|
1155
|
+
const e = this.m.get(key);
|
|
1156
|
+
if (!e)
|
|
1157
|
+
return undefined;
|
|
1158
|
+
if (this.ttlMs > 0 && this.now() - e.at > this.ttlMs) {
|
|
1159
|
+
this.m.delete(key);
|
|
1160
|
+
return undefined;
|
|
1161
|
+
}
|
|
1162
|
+
return e.v;
|
|
1163
|
+
}
|
|
1164
|
+
set(key, value) {
|
|
1165
|
+
this.m.delete(key); // a re-set is a fresh insertion
|
|
1166
|
+
this.m.set(key, { v: value, at: this.now() });
|
|
1167
|
+
if (this.max > 0)
|
|
1168
|
+
while (this.m.size > this.max)
|
|
1169
|
+
this.m.delete(this.m.keys().next().value);
|
|
1170
|
+
}
|
|
1171
|
+
get size() {
|
|
1172
|
+
return this.m.size;
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
/**
|
|
1176
|
+
* V1: fail closed on a reply that does not answer exactly the questions asked, within their
|
|
1177
|
+
* schema. First failure wins, in the §8B order. Returns the error, or undefined when valid.
|
|
1178
|
+
*/
|
|
1179
|
+
export function validateDecision(questions, decision) {
|
|
1180
|
+
const bad = (msg) => new Error(`classifier: invalid decision: ${msg}`);
|
|
1181
|
+
const answers = decision.answers ?? {};
|
|
1182
|
+
const has = (o, k) => Object.prototype.hasOwnProperty.call(o, k);
|
|
1183
|
+
const qKeys = Object.keys(questions).sort();
|
|
1184
|
+
for (const k of qKeys)
|
|
1185
|
+
if (!has(answers, k))
|
|
1186
|
+
return bad(`no answer "${k}"`);
|
|
1187
|
+
const extra = Object.keys(answers)
|
|
1188
|
+
.filter((k) => !has(questions, k))
|
|
1189
|
+
.sort();
|
|
1190
|
+
if (extra.length > 0)
|
|
1191
|
+
return bad(`unexpected answer "${extra[0]}"`);
|
|
1192
|
+
const in01 = (p) => typeof p === "number" && !Number.isNaN(p) && p >= 0 && p <= 1;
|
|
1193
|
+
const dist = (k, probs, offered) => {
|
|
1194
|
+
let sum = 0;
|
|
1195
|
+
const pk = Object.keys(probs ?? {}).sort();
|
|
1196
|
+
for (const o of pk) {
|
|
1197
|
+
if (!offered(o))
|
|
1198
|
+
return bad(`answer "${k}": probability for "${o}", which was not offered`);
|
|
1199
|
+
if (!in01(probs[o]))
|
|
1200
|
+
return bad(`answer "${k}": probability ${probs[o]} outside [0,1]`);
|
|
1201
|
+
sum += probs[o];
|
|
1202
|
+
}
|
|
1203
|
+
if (pk.length > 0 && Math.abs(sum - 1) > 0.02)
|
|
1204
|
+
return bad(`answer "${k}": probabilities sum to ${sum.toFixed(4)}`);
|
|
1205
|
+
return undefined;
|
|
1206
|
+
};
|
|
1207
|
+
for (const k of qKeys) {
|
|
1208
|
+
const raw = questions[k];
|
|
1209
|
+
const q = raw;
|
|
1210
|
+
const a = answers[k];
|
|
1211
|
+
const qType = raw.type;
|
|
1212
|
+
const aType = a?.type;
|
|
1213
|
+
if (aType !== qType)
|
|
1214
|
+
return bad(`answer "${k}" is a ${aType} answer, not ${qType}`);
|
|
1215
|
+
if (isRawQuestion(raw))
|
|
1216
|
+
continue;
|
|
1217
|
+
if (q.type === "noul") {
|
|
1218
|
+
if (!in01(a.noul))
|
|
1219
|
+
return bad(`answer "${k}": noul ${a.noul} outside [0,1]`);
|
|
1220
|
+
}
|
|
1221
|
+
else if (q.type === "choice") {
|
|
1222
|
+
const offered = (o) => has(q.criteria ?? {}, o);
|
|
1223
|
+
if (!offered(a.choice))
|
|
1224
|
+
return bad(`answer "${k}": choice "${a.choice}" was not offered`);
|
|
1225
|
+
const err = dist(k, a.probabilities, offered);
|
|
1226
|
+
if (err)
|
|
1227
|
+
return err;
|
|
1228
|
+
if (!in01(a.confidence))
|
|
1229
|
+
return bad(`answer "${k}": confidence ${a.confidence} outside [0,1]`);
|
|
1230
|
+
}
|
|
1231
|
+
else {
|
|
1232
|
+
const n = (q.criteria ?? []).length;
|
|
1233
|
+
if (typeof a.score !== "number" || Number.isNaN(a.score) || a.score < 0 || a.score > n - 1) {
|
|
1234
|
+
return bad(`answer "${k}": score ${a.score} outside [0,${n - 1}]`);
|
|
1235
|
+
}
|
|
1236
|
+
const offered = (o) => Array.from({ length: n }, (_, i) => String(i)).includes(o);
|
|
1237
|
+
const err = dist(k, a.probabilities, offered);
|
|
1238
|
+
if (err)
|
|
1239
|
+
return err;
|
|
1240
|
+
if (!in01(a.confidence))
|
|
1241
|
+
return bad(`answer "${k}": confidence ${a.confidence} outside [0,1]`);
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
return undefined;
|
|
1245
|
+
}
|
|
1246
|
+
/** 0.8 / 0.2 / 0.6 (jevx's builtin profile). */
|
|
1247
|
+
export const DEFAULT_THRESHOLDS = Object.freeze({ yes: 0.8, no: 0.2, minConfidence: 0.6 });
|
|
1248
|
+
/**
|
|
1249
|
+
* The band of one answer under `Thresholds`: noul `p >= yes` ⇒ yes, `p <= no` ⇒ no, else
|
|
1250
|
+
* uncertain; choice/score `confidence >= minConfidence` ⇒ yes, else uncertain. No near-uniform
|
|
1251
|
+
* demotion (that is `Bands`' rule). A raw answer is uncertain.
|
|
1252
|
+
*/
|
|
1253
|
+
export function thresholdBand(answer, t = DEFAULT_THRESHOLDS) {
|
|
1254
|
+
const a = answer;
|
|
1255
|
+
if (a.type === "noul") {
|
|
1256
|
+
if (a.noul >= t.yes)
|
|
1257
|
+
return "yes";
|
|
1258
|
+
if (a.noul <= t.no)
|
|
1259
|
+
return "no";
|
|
1260
|
+
}
|
|
1261
|
+
else if ((a.type === "choice" || a.type === "score") && a.confidence >= t.minConfidence) {
|
|
1262
|
+
return "yes";
|
|
1263
|
+
}
|
|
1264
|
+
return "uncertain";
|
|
1265
|
+
}
|
|
1266
|
+
/**
|
|
1267
|
+
* Split a question set into chunks of `size`, keys in ASCII order. A helper, not automatic:
|
|
1268
|
+
* joint calibration across a split is unproven, so the host decides to split and merges.
|
|
1269
|
+
*/
|
|
1270
|
+
export function chunkQuestions(questions, size) {
|
|
1271
|
+
if (!(size >= 1))
|
|
1272
|
+
throw new Error(`classifier: chunk size must be at least 1, got ${size}`);
|
|
1273
|
+
const keys = Object.keys(questions).sort();
|
|
1274
|
+
const out = [];
|
|
1275
|
+
for (let i = 0; i < keys.length; i += size) {
|
|
1276
|
+
out.push(Object.fromEntries(keys.slice(i, i + size).map((k) => [k, questions[k]])));
|
|
1277
|
+
}
|
|
1278
|
+
return out;
|
|
1279
|
+
}
|