toolnexus 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -1
- package/dist/acp.d.ts +37 -6
- package/dist/acp.js +239 -52
- package/dist/classifier.d.ts +189 -31
- package/dist/classifier.js +589 -30
- package/dist/client.d.ts +38 -8
- package/dist/client.js +102 -23
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/judge.d.ts +6 -3
- package/dist/judge.js +13 -1
- package/dist/mcp.d.ts +32 -7
- package/dist/mcp.js +125 -36
- package/dist/mcp2026/auth.d.ts +137 -0
- package/dist/mcp2026/auth.js +446 -0
- package/dist/mcp2026/client.d.ts +48 -0
- package/dist/mcp2026/client.js +160 -0
- package/dist/mcp2026/protocol.d.ts +54 -0
- package/dist/mcp2026/protocol.js +113 -0
- package/dist/mcp2026/skills.d.ts +5 -0
- package/dist/mcp2026/skills.js +25 -0
- package/dist/mcp2026/source.d.ts +86 -0
- package/dist/mcp2026/source.js +372 -0
- package/dist/mcp2026/stdio.d.ts +24 -0
- package/dist/mcp2026/stdio.js +99 -0
- package/dist/mcp2026/toolsearch.d.ts +5 -0
- package/dist/mcp2026/toolsearch.js +58 -0
- package/dist/mcphandler.d.ts +169 -0
- package/dist/mcphandler.js +1145 -0
- package/dist/mcpheader.d.ts +5 -0
- package/dist/mcpheader.js +18 -0
- package/dist/mcpserve.d.ts +24 -15
- package/dist/mcpserve.js +105 -2
- package/dist/mcpserverauth.d.ts +48 -0
- package/dist/mcpserverauth.js +203 -0
- package/dist/serve.js +7 -0
- package/dist/skill.d.ts +21 -0
- package/dist/skill.js +44 -0
- package/dist/toolkit.d.ts +20 -1
- package/dist/toolkit.js +25 -3
- package/dist/types.d.ts +22 -0
- package/package.json +4 -3
- package/src/acp.ts +258 -55
- package/src/classifier.ts +677 -46
- package/src/client.ts +130 -27
- package/src/index.ts +2 -0
- package/src/judge.ts +19 -4
- package/src/mcp.ts +162 -43
- package/src/mcp2026/auth.ts +473 -0
- package/src/mcp2026/client.ts +161 -0
- package/src/mcp2026/protocol.ts +135 -0
- package/src/mcp2026/skills.ts +22 -0
- package/src/mcp2026/source.ts +407 -0
- package/src/mcp2026/stdio.ts +102 -0
- package/src/mcp2026/toolsearch.ts +56 -0
- package/src/mcphandler.ts +1252 -0
- package/src/mcpheader.ts +19 -0
- package/src/mcpserve.ts +115 -2
- package/src/mcpserverauth.ts +216 -0
- package/src/serve.ts +7 -0
- package/src/skill.ts +50 -0
- package/src/toolkit.ts +31 -4
- package/src/types.ts +23 -0
package/dist/classifier.js
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Classifier (SPEC.md §8B) — the contract for a JUDGMENT, as `Tool` is the contract for an
|
|
3
|
+
* ACTION. A System One model takes a state plus pre-declared, typed questions and returns
|
|
4
|
+
* calibrated answers with no free text. It has no messages, no tool calling and no streaming,
|
|
5
|
+
* so it never enters the §8 client loop.
|
|
6
|
+
*
|
|
7
|
+
* A classifier INTERPRETS; it never AUTHORISES. Schema validity is not correctness: a decision
|
|
8
|
+
* can be confidently wrong, and "cannot hallucinate" means only that the returned value is in
|
|
9
|
+
* the declared schema. Numeric limits, permission checks and allowlists stay in code. Nothing
|
|
10
|
+
* here is a security control.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
1
13
|
import { redactErrorBody, capErrorBody } from "./client.js";
|
|
2
14
|
import { retryAfterMs, isRetryableStatus } from "./retry.js";
|
|
15
|
+
import { isContentPart } from "./content.js";
|
|
3
16
|
// ---------------------------------------------------------------- constants
|
|
4
17
|
/** The System One endpoint base. */
|
|
5
18
|
export const DEFAULT_CLASSIFIER_BASE_URL = "https://api.typesafe.ai/v1";
|
|
@@ -11,11 +24,74 @@ export const DEFAULT_CLASSIFIER_API_KEY_ENV = "TYPESAFE_API_KEY";
|
|
|
11
24
|
export const DEFAULT_CLASSIFIER_TIMEOUT_MS = 10_000;
|
|
12
25
|
/** `evaluateBatch`: at most this many states in flight by default. */
|
|
13
26
|
export const DEFAULT_BATCH_CONCURRENCY = 16;
|
|
14
|
-
/**
|
|
15
|
-
export const
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
27
|
+
/** TypeSafe's required opt-in header for image ("danger zone") input. */
|
|
28
|
+
export const CLASSIFIER_IMAGES_HEADER = "TypeSafe-Danger-Zone-Images";
|
|
29
|
+
/** Image limits (docs.typesafe.ai/images): checked client-side, never resized. */
|
|
30
|
+
export const CLASSIFIER_IMAGE_MIN_SIDE = 280;
|
|
31
|
+
export const CLASSIFIER_IMAGE_MAX_SIDE = 1568;
|
|
32
|
+
export const CLASSIFIER_IMAGE_MAX_BYTES = 16 * 1024 * 1024;
|
|
33
|
+
export const CLASSIFIER_REQUEST_IMAGE_BYTES = 32 * 1024 * 1024;
|
|
34
|
+
/**
|
|
35
|
+
* The backend table (PROPOSED defaults, owner to confirm — design.md PD-1..PD-4). Vercel and
|
|
36
|
+
* Cloudflare-Jev images stay false until a live probe proves them.
|
|
37
|
+
*/
|
|
38
|
+
export const CLASSIFIER_BACKENDS = Object.freeze({
|
|
39
|
+
typesafe: {
|
|
40
|
+
baseUrl: DEFAULT_CLASSIFIER_BASE_URL,
|
|
41
|
+
model: DEFAULT_CLASSIFIER_MODEL,
|
|
42
|
+
apiKeyEnv: DEFAULT_CLASSIFIER_API_KEY_ENV,
|
|
43
|
+
wire: "systemone",
|
|
44
|
+
imagesHeader: true,
|
|
45
|
+
supports: () => ({ images: true }),
|
|
46
|
+
},
|
|
47
|
+
openrouter: {
|
|
48
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
49
|
+
model: "typesafe/jev-1.14",
|
|
50
|
+
apiKeyEnv: "OPENROUTER_API_KEY",
|
|
51
|
+
wire: "systemone",
|
|
52
|
+
imagesHeader: true,
|
|
53
|
+
supports: () => ({ images: true }),
|
|
54
|
+
},
|
|
55
|
+
cloudflare: {
|
|
56
|
+
baseUrl: "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/ai/run",
|
|
57
|
+
model: "typesafe/jev",
|
|
58
|
+
apiKeyEnv: "CLOUDFLARE_JEV_API_KEY",
|
|
59
|
+
wire: "cloudflare",
|
|
60
|
+
imagesHeader: false,
|
|
61
|
+
supports: (model) => ({ images: model.includes("clef") }),
|
|
62
|
+
},
|
|
63
|
+
vercel: {
|
|
64
|
+
baseUrl: "https://ai-gateway.vercel.sh/typesafe/v1",
|
|
65
|
+
model: "typesafe-ai/jev",
|
|
66
|
+
apiKeyEnv: "AI_GATEWAY_API_KEY",
|
|
67
|
+
wire: "systemone",
|
|
68
|
+
imagesHeader: true,
|
|
69
|
+
supports: () => ({ images: false }),
|
|
70
|
+
},
|
|
71
|
+
custom: {
|
|
72
|
+
baseUrl: "",
|
|
73
|
+
model: DEFAULT_CLASSIFIER_MODEL,
|
|
74
|
+
apiKeyEnv: DEFAULT_CLASSIFIER_API_KEY_ENV,
|
|
75
|
+
wire: "systemone",
|
|
76
|
+
imagesHeader: true,
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
/**
|
|
80
|
+
* Raised BEFORE any network call when the backend table says the backend cannot take what the
|
|
81
|
+
* request carries. Message, exact: `classifier: <backend> (<model>) does not support <feature>`.
|
|
82
|
+
*/
|
|
83
|
+
export class ClassifierUnsupported extends Error {
|
|
84
|
+
backend;
|
|
85
|
+
model;
|
|
86
|
+
feature;
|
|
87
|
+
constructor(backend, model, feature) {
|
|
88
|
+
super(`classifier: ${backend} (${model}) does not support ${feature}`);
|
|
89
|
+
this.backend = backend;
|
|
90
|
+
this.model = model;
|
|
91
|
+
this.feature = feature;
|
|
92
|
+
this.name = "ClassifierUnsupported";
|
|
93
|
+
}
|
|
94
|
+
}
|
|
19
95
|
/** Client-side cap on a choice's named options (§8B). */
|
|
20
96
|
export const MAX_CHOICE_OPTIONS = 255;
|
|
21
97
|
/** Bounds of a score rubric (§8B). */
|
|
@@ -26,6 +102,16 @@ export const MAX_SCORE_LEVELS = 10;
|
|
|
26
102
|
* port; `examples/judge/near-uniform.json` pins both sides at 0.0499 / 0.0501.
|
|
27
103
|
*/
|
|
28
104
|
export const NEAR_UNIFORM_TOLERANCE = 0.05;
|
|
105
|
+
const MODELLED_TYPES = new Set(["noul", "choice", "score"]);
|
|
106
|
+
/** True when `q` is a raw question (its type is not noul/choice/score). Not a type guard: every
|
|
107
|
+
* modelled question is structurally a `RawQuestion` too. */
|
|
108
|
+
export function isRawQuestion(q) {
|
|
109
|
+
return !MODELLED_TYPES.has(q?.type);
|
|
110
|
+
}
|
|
111
|
+
/** A raw question, sent as written. */
|
|
112
|
+
export function raw(type, instructions, criteria) {
|
|
113
|
+
return criteria === undefined ? { type, instructions } : { type, instructions, criteria };
|
|
114
|
+
}
|
|
29
115
|
/** A noul question. Pass `criteria` only when you mean it — absent ≠ empty on the wire. */
|
|
30
116
|
export function noul(instructions, criteria) {
|
|
31
117
|
return criteria === undefined ? { type: "noul", instructions } : { type: "noul", instructions, criteria };
|
|
@@ -47,7 +133,14 @@ export function choiceOver(instructions, items) {
|
|
|
47
133
|
return choice(instructions, items);
|
|
48
134
|
}
|
|
49
135
|
/** The wire form of one question — the object itself, minus an absent `criteria`. */
|
|
50
|
-
export function questionWire(
|
|
136
|
+
export function questionWire(question) {
|
|
137
|
+
if (isRawQuestion(question)) {
|
|
138
|
+
const r = question;
|
|
139
|
+
return r.criteria === undefined
|
|
140
|
+
? { type: r.type, instructions: r.instructions }
|
|
141
|
+
: { type: r.type, instructions: r.instructions, criteria: r.criteria };
|
|
142
|
+
}
|
|
143
|
+
const q = question;
|
|
51
144
|
switch (q.type) {
|
|
52
145
|
case "noul":
|
|
53
146
|
return q.criteria === undefined
|
|
@@ -60,7 +153,13 @@ export function questionWire(q) {
|
|
|
60
153
|
}
|
|
61
154
|
}
|
|
62
155
|
/** Client-side limits, enforced BEFORE the request. The error names the key and the limit. */
|
|
63
|
-
function validateQuestion(key,
|
|
156
|
+
function validateQuestion(key, question) {
|
|
157
|
+
const q = question;
|
|
158
|
+
if (isRawQuestion(question)) {
|
|
159
|
+
if (!question?.type)
|
|
160
|
+
throw new Error(`classifier: question "${key}": a raw question needs a type`);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
64
163
|
if (q.type === "choice") {
|
|
65
164
|
const n = Object.keys(q.criteria ?? {}).length;
|
|
66
165
|
if (n < 1 || n > MAX_CHOICE_OPTIONS) {
|
|
@@ -119,6 +218,10 @@ export function canonicalRequest(model, questions) {
|
|
|
119
218
|
}
|
|
120
219
|
return canonicalJson({ model, questions: qs });
|
|
121
220
|
}
|
|
221
|
+
/** True when an answer is a raw (unmodelled-type) answer. */
|
|
222
|
+
export function isRawAnswer(a) {
|
|
223
|
+
return !!a && typeof a === "object" && !MODELLED_TYPES.has(a.type) && "value" in a;
|
|
224
|
+
}
|
|
122
225
|
/** The legend in LEVEL order, which the map itself loses ("2" sorts before "10" as a level). */
|
|
123
226
|
export function levels(answer) {
|
|
124
227
|
return Object.keys(answer.legend)
|
|
@@ -134,6 +237,7 @@ export class Decision {
|
|
|
134
237
|
answers;
|
|
135
238
|
usage;
|
|
136
239
|
calibrated;
|
|
240
|
+
cached;
|
|
137
241
|
constructor(
|
|
138
242
|
/** What actually answered, which may be more specific than the model that was asked for. */
|
|
139
243
|
model, answers, usage,
|
|
@@ -142,11 +246,23 @@ export class Decision {
|
|
|
142
246
|
* unless it derived them from provider token probabilities. A THRESHOLD TUNED AGAINST ONE
|
|
143
247
|
* BACKEND DOES NOT TRANSFER TO ANOTHER.
|
|
144
248
|
*/
|
|
145
|
-
calibrated
|
|
249
|
+
calibrated,
|
|
250
|
+
/** True when the decision came from `ClassifierOptions.cache` and no request was sent. */
|
|
251
|
+
cached = false) {
|
|
146
252
|
this.model = model;
|
|
147
253
|
this.answers = answers;
|
|
148
254
|
this.usage = usage;
|
|
149
255
|
this.calibrated = calibrated;
|
|
256
|
+
this.cached = cached;
|
|
257
|
+
}
|
|
258
|
+
/** Typed read of a raw (unmodelled-type) answer. Throws on an absent key or a modelled type. */
|
|
259
|
+
raw(key) {
|
|
260
|
+
const a = this.answers[key];
|
|
261
|
+
if (a === undefined)
|
|
262
|
+
throw new Error(`classifier: no answer "${key}" in this decision`);
|
|
263
|
+
if (!isRawAnswer(a))
|
|
264
|
+
throw new Error(`classifier: answer "${key}" is a ${a.type} answer, not raw`);
|
|
265
|
+
return a;
|
|
150
266
|
}
|
|
151
267
|
/** Typed read. Throws (never returns undefined) on a wrong-type or absent key. */
|
|
152
268
|
noul(key) {
|
|
@@ -223,7 +339,10 @@ export function decodeDecision(raw, fallbackModel) {
|
|
|
223
339
|
};
|
|
224
340
|
break;
|
|
225
341
|
default:
|
|
226
|
-
|
|
342
|
+
// The union is OPEN (§8B): an unmodelled type (e.g. Jev 1.14 "box") decodes to a raw
|
|
343
|
+
// answer instead of failing the whole decision.
|
|
344
|
+
;
|
|
345
|
+
answers[key] = { type: String(a.type ?? ""), value: { ...a } };
|
|
227
346
|
}
|
|
228
347
|
}
|
|
229
348
|
const usage = {
|
|
@@ -262,6 +381,9 @@ export class Classifier {
|
|
|
262
381
|
model;
|
|
263
382
|
apiKeyEnv;
|
|
264
383
|
timeoutMs;
|
|
384
|
+
backend;
|
|
385
|
+
preset;
|
|
386
|
+
supports;
|
|
265
387
|
corpus = new Map();
|
|
266
388
|
/** The once-per-question-key set for the degenerate warning, so a per-turn judge does not
|
|
267
389
|
* flood the sink. */
|
|
@@ -269,14 +391,23 @@ export class Classifier {
|
|
|
269
391
|
constructor(opts) {
|
|
270
392
|
this.opts = opts;
|
|
271
393
|
this.style = opts.style ?? "systemone";
|
|
272
|
-
|
|
273
|
-
|
|
394
|
+
this.backend = opts.backend ?? "typesafe";
|
|
395
|
+
if (!Object.prototype.hasOwnProperty.call(CLASSIFIER_BACKENDS, this.backend)) {
|
|
396
|
+
throw new Error(`classifier: unknown backend "${this.backend}" (want "typesafe", "openrouter", "cloudflare", "vercel" or "custom")`);
|
|
397
|
+
}
|
|
398
|
+
const preset = CLASSIFIER_BACKENDS[this.backend];
|
|
399
|
+
this.preset = preset;
|
|
400
|
+
if (this.backend === "custom" && this.style === "systemone" && (!opts.baseUrl || !opts.supports)) {
|
|
401
|
+
throw new Error('classifier: backend "custom" requires baseUrl and supports');
|
|
402
|
+
}
|
|
403
|
+
this.baseUrl = opts.baseUrl || preset.baseUrl;
|
|
274
404
|
this.model = opts.model ?? preset.model;
|
|
275
405
|
this.apiKeyEnv = opts.apiKeyEnv ?? preset.apiKeyEnv;
|
|
276
406
|
const mismatch = backendMismatch(this.baseUrl, this.model);
|
|
277
407
|
if (mismatch)
|
|
278
408
|
throw new Error(mismatch);
|
|
279
409
|
this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS;
|
|
410
|
+
this.supports = opts.supports ? { ...opts.supports } : (preset.supports?.(this.model) ?? { images: false });
|
|
280
411
|
switch (this.style) {
|
|
281
412
|
case "systemone":
|
|
282
413
|
break;
|
|
@@ -309,11 +440,60 @@ export class Classifier {
|
|
|
309
440
|
* INDEPENDENT — one answer is never context for another.
|
|
310
441
|
*/
|
|
311
442
|
async evaluate(state, questions, signal) {
|
|
443
|
+
return this.run(state, questions, signal, false);
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Binds one state (a VALUE, not a session: nothing is sent until `evaluate`). Its `evaluate`
|
|
447
|
+
* runs the same path as `evaluate` PLUS fail-closed validation (`validateDecision`).
|
|
448
|
+
*/
|
|
449
|
+
state(value, opts = {}) {
|
|
450
|
+
return {
|
|
451
|
+
evaluate: async (...qs) => this.run(value, namedQuestions(qs), opts.signal, true),
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
/**
|
|
455
|
+
* Binds a batch of states. Its `evaluate` runs `state(v).evaluate` per state, at most 16 in
|
|
456
|
+
* flight, and returns one `{decision | error}` per state IN ORDER. One state's failure never
|
|
457
|
+
* fails the batch; only a question-set error (duplicate name) throws, before any request.
|
|
458
|
+
*/
|
|
459
|
+
states(...values) {
|
|
460
|
+
return {
|
|
461
|
+
evaluate: async (...qs) => {
|
|
462
|
+
const questions = namedQuestions(qs);
|
|
463
|
+
const out = new Array(values.length);
|
|
464
|
+
let next = 0;
|
|
465
|
+
const worker = async () => {
|
|
466
|
+
while (next < values.length) {
|
|
467
|
+
const i = next++;
|
|
468
|
+
try {
|
|
469
|
+
out[i] = { decision: await this.run(values[i], questions, undefined, true) };
|
|
470
|
+
}
|
|
471
|
+
catch (e) {
|
|
472
|
+
out[i] = { error: e instanceof Error ? e : new Error(String(e)) };
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
};
|
|
476
|
+
await Promise.all(Array.from({ length: Math.min(DEFAULT_BATCH_CONCURRENCY, values.length) }, worker));
|
|
477
|
+
return out;
|
|
478
|
+
},
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
/** The one path. `validate` = fail closed on `validateDecision` (the value builders). */
|
|
482
|
+
async run(state, questions, signal, validate) {
|
|
312
483
|
const start = Date.now();
|
|
313
484
|
const keys = Object.keys(questions ?? {}).sort();
|
|
485
|
+
// L1: count + hash of the canonical request on every evaluate event ("" when unencodable).
|
|
486
|
+
let qHash = "";
|
|
487
|
+
try {
|
|
488
|
+
qHash = sha256Hex(canonicalRequest(this.model, questions ?? {}));
|
|
489
|
+
}
|
|
490
|
+
catch {
|
|
491
|
+
qHash = "";
|
|
492
|
+
}
|
|
493
|
+
const l1 = { questions: keys.length, questionsHash: qHash };
|
|
314
494
|
if (keys.length === 0) {
|
|
315
495
|
const err = new Error("classifier: no questions to evaluate");
|
|
316
|
-
this.emitError(start, err);
|
|
496
|
+
this.emitError(start, err, l1);
|
|
317
497
|
throw err;
|
|
318
498
|
}
|
|
319
499
|
try {
|
|
@@ -328,7 +508,7 @@ export class Classifier {
|
|
|
328
508
|
}
|
|
329
509
|
}
|
|
330
510
|
catch (e) {
|
|
331
|
-
this.emitError(start, e);
|
|
511
|
+
this.emitError(start, e, l1);
|
|
332
512
|
throw e;
|
|
333
513
|
}
|
|
334
514
|
// Detection, never repair (ADR 0020/0021): the request goes out BYTE-UNCHANGED and the
|
|
@@ -347,11 +527,16 @@ export class Classifier {
|
|
|
347
527
|
decision = await this.evaluateLLM(state, questions, signal);
|
|
348
528
|
break;
|
|
349
529
|
default:
|
|
350
|
-
decision = await this.
|
|
530
|
+
decision = await this.evaluateWire(state, questions, signal, validate);
|
|
531
|
+
}
|
|
532
|
+
if (validate && this.style !== "systemone") {
|
|
533
|
+
const verr = validateDecision(questions, decision);
|
|
534
|
+
if (verr)
|
|
535
|
+
throw verr;
|
|
351
536
|
}
|
|
352
537
|
}
|
|
353
538
|
catch (e) {
|
|
354
|
-
this.emitError(start, e);
|
|
539
|
+
this.emitError(start, e, l1);
|
|
355
540
|
throw e;
|
|
356
541
|
}
|
|
357
542
|
this.emit({
|
|
@@ -361,6 +546,8 @@ export class Classifier {
|
|
|
361
546
|
ms: Date.now() - start,
|
|
362
547
|
promptTokens: decision.usage.inputTokens,
|
|
363
548
|
completionTokens: decision.usage.outputTokens,
|
|
549
|
+
...l1,
|
|
550
|
+
cached: decision.cached === true,
|
|
364
551
|
});
|
|
365
552
|
return decision;
|
|
366
553
|
}
|
|
@@ -421,15 +608,26 @@ export class Classifier {
|
|
|
421
608
|
}
|
|
422
609
|
// ---------------------------------------------------------------- backends
|
|
423
610
|
/**
|
|
424
|
-
* Assembles the request: the canonical model + questions, plus
|
|
425
|
-
*
|
|
426
|
-
* bodyTransform → marshal).
|
|
611
|
+
* Assembles the request for the backend's wire: the canonical model + questions, plus the state
|
|
612
|
+
* (VERBATIM unless it carried content parts) and, on systemone, `assets`; then the §8 Gap 1
|
|
613
|
+
* pipeline in §8 order (base → requestParams merge → bodyTransform → marshal).
|
|
427
614
|
*/
|
|
428
|
-
body(
|
|
615
|
+
body(p, questions) {
|
|
429
616
|
const qs = {};
|
|
430
617
|
for (const [key, q] of Object.entries(questions))
|
|
431
618
|
qs[key] = questionWire(q);
|
|
432
|
-
let body
|
|
619
|
+
let body;
|
|
620
|
+
if (this.preset.wire === "cloudflare") {
|
|
621
|
+
const input = { questions: qs, state: p.state };
|
|
622
|
+
if (p.images.length > 0)
|
|
623
|
+
input.images = p.images.map((im) => `data:image/${im.format};base64,${im.data}`);
|
|
624
|
+
body = { model: this.model, input };
|
|
625
|
+
}
|
|
626
|
+
else {
|
|
627
|
+
body = { model: this.model, questions: qs, state: p.state };
|
|
628
|
+
if (p.assets)
|
|
629
|
+
body.assets = p.assets;
|
|
630
|
+
}
|
|
433
631
|
// A requestParams key WINS on collision.
|
|
434
632
|
if (this.opts.requestParams)
|
|
435
633
|
body = { ...body, ...this.opts.requestParams };
|
|
@@ -447,8 +645,44 @@ export class Classifier {
|
|
|
447
645
|
throw new Error("classifier: static: no recorded decision for this request+state");
|
|
448
646
|
return decodeDecision(raw, this.model);
|
|
449
647
|
}
|
|
450
|
-
|
|
451
|
-
|
|
648
|
+
/** Lift images, apply the support table and the limits — all before any request. */
|
|
649
|
+
prepare(state) {
|
|
650
|
+
const { state: sent, images } = liftImages(state);
|
|
651
|
+
if (images.length > 0 && !this.supports.images)
|
|
652
|
+
throw new ClassifierUnsupported(this.backend, this.model, "images");
|
|
653
|
+
checkImages(images);
|
|
654
|
+
let assets;
|
|
655
|
+
if (images.length > 0) {
|
|
656
|
+
assets = {};
|
|
657
|
+
for (const im of images)
|
|
658
|
+
assets[im.ref] = { type: "image", format: im.format, data_base64: im.data };
|
|
659
|
+
}
|
|
660
|
+
return { state: sent, images, assets };
|
|
661
|
+
}
|
|
662
|
+
/** The systemone style on every backend: lift, check, cache, post, decode. */
|
|
663
|
+
async evaluateWire(state, questions, signal, validate) {
|
|
664
|
+
const p = this.prepare(state);
|
|
665
|
+
const cache = this.opts.cache;
|
|
666
|
+
let modelKey = "";
|
|
667
|
+
if (cache) {
|
|
668
|
+
modelKey = classifierModelCacheKey(this.baseUrl, this.model);
|
|
669
|
+
const served = await cache.get(modelKey);
|
|
670
|
+
if (typeof served === "string") {
|
|
671
|
+
const hit = await cache.get(classifierDecisionCacheKey(this.baseUrl, served, p.state, p.assets, questions));
|
|
672
|
+
if (typeof hit === "string") {
|
|
673
|
+
try {
|
|
674
|
+
const d = decodeDecision(JSON.parse(hit), served);
|
|
675
|
+
if (!validateDecision(questions, d))
|
|
676
|
+
return new Decision(d.model, d.answers, d.usage, d.calibrated, true);
|
|
677
|
+
}
|
|
678
|
+
catch {
|
|
679
|
+
// a corrupt entry is a miss
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
const extra = p.images.length > 0 && this.preset.imagesHeader ? { [CLASSIFIER_IMAGES_HEADER]: "true" } : {};
|
|
685
|
+
let text = await this.post(this.body(p, questions), signal, extra);
|
|
452
686
|
let parsed;
|
|
453
687
|
try {
|
|
454
688
|
parsed = JSON.parse(text);
|
|
@@ -456,7 +690,20 @@ export class Classifier {
|
|
|
456
690
|
catch (e) {
|
|
457
691
|
throw new Error(`classifier: response was not JSON: ${e.message}`);
|
|
458
692
|
}
|
|
459
|
-
|
|
693
|
+
if (this.preset.wire === "cloudflare") {
|
|
694
|
+
parsed = unwrapCloudflare(parsed);
|
|
695
|
+
text = canonicalJson(parsed);
|
|
696
|
+
}
|
|
697
|
+
const d = decodeDecision(parsed, this.model);
|
|
698
|
+
const verr = validateDecision(questions, d);
|
|
699
|
+
if (validate && verr)
|
|
700
|
+
throw verr;
|
|
701
|
+
if (cache && !verr) {
|
|
702
|
+
const served = d.model || this.model;
|
|
703
|
+
await cache.set(modelKey, served);
|
|
704
|
+
await cache.set(classifierDecisionCacheKey(this.baseUrl, served, p.state, p.assets, questions), text);
|
|
705
|
+
}
|
|
706
|
+
return d;
|
|
460
707
|
}
|
|
461
708
|
/**
|
|
462
709
|
* The one POST, with the §8 retry budget, reusing the client's `ErrorInfo`/`ErrorTier`
|
|
@@ -467,8 +714,13 @@ export class Classifier {
|
|
|
467
714
|
* is never echoed back (a gateway happily reflects a bad Authorization header into its own
|
|
468
715
|
* 401 text).
|
|
469
716
|
*/
|
|
470
|
-
async post(payload, signal) {
|
|
471
|
-
|
|
717
|
+
async post(payload, signal, extraHeaders = {}) {
|
|
718
|
+
// `endpoint` is what errors name: the CONFIGURED base, ${ENV} unexpanded, so an account id in
|
|
719
|
+
// a templated URL never reaches an error. `url` is what is dialled, expanded at call time.
|
|
720
|
+
let endpoint = this.baseUrl.replace(/\/+$/, "");
|
|
721
|
+
if (this.preset.wire !== "cloudflare")
|
|
722
|
+
endpoint += "/systemone";
|
|
723
|
+
const url = endpoint.includes("${") ? expandEnv(endpoint) : endpoint;
|
|
472
724
|
const retries = this.opts.retries ?? 2;
|
|
473
725
|
const base = this.opts.retryBaseMs ?? 500;
|
|
474
726
|
const classify = this.opts.onError ?? ((info) => (info.retryable ? "retry" : "fail"));
|
|
@@ -491,9 +743,9 @@ export class Classifier {
|
|
|
491
743
|
let res;
|
|
492
744
|
let thrown;
|
|
493
745
|
try {
|
|
494
|
-
res = await doFetch(
|
|
746
|
+
res = await doFetch(url, {
|
|
495
747
|
method: "POST",
|
|
496
|
-
headers: this.requestHeaders(),
|
|
748
|
+
headers: this.requestHeaders(extraHeaders),
|
|
497
749
|
body: payload,
|
|
498
750
|
signal: timer.signal,
|
|
499
751
|
});
|
|
@@ -516,7 +768,7 @@ export class Classifier {
|
|
|
516
768
|
lastErr = new Error(`classifier: POST ${endpoint}: HTTP ${res.status}${cause(res.status, text)}`);
|
|
517
769
|
}
|
|
518
770
|
else {
|
|
519
|
-
lastErr = new Error(`classifier: POST ${endpoint}: ${errText(thrown)}`);
|
|
771
|
+
lastErr = new Error(`classifier: POST ${endpoint}: ${errText(thrown).split(url).join(endpoint)}`);
|
|
520
772
|
}
|
|
521
773
|
const retryable = res ? res.status === 408 || isRetryableStatus(res.status, this.opts.retryableStatuses) : true;
|
|
522
774
|
if (attempt >= retries)
|
|
@@ -529,11 +781,14 @@ export class Classifier {
|
|
|
529
781
|
}
|
|
530
782
|
}
|
|
531
783
|
/** Credential and `${ENV_VAR}` headers resolve AT CALL TIME. Never logged, never returned. */
|
|
532
|
-
requestHeaders() {
|
|
784
|
+
requestHeaders(extra = {}) {
|
|
533
785
|
const headers = { "Content-Type": "application/json" };
|
|
534
786
|
const key = process.env[this.apiKeyEnv];
|
|
535
787
|
if (key)
|
|
536
788
|
headers["Authorization"] = `Bearer ${key}`;
|
|
789
|
+
// The image opt-in header goes BEFORE host headers, which win.
|
|
790
|
+
for (const [k, v] of Object.entries(extra))
|
|
791
|
+
headers[k] = v;
|
|
537
792
|
for (const [k, v] of Object.entries(this.opts.headers ?? {}))
|
|
538
793
|
headers[k] = expandEnv(v);
|
|
539
794
|
return headers;
|
|
@@ -577,7 +832,7 @@ export class Classifier {
|
|
|
577
832
|
emit(ev) {
|
|
578
833
|
this.opts.onMetric?.(ev);
|
|
579
834
|
}
|
|
580
|
-
emitError(start, e) {
|
|
835
|
+
emitError(start, e, l1) {
|
|
581
836
|
this.emit({
|
|
582
837
|
event: "classifier.evaluate",
|
|
583
838
|
model: this.model,
|
|
@@ -586,6 +841,8 @@ export class Classifier {
|
|
|
586
841
|
promptTokens: 0,
|
|
587
842
|
completionTokens: 0,
|
|
588
843
|
error: errText(e),
|
|
844
|
+
...l1,
|
|
845
|
+
cached: false,
|
|
589
846
|
});
|
|
590
847
|
}
|
|
591
848
|
}
|
|
@@ -669,3 +926,305 @@ function sleep(ms) {
|
|
|
669
926
|
setTimeout(resolve, ms);
|
|
670
927
|
});
|
|
671
928
|
}
|
|
929
|
+
// ---------------------------------------------------------------- backends, images, cache, validation
|
|
930
|
+
// SPEC.md §8B "Backends, images, cache and validation" (change add-classifier-backends, ADR 0038).
|
|
931
|
+
// Every default here is a PROPOSED default, owner to confirm (design.md PD-1..PD-15).
|
|
932
|
+
/** Ordered named questions -> the questions map. A duplicate name is an error naming it. */
|
|
933
|
+
function namedQuestions(qs) {
|
|
934
|
+
const out = {};
|
|
935
|
+
for (const { name, question } of qs) {
|
|
936
|
+
if (Object.prototype.hasOwnProperty.call(out, name))
|
|
937
|
+
throw new Error(`duplicate question name ${JSON.stringify(name)}`);
|
|
938
|
+
out[name] = question;
|
|
939
|
+
}
|
|
940
|
+
return out;
|
|
941
|
+
}
|
|
942
|
+
function sha256Hex(s) {
|
|
943
|
+
return createHash("sha256").update(s, "utf8").digest("hex");
|
|
944
|
+
}
|
|
945
|
+
function containsPart(v) {
|
|
946
|
+
if (Array.isArray(v))
|
|
947
|
+
return v.some(containsPart);
|
|
948
|
+
if (v !== null && typeof v === "object") {
|
|
949
|
+
if (isContentPart(v))
|
|
950
|
+
return true;
|
|
951
|
+
return Object.values(v).some(containsPart);
|
|
952
|
+
}
|
|
953
|
+
return false;
|
|
954
|
+
}
|
|
955
|
+
/**
|
|
956
|
+
* The §8B lifting walk. A state with no content part is returned VERBATIM (the same value), so a
|
|
957
|
+
* text-only state is byte-identical to before. Otherwise depth-first, object keys in ASCII order,
|
|
958
|
+
* arrays in order: an image part with inline data becomes a reference (its key when unused, else
|
|
959
|
+
* the smallest unused `image<n>`), a text part becomes its text, any other part is an error.
|
|
960
|
+
*/
|
|
961
|
+
function liftImages(state) {
|
|
962
|
+
if (!containsPart(state))
|
|
963
|
+
return { state, images: [] };
|
|
964
|
+
const used = new Set();
|
|
965
|
+
const images = [];
|
|
966
|
+
const nextRef = () => {
|
|
967
|
+
for (let n = 1;; n++)
|
|
968
|
+
if (!used.has(`image${n}`))
|
|
969
|
+
return `image${n}`;
|
|
970
|
+
};
|
|
971
|
+
const walk = (v, key) => {
|
|
972
|
+
if (Array.isArray(v))
|
|
973
|
+
return v.map((e) => walk(e, ""));
|
|
974
|
+
if (v === null || typeof v !== "object")
|
|
975
|
+
return v;
|
|
976
|
+
if (isContentPart(v)) {
|
|
977
|
+
const part = v;
|
|
978
|
+
if (part.type === "text")
|
|
979
|
+
return part.text;
|
|
980
|
+
if (part.type === "image") {
|
|
981
|
+
if (!part.data)
|
|
982
|
+
throw new Error("classifier: state: an image part needs inline data (a URL image cannot be sent)");
|
|
983
|
+
const ref = key === "" || used.has(key) ? nextRef() : key;
|
|
984
|
+
used.add(ref);
|
|
985
|
+
images.push({ ref, format: part.mimeType, data: part.data });
|
|
986
|
+
return ref;
|
|
987
|
+
}
|
|
988
|
+
throw new Error(`classifier: state: ${part.type} parts are not supported`);
|
|
989
|
+
}
|
|
990
|
+
const obj = v;
|
|
991
|
+
const out = {};
|
|
992
|
+
for (const k of Object.keys(obj).sort())
|
|
993
|
+
out[k] = walk(obj[k], k);
|
|
994
|
+
return out;
|
|
995
|
+
};
|
|
996
|
+
return { state: walk(state, ""), images };
|
|
997
|
+
}
|
|
998
|
+
const IMAGE_FORMATS = { "image/png": "png", "image/jpeg": "jpeg", "image/jpg": "jpeg" };
|
|
999
|
+
/** Format, per-image and per-request limits, in collection order; normalises `format`. */
|
|
1000
|
+
function checkImages(images) {
|
|
1001
|
+
let total = 0;
|
|
1002
|
+
for (const im of images) {
|
|
1003
|
+
const format = IMAGE_FORMATS[im.format];
|
|
1004
|
+
if (!format)
|
|
1005
|
+
throw new Error(`classifier: image "${im.ref}": type "${im.format}" is not supported (PNG or JPEG only)`);
|
|
1006
|
+
im.format = format;
|
|
1007
|
+
if (im.data.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(im.data)) {
|
|
1008
|
+
throw new Error(`classifier: image "${im.ref}": data is not base64`);
|
|
1009
|
+
}
|
|
1010
|
+
const raw = Buffer.from(im.data, "base64");
|
|
1011
|
+
if (raw.length > CLASSIFIER_IMAGE_MAX_BYTES) {
|
|
1012
|
+
throw new Error(`classifier: image "${im.ref}": ${raw.length} bytes, over 16 MiB`);
|
|
1013
|
+
}
|
|
1014
|
+
const size = imageSize(format, raw);
|
|
1015
|
+
if (!size)
|
|
1016
|
+
throw new Error(`classifier: image "${im.ref}": not a readable ${format.toUpperCase()}`);
|
|
1017
|
+
const [w, h] = size;
|
|
1018
|
+
const ok = (n) => n >= CLASSIFIER_IMAGE_MIN_SIDE && n <= CLASSIFIER_IMAGE_MAX_SIDE;
|
|
1019
|
+
if (!ok(w) || !ok(h)) {
|
|
1020
|
+
throw new Error(`classifier: image "${im.ref}": ${w}x${h}; each side must be ${CLASSIFIER_IMAGE_MIN_SIDE}..${CLASSIFIER_IMAGE_MAX_SIDE} px`);
|
|
1021
|
+
}
|
|
1022
|
+
total += raw.length;
|
|
1023
|
+
}
|
|
1024
|
+
if (total > CLASSIFIER_REQUEST_IMAGE_BYTES) {
|
|
1025
|
+
throw new Error(`classifier: image "${images[images.length - 1].ref}": request images total ${total} bytes, over 32 MiB`);
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
/** Width/height from a PNG IHDR or the first JPEG SOF marker; undefined when unreadable. */
|
|
1029
|
+
function imageSize(format, b) {
|
|
1030
|
+
const be16 = (i) => (b[i] << 8) | b[i + 1];
|
|
1031
|
+
const be32 = (i) => ((b[i] << 24) >>> 0) + (b[i + 1] << 16) + (b[i + 2] << 8) + b[i + 3];
|
|
1032
|
+
if (format === "png") {
|
|
1033
|
+
const sig = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
|
1034
|
+
if (b.length < 24 || sig.some((x, i) => b[i] !== x))
|
|
1035
|
+
return undefined;
|
|
1036
|
+
if (String.fromCharCode(b[12], b[13], b[14], b[15]) !== "IHDR")
|
|
1037
|
+
return undefined;
|
|
1038
|
+
return [be32(16), be32(20)];
|
|
1039
|
+
}
|
|
1040
|
+
if (b.length < 4 || b[0] !== 0xff || b[1] !== 0xd8)
|
|
1041
|
+
return undefined;
|
|
1042
|
+
let i = 2;
|
|
1043
|
+
while (i + 3 < b.length) {
|
|
1044
|
+
if (b[i] !== 0xff)
|
|
1045
|
+
return undefined;
|
|
1046
|
+
const m = b[i + 1];
|
|
1047
|
+
if (m === 0xff) {
|
|
1048
|
+
i++;
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
if (m >= 0xc0 && m <= 0xcf && m !== 0xc4 && m !== 0xc8 && m !== 0xcc) {
|
|
1052
|
+
if (i + 8 >= b.length)
|
|
1053
|
+
return undefined;
|
|
1054
|
+
return [be16(i + 7), be16(i + 5)];
|
|
1055
|
+
}
|
|
1056
|
+
i += 2 + be16(i + 2);
|
|
1057
|
+
}
|
|
1058
|
+
return undefined;
|
|
1059
|
+
}
|
|
1060
|
+
/** The System One reply inside a Cloudflare `/ai/run` envelope. Its error list is never echoed. */
|
|
1061
|
+
function unwrapCloudflare(env) {
|
|
1062
|
+
if (env === null || typeof env !== "object" || Array.isArray(env) || env.success !== true) {
|
|
1063
|
+
throw new Error("classifier: cloudflare: envelope not successful");
|
|
1064
|
+
}
|
|
1065
|
+
let result = env.result;
|
|
1066
|
+
if (result !== null && typeof result === "object" && !Array.isArray(result) && typeof result.state === "string") {
|
|
1067
|
+
if (result.state !== "Completed")
|
|
1068
|
+
throw new Error(`classifier: cloudflare: run state "${result.state}"`);
|
|
1069
|
+
result = result.result;
|
|
1070
|
+
}
|
|
1071
|
+
return result;
|
|
1072
|
+
}
|
|
1073
|
+
/** §8B model key: which model an endpoint + requested model (alias) served last. */
|
|
1074
|
+
export function classifierModelCacheKey(endpoint, model) {
|
|
1075
|
+
return sha256Hex(canonicalJson({ endpoint, kind: "model", model }));
|
|
1076
|
+
}
|
|
1077
|
+
/** §8B decision key, on the SERVED model. No assets ⇒ the `assets` member is omitted. */
|
|
1078
|
+
export function classifierDecisionCacheKey(endpoint, servedModel, state, assets, questions) {
|
|
1079
|
+
return sha256Hex(classifierDecisionCacheKeyBytes(endpoint, servedModel, state, assets, questions));
|
|
1080
|
+
}
|
|
1081
|
+
/** The canonical JSON a decision key hashes (exported for conformance tests). */
|
|
1082
|
+
export function classifierDecisionCacheKeyBytes(endpoint, servedModel, state, assets, questions) {
|
|
1083
|
+
const qs = {};
|
|
1084
|
+
for (const [k, q] of Object.entries(questions))
|
|
1085
|
+
qs[k] = questionWire(q);
|
|
1086
|
+
const m = { endpoint, kind: "decision", model: servedModel, questions: qs, state };
|
|
1087
|
+
if (assets && Object.keys(assets).length > 0)
|
|
1088
|
+
m.assets = assets;
|
|
1089
|
+
return canonicalJson(m);
|
|
1090
|
+
}
|
|
1091
|
+
/**
|
|
1092
|
+
* In-process TTL cache with a size cap (oldest-inserted out first). `ttlMs` 0 = no expiry; `max`
|
|
1093
|
+
* 0 = unbounded. toolnexus persists nothing itself: implement `ClassifierCache` to persist.
|
|
1094
|
+
*/
|
|
1095
|
+
export class MemoryCache {
|
|
1096
|
+
ttlMs;
|
|
1097
|
+
max;
|
|
1098
|
+
now;
|
|
1099
|
+
m = new Map();
|
|
1100
|
+
constructor(ttlMs = 0, max = 0, now = Date.now) {
|
|
1101
|
+
this.ttlMs = ttlMs;
|
|
1102
|
+
this.max = max;
|
|
1103
|
+
this.now = now;
|
|
1104
|
+
}
|
|
1105
|
+
get(key) {
|
|
1106
|
+
const e = this.m.get(key);
|
|
1107
|
+
if (!e)
|
|
1108
|
+
return undefined;
|
|
1109
|
+
if (this.ttlMs > 0 && this.now() - e.at > this.ttlMs) {
|
|
1110
|
+
this.m.delete(key);
|
|
1111
|
+
return undefined;
|
|
1112
|
+
}
|
|
1113
|
+
return e.v;
|
|
1114
|
+
}
|
|
1115
|
+
set(key, value) {
|
|
1116
|
+
this.m.delete(key); // a re-set is a fresh insertion
|
|
1117
|
+
this.m.set(key, { v: value, at: this.now() });
|
|
1118
|
+
if (this.max > 0)
|
|
1119
|
+
while (this.m.size > this.max)
|
|
1120
|
+
this.m.delete(this.m.keys().next().value);
|
|
1121
|
+
}
|
|
1122
|
+
get size() {
|
|
1123
|
+
return this.m.size;
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
/**
|
|
1127
|
+
* V1: fail closed on a reply that does not answer exactly the questions asked, within their
|
|
1128
|
+
* schema. First failure wins, in the §8B order. Returns the error, or undefined when valid.
|
|
1129
|
+
*/
|
|
1130
|
+
export function validateDecision(questions, decision) {
|
|
1131
|
+
const bad = (msg) => new Error(`classifier: invalid decision: ${msg}`);
|
|
1132
|
+
const answers = decision.answers ?? {};
|
|
1133
|
+
const has = (o, k) => Object.prototype.hasOwnProperty.call(o, k);
|
|
1134
|
+
const qKeys = Object.keys(questions).sort();
|
|
1135
|
+
for (const k of qKeys)
|
|
1136
|
+
if (!has(answers, k))
|
|
1137
|
+
return bad(`no answer "${k}"`);
|
|
1138
|
+
const extra = Object.keys(answers)
|
|
1139
|
+
.filter((k) => !has(questions, k))
|
|
1140
|
+
.sort();
|
|
1141
|
+
if (extra.length > 0)
|
|
1142
|
+
return bad(`unexpected answer "${extra[0]}"`);
|
|
1143
|
+
const in01 = (p) => typeof p === "number" && !Number.isNaN(p) && p >= 0 && p <= 1;
|
|
1144
|
+
const dist = (k, probs, offered) => {
|
|
1145
|
+
let sum = 0;
|
|
1146
|
+
const pk = Object.keys(probs ?? {}).sort();
|
|
1147
|
+
for (const o of pk) {
|
|
1148
|
+
if (!offered(o))
|
|
1149
|
+
return bad(`answer "${k}": probability for "${o}", which was not offered`);
|
|
1150
|
+
if (!in01(probs[o]))
|
|
1151
|
+
return bad(`answer "${k}": probability ${probs[o]} outside [0,1]`);
|
|
1152
|
+
sum += probs[o];
|
|
1153
|
+
}
|
|
1154
|
+
if (pk.length > 0 && Math.abs(sum - 1) > 0.02)
|
|
1155
|
+
return bad(`answer "${k}": probabilities sum to ${sum.toFixed(4)}`);
|
|
1156
|
+
return undefined;
|
|
1157
|
+
};
|
|
1158
|
+
for (const k of qKeys) {
|
|
1159
|
+
const raw = questions[k];
|
|
1160
|
+
const q = raw;
|
|
1161
|
+
const a = answers[k];
|
|
1162
|
+
const qType = raw.type;
|
|
1163
|
+
const aType = a?.type;
|
|
1164
|
+
if (aType !== qType)
|
|
1165
|
+
return bad(`answer "${k}" is a ${aType} answer, not ${qType}`);
|
|
1166
|
+
if (isRawQuestion(raw))
|
|
1167
|
+
continue;
|
|
1168
|
+
if (q.type === "noul") {
|
|
1169
|
+
if (!in01(a.noul))
|
|
1170
|
+
return bad(`answer "${k}": noul ${a.noul} outside [0,1]`);
|
|
1171
|
+
}
|
|
1172
|
+
else if (q.type === "choice") {
|
|
1173
|
+
const offered = (o) => has(q.criteria ?? {}, o);
|
|
1174
|
+
if (!offered(a.choice))
|
|
1175
|
+
return bad(`answer "${k}": choice "${a.choice}" was not offered`);
|
|
1176
|
+
const err = dist(k, a.probabilities, offered);
|
|
1177
|
+
if (err)
|
|
1178
|
+
return err;
|
|
1179
|
+
if (!in01(a.confidence))
|
|
1180
|
+
return bad(`answer "${k}": confidence ${a.confidence} outside [0,1]`);
|
|
1181
|
+
}
|
|
1182
|
+
else {
|
|
1183
|
+
const n = (q.criteria ?? []).length;
|
|
1184
|
+
if (typeof a.score !== "number" || Number.isNaN(a.score) || a.score < 0 || a.score > n - 1) {
|
|
1185
|
+
return bad(`answer "${k}": score ${a.score} outside [0,${n - 1}]`);
|
|
1186
|
+
}
|
|
1187
|
+
const offered = (o) => Array.from({ length: n }, (_, i) => String(i)).includes(o);
|
|
1188
|
+
const err = dist(k, a.probabilities, offered);
|
|
1189
|
+
if (err)
|
|
1190
|
+
return err;
|
|
1191
|
+
if (!in01(a.confidence))
|
|
1192
|
+
return bad(`answer "${k}": confidence ${a.confidence} outside [0,1]`);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
return undefined;
|
|
1196
|
+
}
|
|
1197
|
+
/** 0.8 / 0.2 / 0.6 (jevx's builtin profile). */
|
|
1198
|
+
export const DEFAULT_THRESHOLDS = Object.freeze({ yes: 0.8, no: 0.2, minConfidence: 0.6 });
|
|
1199
|
+
/**
|
|
1200
|
+
* The band of one answer under `Thresholds`: noul `p >= yes` ⇒ yes, `p <= no` ⇒ no, else
|
|
1201
|
+
* uncertain; choice/score `confidence >= minConfidence` ⇒ yes, else uncertain. No near-uniform
|
|
1202
|
+
* demotion (that is `Bands`' rule). A raw answer is uncertain.
|
|
1203
|
+
*/
|
|
1204
|
+
export function thresholdBand(answer, t = DEFAULT_THRESHOLDS) {
|
|
1205
|
+
const a = answer;
|
|
1206
|
+
if (a.type === "noul") {
|
|
1207
|
+
if (a.noul >= t.yes)
|
|
1208
|
+
return "yes";
|
|
1209
|
+
if (a.noul <= t.no)
|
|
1210
|
+
return "no";
|
|
1211
|
+
}
|
|
1212
|
+
else if ((a.type === "choice" || a.type === "score") && a.confidence >= t.minConfidence) {
|
|
1213
|
+
return "yes";
|
|
1214
|
+
}
|
|
1215
|
+
return "uncertain";
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Split a question set into chunks of `size`, keys in ASCII order. A helper, not automatic:
|
|
1219
|
+
* joint calibration across a split is unproven, so the host decides to split and merges.
|
|
1220
|
+
*/
|
|
1221
|
+
export function chunkQuestions(questions, size) {
|
|
1222
|
+
if (!(size >= 1))
|
|
1223
|
+
throw new Error(`classifier: chunk size must be at least 1, got ${size}`);
|
|
1224
|
+
const keys = Object.keys(questions).sort();
|
|
1225
|
+
const out = [];
|
|
1226
|
+
for (let i = 0; i < keys.length; i += size) {
|
|
1227
|
+
out.push(Object.fromEntries(keys.slice(i, i + size).map((k) => [k, questions[k]])));
|
|
1228
|
+
}
|
|
1229
|
+
return out;
|
|
1230
|
+
}
|