toolnexus 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/classifier.d.ts +189 -31
- package/dist/classifier.js +589 -30
- package/dist/client.d.ts +6 -0
- package/dist/judge.d.ts +6 -3
- package/dist/judge.js +13 -1
- package/dist/serve.js +7 -0
- package/package.json +2 -2
- package/src/classifier.ts +677 -46
- package/src/client.ts +6 -0
- package/src/judge.ts +19 -4
- package/src/serve.ts +7 -0
package/src/classifier.ts
CHANGED
|
@@ -9,9 +9,11 @@
|
|
|
9
9
|
* the declared schema. Numeric limits, permission checks and allowlists stay in code. Nothing
|
|
10
10
|
* here is a security control.
|
|
11
11
|
*/
|
|
12
|
+
import { createHash } from "node:crypto"
|
|
12
13
|
import type { Client } from "./client.js"
|
|
13
14
|
import { type ErrorInfo, type ErrorTier, type MetricEvent, redactErrorBody, capErrorBody } from "./client.js"
|
|
14
15
|
import { retryAfterMs, isRetryableStatus } from "./retry.js"
|
|
16
|
+
import { isContentPart, type ContentPart } from "./content.js"
|
|
15
17
|
|
|
16
18
|
// ---------------------------------------------------------------- constants
|
|
17
19
|
|
|
@@ -29,14 +31,98 @@ export const DEFAULT_BATCH_CONCURRENCY = 16
|
|
|
29
31
|
/**
|
|
30
32
|
* A named BACKEND: `baseUrl`, `model` and `apiKeyEnv` are only jointly valid, so they travel as
|
|
31
33
|
* a UNIT (ADR 0027 D1). `jev-latest` is TypeSafe's spelling; the gateway spells the same model
|
|
32
|
-
* `typesafe/jev-1.
|
|
34
|
+
* `typesafe/jev-1.14`, and mixing the two is issue #91. `custom` is a self-hosted System One
|
|
35
|
+
* endpoint and requires `baseUrl` + `supports` (SPEC §8B "Backends, images, cache and validation").
|
|
33
36
|
*/
|
|
34
|
-
export type ClassifierBackend = "typesafe" | "openrouter"
|
|
37
|
+
export type ClassifierBackend = "typesafe" | "openrouter" | "cloudflare" | "vercel" | "custom"
|
|
35
38
|
|
|
36
|
-
/**
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
/** What a backend+model can take. DATA: a fail-fast for what the table says, never a claim about
|
|
40
|
+
* the vendor. Override per classifier with `ClassifierOptions.supports`. */
|
|
41
|
+
export interface ClassifierSupports {
|
|
42
|
+
images: boolean
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** One row of the backend table. */
|
|
46
|
+
export interface ClassifierBackendPreset {
|
|
47
|
+
/** `""` on `custom`, which requires `baseUrl`. May carry `${NAME}`, expanded at call time. */
|
|
48
|
+
baseUrl: string
|
|
49
|
+
model: string
|
|
50
|
+
apiKeyEnv: string
|
|
51
|
+
/** `systemone`: POST {baseUrl}/systemone. `cloudflare`: POST {baseUrl}, `/ai/run` envelope. */
|
|
52
|
+
wire: "systemone" | "cloudflare"
|
|
53
|
+
/** Send `TypeSafe-Danger-Zone-Images: true` when a request carries images. */
|
|
54
|
+
imagesHeader: boolean
|
|
55
|
+
/** The default support for a model on this backend; absent on `custom`. */
|
|
56
|
+
supports?: (model: string) => ClassifierSupports
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** TypeSafe's required opt-in header for image ("danger zone") input. */
|
|
60
|
+
export const CLASSIFIER_IMAGES_HEADER = "TypeSafe-Danger-Zone-Images"
|
|
61
|
+
/** Image limits (docs.typesafe.ai/images): checked client-side, never resized. */
|
|
62
|
+
export const CLASSIFIER_IMAGE_MIN_SIDE = 280
|
|
63
|
+
export const CLASSIFIER_IMAGE_MAX_SIDE = 1568
|
|
64
|
+
export const CLASSIFIER_IMAGE_MAX_BYTES = 16 * 1024 * 1024
|
|
65
|
+
export const CLASSIFIER_REQUEST_IMAGE_BYTES = 32 * 1024 * 1024
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The backend table (PROPOSED defaults, owner to confirm — design.md PD-1..PD-4). Vercel and
|
|
69
|
+
* Cloudflare-Jev images stay false until a live probe proves them.
|
|
70
|
+
*/
|
|
71
|
+
export const CLASSIFIER_BACKENDS: Readonly<Record<ClassifierBackend, Readonly<ClassifierBackendPreset>>> = Object.freeze({
|
|
72
|
+
typesafe: {
|
|
73
|
+
baseUrl: DEFAULT_CLASSIFIER_BASE_URL,
|
|
74
|
+
model: DEFAULT_CLASSIFIER_MODEL,
|
|
75
|
+
apiKeyEnv: DEFAULT_CLASSIFIER_API_KEY_ENV,
|
|
76
|
+
wire: "systemone",
|
|
77
|
+
imagesHeader: true,
|
|
78
|
+
supports: () => ({ images: true }),
|
|
79
|
+
},
|
|
80
|
+
openrouter: {
|
|
81
|
+
baseUrl: "https://openrouter.ai/api/v1",
|
|
82
|
+
model: "typesafe/jev-1.14",
|
|
83
|
+
apiKeyEnv: "OPENROUTER_API_KEY",
|
|
84
|
+
wire: "systemone",
|
|
85
|
+
imagesHeader: true,
|
|
86
|
+
supports: () => ({ images: true }),
|
|
87
|
+
},
|
|
88
|
+
cloudflare: {
|
|
89
|
+
baseUrl: "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/ai/run",
|
|
90
|
+
model: "typesafe/jev",
|
|
91
|
+
apiKeyEnv: "CLOUDFLARE_JEV_API_KEY",
|
|
92
|
+
wire: "cloudflare",
|
|
93
|
+
imagesHeader: false,
|
|
94
|
+
supports: (model: string) => ({ images: model.includes("clef") }),
|
|
95
|
+
},
|
|
96
|
+
vercel: {
|
|
97
|
+
baseUrl: "https://ai-gateway.vercel.sh/typesafe/v1",
|
|
98
|
+
model: "typesafe-ai/jev",
|
|
99
|
+
apiKeyEnv: "AI_GATEWAY_API_KEY",
|
|
100
|
+
wire: "systemone",
|
|
101
|
+
imagesHeader: true,
|
|
102
|
+
supports: () => ({ images: false }),
|
|
103
|
+
},
|
|
104
|
+
custom: {
|
|
105
|
+
baseUrl: "",
|
|
106
|
+
model: DEFAULT_CLASSIFIER_MODEL,
|
|
107
|
+
apiKeyEnv: DEFAULT_CLASSIFIER_API_KEY_ENV,
|
|
108
|
+
wire: "systemone",
|
|
109
|
+
imagesHeader: true,
|
|
110
|
+
},
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Raised BEFORE any network call when the backend table says the backend cannot take what the
|
|
115
|
+
* request carries. Message, exact: `classifier: <backend> (<model>) does not support <feature>`.
|
|
116
|
+
*/
|
|
117
|
+
export class ClassifierUnsupported extends Error {
|
|
118
|
+
constructor(
|
|
119
|
+
readonly backend: string,
|
|
120
|
+
readonly model: string,
|
|
121
|
+
readonly feature: string,
|
|
122
|
+
) {
|
|
123
|
+
super(`classifier: ${backend} (${model}) does not support ${feature}`)
|
|
124
|
+
this.name = "ClassifierUnsupported"
|
|
125
|
+
}
|
|
40
126
|
}
|
|
41
127
|
|
|
42
128
|
/** Client-side cap on a choice's named options (§8B). */
|
|
@@ -97,9 +183,36 @@ export interface ScoreQuestion {
|
|
|
97
183
|
criteria: string[]
|
|
98
184
|
}
|
|
99
185
|
|
|
100
|
-
/** The
|
|
186
|
+
/** The modelled question types, discriminated by `type`. */
|
|
101
187
|
export type Question = NoulQuestion | ChoiceQuestion | ScoreQuestion
|
|
102
188
|
|
|
189
|
+
/**
|
|
190
|
+
* The OPEN end of the question union (§8B): a System One type the library does not model (e.g.
|
|
191
|
+
* Jev 1.14 `box`), sent as written with no client-side limits. `type` must be non-empty.
|
|
192
|
+
*/
|
|
193
|
+
export interface RawQuestion {
|
|
194
|
+
type: string
|
|
195
|
+
instructions: string
|
|
196
|
+
/** Sent verbatim when present, absent otherwise. */
|
|
197
|
+
criteria?: unknown
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Every question `evaluate` accepts: the modelled three plus a raw one. */
|
|
201
|
+
export type AnyQuestion = Question | RawQuestion
|
|
202
|
+
|
|
203
|
+
const MODELLED_TYPES = new Set(["noul", "choice", "score"])
|
|
204
|
+
|
|
205
|
+
/** True when `q` is a raw question (its type is not noul/choice/score). Not a type guard: every
|
|
206
|
+
* modelled question is structurally a `RawQuestion` too. */
|
|
207
|
+
export function isRawQuestion(q: AnyQuestion): boolean {
|
|
208
|
+
return !MODELLED_TYPES.has(q?.type as string)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** A raw question, sent as written. */
|
|
212
|
+
export function raw(type: string, instructions: string, criteria?: unknown): RawQuestion {
|
|
213
|
+
return criteria === undefined ? { type, instructions } : { type, instructions, criteria }
|
|
214
|
+
}
|
|
215
|
+
|
|
103
216
|
/** A noul question. Pass `criteria` only when you mean it — absent ≠ empty on the wire. */
|
|
104
217
|
export function noul(instructions: string, criteria?: NoulCriteria): NoulQuestion {
|
|
105
218
|
return criteria === undefined ? { type: "noul", instructions } : { type: "noul", instructions, criteria }
|
|
@@ -125,7 +238,14 @@ export function choiceOver(instructions: string, items: Record<string, string>):
|
|
|
125
238
|
}
|
|
126
239
|
|
|
127
240
|
/** The wire form of one question — the object itself, minus an absent `criteria`. */
|
|
128
|
-
export function questionWire(
|
|
241
|
+
export function questionWire(question: AnyQuestion): Record<string, unknown> {
|
|
242
|
+
if (isRawQuestion(question)) {
|
|
243
|
+
const r = question as RawQuestion
|
|
244
|
+
return r.criteria === undefined
|
|
245
|
+
? { type: r.type, instructions: r.instructions }
|
|
246
|
+
: { type: r.type, instructions: r.instructions, criteria: r.criteria }
|
|
247
|
+
}
|
|
248
|
+
const q = question as Question
|
|
129
249
|
switch (q.type) {
|
|
130
250
|
case "noul":
|
|
131
251
|
return q.criteria === undefined
|
|
@@ -139,7 +259,12 @@ export function questionWire(q: Question): Record<string, unknown> {
|
|
|
139
259
|
}
|
|
140
260
|
|
|
141
261
|
/** Client-side limits, enforced BEFORE the request. The error names the key and the limit. */
|
|
142
|
-
function validateQuestion(key: string,
|
|
262
|
+
function validateQuestion(key: string, question: AnyQuestion): void {
|
|
263
|
+
const q = question as Question
|
|
264
|
+
if (isRawQuestion(question)) {
|
|
265
|
+
if (!question?.type) throw new Error(`classifier: question "${key}": a raw question needs a type`)
|
|
266
|
+
return
|
|
267
|
+
}
|
|
143
268
|
if (q.type === "choice") {
|
|
144
269
|
const n = Object.keys(q.criteria ?? {}).length
|
|
145
270
|
if (n < 1 || n > MAX_CHOICE_OPTIONS) {
|
|
@@ -192,7 +317,7 @@ export function canonicalJson(value: unknown): string {
|
|
|
192
317
|
* ways across our own seven runtimes). Do not re-widen this: a caller who needs their state
|
|
193
318
|
* pinned canonicalises it themselves before handing it over.
|
|
194
319
|
*/
|
|
195
|
-
export function canonicalRequest(model: string, questions: Record<string,
|
|
320
|
+
export function canonicalRequest(model: string, questions: Record<string, AnyQuestion>): string {
|
|
196
321
|
const qs: Record<string, unknown> = {}
|
|
197
322
|
for (const [key, q] of Object.entries(questions)) {
|
|
198
323
|
if (!q) throw new Error(`classifier: question "${key}" is missing`)
|
|
@@ -241,6 +366,22 @@ export interface ScoreAnswer {
|
|
|
241
366
|
*/
|
|
242
367
|
export type DecisionAnswer = NoulAnswer | ChoiceAnswer | ScoreAnswer
|
|
243
368
|
|
|
369
|
+
/**
|
|
370
|
+
* An answer of a type the library does not model (the open union, §8B). It may appear in
|
|
371
|
+
* `Decision.answers` at runtime (read it with `decision.raw(key)` or `isRawAnswer`); `value` is the
|
|
372
|
+
* whole answer object as decoded, `type` included. It is kept out of `DecisionAnswer` so existing
|
|
373
|
+
* `switch (a.type)` narrowing keeps compiling.
|
|
374
|
+
*/
|
|
375
|
+
export interface RawAnswer {
|
|
376
|
+
type: string
|
|
377
|
+
value: Record<string, unknown>
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** True when an answer is a raw (unmodelled-type) answer. */
|
|
381
|
+
export function isRawAnswer(a: unknown): a is RawAnswer {
|
|
382
|
+
return !!a && typeof a === "object" && !MODELLED_TYPES.has((a as any).type) && "value" in (a as any)
|
|
383
|
+
}
|
|
384
|
+
|
|
244
385
|
/** The wire's usage block. `cost` is absent on some backends. */
|
|
245
386
|
export interface ClassifierUsage {
|
|
246
387
|
inputTokens: number
|
|
@@ -271,8 +412,18 @@ export class Decision {
|
|
|
271
412
|
* BACKEND DOES NOT TRANSFER TO ANOTHER.
|
|
272
413
|
*/
|
|
273
414
|
readonly calibrated: boolean,
|
|
415
|
+
/** True when the decision came from `ClassifierOptions.cache` and no request was sent. */
|
|
416
|
+
readonly cached: boolean = false,
|
|
274
417
|
) {}
|
|
275
418
|
|
|
419
|
+
/** Typed read of a raw (unmodelled-type) answer. Throws on an absent key or a modelled type. */
|
|
420
|
+
raw(key: string): RawAnswer {
|
|
421
|
+
const a: unknown = this.answers[key]
|
|
422
|
+
if (a === undefined) throw new Error(`classifier: no answer "${key}" in this decision`)
|
|
423
|
+
if (!isRawAnswer(a)) throw new Error(`classifier: answer "${key}" is a ${(a as any).type} answer, not raw`)
|
|
424
|
+
return a
|
|
425
|
+
}
|
|
426
|
+
|
|
276
427
|
/** Typed read. Throws (never returns undefined) on a wrong-type or absent key. */
|
|
277
428
|
noul(key: string): NoulAnswer {
|
|
278
429
|
return this.typed(key, "noul")
|
|
@@ -345,7 +496,9 @@ export function decodeDecision(raw: unknown, fallbackModel: string): Decision {
|
|
|
345
496
|
}
|
|
346
497
|
break
|
|
347
498
|
default:
|
|
348
|
-
|
|
499
|
+
// The union is OPEN (§8B): an unmodelled type (e.g. Jev 1.14 "box") decodes to a raw
|
|
500
|
+
// answer instead of failing the whole decision.
|
|
501
|
+
;(answers as Record<string, unknown>)[key] = { type: String(a.type ?? ""), value: { ...a } } satisfies RawAnswer
|
|
349
502
|
}
|
|
350
503
|
}
|
|
351
504
|
const usage: ClassifierUsage = {
|
|
@@ -376,7 +529,12 @@ function strings(v: unknown): Record<string, string> {
|
|
|
376
529
|
|
|
377
530
|
// ---------------------------------------------------------------- options
|
|
378
531
|
|
|
379
|
-
/**
|
|
532
|
+
/**
|
|
533
|
+
* `"systemone" | "llm" | "custom" | "static"` — the same slot as the client's `style`.
|
|
534
|
+
*
|
|
535
|
+
* `"llm"` is DEPRECATED (ADR 0038 D3: System One is the single product wire). It keeps working
|
|
536
|
+
* byte-identically, as an alias, for at least one minor release; there is no runtime warning.
|
|
537
|
+
*/
|
|
380
538
|
export type ClassifierStyle = "systemone" | "llm" | "custom" | "static"
|
|
381
539
|
|
|
382
540
|
/**
|
|
@@ -387,7 +545,7 @@ export type ClassifierStyle = "systemone" | "llm" | "custom" | "static"
|
|
|
387
545
|
*/
|
|
388
546
|
export interface RecordedDecision {
|
|
389
547
|
state: unknown
|
|
390
|
-
questions: Record<string,
|
|
548
|
+
questions: Record<string, AnyQuestion>
|
|
391
549
|
/** The recorded backend response body, verbatim. */
|
|
392
550
|
response: unknown
|
|
393
551
|
}
|
|
@@ -395,7 +553,7 @@ export interface RecordedDecision {
|
|
|
395
553
|
/** The host's own backend (`style: "custom"`). Every wire option is ignored. */
|
|
396
554
|
export type EvaluateFn = (
|
|
397
555
|
state: unknown,
|
|
398
|
-
questions: Record<string,
|
|
556
|
+
questions: Record<string, AnyQuestion>,
|
|
399
557
|
signal?: AbortSignal,
|
|
400
558
|
) => Promise<Decision> | Decision
|
|
401
559
|
|
|
@@ -406,7 +564,7 @@ export type EvaluateFn = (
|
|
|
406
564
|
export interface ClassifierOptions {
|
|
407
565
|
/** Default `"systemone"`. */
|
|
408
566
|
style?: ClassifierStyle
|
|
409
|
-
/** A named `
|
|
567
|
+
/** A named backend row (`"typesafe"` | `"openrouter"` | `"cloudflare"` | `"vercel"` | `"custom"`), set as a
|
|
410
568
|
* UNIT. Default `"typesafe"`. The three individual options still win where given, for a
|
|
411
569
|
* self-hosted origin — but a known-bad mixture is refused at construction, not 700 ms later
|
|
412
570
|
* as an "Unknown model" 400. */
|
|
@@ -451,12 +609,37 @@ export interface ClassifierOptions {
|
|
|
451
609
|
/** Emits `classifier.evaluate` events into the SAME §8 sink, and carries the
|
|
452
610
|
* degenerate-criteria warning as `classifier.warning`. */
|
|
453
611
|
onMetric?: (ev: MetricEvent) => void
|
|
454
|
-
/** `style: "llm"` only — the §8 Client to emulate over.
|
|
612
|
+
/** `style: "llm"` only — the §8 Client to emulate over.
|
|
613
|
+
* @deprecated `style: "llm"` is deprecated (System One is the single wire); it still works. */
|
|
455
614
|
client?: Client
|
|
456
615
|
/** `style: "custom"` only — the host's own function. */
|
|
457
616
|
evaluate?: EvaluateFn
|
|
458
617
|
/** `style: "static"` only — the recorded corpus. */
|
|
459
618
|
decisions?: RecordedDecision[]
|
|
619
|
+
/** Overrides the backend table's support row for this classifier. Required with `backend: "custom"`. */
|
|
620
|
+
supports?: ClassifierSupports
|
|
621
|
+
/** Opt-in answer cache (§8B). Absent ⇒ no caching. Consulted by style `systemone` only; only
|
|
622
|
+
* replies that pass `validateDecision` are stored. Methods may return values or Promises. */
|
|
623
|
+
cache?: ClassifierCache
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/** An injected answer cache. Values are opaque strings: the served model under a model key, the
|
|
627
|
+
* System One reply body under a decision key. */
|
|
628
|
+
export interface ClassifierCache {
|
|
629
|
+
get(key: string): string | undefined | null | Promise<string | undefined | null>
|
|
630
|
+
set(key: string, value: string): void | Promise<void>
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/** One state's outcome from `states(...).evaluate`: exactly one of `decision` / `error` is set. */
|
|
634
|
+
export interface BatchResult {
|
|
635
|
+
decision?: Decision
|
|
636
|
+
error?: Error
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/** A named question (`judge.noul(...)` etc. build these). */
|
|
640
|
+
export interface NamedClassifierQuestion {
|
|
641
|
+
name: string
|
|
642
|
+
question: AnyQuestion
|
|
460
643
|
}
|
|
461
644
|
|
|
462
645
|
// ---------------------------------------------------------------- classifier
|
|
@@ -468,6 +651,9 @@ export class Classifier {
|
|
|
468
651
|
private readonly model: string
|
|
469
652
|
private readonly apiKeyEnv: string
|
|
470
653
|
private readonly timeoutMs: number
|
|
654
|
+
private readonly backend: ClassifierBackend
|
|
655
|
+
private readonly preset: Readonly<ClassifierBackendPreset>
|
|
656
|
+
private readonly supports: ClassifierSupports
|
|
471
657
|
private readonly corpus = new Map<string, unknown>()
|
|
472
658
|
/** The once-per-question-key set for the degenerate warning, so a per-turn judge does not
|
|
473
659
|
* flood the sink. */
|
|
@@ -475,13 +661,24 @@ export class Classifier {
|
|
|
475
661
|
|
|
476
662
|
constructor(private readonly opts: ClassifierOptions) {
|
|
477
663
|
this.style = opts.style ?? "systemone"
|
|
478
|
-
|
|
479
|
-
|
|
664
|
+
this.backend = opts.backend ?? "typesafe"
|
|
665
|
+
if (!Object.prototype.hasOwnProperty.call(CLASSIFIER_BACKENDS, this.backend)) {
|
|
666
|
+
throw new Error(
|
|
667
|
+
`classifier: unknown backend "${this.backend}" (want "typesafe", "openrouter", "cloudflare", "vercel" or "custom")`,
|
|
668
|
+
)
|
|
669
|
+
}
|
|
670
|
+
const preset = CLASSIFIER_BACKENDS[this.backend]
|
|
671
|
+
this.preset = preset
|
|
672
|
+
if (this.backend === "custom" && this.style === "systemone" && (!opts.baseUrl || !opts.supports)) {
|
|
673
|
+
throw new Error('classifier: backend "custom" requires baseUrl and supports')
|
|
674
|
+
}
|
|
675
|
+
this.baseUrl = opts.baseUrl || preset.baseUrl
|
|
480
676
|
this.model = opts.model ?? preset.model
|
|
481
677
|
this.apiKeyEnv = opts.apiKeyEnv ?? preset.apiKeyEnv
|
|
482
678
|
const mismatch = backendMismatch(this.baseUrl, this.model)
|
|
483
679
|
if (mismatch) throw new Error(mismatch)
|
|
484
680
|
this.timeoutMs = opts.timeoutMs ?? DEFAULT_CLASSIFIER_TIMEOUT_MS
|
|
681
|
+
this.supports = opts.supports ? { ...opts.supports } : (preset.supports?.(this.model) ?? { images: false })
|
|
485
682
|
switch (this.style) {
|
|
486
683
|
case "systemone":
|
|
487
684
|
break
|
|
@@ -511,12 +708,67 @@ export class Classifier {
|
|
|
511
708
|
* The whole contract: a state plus typed questions in, a Decision out. Questions are
|
|
512
709
|
* INDEPENDENT — one answer is never context for another.
|
|
513
710
|
*/
|
|
514
|
-
async evaluate(state: unknown, questions: Record<string,
|
|
711
|
+
async evaluate(state: unknown, questions: Record<string, AnyQuestion>, signal?: AbortSignal): Promise<Decision> {
|
|
712
|
+
return this.run(state, questions, signal, false)
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
/**
|
|
716
|
+
* Binds one state (a VALUE, not a session: nothing is sent until `evaluate`). Its `evaluate`
|
|
717
|
+
* runs the same path as `evaluate` PLUS fail-closed validation (`validateDecision`).
|
|
718
|
+
*/
|
|
719
|
+
state(value: unknown, opts: { signal?: AbortSignal } = {}): { evaluate(...questions: NamedClassifierQuestion[]): Promise<Decision> } {
|
|
720
|
+
return {
|
|
721
|
+
evaluate: async (...qs: NamedClassifierQuestion[]) => this.run(value, namedQuestions(qs), opts.signal, true),
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* Binds a batch of states. Its `evaluate` runs `state(v).evaluate` per state, at most 16 in
|
|
727
|
+
* flight, and returns one `{decision | error}` per state IN ORDER. One state's failure never
|
|
728
|
+
* fails the batch; only a question-set error (duplicate name) throws, before any request.
|
|
729
|
+
*/
|
|
730
|
+
states(...values: unknown[]): { evaluate(...questions: NamedClassifierQuestion[]): Promise<BatchResult[]> } {
|
|
731
|
+
return {
|
|
732
|
+
evaluate: async (...qs: NamedClassifierQuestion[]) => {
|
|
733
|
+
const questions = namedQuestions(qs)
|
|
734
|
+
const out: BatchResult[] = new Array(values.length)
|
|
735
|
+
let next = 0
|
|
736
|
+
const worker = async () => {
|
|
737
|
+
while (next < values.length) {
|
|
738
|
+
const i = next++
|
|
739
|
+
try {
|
|
740
|
+
out[i] = { decision: await this.run(values[i], questions, undefined, true) }
|
|
741
|
+
} catch (e) {
|
|
742
|
+
out[i] = { error: e instanceof Error ? e : new Error(String(e)) }
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
await Promise.all(Array.from({ length: Math.min(DEFAULT_BATCH_CONCURRENCY, values.length) }, worker))
|
|
747
|
+
return out
|
|
748
|
+
},
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/** The one path. `validate` = fail closed on `validateDecision` (the value builders). */
|
|
753
|
+
private async run(
|
|
754
|
+
state: unknown,
|
|
755
|
+
questions: Record<string, AnyQuestion>,
|
|
756
|
+
signal: AbortSignal | undefined,
|
|
757
|
+
validate: boolean,
|
|
758
|
+
): Promise<Decision> {
|
|
515
759
|
const start = Date.now()
|
|
516
760
|
const keys = Object.keys(questions ?? {}).sort()
|
|
761
|
+
// L1: count + hash of the canonical request on every evaluate event ("" when unencodable).
|
|
762
|
+
let qHash = ""
|
|
763
|
+
try {
|
|
764
|
+
qHash = sha256Hex(canonicalRequest(this.model, questions ?? {}))
|
|
765
|
+
} catch {
|
|
766
|
+
qHash = ""
|
|
767
|
+
}
|
|
768
|
+
const l1 = { questions: keys.length, questionsHash: qHash }
|
|
517
769
|
if (keys.length === 0) {
|
|
518
770
|
const err = new Error("classifier: no questions to evaluate")
|
|
519
|
-
this.emitError(start, err)
|
|
771
|
+
this.emitError(start, err, l1)
|
|
520
772
|
throw err
|
|
521
773
|
}
|
|
522
774
|
try {
|
|
@@ -529,7 +781,7 @@ export class Classifier {
|
|
|
529
781
|
validateQuestion(key, q)
|
|
530
782
|
}
|
|
531
783
|
} catch (e) {
|
|
532
|
-
this.emitError(start, e)
|
|
784
|
+
this.emitError(start, e, l1)
|
|
533
785
|
throw e
|
|
534
786
|
}
|
|
535
787
|
// Detection, never repair (ADR 0020/0021): the request goes out BYTE-UNCHANGED and the
|
|
@@ -549,10 +801,14 @@ export class Classifier {
|
|
|
549
801
|
decision = await this.evaluateLLM(state, questions, signal)
|
|
550
802
|
break
|
|
551
803
|
default:
|
|
552
|
-
decision = await this.
|
|
804
|
+
decision = await this.evaluateWire(state, questions, signal, validate)
|
|
805
|
+
}
|
|
806
|
+
if (validate && this.style !== "systemone") {
|
|
807
|
+
const verr = validateDecision(questions, decision)
|
|
808
|
+
if (verr) throw verr
|
|
553
809
|
}
|
|
554
810
|
} catch (e) {
|
|
555
|
-
this.emitError(start, e)
|
|
811
|
+
this.emitError(start, e, l1)
|
|
556
812
|
throw e
|
|
557
813
|
}
|
|
558
814
|
this.emit({
|
|
@@ -562,6 +818,8 @@ export class Classifier {
|
|
|
562
818
|
ms: Date.now() - start,
|
|
563
819
|
promptTokens: decision.usage.inputTokens,
|
|
564
820
|
completionTokens: decision.usage.outputTokens,
|
|
821
|
+
...l1,
|
|
822
|
+
cached: decision.cached === true,
|
|
565
823
|
})
|
|
566
824
|
return decision
|
|
567
825
|
}
|
|
@@ -574,7 +832,7 @@ export class Classifier {
|
|
|
574
832
|
*/
|
|
575
833
|
async evaluateBatch(
|
|
576
834
|
states: readonly unknown[],
|
|
577
|
-
questions: Record<string,
|
|
835
|
+
questions: Record<string, AnyQuestion>,
|
|
578
836
|
opts: { signal?: AbortSignal; concurrency?: number } = {},
|
|
579
837
|
): Promise<Decision[]> {
|
|
580
838
|
if (!states || states.length === 0) throw new Error("classifier: no states to evaluate")
|
|
@@ -606,9 +864,9 @@ export class Classifier {
|
|
|
606
864
|
* nothing about the request. Repairing would invent option descriptions the caller did not
|
|
607
865
|
* write, and the library has no way to know what the options mean.
|
|
608
866
|
*/
|
|
609
|
-
private reportDegenerate(keys: string[], questions: Record<string,
|
|
867
|
+
private reportDegenerate(keys: string[], questions: Record<string, AnyQuestion>): void {
|
|
610
868
|
for (const key of keys) {
|
|
611
|
-
const q = questions[key]
|
|
869
|
+
const q = questions[key] as Question
|
|
612
870
|
if (q.type !== "choice") continue
|
|
613
871
|
const reason = degenerateCriteria(q.criteria)
|
|
614
872
|
if (!reason || this.warned.has(key)) continue
|
|
@@ -627,14 +885,22 @@ export class Classifier {
|
|
|
627
885
|
// ---------------------------------------------------------------- backends
|
|
628
886
|
|
|
629
887
|
/**
|
|
630
|
-
* Assembles the request: the canonical model + questions, plus
|
|
631
|
-
*
|
|
632
|
-
* bodyTransform → marshal).
|
|
888
|
+
* Assembles the request for the backend's wire: the canonical model + questions, plus the state
|
|
889
|
+
* (VERBATIM unless it carried content parts) and, on systemone, `assets`; then the §8 Gap 1
|
|
890
|
+
* pipeline in §8 order (base → requestParams merge → bodyTransform → marshal).
|
|
633
891
|
*/
|
|
634
|
-
private body(
|
|
892
|
+
private body(p: PreparedRequest, questions: Record<string, AnyQuestion>): string {
|
|
635
893
|
const qs: Record<string, unknown> = {}
|
|
636
894
|
for (const [key, q] of Object.entries(questions)) qs[key] = questionWire(q)
|
|
637
|
-
let body: Record<string, any>
|
|
895
|
+
let body: Record<string, any>
|
|
896
|
+
if (this.preset.wire === "cloudflare") {
|
|
897
|
+
const input: Record<string, unknown> = { questions: qs, state: p.state }
|
|
898
|
+
if (p.images.length > 0) input.images = p.images.map((im) => `data:image/${im.format};base64,${im.data}`)
|
|
899
|
+
body = { model: this.model, input }
|
|
900
|
+
} else {
|
|
901
|
+
body = { model: this.model, questions: qs, state: p.state }
|
|
902
|
+
if (p.assets) body.assets = p.assets
|
|
903
|
+
}
|
|
638
904
|
// A requestParams key WINS on collision.
|
|
639
905
|
if (this.opts.requestParams) body = { ...body, ...this.opts.requestParams }
|
|
640
906
|
if (this.opts.bodyTransform) body = this.opts.bodyTransform(body) || body
|
|
@@ -642,29 +908,76 @@ export class Classifier {
|
|
|
642
908
|
}
|
|
643
909
|
|
|
644
910
|
/** Identifies a recorded decision by the canonical request AND the state. */
|
|
645
|
-
private staticKey(state: unknown, questions: Record<string,
|
|
911
|
+
private staticKey(state: unknown, questions: Record<string, AnyQuestion>): string {
|
|
646
912
|
return canonicalRequest(this.model, questions) + "\u0000" + canonicalJson(state)
|
|
647
913
|
}
|
|
648
914
|
|
|
649
|
-
private evaluateStatic(state: unknown, questions: Record<string,
|
|
915
|
+
private evaluateStatic(state: unknown, questions: Record<string, AnyQuestion>): Decision {
|
|
650
916
|
const raw = this.corpus.get(this.staticKey(state, questions))
|
|
651
917
|
if (raw === undefined) throw new Error("classifier: static: no recorded decision for this request+state")
|
|
652
918
|
return decodeDecision(raw, this.model)
|
|
653
919
|
}
|
|
654
920
|
|
|
655
|
-
|
|
921
|
+
/** Lift images, apply the support table and the limits — all before any request. */
|
|
922
|
+
private prepare(state: unknown): PreparedRequest {
|
|
923
|
+
const { state: sent, images } = liftImages(state)
|
|
924
|
+
if (images.length > 0 && !this.supports.images) throw new ClassifierUnsupported(this.backend, this.model, "images")
|
|
925
|
+
checkImages(images)
|
|
926
|
+
let assets: Record<string, unknown> | undefined
|
|
927
|
+
if (images.length > 0) {
|
|
928
|
+
assets = {}
|
|
929
|
+
for (const im of images) assets[im.ref] = { type: "image", format: im.format, data_base64: im.data }
|
|
930
|
+
}
|
|
931
|
+
return { state: sent, images, assets }
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/** The systemone style on every backend: lift, check, cache, post, decode. */
|
|
935
|
+
private async evaluateWire(
|
|
656
936
|
state: unknown,
|
|
657
|
-
questions: Record<string,
|
|
658
|
-
signal
|
|
937
|
+
questions: Record<string, AnyQuestion>,
|
|
938
|
+
signal: AbortSignal | undefined,
|
|
939
|
+
validate: boolean,
|
|
659
940
|
): Promise<Decision> {
|
|
660
|
-
const
|
|
941
|
+
const p = this.prepare(state)
|
|
942
|
+
const cache = this.opts.cache
|
|
943
|
+
let modelKey = ""
|
|
944
|
+
if (cache) {
|
|
945
|
+
modelKey = classifierModelCacheKey(this.baseUrl, this.model)
|
|
946
|
+
const served = await cache.get(modelKey)
|
|
947
|
+
if (typeof served === "string") {
|
|
948
|
+
const hit = await cache.get(classifierDecisionCacheKey(this.baseUrl, served, p.state, p.assets, questions))
|
|
949
|
+
if (typeof hit === "string") {
|
|
950
|
+
try {
|
|
951
|
+
const d = decodeDecision(JSON.parse(hit), served)
|
|
952
|
+
if (!validateDecision(questions, d)) return new Decision(d.model, d.answers, d.usage, d.calibrated, true)
|
|
953
|
+
} catch {
|
|
954
|
+
// a corrupt entry is a miss
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
const extra: Record<string, string> =
|
|
960
|
+
p.images.length > 0 && this.preset.imagesHeader ? { [CLASSIFIER_IMAGES_HEADER]: "true" } : {}
|
|
961
|
+
let text = await this.post(this.body(p, questions), signal, extra)
|
|
661
962
|
let parsed: unknown
|
|
662
963
|
try {
|
|
663
964
|
parsed = JSON.parse(text)
|
|
664
965
|
} catch (e) {
|
|
665
966
|
throw new Error(`classifier: response was not JSON: ${(e as Error).message}`)
|
|
666
967
|
}
|
|
667
|
-
|
|
968
|
+
if (this.preset.wire === "cloudflare") {
|
|
969
|
+
parsed = unwrapCloudflare(parsed)
|
|
970
|
+
text = canonicalJson(parsed)
|
|
971
|
+
}
|
|
972
|
+
const d = decodeDecision(parsed, this.model)
|
|
973
|
+
const verr = validateDecision(questions, d)
|
|
974
|
+
if (validate && verr) throw verr
|
|
975
|
+
if (cache && !verr) {
|
|
976
|
+
const served = d.model || this.model
|
|
977
|
+
await cache.set(modelKey, served)
|
|
978
|
+
await cache.set(classifierDecisionCacheKey(this.baseUrl, served, p.state, p.assets, questions), text)
|
|
979
|
+
}
|
|
980
|
+
return d
|
|
668
981
|
}
|
|
669
982
|
|
|
670
983
|
/**
|
|
@@ -676,8 +989,12 @@ export class Classifier {
|
|
|
676
989
|
* is never echoed back (a gateway happily reflects a bad Authorization header into its own
|
|
677
990
|
* 401 text).
|
|
678
991
|
*/
|
|
679
|
-
private async post(payload: string, signal?: AbortSignal): Promise<string> {
|
|
680
|
-
|
|
992
|
+
private async post(payload: string, signal?: AbortSignal, extraHeaders: Record<string, string> = {}): Promise<string> {
|
|
993
|
+
// `endpoint` is what errors name: the CONFIGURED base, ${ENV} unexpanded, so an account id in
|
|
994
|
+
// a templated URL never reaches an error. `url` is what is dialled, expanded at call time.
|
|
995
|
+
let endpoint = this.baseUrl.replace(/\/+$/, "")
|
|
996
|
+
if (this.preset.wire !== "cloudflare") endpoint += "/systemone"
|
|
997
|
+
const url = endpoint.includes("${") ? expandEnv(endpoint) : endpoint
|
|
681
998
|
const retries = this.opts.retries ?? 2
|
|
682
999
|
const base = this.opts.retryBaseMs ?? 500
|
|
683
1000
|
const classify = this.opts.onError ?? ((info: ErrorInfo): ErrorTier => (info.retryable ? "retry" : "fail"))
|
|
@@ -700,9 +1017,9 @@ export class Classifier {
|
|
|
700
1017
|
let res: Response | undefined
|
|
701
1018
|
let thrown: unknown
|
|
702
1019
|
try {
|
|
703
|
-
res = await doFetch(
|
|
1020
|
+
res = await doFetch(url, {
|
|
704
1021
|
method: "POST",
|
|
705
|
-
headers: this.requestHeaders(),
|
|
1022
|
+
headers: this.requestHeaders(extraHeaders),
|
|
706
1023
|
body: payload,
|
|
707
1024
|
signal: timer.signal,
|
|
708
1025
|
})
|
|
@@ -720,7 +1037,7 @@ export class Classifier {
|
|
|
720
1037
|
retryAfter = res.headers.get("retry-after")
|
|
721
1038
|
lastErr = new Error(`classifier: POST ${endpoint}: HTTP ${res.status}${cause(res.status, text)}`)
|
|
722
1039
|
} else {
|
|
723
|
-
lastErr = new Error(`classifier: POST ${endpoint}: ${errText(thrown)}`)
|
|
1040
|
+
lastErr = new Error(`classifier: POST ${endpoint}: ${errText(thrown).split(url).join(endpoint)}`)
|
|
724
1041
|
}
|
|
725
1042
|
const retryable = res ? res.status === 408 || isRetryableStatus(res.status, this.opts.retryableStatuses) : true
|
|
726
1043
|
if (attempt >= retries) throw lastErr
|
|
@@ -732,10 +1049,12 @@ export class Classifier {
|
|
|
732
1049
|
}
|
|
733
1050
|
|
|
734
1051
|
/** Credential and `${ENV_VAR}` headers resolve AT CALL TIME. Never logged, never returned. */
|
|
735
|
-
private requestHeaders(): Record<string, string> {
|
|
1052
|
+
private requestHeaders(extra: Record<string, string> = {}): Record<string, string> {
|
|
736
1053
|
const headers: Record<string, string> = { "Content-Type": "application/json" }
|
|
737
1054
|
const key = process.env[this.apiKeyEnv]
|
|
738
1055
|
if (key) headers["Authorization"] = `Bearer ${key}`
|
|
1056
|
+
// The image opt-in header goes BEFORE host headers, which win.
|
|
1057
|
+
for (const [k, v] of Object.entries(extra)) headers[k] = v
|
|
739
1058
|
for (const [k, v] of Object.entries(this.opts.headers ?? {})) headers[k] = expandEnv(v)
|
|
740
1059
|
return headers
|
|
741
1060
|
}
|
|
@@ -748,7 +1067,7 @@ export class Classifier {
|
|
|
748
1067
|
*/
|
|
749
1068
|
private async evaluateLLM(
|
|
750
1069
|
state: unknown,
|
|
751
|
-
questions: Record<string,
|
|
1070
|
+
questions: Record<string, AnyQuestion>,
|
|
752
1071
|
signal?: AbortSignal,
|
|
753
1072
|
): Promise<Decision> {
|
|
754
1073
|
const qs: Record<string, unknown> = {}
|
|
@@ -789,7 +1108,7 @@ export class Classifier {
|
|
|
789
1108
|
this.opts.onMetric?.(ev)
|
|
790
1109
|
}
|
|
791
1110
|
|
|
792
|
-
private emitError(start: number, e: unknown): void {
|
|
1111
|
+
private emitError(start: number, e: unknown, l1: { questions: number; questionsHash: string }): void {
|
|
793
1112
|
this.emit({
|
|
794
1113
|
event: "classifier.evaluate",
|
|
795
1114
|
model: this.model,
|
|
@@ -798,6 +1117,8 @@ export class Classifier {
|
|
|
798
1117
|
promptTokens: 0,
|
|
799
1118
|
completionTokens: 0,
|
|
800
1119
|
error: errText(e),
|
|
1120
|
+
...l1,
|
|
1121
|
+
cached: false,
|
|
801
1122
|
})
|
|
802
1123
|
}
|
|
803
1124
|
}
|
|
@@ -883,3 +1204,313 @@ function sleep(ms: number): Promise<void> {
|
|
|
883
1204
|
})
|
|
884
1205
|
}
|
|
885
1206
|
|
|
1207
|
+
|
|
1208
|
+
// ---------------------------------------------------------------- backends, images, cache, validation
|
|
1209
|
+
// SPEC.md §8B "Backends, images, cache and validation" (change add-classifier-backends, ADR 0038).
|
|
1210
|
+
// Every default here is a PROPOSED default, owner to confirm (design.md PD-1..PD-15).
|
|
1211
|
+
|
|
1212
|
+
/** Ordered named questions -> the questions map. A duplicate name is an error naming it. */
|
|
1213
|
+
function namedQuestions(qs: readonly NamedClassifierQuestion[]): Record<string, AnyQuestion> {
|
|
1214
|
+
const out: Record<string, AnyQuestion> = {}
|
|
1215
|
+
for (const { name, question } of qs) {
|
|
1216
|
+
if (Object.prototype.hasOwnProperty.call(out, name)) throw new Error(`duplicate question name ${JSON.stringify(name)}`)
|
|
1217
|
+
out[name] = question
|
|
1218
|
+
}
|
|
1219
|
+
return out
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
function sha256Hex(s: string): string {
|
|
1223
|
+
return createHash("sha256").update(s, "utf8").digest("hex")
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
interface LiftedImage {
|
|
1227
|
+
ref: string
|
|
1228
|
+
/** The MIME type until `checkImages` normalises it to `png` | `jpeg`. */
|
|
1229
|
+
format: string
|
|
1230
|
+
data: string
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
interface PreparedRequest {
|
|
1234
|
+
state: unknown
|
|
1235
|
+
images: LiftedImage[]
|
|
1236
|
+
assets?: Record<string, unknown>
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
function containsPart(v: unknown): boolean {
|
|
1240
|
+
if (Array.isArray(v)) return v.some(containsPart)
|
|
1241
|
+
if (v !== null && typeof v === "object") {
|
|
1242
|
+
if (isContentPart(v)) return true
|
|
1243
|
+
return Object.values(v as Record<string, unknown>).some(containsPart)
|
|
1244
|
+
}
|
|
1245
|
+
return false
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
/**
|
|
1249
|
+
* The §8B lifting walk. A state with no content part is returned VERBATIM (the same value), so a
|
|
1250
|
+
* text-only state is byte-identical to before. Otherwise depth-first, object keys in ASCII order,
|
|
1251
|
+
* arrays in order: an image part with inline data becomes a reference (its key when unused, else
|
|
1252
|
+
* the smallest unused `image<n>`), a text part becomes its text, any other part is an error.
|
|
1253
|
+
*/
|
|
1254
|
+
function liftImages(state: unknown): { state: unknown; images: LiftedImage[] } {
|
|
1255
|
+
if (!containsPart(state)) return { state, images: [] }
|
|
1256
|
+
const used = new Set<string>()
|
|
1257
|
+
const images: LiftedImage[] = []
|
|
1258
|
+
const nextRef = () => {
|
|
1259
|
+
for (let n = 1; ; n++) if (!used.has(`image${n}`)) return `image${n}`
|
|
1260
|
+
}
|
|
1261
|
+
const walk = (v: unknown, key: string): unknown => {
|
|
1262
|
+
if (Array.isArray(v)) return v.map((e) => walk(e, ""))
|
|
1263
|
+
if (v === null || typeof v !== "object") return v
|
|
1264
|
+
if (isContentPart(v)) {
|
|
1265
|
+
const part = v as ContentPart
|
|
1266
|
+
if (part.type === "text") return part.text
|
|
1267
|
+
if (part.type === "image") {
|
|
1268
|
+
if (!part.data) throw new Error("classifier: state: an image part needs inline data (a URL image cannot be sent)")
|
|
1269
|
+
const ref = key === "" || used.has(key) ? nextRef() : key
|
|
1270
|
+
used.add(ref)
|
|
1271
|
+
images.push({ ref, format: part.mimeType, data: part.data })
|
|
1272
|
+
return ref
|
|
1273
|
+
}
|
|
1274
|
+
throw new Error(`classifier: state: ${part.type} parts are not supported`)
|
|
1275
|
+
}
|
|
1276
|
+
const obj = v as Record<string, unknown>
|
|
1277
|
+
const out: Record<string, unknown> = {}
|
|
1278
|
+
for (const k of Object.keys(obj).sort()) out[k] = walk(obj[k], k)
|
|
1279
|
+
return out
|
|
1280
|
+
}
|
|
1281
|
+
return { state: walk(state, ""), images }
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
const IMAGE_FORMATS: Record<string, string> = { "image/png": "png", "image/jpeg": "jpeg", "image/jpg": "jpeg" }
|
|
1285
|
+
|
|
1286
|
+
/** Format, per-image and per-request limits, in collection order; normalises `format`. */
|
|
1287
|
+
function checkImages(images: LiftedImage[]): void {
|
|
1288
|
+
let total = 0
|
|
1289
|
+
for (const im of images) {
|
|
1290
|
+
const format = IMAGE_FORMATS[im.format]
|
|
1291
|
+
if (!format) throw new Error(`classifier: image "${im.ref}": type "${im.format}" is not supported (PNG or JPEG only)`)
|
|
1292
|
+
im.format = format
|
|
1293
|
+
if (im.data.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(im.data)) {
|
|
1294
|
+
throw new Error(`classifier: image "${im.ref}": data is not base64`)
|
|
1295
|
+
}
|
|
1296
|
+
const raw = Buffer.from(im.data, "base64")
|
|
1297
|
+
if (raw.length > CLASSIFIER_IMAGE_MAX_BYTES) {
|
|
1298
|
+
throw new Error(`classifier: image "${im.ref}": ${raw.length} bytes, over 16 MiB`)
|
|
1299
|
+
}
|
|
1300
|
+
const size = imageSize(format, raw)
|
|
1301
|
+
if (!size) throw new Error(`classifier: image "${im.ref}": not a readable ${format.toUpperCase()}`)
|
|
1302
|
+
const [w, h] = size
|
|
1303
|
+
const ok = (n: number) => n >= CLASSIFIER_IMAGE_MIN_SIDE && n <= CLASSIFIER_IMAGE_MAX_SIDE
|
|
1304
|
+
if (!ok(w) || !ok(h)) {
|
|
1305
|
+
throw new Error(
|
|
1306
|
+
`classifier: image "${im.ref}": ${w}x${h}; each side must be ${CLASSIFIER_IMAGE_MIN_SIDE}..${CLASSIFIER_IMAGE_MAX_SIDE} px`,
|
|
1307
|
+
)
|
|
1308
|
+
}
|
|
1309
|
+
total += raw.length
|
|
1310
|
+
}
|
|
1311
|
+
if (total > CLASSIFIER_REQUEST_IMAGE_BYTES) {
|
|
1312
|
+
throw new Error(`classifier: image "${images[images.length - 1].ref}": request images total ${total} bytes, over 32 MiB`)
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
/** Width/height from a PNG IHDR or the first JPEG SOF marker; undefined when unreadable. */
|
|
1317
|
+
function imageSize(format: string, b: Uint8Array): [number, number] | undefined {
|
|
1318
|
+
const be16 = (i: number) => (b[i] << 8) | b[i + 1]
|
|
1319
|
+
const be32 = (i: number) => ((b[i] << 24) >>> 0) + (b[i + 1] << 16) + (b[i + 2] << 8) + b[i + 3]
|
|
1320
|
+
if (format === "png") {
|
|
1321
|
+
const sig = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
|
|
1322
|
+
if (b.length < 24 || sig.some((x, i) => b[i] !== x)) return undefined
|
|
1323
|
+
if (String.fromCharCode(b[12], b[13], b[14], b[15]) !== "IHDR") return undefined
|
|
1324
|
+
return [be32(16), be32(20)]
|
|
1325
|
+
}
|
|
1326
|
+
if (b.length < 4 || b[0] !== 0xff || b[1] !== 0xd8) return undefined
|
|
1327
|
+
let i = 2
|
|
1328
|
+
while (i + 3 < b.length) {
|
|
1329
|
+
if (b[i] !== 0xff) return undefined
|
|
1330
|
+
const m = b[i + 1]
|
|
1331
|
+
if (m === 0xff) {
|
|
1332
|
+
i++
|
|
1333
|
+
continue
|
|
1334
|
+
}
|
|
1335
|
+
if (m >= 0xc0 && m <= 0xcf && m !== 0xc4 && m !== 0xc8 && m !== 0xcc) {
|
|
1336
|
+
if (i + 8 >= b.length) return undefined
|
|
1337
|
+
return [be16(i + 7), be16(i + 5)]
|
|
1338
|
+
}
|
|
1339
|
+
i += 2 + be16(i + 2)
|
|
1340
|
+
}
|
|
1341
|
+
return undefined
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
/** The System One reply inside a Cloudflare `/ai/run` envelope. Its error list is never echoed. */
|
|
1345
|
+
function unwrapCloudflare(env: unknown): unknown {
|
|
1346
|
+
if (env === null || typeof env !== "object" || Array.isArray(env) || (env as any).success !== true) {
|
|
1347
|
+
throw new Error("classifier: cloudflare: envelope not successful")
|
|
1348
|
+
}
|
|
1349
|
+
let result = (env as any).result
|
|
1350
|
+
if (result !== null && typeof result === "object" && !Array.isArray(result) && typeof result.state === "string") {
|
|
1351
|
+
if (result.state !== "Completed") throw new Error(`classifier: cloudflare: run state "${result.state}"`)
|
|
1352
|
+
result = result.result
|
|
1353
|
+
}
|
|
1354
|
+
return result
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
/** §8B model key: which model an endpoint + requested model (alias) served last. */
|
|
1358
|
+
export function classifierModelCacheKey(endpoint: string, model: string): string {
|
|
1359
|
+
return sha256Hex(canonicalJson({ endpoint, kind: "model", model }))
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
/** §8B decision key, on the SERVED model. No assets ⇒ the `assets` member is omitted. */
|
|
1363
|
+
export function classifierDecisionCacheKey(
|
|
1364
|
+
endpoint: string,
|
|
1365
|
+
servedModel: string,
|
|
1366
|
+
state: unknown,
|
|
1367
|
+
assets: Record<string, unknown> | undefined,
|
|
1368
|
+
questions: Record<string, AnyQuestion>,
|
|
1369
|
+
): string {
|
|
1370
|
+
return sha256Hex(classifierDecisionCacheKeyBytes(endpoint, servedModel, state, assets, questions))
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
/** The canonical JSON a decision key hashes (exported for conformance tests). */
|
|
1374
|
+
export function classifierDecisionCacheKeyBytes(
|
|
1375
|
+
endpoint: string,
|
|
1376
|
+
servedModel: string,
|
|
1377
|
+
state: unknown,
|
|
1378
|
+
assets: Record<string, unknown> | undefined,
|
|
1379
|
+
questions: Record<string, AnyQuestion>,
|
|
1380
|
+
): string {
|
|
1381
|
+
const qs: Record<string, unknown> = {}
|
|
1382
|
+
for (const [k, q] of Object.entries(questions)) qs[k] = questionWire(q)
|
|
1383
|
+
const m: Record<string, unknown> = { endpoint, kind: "decision", model: servedModel, questions: qs, state }
|
|
1384
|
+
if (assets && Object.keys(assets).length > 0) m.assets = assets
|
|
1385
|
+
return canonicalJson(m)
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/**
|
|
1389
|
+
* In-process TTL cache with a size cap (oldest-inserted out first). `ttlMs` 0 = no expiry; `max`
|
|
1390
|
+
* 0 = unbounded. toolnexus persists nothing itself: implement `ClassifierCache` to persist.
|
|
1391
|
+
*/
|
|
1392
|
+
export class MemoryCache implements ClassifierCache {
|
|
1393
|
+
private readonly m = new Map<string, { v: string; at: number }>()
|
|
1394
|
+
constructor(
|
|
1395
|
+
readonly ttlMs = 0,
|
|
1396
|
+
readonly max = 0,
|
|
1397
|
+
private readonly now: () => number = Date.now,
|
|
1398
|
+
) {}
|
|
1399
|
+
get(key: string): string | undefined {
|
|
1400
|
+
const e = this.m.get(key)
|
|
1401
|
+
if (!e) return undefined
|
|
1402
|
+
if (this.ttlMs > 0 && this.now() - e.at > this.ttlMs) {
|
|
1403
|
+
this.m.delete(key)
|
|
1404
|
+
return undefined
|
|
1405
|
+
}
|
|
1406
|
+
return e.v
|
|
1407
|
+
}
|
|
1408
|
+
set(key: string, value: string): void {
|
|
1409
|
+
this.m.delete(key) // a re-set is a fresh insertion
|
|
1410
|
+
this.m.set(key, { v: value, at: this.now() })
|
|
1411
|
+
if (this.max > 0) while (this.m.size > this.max) this.m.delete(this.m.keys().next().value as string)
|
|
1412
|
+
}
|
|
1413
|
+
get size(): number {
|
|
1414
|
+
return this.m.size
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
/**
|
|
1419
|
+
* V1: fail closed on a reply that does not answer exactly the questions asked, within their
|
|
1420
|
+
* schema. First failure wins, in the §8B order. Returns the error, or undefined when valid.
|
|
1421
|
+
*/
|
|
1422
|
+
export function validateDecision(
|
|
1423
|
+
questions: Record<string, AnyQuestion>,
|
|
1424
|
+
decision: { answers: Record<string, unknown> },
|
|
1425
|
+
): Error | undefined {
|
|
1426
|
+
const bad = (msg: string) => new Error(`classifier: invalid decision: ${msg}`)
|
|
1427
|
+
const answers = decision.answers ?? {}
|
|
1428
|
+
const has = (o: object, k: string) => Object.prototype.hasOwnProperty.call(o, k)
|
|
1429
|
+
const qKeys = Object.keys(questions).sort()
|
|
1430
|
+
for (const k of qKeys) if (!has(answers, k)) return bad(`no answer "${k}"`)
|
|
1431
|
+
const extra = Object.keys(answers)
|
|
1432
|
+
.filter((k) => !has(questions, k))
|
|
1433
|
+
.sort()
|
|
1434
|
+
if (extra.length > 0) return bad(`unexpected answer "${extra[0]}"`)
|
|
1435
|
+
const in01 = (p: unknown) => typeof p === "number" && !Number.isNaN(p) && p >= 0 && p <= 1
|
|
1436
|
+
const dist = (k: string, probs: Record<string, number>, offered: (o: string) => boolean): Error | undefined => {
|
|
1437
|
+
let sum = 0
|
|
1438
|
+
const pk = Object.keys(probs ?? {}).sort()
|
|
1439
|
+
for (const o of pk) {
|
|
1440
|
+
if (!offered(o)) return bad(`answer "${k}": probability for "${o}", which was not offered`)
|
|
1441
|
+
if (!in01(probs[o])) return bad(`answer "${k}": probability ${probs[o]} outside [0,1]`)
|
|
1442
|
+
sum += probs[o]
|
|
1443
|
+
}
|
|
1444
|
+
if (pk.length > 0 && Math.abs(sum - 1) > 0.02) return bad(`answer "${k}": probabilities sum to ${sum.toFixed(4)}`)
|
|
1445
|
+
return undefined
|
|
1446
|
+
}
|
|
1447
|
+
for (const k of qKeys) {
|
|
1448
|
+
const raw = questions[k]
|
|
1449
|
+
const q = raw as Question
|
|
1450
|
+
const a = answers[k] as any
|
|
1451
|
+
const qType = raw.type
|
|
1452
|
+
const aType = a?.type
|
|
1453
|
+
if (aType !== qType) return bad(`answer "${k}" is a ${aType} answer, not ${qType}`)
|
|
1454
|
+
if (isRawQuestion(raw)) continue
|
|
1455
|
+
if (q.type === "noul") {
|
|
1456
|
+
if (!in01(a.noul)) return bad(`answer "${k}": noul ${a.noul} outside [0,1]`)
|
|
1457
|
+
} else if (q.type === "choice") {
|
|
1458
|
+
const offered = (o: string) => has(q.criteria ?? {}, o)
|
|
1459
|
+
if (!offered(a.choice)) return bad(`answer "${k}": choice "${a.choice}" was not offered`)
|
|
1460
|
+
const err = dist(k, a.probabilities, offered)
|
|
1461
|
+
if (err) return err
|
|
1462
|
+
if (!in01(a.confidence)) return bad(`answer "${k}": confidence ${a.confidence} outside [0,1]`)
|
|
1463
|
+
} else {
|
|
1464
|
+
const n = (q.criteria ?? []).length
|
|
1465
|
+
if (typeof a.score !== "number" || Number.isNaN(a.score) || a.score < 0 || a.score > n - 1) {
|
|
1466
|
+
return bad(`answer "${k}": score ${a.score} outside [0,${n - 1}]`)
|
|
1467
|
+
}
|
|
1468
|
+
const offered = (o: string) => Array.from({ length: n }, (_, i) => String(i)).includes(o)
|
|
1469
|
+
const err = dist(k, a.probabilities, offered)
|
|
1470
|
+
if (err) return err
|
|
1471
|
+
if (!in01(a.confidence)) return bad(`answer "${k}": confidence ${a.confidence} outside [0,1]`)
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
return undefined
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
/** jevx's INCLUSIVE cut-points (T1). They sit alongside `Bands` (exclusive), which stays the
|
|
1478
|
+
* default of `ask` / `gate`. */
|
|
1479
|
+
export interface Thresholds {
|
|
1480
|
+
yes: number
|
|
1481
|
+
no: number
|
|
1482
|
+
minConfidence: number
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/** 0.8 / 0.2 / 0.6 (jevx's builtin profile). */
|
|
1486
|
+
export const DEFAULT_THRESHOLDS: Readonly<Thresholds> = Object.freeze({ yes: 0.8, no: 0.2, minConfidence: 0.6 })
|
|
1487
|
+
|
|
1488
|
+
/**
|
|
1489
|
+
* The band of one answer under `Thresholds`: noul `p >= yes` ⇒ yes, `p <= no` ⇒ no, else
|
|
1490
|
+
* uncertain; choice/score `confidence >= minConfidence` ⇒ yes, else uncertain. No near-uniform
|
|
1491
|
+
* demotion (that is `Bands`' rule). A raw answer is uncertain.
|
|
1492
|
+
*/
|
|
1493
|
+
export function thresholdBand(answer: DecisionAnswer | RawAnswer, t: Thresholds = DEFAULT_THRESHOLDS): "yes" | "no" | "uncertain" {
|
|
1494
|
+
const a = answer as any
|
|
1495
|
+
if (a.type === "noul") {
|
|
1496
|
+
if (a.noul >= t.yes) return "yes"
|
|
1497
|
+
if (a.noul <= t.no) return "no"
|
|
1498
|
+
} else if ((a.type === "choice" || a.type === "score") && a.confidence >= t.minConfidence) {
|
|
1499
|
+
return "yes"
|
|
1500
|
+
}
|
|
1501
|
+
return "uncertain"
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
/**
|
|
1505
|
+
* Split a question set into chunks of `size`, keys in ASCII order. A helper, not automatic:
|
|
1506
|
+
* joint calibration across a split is unproven, so the host decides to split and merges.
|
|
1507
|
+
*/
|
|
1508
|
+
export function chunkQuestions<Q extends AnyQuestion>(questions: Record<string, Q>, size: number): Record<string, Q>[] {
|
|
1509
|
+
if (!(size >= 1)) throw new Error(`classifier: chunk size must be at least 1, got ${size}`)
|
|
1510
|
+
const keys = Object.keys(questions).sort()
|
|
1511
|
+
const out: Record<string, Q>[] = []
|
|
1512
|
+
for (let i = 0; i < keys.length; i += size) {
|
|
1513
|
+
out.push(Object.fromEntries(keys.slice(i, i + size).map((k) => [k, questions[k]])))
|
|
1514
|
+
}
|
|
1515
|
+
return out
|
|
1516
|
+
}
|