pi-smart-router 1.1.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/smart-router/delegate-stream.ts +7 -1
- package/.pi/extensions/smart-router/fleet-bootstrap.ts +2 -0
- package/.pi/extensions/smart-router/pi-model-scope.ts +18 -3
- package/.pi/extensions/smart-router/routing-context.ts +10 -0
- package/README.md +78 -0
- package/config/benchmark-profiles.json +2 -2
- package/config/operator-config.json.example +7 -0
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +3 -1
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/pi-model-mapper.d.ts +12 -1
- package/dist/config/pi-model-mapper.d.ts.map +1 -1
- package/dist/config/pi-model-mapper.js +33 -8
- package/dist/config/pi-model-mapper.js.map +1 -1
- package/dist/domain/matching/embedding-provider.d.ts +63 -1
- package/dist/domain/matching/embedding-provider.d.ts.map +1 -1
- package/dist/domain/matching/embedding-provider.js +117 -0
- package/dist/domain/matching/embedding-provider.js.map +1 -1
- package/dist/domain/matching/encoder-gate.d.ts +73 -0
- package/dist/domain/matching/encoder-gate.d.ts.map +1 -0
- package/dist/domain/matching/encoder-gate.js +66 -0
- package/dist/domain/matching/encoder-gate.js.map +1 -0
- package/dist/domain/matching/hydra-matcher.d.ts +24 -3
- package/dist/domain/matching/hydra-matcher.d.ts.map +1 -1
- package/dist/domain/matching/hydra-matcher.js +21 -2
- package/dist/domain/matching/hydra-matcher.js.map +1 -1
- package/dist/domain/pipeline/router-pipeline.d.ts.map +1 -1
- package/dist/domain/pipeline/router-pipeline.js +12 -0
- package/dist/domain/pipeline/router-pipeline.js.map +1 -1
- package/dist/domain/types/entities.d.ts +15 -0
- package/dist/domain/types/entities.d.ts.map +1 -1
- package/dist/domain/types/schemas.d.ts +63 -7
- package/dist/domain/types/schemas.d.ts.map +1 -1
- package/dist/domain/types/schemas.js +46 -3
- package/dist/domain/types/schemas.js.map +1 -1
- package/dist/infrastructure/pricing/price-broker.d.ts +6 -1
- package/dist/infrastructure/pricing/price-broker.d.ts.map +1 -1
- package/dist/infrastructure/pricing/price-broker.js +16 -1
- package/dist/infrastructure/pricing/price-broker.js.map +1 -1
- package/package.json +3 -3
- package/src/config/defaults.ts +3 -0
- package/src/config/pi-model-mapper.ts +48 -9
- package/src/domain/matching/embedding-provider.ts +205 -1
- package/src/domain/matching/encoder-gate.ts +121 -0
- package/src/domain/matching/hydra-matcher.ts +53 -3
- package/src/domain/pipeline/router-pipeline.ts +13 -0
- package/src/domain/types/entities.ts +15 -0
- package/src/domain/types/schemas.ts +56 -7
- package/src/infrastructure/pricing/price-broker.ts +18 -2
|
@@ -33,6 +33,13 @@ export type CapabilitySource = 'benchmark' | 'pattern_default';
|
|
|
33
33
|
/** ModelProfile plus operator-visible capability provenance (SP-174). */
|
|
34
34
|
export interface MappedModelProfile extends ModelProfile {
|
|
35
35
|
readonly capability_source: CapabilitySource;
|
|
36
|
+
/**
|
|
37
|
+
* Context limits carried from the pi model registry (`Model.contextWindow` /
|
|
38
|
+
* `Model.maxTokens`) — SP-299 / #177. Kept separate from `limits` (YAML
|
|
39
|
+
* overrides) so `resolveLimits` can rank them below LiteLLM catalog data
|
|
40
|
+
* and above tier defaults.
|
|
41
|
+
*/
|
|
42
|
+
readonly pi_registry_limits?: ModelLimits;
|
|
36
43
|
}
|
|
37
44
|
|
|
38
45
|
const benchmarkCapabilitiesSchema = z.object({
|
|
@@ -181,7 +188,7 @@ function withBenchmarkCapabilities(
|
|
|
181
188
|
};
|
|
182
189
|
}
|
|
183
190
|
|
|
184
|
-
/** Pi registry `Model.cost` shape — per-token USD rates. */
|
|
191
|
+
/** Pi registry `Model.cost` shape — per-1M-token USD rates (SP-298). */
|
|
185
192
|
export interface PiRegistryCost {
|
|
186
193
|
readonly input: number;
|
|
187
194
|
readonly output: number;
|
|
@@ -194,6 +201,10 @@ export interface PiModelInput {
|
|
|
194
201
|
readonly id: string;
|
|
195
202
|
readonly name?: string;
|
|
196
203
|
readonly cost?: PiRegistryCost;
|
|
204
|
+
/** Pi registry `Model.contextWindow` — total input token window (SP-299 / #177). */
|
|
205
|
+
readonly contextWindow?: number;
|
|
206
|
+
/** Pi registry `Model.maxTokens` — max output tokens (SP-299 / #177). */
|
|
207
|
+
readonly maxTokens?: number;
|
|
197
208
|
}
|
|
198
209
|
|
|
199
210
|
interface ModelPatternRule {
|
|
@@ -355,19 +366,24 @@ function matchPatternRules(id: string): ModelFamilyDefaults | undefined {
|
|
|
355
366
|
}
|
|
356
367
|
|
|
357
368
|
/**
|
|
358
|
-
* Registry cost → USD per 1M tokens: average of input and output rates
|
|
359
|
-
*
|
|
360
|
-
*
|
|
369
|
+
* Registry cost → USD per 1M tokens: average of input and output rates.
|
|
370
|
+
* Pi registry `Model.cost` rates are already denominated per 1M tokens
|
|
371
|
+
* (SP-298), so no scaling is applied — unlike LiteLLM per-token rates handled
|
|
372
|
+
* by `computeCostPer1MTokens` in litellm-fetch (SP-045/SP-046).
|
|
373
|
+
* Returns undefined when both input and output rates are zero (use pattern
|
|
374
|
+
* default), or when either rate is negative — negative values are vendor
|
|
375
|
+
* unknown/sentinel rates, not valid pricing, so cost stays unknown (SP-298).
|
|
361
376
|
*/
|
|
362
377
|
function deriveFallbackCostPer1M(cost: PiRegistryCost): number | undefined {
|
|
363
378
|
const { input, output } = cost;
|
|
364
379
|
if (input === 0 && output === 0) {
|
|
365
380
|
return undefined;
|
|
366
381
|
}
|
|
382
|
+
if (input < 0 || output < 0) {
|
|
383
|
+
return undefined;
|
|
384
|
+
}
|
|
367
385
|
|
|
368
|
-
|
|
369
|
-
const outputPer1M = output * 1_000_000;
|
|
370
|
-
return (inputPer1M + outputPer1M) / 2;
|
|
386
|
+
return (input + output) / 2;
|
|
371
387
|
}
|
|
372
388
|
|
|
373
389
|
function resolveFallbackCost(
|
|
@@ -382,6 +398,23 @@ function resolveFallbackCost(
|
|
|
382
398
|
return fromRegistry ?? defaults.pricing.fallback_cost_per_1m;
|
|
383
399
|
}
|
|
384
400
|
|
|
401
|
+
/**
|
|
402
|
+
* Pi registry contextWindow/maxTokens → ModelLimits (SP-299 / #177).
|
|
403
|
+
* Non-positive or non-finite registry values are vendor sentinels, not real
|
|
404
|
+
* limits — ignored so tier defaults still apply.
|
|
405
|
+
*/
|
|
406
|
+
function derivePiRegistryLimits(input: PiModelInput): ModelLimits | undefined {
|
|
407
|
+
const limits: { max_input_tokens?: number; max_output_tokens?: number } = {};
|
|
408
|
+
const { contextWindow, maxTokens } = input;
|
|
409
|
+
if (contextWindow !== undefined && Number.isFinite(contextWindow) && contextWindow > 0) {
|
|
410
|
+
limits.max_input_tokens = Math.floor(contextWindow);
|
|
411
|
+
}
|
|
412
|
+
if (maxTokens !== undefined && Number.isFinite(maxTokens) && maxTokens > 0) {
|
|
413
|
+
limits.max_output_tokens = Math.floor(maxTokens);
|
|
414
|
+
}
|
|
415
|
+
return Object.keys(limits).length > 0 ? limits : undefined;
|
|
416
|
+
}
|
|
417
|
+
|
|
385
418
|
function buildProfile(
|
|
386
419
|
input: PiModelInput,
|
|
387
420
|
defaults: ModelFamilyDefaults,
|
|
@@ -410,11 +443,17 @@ function buildProfile(
|
|
|
410
443
|
? { ...profile, performance: { ...defaults.performance } }
|
|
411
444
|
: profile;
|
|
412
445
|
|
|
446
|
+
const piRegistryLimits = derivePiRegistryLimits(input);
|
|
447
|
+
const withLimits =
|
|
448
|
+
piRegistryLimits !== undefined
|
|
449
|
+
? { ...withPerformance, pi_registry_limits: piRegistryLimits }
|
|
450
|
+
: withPerformance;
|
|
451
|
+
|
|
413
452
|
if (defaults.endpoint !== undefined) {
|
|
414
|
-
return { ...
|
|
453
|
+
return { ...withLimits, endpoint: defaults.endpoint };
|
|
415
454
|
}
|
|
416
455
|
|
|
417
|
-
return
|
|
456
|
+
return withLimits;
|
|
418
457
|
}
|
|
419
458
|
|
|
420
459
|
/**
|
|
@@ -10,8 +10,12 @@ import { createHash } from 'node:crypto';
|
|
|
10
10
|
import { promises as fs } from 'node:fs';
|
|
11
11
|
import path from 'node:path';
|
|
12
12
|
|
|
13
|
-
import type { Encoder } from '../types/schemas.js';
|
|
13
|
+
import type { Encoder, EncoderCascadeConfig } from '../types/schemas.js';
|
|
14
14
|
import { DEFAULT_ENCODER } from '../types/schemas.js';
|
|
15
|
+
import {
|
|
16
|
+
selectEncoderForPrompt,
|
|
17
|
+
type EncoderGateReasonCode,
|
|
18
|
+
} from './encoder-gate.js';
|
|
15
19
|
|
|
16
20
|
export const EMBEDDING_DIM = 384;
|
|
17
21
|
|
|
@@ -360,3 +364,203 @@ export async function createTextEmbedder(
|
|
|
360
364
|
}
|
|
361
365
|
}
|
|
362
366
|
}
|
|
367
|
+
|
|
368
|
+
// ─── Cascading embedder (SP-292, #173 part 2) ────────────────────────────────
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Decision telemetry for the most recent cascade embed. Mirrors the routing
|
|
372
|
+
* feature-sidecar fields from #173 so the pipeline can copy them onto live
|
|
373
|
+
* decisions without reshaping (SP-292).
|
|
374
|
+
*/
|
|
375
|
+
export interface CascadeEmbedderTelemetry {
|
|
376
|
+
/** Encoder that produced the embedding (never mixed across spaces). */
|
|
377
|
+
readonly encoder_selected: Encoder;
|
|
378
|
+
/** Gate token estimate for the embedded prompt. */
|
|
379
|
+
readonly token_estimate: number;
|
|
380
|
+
/** Configured cascade threshold in effect for the decision. */
|
|
381
|
+
readonly cascade_threshold: number;
|
|
382
|
+
/** SP-291 gate reason code (`under_threshold` / `over_threshold` / `granite_fallback`). */
|
|
383
|
+
readonly reason_code: EncoderGateReasonCode;
|
|
384
|
+
/**
|
|
385
|
+
* Present when an over-threshold prompt degraded to the primary encoder
|
|
386
|
+
* because the long-context session was unavailable (degrade, never mix).
|
|
387
|
+
*/
|
|
388
|
+
readonly cascade_fallback_reason: EncoderGateReasonCode | null;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
export interface CascadingTextEmbedder extends TextEmbedder {
|
|
392
|
+
/**
|
|
393
|
+
* Telemetry from the most recently completed embed(); null before the
|
|
394
|
+
* first embed. Concurrent embeds race on this snapshot — callers on the
|
|
395
|
+
* routing hot path embed sequentially per request.
|
|
396
|
+
*/
|
|
397
|
+
lastTelemetry(): CascadeEmbedderTelemetry | null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** Type guard: does this embedder carry cascade decision telemetry? */
|
|
401
|
+
export function isCascadingTextEmbedder(
|
|
402
|
+
embedder: TextEmbedder,
|
|
403
|
+
): embedder is CascadingTextEmbedder {
|
|
404
|
+
return (
|
|
405
|
+
typeof (embedder as Partial<CascadingTextEmbedder>).lastTelemetry ===
|
|
406
|
+
'function'
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
export interface CreateCascadingTextEmbedderOptions {
|
|
411
|
+
readonly pinOptions?: OnnxPinOptions;
|
|
412
|
+
/**
|
|
413
|
+
* Primary encoder for under-threshold / fallback embeds.
|
|
414
|
+
* Default: DEFAULT_ENCODER ('minilm'), matching `hydra.encoder` defaults.
|
|
415
|
+
*/
|
|
416
|
+
readonly primaryEncoder?: Encoder;
|
|
417
|
+
/**
|
|
418
|
+
* Session factory override for tests. Default: createTextEmbedder bound to
|
|
419
|
+
* the cascade artifact cache path and pin options.
|
|
420
|
+
*/
|
|
421
|
+
readonly sessionFactory?: (encoder: Encoder) => Promise<TextEmbedder>;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Lazy dual-session cascading embedder (SP-292, #173 part 2).
|
|
426
|
+
*
|
|
427
|
+
* Owns at most one ONNX session per encoder: the primary session loads on the
|
|
428
|
+
* first embed, the long-context (Granite) session loads only on the first
|
|
429
|
+
* over-threshold prompt — memory is paid only when the path fires. Selection
|
|
430
|
+
* is the SP-291 pure gate; token estimates default to prompt length (the
|
|
431
|
+
* turn-envelope-parity estimator).
|
|
432
|
+
*
|
|
433
|
+
* Failure behavior: degrade, never mix. If the long-context session fails to
|
|
434
|
+
* load or embed, the request is served by the primary encoder with reason
|
|
435
|
+
* code `granite_fallback` and the long-context encoder is marked unavailable
|
|
436
|
+
* for subsequent prompts. Never silently compares cross-encoder vectors.
|
|
437
|
+
*
|
|
438
|
+
* dispose() closes every session that was actually loaded (idempotent);
|
|
439
|
+
* embed() after dispose fails closed (SP-260 precedent).
|
|
440
|
+
*/
|
|
441
|
+
export function createCascadingTextEmbedder(
|
|
442
|
+
config: EncoderCascadeConfig,
|
|
443
|
+
artifactCachePath: string,
|
|
444
|
+
options?: CreateCascadingTextEmbedderOptions,
|
|
445
|
+
): CascadingTextEmbedder {
|
|
446
|
+
const primaryEncoder = options?.primaryEncoder ?? DEFAULT_ENCODER;
|
|
447
|
+
const longContextEncoder = config.long_context_encoder;
|
|
448
|
+
const factory =
|
|
449
|
+
options?.sessionFactory ??
|
|
450
|
+
((encoder: Encoder) =>
|
|
451
|
+
createTextEmbedder(encoder, artifactCachePath, options?.pinOptions));
|
|
452
|
+
|
|
453
|
+
const sessions = new Map<Encoder, Promise<TextEmbedder>>();
|
|
454
|
+
let longContextAvailable = true;
|
|
455
|
+
let disposed = false;
|
|
456
|
+
let last: CascadeEmbedderTelemetry | null = null;
|
|
457
|
+
|
|
458
|
+
function sessionFor(encoder: Encoder): Promise<TextEmbedder> {
|
|
459
|
+
let pending = sessions.get(encoder);
|
|
460
|
+
if (pending === undefined) {
|
|
461
|
+
pending = factory(encoder);
|
|
462
|
+
sessions.set(encoder, pending);
|
|
463
|
+
// A rejected session promise must not be cached forever — drop it so a
|
|
464
|
+
// later retry observes a fresh load attempt.
|
|
465
|
+
pending.catch(() => sessions.delete(encoder));
|
|
466
|
+
}
|
|
467
|
+
return pending;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
function telemetry(
|
|
471
|
+
decision: {
|
|
472
|
+
readonly encoder: Encoder;
|
|
473
|
+
readonly reason_code: EncoderGateReasonCode;
|
|
474
|
+
readonly token_estimate: number;
|
|
475
|
+
},
|
|
476
|
+
fallbackReason: EncoderGateReasonCode | null,
|
|
477
|
+
): CascadeEmbedderTelemetry {
|
|
478
|
+
return {
|
|
479
|
+
encoder_selected: decision.encoder,
|
|
480
|
+
token_estimate: decision.token_estimate,
|
|
481
|
+
cascade_threshold: config.token_threshold,
|
|
482
|
+
reason_code: fallbackReason ?? decision.reason_code,
|
|
483
|
+
cascade_fallback_reason: fallbackReason,
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
return {
|
|
488
|
+
async embed(text: string): Promise<Float32Array> {
|
|
489
|
+
if (disposed) {
|
|
490
|
+
throw new Error(
|
|
491
|
+
'CascadingTextEmbedder has been disposed; embed() fails closed. ' +
|
|
492
|
+
'Create a new embedder via createCascadingTextEmbedder to continue.',
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
const decision = selectEncoderForPrompt(text, config, {
|
|
497
|
+
primaryEncoder,
|
|
498
|
+
longContextAvailable,
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
if (decision.encoder === longContextEncoder && longContextAvailable) {
|
|
502
|
+
try {
|
|
503
|
+
const session = await sessionFor(longContextEncoder);
|
|
504
|
+
const vector = await session.embed(text);
|
|
505
|
+
last = telemetry(decision, null);
|
|
506
|
+
return vector;
|
|
507
|
+
} catch (error) {
|
|
508
|
+
// Degrade, never mix (#173): Granite session/artifacts unavailable →
|
|
509
|
+
// serve with the primary encoder and make the fallback explicit.
|
|
510
|
+
longContextAvailable = false;
|
|
511
|
+
sessions.delete(longContextEncoder);
|
|
512
|
+
console.warn(
|
|
513
|
+
'Encoder cascade: long-context encoder unavailable; degrading to primary encoder (granite_fallback)',
|
|
514
|
+
{
|
|
515
|
+
long_context_encoder: longContextEncoder,
|
|
516
|
+
primary_encoder: primaryEncoder,
|
|
517
|
+
token_estimate: decision.token_estimate,
|
|
518
|
+
error: error instanceof Error ? error.message : String(error),
|
|
519
|
+
},
|
|
520
|
+
);
|
|
521
|
+
const session = await sessionFor(primaryEncoder);
|
|
522
|
+
const vector = await session.embed(text);
|
|
523
|
+
last = telemetry(
|
|
524
|
+
{ ...decision, encoder: primaryEncoder },
|
|
525
|
+
'granite_fallback',
|
|
526
|
+
);
|
|
527
|
+
return vector;
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
const session = await sessionFor(primaryEncoder);
|
|
532
|
+
const vector = await session.embed(text);
|
|
533
|
+
last = telemetry(
|
|
534
|
+
{ ...decision, encoder: primaryEncoder },
|
|
535
|
+
decision.reason_code === 'granite_fallback' ? 'granite_fallback' : null,
|
|
536
|
+
);
|
|
537
|
+
return vector;
|
|
538
|
+
},
|
|
539
|
+
|
|
540
|
+
lastTelemetry(): CascadeEmbedderTelemetry | null {
|
|
541
|
+
return last;
|
|
542
|
+
},
|
|
543
|
+
|
|
544
|
+
async dispose(): Promise<void> {
|
|
545
|
+
if (disposed) return; // idempotent: safe for shared-factory callers
|
|
546
|
+
disposed = true;
|
|
547
|
+
const pending = [...sessions.values()];
|
|
548
|
+
sessions.clear();
|
|
549
|
+
const failures: string[] = [];
|
|
550
|
+
for (const sessionPromise of pending) {
|
|
551
|
+
try {
|
|
552
|
+
const session = await sessionPromise;
|
|
553
|
+
await session.dispose();
|
|
554
|
+
} catch (error) {
|
|
555
|
+
failures.push(error instanceof Error ? error.message : String(error));
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
// Fail loud: partially released cascade sessions must not pass silently.
|
|
559
|
+
if (failures.length > 0) {
|
|
560
|
+
throw new Error(
|
|
561
|
+
`CascadingTextEmbedder dispose failed for ${failures.length} session(s): ${failures.join('; ')}`,
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
},
|
|
565
|
+
};
|
|
566
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encoder cascade gate (SP-291, #173 part 1) — pure domain, unit-testable.
|
|
3
|
+
*
|
|
4
|
+
* Opt-in per-prompt encoder selection: when `hydra.encoder_cascade.enabled`
|
|
5
|
+
* is true, prompts whose estimated token count reaches `token_threshold` are
|
|
6
|
+
* routed to the long-context encoder (Granite 97M, SP-156) instead of the
|
|
7
|
+
* primary `hydra.encoder` (MiniLM, 512-token window). Default off: existing
|
|
8
|
+
* single-encoder installs are unaffected and pay zero added cost — the gate
|
|
9
|
+
* is pre-embedding arithmetic only.
|
|
10
|
+
*
|
|
11
|
+
* Token estimate parity: this module reuses the same estimator as the
|
|
12
|
+
* turn-envelope stage (`src/domain/pipeline/turn-envelope-stage.ts`):
|
|
13
|
+
* `estimated_input_tokens ?? prompt_text.length`. Callers pass
|
|
14
|
+
* `RoutingRequest.estimated_input_tokens` via options when present; the
|
|
15
|
+
* fallback is the prompt's character length.
|
|
16
|
+
*
|
|
17
|
+
* Hard constraint (#173): encoders embed into different vector spaces — the
|
|
18
|
+
* gate only SELECTS an encoder; per-encoder artifacts (centroids, learned
|
|
19
|
+
* projection) and the cascading embedder land in SP-292. Never mix spaces.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import type { Encoder, EncoderCascadeConfig } from '../types/schemas.js';
|
|
23
|
+
import { DEFAULT_ENCODER } from '../types/schemas.js';
|
|
24
|
+
|
|
25
|
+
/** Gate outcomes per #173; attached to decision telemetry. */
|
|
26
|
+
export type EncoderGateReasonCode =
|
|
27
|
+
/** Cascade disabled — every prompt uses the primary encoder. */
|
|
28
|
+
| 'cascade_disabled'
|
|
29
|
+
/** Estimate below the configured threshold — primary encoder. */
|
|
30
|
+
| 'under_threshold'
|
|
31
|
+
/** Estimate at/over the threshold — long-context encoder selected. */
|
|
32
|
+
| 'over_threshold'
|
|
33
|
+
/**
|
|
34
|
+
* Over threshold but the long-context encoder is unavailable — degrade to
|
|
35
|
+
* the primary encoder (continuity of routing wins; never mix vector spaces).
|
|
36
|
+
*/
|
|
37
|
+
| 'granite_fallback';
|
|
38
|
+
|
|
39
|
+
export interface EncoderGateDecision {
|
|
40
|
+
readonly encoder: Encoder;
|
|
41
|
+
readonly reason_code: EncoderGateReasonCode;
|
|
42
|
+
readonly token_estimate: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface EncoderGateOptions {
|
|
46
|
+
/**
|
|
47
|
+
* Precomputed token estimate — pass `RoutingRequest.estimated_input_tokens`
|
|
48
|
+
* when set. Falls back to `prompt.length`, exactly mirroring the
|
|
49
|
+
* turn-envelope stage estimator (`estimated_input_tokens ?? prompt_text.length`).
|
|
50
|
+
*/
|
|
51
|
+
readonly estimatedTokens?: number | undefined;
|
|
52
|
+
/**
|
|
53
|
+
* Primary encoder used when the cascade is disabled or does not fire.
|
|
54
|
+
* Default: DEFAULT_ENCODER ('minilm'), matching `hydra.encoder` defaults.
|
|
55
|
+
*/
|
|
56
|
+
readonly primaryEncoder?: Encoder | undefined;
|
|
57
|
+
/**
|
|
58
|
+
* Long-context encoder availability signal (SP-292 embedder layer owns
|
|
59
|
+
* session/artifact health). When explicitly false, an over-threshold prompt
|
|
60
|
+
* degrades to the primary encoder with reason `granite_fallback`.
|
|
61
|
+
* Default: assumed available.
|
|
62
|
+
*/
|
|
63
|
+
readonly longContextAvailable?: boolean;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Turn-envelope-parity token estimator: a caller-supplied estimate wins;
|
|
68
|
+
* otherwise the prompt's character length is the proxy (same formula as
|
|
69
|
+
* `turn-envelope-stage.ts` step 2b breakeven arithmetic).
|
|
70
|
+
*/
|
|
71
|
+
export function estimatePromptTokens(
|
|
72
|
+
prompt: string,
|
|
73
|
+
estimatedTokens?: number,
|
|
74
|
+
): number {
|
|
75
|
+
return estimatedTokens ?? prompt.length;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Select the encoder for a prompt. Pure: no I/O, no ONNX sessions.
|
|
80
|
+
*
|
|
81
|
+
* Boundary semantics (per #173 eval table): `token_estimate >= token_threshold`
|
|
82
|
+
* routes to the long-context encoder; strictly below stays on the primary.
|
|
83
|
+
*/
|
|
84
|
+
export function selectEncoderForPrompt(
|
|
85
|
+
prompt: string,
|
|
86
|
+
config: EncoderCascadeConfig,
|
|
87
|
+
options?: EncoderGateOptions,
|
|
88
|
+
): EncoderGateDecision {
|
|
89
|
+
const primary = options?.primaryEncoder ?? DEFAULT_ENCODER;
|
|
90
|
+
const tokenEstimate = estimatePromptTokens(prompt, options?.estimatedTokens);
|
|
91
|
+
|
|
92
|
+
if (!config.enabled) {
|
|
93
|
+
return {
|
|
94
|
+
encoder: primary,
|
|
95
|
+
reason_code: 'cascade_disabled',
|
|
96
|
+
token_estimate: tokenEstimate,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (tokenEstimate < config.token_threshold) {
|
|
101
|
+
return {
|
|
102
|
+
encoder: primary,
|
|
103
|
+
reason_code: 'under_threshold',
|
|
104
|
+
token_estimate: tokenEstimate,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (options?.longContextAvailable === false) {
|
|
109
|
+
return {
|
|
110
|
+
encoder: primary,
|
|
111
|
+
reason_code: 'granite_fallback',
|
|
112
|
+
token_estimate: tokenEstimate,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return {
|
|
117
|
+
encoder: config.long_context_encoder,
|
|
118
|
+
reason_code: 'over_threshold',
|
|
119
|
+
token_estimate: tokenEstimate,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
@@ -19,8 +19,11 @@ import { z } from 'zod';
|
|
|
19
19
|
import { DEFAULT_OPERATOR_CONFIG } from '../../config/defaults.js';
|
|
20
20
|
import { buildHydraInput } from './hydra-input.js';
|
|
21
21
|
import {
|
|
22
|
+
createCascadingTextEmbedder,
|
|
22
23
|
createTextEmbedder,
|
|
24
|
+
isCascadingTextEmbedder,
|
|
23
25
|
EMBEDDING_DIM,
|
|
26
|
+
type CascadeEmbedderTelemetry,
|
|
24
27
|
type TextEmbedder,
|
|
25
28
|
} from './embedding-provider.js';
|
|
26
29
|
import {
|
|
@@ -29,7 +32,12 @@ import {
|
|
|
29
32
|
type ModernBertHeadsPredictor,
|
|
30
33
|
} from './modernbert-heads.js';
|
|
31
34
|
import { HYDRA_WEIGHTS_MISSING_REASON_CODE } from './missing-weights-reason-codes.js';
|
|
32
|
-
import type {
|
|
35
|
+
import type {
|
|
36
|
+
Encoder,
|
|
37
|
+
EncoderCascadeConfig,
|
|
38
|
+
HydraConfig,
|
|
39
|
+
HydraHeads,
|
|
40
|
+
} from '../types/schemas.js';
|
|
33
41
|
import { DEFAULT_ENCODER, DEFAULT_HYDRA_HEADS } from '../types/schemas.js';
|
|
34
42
|
import {
|
|
35
43
|
scoreMultiObjective,
|
|
@@ -69,6 +77,12 @@ export interface EmbeddingProvider {
|
|
|
69
77
|
* Absent/empty when learned weights are active.
|
|
70
78
|
*/
|
|
71
79
|
requirementReasonCodes?(): readonly string[];
|
|
80
|
+
/**
|
|
81
|
+
* Cascade decision telemetry from the most recent extraction (SP-292, #173).
|
|
82
|
+
* Present only on cascading-embedder-backed providers; null before the first
|
|
83
|
+
* embed. Absent on single-encoder and ModernBERT K=4 providers.
|
|
84
|
+
*/
|
|
85
|
+
cascadeTelemetry?(): CascadeEmbedderTelemetry | null;
|
|
72
86
|
dispose(): Promise<void>;
|
|
73
87
|
}
|
|
74
88
|
|
|
@@ -94,6 +108,13 @@ export interface MatchResult {
|
|
|
94
108
|
* always populates it.
|
|
95
109
|
*/
|
|
96
110
|
readonly requirement_reason_codes?: readonly string[];
|
|
111
|
+
/**
|
|
112
|
+
* Encoder cascade decision telemetry for this match (SP-292, #173):
|
|
113
|
+
* `encoder_selected`, `token_estimate`, `cascade_threshold`, and the
|
|
114
|
+
* fallback reason when an over-threshold prompt degraded to the primary
|
|
115
|
+
* encoder. Absent on legacy call paths and non-cascade providers.
|
|
116
|
+
*/
|
|
117
|
+
readonly cascade_telemetry?: CascadeEmbedderTelemetry | null;
|
|
97
118
|
readonly elapsedMs: number;
|
|
98
119
|
readonly budgetExceeded: boolean;
|
|
99
120
|
}
|
|
@@ -414,6 +435,10 @@ export function wrapHydraEmbeddingProvider(
|
|
|
414
435
|
return { requirements, embedding: Array.from(embedding) };
|
|
415
436
|
},
|
|
416
437
|
|
|
438
|
+
...(isCascadingTextEmbedder(embedder)
|
|
439
|
+
? { cascadeTelemetry: () => embedder.lastTelemetry() }
|
|
440
|
+
: {}),
|
|
441
|
+
|
|
417
442
|
async dispose(): Promise<void> {
|
|
418
443
|
await embedder.dispose();
|
|
419
444
|
},
|
|
@@ -595,12 +620,20 @@ export class HydraMatcher {
|
|
|
595
620
|
|
|
596
621
|
const elapsedMs = performance.now() - start;
|
|
597
622
|
|
|
623
|
+
// SP-292 / #173: snapshot the cascade decision for this match (null on
|
|
624
|
+
// non-cascade providers and before any embed ran).
|
|
625
|
+
const cascadeTelemetry = this.provider.cascadeTelemetry?.() ?? null;
|
|
626
|
+
|
|
598
627
|
return {
|
|
599
628
|
selected,
|
|
600
629
|
candidates: rankedCandidates,
|
|
601
630
|
requirements,
|
|
602
631
|
embedding: detailed ? detailed.embedding : null,
|
|
603
632
|
requirement_reason_codes: this.requirementReasonCodes,
|
|
633
|
+
// SP-292 / #173: cascade decision telemetry rides the match result so
|
|
634
|
+
// the pipeline can copy it onto the decision feature sidecar. Absent
|
|
635
|
+
// on single-encoder providers (cascade disabled path unchanged).
|
|
636
|
+
...(cascadeTelemetry ? { cascade_telemetry: cascadeTelemetry } : {}),
|
|
604
637
|
elapsedMs,
|
|
605
638
|
budgetExceeded: elapsedMs > this.budgetMs,
|
|
606
639
|
};
|
|
@@ -632,6 +665,14 @@ export class HydraMatcher {
|
|
|
632
665
|
export interface CreateOnnxEmbeddingProviderOptions {
|
|
633
666
|
readonly encoder?: Encoder;
|
|
634
667
|
readonly projectionWeightsPath?: string;
|
|
668
|
+
/**
|
|
669
|
+
* Opt-in encoder cascade (SP-292, #173). When enabled, the provider is
|
|
670
|
+
* backed by a lazy dual-session cascading embedder that routes
|
|
671
|
+
* over-threshold prompts to the long-context encoder and degrades to the
|
|
672
|
+
* primary encoder with an explicit fallback reason when unavailable.
|
|
673
|
+
* Default: disabled (single-encoder path unchanged).
|
|
674
|
+
*/
|
|
675
|
+
readonly encoderCascade?: EncoderCascadeConfig;
|
|
635
676
|
}
|
|
636
677
|
|
|
637
678
|
export interface CreateHydraEmbeddingProviderOptions extends CreateOnnxEmbeddingProviderOptions {
|
|
@@ -649,7 +690,13 @@ export async function createOnnxEmbeddingProvider(
|
|
|
649
690
|
options?: CreateOnnxEmbeddingProviderOptions,
|
|
650
691
|
): Promise<EmbeddingProvider> {
|
|
651
692
|
const encoder = options?.encoder ?? DEFAULT_ENCODER;
|
|
652
|
-
|
|
693
|
+
// SP-292 / #173: cascade path only when the operator opted in; otherwise
|
|
694
|
+
// the single-encoder createTextEmbedder behavior is byte-identical.
|
|
695
|
+
const embedder = options?.encoderCascade?.enabled
|
|
696
|
+
? createCascadingTextEmbedder(options.encoderCascade, artifactCachePath, {
|
|
697
|
+
primaryEncoder: encoder,
|
|
698
|
+
})
|
|
699
|
+
: await createTextEmbedder(encoder, artifactCachePath);
|
|
653
700
|
const projectionWeights = resolveHydraProjectionWeights(
|
|
654
701
|
options?.projectionWeightsPath ? { filePath: options.projectionWeightsPath } : undefined,
|
|
655
702
|
);
|
|
@@ -687,7 +734,7 @@ export async function createHydraEmbeddingProvider(
|
|
|
687
734
|
*/
|
|
688
735
|
export async function createHydraMatcherFromHydraConfig(
|
|
689
736
|
hydraConfig: Pick<HydraConfig, 'artifact_cache_path'> &
|
|
690
|
-
Partial<Pick<HydraConfig, 'encoder' | 'hydra_heads'>>,
|
|
737
|
+
Partial<Pick<HydraConfig, 'encoder' | 'hydra_heads' | 'encoder_cascade'>>,
|
|
691
738
|
options?: Omit<HydraMatcherConfig, 'artifactCachePath' | 'encoder' | 'hydraHeads'>,
|
|
692
739
|
): Promise<HydraMatcher> {
|
|
693
740
|
const encoder = hydraConfig.encoder ?? DEFAULT_ENCODER;
|
|
@@ -695,6 +742,9 @@ export async function createHydraMatcherFromHydraConfig(
|
|
|
695
742
|
const provider = await createHydraEmbeddingProvider(hydraConfig.artifact_cache_path, {
|
|
696
743
|
encoder,
|
|
697
744
|
hydraHeads,
|
|
745
|
+
...(hydraConfig.encoder_cascade === undefined
|
|
746
|
+
? {}
|
|
747
|
+
: { encoderCascade: hydraConfig.encoder_cascade }),
|
|
698
748
|
...(options?.projectionWeightsPath === undefined
|
|
699
749
|
? {}
|
|
700
750
|
: { projectionWeightsPath: options.projectionWeightsPath }),
|
|
@@ -407,6 +407,19 @@ export class RouterPipeline {
|
|
|
407
407
|
// Absent (not null) unless capture is enabled so the wire decision stays
|
|
408
408
|
// byte-identical when the operator has not opted in.
|
|
409
409
|
...(ctx.hydraResult?.embedding ? { embedding: ctx.hydraResult.embedding } : {}),
|
|
410
|
+
// SP-292 / #173: encoder cascade decision telemetry (encoder_selected,
|
|
411
|
+
// token_estimate, cascade_threshold, cascade_fallback_reason). Present
|
|
412
|
+
// only when the cascade is enabled and hydra_match ran; absent on the
|
|
413
|
+
// single-encoder path so the wire decision stays byte-identical.
|
|
414
|
+
...(ctx.hydraResult?.cascade_telemetry
|
|
415
|
+
? {
|
|
416
|
+
encoder_selected: ctx.hydraResult.cascade_telemetry.encoder_selected,
|
|
417
|
+
token_estimate: ctx.hydraResult.cascade_telemetry.token_estimate,
|
|
418
|
+
cascade_threshold: ctx.hydraResult.cascade_telemetry.cascade_threshold,
|
|
419
|
+
cascade_fallback_reason:
|
|
420
|
+
ctx.hydraResult.cascade_telemetry.cascade_fallback_reason,
|
|
421
|
+
}
|
|
422
|
+
: {}),
|
|
410
423
|
candidates: this.mergeFeatureCandidates(ctx),
|
|
411
424
|
tier_hint: ctx.tierHint,
|
|
412
425
|
tier_hint_reason_code: ctx.tierHintReasonCode,
|
|
@@ -404,6 +404,21 @@ export interface RoutingFeatureSidecar {
|
|
|
404
404
|
readonly prewarm_accepted?: boolean | null;
|
|
405
405
|
/** Telemetry-visible reason the acceptance guard disabled prewarm (SP-217, #117). */
|
|
406
406
|
readonly prewarm_disabled_reason?: string | null;
|
|
407
|
+
/**
|
|
408
|
+
* Encoder selected for HyDRA embedding (SP-292, #173). Present only when
|
|
409
|
+
* the encoder cascade is enabled and the hydra_match stage ran.
|
|
410
|
+
* Union mirrors `Encoder` in schemas.ts (entities.ts is import-free).
|
|
411
|
+
*/
|
|
412
|
+
readonly encoder_selected?: 'minilm' | 'granite' | null;
|
|
413
|
+
/** Gate token estimate used for cascade selection (SP-292, #173). */
|
|
414
|
+
readonly token_estimate?: number | null;
|
|
415
|
+
/** Configured cascade token threshold in effect (SP-292, #173). */
|
|
416
|
+
readonly cascade_threshold?: number | null;
|
|
417
|
+
/**
|
|
418
|
+
* Fallback reason when an over-threshold prompt degraded to the primary
|
|
419
|
+
* encoder (degrade, never mix; SP-292, #173). Null when no fallback fired.
|
|
420
|
+
*/
|
|
421
|
+
readonly cascade_fallback_reason?: string | null;
|
|
407
422
|
}
|
|
408
423
|
|
|
409
424
|
export interface RoutingDecision {
|