pi-smart-router 0.2.0 → 0.4.0

package/src/domain/pipeline/router-pipeline.ts

@@ -11,6 +11,8 @@
 
 import type {
   ModelProfile,
+  PlanningDelegateConfig,
+  PlanningDelegateObservability,
   PriceCatalog,
   RoutingDecision,
   RoutingFeatureSidecar,
@@ -49,6 +51,10 @@ import {
   estimateRoutingCost,
   enrichRoutingDecisionWithContextFit,
   enrichRoutingDecisionWithTierSelection,
+  createPlanningDelegateObservability,
+  PLANNING_DELEGATE,
+  PLANNING_DELEGATE_DISABLED,
+  PLANNING_DIRECT_FRONTIER,
 } from '../../infrastructure/telemetry/routing-telemetry.js';
 import type { HydraMatcher as HydraMatcherType, MatchResult } from '../matching/hydra-matcher.js';
 import type { ClusterMatcher, ClusterMatchResult } from '../matching/cluster-matcher.js';
@@ -58,6 +64,11 @@ import {
   buildTierFeatures,
   scoreLowIntensity,
 } from '../routing/tier-features.js';
+import {
+  applyIsotonicCalibratorTimed,
+  resolveIsotonicCalibrator,
+  type IsotonicCalibratorArtifact,
+} from '../routing/isotonic-calibrator.js';
 import {
   predictPSuccessCheapTimed,
   resolvePSuccessWeights,
@@ -168,8 +179,13 @@ export interface PipelineOptions {
   /** Preloaded P(success) weights for tests; lazy-loads artifact when omitted (SP-105). */
   readonly pSuccessWeights?: PSuccessWeights;
   readonly pSuccessWeightsPath?: string;
+  /** Preloaded isotonic calibrator for tests; lazy-loads bundle when omitted (SP-133). */
+  readonly isotonicCalibrator?: IsotonicCalibratorArtifact | null;
+  readonly routingCalibrationPath?: string;
   /** SAAR pin policy (SP-123). Must match sessionPinner.saarConfig when enabled. */
   readonly saarConfig?: SaarConfig;
+  /** Planning delegate operator knobs (SP-143, #71). Defaults to operator config. */
+  readonly planningDelegateConfig?: PlanningDelegateConfig;
 }
 
 // ─── Orchestrator ────────────────────────────────────────────────────────────
@@ -194,17 +210,23 @@ export class RouterPipeline {
   private currentTierHintReasonCode: string | null = null;
   private currentLowIntensityScore: number | null = null;
   private currentPSuccessCheap: number | null = null;
+  private currentPSuccessRaw: number | null = null;
+  private currentPSuccessCalibrated: number | null = null;
   private currentPSuccessAlpha: number | null = null;
   private currentExpectedCostByTier: ExpectedCostBreakdown[] | null = null;
   private currentLocalEligibleReason: string | null = null;
   private pSuccessWeightsLoaded = false;
   private cachedPSuccessWeights: PSuccessWeights | null = null;
+  private isotonicCalibratorLoaded = false;
+  private cachedIsotonicCalibrator: IsotonicCalibratorArtifact | null = null;
   private currentContextFitRejected: readonly CandidateScore[] = [];
   private currentContextFitViableCount = 0;
   private contextOverflowPreferredProvider: string | null = null;
   private contextOverflowTriggered = false;
   /** Internal breakeven gate reason for SP-126 explain wiring. */
   private currentBreakevenReason: string | null = null;
+  /** Planning delegate observability for SP-143 explain/telemetry wiring. */
+  private currentPlanningDelegate: PlanningDelegateObservability | null = null;
 
   constructor(fleet: readonly ModelProfile[], options?: PipelineOptions) {
     this.fleet = fleet;
@@ -243,6 +265,8 @@ export class RouterPipeline {
     this.currentTierHintReasonCode = null;
     this.currentLowIntensityScore = null;
     this.currentPSuccessCheap = null;
+    this.currentPSuccessRaw = null;
+    this.currentPSuccessCalibrated = null;
     this.currentPSuccessAlpha = null;
     this.currentExpectedCostByTier = null;
     this.currentLocalEligibleReason = null;
@@ -251,6 +275,7 @@ export class RouterPipeline {
     this.contextOverflowPreferredProvider = null;
     this.contextOverflowTriggered = false;
     this.currentBreakevenReason = null;
+    this.currentPlanningDelegate = null;
 
     let currentStage: NamedPipelineStage | undefined;
 
@@ -341,8 +366,13 @@ export class RouterPipeline {
       tier_hint_reason_code: this.currentTierHintReasonCode,
       low_intensity_score: this.currentLowIntensityScore,
       p_success_cheap: this.currentPSuccessCheap,
+      p_success_raw: this.currentPSuccessRaw,
+      p_success_calibrated: this.currentPSuccessCalibrated,
       p_success_alpha: this.currentPSuccessAlpha,
       local_eligible_reason: this.currentLocalEligibleReason,
+      ...(this.currentPlanningDelegate
+        ? { planning_delegate: this.currentPlanningDelegate }
+        : {}),
     };
 
     const withBaseFeatures = { ...decision, features };
@@ -945,54 +975,155 @@ export class RouterPipeline {
     const tierCandidates = this.activeFleet.filter(
       (m) => m.tier === targetTier && m.healthy !== false,
     );
-    const model = selectLowestCostModel(tierCandidates);
-    if (!model) {
+    const targetModel = selectLowestCostModel(tierCandidates);
+    if (!targetModel) {
       return { decided: false, stage: 'turn_envelope' };
     }
 
     const pinner = this.options.sessionPinner;
     const pin = pinner?.getPin(request.session_id) ?? null;
-    if (pin) {
-      const pinnedModel = this.activeFleet.find(
-        (m) => m.id === pin.pinned_model_id && m.healthy !== false,
+    const pinnedModel =
+      pin !== null
+        ? this.activeFleet.find(
+            (m) => m.id === pin.pinned_model_id && m.healthy !== false,
+          )
+        : undefined;
+
+    // SP-143: cache-preserving planning delegate when warm economical pin would switch to frontier.
+    if (
+      turnType === 'planning' &&
+      pinnedModel &&
+      pinnedModel.tier === 'economical-cloud' &&
+      pinnedModel.id !== targetModel.id
+    ) {
+      const delegateDecision = this.tryPlanningDelegateDecision(
+        request,
+        pinnedModel,
+        targetModel,
       );
-      if (pinnedModel && pinnedModel.id !== model.id) {
-        const skipBreakeven =
-          turnType === 'planning' && this.isSaarPlanningBufferActive(request);
-        if (!skipBreakeven) {
-          const tokenEstimate =
-            request.estimated_input_tokens ?? request.prompt_text.length;
-          const breakeven = evaluateModelSwitchBreakeven(
-            pinnedModel,
-            model,
-            tokenEstimate,
-            tokenEstimate,
-            this.options.saarConfig,
-          );
-          if (!breakeven.shouldSwitch) {
-            this.currentBreakevenReason = 'breakeven_blocked';
-            return { decided: false, stage: 'turn_envelope' };
-          }
-          this.currentBreakevenReason = 'breakeven_pass';
+      if (delegateDecision) {
+        return delegateDecision;
+      }
+    }
+
+    if (pinnedModel && pinnedModel.id !== targetModel.id) {
+      const skipBreakeven =
+        turnType === 'planning' && this.isSaarPlanningBufferActive(request);
+      if (!skipBreakeven) {
+        const tokenEstimate =
+          request.estimated_input_tokens ?? request.prompt_text.length;
+        const breakeven = evaluateModelSwitchBreakeven(
+          pinnedModel,
+          targetModel,
+          tokenEstimate,
+          tokenEstimate,
+          this.options.saarConfig,
+        );
+        if (!breakeven.shouldSwitch) {
+          this.currentBreakevenReason = 'breakeven_blocked';
+          return { decided: false, stage: 'turn_envelope' };
         }
+        this.currentBreakevenReason = 'breakeven_pass';
       }
     }
 
+    const directReasonCode = `turn_${turnType}`;
+    let planningDirectFallback: string | null = null;
+
+    if (
+      turnType === 'planning' &&
+      pinnedModel &&
+      pinnedModel.tier === 'economical-cloud' &&
+      pinnedModel.id !== targetModel.id
+    ) {
+      planningDirectFallback = this.resolvePlanningDirectFallbackReason();
+    }
+
+    if (planningDirectFallback) {
+      this.setPlanningDelegateDirectFallback(targetModel.id, planningDirectFallback);
+    }
+
     return {
       decided: true,
       stage: 'turn_envelope',
-      decision: this.withEstimatedCost(request, model, {
+      decision: this.withEstimatedCost(request, targetModel, {
         request_id: request.request_id,
-        selected_model_id: model.id,
+        selected_model_id: targetModel.id,
         tier: targetTier,
         stage: 'turn_envelope',
-        reason_code: `turn_${turnType}`,
+        reason_code: planningDirectFallback
+          ? PLANNING_DIRECT_FRONTIER
+          : directReasonCode,
+        routing_latency_ms: 0,
+        pin_reason: null,
+      }),
+    };
+  }
+
+  private resolvePlanningDirectFallbackReason(): string | null {
+    const config = this.resolvePlanningDelegateConfig();
+    if (!config.enabled) {
+      return PLANNING_DELEGATE_DISABLED;
+    }
+    return null;
+  }
+
+  private resolvePlanningDelegateConfig(): PlanningDelegateConfig {
+    return (
+      this.options.planningDelegateConfig ??
+      DEFAULT_OPERATOR_CONFIG.planning_delegate
+    );
+  }
+
+  /**
+   * SP-143: emit planning_delegate when enabled; otherwise record direct fallback
+   * observability and return null so breakeven-gated direct frontier can proceed.
+   */
+  private tryPlanningDelegateDecision(
+    request: RoutingRequest,
+    pinnedModel: ModelProfile,
+    frontierModel: ModelProfile,
+  ): StageResult | null {
+    const config = this.resolvePlanningDelegateConfig();
+    if (!config.enabled) {
+      return null;
+    }
+
+    this.currentPlanningDelegate = createPlanningDelegateObservability({
+      path: 'delegate',
+      primary_model_id: pinnedModel.id,
+      delegate_model_id: frontierModel.id,
+      compressed_context: config.compressed_context,
+      planning_delegate_reason_code: PLANNING_DELEGATE,
+    });
+
+    return {
+      decided: true,
+      stage: 'turn_envelope',
+      decision: this.withEstimatedCost(request, pinnedModel, {
+        request_id: request.request_id,
+        selected_model_id: pinnedModel.id,
+        tier: pinnedModel.tier,
+        stage: 'turn_envelope',
+        reason_code: PLANNING_DELEGATE,
         routing_latency_ms: 0,
         pin_reason: null,
       }),
     };
   }
 
+  private setPlanningDelegateDirectFallback(
+    delegateModelId: string | null,
+    fallbackReason: string,
+  ): void {
+    this.currentPlanningDelegate = createPlanningDelegateObservability({
+      path: 'direct',
+      delegate_model_id: delegateModelId,
+      planning_delegate_reason_code: PLANNING_DIRECT_FRONTIER,
+      fallback_reason: fallbackReason,
+    });
+  }
+
   // ─── Low-intensity tier gate (SP-103, #58) ───────────────────────────────
 
   /**
@@ -1027,7 +1158,13 @@ export class RouterPipeline {
     const weights = this.resolvePSuccessWeights();
     const pFeatures = tierFeaturesToPSuccessFeatures(tierFeatures);
     const pResult = predictPSuccessCheapTimed(pFeatures, weights);
-    this.currentPSuccessCheap = pResult.probability;
+    const calibrator = this.resolveIsotonicCalibrator();
+    const calibratedResult = applyIsotonicCalibratorTimed(pResult.probability, calibrator);
+    const pSuccessForGate = calibratedResult.calibrated;
+
+    this.currentPSuccessRaw = pResult.probability;
+    this.currentPSuccessCalibrated = pSuccessForGate;
+    this.currentPSuccessCheap = pSuccessForGate;
 
     const structuralHint = this.resolveTierHint(
       score,
@@ -1041,10 +1178,14 @@ export class RouterPipeline {
       ? (() => {
           const selection = this.selectExpectedCostTierHint(
             request,
-            pResult.probability,
+            pSuccessForGate,
             alpha,
           );
-          this.logExpectedCostExplain(pResult.probability, alpha, selection);
+          this.logExpectedCostExplain(pSuccessForGate, alpha, selection, {
+            p_success_raw: pResult.probability,
+            p_success_calibrated: pSuccessForGate,
+            calibration_applied: calibratedResult.calibration_applied,
+          });
           return {
             tierHint: selection.tierHint,
             reasonCode: selection.reasonCode,
@@ -1106,10 +1247,18 @@ export class RouterPipeline {
       tierCosts: readonly ExpectedCostBreakdown[];
       rationale: string;
     },
+    calibration?: {
+      readonly p_success_raw: number;
+      readonly p_success_calibrated: number;
+      readonly calibration_applied: boolean;
+    },
   ): void {
     console.info('Expected-cost tier gate', {
       reason: selection.reasonCode,
       p_success_cheap: pSuccessCheap,
+      p_success_raw: calibration?.p_success_raw ?? pSuccessCheap,
+      p_success_calibrated: calibration?.p_success_calibrated ?? pSuccessCheap,
+      calibration_applied: calibration?.calibration_applied ?? false,
       alpha,
       chosen_tier: selection.tierHint,
       rationale: selection.rationale,
@@ -1140,6 +1289,23 @@ export class RouterPipeline {
     return this.cachedPSuccessWeights!;
   }
 
+  private resolveIsotonicCalibrator(): IsotonicCalibratorArtifact | null {
+    if (this.options.isotonicCalibrator !== undefined) {
+      return this.options.isotonicCalibrator;
+    }
+
+    if (!this.isotonicCalibratorLoaded) {
+      this.cachedIsotonicCalibrator = resolveIsotonicCalibrator({
+        ...(this.options.routingCalibrationPath !== undefined
+          ? { filePath: this.options.routingCalibrationPath }
+          : {}),
+      });
+      this.isotonicCalibratorLoaded = true;
+    }
+
+    return this.cachedIsotonicCalibrator;
+  }
+
   private resolveTierHint(
     score: number,
     highThreshold: number,

package/src/domain/routing/isotonic-calibrator.ts

@@ -0,0 +1,255 @@
+/**
+ * Online isotonic P(success) calibrator for low_intensity gate (SP-133).
+ *
+ * Loads monotonic knot tables from routing-calibration bundle at serve time.
+ * O(log n) binary search lookup with <5ms budget; falls back to raw logistic
+ * when the artifact is missing or under-trained.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+import { z } from 'zod';
+
+import { MIN_TRAINING_SAMPLES } from './p-success-classifier.js';
+
+export const ISOTONIC_CALIBRATOR_ARTIFACT_VERSION = 1 as const;
+export const ISOTONIC_LOOKUP_BUDGET_MS = 5;
+export const DEFAULT_ROUTING_CALIBRATION_PATH = resolve('config', 'routing-calibration.json');
+
+export interface IsotonicCalibratorArtifact {
+  readonly version: typeof ISOTONIC_CALIBRATOR_ARTIFACT_VERSION;
+  readonly min_training_samples: number;
+  readonly x_knots: readonly number[];
+  readonly y_knots: readonly number[];
+  readonly trained_sample_count: number;
+  readonly holdout_ece_raw: number | null;
+  readonly holdout_ece_calibrated: number | null;
+}
+
+export interface IsotonicCalibratorResult {
+  readonly calibrated: number;
+  readonly elapsed_ms: number;
+  readonly within_budget: boolean;
+  readonly calibration_applied: boolean;
+}
+
+export interface LoadIsotonicCalibratorOptions {
+  readonly filePath?: string;
+}
+
+export class IsotonicCalibratorLoaderError extends Error {
+  override readonly name = 'IsotonicCalibratorLoaderError';
+
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+  }
+}
+
+const IsotonicCalibratorArtifactSchema = z.object({
+  version: z.literal(ISOTONIC_CALIBRATOR_ARTIFACT_VERSION),
+  min_training_samples: z.number().int().min(0),
+  x_knots: z.array(z.number().finite().min(0).max(1)).min(2),
+  y_knots: z.array(z.number().finite().min(0).max(1)).min(2),
+  trained_sample_count: z.number().int().min(0),
+  holdout_ece_raw: z.number().finite().min(0).max(1).nullable(),
+  holdout_ece_calibrated: z.number().finite().min(0).max(1).nullable(),
+});
+
+function clamp01(value: number): number {
+  if (value <= 0) return 0;
+  if (value >= 1) return 1;
+  return value;
+}
+
+function formatZodIssues(error: { issues: readonly { path: readonly PropertyKey[]; message: string }[] }): string {
+  return error.issues
+    .map((issue) => `  - ${issue.path.join('.')}: ${issue.message}`)
+    .join('\n');
+}
+
+/** True when artifact has enough labeled samples for serve-time calibration. */
+export function isIsotonicCalibratorTrained(artifact: IsotonicCalibratorArtifact): boolean {
+  return artifact.trained_sample_count >= artifact.min_training_samples;
+}
+
+export function createDefaultIsotonicCalibratorArtifact(): IsotonicCalibratorArtifact {
+  return {
+    version: ISOTONIC_CALIBRATOR_ARTIFACT_VERSION,
+    min_training_samples: MIN_TRAINING_SAMPLES,
+    x_knots: [0, 1],
+    y_knots: [0, 1],
+    trained_sample_count: 0,
+    holdout_ece_raw: null,
+    holdout_ece_calibrated: null,
+  };
+}
+
+/** Piecewise-constant isotonic lookup with O(log n) binary search on knots. */
+export function applyIsotonicLookup(
+  rawScore: number,
+  xKnots: readonly number[],
+  yKnots: readonly number[],
+): number {
+  if (xKnots.length === 0 || yKnots.length === 0 || xKnots.length !== yKnots.length) {
+    return clamp01(rawScore);
+  }
+
+  const score = clamp01(rawScore);
+  if (score <= xKnots[0]!) {
+    return clamp01(yKnots[0]!);
+  }
+
+  const lastIndex = xKnots.length - 1;
+  if (score >= xKnots[lastIndex]!) {
+    return clamp01(yKnots[lastIndex]!);
+  }
+
+  let low = 0;
+  let high = lastIndex;
+  while (low < high) {
+    const mid = Math.floor((low + high + 1) / 2);
+    if (xKnots[mid]! <= score) {
+      low = mid;
+    } else {
+      high = mid - 1;
+    }
+  }
+
+  return clamp01(yKnots[low]!);
+}
+
+/**
+ * Apply isotonic calibration to a raw logistic P(success) score.
+ * Returns the raw score when the artifact is missing or under-trained.
+ */
+export function applyIsotonicCalibrator(
+  rawScore: number,
+  artifact: IsotonicCalibratorArtifact | null,
+): number {
+  if (artifact === null || !isIsotonicCalibratorTrained(artifact)) {
+    return clamp01(rawScore);
+  }
+
+  return applyIsotonicLookup(rawScore, artifact.x_knots, artifact.y_knots);
+}
+
+/** Apply isotonic calibration with elapsed timing guard for the online routing budget. */
+export function applyIsotonicCalibratorTimed(
+  rawScore: number,
+  artifact: IsotonicCalibratorArtifact | null,
+  budgetMs: number = ISOTONIC_LOOKUP_BUDGET_MS,
+): IsotonicCalibratorResult {
+  const start = performance.now();
+  const calibration_applied = artifact !== null && isIsotonicCalibratorTrained(artifact);
+  const calibrated = applyIsotonicCalibrator(rawScore, artifact);
+  const elapsed_ms = performance.now() - start;
+
+  if (elapsed_ms > budgetMs) {
+    console.warn('Isotonic P(success) lookup exceeded latency budget', {
+      elapsed_ms,
+      budget_ms: budgetMs,
+      knot_count: artifact?.x_knots.length ?? 0,
+    });
+  }
+
+  return {
+    calibrated,
+    elapsed_ms,
+    within_budget: elapsed_ms <= budgetMs,
+    calibration_applied,
+  };
+}
+
+/** Parse and validate an isotonic calibrator artifact from JSON text. */
+export function parseIsotonicCalibratorJson(raw: string): IsotonicCalibratorArtifact {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new IsotonicCalibratorLoaderError(`Failed to parse JSON: ${message}`, { cause: err });
+  }
+
+  const result = IsotonicCalibratorArtifactSchema.safeParse(parsed);
+  if (!result.success) {
+    throw new IsotonicCalibratorLoaderError(
+      `Invalid isotonic calibrator artifact:\n${formatZodIssues(result.error)}`,
+      { cause: result.error },
+    );
+  }
+
+  return result.data;
+}
+
+/** Extract isotonic_calibrator from a routing-calibration bundle object. */
+export function parseIsotonicCalibratorFromBundle(parsed: unknown): IsotonicCalibratorArtifact {
+  if (typeof parsed !== 'object' || parsed === null) {
+    throw new IsotonicCalibratorLoaderError('Routing calibration bundle must be a JSON object');
+  }
+
+  const isotonic = (parsed as Record<string, unknown>).isotonic_calibrator;
+  if (isotonic === undefined) {
+    throw new IsotonicCalibratorLoaderError('Routing calibration bundle missing isotonic_calibrator');
+  }
+
+  const result = IsotonicCalibratorArtifactSchema.safeParse(isotonic);
+  if (!result.success) {
+    throw new IsotonicCalibratorLoaderError(
+      `Invalid isotonic_calibrator in routing calibration bundle:\n${formatZodIssues(result.error)}`,
+      { cause: result.error },
+    );
+  }
+
+  return result.data;
+}
+
+/**
+ * Load isotonic calibrator from routing-calibration bundle on disk.
+ * Returns null when the bundle file is missing.
+ */
+export function loadIsotonicCalibrator(
+  options?: LoadIsotonicCalibratorOptions,
+): IsotonicCalibratorArtifact | null {
+  const filePath = options?.filePath ?? DEFAULT_ROUTING_CALIBRATION_PATH;
+
+  if (!existsSync(filePath)) {
+    return null;
+  }
+
+  let raw: string;
+  try {
+    raw = readFileSync(filePath, 'utf8');
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new IsotonicCalibratorLoaderError(`Failed to read routing calibration file: ${message}`, {
+      cause: err,
+    });
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new IsotonicCalibratorLoaderError(`Failed to parse routing calibration JSON: ${message}`, {
+      cause: err,
+    });
+  }
+
+  return parseIsotonicCalibratorFromBundle(parsed);
+}
+
+/** Resolve calibrator for online inference — missing or invalid artifacts fall back safely. */
+export function resolveIsotonicCalibrator(
+  options?: LoadIsotonicCalibratorOptions,
+): IsotonicCalibratorArtifact | null {
+  try {
+    return loadIsotonicCalibrator(options);
+  } catch (err: unknown) {
+    console.warn('Isotonic calibrator artifact invalid; using raw logistic fallback', {
+      error: err instanceof Error ? err.message : String(err),
+    });
+    return null;
+  }
+}