pi-smart-router 0.1.0

package/.pi/extensions/smart-router/command-formatters.ts

@@ -0,0 +1,167 @@
+import {
+  DEFAULT_HISTORY_LIMIT,
+  MAX_HISTORY_LIMIT,
+} from '../../../src/infrastructure/telemetry/telemetry-limits.js';
+import type { RoutingDecision, RoutingTelemetry } from '../../../src/domain/types/index.js';
+import { SMART_ROUTER_USAGE } from './commands.js';
+import {
+  DEFAULT_TELEMETRY_CONTRIB_EXPORT_LIMIT,
+  parseExportTelemetryContribArgs,
+} from '../../../src/cli/smart-router-cli.js';
+import {
+  DEFAULT_DATASET_EXPORT_LIMIT,
+  MAX_DATASET_EXPORT_LIMIT,
+} from './dataset-export.js';
+import { formatPricingStalenessLine } from './pricing-lifecycle.js';
+import type { FleetMode, SmartRouterCommand, SmartRouterRuntime } from './types.js';
+
+export function parseHistoryLimit(raw: string | undefined): number {
+  if (raw === undefined) {
+    return DEFAULT_HISTORY_LIMIT;
+  }
+
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isFinite(parsed) || parsed < 1) {
+    throw new Error(`Usage: ${SMART_ROUTER_USAGE}`);
+  }
+
+  return Math.min(parsed, MAX_HISTORY_LIMIT);
+}
+
+export function parseExportLimit(tokens: string[]): number {
+  let limit = DEFAULT_DATASET_EXPORT_LIMIT;
+
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    if (token === '--limit') {
+      const raw = tokens[i + 1];
+      if (raw === undefined) {
+        throw new Error(`Usage: ${SMART_ROUTER_USAGE}`);
+      }
+      const parsed = Number.parseInt(raw, 10);
+      if (!Number.isFinite(parsed) || parsed < 1) {
+        throw new Error(`Usage: ${SMART_ROUTER_USAGE}`);
+      }
+      limit = Math.min(parsed, MAX_DATASET_EXPORT_LIMIT);
+      i += 1;
+      continue;
+    }
+
+    if (token?.startsWith('--limit=')) {
+      const raw = token.slice('--limit='.length);
+      const parsed = Number.parseInt(raw, 10);
+      if (!Number.isFinite(parsed) || parsed < 1) {
+        throw new Error(`Usage: ${SMART_ROUTER_USAGE}`);
+      }
+      limit = Math.min(parsed, MAX_DATASET_EXPORT_LIMIT);
+      continue;
+    }
+
+    throw new Error(`Usage: ${SMART_ROUTER_USAGE}`);
+  }
+
+  return limit;
+}
+
+export function parseSmartRouterArgs(args: string): SmartRouterCommand {
+  const tokens = args.trim().split(/\s+/).filter(Boolean);
+  if (tokens.length === 0 || tokens[0] === 'status') {
+    return { command: 'status' };
+  }
+
+  if (tokens[0] === 'history') {
+    return { command: 'history', limit: parseHistoryLimit(tokens[1]) };
+  }
+
+  if (tokens[0] === 'mode' && (tokens[1] === 'scoped' || tokens[1] === 'all')) {
+    return { command: 'mode', mode: tokens[1] };
+  }
+
+  if (tokens[0] === 'pricing' && tokens[1] === 'refresh') {
+    return { command: 'pricing', subcommand: 'refresh' };
+  }
+
+  if (tokens[0] === 'export' && tokens[1] === 'dataset') {
+    return {
+      command: 'export',
+      subcommand: 'dataset',
+      limit: parseExportLimit(tokens.slice(2)),
+    };
+  }
+
+  if (tokens[0] === 'export' && tokens[1] === 'telemetry-contrib') {
+    const { limit } = parseExportTelemetryContribArgs(tokens.join(' '));
+    return {
+      command: 'export',
+      subcommand: 'telemetry-contrib',
+      limit: Math.min(limit, DEFAULT_TELEMETRY_CONTRIB_EXPORT_LIMIT),
+    };
+  }
+
+  if (tokens[0] === 'feedback' && (tokens[1] === 'good' || tokens[1] === 'bad')) {
+    return { command: 'feedback', rating: tokens[1] };
+  }
+
+  if (tokens[0] === 'unpin' && tokens.length === 1) {
+    return { command: 'unpin' };
+  }
+
+  throw new Error(`Usage: ${SMART_ROUTER_USAGE}`);
+}
+
+export function formatStatusMessage(
+  runtime: SmartRouterRuntime,
+  decision: RoutingDecision | undefined,
+): string {
+  const lines = [
+    `Fleet mode: ${runtime.fleetMode}`,
+    `Fleet size: ${runtime.streamDeps.fleet.length}`,
+  ];
+
+  const fleetMembers = runtime.streamDeps.fleet
+    .map((profile) => `${profile.provider}/${profile.id}`)
+    .sort();
+  if (fleetMembers.length > 0) {
+    lines.push('Fleet members:');
+    for (const member of fleetMembers) {
+      lines.push(`  - ${member}`);
+    }
+  } else {
+    lines.push('Fleet members: (none)');
+  }
+
+  const stalenessLine = formatPricingStalenessLine(runtime.priceCatalog);
+  if (stalenessLine) {
+    lines.push(`Pricing: ${stalenessLine}`);
+  } else if (runtime.priceCatalog) {
+    lines.push(`Pricing: fresh (last_updated ${runtime.priceCatalog.last_updated})`);
+  }
+
+  if (!decision) {
+    lines.push('Last routing decision: (none yet)');
+    return lines.join('\n');
+  }
+
+  lines.push(
+    `Model: ${decision.selected_model_id}`,
+    `Stage: ${decision.stage}`,
+    `Reason: ${decision.reason_code}`,
+    `Latency: ${decision.routing_latency_ms}ms`,
+  );
+  return lines.join('\n');
+}
+
+export function formatHistoryMessage(entries: readonly RoutingTelemetry[]): string {
+  if (entries.length === 0) {
+    return 'No routing history yet.';
+  }
+
+  return entries
+    .map(
+      (entry) =>
+        `${entry.timestamp} | ${entry.selected_model_id} | ${entry.stage} | ${entry.turn_type} | ${entry.routing_latency_ms}ms`,
+    )
+    .join('\n');
+}
+
+export type { FleetMode };

package/.pi/extensions/smart-router/commands.ts

@@ -0,0 +1,242 @@
+import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
+
+import {
+  formatHistoryMessage,
+  formatStatusMessage,
+  parseSmartRouterArgs,
+} from './command-formatters.js';
+import { exportDatasetToFile } from './dataset-export.js';
+import { exportTelemetryContrib } from '../../../src/cli/smart-router-cli.js';
+import { bindSharedModelRegistry, rebuildFleet } from './fleet-bootstrap.js';
+import { refreshPricingCatalog } from './pricing-lifecycle.js';
+import { FLEET_MODE_ENTRY_TYPE } from './session-lifecycle.js';
+import type { SmartRouterRuntime } from './types.js';
+
+export const SMART_ROUTER_USAGE =
+  '/smart-router [status] | history [limit] | mode scoped|all | pricing refresh | export dataset [--limit N] | export telemetry-contrib [--limit N] | feedback good|bad | unpin';
+
+type CompletionItem = { value: string; label: string };
+
+const TOP_LEVEL: CompletionItem[] = [
+  { value: 'status', label: 'Show last routing decision' },
+  { value: 'history', label: 'Show recent routing history' },
+  { value: 'mode', label: 'Switch fleet mode (scoped or all)' },
+  { value: 'pricing', label: 'Manage pricing catalog' },
+  { value: 'export', label: 'Export opt-in routing dataset' },
+  { value: 'feedback', label: 'Label last routing outcome good or bad' },
+  { value: 'unpin', label: 'Clear current session pin' },
+];
+
+const MODE_COMPLETIONS: CompletionItem[] = [
+  { value: 'mode scoped', label: 'Route among scoped models only' },
+  { value: 'mode all', label: 'Route among all authenticated models' },
+];
+
+const PRICING_COMPLETIONS: CompletionItem[] = [
+  { value: 'pricing refresh', label: 'Fetch LiteLLM rates and rebuild fleet' },
+];
+
+const EXPORT_COMPLETIONS: CompletionItem[] = [
+  { value: 'export dataset', label: 'Export privacy-safe dataset JSONL' },
+  { value: 'export telemetry-contrib', label: 'Export community telemetry JSON' },
+];
+
+const FEEDBACK_COMPLETIONS: CompletionItem[] = [
+  { value: 'feedback good', label: 'Mark last routing outcome as good' },
+  { value: 'feedback bad', label: 'Mark last routing outcome as bad' },
+];
+
+/** Full invocations used to keep completions and parseSmartRouterArgs in sync. */
+export const SMART_ROUTER_FULL_INVOCATIONS = [
+  '',
+  'status',
+  'history',
+  'history 10',
+  'mode scoped',
+  'mode all',
+  'pricing refresh',
+  'export dataset',
+  'export dataset --limit 100',
+  'export telemetry-contrib',
+  'export telemetry-contrib --limit 100',
+  'feedback good',
+  'feedback bad',
+  'unpin',
+] as const;
+
+function filterByPrefix(items: CompletionItem[], prefix: string): CompletionItem[] {
+  return items.filter((item) => item.value.startsWith(prefix));
+}
+
+export function getSmartRouterArgumentCompletions(prefix: string): CompletionItem[] | null {
+  const trimmed = prefix.trimStart();
+  const tokens = trimmed.split(/\s+/).filter(Boolean);
+
+  if (tokens[0] === 'mode') {
+    const subPrefix = tokens.slice(1).join(' ');
+    const filtered = filterByPrefix(MODE_COMPLETIONS, `mode${subPrefix ? ` ${subPrefix}` : ''}`);
+    return filtered.length > 0 ? filtered : null;
+  }
+
+  if (tokens[0] === 'pricing') {
+    const subPrefix = tokens.slice(1).join(' ');
+    const filtered = filterByPrefix(PRICING_COMPLETIONS, `pricing${subPrefix ? ` ${subPrefix}` : ''}`);
+    return filtered.length > 0 ? filtered : null;
+  }
+
+  if (tokens[0] === 'export') {
+    const subPrefix = tokens.slice(1).join(' ');
+    const filtered = filterByPrefix(EXPORT_COMPLETIONS, `export${subPrefix ? ` ${subPrefix}` : ''}`);
+    return filtered.length > 0 ? filtered : null;
+  }
+
+  if (tokens[0] === 'feedback') {
+    const subPrefix = tokens.slice(1).join(' ');
+    const filtered = filterByPrefix(FEEDBACK_COMPLETIONS, `feedback${subPrefix ? ` ${subPrefix}` : ''}`);
+    return filtered.length > 0 ? filtered : null;
+  }
+
+  if (tokens[0] === 'history') {
+    return [{ value: 'history', label: 'Show recent routing history' }];
+  }
+
+  const firstToken = tokens[0] ?? '';
+  const filtered = filterByPrefix(TOP_LEVEL, firstToken);
+  return filtered.length > 0 ? filtered : null;
+}
+
+export function registerSmartRouterCommand(
+  pi: ExtensionAPI,
+  runtime: SmartRouterRuntime,
+): void {
+  pi.registerCommand('smart-router', {
+    description:
+      'Show routing status/history, switch fleet mode (scoped|all), refresh pricing, or export dataset',
+    getArgumentCompletions: getSmartRouterArgumentCompletions,
+    handler: async (args, ctx) => {
+      try {
+        bindSharedModelRegistry(runtime, ctx.modelRegistry);
+        const parsed = parseSmartRouterArgs(args);
+
+        if (parsed.command === 'status') {
+          ctx.ui.notify(formatStatusMessage(runtime, runtime.lastDecision), 'info');
+          return;
+        }
+
+        if (parsed.command === 'history') {
+          const rows = await runtime.store.listTelemetry({ limit: parsed.limit });
+          ctx.ui.notify(formatHistoryMessage(rows), 'info');
+          return;
+        }
+
+        if (parsed.command === 'pricing') {
+          const { modelCount, lastUpdated } = await refreshPricingCatalog(runtime);
+          await rebuildFleet(runtime, pi, ctx.cwd);
+          ctx.ui.notify(
+            `Pricing refreshed: ${modelCount} models loaded (last_updated: ${lastUpdated}). Fleet rebuilt (${runtime.streamDeps.fleet.length} models).`,
+            'info',
+          );
+          return;
+        }
+
+        if (parsed.command === 'export' && parsed.subcommand === 'dataset') {
+          const result = await exportDatasetToFile(runtime.store, ctx.cwd, parsed.limit);
+          if (!result) {
+            ctx.ui.notify('No routing dataset records to export.', 'info');
+            return;
+          }
+          ctx.ui.notify(
+            `Exported ${result.recordCount} dataset record(s) to ${result.path}`,
+            'info',
+          );
+          return;
+        }
+
+        if (parsed.command === 'export' && parsed.subcommand === 'telemetry-contrib') {
+          const result = await exportTelemetryContrib({
+            store: runtime.store,
+            cwd: ctx.cwd,
+            limit: parsed.limit,
+          });
+          if (!result.path) {
+            ctx.ui.notify(
+              result.recordCount === 0
+                ? 'No telemetry-contrib records to export (opt in with SMART_ROUTER_DATASET=1).'
+                : 'No telemetry-contrib records written.',
+              'info',
+            );
+            return;
+          }
+          ctx.ui.notify(
+            `Exported ${result.recordCount} telemetry-contrib record(s) to ${result.path}`,
+            'info',
+          );
+          return;
+        }
+
+        if (parsed.command === 'feedback') {
+          const sessionId = ctx.sessionManager.getSessionId();
+          const snapshot = runtime.sessionRouting.get(sessionId);
+          if (!snapshot) {
+            ctx.ui.notify('No recent auto-routed request to label.', 'info');
+            return;
+          }
+
+          const record = runtime.outcomeRecorder?.recordFeedback(
+            snapshot,
+            sessionId,
+            parsed.rating,
+          );
+          if (!record) {
+            ctx.ui.notify('Outcome labels require SMART_ROUTER_DATASET=1.', 'info');
+            return;
+          }
+
+          ctx.ui.notify(
+            `Recorded ${parsed.rating} feedback for request ${snapshot.lastRequestId}.`,
+            'info',
+          );
+          return;
+        }
+
+        if (parsed.command === 'unpin') {
+          const sessionId = ctx.sessionManager.getSessionId();
+          const sessionPinner = runtime.streamDeps.sessionPinner;
+          if (!sessionPinner) {
+            ctx.ui.notify('Session pinner unavailable.', 'error');
+            return;
+          }
+
+          const pin = sessionPinner.getPin(sessionId);
+          if (!pin) {
+            ctx.ui.notify('No session pin to clear.', 'info');
+            return;
+          }
+
+          sessionPinner.breakPin(sessionId);
+          ctx.ui.notify(
+            `Cleared session pin (was ${pin.pinned_model_id}). Next request will run full routing.`,
+            'info',
+          );
+          return;
+        }
+
+        if (parsed.mode === runtime.fleetMode) {
+          ctx.ui.notify(`Fleet mode already set to ${parsed.mode}`, 'info');
+          return;
+        }
+
+        runtime.fleetMode = parsed.mode;
+        await rebuildFleet(runtime, pi, ctx.cwd);
+        pi.appendEntry(FLEET_MODE_ENTRY_TYPE, { mode: parsed.mode });
+        ctx.ui.notify(
+          `Fleet mode set to ${parsed.mode} (${runtime.streamDeps.fleet.length} models)`,
+          'info',
+        );
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        ctx.ui.notify(message, 'error');
+      }
+    },
+  });
+}

package/.pi/extensions/smart-router/dataset-export.ts

@@ -0,0 +1,142 @@
+import { join } from 'node:path';
+import { mkdirSync, writeFileSync } from 'node:fs';
+
+import { createHash } from 'node:crypto';
+
+import {
+  attachOutcomeLabelsToExport,
+  indexOutcomesByRequestId,
+} from '../../../src/domain/routing/p-success-classifier.js';
+import { DATASET_MAX_ENTRIES } from '../../../src/infrastructure/telemetry/dataset-limits.js';
+import {
+  DatasetRecorder,
+  DATASET_ENABLED_NOTIFY_MESSAGE,
+} from '../../../src/infrastructure/telemetry/dataset-recorder.js';
+import { OutcomeRecorder } from '../../../src/infrastructure/telemetry/outcome-recorder.js';
+import type { RoutingDatasetRecord, RoutingOutcomeRecord } from '../../../src/domain/types/index.js';
+import type { StorePort } from '../../../src/domain/types/store-port.js';
+
+const DEFAULT_DATASET_EXPORT_DIR = '.pi-smart-router/exports';
+const DEFAULT_DATASET_EXPORT_LIMIT = DATASET_MAX_ENTRIES;
+const MAX_DATASET_EXPORT_LIMIT = DATASET_MAX_ENTRIES;
+
+export const DATASET_EXPORT_FORBIDDEN_KEYS = [
+  'session_id',
+  'prompt_text',
+  'messages',
+  'prompt',
+  'pepper',
+  'install_pepper',
+  'dataset_pepper',
+  'dataset_key',
+] as const;
+
+export function createExtensionDatasetRecorder(
+  store: StorePort,
+  cwdOrNotify?: string | ((message: string) => void),
+  notifyEnabled?: (message: string) => void,
+): DatasetRecorder {
+  let cwd = process.cwd();
+  let notify: ((message: string) => void) | undefined;
+
+  if (typeof cwdOrNotify === 'string') {
+    cwd = cwdOrNotify;
+    notify = notifyEnabled;
+  } else if (typeof cwdOrNotify === 'function') {
+    notify = cwdOrNotify;
+  }
+
+  return new DatasetRecorder({
+    cwd,
+    onRecord: (record) => {
+      store.appendDatasetRecord(record);
+    },
+    onFirstEnable: () => {
+      notify?.(DATASET_ENABLED_NOTIFY_MESSAGE);
+    },
+  });
+}
+
+export function createExtensionOutcomeRecorder(store: StorePort): OutcomeRecorder {
+  return new OutcomeRecorder({
+    onRecord: (record) => {
+      store.appendOutcomeRecord(record);
+    },
+  });
+}
+
+export function hashSessionIdForExport(sessionId: string): string {
+  return createHash('sha256').update(sessionId).digest('hex');
+}
+
+/** Map a stored dataset row to a privacy-safe JSON export object (Tier 1 only). */
+export function toDatasetExportRecord(
+  record: RoutingDatasetRecord,
+): Record<string, unknown> {
+  const raw = record as RoutingDatasetRecord & Record<string, unknown>;
+  const exportable: Record<string, unknown> = { ...record };
+
+  for (const key of DATASET_EXPORT_FORBIDDEN_KEYS) {
+    delete exportable[key];
+  }
+
+  const sessionId = raw.session_id;
+  if (typeof sessionId === 'string' && sessionId.length > 0) {
+    delete exportable.session_id;
+    exportable.session_id_hash = hashSessionIdForExport(sessionId);
+  }
+
+  return exportable;
+}
+
+export function formatDatasetExportJsonl(
+  records: readonly RoutingDatasetRecord[],
+  outcomeRecords: readonly RoutingOutcomeRecord[] = [],
+): string {
+  const outcomesByRequest = indexOutcomesByRequestId(outcomeRecords);
+
+  return records
+    .map((record) => {
+      const exportable = toDatasetExportRecord(record);
+      const linkedOutcomes = outcomesByRequest.get(record.request_id) ?? [];
+      return JSON.stringify(attachOutcomeLabelsToExport(exportable, linkedOutcomes));
+    })
+    .join('\n');
+}
+
+export function formatDatasetExportTimestamp(date: Date = new Date()): string {
+  return date.toISOString().replace(/[:.]/g, '-');
+}
+
+export function getDatasetExportPath(cwd: string, timestamp: string): string {
+  return join(cwd, DEFAULT_DATASET_EXPORT_DIR, `dataset-${timestamp}.jsonl`);
+}
+
+export interface DatasetExportResult {
+  readonly path: string;
+  readonly recordCount: number;
+}
+
+export async function exportDatasetToFile(
+  store: StorePort,
+  cwd: string,
+  limit: number,
+): Promise<DatasetExportResult | null> {
+  const records = await store.listDatasetRecords({ limit });
+  if (records.length === 0) {
+    return null;
+  }
+
+  const outcomeRecords = await store.listOutcomeRecords({ limit });
+
+  const timestamp = formatDatasetExportTimestamp();
+  const exportDir = join(cwd, DEFAULT_DATASET_EXPORT_DIR);
+  mkdirSync(exportDir, { recursive: true });
+  const exportPath = getDatasetExportPath(cwd, timestamp);
+  const jsonl = formatDatasetExportJsonl(records, outcomeRecords);
+  writeFileSync(exportPath, jsonl.length > 0 ? `${jsonl}\n` : '', 'utf8');
+
+  return { path: exportPath, recordCount: records.length };
+}
+
+export { DEFAULT_DATASET_EXPORT_LIMIT, MAX_DATASET_EXPORT_LIMIT };

package/.pi/extensions/smart-router/delegate-stream.ts

@@ -0,0 +1,105 @@
+import {
+  type Api,
+  type AssistantMessage,
+  type AssistantMessageEvent,
+  type Context,
+  type Model,
+  type SimpleStreamOptions,
+  streamSimple as defaultDelegateStream,
+} from '@earendil-works/pi-ai/compat';
+
+import { parseAssistantMessageError } from '../../../src/infrastructure/delegation/provider-error.js';
+import {
+  buildDelegationContext,
+  modelToExecutionModel,
+  resolveDelegationOptions,
+  type DelegatedStreamResult,
+  type DelegationHeadroomContext,
+} from './delegation-runtime.js';
+import type { StreamDelegationDeps } from './types.js';
+
+export async function collectDelegatedStream(
+  targetModel: Model<Api>,
+  context: Context,
+  deps: StreamDelegationDeps,
+  options: SimpleStreamOptions | undefined,
+  headroomContext?: DelegationHeadroomContext,
+): Promise<DelegatedStreamResult> {
+  if (options?.signal?.aborted) {
+    throw new Error('Request was aborted');
+  }
+
+  const delegationOptions = await resolveDelegationOptions(
+    deps.modelRegistry,
+    targetModel,
+    options,
+    headroomContext,
+  );
+  const delegateStream = deps.delegateStream ?? defaultDelegateStream;
+  const inner = delegateStream(targetModel, context, delegationOptions);
+  const events: AssistantMessageEvent[] = [];
+  let finalMessage: AssistantMessage | undefined;
+
+  for await (const event of inner) {
+    if (options?.signal?.aborted) {
+      throw new Error('Request was aborted');
+    }
+    events.push(event);
+
+    if (event.type === 'done') {
+      finalMessage = event.message;
+    } else if (event.type === 'error') {
+      finalMessage = event.error;
+    }
+  }
+
+  const failed =
+    finalMessage !== undefined &&
+    (finalMessage.stopReason === 'error' || finalMessage.stopReason === 'aborted');
+
+  return { finalMessage, failed, events };
+}
+
+export async function delegateWithOutcome(
+  targetModel: Model<Api>,
+  context: Context,
+  deps: StreamDelegationDeps,
+  options: SimpleStreamOptions | undefined,
+  sessionId: string | undefined,
+  headroomContext?: DelegationHeadroomContext,
+): Promise<DelegatedStreamResult> {
+  const delegationContext = buildDelegationContext(
+    context,
+    targetModel,
+    deps,
+    sessionId,
+  );
+
+  const result = await collectDelegatedStream(
+    targetModel,
+    delegationContext,
+    deps,
+    options,
+    headroomContext,
+  );
+
+  if (!result.finalMessage) {
+    return result;
+  }
+
+  if (result.failed) {
+    const parsed = parseAssistantMessageError(result.finalMessage);
+    deps.router.dispatch.recordOutcome(targetModel.id, parsed);
+  } else {
+    deps.router.dispatch.recordOutcome(targetModel.id);
+    if (sessionId) {
+      deps.executionLedger.recordSuccess(sessionId, modelToExecutionModel(targetModel));
+    }
+    deps.onDelegatedModel?.({
+      provider: targetModel.provider,
+      id: targetModel.id,
+    });
+  }
+
+  return result;
+}