tryaii-dre 0.2.0

package/dist/router.js

@@ -0,0 +1,259 @@
+/**
+ * Main Router -- the primary public API for TryAii-DRE.
+ *
+ * Usage:
+ *   import { Router } from 'tryaii-dre';
+ *
+ *   const router = new Router();
+ *   const result = await router.route('Write a Python function to merge sorted arrays');
+ *   console.log(result.bestModel);   // e.g., "gpt-5.2"
+ *   console.log(result.scores);      // Top models with scores and reasoning
+ *
+ * The default `LocalEmbeddingProvider` is async-only, so `route()` itself is
+ * async. Callers that have injected a sync embedding provider can use the
+ * niche `routeSync()` method for a blocking call.
+ */
+import { BenchmarkRegistry } from './benchmarks/registry.js';
+import { NormalizationRange } from './scoring/benchmarks.js';
+import { CentroidLoader } from './centroids/loader.js';
+import { EmbeddingClassifier } from './classifiers/embedding.js';
+import { createDefaultConfig, centroidFilePath } from './config.js';
+import { LocalEmbeddingProvider } from './embeddings/local.js';
+import { ModelRegistry } from './registry/models.js';
+import { ScoringEngine } from './scoring/engine.js';
+import { DEFAULT_PRIORITIES } from './scoring/priorities.js';
+/**
+ * Maximum allowed prompt length (characters). Prompts longer than this are
+ * truncated before classification to avoid OOM in the embedding model. Must
+ * stay in sync with the Python SDK's MAX_PROMPT_LENGTH (classifiers/base.py).
+ */
+export const MAX_PROMPT_LENGTH = 100000;
+/** Get list of model IDs in ranked order from a RouteResult. */
+export function routeResultTopK(result) {
+    return result.scores.map((s) => s.modelId);
+}
+/** Get the score of the top model. */
+export function routeResultBestScore(result) {
+    return result.scores[0]?.finalScore ?? 0;
+}
+/** Get the reasoning for why the top model was chosen. */
+export function routeResultBestReasoning(result) {
+    return result.scores[0]?.reasoning ?? '';
+}
+/**
+ * Semantic AI model router.
+ *
+ * Analyzes user prompts using embeddings, matches them against benchmark
+ * centroids, and recommends the best AI model based on benchmark performance,
+ * pricing, latency, and user priorities.
+ */
+export class Router {
+    constructor(opts) {
+        this._centroidLoader = null;
+        this._classifier = null;
+        this._config = createDefaultConfig(opts?.config);
+        // Model registry
+        this._registry = opts?.registry ?? ModelRegistry.default();
+        // Benchmark registry
+        this._benchmarkRegistry = opts?.benchmarkRegistry ?? BenchmarkRegistry.default();
+        // Scoring engine with normalizer from benchmark registry
+        const normalizer = this._benchmarkRegistry.getNormalizer();
+        this._scoringEngine = new ScoringEngine(normalizer);
+        // Embedding provider (lazy -- only initialized when needed)
+        this._embeddingProvider = opts?.embeddingProvider ?? null;
+    }
+    /**
+     * Lazy-initialize the embedding provider + centroid loader.
+     *
+     * The loader is stored on `this` so that `addBenchmark()` mutates the
+     * same instance the classifier reads from -- custom benchmarks added
+     * via the Router become visible to subsequent `route()` calls without
+     * re-instantiating anything.
+     */
+    _ensureCentroidLoader() {
+        if (this._centroidLoader !== null)
+            return this._centroidLoader;
+        if (this._embeddingProvider === null) {
+            this._embeddingProvider = new LocalEmbeddingProvider(`Xenova/${this._config.embeddingModel}`);
+        }
+        this._centroidLoader = new CentroidLoader(this._embeddingProvider, centroidFilePath(this._config));
+        return this._centroidLoader;
+    }
+    /** Lazy-initialize the embedding classifier on first use. */
+    _ensureClassifier() {
+        if (this._classifier !== null)
+            return this._classifier;
+        const centroidLoader = this._ensureCentroidLoader();
+        // `_embeddingProvider` is non-null after _ensureCentroidLoader returns.
+        const provider = this._embeddingProvider;
+        this._classifier = new EmbeddingClassifier(provider, centroidLoader, {
+            embeddingCacheSize: this._config.cache.embeddingCacheSize,
+            classificationCacheSize: this._config.cache.classificationCacheSize,
+            ttlSeconds: this._config.cache.ttlSeconds,
+        });
+        return this._classifier;
+    }
+    /**
+     * Route a prompt to the best AI model.
+     *
+     * Async by default -- works with any embedding provider, including the
+     * default async `LocalEmbeddingProvider`. For a blocking call backed by a
+     * sync provider, see `routeSync()`.
+     *
+     * @param prompt - The user's input text to classify and route.
+     * @param opts - Routing options (priorities, filters, topK).
+     * @returns RouteResult with the best model and full scoring breakdown.
+     */
+    async route(prompt, opts) {
+        prompt = Router._normalizePrompt(prompt);
+        const classifier = this._ensureClassifier();
+        const classification = await classifier.classifyAsync(prompt);
+        return this._buildResult(classification, opts);
+    }
+    /**
+     * Synchronous version of `route()`.
+     *
+     * Requires the injected embedding provider to support sync calls
+     * (`supportsSync === true`). Throws otherwise -- the default
+     * `LocalEmbeddingProvider` is async-only, so calling `routeSync()` on a
+     * default `Router` will fail. Inject a sync provider (e.g. a custom
+     * cached provider) to use this path.
+     */
+    routeSync(prompt, opts) {
+        prompt = Router._normalizePrompt(prompt);
+        const classifier = this._ensureClassifier();
+        if (this._embeddingProvider !== null && !this._embeddingProvider.supportsSync) {
+            throw new Error(`routeSync() requires an embedding provider that supports sync calls. ` +
+                `${this._embeddingProvider.constructor.name} is async-only -- ` +
+                `use route() (async) instead, or inject a sync provider.`);
+        }
+        const classification = classifier.classify(prompt);
+        return this._buildResult(classification, opts);
+    }
+    /**
+     * Validate and normalize a prompt before classification.
+     *
+     * Rejects empty/non-string prompts and truncates to MAX_PROMPT_LENGTH,
+     * matching the Python Router (which raises ValueError on empty/non-string
+     * and slices to the same limit) so both SDKs behave identically.
+     */
+    static _normalizePrompt(prompt) {
+        if (typeof prompt !== 'string' || prompt.length === 0) {
+            throw new Error('prompt must be a non-empty string');
+        }
+        if (prompt.length > MAX_PROMPT_LENGTH) {
+            return prompt.slice(0, MAX_PROMPT_LENGTH);
+        }
+        return prompt;
+    }
+    /** Shared post-classification path: filter models, score, return RouteResult. */
+    _buildResult(classification, opts) {
+        const priorities = opts?.priorities ?? DEFAULT_PRIORITIES;
+        const topK = opts?.topK ?? 5;
+        let models = this._registry.allModels;
+        if (opts?.filterProvider) {
+            const providerLower = opts.filterProvider.toLowerCase();
+            models = models.filter((m) => m.provider.toLowerCase() === providerLower);
+        }
+        if (opts?.filterCapability) {
+            const cap = opts.filterCapability;
+            models = models.filter((m) => m.capabilities.includes(cap));
+        }
+        if (opts?.filterMaxCost != null) {
+            const maxCost = opts.filterMaxCost;
+            models = models.filter((m) => m.pricing != null && m.pricing.inputPer1k <= maxCost);
+        }
+        if (models.length === 0) {
+            return {
+                bestModel: '',
+                scores: [],
+                classification,
+                priorities,
+            };
+        }
+        const scores = this._scoringEngine.scoreModels(models, classification.benchmarkScores, priorities, topK);
+        const best = scores[0]?.modelId ?? '';
+        return {
+            bestModel: best,
+            scores,
+            classification,
+            priorities,
+        };
+    }
+    /**
+     * Shortcut to add a model to the registry.
+     *
+     * @see ModelRegistry.add()
+     */
+    addModel(opts) {
+        return this._registry.add(opts);
+    }
+    /**
+     * Add a custom benchmark to the routing system.
+     *
+     * Async by default -- works with any embedding provider, including the
+     * async-only `LocalEmbeddingProvider`. The new centroid is generated
+     * immediately and added to the shared centroid loader, so subsequent
+     * `route()` calls see it without any restart.
+     *
+     * For sync-provider callers who want a blocking setup step, use
+     * `addBenchmarkSync()`.
+     *
+     * @param name - Benchmark name (e.g., "CustomerSupportQA").
+     * @param queries - Representative prompts for this benchmark (10-20 recommended).
+     * @param description - Human-readable description.
+     * @param minScore - Minimum score for normalization.
+     * @param maxScore - Maximum score for normalization.
+     */
+    async addBenchmark(name, queries, description = '', minScore = 0, maxScore = 100) {
+        this._registerBenchmark(name, queries, description, minScore, maxScore);
+        const loader = this._ensureCentroidLoader();
+        await loader.addBenchmarkCentroidAsync(name, queries);
+    }
+    /**
+     * Synchronous version of `addBenchmark()`.
+     *
+     * Requires the injected embedding provider to support sync calls
+     * (`supportsSync === true`). The default `LocalEmbeddingProvider` is
+     * async-only, so calling `addBenchmarkSync()` on a default `Router`
+     * will throw.
+     */
+    addBenchmarkSync(name, queries, description = '', minScore = 0, maxScore = 100) {
+        this._registerBenchmark(name, queries, description, minScore, maxScore);
+        const loader = this._ensureCentroidLoader();
+        if (this._embeddingProvider !== null && !this._embeddingProvider.supportsSync) {
+            throw new Error(`addBenchmarkSync() requires an embedding provider that supports sync calls. ` +
+                `${this._embeddingProvider.constructor.name} is async-only -- ` +
+                `use addBenchmark() (async) instead, or inject a sync provider.`);
+        }
+        loader.addBenchmarkCentroid(name, queries);
+    }
+    /** Register a benchmark in the registry and rebuild the scoring normalizer. */
+    _registerBenchmark(name, queries, description, minScore, maxScore) {
+        const benchmark = {
+            name,
+            description,
+            trainingQueries: queries,
+            normalization: new NormalizationRange(minScore, maxScore, description),
+            broadCategory: 'TECHNICAL',
+            subcategories: [],
+            metadata: {},
+        };
+        this._benchmarkRegistry.register(benchmark);
+        const normalizer = this._benchmarkRegistry.getNormalizer();
+        this._scoringEngine = new ScoringEngine(normalizer);
+    }
+    /** Access the model registry. */
+    get models() {
+        return this._registry;
+    }
+    /** Access the benchmark registry. */
+    get benchmarks() {
+        return this._benchmarkRegistry;
+    }
+    /** Access the configuration. */
+    get config() {
+        return this._config;
+    }
+}
+//# sourceMappingURL=router.js.map

package/dist/router.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,iBAAiB,EAAuB,MAAM,0BAA0B,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAmB,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAa,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAc,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAc,MAAM,yBAAyB,CAAC;AAGzE;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,MAAO,CAAC;AAqBzC,gEAAgE;AAChE,MAAM,UAAU,eAAe,CAAC,MAAmB;IACjD,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;AAC7C,CAAC;AAED,sCAAsC;AACtC,MAAM,UAAU,oBAAoB,CAAC,MAAmB;IACtD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,wBAAwB,CAAC,MAAmB;IAC1D,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;AAC3C,CAAC;AAgBD;;;;;;GAMG;AACH,MAAM,OAAO,MAAM;IASjB,YAAY,IAKX;QARO,oBAAe,GAA0B,IAAI,CAAC;QAC9C,gBAAW,GAA+B,IAAI,CAAC;QAQrD,IAAI,CAAC,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAEjD,iBAAiB;QACjB,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,QAAQ,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;QAE3D,qBAAqB;QACrB,IAAI,CAAC,kBAAkB,GAAG,IAAI,EAAE,iBAAiB,IAAI,iBAAiB,CAAC,OAAO,EAAE,CAAC;QAEjF,yDAAyD;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,EAAE,CAAC;QAC3D,IAAI,CAAC,cAAc,GAAG,IAAI,aAAa,CAAC,UAAU,CAAC,CAAC;QAEpD,4DAA4D;QAC5D,IAAI,CAAC,kBAAkB,GAAG,IAAI,EAAE,iBAAiB,IAAI,IAAI,CAAC;IAC5D,CAAC;IAED;;;;;;;OAOG;IACK,qBAAqB;QAC3B,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,eAAe,CAAC;QAE/D,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC,kBAAkB,GAAG,IAAI,sBAAsB,CAClD,UAAU,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CACxC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,cAAc,CACvC,IAAI,CAAC,kBAAkB,EACvB,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAC/B,CAAC;QAEF,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED,6DAA6D;IACrD,iBAAiB;QACvB,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAEvD,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACpD,wEAAwE;QACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAA2C,CAAC;QAElE,IAAI,CAAC,WAAW,GAAG,IAAI,mBAAmB,CACxC,QAAQ,EACR,cAAc,EACd;YACE,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB;YACzD,uBAAuB,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,uBAAuB;YACnE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU;SAC1C,CACF,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,MAAc,EAAE,IAAmB;QAC7C,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC5C,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,SAAS,CAAC,MAAc,EAAE,IAAmB;QAC3C,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,CAAC;YAC9E,MAAM,IAAI,KAAK,CACb,uEAAuE;gBACrE,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,oBAAoB;gBAC/D,yDAAyD,CAC5D,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,gBAAgB,CAAC,MAAc;QAC5C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;YACtC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,iFAAiF;IACzE,YAAY,CAClB,cAAoC,EACpC,IAAmB;QAEnB,MAAM,UAAU,GAAG,IAAI,EAAE,UAAU,IAAI,kBAAkB,CAAC;QAC1D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;QAE7B,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;QACtC,IAAI,IAAI,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC;YACxD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC,CAAC;QAC5E,CAAC;QACD,IAAI,IAAI,EAAE,gBAAgB,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAClC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,EAAE,aAAa,IAAI,IAAI,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC;YACnC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,SAAS,EAAE,EAAE;gBACb,MAAM,EAAE,EAAE;gBACV,cAAc;gBACd,UAAU;aACX,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,CAC5C,MAAM,EACN,cAAc,CAAC,eAAe,EAC9B,UAAU,EACV,IAAI,CACL,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC;QAEtC,OAAO;YACL,SAAS,EAAE,IAAI;YACf,MAAM;YACN,cAAc;YACd,UAAU;SACX,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,QAAQ,CAAC,IAQR;QACC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAChB,IAAY,EACZ,OAAiB,EACjB,WAAW,GAAG,EAAE,EAChB,QAAQ,GAAG,CAAC,EACZ,QAAQ,GAAG,GAAG;QAEd,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC5C,MAAM,MAAM,CAAC,yBAAyB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;OAOG;IACH,gBAAgB,CACd,IAAY,EACZ,OAAiB,EACjB,WAAW,GAAG,EAAE,EAChB,QAAQ,GAAG,CAAC,EACZ,QAAQ,GAAG,GAAG;QAEd,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC5C,IAAI,IAAI,CAAC,kBAAkB,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,CAAC;YAC9E,MAAM,IAAI,KAAK,CACb,8EAA8E;gBAC5E,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,oBAAoB;gBAC/D,gEAAgE,CACnE,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,+EAA+E;IACvE,kBAAkB,CACxB,IAAY,EACZ,OAAiB,EACjB,WAAmB,EACnB,QAAgB,EAChB,QAAgB;QAEhB,MAAM,SAAS,GAAwB;YACrC,IAAI;YACJ,WAAW;YACX,eAAe,EAAE,OAAO;YACxB,aAAa,EAAE,IAAI,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC;YACtE,aAAa,EAAE,WAAW;YAC1B,aAAa,EAAE,EAAE;YACjB,QAAQ,EAAE,EAAE;SACb,CAAC;QACF,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,EAAE,CAAC;QAC3D,IAAI,CAAC,cAAc,GAAG,IAAI,aAAa,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAED,iCAAiC;IACjC,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,qCAAqC;IACrC,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED,gCAAgC;IAChC,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}

package/dist/scoring/benchmarks.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Benchmark score normalization.
+ *
+ * Different benchmarks use different scales (0-100%, ELO ratings, etc.).
+ * This module normalizes them all to a 0-1 range for fair comparison.
+ */
+export declare class NormalizationRange {
+    readonly minScore: number;
+    readonly maxScore: number;
+    readonly description: string;
+    constructor(minScore: number, maxScore: number, description?: string);
+    /** Normalize a raw benchmark score to 0-1. */
+    normalize(rawScore: number): number;
+}
+/** Standard benchmark normalization ranges. */
+export declare const NORMALIZATION_RANGES: Record<string, NormalizationRange>;
+/**
+ * Normalizes benchmark scores across different scales.
+ *
+ * Supports standard benchmarks out of the box and allows
+ * registering custom normalization ranges.
+ */
+export declare class BenchmarkNormalizer {
+    private _ranges;
+    constructor();
+    /** Normalize a raw benchmark score to 0-1. */
+    normalize(benchmark: string, rawScore: number): number;
+    /** Register a custom normalization range for a benchmark. */
+    registerRange(benchmark: string, minScore: number, maxScore: number, description?: string): void;
+    /** Get the normalization range for a benchmark. */
+    getRange(benchmark: string): NormalizationRange | undefined;
+    /** List all benchmarks with registered normalization ranges. */
+    get knownBenchmarks(): string[];
+}
+//# sourceMappingURL=benchmarks.d.ts.map

package/dist/scoring/benchmarks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../src/scoring/benchmarks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,qBAAa,kBAAkB;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;gBAEjB,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,SAAK;IAMhE,8CAA8C;IAC9C,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;CAKpC;AAED,+CAA+C;AAC/C,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAanE,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAAkC;;IAMjD,8CAA8C;IAC9C,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAStD,6DAA6D;IAC7D,aAAa,CACX,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,WAAW,SAAK,GACf,IAAI;IAIP,mDAAmD;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS;IAI3D,gEAAgE;IAChE,IAAI,eAAe,IAAI,MAAM,EAAE,CAE9B;CACF"}

package/dist/scoring/benchmarks.js

@@ -0,0 +1,68 @@
+/**
+ * Benchmark score normalization.
+ *
+ * Different benchmarks use different scales (0-100%, ELO ratings, etc.).
+ * This module normalizes them all to a 0-1 range for fair comparison.
+ */
+export class NormalizationRange {
+    constructor(minScore, maxScore, description = '') {
+        this.minScore = minScore;
+        this.maxScore = maxScore;
+        this.description = description;
+    }
+    /** Normalize a raw benchmark score to 0-1. */
+    normalize(rawScore) {
+        if (this.maxScore === this.minScore)
+            return 0.5;
+        const normalized = (rawScore - this.minScore) / (this.maxScore - this.minScore);
+        return Math.max(0.0, Math.min(1.0, normalized));
+    }
+}
+/** Standard benchmark normalization ranges. */
+export const NORMALIZATION_RANGES = {
+    'MMLU': new NormalizationRange(25, 95, 'Academic knowledge across 57 subjects'),
+    'HellaSwag': new NormalizationRange(50, 98, 'Commonsense reasoning'),
+    'HumanEval': new NormalizationRange(20, 95, 'Code generation'),
+    'SWE-bench': new NormalizationRange(5, 85, 'Real-world software engineering'),
+    'TruthfulQA': new NormalizationRange(20, 85, 'Truthful question answering'),
+    'ARC': new NormalizationRange(0, 95, 'Science exam questions'),
+    'GSM8K': new NormalizationRange(20, 98, 'Grade school math'),
+    'DROP': new NormalizationRange(30, 90, 'Reading comprehension with arithmetic'),
+    'SuperGLUE': new NormalizationRange(40, 95, 'Natural language understanding'),
+    'Chatbot Arena (LMSys)': new NormalizationRange(1000, 1550, 'Human-rated chat quality'),
+    'MT-Bench': new NormalizationRange(5, 10, 'Multi-turn conversation quality'),
+    'LiveBench': new NormalizationRange(0, 100, 'Fresh, contamination-resistant evaluation'),
+};
+/**
+ * Normalizes benchmark scores across different scales.
+ *
+ * Supports standard benchmarks out of the box and allows
+ * registering custom normalization ranges.
+ */
+export class BenchmarkNormalizer {
+    constructor() {
+        this._ranges = new Map(Object.entries(NORMALIZATION_RANGES));
+    }
+    /** Normalize a raw benchmark score to 0-1. */
+    normalize(benchmark, rawScore) {
+        const range = this._ranges.get(benchmark);
+        if (!range) {
+            // Unknown benchmark -- assume 0-100 percentage scale
+            return Math.max(0.0, Math.min(1.0, rawScore / 100.0));
+        }
+        return range.normalize(rawScore);
+    }
+    /** Register a custom normalization range for a benchmark. */
+    registerRange(benchmark, minScore, maxScore, description = '') {
+        this._ranges.set(benchmark, new NormalizationRange(minScore, maxScore, description));
+    }
+    /** Get the normalization range for a benchmark. */
+    getRange(benchmark) {
+        return this._ranges.get(benchmark);
+    }
+    /** List all benchmarks with registered normalization ranges. */
+    get knownBenchmarks() {
+        return [...this._ranges.keys()];
+    }
+}
+//# sourceMappingURL=benchmarks.js.map

package/dist/scoring/benchmarks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmarks.js","sourceRoot":"","sources":["../../src/scoring/benchmarks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,OAAO,kBAAkB;IAK7B,YAAY,QAAgB,EAAE,QAAgB,EAAE,WAAW,GAAG,EAAE;QAC9D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,8CAA8C;IAC9C,SAAS,CAAC,QAAgB;QACxB,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ;YAAE,OAAO,GAAG,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC;IAClD,CAAC;CACF;AAED,+CAA+C;AAC/C,MAAM,CAAC,MAAM,oBAAoB,GAAuC;IACtE,MAAM,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,uCAAuC,CAAC;IAC/E,WAAW,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,uBAAuB,CAAC;IACpE,WAAW,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,iBAAiB,CAAC;IAC9D,WAAW,EAAE,IAAI,kBAAkB,CAAC,CAAC,EAAE,EAAE,EAAE,iCAAiC,CAAC;IAC7E,YAAY,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,6BAA6B,CAAC;IAC3E,KAAK,EAAE,IAAI,kBAAkB,CAAC,CAAC,EAAE,EAAE,EAAE,wBAAwB,CAAC;IAC9D,OAAO,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,mBAAmB,CAAC;IAC5D,MAAM,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,uCAAuC,CAAC;IAC/E,WAAW,EAAE,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,gCAAgC,CAAC;IAC7E,uBAAuB,EAAE,IAAI,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,0BAA0B,CAAC;IACvF,UAAU,EAAE,IAAI,kBAAkB,CAAC,CAAC,EAAE,EAAE,EAAE,iCAAiC,CAAC;IAC5E,WAAW,EAAE,IAAI,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,2CAA2C,CAAC;CACzF,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,mBAAmB;IAG9B;QACE,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,8CAA8C;IAC9C,SAAS,CAAC,SAAiB,EAAE,QAAgB;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,qDAAqD;YACrD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,6DAA6D;IAC7D,aAAa,CACX,SAAiB,EACjB,QAAgB,EAChB,QAAgB,EAChB,WAAW,GAAG,EAAE;QAEhB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;IACvF,CAAC;IAED,mDAAmD;IACnD,QAAQ,CAAC,SAAiB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACrC,CAAC;IAED,gEAAgE;IAChE,IAAI,eAAe;QACjB,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;CACF"}

package/dist/scoring/engine.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Dynamic model scoring engine.
+ *
+ * Combines benchmark performance, cost, and speed into a single score
+ * weighted by user priorities. This is the heart of the routing logic.
+ */
+import { ModelInfo } from '../registry/models.js';
+import { BenchmarkNormalizer } from './benchmarks.js';
+import { Priorities } from './priorities.js';
+export interface ModelScore {
+    modelId: string;
+    finalScore: number;
+    qualityScore: number;
+    costScore: number;
+    speedScore: number;
+    qualityContribution: number;
+    costContribution: number;
+    speedContribution: number;
+    topBenchmarks: Array<[string, number]>;
+    reasoning: string;
+}
+/** Speed tier -> numeric score. */
+export declare const SPEED_SCORES: Record<string, number>;
+/**
+ * Scores models against a classified prompt.
+ *
+ * Takes benchmark similarity scores (from the classifier) and user priorities,
+ * then ranks all available models using a three-factor weighted algorithm:
+ *
+ *     final = (quality * qW + cost * cW + speed * sW) / (qW + cW + sW)
+ *
+ * Where weights are derived from user priorities (1-5 scale).
+ */
+export declare class ScoringEngine {
+    private _normalizer;
+    constructor(normalizer?: BenchmarkNormalizer);
+    /**
+     * Score and rank models based on benchmark similarities and priorities.
+     */
+    scoreModels(models: ModelInfo[], benchmarkSimilarities: Record<string, number>, priorities?: Priorities, topK?: number): ModelScore[];
+    private _scoreSingleModel;
+}
+//# sourceMappingURL=engine.d.ts.map

package/dist/scoring/engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../../src/scoring/engine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAsB,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,mCAAmC;AACnC,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAM/C,CAAC;AAwDF;;;;;;;;;GASG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,WAAW,CAAsB;gBAE7B,UAAU,CAAC,EAAE,mBAAmB;IAI5C;;OAEG;IACH,WAAW,CACT,MAAM,EAAE,SAAS,EAAE,EACnB,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7C,UAAU,GAAE,UAA+B,EAC3C,IAAI,SAAI,GACP,UAAU,EAAE;IA2Ff,OAAO,CAAC,iBAAiB;CAwH1B"}