pi-provider-litellm 2.2.1 → 3.0.0

package/src/protocols.ts

@@ -0,0 +1,42 @@
+import type { ProviderStreams } from "@earendil-works/pi-ai";
+import { anthropicMessagesApi, openAICompletionsApi, openAIResponsesApi } from "@earendil-works/pi-ai/compat";
+import { normalizeBaseUrl } from "./discover.js";
+import type { LiteLLMApi } from "./types.js";
+
+type LiteLLMProtocol = {
+  createApi: () => ProviderStreams;
+  modelBaseUrl(root: string): string;
+};
+
+export const LITELLM_PROTOCOLS = {
+  "anthropic-messages": {
+    createApi: anthropicMessagesApi,
+    modelBaseUrl: (root) => root,
+  },
+  "openai-completions": {
+    createApi: openAICompletionsApi,
+    modelBaseUrl: (root) => `${root}/v1`,
+  },
+  "openai-responses": {
+    createApi: openAIResponsesApi,
+    modelBaseUrl: (root) => `${root}/v1`,
+  },
+} satisfies Record<LiteLLMApi, LiteLLMProtocol>;
+
+export const LITELLM_API_NAMES = Object.keys(LITELLM_PROTOCOLS) as LiteLLMApi[];
+
+export function isLiteLLMApi(api: unknown): api is LiteLLMApi {
+  return typeof api === "string" && Object.hasOwn(LITELLM_PROTOCOLS, api);
+}
+
+export function resolveModelBaseUrl(baseUrl: string, api: LiteLLMApi, allowInsecureHttp = false): string {
+  return LITELLM_PROTOCOLS[api].modelBaseUrl(normalizeBaseUrl(baseUrl, allowInsecureHttp));
+}
+
+export function createLiteLLMProtocolApis(): Record<LiteLLMApi, ProviderStreams> {
+  return {
+    "anthropic-messages": LITELLM_PROTOCOLS["anthropic-messages"].createApi(),
+    "openai-completions": LITELLM_PROTOCOLS["openai-completions"].createApi(),
+    "openai-responses": LITELLM_PROTOCOLS["openai-responses"].createApi(),
+  };
+}

package/src/provider.ts

@@ -0,0 +1,248 @@
+import {
+  type ApiStreamOptions,
+  type Context,
+  type Credential,
+  createProvider,
+  type Model,
+  type Provider,
+  type ProviderAuth,
+  type SimpleStreamOptions,
+} from "@earendil-works/pi-ai";
+import { LITELLM_DISCOVERY_VERSION } from "./backend-identity.js";
+import { enrichCachedModel, normalizeBaseUrl, restoreCachedModelPolicy } from "./discover.js";
+import { createLiteLLMProtocolApis, isLiteLLMApi, LITELLM_API_NAMES, resolveModelBaseUrl } from "./protocols.js";
+import type { DiscoveredModel, DiscoveryResult, LiteLLMApi, LiteLLMModel } from "./types.js";
+
+export type LiteLLMProviderOptions = {
+  id: string;
+  name: string;
+  baseUrl: string;
+  auth: ProviderAuth;
+  /** Catalog discovered during activation. Pi's startup refresh never allows network access, so
+   * without a seed the provider stays empty until the user opens /model. */
+  models?: readonly LiteLLMModel[];
+  allowInsecureHttp?: boolean;
+  resolveCredentialRoot: (credential?: Credential, requestBaseUrl?: string, apiKey?: string) => string | undefined;
+  discover(credential: Credential, signal?: AbortSignal): Promise<DiscoveryResult & { baseUrl?: string }>;
+};
+
+export function toNativeModels(
+  provider: string,
+  baseUrl: string,
+  models: readonly DiscoveredModel[],
+  allowInsecureHttp = false,
+): LiteLLMModel[] {
+  return models.map((model) => ({
+    ...model,
+    provider,
+    baseUrl: resolveModelBaseUrl(baseUrl, model.api, allowInsecureHttp),
+  }));
+}
+
+export const DEFAULT_LITELLM_BASE_URL = "https://litellm.example.com";
+const CHAT_TOOL_CAP = 128;
+const PLACEHOLDER_HOSTNAMES = new Set([canonicalHostname(new URL(DEFAULT_LITELLM_BASE_URL).hostname)]);
+
+function canonicalHostname(hostname: string): string {
+  return hostname.replace(/\.$/, "");
+}
+
+function refreshRequired(message: string): Error {
+  return new Error(`${message}; a network refresh with a valid LiteLLM base URL is required`);
+}
+
+function proxyRoot(
+  baseUrl: string,
+  subject: string,
+  allowInsecureHttp = false,
+  verb = "has",
+): { root: string; canonical: string; hostname: string } {
+  try {
+    const root = normalizeBaseUrl(baseUrl, allowInsecureHttp);
+    const url = new URL(root);
+    if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error("unsupported protocol");
+    return { root, canonical: url.href, hostname: url.hostname };
+  } catch {
+    throw refreshRequired(`${subject} ${verb} an invalid LiteLLM model URL`);
+  }
+}
+
+function activeCredentialRoot(
+  root: string,
+  allowInsecureHttp = false,
+): { root: string; canonical: string; hostname: string } {
+  const active = proxyRoot(root, "Active credentials", allowInsecureHttp, "have");
+  if (isPlaceholderHost(active.hostname)) {
+    throw refreshRequired("Active credentials use a placeholder LiteLLM model host");
+  }
+  return active;
+}
+
+export function isPlaceholderHost(hostname: string): boolean {
+  return PLACEHOLDER_HOSTNAMES.has(canonicalHostname(hostname));
+}
+
+function modelRootError(
+  model: Model<LiteLLMApi>,
+  active: { root: string; canonical: string; hostname: string },
+  allowInsecureHttp = false,
+): Error | undefined {
+  if (!isLiteLLMApi(model.api)) {
+    return new Error(
+      `LiteLLM model ${model.id} declares unsupported protocol "${String(model.api)}"; ` +
+        `set "api" to one of ${LITELLM_API_NAMES.join(", ")} in models.json`,
+    );
+  }
+  let stored: { root: string; canonical: string; hostname: string };
+  try {
+    stored = proxyRoot(model.baseUrl, "Cached model", allowInsecureHttp);
+  } catch (error) {
+    return error instanceof Error ? error : refreshRequired("Cached model has an invalid LiteLLM model URL");
+  }
+  if (isPlaceholderHost(stored.hostname)) {
+    return refreshRequired("Cached model uses a placeholder LiteLLM model host");
+  }
+  if (stored.canonical !== active.canonical) {
+    return refreshRequired(
+      `Cached model has stale LiteLLM model root ${stored.root}; active credentials use ${active.root}`,
+    );
+  }
+}
+
+function requestModel(
+  provider: string,
+  model: Model<LiteLLMApi>,
+  context: Context,
+  credentialRoot: string | undefined,
+  allowInsecureHttp = false,
+): Model<LiteLLMApi> {
+  if (!credentialRoot) throw refreshRequired("Active credentials do not identify a LiteLLM model host");
+  const active = activeCredentialRoot(credentialRoot, allowInsecureHttp);
+  const error = modelRootError(model, active, allowInsecureHttp);
+  if (error) throw error;
+  if (
+    model.api === "openai-completions" &&
+    (model as LiteLLMModel).litellmBackendFamily === "openai" &&
+    context.tools &&
+    context.tools.length > CHAT_TOOL_CAP
+  ) {
+    throw new Error(
+      `LiteLLM model ${model.id} uses Chat Completions with ${context.tools.length} tools, exceeding the ` +
+        `${CHAT_TOOL_CAP}-tool cap; route this model via Responses or reduce enabled extensions`,
+    );
+  }
+  return { ...model, provider, baseUrl: resolveModelBaseUrl(active.root, model.api, allowInsecureHttp) };
+}
+
+export function createLiteLLMProvider(options: LiteLLMProviderOptions): Provider<LiteLLMApi> {
+  const reportedAvailabilityDiagnostics = new Set<string>();
+  const reportUnavailable = (message: string): void => {
+    if (reportedAvailabilityDiagnostics.has(message)) return;
+    reportedAvailabilityDiagnostics.add(message);
+    process.stderr.write(`LiteLLM (${options.id}): ${message}\n`);
+  };
+  const provider = createProvider<LiteLLMApi>({
+    id: options.id,
+    name: options.name,
+    baseUrl: options.baseUrl,
+    auth: options.auth,
+    models: options.models ?? [],
+    async fetchModels(context) {
+      if (!context.credential) throw new Error("LiteLLM model discovery requires a credential");
+      const result = await options.discover(context.credential, context.signal);
+      return toNativeModels(options.id, result.baseUrl ?? options.baseUrl, result.models, options.allowInsecureHttp);
+    },
+    filterModels(models, credential) {
+      let active: { root: string; canonical: string; hostname: string };
+      try {
+        const root = options.resolveCredentialRoot(credential);
+        if (!root) {
+          if (models.length > 0) {
+            reportUnavailable(
+              `${models.length} model(s) hidden because no LiteLLM base URL is configured; ` +
+                "set LITELLM_BASE_URL or run /login litellm",
+            );
+          }
+          return [];
+        }
+        active = activeCredentialRoot(root, options.allowInsecureHttp);
+      } catch (error) {
+        reportUnavailable(error instanceof Error ? error.message : String(error));
+        return [];
+      }
+      const available: Model<LiteLLMApi>[] = [];
+      for (const model of models) {
+        const error = modelRootError(model, active, options.allowInsecureHttp);
+        if (error) {
+          reportUnavailable(error.message);
+          continue;
+        }
+        available.push({
+          ...model,
+          baseUrl: resolveModelBaseUrl(active.root, model.api, options.allowInsecureHttp),
+        });
+      }
+      return available;
+    },
+    api: createLiteLLMProtocolApis(),
+  });
+  const refreshModels = provider.refreshModels;
+  const guardedProvider: Provider<LiteLLMApi> = {
+    ...provider,
+    stream: <T extends LiteLLMApi>(model: Model<T>, context: Context, requestOptions?: ApiStreamOptions<T>) =>
+      provider.stream(
+        requestModel(
+          options.id,
+          model,
+          context,
+          options.resolveCredentialRoot(undefined, requestOptions?.env?.LITELLM_BASE_URL, requestOptions?.apiKey),
+          options.allowInsecureHttp,
+        ),
+        context,
+        requestOptions,
+      ),
+    streamSimple: (model: Model<LiteLLMApi>, context: Context, requestOptions?: SimpleStreamOptions) =>
+      provider.streamSimple(
+        requestModel(
+          options.id,
+          model,
+          context,
+          options.resolveCredentialRoot(undefined, requestOptions?.env?.LITELLM_BASE_URL, requestOptions?.apiKey),
+          options.allowInsecureHttp,
+        ),
+        context,
+        requestOptions,
+      ),
+  };
+  if (!refreshModels) return guardedProvider;
+  return {
+    ...guardedProvider,
+    refreshModels: async (context) => {
+      const storedModels = context.stored?.models ?? [];
+      let legacyCount = 0;
+      const models = storedModels.map((model) => {
+        if ((model as LiteLLMModel).litellmDiscoveryVersion !== LITELLM_DISCOVERY_VERSION) {
+          legacyCount++;
+          return restoreCachedModelPolicy(model);
+        }
+        return enrichCachedModel(model);
+      });
+      try {
+        await refreshModels({
+          ...context,
+          // The current pi-ai provider has no freshness gate, but force is the documented refresh contract field.
+          force: context.force || (context.allowNetwork && legacyCount > 0),
+          stored: context.stored && { ...context.stored, models },
+        });
+      } catch (error) {
+        if (context.allowNetwork && legacyCount > 0) {
+          reportUnavailable(
+            `keeping cached models whose discovery metadata version does not match ` +
+              `${LITELLM_DISCOVERY_VERSION} because the required network refresh failed`,
+          );
+        }
+        throw error;
+      }
+    },
+  };
+}

package/src/public-catalog.ts

@@ -0,0 +1,272 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { Api, Model } from "@earendil-works/pi-ai";
+import { getModels, getProviders } from "@earendil-works/pi-ai/compat";
+import type { BuiltinProvider } from "@earendil-works/pi-ai/providers/all";
+
+export const MODELS_DEV_URL = "https://models.dev/api.json";
+const MODELS_DEV_CACHE_TTL_MS = 28 * 24 * 60 * 60 * 1000;
+const DEFAULT_TIMEOUT_MS = 5000;
+const KNOWN_PROVIDERS = new Set<string>(getProviders());
+
+export interface PublicCatalogRecord {
+  source: "models.dev" | "pi-vendor" | "pi-adapter";
+  provider: string;
+  modelId: string;
+  limits?: { context?: number; output?: number };
+  cost?: { input?: number; output?: number; cacheRead?: number; cacheWrite?: number };
+  modalities?: Array<"text" | "image">;
+  effortLevels?: string[];
+  thinkingLevelMap?: Record<string, unknown>;
+}
+
+export interface PublicCatalog {
+  lookup(provider: string | undefined, id: string): PublicCatalogRecord | undefined;
+}
+
+interface ModelsDevModel {
+  modalities?: { input?: string[] };
+  limit?: { context?: number; input?: number; output?: number };
+  cost?: { input?: number; output?: number; cache_read?: number; cache_write?: number };
+  reasoning_options?: { type?: string; values?: string[] } | Array<{ type?: string; values?: string[] }>;
+}
+
+type ModelsDevCatalog = Record<string, { models?: Record<string, ModelsDevModel> }>;
+interface CacheFile {
+  fetchedAt: number;
+  catalog: ModelsDevCatalog;
+}
+
+export interface LoadPublicCatalogOptions {
+  cachePath?: string;
+  offline?: boolean;
+  timeoutMs?: number;
+  signal?: AbortSignal;
+  fetch?: typeof fetch;
+}
+
+const memoryCaches = new Map<string, CacheFile>();
+const refreshes = new Map<string, Promise<ModelsDevCatalog | undefined>>();
+
+const PROVIDER_ALIASES: Readonly<Record<string, readonly string[]>> = {
+  anthropic: ["anthropic"],
+  azure: ["azure", "openai"],
+  azure_ai: ["azure", "openai"],
+  bedrock: ["amazon-bedrock"],
+  bedrock_converse: ["amazon-bedrock"],
+  deepseek: ["deepseek"],
+  fireworks: ["fireworks-ai"],
+  fireworks_ai: ["fireworks-ai"],
+  gemini: ["google"],
+  moonshot: ["moonshotai"],
+  nvidia_nim: ["nvidia"],
+  openai: ["openai"],
+  together_ai: ["together"],
+  vertex_ai: ["google-vertex"],
+};
+
+const PI_PROVIDER_ALIASES: Readonly<Record<string, readonly string[]>> = {
+  "amazon-bedrock": ["amazon-bedrock"],
+  azure: ["azure-openai-responses"],
+  chatgpt: ["openai-codex"],
+  "fireworks-ai": ["fireworks"],
+  openai: ["openai"],
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function finite(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+// A token price below zero is malformed catalog data, not a discount; treating it as
+// missing lets the normal incomplete-cost handling apply instead of publishing it.
+function price(value: unknown): number | undefined {
+  const parsed = finite(value);
+  return parsed !== undefined && parsed >= 0 ? parsed : undefined;
+}
+
+function normalizeCatalog(value: unknown): ModelsDevCatalog | undefined {
+  if (!isRecord(value)) return undefined;
+  const result: ModelsDevCatalog = {};
+  for (const [provider, rawProvider] of Object.entries(value)) {
+    if (!isRecord(rawProvider) || !isRecord(rawProvider.models)) continue;
+    const models: Record<string, ModelsDevModel> = {};
+    for (const [id, rawModel] of Object.entries(rawProvider.models)) {
+      if (!isRecord(rawModel)) continue;
+      models[id] = rawModel as ModelsDevModel;
+    }
+    result[provider] = { models };
+  }
+  return result;
+}
+
+async function writeJsonAtomic(path: string, value: unknown): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  const temporaryPath = `${path}.${process.pid}.${Date.now()}.tmp`;
+  await writeFile(temporaryPath, JSON.stringify(value, null, 2), "utf8");
+  await rename(temporaryPath, path);
+}
+
+async function readCache(path: string): Promise<CacheFile | undefined> {
+  try {
+    const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (!isRecord(parsed)) return undefined;
+    const fetchedAt = finite(parsed.fetchedAt);
+    if (fetchedAt === undefined || fetchedAt < 0 || fetchedAt > Date.now()) return undefined;
+    const catalog = normalizeCatalog(parsed.catalog);
+    return catalog ? { fetchedAt, catalog } : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+async function refreshCatalog(key: string, options: LoadPublicCatalogOptions): Promise<ModelsDevCatalog | undefined> {
+  const active = refreshes.get(key);
+  if (active) return active;
+  const refresh = (async () => {
+    try {
+      const fetchImpl = options.fetch ?? globalThis.fetch;
+      const timeout = AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+      const response = await fetchImpl(MODELS_DEV_URL, {
+        headers: { Accept: "application/json" },
+        signal: options.signal ? AbortSignal.any([options.signal, timeout]) : timeout,
+      });
+      if (!response.ok) return undefined;
+      const catalog = normalizeCatalog(await response.json());
+      if (!catalog) return undefined;
+      const cache = { fetchedAt: Date.now(), catalog };
+      memoryCaches.set(key, cache);
+      if (options.cachePath) await writeJsonAtomic(options.cachePath, cache).catch(() => undefined);
+      return catalog;
+    } catch {
+      return undefined;
+    } finally {
+      refreshes.delete(key);
+    }
+  })();
+  refreshes.set(key, refresh);
+  return refresh;
+}
+
+function providerCandidates(provider: string | undefined): string[] {
+  if (!provider) return [];
+  const normalized = provider.trim().toLowerCase();
+  return [...new Set(PROVIDER_ALIASES[normalized] ?? [normalized])];
+}
+
+function lookupIds(id: string): string[] {
+  const result = new Set([id]);
+  if (id.includes("/")) result.add(id.slice(id.indexOf("/") + 1));
+  return [...result];
+}
+
+function mapModelsDev(provider: string, modelId: string, model: ModelsDevModel): PublicCatalogRecord {
+  const context = finite(model.limit?.context) ?? finite(model.limit?.input);
+  const output = finite(model.limit?.output);
+  const input = price(model.cost?.input);
+  const outputCost = price(model.cost?.output);
+  const cacheRead = price(model.cost?.cache_read);
+  const cacheWrite = price(model.cost?.cache_write);
+  const options = Array.isArray(model.reasoning_options) ? model.reasoning_options : [model.reasoning_options];
+  const effortLevels = options
+    .filter((option) => option?.type === "effort")
+    .flatMap((option) => option?.values ?? [])
+    .filter((value): value is string => typeof value === "string");
+  return {
+    source: "models.dev",
+    provider,
+    modelId,
+    ...(context !== undefined || output !== undefined ? { limits: { context, output } } : {}),
+    ...(input !== undefined || outputCost !== undefined || cacheRead !== undefined || cacheWrite !== undefined
+      ? {
+          cost: {
+            ...(input !== undefined ? { input } : {}),
+            ...(outputCost !== undefined ? { output: outputCost } : {}),
+            ...(cacheRead !== undefined ? { cacheRead } : {}),
+            ...(cacheWrite !== undefined ? { cacheWrite } : {}),
+          },
+        }
+      : {}),
+    ...(Array.isArray(model.modalities?.input)
+      ? { modalities: model.modalities.input.includes("image") ? ["text", "image"] : ["text"] }
+      : {}),
+    ...(effortLevels.length > 0 ? { effortLevels } : {}),
+  };
+}
+
+function toPiRecord(source: "pi-vendor" | "pi-adapter", provider: string, model: Model<Api>): PublicCatalogRecord {
+  const thinkingLevelMap = model.thinkingLevelMap as Record<string, unknown> | undefined;
+  const effortLevels = thinkingLevelMap
+    ? Object.entries(thinkingLevelMap)
+        .filter(([, value]) => value !== null)
+        .map(([level]) => level)
+    : [];
+  return {
+    source,
+    provider,
+    modelId: model.id,
+    limits: { context: model.contextWindow, output: model.maxTokens },
+    cost: {
+      input: model.cost.input,
+      output: model.cost.output,
+      cacheRead: model.cost.cacheRead,
+      cacheWrite: model.cost.cacheWrite,
+    },
+    modalities: model.input.includes("image") ? ["text", "image"] : ["text"],
+    ...(effortLevels.length > 0 ? { effortLevels } : {}),
+    ...(thinkingLevelMap ? { thinkingLevelMap } : {}),
+  };
+}
+
+function findPiModel(provider: string, ids: readonly string[]): Model<Api> | undefined {
+  if (!KNOWN_PROVIDERS.has(provider)) return undefined;
+  const models = getModels(provider as BuiltinProvider);
+  return models.find((model) => ids.includes(model.id) || ids.includes(`${provider}/${model.id}`));
+}
+
+export async function loadPublicCatalog(options: LoadPublicCatalogOptions = {}): Promise<PublicCatalog> {
+  const key = options.cachePath ?? MODELS_DEV_URL;
+  let cache = memoryCaches.get(key);
+  if (!cache && options.cachePath) {
+    cache = await readCache(options.cachePath);
+    if (cache) memoryCaches.set(key, cache);
+  }
+  const offline = options.offline ?? process.env.LITELLM_OFFLINE === "1";
+  let catalog = cache?.catalog;
+  if (!offline) {
+    if (!cache) catalog = await refreshCatalog(key, options);
+    else if (Date.now() - cache.fetchedAt >= MODELS_DEV_CACHE_TTL_MS) void refreshCatalog(key, options);
+  }
+
+  return {
+    lookup(provider, id) {
+      const providers = providerCandidates(provider);
+      const ids = lookupIds(id);
+      for (const candidate of providers) {
+        const models = catalog?.[candidate]?.models;
+        for (const modelId of ids) {
+          const model = models?.[modelId];
+          if (model) return mapModelsDev(candidate, modelId, model);
+        }
+      }
+      const vendor = providers.find((candidate) => candidate !== "azure") ?? providers[0];
+      if (vendor) {
+        for (const piProvider of PI_PROVIDER_ALIASES[vendor] ?? [vendor]) {
+          const model = findPiModel(piProvider, ids);
+          if (model) return toPiRecord("pi-vendor", piProvider, model);
+        }
+      }
+      const adapter = provider?.trim().toLowerCase();
+      if (adapter) {
+        for (const piProvider of PI_PROVIDER_ALIASES[adapter] ?? [adapter]) {
+          const model = findPiModel(piProvider, ids);
+          if (model) return toPiRecord("pi-adapter", piProvider, model);
+        }
+      }
+      return undefined;
+    },
+  };
+}