responses-proxy 0.1.0

package/dist/prompt-cache.js

@@ -0,0 +1,229 @@
+import { createHash } from "node:crypto";
+export function buildPromptCacheLayout(request, options = {}) {
+    const inputItems = Array.isArray(request.input) ? request.input : undefined;
+    const split = splitInputForPromptCache(inputItems, options.summaryKeepRecentItems ?? 6);
+    const stableInputItems = split.stableItems;
+    const dynamicInputItems = split.dynamicItems;
+    const shouldSummarize = options.stableSummarizationEnabled === true &&
+        stableInputItems.length >= (options.summaryTriggerItems ?? 14) &&
+        dynamicInputItems.length > 0;
+    const summarizedStableItems = shouldSummarize
+        ? summarizeStableItems(stableInputItems)
+        : stableInputItems;
+    const includeStableConversationPrefix = shouldSummarize && summarizedStableItems.length > 0;
+    const stablePrefix = omitUndefined({
+        model: request.model,
+        instructions: mergeStableInstructions(typeof request.instructions === "string" ? request.instructions : undefined, shouldSummarize ? buildStableSummaryBlock(stableInputItems) : undefined),
+        tools: request.tools,
+        tool_choice: request.tool_choice,
+        parallel_tool_calls: request.parallel_tool_calls,
+        reasoning: request.reasoning,
+        text: request.text,
+        max_output_tokens: request.max_output_tokens,
+        max_tool_calls: request.max_tool_calls,
+        temperature: request.temperature,
+        top_p: request.top_p,
+        metadata: request.metadata,
+        user: request.user,
+        truncation: request.truncation,
+        include: request.include,
+        input: includeStableConversationPrefix ? summarizedStableItems : undefined,
+    });
+    const dynamicTail = omitUndefined({
+        input: includeStableConversationPrefix
+            ? dynamicInputItems.length > 0
+                ? dynamicInputItems
+                : request.input
+            : request.input,
+    });
+    const familySignature = stableStringify(omitUndefined({
+        model: request.model,
+        instructions: typeof request.instructions === "string" ? request.instructions : undefined,
+        tools: request.tools,
+        tool_choice: request.tool_choice,
+        parallel_tool_calls: request.parallel_tool_calls,
+        reasoning: request.reasoning,
+        text: request.text,
+        max_output_tokens: request.max_output_tokens,
+        max_tool_calls: request.max_tool_calls,
+        temperature: request.temperature,
+        top_p: request.top_p,
+        metadata: request.metadata,
+        user: request.user,
+        truncation: request.truncation,
+        include: request.include,
+    }));
+    const modelSlug = slugify(typeof request.model === "string" ? request.model : "unknown-model");
+    const familyId = `family:${modelSlug}:core:${shortHash(familySignature)}`;
+    const staticKey = `static:${familyId}:${shortHash(stableStringify(stablePrefix))}`;
+    const requestKey = `request:${staticKey}:${shortHash(stableStringify(dynamicTail))}`;
+    const promptCacheRetention = resolvePromptCacheRetention(familyId, staticKey, options);
+    return {
+        familyId,
+        staticKey,
+        requestKey,
+        stablePrefix,
+        dynamicTail,
+        summaryApplied: shouldSummarize,
+        summaryItemCount: shouldSummarize ? stableInputItems.length : 0,
+        promptCacheKey: options.enabled ? requestKey : undefined,
+        promptCacheRetention: options.enabled ? promptCacheRetention : undefined,
+    };
+}
+function splitInputForPromptCache(inputItems, keepRecentItems) {
+    if (!inputItems || inputItems.length === 0) {
+        return {
+            stableItems: [],
+            dynamicItems: [],
+        };
+    }
+    const minimumBoundaryIndex = Math.max(0, inputItems.length - keepRecentItems);
+    let boundaryIndex = minimumBoundaryIndex;
+    for (let index = inputItems.length - 1; index >= minimumBoundaryIndex; index -= 1) {
+        const item = inputItems[index];
+        if (isRecord(item) && item.role === "user") {
+            boundaryIndex = index;
+            break;
+        }
+    }
+    return {
+        stableItems: inputItems.slice(0, boundaryIndex),
+        dynamicItems: inputItems.slice(boundaryIndex),
+    };
+}
+function summarizeStableItems(items) {
+    const summary = buildStableSummaryBlock(items);
+    if (!summary) {
+        return items;
+    }
+    return [
+        {
+            role: "assistant",
+            content: [
+                {
+                    type: "input_text",
+                    text: summary,
+                },
+            ],
+        },
+    ];
+}
+function buildStableSummaryBlock(items) {
+    if (items.length === 0) {
+        return undefined;
+    }
+    const lines = items.map((item, index) => summarizeInputItem(item, index + 1)).filter(Boolean);
+    if (lines.length === 0) {
+        return undefined;
+    }
+    return [
+        "Stable conversation summary generated by responses-proxy.",
+        "This summary replaces older conversation history to preserve a reusable upstream prompt prefix.",
+        ...lines,
+    ].join("\n");
+}
+function summarizeInputItem(item, index) {
+    if (!isRecord(item)) {
+        return undefined;
+    }
+    if (typeof item.role === "string") {
+        const content = summarizeContent(item.content);
+        return content ? `${index}. ${item.role}: ${content}` : `${index}. ${item.role}`;
+    }
+    if (item.type === "function_call") {
+        const name = typeof item.name === "string" ? item.name : "function";
+        const args = typeof item.arguments === "string" ? compactText(item.arguments, 160) : "";
+        return `${index}. function_call ${name}${args ? ` args=${args}` : ""}`;
+    }
+    if (item.type === "function_call_output") {
+        const output = typeof item.output === "string" ? compactText(item.output, 160) : "";
+        return `${index}. function_call_output${output ? ` ${output}` : ""}`;
+    }
+    return `${index}. ${compactText(stableStringify(item), 160)}`;
+}
+function summarizeContent(content) {
+    if (typeof content === "string") {
+        return compactText(content, 160);
+    }
+    if (!Array.isArray(content)) {
+        return undefined;
+    }
+    const fragments = content
+        .map((part) => {
+        if (!isRecord(part)) {
+            return "";
+        }
+        if (typeof part.text === "string") {
+            return part.text;
+        }
+        if (typeof part.image_url === "string") {
+            return `[image:${compactText(part.image_url, 64)}]`;
+        }
+        return "";
+    })
+        .filter(Boolean)
+        .join(" ");
+    return fragments ? compactText(fragments, 160) : undefined;
+}
+function mergeStableInstructions(baseInstructions, summaryBlock) {
+    const parts = [baseInstructions, summaryBlock].filter((value) => typeof value === "string" && value.trim().length > 0);
+    if (parts.length === 0) {
+        return undefined;
+    }
+    return parts.join("\n\n");
+}
+function resolvePromptCacheRetention(familyId, staticKey, options) {
+    if (options.retentionByStaticKeyEnabled) {
+        const bestStaticMatch = [...(options.staticKeyRetentionRules ?? [])]
+            .filter((rule) => staticKey.startsWith(rule.prefix))
+            .sort((left, right) => right.prefix.length - left.prefix.length)[0];
+        if (bestStaticMatch) {
+            return bestStaticMatch.retention;
+        }
+    }
+    if (!options.retentionByFamilyEnabled) {
+        return options.defaultRetention;
+    }
+    const bestMatch = [...(options.familyRetentionRules ?? [])]
+        .filter((rule) => familyId.startsWith(rule.prefix))
+        .sort((left, right) => right.prefix.length - left.prefix.length)[0];
+    return bestMatch?.retention ?? options.defaultRetention;
+}
+function stableStringify(value) {
+    return JSON.stringify(sortUnknown(value));
+}
+function sortUnknown(value) {
+    if (Array.isArray(value)) {
+        return value.map((item) => sortUnknown(item));
+    }
+    if (!isRecord(value)) {
+        return value;
+    }
+    return Object.fromEntries(Object.entries(value)
+        .filter(([, nested]) => nested !== undefined)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([key, nested]) => [key, sortUnknown(nested)]));
+}
+function omitUndefined(value) {
+    return Object.fromEntries(Object.entries(value).filter(([, nested]) => nested !== undefined));
+}
+function shortHash(value) {
+    return createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+function slugify(value) {
+    return value
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "") || "unknown";
+}
+function compactText(value, maxLength) {
+    const compact = value.replace(/\s+/g, " ").trim();
+    if (compact.length <= maxLength) {
+        return compact;
+    }
+    return `${compact.slice(0, maxLength - 1)}…`;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/dist/provider-health-service.js

@@ -0,0 +1,404 @@
+import { fetchProviderUsage } from "./provider-usage.js";
+const DEFAULT_HEALTH_THRESHOLDS = {
+    responseTime: {
+        good: 2000, // 2s
+        degraded: 5000 // 5s
+    },
+    errorRate: {
+        good: 0.02, // 2%
+        degraded: 0.1 // 10%
+    },
+    quotaUsage: {
+        warning: 80, // 80%
+        critical: 95 // 95%
+    }
+};
+export class ProviderHealthService {
+    providerRepository;
+    chatGptOAuthStore;
+    kiroTokenStore;
+    healthCheckInterval;
+    thresholds;
+    healthCache = new Map();
+    healthCheckIntervals = new Map();
+    responseTimeHistory = new Map();
+    errorHistory = new Map();
+    constructor(providerRepository, chatGptOAuthStore, kiroTokenStore, healthCheckInterval = 30000, // 30 seconds
+    thresholds = DEFAULT_HEALTH_THRESHOLDS) {
+        this.providerRepository = providerRepository;
+        this.chatGptOAuthStore = chatGptOAuthStore;
+        this.kiroTokenStore = kiroTokenStore;
+        this.healthCheckInterval = healthCheckInterval;
+        this.thresholds = thresholds;
+    }
+    /**
+     * Start health monitoring for all providers
+     */
+    startHealthMonitoring() {
+        const providers = this.providerRepository.listProviders();
+        for (const provider of providers) {
+            this.startProviderHealthCheck(provider.id);
+        }
+        console.log(`Started health monitoring for ${providers.length} providers`);
+    }
+    /**
+     * Stop health monitoring for all providers
+     */
+    stopHealthMonitoring() {
+        for (const [providerId, interval] of this.healthCheckIntervals) {
+            clearInterval(interval);
+        }
+        this.healthCheckIntervals.clear();
+        console.log('Stopped health monitoring for all providers');
+    }
+    /**
+     * Start health checking for a specific provider
+     */
+    startProviderHealthCheck(providerId) {
+        // Clear existing interval if any
+        const existingInterval = this.healthCheckIntervals.get(providerId);
+        if (existingInterval) {
+            clearInterval(existingInterval);
+        }
+        // Perform initial health check
+        this.checkProviderHealth(providerId);
+        // Set up recurring health checks
+        const interval = setInterval(() => {
+            this.checkProviderHealth(providerId);
+        }, this.healthCheckInterval);
+        this.healthCheckIntervals.set(providerId, interval);
+    }
+    /**
+     * Stop health checking for a specific provider
+     */
+    stopProviderHealthCheck(providerId) {
+        const interval = this.healthCheckIntervals.get(providerId);
+        if (interval) {
+            clearInterval(interval);
+            this.healthCheckIntervals.delete(providerId);
+        }
+    }
+    /**
+     * Get current health metrics for a provider
+     */
+    getProviderHealth(providerId) {
+        return this.healthCache.get(providerId) || null;
+    }
+    /**
+     * Get health metrics for all providers
+     */
+    getAllProviderHealth() {
+        return new Map(this.healthCache);
+    }
+    /**
+     * Force a health check for a specific provider
+     */
+    async forceHealthCheck(providerId) {
+        return this.checkProviderHealth(providerId);
+    }
+    /**
+     * Record a request result for health tracking
+     */
+    recordRequestResult(providerId, responseTime, isError) {
+        // Update response time history
+        const responseHistory = this.responseTimeHistory.get(providerId) || [];
+        responseHistory.push(responseTime);
+        // Keep only last 100 response times
+        if (responseHistory.length > 100) {
+            responseHistory.shift();
+        }
+        this.responseTimeHistory.set(providerId, responseHistory);
+        // Update error history
+        const errorHistory = this.errorHistory.get(providerId) || [];
+        errorHistory.push({
+            timestamp: Date.now(),
+            error: isError
+        });
+        // Keep only last hour of error history
+        const oneHourAgo = Date.now() - (60 * 60 * 1000);
+        const recentErrors = errorHistory.filter(e => e.timestamp > oneHourAgo);
+        this.errorHistory.set(providerId, recentErrors);
+        // Update cached health metrics if they exist
+        const cachedHealth = this.healthCache.get(providerId);
+        if (cachedHealth) {
+            this.updateHealthMetricsFromHistory(providerId, cachedHealth);
+        }
+    }
+    /**
+     * Perform health check for a specific provider
+     */
+    async checkProviderHealth(providerId) {
+        try {
+            const provider = this.providerRepository.getProvider(providerId);
+            if (!provider) {
+                return {
+                    success: false,
+                    error: `Provider ${providerId} not found`
+                };
+            }
+            const now = Date.now();
+            // Check quota status
+            const quotaStatus = await this.checkProviderQuota(provider);
+            // Check account status
+            const accountStatus = await this.checkAccountStatus(provider);
+            // Get response time metrics from history
+            const responseTimeMetrics = this.calculateResponseTimeMetrics(providerId);
+            // Get error rate metrics from history
+            const errorRateMetrics = this.calculateErrorRateMetrics(providerId);
+            // Calculate overall health score
+            const healthScore = this.calculateHealthScore(responseTimeMetrics, errorRateMetrics, quotaStatus, accountStatus);
+            const metrics = {
+                providerId,
+                isHealthy: healthScore >= 70, // Healthy if score >= 70
+                healthScore,
+                responseTime: responseTimeMetrics,
+                errorRate: errorRateMetrics,
+                quotaStatus,
+                accountStatus,
+                lastChecked: now,
+                lastUpdated: now
+            };
+            // Cache the metrics
+            this.healthCache.set(providerId, metrics);
+            return {
+                success: true,
+                metrics
+            };
+        }
+        catch (error) {
+            console.error(`Health check failed for provider ${providerId}:`, error);
+            return {
+                success: false,
+                error: error instanceof Error ? error.message : String(error)
+            };
+        }
+    }
+    /**
+     * Check provider quota status
+     */
+    async checkProviderQuota(provider) {
+        if (!provider.capabilities.usageCheckEnabled || !provider.capabilities.usageCheckUrl) {
+            return {
+                usagePercent: 0,
+                remaining: -1,
+                limit: -1
+            };
+        }
+        try {
+            const usage = await fetchProviderUsage({
+                apiKey: provider.providerApiKeys[0],
+                requestId: `health-${provider.id}-${Date.now()}`,
+                logger: {
+                    info: (...args) => console.log(...args),
+                    warn: (...args) => console.warn(...args),
+                    error: (...args) => console.error(...args),
+                    debug: (...args) => console.debug(...args),
+                    trace: (...args) => console.trace(...args),
+                    fatal: (...args) => console.error(...args),
+                    silent: () => { },
+                },
+                timeoutMs: 5000,
+                url: provider.capabilities.usageCheckUrl,
+            });
+            if (usage) {
+                const limit = usage.limit ?? -1;
+                const used = usage.used ?? 0;
+                const remaining = usage.remaining ?? (limit !== -1 ? limit - used : -1);
+                const usagePercent = limit > 0 ? (used / limit) * 100 : 0;
+                const resetTime = typeof usage.raw?.resetTime === 'string' ? usage.raw.resetTime : undefined;
+                return {
+                    usagePercent,
+                    remaining,
+                    limit,
+                    resetTime
+                };
+            }
+        }
+        catch (error) {
+            console.warn(`Failed to check quota for provider ${provider.id}:`, error);
+        }
+        return {
+            usagePercent: 0,
+            remaining: -1,
+            limit: -1
+        };
+    }
+    /**
+     * Check account status for the provider
+     */
+    async checkAccountStatus(provider) {
+        let hasValidAccounts = true;
+        let accountsNearExpiry = false;
+        let activeAccounts = 0;
+        let totalAccounts = 0;
+        try {
+            if (provider.authMode === 'chatgpt_oauth' && this.chatGptOAuthStore) {
+                const accounts = this.chatGptOAuthStore.listAccounts();
+                totalAccounts = accounts.length;
+                for (const account of accounts) {
+                    if (account.accessToken && account.refreshToken) {
+                        activeAccounts++;
+                        // Check if token is near expiry (within 24 hours)
+                        if (account.expiresAt) {
+                            const expiryTime = new Date(account.expiresAt).getTime();
+                            const twentyFourHours = 24 * 60 * 60 * 1000;
+                            if (expiryTime - Date.now() < twentyFourHours) {
+                                accountsNearExpiry = true;
+                            }
+                        }
+                    }
+                }
+                hasValidAccounts = activeAccounts > 0;
+            }
+            else if (provider.authMode === 'kiro' && this.kiroTokenStore) {
+                const accounts = this.kiroTokenStore.listAccounts();
+                totalAccounts = accounts.length;
+                for (const account of accounts) {
+                    if (account.isActive && account.accessToken) {
+                        activeAccounts++;
+                        // Check if Kiro token is near expiry
+                        if (account.expiresAt) {
+                            const expiryTime = new Date(account.expiresAt).getTime();
+                            const twentyFourHours = 24 * 60 * 60 * 1000;
+                            if (expiryTime - Date.now() < twentyFourHours) {
+                                accountsNearExpiry = true;
+                            }
+                        }
+                    }
+                }
+                hasValidAccounts = activeAccounts > 0;
+            }
+            else if (provider.authMode === 'api_key') {
+                // For API key providers, check if they have keys configured
+                totalAccounts = provider.providerApiKeys.length;
+                activeAccounts = provider.providerApiKeys.length;
+                hasValidAccounts = provider.providerApiKeys.length > 0;
+            }
+        }
+        catch (error) {
+            console.warn(`Failed to check account status for provider ${provider.id}:`, error);
+            hasValidAccounts = false;
+        }
+        return {
+            hasValidAccounts,
+            accountsNearExpiry,
+            activeAccounts,
+            totalAccounts
+        };
+    }
+    /**
+     * Calculate response time metrics from history
+     */
+    calculateResponseTimeMetrics(providerId) {
+        const history = this.responseTimeHistory.get(providerId) || [];
+        if (history.length === 0) {
+            return {
+                average: 1000, // Default 1s
+                p95: 1000,
+                p99: 1000
+            };
+        }
+        const sorted = [...history].sort((a, b) => a - b);
+        const average = history.reduce((sum, time) => sum + time, 0) / history.length;
+        const p95Index = Math.floor(sorted.length * 0.95);
+        const p99Index = Math.floor(sorted.length * 0.99);
+        return {
+            average,
+            p95: sorted[p95Index] || average,
+            p99: sorted[p99Index] || average
+        };
+    }
+    /**
+     * Calculate error rate metrics from history
+     */
+    calculateErrorRateMetrics(providerId) {
+        const history = this.errorHistory.get(providerId) || [];
+        if (history.length === 0) {
+            return {
+                rate: 0,
+                recentErrors: 0,
+                totalRequests: 0
+            };
+        }
+        const recentErrors = history.filter(e => e.error).length;
+        const totalRequests = history.length;
+        const rate = totalRequests > 0 ? recentErrors / totalRequests : 0;
+        return {
+            rate,
+            recentErrors,
+            totalRequests
+        };
+    }
+    /**
+     * Calculate overall health score (0-100)
+     */
+    calculateHealthScore(responseTime, errorRate, quotaStatus, accountStatus) {
+        let score = 100;
+        // Response time factor (25% weight)
+        if (responseTime.average > this.thresholds.responseTime.degraded) {
+            score -= 25;
+        }
+        else if (responseTime.average > this.thresholds.responseTime.good) {
+            score -= 12;
+        }
+        // Error rate factor (30% weight)
+        if (errorRate.rate > this.thresholds.errorRate.degraded) {
+            score -= 30;
+        }
+        else if (errorRate.rate > this.thresholds.errorRate.good) {
+            score -= 15;
+        }
+        // Quota status factor (25% weight)
+        if (quotaStatus.usagePercent > this.thresholds.quotaUsage.critical) {
+            score -= 25;
+        }
+        else if (quotaStatus.usagePercent > this.thresholds.quotaUsage.warning) {
+            score -= 12;
+        }
+        // Account status factor (20% weight)
+        if (!accountStatus.hasValidAccounts) {
+            score -= 20;
+        }
+        else if (accountStatus.accountsNearExpiry) {
+            score -= 10;
+        }
+        return Math.max(0, Math.min(100, score));
+    }
+    /**
+     * Update health metrics from request history
+     */
+    updateHealthMetricsFromHistory(providerId, metrics) {
+        metrics.responseTime = this.calculateResponseTimeMetrics(providerId);
+        metrics.errorRate = this.calculateErrorRateMetrics(providerId);
+        metrics.healthScore = this.calculateHealthScore(metrics.responseTime, metrics.errorRate, metrics.quotaStatus, metrics.accountStatus);
+        metrics.isHealthy = metrics.healthScore >= 70;
+        metrics.lastUpdated = Date.now();
+    }
+    /**
+     * Get health summary for all providers
+     */
+    getHealthSummary() {
+        const allHealth = Array.from(this.healthCache.values());
+        const totalProviders = allHealth.length;
+        if (totalProviders === 0) {
+            return {
+                totalProviders: 0,
+                healthyProviders: 0,
+                degradedProviders: 0,
+                unhealthyProviders: 0,
+                averageHealthScore: 0
+            };
+        }
+        const healthyProviders = allHealth.filter(h => h.healthScore >= 70).length;
+        const degradedProviders = allHealth.filter(h => h.healthScore >= 40 && h.healthScore < 70).length;
+        const unhealthyProviders = allHealth.filter(h => h.healthScore < 40).length;
+        const averageHealthScore = allHealth.reduce((sum, h) => sum + h.healthScore, 0) / totalProviders;
+        return {
+            totalProviders,
+            healthyProviders,
+            degradedProviders,
+            unhealthyProviders,
+            averageHealthScore: Math.round(averageHealthScore)
+        };
+    }
+}