swayrouter 1.0.0

package/src/services/accountFallback.js

@@ -0,0 +1,241 @@
+import { ERROR_RULES, BACKOFF_CONFIG, TRANSIENT_COOLDOWN_MS, classifyError, ERROR_TIERS, isDeprioitized } from "../config/errorConfig.js";
+
+import { fnv1a } from "@/lib/security/payloadCapture.js";
+
+export function getQuotaCooldown(backoffLevel = 0) {
+  const level = Math.max(0, backoffLevel - 1);
+  const cooldown = BACKOFF_CONFIG.base * Math.pow(2, level);
+  return Math.min(cooldown, BACKOFF_CONFIG.max);
+}
+
+export function checkFallbackError(status, errorText, backoffLevel = 0) {
+  const lowerError = errorText
+    ? (typeof errorText === "string" ? errorText : JSON.stringify(errorText)).toLowerCase()
+    : "";
+
+  for (const rule of ERROR_RULES) {
+
+    if (rule.text && lowerError && lowerError.includes(rule.text)) {
+      if (rule.backoff) {
+        const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel);
+        return { shouldFallback: true, cooldownMs: getQuotaCooldown(newLevel), newBackoffLevel: newLevel };
+      }
+      return { shouldFallback: true, cooldownMs: rule.cooldownMs };
+    }
+
+    if (rule.status && rule.status === status) {
+      if (rule.backoff) {
+        const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel);
+        return { shouldFallback: true, cooldownMs: getQuotaCooldown(newLevel), newBackoffLevel: newLevel };
+      }
+      return { shouldFallback: true, cooldownMs: rule.cooldownMs };
+    }
+  }
+
+  return { shouldFallback: true, cooldownMs: TRANSIENT_COOLDOWN_MS };
+}
+
+export function isAccountUnavailable(unavailableUntil) {
+  if (!unavailableUntil) return false;
+  return new Date(unavailableUntil).getTime() > Date.now();
+}
+
+export function getUnavailableUntil(cooldownMs) {
+  return new Date(Date.now() + cooldownMs).toISOString();
+}
+
+export function getEarliestRateLimitedUntil(accounts) {
+  let earliest = null;
+  const now = Date.now();
+  for (const acc of accounts) {
+    if (!acc.rateLimitedUntil) continue;
+    const until = new Date(acc.rateLimitedUntil).getTime();
+    if (until <= now) continue;
+    if (!earliest || until < earliest) earliest = until;
+  }
+  if (!earliest) return null;
+  return new Date(earliest).toISOString();
+}
+
+export function formatRetryAfter(rateLimitedUntil) {
+  if (!rateLimitedUntil) return "";
+  const diffMs = new Date(rateLimitedUntil).getTime() - Date.now();
+  if (diffMs <= 0) return "reset after 0s";
+  const totalSec = Math.ceil(diffMs / 1000);
+  const h = Math.floor(totalSec / 3600);
+  const m = Math.floor((totalSec % 3600) / 60);
+  const s = totalSec % 60;
+  const parts = [];
+  if (h > 0) parts.push(`${h}h`);
+  if (m > 0) parts.push(`${m}m`);
+  if (s > 0 || parts.length === 0) parts.push(`${s}s`);
+  return `reset after ${parts.join(" ")}`;
+}
+
+export const MODEL_LOCK_PREFIX = "modelLock_";
+
+export const MODEL_LOCK_ALL = `${MODEL_LOCK_PREFIX}__all`;
+
+export function getModelLockKey(model) {
+  return model ? `${MODEL_LOCK_PREFIX}${model}` : MODEL_LOCK_ALL;
+}
+
+export function isModelLockActive(connection, model) {
+  const key = getModelLockKey(model);
+  const expiry = connection[key] || connection[MODEL_LOCK_ALL];
+  if (!expiry) return false;
+  return new Date(expiry).getTime() > Date.now();
+}
+
+export function getEarliestModelLockUntil(connection) {
+  if (!connection) return null;
+  let earliest = null;
+  const now = Date.now();
+  for (const [key, val] of Object.entries(connection)) {
+    if (!key.startsWith(MODEL_LOCK_PREFIX) || !val) continue;
+    const t = new Date(val).getTime();
+    if (t <= now) continue;
+    if (!earliest || t < earliest) earliest = t;
+  }
+  return earliest ? new Date(earliest).toISOString() : null;
+}
+
+export function buildModelLockUpdate(model, cooldownMs) {
+  const key = getModelLockKey(model);
+  return { [key]: new Date(Date.now() + cooldownMs).toISOString() };
+}
+
+export function buildClearModelLocksUpdate(connection) {
+  const cleared = {};
+  for (const key of Object.keys(connection)) {
+    if (key.startsWith(MODEL_LOCK_PREFIX)) cleared[key] = null;
+  }
+  return cleared;
+}
+
+export function filterAvailableAccounts(accounts, excludeId = null) {
+  const now = Date.now();
+  return accounts.filter(acc => {
+    if (excludeId && acc.id === excludeId) return false;
+    if (acc.rateLimitedUntil) {
+      const until = new Date(acc.rateLimitedUntil).getTime();
+      if (until > now) return false;
+    }
+    return true;
+  });
+}
+
+export function resetAccountState(account) {
+  if (!account) return account;
+  return {
+    ...account,
+    rateLimitedUntil: null,
+    backoffLevel: 0,
+    lastError: null,
+    status: "active"
+  };
+}
+
+export function applyErrorState(account, status, errorText) {
+  if (!account) return account;
+
+  const backoffLevel = account.backoffLevel || 0;
+  const { cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel);
+
+  return {
+    ...account,
+    rateLimitedUntil: cooldownMs > 0 ? getUnavailableUntil(cooldownMs) : null,
+    backoffLevel: newBackoffLevel ?? backoffLevel,
+    lastError: { status, message: errorText, timestamp: new Date().toISOString() },
+    status: "error"
+  };
+}
+
+export const DEPRIORITIZE_FIELD = "deprioritizeUntil";
+
+export function isDeprioritized(connection) {
+  if (!connection) return false;
+  const until = connection[DEPRIORITIZE_FIELD];
+  if (!until) return false;
+  return isDeprioitized(Number(until) || Date.parse(until));
+}
+
+export function buildDeprioritizeUpdate(deprioitizeUntilMs) {
+  return { [DEPRIORITIZE_FIELD]: deprioitizeUntilMs || null };
+}
+
+export function buildClearDeprioritizeUpdate() {
+  return { [DEPRIORITIZE_FIELD]: null };
+}
+
+export function sortWithDeprioritization(connections) {
+  if (!Array.isArray(connections) || connections.length <= 1) return connections || [];
+  const cmp = (a, b) => (a.priority || 999) - (b.priority || 999);
+  const top = connections.filter((c) => !isDeprioritized(c)).sort(cmp);
+  const bottom = connections.filter((c) => isDeprioritized(c)).sort(cmp);
+  return [...top, ...bottom];
+}
+
+export function pickCacheAffine(connections, { cacheKey, excludeIds } = {}) {
+  const all = connections || [];
+  if (all.length === 0) return { connection: null, affinity: "miss" };
+  const excluded = excludeIds instanceof Set ? excludeIds : (excludeIds ? new Set(excludeIds) : null);
+
+  if (!cacheKey) {
+
+    const firstAvail = excluded ? all.find((c) => !excluded.has(c.id)) : all[0];
+    return { connection: firstAvail || null, affinity: "miss" };
+  }
+
+  const n = all.length;
+  const bucket = fnv1a(cacheKey) % n;
+  const primary = all[bucket] || null;
+
+  if (primary && (!excluded || !excluded.has(primary.id))) {
+    return { connection: primary, affinity: "hit" };
+  }
+
+  for (let step = 1; step <= n; step++) {
+    const cand = all[(bucket + step) % n];
+    if (!cand || (excluded && excluded.has(cand.id))) continue;
+    return { connection: cand, affinity: "rotate" };
+  }
+
+  return { connection: null, affinity: "rotate" };
+}
+
+export function extractCacheKey(body) {
+  if (!body || typeof body !== "object") return null;
+
+  const contentText = (content) => {
+    if (typeof content === "string") return content;
+    if (Array.isArray(content)) {
+      const parts = [];
+      for (const part of content) {
+        if (part && typeof part.text === "string") parts.push(part.text);
+        else if (part && typeof part.type === "string" && typeof part.text === "string") parts.push(part.text);
+      }
+      return parts.join(" ");
+    }
+    return "";
+  };
+
+  if (Array.isArray(body.messages) && body.messages.length > 0) {
+    for (const m of body.messages) {
+      const t = contentText(m?.content);
+      if (t) return t.slice(0, 256);
+    }
+  }
+
+  if (Array.isArray(body.contents) && body.contents.length > 0) {
+    const parts = body.contents[0]?.parts;
+    if (Array.isArray(parts)) {
+      const t = parts.map((p) => (p?.text ? String(p.text) : "")).join(" ");
+      if (t) return t.slice(0, 256);
+    }
+  }
+
+  if (typeof body.prompt === "string" && body.prompt) return body.prompt.slice(0, 256);
+  if (typeof body.input === "string" && body.input) return body.input.slice(0, 256);
+  return null;
+}

package/src/services/accountPool.js

@@ -0,0 +1,389 @@
+import { deriveResourceProfile } from "@/lib/resourceProfile";
+
+export const ACCOUNT_POOL_DEFAULTS = Object.freeze({
+  maxConcurrentPerAccount: 30,
+  maxConcurrentPerProvider: 64,
+  maxQueuePerProvider: 1024,
+  queueWaitTimeoutMs: 30_000,
+  cacheTtlMs: 5_000,
+});
+
+const GLOBAL_STATE_KEY = "__swayAccountPoolState";
+
+function createState() {
+  return {
+    accountLoads: new Map(),
+    providerLoads: new Map(),
+    providerQueues: new Map(),
+    providerCache: new Map(),
+  };
+}
+
+const state = globalThis[GLOBAL_STATE_KEY] ||= createState();
+state.providerLoads ||= new Map();
+
+function positiveInt(value, fallback, { min = 1, max = 1_000_000 } = {}) {
+  const number = Number(value);
+  if (!Number.isFinite(number) || number < min) return fallback;
+  return Math.min(Math.floor(number), max);
+}
+
+function firstDefined(...values) {
+  return values.find((value) => value !== undefined && value !== null && value !== "");
+}
+
+function deriveAutomaticProviderLimit() {
+  try {
+    const profile = deriveResourceProfile(undefined, process.env.SWAY_PERFORMANCE_PROFILE);
+    const configuredRouterMax = positiveInt(
+      process.env.SWAY_ROUTER_MAX_CONCURRENT,
+      profile.maxConcurrent,
+      { max: 100_000 },
+    );
+    return Math.max(1, Math.min(configuredRouterMax, profile.maxConcurrentPerProvider));
+  } catch {
+    return ACCOUNT_POOL_DEFAULTS.maxConcurrentPerProvider;
+  }
+}
+
+const AUTOMATIC_MAX_CONCURRENT_PER_PROVIDER = deriveAutomaticProviderLimit();
+
+export function resolveAccountPoolConfig(settings = {}, providerOverride = {}) {
+  const globalPool = settings.accountPool || {};
+  const providerPool = providerOverride.accountPool || {};
+  const env = process.env;
+
+  const enabled = firstDefined(
+    providerPool.enabled,
+    providerOverride.accountPoolEnabled,
+    globalPool.enabled,
+    settings.accountPoolEnabled,
+    true,
+  ) !== false;
+
+  return {
+    enabled,
+    maxConcurrentPerAccount: positiveInt(
+      firstDefined(
+        providerPool.maxConcurrentPerAccount,
+        providerOverride.maxConcurrentPerAccount,
+        globalPool.maxConcurrentPerAccount,
+        settings.maxConcurrentPerAccount,
+        env.SWAY_MAX_CONCURRENT_PER_ACCOUNT,
+        ACCOUNT_POOL_DEFAULTS.maxConcurrentPerAccount,
+      ),
+      ACCOUNT_POOL_DEFAULTS.maxConcurrentPerAccount,
+      { max: 100_000 },
+    ),
+    maxConcurrentPerProvider: positiveInt(
+      firstDefined(
+        providerPool.maxConcurrentPerProvider,
+        providerOverride.maxConcurrentPerProvider,
+        globalPool.maxConcurrentPerProvider,
+        settings.maxConcurrentPerProvider,
+        env.SWAY_MAX_CONCURRENT_PER_PROVIDER,
+        AUTOMATIC_MAX_CONCURRENT_PER_PROVIDER,
+      ),
+      AUTOMATIC_MAX_CONCURRENT_PER_PROVIDER,
+      { max: 100_000 },
+    ),
+    maxQueuePerProvider: positiveInt(
+      firstDefined(
+        providerPool.maxQueuePerProvider,
+        providerOverride.maxQueuePerProvider,
+        globalPool.maxQueuePerProvider,
+        settings.maxQueuePerProvider,
+        env.SWAY_MAX_ACCOUNT_QUEUE,
+        ACCOUNT_POOL_DEFAULTS.maxQueuePerProvider,
+      ),
+      ACCOUNT_POOL_DEFAULTS.maxQueuePerProvider,
+      { max: 1_000_000 },
+    ),
+    queueWaitTimeoutMs: positiveInt(
+      firstDefined(
+        providerPool.queueWaitTimeoutMs,
+        providerOverride.queueWaitTimeoutMs,
+        globalPool.queueWaitTimeoutMs,
+        settings.queueWaitTimeoutMs,
+        env.SWAY_ACCOUNT_QUEUE_TIMEOUT_MS,
+        ACCOUNT_POOL_DEFAULTS.queueWaitTimeoutMs,
+      ),
+      ACCOUNT_POOL_DEFAULTS.queueWaitTimeoutMs,
+      { max: 10 * 60 * 1000 },
+    ),
+    cacheTtlMs: positiveInt(
+      firstDefined(
+        providerPool.cacheTtlMs,
+        providerOverride.cacheTtlMs,
+        globalPool.cacheTtlMs,
+        settings.accountPoolCacheTtlMs,
+        env.SWAY_ACCOUNT_POOL_CACHE_TTL_MS,
+        ACCOUNT_POOL_DEFAULTS.cacheTtlMs,
+      ),
+      ACCOUNT_POOL_DEFAULTS.cacheTtlMs,
+      { max: 5 * 60 * 1000 },
+    ),
+  };
+}
+
+function getAccountLoadEntry(connectionId, provider = null) {
+  if (!connectionId) return null;
+  let entry = state.accountLoads.get(connectionId);
+  if (!entry) {
+    entry = { provider, active: 0, lastAssignedAt: 0 };
+    state.accountLoads.set(connectionId, entry);
+  } else if (provider && !entry.provider) {
+    entry.provider = provider;
+  }
+  return entry;
+}
+
+export function getAccountLoad(connectionId) {
+  return state.accountLoads.get(connectionId)?.active || 0;
+}
+
+export function getProviderLoad(provider) {
+  return state.providerLoads.get(provider)?.active || 0;
+}
+
+export function getAccountQueueLoad(connectionId) {
+  const provider = state.accountLoads.get(connectionId)?.provider;
+  if (!provider) return 0;
+  const queue = state.providerQueues.get(provider);
+  if (!queue) return 0;
+  return [...queue.waiters].filter((waiter) => !waiter.connectionIds || waiter.connectionIds.has(connectionId)).length;
+}
+
+export function pickLeastInflightConnection(connections = []) {
+  let best = null;
+  let bestActive = Infinity;
+  let bestLastAssigned = Infinity;
+  let bestPriority = Infinity;
+
+  for (const connection of connections) {
+    const active = getAccountLoad(connection?.id);
+    const lastAssigned = state.accountLoads.get(connection?.id)?.lastAssignedAt || 0;
+    const priority = Number.isFinite(Number(connection?.priority)) ? Number(connection.priority) : 999;
+    if (
+      active < bestActive
+      || (active === bestActive && lastAssigned < bestLastAssigned)
+      || (active === bestActive && lastAssigned === bestLastAssigned && priority < bestPriority)
+    ) {
+      best = connection;
+      bestActive = active;
+      bestLastAssigned = lastAssigned;
+      bestPriority = priority;
+    }
+  }
+  return best;
+}
+
+export function rankConnectionsByInflight(connections = []) {
+  return [...connections]
+    .map((connection, index) => ({ connection, index }))
+    .sort((left, right) => {
+      const a = left.connection;
+      const b = right.connection;
+      const activeDiff = getAccountLoad(a?.id) - getAccountLoad(b?.id);
+      if (activeDiff !== 0) return activeDiff;
+
+      const aLast = state.accountLoads.get(a?.id)?.lastAssignedAt || 0;
+      const bLast = state.accountLoads.get(b?.id)?.lastAssignedAt || 0;
+      if (aLast !== bLast) return aLast - bLast;
+
+      const aPriority = Number.isFinite(Number(a?.priority)) ? Number(a.priority) : 999;
+      const bPriority = Number.isFinite(Number(b?.priority)) ? Number(b.priority) : 999;
+      if (aPriority !== bPriority) return aPriority - bPriority;
+      return left.index - right.index;
+    })
+    .map(({ connection }) => connection);
+}
+
+function notifyProviderQueue(provider, releasedConnectionId) {
+  const queue = state.providerQueues.get(provider);
+  if (!queue || queue.waiters.size === 0) return;
+
+  for (const waiter of queue.waiters) {
+    if (waiter.connectionIds && !waiter.connectionIds.has(releasedConnectionId)) continue;
+    waiter.cleanup();
+    waiter.resolve({ ok: true, releasedConnectionId });
+
+    break;
+  }
+}
+
+function getProviderLoadEntry(provider) {
+  if (!provider) return null;
+  let entry = state.providerLoads.get(provider);
+  if (!entry) {
+    entry = { active: 0 };
+    state.providerLoads.set(provider, entry);
+  }
+  return entry;
+}
+
+export function tryAcquireAccountSlot(provider, connectionId, config = {}) {
+  if (!config.enabled || !connectionId) return () => {};
+
+  const entry = getAccountLoadEntry(connectionId, provider);
+  const providerEntry = getProviderLoadEntry(provider);
+  const max = positiveInt(config.maxConcurrentPerAccount, ACCOUNT_POOL_DEFAULTS.maxConcurrentPerAccount, { max: 100_000 });
+  const providerMax = positiveInt(
+    config.maxConcurrentPerProvider,
+    AUTOMATIC_MAX_CONCURRENT_PER_PROVIDER,
+    { max: 100_000 },
+  );
+  if (entry.active >= max || providerEntry.active >= providerMax) return null;
+
+  entry.active += 1;
+  providerEntry.active += 1;
+  entry.lastAssignedAt = Date.now();
+  let released = false;
+
+  return () => {
+    if (released) return;
+    released = true;
+    entry.active = Math.max(0, entry.active - 1);
+    providerEntry.active = Math.max(0, providerEntry.active - 1);
+    notifyProviderQueue(provider, connectionId);
+  };
+}
+
+function providerQueue(provider) {
+  let queue = state.providerQueues.get(provider);
+  if (!queue) {
+    queue = { waiters: new Set() };
+    state.providerQueues.set(provider, queue);
+  }
+  return queue;
+}
+
+function hasCapacity(provider, connectionIds, config) {
+  const max = positiveInt(
+    config.maxConcurrentPerAccount,
+    ACCOUNT_POOL_DEFAULTS.maxConcurrentPerAccount,
+    { max: 100_000 },
+  );
+  const providerMax = positiveInt(
+    config.maxConcurrentPerProvider,
+    AUTOMATIC_MAX_CONCURRENT_PER_PROVIDER,
+    { max: 100_000 },
+  );
+  if (getProviderLoad(provider) >= providerMax) return false;
+  return connectionIds.some((connectionId) => {
+    const entry = state.accountLoads.get(connectionId);
+    return !entry || entry.active < max;
+  });
+}
+
+export function waitForAccountCapacity(provider, connectionIds, config = {}, signal = null) {
+  if (!config.enabled) return Promise.resolve({ ok: true, disabled: true });
+
+  const wildcard = connectionIds == null;
+  const ids = wildcard ? null : [...new Set((connectionIds || []).filter(Boolean))];
+  if (!wildcard && (ids.length === 0 || hasCapacity(provider, ids, config))) {
+    return Promise.resolve({ ok: true, immediate: true });
+  }
+
+  const queue = providerQueue(provider);
+  const maxQueue = positiveInt(config.maxQueuePerProvider, ACCOUNT_POOL_DEFAULTS.maxQueuePerProvider, { max: 1_000_000 });
+  if (queue.waiters.size >= maxQueue) {
+    return Promise.resolve({ ok: false, reason: "queue_full" });
+  }
+
+  if (signal?.aborted) return Promise.resolve({ ok: false, reason: "aborted" });
+
+  return new Promise((resolve) => {
+    const waiter = {
+      connectionIds: ids ? new Set(ids) : null,
+      timer: null,
+      abortHandler: null,
+      cleanup: () => {
+        queue.waiters.delete(waiter);
+        if (waiter.timer) clearTimeout(waiter.timer);
+        if (signal && waiter.abortHandler) signal.removeEventListener("abort", waiter.abortHandler);
+      },
+      resolve,
+    };
+
+    const finish = (result) => {
+      waiter.cleanup();
+      resolve(result);
+    };
+
+    waiter.resolve = finish;
+    waiter.timer = setTimeout(() => finish({ ok: false, reason: "timeout" }), config.queueWaitTimeoutMs);
+    waiter.timer?.unref?.();
+    waiter.abortHandler = () => finish({ ok: false, reason: "aborted" });
+    signal?.addEventListener("abort", waiter.abortHandler, { once: true });
+    queue.waiters.add(waiter);
+  });
+}
+
+function toRoutingConnection(connection) {
+  const routing = { ...connection };
+  for (const key of [
+    "apiKey", "accessToken", "refreshToken", "idToken", "copilotToken",
+    "providerSpecificData", "tokenType", "scope", "expiresAt", "expiresIn",
+  ]) delete routing[key];
+  return routing;
+}
+
+function pruneInactiveAccountLoads(provider, activeIds) {
+  for (const [connectionId, entry] of state.accountLoads.entries()) {
+    if (entry.provider === provider && entry.active === 0 && !activeIds.has(connectionId)) {
+      state.accountLoads.delete(connectionId);
+    }
+  }
+}
+
+export async function getCachedProviderConnections(provider, loader, ttlMs = Number(ACCOUNT_POOL_DEFAULTS.cacheTtlMs)) {
+  const key = String(provider || "");
+  const now = Date.now();
+  const hit = state.providerCache.get(key);
+  if (hit?.value && hit.expiresAt > now) return hit.value;
+  if (hit?.promise) return hit.promise;
+
+  const promise = Promise.resolve()
+    .then(loader)
+    .then((connections) => {
+      const value = Array.isArray(connections) ? connections.map(toRoutingConnection) : [];
+      pruneInactiveAccountLoads(key, new Set(value.map((connection) => connection.id)));
+      state.providerCache.set(key, { value, expiresAt: Date.now() + Math.max(0, ttlMs) });
+      return value;
+    })
+    .catch((error) => {
+      state.providerCache.delete(key);
+      throw error;
+    });
+
+  state.providerCache.set(key, { promise });
+  return promise;
+}
+
+export function invalidateProviderAccountPool(provider) {
+  if (provider) state.providerCache.delete(String(provider));
+  else state.providerCache.clear();
+}
+
+export function getAccountPoolSnapshot(provider = null) {
+  const accounts = [];
+  for (const [connectionId, entry] of state.accountLoads.entries()) {
+    if (provider && entry.provider !== provider) continue;
+    accounts.push({ connectionId, provider: entry.provider, active: entry.active });
+  }
+  return {
+    provider,
+    activeProvider: provider ? getProviderLoad(provider) : null,
+    activeProviders: [...state.providerLoads.entries()].map(([id, entry]) => ({ provider: id, active: entry.active })),
+    activeAccounts: accounts,
+    queued: provider ? (state.providerQueues.get(provider)?.waiters.size || 0) : [...state.providerQueues.values()].reduce((sum, queue) => sum + queue.waiters.size, 0),
+  };
+}
+
+export function resetAccountPoolForTests() {
+  state.accountLoads.clear();
+  state.providerLoads.clear();
+  state.providerQueues.clear();
+  state.providerCache.clear();
+}