triflux 10.4.0 → 10.7.0

package/hub/account-broker.mjs

@@ -1,35 +1,45 @@
 // hub/account-broker.mjs — Multi-account CLI pool broker
-// Manages lease/release/cooldown for Codex and Gemini accounts.
+// Manages lease/release/cooldown/circuit-breaker for Codex and Gemini accounts.
+// Per-account circuit breaker: one bad account does not block others.
 // Singleton export. All state changes create new objects (immutable pattern).
 
-import { readFileSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
-import { homedir } from 'node:os';
-import * as z from 'zod';
+import { EventEmitter } from "node:events";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, sep } from "node:path";
+import * as z from "zod";
 
 // ── Zod schema ───────────────────────────────────────────────────
 
-const AccountSchema = z.object({
-  id: z.string().min(1),
-  mode: z.enum(['profile', 'env', 'auth']),
-  profile: z.string().optional(),
-  env: z.record(z.string(), z.string()).optional(),
-  authFile: z.string().optional(),
-  tier: z.enum(['pro', 'plus', 'free', 'unknown']).optional().default('unknown'),
-}).superRefine((val, ctx) => {
-  if (val.mode === 'auth' && !val.authFile) {
-    ctx.addIssue({
-      code: z.ZodIssueCode.custom,
-      message: 'authFile is required when mode is "auth"',
-      path: ['authFile'],
-    });
-  }
-});
+const AccountSchema = z
+  .object({
+    id: z.string().min(1),
+    mode: z.enum(["profile", "env", "auth"]),
+    profile: z.string().optional(),
+    env: z.record(z.string(), z.string()).optional(),
+    authFile: z.string().optional(),
+    host: z.string().min(1).optional(),
+    tier: z
+      .enum(["pro", "plus", "free", "unknown"])
+      .optional()
+      .default("unknown"),
+  })
+  .superRefine((val, ctx) => {
+    if (val.mode === "auth" && !val.authFile) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'authFile is required when mode is "auth"',
+        path: ["authFile"],
+      });
+    }
+  });
 
 const ConfigSchema = z.object({
-  defaults: z.object({
-    cooldownMs: z.number().int().positive().optional(),
-  }).optional(),
+  defaults: z
+    .object({
+      cooldownMs: z.number().int().positive().optional(),
+    })
+    .optional(),
   codex: z.array(AccountSchema).optional(),
   gemini: z.array(AccountSchema).optional(),
 });
@@ -37,7 +47,9 @@ const ConfigSchema = z.object({
 const DEFAULT_COOLDOWN_MS = 300_000; // 5 minutes
 const TIER_PRIORITY = { pro: 0, plus: 1, unknown: 2, free: 3 };
 const LEASE_TTL_MS = 30 * 60 * 1000; // 30 minutes
-const AUTH_BASE_PATH = join(homedir(), '.claude', 'cache', 'tfx-hub');
+const CIRCUIT_WINDOW_MS = 10 * 60_000; // 10 minutes
+const CIRCUIT_MAX_FAILURES = 3;
+const AUTH_BASE_PATH = join(homedir(), ".claude", "cache", "tfx-hub");
 
 // ── env var resolution ───────────────────────────────────────────
 
@@ -45,9 +57,9 @@ function resolveEnvValues(env) {
   if (!env) return undefined;
   const resolved = {};
   for (const [key, value] of Object.entries(env)) {
-    if (typeof value === 'string' && value.startsWith('$')) {
+    if (typeof value === "string" && value.startsWith("$")) {
       const varName = value.slice(1);
-      resolved[key] = process.env[varName] ?? '';
+      resolved[key] = process.env[varName] ?? "";
     } else {
       resolved[key] = value;
     }
@@ -55,14 +67,24 @@ function resolveEnvValues(env) {
   return resolved;
 }
 
+function isRemoteAccount(account) {
+  return Boolean(account.host);
+}
+
+function getRemainingLeaseMs(account, now) {
+  if (!account.busy || account.leasedAt === null) return 0;
+  return Math.max(0, LEASE_TTL_MS - (now - account.leasedAt));
+}
+
 // ── AccountBroker ────────────────────────────────────────────────
 
-class AccountBroker {
+class AccountBroker extends EventEmitter {
   #config;
   #state; // Map<accountId, accountState>
   #roundRobinIndex; // Map<provider, number>
 
   constructor(config) {
+    super();
     const parsed = ConfigSchema.parse(config);
     this.#config = parsed;
 
@@ -70,8 +92,8 @@ class AccountBroker {
     this.#roundRobinIndex = new Map();
 
     const allAccounts = [
-      ...(parsed.codex || []).map((a) => ({ ...a, provider: 'codex' })),
-      ...(parsed.gemini || []).map((a) => ({ ...a, provider: 'gemini' })),
+      ...(parsed.codex || []).map((a) => ({ ...a, provider: "codex" })),
+      ...(parsed.gemini || []).map((a) => ({ ...a, provider: "gemini" })),
     ];
 
     for (const account of allAccounts) {
@@ -82,22 +104,73 @@ class AccountBroker {
         profile: account.profile,
         env: account.env,
         authFile: account.authFile,
-        tier: account.tier ?? 'unknown',
+        host: account.host,
+        tier: account.tier ?? "unknown",
         busy: false,
         leasedAt: null,
         cooldownUntil: 0,
-        failures: 0,
+        // per-account circuit breaker state
+        failureTimestamps: [], // timestamp array for window-based decay
+        circuitOpenedAt: 0,
+        circuitTrialInFlight: false,
         lastUsedAt: 0,
         totalSessions: 0,
       });
     }
   }
 
+  // ── per-account circuit breaker ─────────────────────────────────
+
+  #getCircuitState(acct, now) {
+    const validFailures = acct.failureTimestamps.filter(
+      (ts) => now - ts < CIRCUIT_WINDOW_MS,
+    );
+    const withinWindow =
+      acct.circuitOpenedAt && now - acct.circuitOpenedAt < CIRCUIT_WINDOW_MS;
+    if (withinWindow) return { state: "open", failures: validFailures };
+    if (acct.circuitOpenedAt)
+      return { state: "half-open", failures: validFailures };
+    return { state: "closed", failures: validFailures };
+  }
+
+  #isCircuitBlocked(acct, now) {
+    const circuit = this.#getCircuitState(acct, now);
+    if (circuit.state === "open") return true;
+    if (circuit.state === "half-open" && acct.circuitTrialInFlight) return true;
+    return false;
+  }
+
+  #recordCircuitFailure(acct, isHalfOpen, now) {
+    const validFailures = [
+      ...acct.failureTimestamps.filter((ts) => now - ts < CIRCUIT_WINDOW_MS),
+      now,
+    ];
+    const shouldOpen =
+      isHalfOpen || validFailures.length >= CIRCUIT_MAX_FAILURES;
+    return {
+      failureTimestamps: validFailures,
+      circuitOpenedAt: shouldOpen ? now : acct.circuitOpenedAt,
+      circuitTrialInFlight: false,
+    };
+  }
+
+  #resetCircuit() {
+    return {
+      failureTimestamps: [],
+      circuitOpenedAt: 0,
+      circuitTrialInFlight: false,
+    };
+  }
+
   // ── lease TTL pruning ──────────────────────────────────────────
 
   #pruneExpiredLeases(now) {
     for (const [id, acct] of this.#state) {
-      if (acct.busy && acct.leasedAt !== null && now - acct.leasedAt > LEASE_TTL_MS) {
+      if (
+        acct.busy &&
+        acct.leasedAt !== null &&
+        now - acct.leasedAt > LEASE_TTL_MS
+      ) {
         this.#state.set(id, { ...acct, busy: false, leasedAt: null });
       }
     }
@@ -105,16 +178,36 @@ class AccountBroker {
 
   // ── lease ─────────────────────────────────────────────────────
 
-  lease({ provider }) {
+  lease({ provider, remote = false } = {}) {
     const now = Date.now();
     this.#pruneExpiredLeases(now);
 
-    const accounts = [...this.#state.values()].filter((a) => a.provider === provider);
+    const wantsRemote = remote === true;
+    const accounts = [...this.#state.values()].filter(
+      (a) => a.provider === provider && isRemoteAccount(a) === wantsRemote,
+    );
     if (!accounts.length) return null;
 
-    // group available accounts by tier, preserving insertion order within each tier
-    const available = accounts.filter((a) => !a.busy && a.cooldownUntil <= now);
-    if (!available.length) return null;
+    // filter: not busy, not in cooldown, circuit not blocked
+    const available = accounts.filter(
+      (a) =>
+        !a.busy && a.cooldownUntil <= now && !this.#isCircuitBlocked(a, now),
+    );
+
+    if (!available.length) {
+      // check if any accounts exist but all are blocked by circuit
+      const circuitBlocked = accounts.filter(
+        (a) =>
+          !a.busy && a.cooldownUntil <= now && this.#isCircuitBlocked(a, now),
+      );
+      if (circuitBlocked.length) {
+        this.emit("noAvailableAccounts", {
+          provider,
+          count: circuitBlocked.length,
+        });
+      }
+      return null;
+    }
 
     // sort by tier priority; stable sort preserves original order within same priority
     const sorted = [...available].sort(
@@ -125,8 +218,23 @@ class AccountBroker {
     const bestTier = sorted[0].tier;
     const sameTierAccounts = sorted.filter((a) => a.tier === bestTier);
 
+    // detect tier fallback
+    const highestTier = accounts.reduce(
+      (best, a) => Math.min(best, TIER_PRIORITY[a.tier] ?? 2),
+      Infinity,
+    );
+    if ((TIER_PRIORITY[bestTier] ?? 2) > highestTier) {
+      this.emit("tierFallback", {
+        provider,
+        from: Object.entries(TIER_PRIORITY).find(
+          ([, v]) => v === highestTier,
+        )?.[0],
+        to: bestTier,
+      });
+    }
+
     // use a per-provider+tier round-robin key to distribute within the tier
-    const rrKey = `${provider}:${bestTier}`;
+    const rrKey = `${provider}:${bestTier}:${wantsRemote ? "remote" : "local"}`;
     const rrCurrent = this.#roundRobinIndex.get(rrKey) ?? 0;
     const tierCount = sameTierAccounts.length;
     const idx = rrCurrent % tierCount;
@@ -135,6 +243,10 @@ class AccountBroker {
     // advance round-robin index for this tier
     this.#roundRobinIndex.set(rrKey, (idx + 1) % tierCount);
 
+    // mark half-open trial if applicable
+    const circuit = this.#getCircuitState(acct, now);
+    const isHalfOpen = circuit.state === "half-open";
+
     // update state (immutable)
     this.#state.set(acct.id, {
       ...acct,
@@ -142,14 +254,45 @@ class AccountBroker {
       leasedAt: now,
       lastUsedAt: now,
       totalSessions: acct.totalSessions + 1,
+      circuitTrialInFlight: isHalfOpen ? true : acct.circuitTrialInFlight,
+    });
+
+    this.emit("lease", {
+      id: acct.id,
+      provider,
+      tier: acct.tier,
+      halfOpen: isHalfOpen,
     });
 
+    // path traversal guard for authFile
+    let authFile;
+    if (acct.mode === "auth") {
+      const resolved = join(AUTH_BASE_PATH, acct.authFile);
+      if (!resolved.startsWith(AUTH_BASE_PATH + sep)) {
+        this.emit("securityViolation", {
+          id: acct.id,
+          authFile: acct.authFile,
+        });
+        // undo the lease — path traversal blocked
+        this.#state.set(acct.id, {
+          ...this.#state.get(acct.id),
+          busy: false,
+          leasedAt: null,
+        });
+        return null;
+      }
+      authFile = resolved;
+    }
+
     return {
       id: acct.id,
       mode: acct.mode,
-      profile: acct.mode === 'profile' ? acct.profile : undefined,
-      env: acct.mode === 'env' ? resolveEnvValues(acct.env) : undefined,
-      authFile: acct.mode === 'auth' ? join(AUTH_BASE_PATH, acct.authFile) : undefined,
+      remote: isRemoteAccount(acct),
+      host: acct.host,
+      halfOpen: isHalfOpen,
+      profile: acct.mode === "profile" ? acct.profile : undefined,
+      env: acct.mode === "env" ? resolveEnvValues(acct.env) : undefined,
+      authFile,
     };
   }
 
@@ -157,26 +300,45 @@ class AccountBroker {
 
   release(accountId, result) {
     const acct = this.#state.get(accountId);
-    if (!acct) return;
+    if (!acct?.busy) return;
 
+    const now = Date.now();
     const ok = result?.ok === true;
-    const newFailures = ok ? 0 : acct.failures + 1;
+    const circuit = this.#getCircuitState(acct, now);
+    const isHalfOpen = circuit.state === "half-open";
+
+    let circuitUpdate;
+    if (ok) {
+      circuitUpdate = this.#resetCircuit();
+      if (isHalfOpen) {
+        this.emit("circuitClose", { id: accountId });
+      }
+    } else {
+      circuitUpdate = this.#recordCircuitFailure(acct, isHalfOpen, now);
+      if (circuitUpdate.circuitOpenedAt !== acct.circuitOpenedAt) {
+        this.emit("circuitOpen", {
+          id: accountId,
+          failures: circuitUpdate.failureTimestamps.length,
+        });
+      }
+    }
+
     const cooldownMs = this.#config.defaults?.cooldownMs ?? DEFAULT_COOLDOWN_MS;
 
+    // rate-limit style cooldown: if circuit just opened, also set cooldown
+    const shouldCooldown =
+      !ok && circuitUpdate.circuitOpenedAt !== acct.circuitOpenedAt;
+
     const updated = {
       ...acct,
       busy: false,
       leasedAt: null,
-      failures: newFailures,
+      ...circuitUpdate,
+      cooldownUntil: shouldCooldown ? now + cooldownMs : acct.cooldownUntil,
     };
 
-    // consecutive failure guard: 3+ failures → auto-cooldown
-    if (newFailures >= 3) {
-      updated.cooldownUntil = Date.now() + cooldownMs;
-      updated.failures = 0; // reset after cooldown triggered
-    }
-
     this.#state.set(accountId, updated);
+    this.emit("release", { id: accountId, ok });
   }
 
   // ── markRateLimited ───────────────────────────────────────────
@@ -195,7 +357,14 @@ class AccountBroker {
   // ── snapshot ──────────────────────────────────────────────────
 
   snapshot() {
-    return [...this.#state.values()].map((acct) => ({ ...acct }));
+    const now = Date.now();
+    this.#pruneExpiredLeases(now);
+    return [...this.#state.values()].map((acct) => ({
+      ...acct,
+      failureTimestamps: [...acct.failureTimestamps],
+      remainingMs: getRemainingLeaseMs(acct, now),
+      circuitState: this.#getCircuitState(acct, now).state,
+    }));
   }
 
   // ── nextAvailableEta ──────────────────────────────────────────
@@ -204,7 +373,9 @@ class AccountBroker {
     const now = Date.now();
     this.#pruneExpiredLeases(now);
 
-    const accounts = [...this.#state.values()].filter((a) => a.provider === provider);
+    const accounts = [...this.#state.values()].filter(
+      (a) => a.provider === provider,
+    );
     if (!accounts.length) return null;
 
     // find minimum cooldownUntil among accounts that are in cooldown or busy
@@ -214,7 +385,9 @@ class AccountBroker {
         // this account is available now — no ETA needed
         return null;
       }
-      const eta = acct.busy ? (acct.leasedAt ?? now) + LEASE_TTL_MS : acct.cooldownUntil;
+      const eta = acct.busy
+        ? (acct.leasedAt ?? now) + LEASE_TTL_MS
+        : acct.cooldownUntil;
       if (earliest === null || eta < earliest) {
         earliest = eta;
       }
@@ -226,11 +399,21 @@ class AccountBroker {
 // ── Config loader ────────────────────────────────────────────────
 
 function loadConfig() {
-  const configPath = join(homedir(), '.claude', 'cache', 'tfx-hub', 'accounts.json');
+  const configPath = join(
+    homedir(),
+    ".claude",
+    "cache",
+    "tfx-hub",
+    "accounts.json",
+  );
   if (!existsSync(configPath)) return null;
   try {
-    return JSON.parse(readFileSync(configPath, 'utf8'));
-  } catch {
+    return JSON.parse(readFileSync(configPath, "utf8"));
+  } catch (err) {
+    console.error(
+      "[account-broker] Failed to parse accounts.json:",
+      err.message,
+    );
     return null;
   }
 }
@@ -242,10 +425,23 @@ function createBroker() {
   if (!config) return null;
   try {
     return new AccountBroker(config);
-  } catch {
+  } catch (err) {
+    console.error("[account-broker] Failed to create broker:", err.message);
     return null;
   }
 }
 
-export const broker = createBroker();
-export { AccountBroker };
+/** Re-read config and replace the module-level singleton. ESM live binding propagates to all importers. */
+function reloadBroker() {
+  const config = loadConfig();
+  if (!config) return { ok: false, error: "Config not found or invalid" };
+  try {
+    broker = new AccountBroker(config);
+    return { ok: true, broker };
+  } catch (err) {
+    return { ok: false, error: err.message };
+  }
+}
+
+export let broker = createBroker();
+export { AccountBroker, reloadBroker };