vidspotai-shared 1.0.81 → 1.0.82

package/lib/services/aiGen/providers/google/googleApiKeys.js

@@ -0,0 +1,137 @@
+"use strict";
+/**
+ * Pure (SDK-free, Redis-free) helpers for the Google AI Studio API key pool.
+ *
+ * ── DECISION RECORD (2026-06-15) ────────────────────────────────────────────
+ * Problem: Veo on a single Gemini Developer API key is quota-starved — Tier 2
+ * is ~4 req/min and ~50/day per Veo model — which serialized bursts of jobs
+ * into a multi-hour queue (~370 min observed avg latency).
+ *
+ * Chosen fix (today): a multi-key pool over TWO Google AI Studio keys that live
+ * in SEPARATE GCP projects / billing accounts, so their quotas are independent
+ * and ADD UP. Keys are used in PRIORITY order:
+ *   - key[0] = the NEW key (vidspotai project), currently **Tier 1** (2/min,
+ *     10/day per Veo model). Used FIRST, deliberately, to drive usage and
+ *     promote its billing account up the tier ladder.
+ *   - key[1] = the CURRENT key, **Tier 2** (4/min, 50/day). Used once key[0]
+ *     is out of per-minute / per-day budget.
+ * Aggregate Veo budget = T1 + T2 = 6/min, 60/day. When BOTH are exhausted, the
+ * job-start capacity selector (videoJobProcessor) spills to another provider.
+ *
+ * When key[1]'s account is billed it moves to Tier 3; bump its tier in
+ * GOOGLE_API_KEY_TIERS then (no code change — the ladder below handles it).
+ *
+ * Vertex AI (DEFERRED, on record for the future): Veo is also available via
+ * Vertex, where quota is **per-project** and the current billing account is
+ * Tier-2-per-project — so we could create multiple projects under one account
+ * to scale further. We are NOT doing Vertex now: the two AI-Studio keys across
+ * two billing accounts already cover our needed headroom, and the Vertex path
+ * needs a different output flow (GCS `gs://` URIs rather than the Files API)
+ * that we'd rather build + test deliberately when the extra capacity is needed.
+ *
+ * ── CONFIG ──────────────────────────────────────────────────────────────────
+ *   GOOGLE_API_KEYS       — comma-separated keys in PRIORITY order. Falls back
+ *                           to the single legacy GOOGLE_API_KEY when unset.
+ *   GOOGLE_API_KEY_TIERS  — comma-separated tier numbers (1/2/3) aligned to
+ *                           GOOGLE_API_KEYS. Missing/extra entries default to 2.
+ *   GOOGLE_API_KEY        — legacy single key; also the client used to poll
+ *                           tasks submitted before the pool existed (un-tagged).
+ *
+ * Per-key budget is derived from the model config's Tier-2 baseline
+ * (requestPerMin / requestPerDay) scaled by each key's tier (see TIER_FACTORS);
+ * the pool gives us the SUM across keys.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseGoogleApiKeys = parseGoogleApiKeys;
+exports.parseGoogleApiKeyTiers = parseGoogleApiKeyTiers;
+exports.googleApiKeyCount = googleApiKeyCount;
+exports.scaleLimitForTier = scaleLimitForTier;
+exports.googleAggregateFactor = googleAggregateFactor;
+exports.googleKeyId = googleKeyId;
+exports.encodeVeoTask = encodeVeoTask;
+exports.decodeVeoTask = decodeVeoTask;
+/**
+ * Per-Veo-model rate ladder, expressed as a factor of the Tier-2 baseline that
+ * the model configs encode. Verified against Google's Veo quota
+ * (predict_long_running_requests_per_model): RPM 2/4/10 and RPD 10/50/500 for
+ * Tier 1/2/3. Only Veo configures per-key min/day caps today, so these factors
+ * only materially affect Veo; models with no caps (factor × 0 = 0) are
+ * unaffected.
+ */
+const TIER_FACTORS = {
+    1: { rpm: 2 / 4, rpd: 10 / 50 }, // 0.5×, 0.2×
+    2: { rpm: 1, rpd: 1 },
+    3: { rpm: 10 / 4, rpd: 500 / 50 }, // 2.5×, 10×
+};
+function tierFactor(tier) {
+    return TIER_FACTORS[tier] ?? TIER_FACTORS[2];
+}
+/** Parse the configured keys in priority order, de-duped, empties removed. */
+function parseGoogleApiKeys() {
+    const multi = (process.env.GOOGLE_API_KEYS ?? "").trim();
+    const raw = multi || (process.env.GOOGLE_API_KEY ?? "").trim();
+    const seen = new Set();
+    const keys = [];
+    for (const part of raw.split(",")) {
+        const k = part.trim();
+        if (k && !seen.has(k)) {
+            seen.add(k);
+            keys.push(k);
+        }
+    }
+    return keys;
+}
+/** Tier per key (aligned to parseGoogleApiKeys order); defaults to 2. */
+function parseGoogleApiKeyTiers() {
+    const keys = parseGoogleApiKeys();
+    const raw = (process.env.GOOGLE_API_KEY_TIERS ?? "").trim();
+    const parsed = raw
+        ? raw.split(",").map((s) => Number(s.trim()))
+        : [];
+    return keys.map((_, i) => {
+        const t = parsed[i];
+        return t === 1 || t === 2 || t === 3 ? t : 2;
+    });
+}
+/** Number of configured Google keys (>= 1 so callers can multiply safely). */
+function googleApiKeyCount() {
+    return Math.max(1, parseGoogleApiKeys().length);
+}
+/** Per-key rate cap for a given Tier-2 baseline and tier. */
+function scaleLimitForTier(baseline, tier, kind) {
+    if (!baseline)
+        return 0;
+    return Math.max(1, Math.round(baseline * tierFactor(tier)[kind]));
+}
+/**
+ * Aggregate multiplier across all configured keys, for scaling a Google
+ * model's Tier-2 baseline into the pool-wide budget. Returns 1 for a single
+ * default key (legacy behavior unchanged).
+ */
+function googleAggregateFactor(kind) {
+    const tiers = parseGoogleApiKeyTiers();
+    if (tiers.length <= 1)
+        return 1; // legacy single key → baseline unchanged
+    return tiers.reduce((sum, t) => sum + tierFactor(t)[kind], 0);
+}
+/** Stable id for the key at a given priority index. */
+function googleKeyId(index) {
+    return `k${index}`;
+}
+const TASK_TAG_RE = /^gk:([a-z0-9]+)::([\s\S]+)$/;
+/**
+ * Tag a Veo operation name with the id of the key that created it, so polling
+ * + download can re-select the same project-scoped client. With a single key
+ * we return the bare operation name (no tag) — byte-for-byte the legacy format,
+ * so nothing changes until a real pool is configured.
+ */
+function encodeVeoTask(keyId, operationName, poolSize) {
+    return poolSize > 1 ? `gk:${keyId}::${operationName}` : operationName;
+}
+/** Reverse of encodeVeoTask. `keyId` is undefined for legacy/un-tagged tasks. */
+function decodeVeoTask(task) {
+    const m = TASK_TAG_RE.exec(task);
+    if (m)
+        return { keyId: m[1], operationName: m[2] };
+    return { operationName: task };
+}

package/lib/services/aiGen/providers/google/googleKeyPool.d.ts

@@ -0,0 +1,52 @@
+import { GoogleGenAI } from "@google/genai";
+import { googleApiKeyCount } from "./googleApiKeys";
+/**
+ * Google AI Studio multi-key pool.
+ *
+ * Each configured key is a separate GCP project / billing account with its own
+ * Tier-2 quota, so N keys give us N× the per-model daily + per-minute budget.
+ * The model-level rate limiter (DistributedRateLimiter) enforces the AGGREGATE
+ * (per-key config × key count — see getAiGenModelRateLimiter); this pool decides
+ * WHICH key serves each individual submit, in priority order, so we drain the
+ * first key's budget before leaning on the next.
+ *
+ * Veo operations are project-scoped — a task submitted with key K can only be
+ * polled / downloaded with key K — so submit tags the task with its key id
+ * (encodeVeoTask) and the poll path re-selects the client via clientById().
+ *
+ * Single-key / unconfigured pool: behaves exactly like the legacy single
+ * `GoogleGenAI({ apiKey })` client (no tagging, no Redis routing).
+ */
+interface KeyEntry {
+    id: string;
+    apiKey: string;
+    client: GoogleGenAI;
+    /** Gemini API tier (1/2/3) of this key's billing account. */
+    tier: number;
+}
+declare class GoogleKeyPool {
+    private readonly entries;
+    constructor();
+    get size(): number;
+    /** The highest-priority key's client (index 0). */
+    get primaryClient(): GoogleGenAI;
+    /** Resolve a tagged task's key id back to its client (undefined if unknown). */
+    clientById(id: string | undefined): GoogleGenAI | undefined;
+    /**
+     * Pick the key that should serve a new submission. Walks keys in priority
+     * order and returns the first whose per-key minute + day budgets still have
+     * headroom, consuming a slot on it. The caps are the model's Tier-2 baseline
+     * (baselineMin / baselineDay) scaled to each key's own tier — so a Tier-1
+     * key gets a smaller share than a Tier-2 one. Falls back to the primary key
+     * if Redis is down or all are at cap (the aggregate model-level gate should
+     * have prevented the latter).
+     */
+    pickForSubmit(modelId: string, baselineMin: number, baselineDay: number): Promise<KeyEntry>;
+    /** Best-effort increment of a key's per-day + per-minute routing counters. */
+    private consume;
+}
+/** Lazily-built process-wide pool. */
+export declare function getGoogleKeyPool(): GoogleKeyPool;
+export { googleApiKeyCount };
+export type { GoogleKeyPool };
+//# sourceMappingURL=googleKeyPool.d.ts.map

package/lib/services/aiGen/providers/google/googleKeyPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"googleKeyPool.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/google/googleKeyPool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,OAAO,EACL,iBAAiB,EAKlB,MAAM,iBAAiB,CAAC;AAEzB;;;;;;;;;;;;;;;;GAgBG;AAEH,UAAU,QAAQ;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;CACd;AAcD,cAAM,aAAa;IACjB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;;IA4BrC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,mDAAmD;IACnD,IAAI,aAAa,IAAI,WAAW,CAE/B;IAED,gFAAgF;IAChF,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS;IAK3D;;;;;;;;OAQG;IACG,aAAa,CACjB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,QAAQ,CAAC;IA2CpB,8EAA8E;YAChE,OAAO;CAUtB;AAID,sCAAsC;AACtC,wBAAgB,gBAAgB,IAAI,aAAa,CAGhD;AAED,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAC7B,YAAY,EAAE,aAAa,EAAE,CAAC"}

package/lib/services/aiGen/providers/google/googleKeyPool.js

@@ -0,0 +1,129 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.googleApiKeyCount = void 0;
+exports.getGoogleKeyPool = getGoogleKeyPool;
+const genai_1 = require("@google/genai");
+const logger_1 = require("../../../../utils/logger");
+const redis_service_1 = require("../../../redis.service");
+const googleApiKeys_1 = require("./googleApiKeys");
+Object.defineProperty(exports, "googleApiKeyCount", { enumerable: true, get: function () { return googleApiKeys_1.googleApiKeyCount; } });
+function utcDateKey() {
+    return new Date().toISOString().slice(0, 10);
+}
+function secsUntilMidnight() {
+    const now = new Date();
+    const midnight = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1));
+    return Math.ceil((midnight.getTime() - now.getTime()) / 1000);
+}
+class GoogleKeyPool {
+    constructor() {
+        const keys = (0, googleApiKeys_1.parseGoogleApiKeys)();
+        const tiers = (0, googleApiKeys_1.parseGoogleApiKeyTiers)();
+        this.entries = keys.map((apiKey, i) => ({
+            id: (0, googleApiKeys_1.googleKeyId)(i),
+            apiKey,
+            client: new genai_1.GoogleGenAI({ apiKey }),
+            tier: tiers[i] ?? 2,
+        }));
+        if (!this.entries.length) {
+            // Mirror the legacy constructor: a bang-asserted GOOGLE_API_KEY. Building
+            // a client with undefined defers the failure to first call, same as before.
+            this.entries.push({
+                id: (0, googleApiKeys_1.googleKeyId)(0),
+                apiKey: process.env.GOOGLE_API_KEY ?? "",
+                client: new genai_1.GoogleGenAI({ apiKey: process.env.GOOGLE_API_KEY }),
+                tier: 2,
+            });
+        }
+        if (this.entries.length > 1) {
+            logger_1.logger.info("googleKeyPool: multi-key pool active", {
+                keys: this.entries.map((e) => ({ id: e.id, tier: e.tier })),
+            });
+        }
+    }
+    get size() {
+        return this.entries.length;
+    }
+    /** The highest-priority key's client (index 0). */
+    get primaryClient() {
+        return this.entries[0].client;
+    }
+    /** Resolve a tagged task's key id back to its client (undefined if unknown). */
+    clientById(id) {
+        if (!id)
+            return undefined;
+        return this.entries.find((e) => e.id === id)?.client;
+    }
+    /**
+     * Pick the key that should serve a new submission. Walks keys in priority
+     * order and returns the first whose per-key minute + day budgets still have
+     * headroom, consuming a slot on it. The caps are the model's Tier-2 baseline
+     * (baselineMin / baselineDay) scaled to each key's own tier — so a Tier-1
+     * key gets a smaller share than a Tier-2 one. Falls back to the primary key
+     * if Redis is down or all are at cap (the aggregate model-level gate should
+     * have prevented the latter).
+     */
+    async pickForSubmit(modelId, baselineMin, baselineDay) {
+        if (this.entries.length === 1)
+            return this.entries[0];
+        const client = redis_service_1.redis.getClient();
+        if (!client)
+            return this.entries[0];
+        const date = utcDateKey();
+        for (const entry of this.entries) {
+            try {
+                const perKeyMinLimit = (0, googleApiKeys_1.scaleLimitForTier)(baselineMin, entry.tier, "rpm");
+                const perKeyDayLimit = (0, googleApiKeys_1.scaleLimitForTier)(baselineDay, entry.tier, "rpd");
+                const dayKey = `gkpool:${entry.id}:${modelId}:day:${date}`;
+                const minKey = `gkpool:${entry.id}:${modelId}:min`;
+                const [dayRaw, minRaw] = await Promise.all([
+                    client.get(dayKey),
+                    client.get(minKey),
+                ]);
+                const dayUsed = dayRaw ? Number(dayRaw) : 0;
+                const minUsed = minRaw ? Number(minRaw) : 0;
+                const dayOk = perKeyDayLimit <= 0 || dayUsed < perKeyDayLimit;
+                const minOk = perKeyMinLimit <= 0 || minUsed < perKeyMinLimit;
+                if (dayOk && minOk) {
+                    await this.consume(entry, modelId, date);
+                    return entry;
+                }
+            }
+            catch (err) {
+                logger_1.logger.warn("googleKeyPool: routing read failed, trying next key", {
+                    keyId: entry.id,
+                    err: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        // All at cap / errored — use the primary and let Google's own 429 + the
+        // provider-fallback chain handle it.
+        logger_1.logger.warn("googleKeyPool: no key with headroom — using primary", {
+            modelId,
+            poolSize: this.entries.length,
+        });
+        await this.consume(this.entries[0], modelId, date).catch(() => undefined);
+        return this.entries[0];
+    }
+    /** Best-effort increment of a key's per-day + per-minute routing counters. */
+    async consume(entry, modelId, date) {
+        const client = redis_service_1.redis.getClient();
+        if (!client)
+            return;
+        const dayKey = `gkpool:${entry.id}:${modelId}:day:${date}`;
+        const minKey = `gkpool:${entry.id}:${modelId}:min`;
+        const newDay = await client.incr(dayKey);
+        if (newDay === 1)
+            await client.expire(dayKey, secsUntilMidnight());
+        const newMin = await client.incr(minKey);
+        if (newMin === 1)
+            await client.expire(minKey, 60);
+    }
+}
+let pool = null;
+/** Lazily-built process-wide pool. */
+function getGoogleKeyPool() {
+    if (!pool)
+        pool = new GoogleKeyPool();
+    return pool;
+}

package/lib/services/aiGen/providers/pixverse/pixverse.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pixverse.service.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/pixverse/pixverse.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AA0FlB,qBAAa,eAAgB,SAAQ,wBAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkD;IAKpE,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAmK3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA0FjD,aAAa,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM;CAkDjD"}
+{"version":3,"file":"pixverse.service.d.ts","sourceRoot":"","sources":["../../../../../src/services/aiGen/providers/pixverse/pixverse.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AACxE,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,UAAU,CAAC;AA0FlB,qBAAa,eAAgB,SAAQ,wBAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkD;IAKpE,aAAa,CACjB,MAAM,EAAE,qBAAqB,GAC5B,OAAO,CAAC,qBAAqB,CAAC;IAyK3B,gBAAgB,CAAC,EACrB,IAAI,EACJ,cAAc,EACd,cAAyB,GAC1B,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA0FjD,aAAa,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM;CAkDjD"}

package/lib/services/aiGen/providers/pixverse/pixverse.service.js

@@ -211,7 +211,13 @@ class PixVerseService extends baseAiGenProvider_service_1.BaseAiGenProviderServi
         if (isImageToVideo) {
             body.image_url = params.inputImageUrl;
         }
-        const endpoint = isImageToVideo ? `${this.baseUrl}/image/generate` : `${this.baseUrl}/text/generate`;
+        // I2V path is `/img/generate` — NOT `/image/generate`, which returns a bare
+        // "404 page not found" (verified 2026-06-15 via PixVerse OpenAPI smoke test
+        // + docs https://docs.platform.pixverse.ai/image-to-video-generation-13016633e0).
+        // This affected ALL PixVerse image-to-video models; it surfaced in prod as
+        // "pixverse-v6 404" only because v6 sits low in the I2V fallback chain and
+        // was the model selected when the rare fallback fired.
+        const endpoint = isImageToVideo ? `${this.baseUrl}/img/generate` : `${this.baseUrl}/text/generate`;
         const data = await pixverseFetch(endpoint, {
             method: "POST",
             headers: {

package/lib/services/bullmq.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bullmq.service.d.ts","sourceRoot":"","sources":["../../src/services/bullmq.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAgB,EAAwB,YAAY,EAAE,MAAM,SAAS,CAAC;AAItE,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAc1C,UAAU,oBAAoB;IAC5B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,aAAa;IACjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgB;IACvC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,YAAY,CAAgB;IAIpC,OAAO,CAAC,UAAU,CAAC,CAAc;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO;WAQO,WAAW,CAAC,OAAO,EAAE,oBAAoB;IAOvD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,aAAa;IAOrB,sDAAsD;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBlC,gBAAgB;IACH,MAAM,CACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,OAAO,EACb,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAUpC,YAAY,CACjB,UAAU,EAAE,MAAM,EAAE,EACpB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,EACtC,eAAe,CAAC,EAAE,MAAM,CACtB,MAAM,EACN;QACE,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CACF;CAkHJ;AASD,eAAO,MAAM,MAAM,eAGjB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,GAAG,CAAC"}
+{"version":3,"file":"bullmq.service.d.ts","sourceRoot":"","sources":["../../src/services/bullmq.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,GAAG,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAgB,EAAwB,YAAY,EAAE,MAAM,SAAS,CAAC;AAItE,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAc1C,UAAU,oBAAoB;IAC5B,YAAY,EAAE,YAAY,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,aAAa;IACjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgB;IACvC,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,YAAY,CAAgB;IAIpC,OAAO,CAAC,UAAU,CAAC,CAAc;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAS;IAE7B,OAAO;WAQO,WAAW,CAAC,OAAO,EAAE,oBAAoB;IAOvD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,aAAa;IAOrB,sDAAsD;IACzC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiBlC,gBAAgB;IACH,MAAM,CACjB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,OAAO,EACb,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE;IAUpC,YAAY,CACjB,UAAU,EAAE,MAAM,EAAE,EACpB,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,EACtC,eAAe,CAAC,EAAE,MAAM,CACtB,MAAM,EACN;QACE,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CACF;CAyIJ;AASD,eAAO,MAAM,MAAM,eAGjB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,GAAG,CAAC"}

package/lib/services/bullmq.service.js

@@ -166,7 +166,28 @@ class BullMQService {
                 if (this.shuttingDown)
                     return;
                 this.shuttingDown = true;
-                logger_1.logger.info("Shutting down BullMQ workers gracefully...");
+                // Hard backstop: worker.close() waits for the in-flight job's processor
+                // to finish, and a video job can legitimately run for the full poll
+                // window (~15 min) + finalization. If a job is wedged (e.g. an
+                // unforeseen hang — the rate-limiter pile-up that used to cause this is
+                // now bounded, see DistributedRateLimiter.waitUntilAvailable), an
+                // unbounded close() would hold the old Railway container alive
+                // indefinitely on a rolling deploy. Force-exit after a generous bound
+                // so a stuck process can never block a deploy. Default 20 min (> the
+                // 15-min legit job ceiling); override with SHUTDOWN_HARD_TIMEOUT_MS to
+                // match the Railway service's drain/grace window.
+                const hardTimeoutMs = Number(process.env.SHUTDOWN_HARD_TIMEOUT_MS) || 20 * 60000;
+                const hardExit = setTimeout(() => {
+                    logger_1.logger.error("Graceful shutdown exceeded hard timeout — force-exiting", {
+                        hardTimeoutMs,
+                        activeWorkers: this.workers.length,
+                    });
+                    process.exit(1);
+                }, hardTimeoutMs);
+                // Don't let this timer itself keep the event loop alive once everything
+                // else has settled.
+                hardExit.unref();
+                logger_1.logger.info("Shutting down BullMQ workers gracefully...", { hardTimeoutMs });
                 for (const worker of this.workers) {
                     await worker.close();
                     logger_1.logger.info(`Worker for queue closed`, { queueName: worker.name });
@@ -180,6 +201,7 @@ class BullMQService {
                 if (this.connection) {
                     await this.connection.quit();
                 }
+                clearTimeout(hardExit);
                 logger_1.logger.info("All workers closed. Exiting process.");
                 process.exit(0);
             };

package/lib/services/index.d.ts

@@ -10,6 +10,7 @@ export * from "./notification.service";
 export * from "./credits/pricing";
 export * from "./analytics.service";
 export * from "./tts";
+export * from "./translation";
 export * from "./musicGen";
 export * from "./audioAnalysis";
 export * from "./asr";

package/lib/services/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,SAAS,CAAC;AACxB,cAAc,kBAAkB,CAAC;AACjC,cAAc,OAAO,CAAC;AACtB,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,OAAO,CAAC;AACtB,cAAc,SAAS,CAAC;AACxB,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/services/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,SAAS,CAAC;AACxB,cAAc,kBAAkB,CAAC;AACjC,cAAc,OAAO,CAAC;AACtB,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,OAAO,CAAC;AACtB,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,OAAO,CAAC;AACtB,cAAc,SAAS,CAAC;AACxB,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC"}

package/lib/services/index.js

@@ -26,6 +26,7 @@ __exportStar(require("./notification.service"), exports);
 __exportStar(require("./credits/pricing"), exports);
 __exportStar(require("./analytics.service"), exports);
 __exportStar(require("./tts"), exports);
+__exportStar(require("./translation"), exports);
 __exportStar(require("./musicGen"), exports);
 __exportStar(require("./audioAnalysis"), exports);
 __exportStar(require("./asr"), exports);

package/lib/services/rateLimiter/distributedRateLimiter.service.d.ts

@@ -1,5 +1,18 @@
 import { TAiGenModel } from "../../globals/aiModels";
 import { TtsProvider } from "../../globals/ttsModels/types";
+/**
+ * Capacity-planning thresholds (used by previewCapacity + the bounded gate).
+ *
+ * MAX_ACCEPTABLE_WAIT_MS — a model whose projected wait for a submit slot
+ *   exceeds this is considered "not available" by previewCapacity, so the
+ *   job-start selector spills to a less-loaded model in the same tier chain.
+ * DAILY_NEAR_RESET_MS — when a model's per-day cap is exhausted, we only wait
+ *   it out if the UTC-midnight reset is within this window; otherwise the
+ *   bounded gate fast-fails (VIDEO_PROVIDER_RATE_LIMITED) so the caller can
+ *   fall back rather than block for hours.
+ */
+export declare const MAX_ACCEPTABLE_WAIT_MS = 180000;
+export declare const DAILY_NEAR_RESET_MS = 120000;
 /**
  * DistributedRateLimiter — three-tier provider quota gate.
  *
@@ -39,6 +52,29 @@ export interface LimiterSnapshot {
     perMinLimit?: number;
     perDayLimit?: number;
 }
+/**
+ * Read-only capacity estimate for a model — does NOT consume a slot. Used by
+ * the job-start model selector to decide whether to spill to another model in
+ * the tier chain before triggering. `available` folds the projected-wait and
+ * daily-reset thresholds into a single go/no-go.
+ */
+export interface CapacityPreview {
+    modelKey: string;
+    /** Rough estimate of how long until a NEW submit can proceed, in ms. */
+    projectedWaitMs: number;
+    /** Whether a new submission can proceed within the acceptable wait window. */
+    available: boolean;
+    /** Per-day cap is configured and currently exhausted. */
+    dailyExhausted: boolean;
+    secsUntilDayReset: number;
+    perMinUsed: number;
+    perMinLimit: number;
+    perDayUsed: number;
+    perDayLimit: number;
+    concurrentActive: number;
+    concurrentLimit: number;
+    concurrentQueueDepth: number;
+}
 export declare class DistributedRateLimiter {
     private readonly modelKey;
     private readonly concurrentLimit;
@@ -52,15 +88,34 @@ export declare class DistributedRateLimiter {
      * a slot in each. Caller MUST call releaseSlot() when the in-flight task
      * settles — per-min / per-day auto-expire but the concurrent counter does
      * not. Mirrors the old AiGenModelRateLimiter API.
+     *
+     * `opts.maxWaitMs` bounds the rate-budget wait: if satisfying the per-day /
+     * per-minute gate would take longer than this, the call throws a
+     * `UserFacingError(VIDEO_PROVIDER_RATE_LIMITED)` instead of sleep-looping for
+     * hours. This is the backstop that stops jobs from piling up in-process when
+     * a daily cap is hit — the submit path passes it; legacy callers that omit
+     * it keep the original (unbounded) behavior.
      */
-    waitUntilAvailable(): Promise<void>;
+    waitUntilAvailable(opts?: {
+        maxWaitMs?: number;
+    }): Promise<void>;
     /**
-     * Bumps per-min + per-day only — does NOT count toward the concurrent cap.
-     * Use for short-lived calls that share a provider's rate budget but aren't
-     * "in-flight tasks" (status polls, sync image/TTS gens that complete in the
-     * same call). No release is needed — Redis auto-expires the counters.
+     * Bumps per-MINUTE only — does NOT count toward the concurrent cap and does
+     * NOT consume/await the per-day gate. Use for short-lived calls that share a
+     * provider's per-minute budget but must not be blocked by the daily SUBMIT
+     * cap: status polls of already-running tasks (failing a poll on daily quota
+     * would abandon a video that's already generating), plus sync image/TTS gens
+     * (which have no per-day limit configured anyway). No release needed — Redis
+     * auto-expires the counters.
      */
     waitUntilAvailableForOneShot(): Promise<void>;
+    /**
+     * Read-only capacity estimate — does NOT consume a slot. Reads current
+     * per-minute / per-day usage and concurrency pressure from Redis and
+     * projects how long a new submission would wait. Fails open (available:true,
+     * wait:0) on Redis errors / no client, matching the gate's fail-open policy.
+     */
+    previewCapacity(maxAcceptableWaitMs?: number): Promise<CapacityPreview>;
     /** Free a concurrent slot acquired via waitUntilAvailable(). */
     releaseSlot(): void;
     get activeConcurrentCount(): number;

package/lib/services/rateLimiter/distributedRateLimiter.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"distributedRateLimiter.service.d.ts","sourceRoot":"","sources":["../../../src/services/rateLimiter/distributedRateLimiter.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAU5D;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,UAAU,4BAA4B;IACpC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAuDD,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAErC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,SAAS,CAAyB;gBAE9B,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,4BAA4B;IAOlE;;;;;OAKG;IACG,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAUzC;;;;;OAKG;IACG,4BAA4B,IAAI,OAAO,CAAC,IAAI,CAAC;IAInD,gEAAgE;IAChE,WAAW,IAAI,IAAI;IAInB,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAEK,QAAQ,IAAI,OAAO,CAAC,eAAe,CAAC;YAoC5B,iBAAiB;IAc/B,OAAO,CAAC,iBAAiB;YAQX,iBAAiB;CA6DhC;AAwBD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,WAAW,GACpB,sBAAsB,CAaxB;AAED,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,WAAW,GACpB,sBAAsB,CAexB;AAED;4DAC4D;AAC5D,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAM1E"}
+{"version":3,"file":"distributedRateLimiter.service.d.ts","sourceRoot":"","sources":["../../../src/services/rateLimiter/distributedRateLimiter.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAErE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAM5D;;;;;;;;;;GAUG;AACH,eAAO,MAAM,sBAAsB,SAAU,CAAC;AAC9C,eAAO,MAAM,mBAAmB,SAAU,CAAC;AAQ3C;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,UAAU,4BAA4B;IACpC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,eAAe,EAAE,MAAM,CAAC;IACxB,8EAA8E;IAC9E,SAAS,EAAE,OAAO,CAAC;IACnB,yDAAyD;IACzD,cAAc,EAAE,OAAO,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AA4DD,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAErC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,SAAS,CAAyB;gBAE9B,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,4BAA4B;IAOlE;;;;;;;;;;;;OAYG;IACG,kBAAkB,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtE;;;;;;;;OAQG;IACG,4BAA4B,IAAI,OAAO,CAAC,IAAI,CAAC;IAInD;;;;;OAKG;IACG,eAAe,CACnB,mBAAmB,GAAE,MAA+B,GACnD,OAAO,CAAC,eAAe,CAAC;IAgF3B,gEAAgE;IAChE,WAAW,IAAI,IAAI;IAInB,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAEK,QAAQ,IAAI,OAAO,CAAC,eAAe,CAAC;YAoC5B,iBAAiB;IAc/B,OAAO,CAAC,iBAAiB;YAQX,iBAAiB;CAoHhC;AAwBD,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,WAAW,GACpB,sBAAsB,CA0BxB;AAED,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,WAAW,GACpB,sBAAsB,CAexB;AAED;4DAC4D;AAC5D,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAM1E"}