zengate 1.0.0

package/src/server/limiter.js

@@ -0,0 +1,72 @@
+import { ApiError } from './errors.js';
+
+/**
+ * Concurrency gate with a bounded FIFO queue.
+ *
+ * `acquire(signal, weight)` resolves to a release function once `weight`
+ * slots are free (a request with n choices runs n generations, so it costs
+ * n slots). The slots are released exactly once: when the caller calls
+ * release, or when `signal` aborts (timeout or client disconnect), so work
+ * that ignores cancellation can never hold a slot forever.
+ * @param {{ maxConcurrent: number, maxQueue: number }} options
+ */
+export function createLimiter({ maxConcurrent, maxQueue }) {
+    let active = 0;
+    const waiting = [];
+
+    const fits = (weight) => active + weight <= maxConcurrent;
+
+    const pump = () => {
+        while (waiting.length > 0 && fits(waiting[0].weight)) waiting.shift().grant();
+    };
+
+    const makeRelease = (signal, weight) => {
+        let released = false;
+        const release = () => {
+            if (released) return;
+            released = true;
+            signal?.removeEventListener('abort', release);
+            active -= weight;
+            pump();
+        };
+        signal?.addEventListener('abort', release, { once: true });
+        return release;
+    };
+
+    const abortedError = () => new ApiError(499, 'Request cancelled.', { code: 'cancelled' });
+
+    function acquire(signal, requested = 1) {
+        const weight = Math.min(Math.max(1, Math.floor(requested) || 1), maxConcurrent);
+        if (signal?.aborted) return Promise.reject(abortedError());
+        if (waiting.length === 0 && fits(weight)) {
+            active += weight;
+            return Promise.resolve(makeRelease(signal, weight));
+        }
+        if (waiting.length >= maxQueue) {
+            return Promise.reject(new ApiError(429, 'Gateway is busy; retry shortly.', { code: 'server_busy', retryAfter: 2 }));
+        }
+        return new Promise((resolve, reject) => {
+            const entry = {
+                weight,
+                grant() {
+                    signal?.removeEventListener('abort', onAbort);
+                    active += weight;
+                    resolve(makeRelease(signal, weight));
+                },
+            };
+            const onAbort = () => {
+                const index = waiting.indexOf(entry);
+                if (index >= 0) waiting.splice(index, 1);
+                reject(abortedError());
+                pump();
+            };
+            signal?.addEventListener('abort', onAbort, { once: true });
+            waiting.push(entry);
+        });
+    }
+
+    return Object.freeze({
+        acquire,
+        stats: () => ({ active, queued: waiting.length, maxConcurrent, maxQueue }),
+    });
+}

package/src/server/metrics.js

@@ -0,0 +1,31 @@
+/** Minimal in-process counters exposed as JSON on GET /metrics. */
+export function createMetrics() {
+    const startedAt = Date.now();
+    const byStatus = new Map();
+    let requests = 0;
+    let authFailures = 0;
+    let rateLimited = 0;
+
+    const middleware = (req, res, next) => {
+        requests += 1;
+        res.on('finish', () => {
+            const key = String(res.statusCode);
+            byStatus.set(key, (byStatus.get(key) || 0) + 1);
+        });
+        next();
+    };
+
+    return Object.freeze({
+        middleware,
+        authFailure: () => { authFailures += 1; },
+        rateLimited: () => { rateLimited += 1; },
+        snapshot: (extra = {}) => ({
+            uptime_s: Math.round((Date.now() - startedAt) / 1000),
+            requests,
+            auth_failures: authFailures,
+            rate_limited: rateLimited,
+            responses_by_status: Object.fromEntries(byStatus),
+            ...extra,
+        }),
+    });
+}

package/src/server/middleware.js

@@ -0,0 +1,94 @@
+import crypto from 'node:crypto';
+import { ApiError, sendError } from './errors.js';
+
+const PUBLIC_PATHS = new Set(['/health', '/ready']);
+
+/** Constant-time comparison; hashing first hides the key length. */
+function safeEqual(a, b) {
+    const ha = crypto.createHash('sha256').update(a).digest();
+    const hb = crypto.createHash('sha256').update(b).digest();
+    return crypto.timingSafeEqual(ha, hb);
+}
+
+function presentedKey(req) {
+    const header = req.headers.authorization || '';
+    const match = /^Bearer\s+(.+)$/i.exec(header);
+    return match ? match[1].trim() : '';
+}
+
+/**
+ * Gateway authentication. Clients send `Authorization: Bearer <API_KEY>`,
+ * which is what every OpenAI SDK does with its api key setting.
+ */
+export function authMiddleware({ apiKeys, allowNoAuth, onFailure }) {
+    return (req, res, next) => {
+        req.clientId = 'anonymous';
+        if (allowNoAuth || PUBLIC_PATHS.has(req.path) || req.method === 'OPTIONS') return next();
+        const key = presentedKey(req);
+        if (key && apiKeys.some((candidate) => safeEqual(key, candidate))) {
+            // Opaque per-key id: scopes stored responses to the key that made them.
+            req.clientId = crypto.createHash('sha256').update(key).digest('hex').slice(0, 32);
+            return next();
+        }
+        onFailure?.();
+        res.set('WWW-Authenticate', 'Bearer');
+        return sendError(res, new ApiError(401, key
+            ? 'Incorrect API key provided.'
+            : 'Missing API key. Send it as "Authorization: Bearer <API_KEY>".', { code: 'invalid_api_key' }));
+    };
+}
+
+/**
+ * Per-client token bucket (burst = limit, refill = limit per minute).
+ * O(1) memory per client; idle entries are swept every minute.
+ */
+export function rateLimitMiddleware({ perMinute, onLimited, maxClients = 50000 }) {
+    const buckets = new Map();
+    const refillPerMs = perMinute / 60000;
+    const sweep = setInterval(() => {
+        const cutoff = Date.now() - 120000;
+        for (const [ip, bucket] of buckets) if (bucket.at < cutoff) buckets.delete(ip);
+    }, 60000);
+    sweep.unref();
+
+    const middleware = (req, res, next) => {
+        if (perMinute <= 0 || PUBLIC_PATHS.has(req.path)) return next();
+        const ip = req.ip || req.socket?.remoteAddress || 'unknown';
+        const now = Date.now();
+        let bucket = buckets.get(ip);
+        if (!bucket) {
+            if (buckets.size >= maxClients) buckets.delete(buckets.keys().next().value);
+            bucket = { tokens: perMinute, at: now };
+            buckets.set(ip, bucket);
+        }
+        bucket.tokens = Math.min(perMinute, bucket.tokens + (now - bucket.at) * refillPerMs);
+        bucket.at = now;
+        if (bucket.tokens < 1) {
+            onLimited?.();
+            const retryAfter = Math.max(1, Math.ceil((1 - bucket.tokens) / refillPerMs / 1000));
+            return sendError(res, new ApiError(429, 'Rate limit reached for this client; slow down.', { code: 'rate_limit_exceeded', retryAfter }));
+        }
+        bucket.tokens -= 1;
+        return next();
+    };
+    middleware.close = () => clearInterval(sweep);
+    return middleware;
+}
+
+export function securityHeaders(req, res, next) {
+    res.set({
+        'X-Content-Type-Options': 'nosniff',
+        'X-Frame-Options': 'DENY',
+        'Referrer-Policy': 'no-referrer',
+        'Cache-Control': 'no-store',
+    });
+    next();
+}
+
+/** Per-request id (echoes a client-supplied one when it looks sane). */
+export function requestId(req, res, next) {
+    const incoming = req.headers['x-request-id'];
+    req.id = typeof incoming === 'string' && /^[\w.-]{1,64}$/.test(incoming) ? incoming : `req_${crypto.randomBytes(12).toString('hex')}`;
+    res.set('x-request-id', req.id);
+    next();
+}

package/src/server/slot.js

@@ -0,0 +1,32 @@
+import { ApiError } from './errors.js';
+
+/**
+ * Adds `req.withSlot(fn, weight)`: runs `fn(signal)` holding `weight` concurrency slots.
+ * The signal aborts when the client disconnects (499) or REQUEST_TIMEOUT_MS
+ * elapses (504); either way the slot is released immediately, even if the
+ * work ignores the signal.
+ */
+export function slotMiddleware({ limiter, timeoutMs }) {
+    return (req, res, next) => {
+        req.withSlot = async (fn, weight = 1) => {
+            const controller = new AbortController();
+            const onClose = () => {
+                if (!res.writableFinished) controller.abort(new ApiError(499, 'Client closed the request.', { code: 'cancelled' }));
+            };
+            res.on('close', onClose);
+            const timer = setTimeout(() => controller.abort(new ApiError(504,
+                `The model did not finish within ${Math.round(timeoutMs / 1000)}s (REQUEST_TIMEOUT_MS).`, { code: 'timeout' })), timeoutMs);
+            timer.unref();
+            let release;
+            try {
+                release = await limiter.acquire(controller.signal, weight);
+                return await fn(controller.signal);
+            } finally {
+                clearTimeout(timer);
+                res.off('close', onClose);
+                release?.();
+            }
+        };
+        next();
+    };
+}