ts-server-lib 0.0.48

package/utils/TSFifo.js

@@ -0,0 +1,145 @@
+"use strict";
+/**
+ * TSFifo — O(1) FIFO queue. The canonical replacement for `array.shift()` and `array.splice(0, n)`.
+ *
+ * ### Why this exists as a primitive
+ *
+ * `shift()` re-indexes every remaining element, so draining a queue of N with `shift()` is O(N²). The
+ * cost arrives at exactly the worst moment: queues are only deep when the system is already stressed, so
+ * the drain gets more expensive the more there is to drain. That is a positive feedback loop, not a
+ * constant factor — the same shape as the O(N) in-flight scan that was removed from the sports feed's
+ * pressure signal for the same reason.
+ *
+ * ### Why it lives HERE, in ts-server-lib
+ *
+ * It started in core-service, which was one layer too high: {@link TSRQWPool}'s own task backlog — the
+ * FIFTH instance of this defect — could not use it, because core-service depends on ts-server-lib and not
+ * the reverse. Copying it down would have produced two implementations of a module whose entire purpose is
+ * being the only one. So it sits at the bottom layer and core-service re-exports it, leaving
+ * `import { TSFifo } from 'core-service'` working for every service-side caller.
+ *
+ * It had already been diagnosed and hand-fixed once (`sports-service` `capacity-pump.ts`, head pointer plus
+ * periodic compaction, with the reasoning written out) while four other queues kept their own shape:
+ *
+ *   - `feed/amqp/dispatch-admission.ts` — the admission gate's `hotQueue` / `moneyQueue`. Its own doc
+ *     records `hotQueued` reaching **683** with 24.2 s waits; releasing 683 waiters by `shift()` is ~233 K
+ *     element moves, on the money path, during a backlog.
+ *   - `feed/amqp/worker.ts` — `pendingOutcomePublishes`, an uncapped array drained `splice(0, 32)` at a
+ *     time, so each batch re-indexes everything still queued behind it.
+ *   - `resolvers/outcomes.ts` — a 64-cap subscriber backlog. Cheap at that size, but it is the site that
+ *     proves the point: its correct eviction is `'drop-oldest'`, the OPPOSITE of the admission queue's, and
+ *     that decision was previously an unexplained `queue.shift()` in a callback.
+ *   - `db/TSRQW.ts` `TSRQWPool.tasks` — the parse-pool backlog, right in the AMQP drain path. Inert while
+ *     the pool keeps up (`parseQueued ~0` measured) and quadratic exactly when it stops keeping up, which
+ *     is the moment a super-linear drain hurts most.
+ *
+ * Five queues, five shapes, one of them correct. That is what a per-site fix always converges to, so the
+ * shape is not the interesting part of this module — having exactly one of it is, and naming the overflow
+ * policy so the difference between those sites is a type rather than a comment.
+ *
+ * ALL FIVE now use this class; a sixth hand-rolled queue is the thing to reject in review.
+ *
+ * ### Design
+ *
+ * A head index makes `shift` O(1); the consumed prefix is reclaimed in one `splice` once it passes
+ * {@link TSFIFO_COMPACT_THRESHOLD}, which keeps the backing array from growing without bound while keeping
+ * amortised cost O(1). `maxSize` is optional and, when set, makes overflow an explicit, observable event
+ * rather than unbounded memory — a queue with no ceiling is a slower way to run out of RAM.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TSFifo = exports.TSFIFO_COMPACT_THRESHOLD = void 0;
+/**
+ * Consumed slots tolerated before the backing array is compacted.
+ *
+ * Compaction is one O(size) `splice` amortised over that many O(1) shifts, so a larger value trades
+ * transient memory for fewer memmoves. 1024 matches the value `capacity-pump` arrived at empirically.
+ */
+exports.TSFIFO_COMPACT_THRESHOLD = 1024;
+class TSFifo {
+    items = [];
+    head = 0;
+    peak = 0;
+    lost = 0;
+    compactThreshold;
+    maxSize;
+    overflow;
+    constructor(opts = {}) {
+        this.compactThreshold = opts.compactThreshold ?? exports.TSFIFO_COMPACT_THRESHOLD;
+        this.maxSize = opts.maxSize;
+        this.overflow = opts.overflow ?? 'reject-new';
+    }
+    /** Items currently queued. */
+    get size() {
+        return this.items.length - this.head;
+    }
+    /** Highest `size` observed since construction — the number a gauge should report. */
+    get peakSize() {
+        return this.peak;
+    }
+    /**
+     * Items lost to `maxSize` — rejected arrivals under `'reject-new'`, evicted heads under `'drop-oldest'`.
+     * Non-zero means the cap is doing something and should be visible: a silent drop is how data loss hides.
+     */
+    get dropped() {
+        return this.lost;
+    }
+    /**
+     * Append. Returns false only under `'reject-new'` overflow when the queue is full — nothing was
+     * enqueued. Under `'drop-oldest'` it always returns true, having evicted the head to make room.
+     */
+    push(item) {
+        if (this.maxSize !== undefined && this.size >= this.maxSize) {
+            if (this.overflow === 'reject-new') {
+                this.lost += 1;
+                return false;
+            }
+            // 'drop-oldest': evict the head, then fall through to append. Counted the same way — the caller
+            // needs to know it lost data regardless of which end went.
+            this.shift();
+            this.lost += 1;
+        }
+        this.items.push(item);
+        if (this.size > this.peak)
+            this.peak = this.size;
+        return true;
+    }
+    /** Remove and return the oldest item, or undefined when empty. O(1). */
+    shift() {
+        if (this.size === 0)
+            return undefined;
+        const item = this.items[this.head];
+        // Release the reference before advancing: without this the backing array pins every dequeued item
+        // until the next compaction, which for a 1024-slot prefix of large payloads is real memory.
+        this.items[this.head] = undefined;
+        this.head += 1;
+        this.compactIfNeeded();
+        return item;
+    }
+    /** Oldest item without removing it, or undefined when empty. */
+    peek() {
+        return this.size === 0 ? undefined : this.items[this.head];
+    }
+    /** Drain up to `max` items, oldest first. Amortised O(returned), never O(remaining). */
+    drain(max) {
+        const out = [];
+        for (let i = 0; i < max; i++) {
+            const item = this.shift();
+            if (item === undefined)
+                break;
+            out.push(item);
+        }
+        return out;
+    }
+    /** Discard everything. Use on shutdown/reset paths. */
+    clear() {
+        this.items = [];
+        this.head = 0;
+    }
+    compactIfNeeded() {
+        if (this.head < this.compactThreshold)
+            return;
+        this.items.splice(0, this.head);
+        this.head = 0;
+    }
+}
+exports.TSFifo = TSFifo;

package/utils/TSFile.d.ts

@@ -0,0 +1,36 @@
+type FileType = {
+    filename: string;
+    type: string;
+    ext?: string;
+    mime?: string;
+    charset?: string;
+};
+export declare class TSFile {
+    static list(path?: string): FileType[];
+    static info(filename: string): FileType;
+    private static readFileHead;
+    /** Prefer POSIX `file(1)` when present; otherwise extension + optional magic-byte sniff (Windows-friendly). */
+    static mime(filename: string): {
+        mime: string;
+        charset?: string;
+        ext?: string;
+    };
+    private static mimeFromFileCommand;
+    private static mimeFallback;
+    private static charsetForMime;
+    static extension(mime: string): string | undefined;
+    static mimeFromExtension(ext: string): string;
+    static mimeFromFilename(filename: string): string | undefined;
+    static isImageMime(mime: string | undefined | null): boolean;
+    static isVideoMime(mime: string | undefined | null): boolean;
+    static isImageFilename(filename: string): boolean;
+    static isVideoFilename(filename: string): boolean;
+    private static bytesMatch;
+    static sniffMimeFromMagicBytes(buffer: ArrayBuffer | Uint8Array): "image/gif" | "image/jpeg" | "image/png" | "video/mp4" | "video/webm" | undefined;
+    private static headerSuggestsBinaryVideo;
+    private static normalizeVideoExtMime;
+    static resolveMediaContentType(filename: string, headerContentType: string | undefined | null, buffer: ArrayBuffer | Uint8Array): string | undefined;
+    static readJson<T = unknown>(filename: string): Promise<T>;
+    static writeJson(filename: string, data: unknown, space?: number): Promise<void>;
+}
+export {};

package/utils/TSFile.js

@@ -0,0 +1,244 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TSFile = void 0;
+const child_process_1 = require("child_process");
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const mimeDB = __importStar(require("./mime.json"));
+const STAT_METHODS = ['isFile', 'isDirectory', 'isSymbolicLink', 'isFIFO', 'isSocket', 'isCharacterDevice', 'isBlockDevice'];
+const OCTET_STREAM = 'application/octet-stream';
+const MIME_MAP = (mimeDB.default ??
+    mimeDB);
+const EXT_TO_MIME = new Map();
+const MEDIA_MIME_OVERRIDES = {
+    mp4: 'video/mp4',
+    m4v: 'video/mp4',
+    mov: 'video/quicktime',
+    webm: 'video/webm',
+    mkv: 'video/x-matroska',
+    avi: 'video/x-msvideo',
+    jpg: 'image/jpeg'
+};
+for (const [mime, data] of Object.entries(MIME_MAP)) {
+    for (const ext of data.extensions ?? []) {
+        const key = ext.toLowerCase();
+        if (!EXT_TO_MIME.has(key)) {
+            EXT_TO_MIME.set(key, mime);
+        }
+    }
+}
+class TSFile {
+    static list(path = './') {
+        const l = (0, fs_1.readdirSync)(path);
+        return l.map(f => TSFile.info((0, path_1.join)(path, f)));
+    }
+    static info(filename) {
+        let type = 'Unknown';
+        const s = (0, fs_1.existsSync)(filename) ? (0, fs_1.lstatSync)(filename) : null;
+        for (const method of STAT_METHODS) {
+            const fn = s ? s[method] : null;
+            if (fn && typeof fn === 'function' && fn.call(s)) {
+                type = method.replace(/^is/, '');
+                break;
+            }
+        }
+        return { filename, type, ...TSFile.mime(filename) };
+    }
+    static readFileHead(filename, maxLen) {
+        let fd;
+        try {
+            fd = (0, fs_1.openSync)(filename, 'r');
+        }
+        catch {
+            return undefined;
+        }
+        try {
+            const buf = Buffer.allocUnsafe(maxLen);
+            const n = (0, fs_1.readSync)(fd, buf, 0, maxLen, 0);
+            return new Uint8Array(buf.buffer, buf.byteOffset, n);
+        }
+        finally {
+            (0, fs_1.closeSync)(fd);
+        }
+    }
+    /** Prefer POSIX `file(1)` when present; otherwise extension + optional magic-byte sniff (Windows-friendly). */
+    static mime(filename) {
+        const fromFile = TSFile.mimeFromFileCommand(filename);
+        if (fromFile)
+            return fromFile;
+        return TSFile.mimeFallback(filename);
+    }
+    static mimeFromFileCommand(filename) {
+        if (process.platform === 'win32')
+            return null;
+        try {
+            const result = (0, child_process_1.execFileSync)('file', ['--mime-type', '-i', filename], { encoding: 'utf-8', maxBuffer: 1024 * 1024 });
+            const parts = result.split(/\s+/).filter(Boolean);
+            parts.shift();
+            const [mime, charset] = parts.map(p => p.replace(/;$/, ''));
+            return { mime, charset, ext: TSFile.extension(mime) };
+        }
+        catch {
+            return null;
+        }
+    }
+    static mimeFallback(filename) {
+        const extFromPath = (0, path_1.extname)(filename).replace(/^\./, '').toLowerCase();
+        const fromExt = TSFile.mimeFromExtension(extFromPath);
+        if (!(0, fs_1.existsSync)(filename)) {
+            const mime = fromExt ?? OCTET_STREAM;
+            return {
+                mime,
+                charset: TSFile.charsetForMime(mime),
+                ext: TSFile.extension(mime)
+            };
+        }
+        const st = (0, fs_1.lstatSync)(filename);
+        if (st.isDirectory()) {
+            return { mime: 'inode/directory', charset: 'binary', ext: undefined };
+        }
+        if (!st.isFile()) {
+            return { mime: fromExt ?? OCTET_STREAM, charset: 'binary', ext: TSFile.extension(fromExt ?? OCTET_STREAM) };
+        }
+        const head = TSFile.readFileHead(filename, 512);
+        const sniff = head ? TSFile.sniffMimeFromMagicBytes(head) : undefined;
+        const mime = sniff ?? fromExt ?? OCTET_STREAM;
+        return {
+            mime,
+            charset: TSFile.charsetForMime(mime),
+            ext: TSFile.extension(mime)
+        };
+    }
+    static charsetForMime(mime) {
+        const m = mime.toLowerCase();
+        if (m.startsWith('text/'))
+            return 'utf-8';
+        if (m.endsWith('+json') || m.endsWith('+xml'))
+            return 'utf-8';
+        if (m === 'application/javascript' || m === 'application/json')
+            return 'utf-8';
+        return 'binary';
+    }
+    static extension(mime) {
+        const mimeData = MIME_MAP[mime];
+        const { extensions } = mimeData || {};
+        return extensions?.[0];
+    }
+    static mimeFromExtension(ext) {
+        const normalized = ext.replace(/^\./, '').toLowerCase();
+        return MEDIA_MIME_OVERRIDES[normalized] ?? EXT_TO_MIME.get(normalized);
+    }
+    static mimeFromFilename(filename) {
+        const ext = (0, path_1.extname)(filename).replace(/^\./, '');
+        if (!ext)
+            return undefined;
+        return TSFile.mimeFromExtension(ext);
+    }
+    static isImageMime(mime) {
+        return typeof mime === 'string' && mime.toLowerCase().startsWith('image/');
+    }
+    static isVideoMime(mime) {
+        return typeof mime === 'string' && mime.toLowerCase().startsWith('video/');
+    }
+    static isImageFilename(filename) {
+        return TSFile.isImageMime(TSFile.mimeFromFilename(filename));
+    }
+    static isVideoFilename(filename) {
+        return TSFile.isVideoMime(TSFile.mimeFromFilename(filename));
+    }
+    static bytesMatch(u, offset, seq) {
+        if (u.byteLength < offset + seq.length)
+            return false;
+        for (let i = 0; i < seq.length; i++) {
+            if (u[offset + i] !== seq[i])
+                return false;
+        }
+        return true;
+    }
+    static sniffMimeFromMagicBytes(buffer) {
+        const u = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+        if (u.byteLength < 12)
+            return undefined;
+        if (TSFile.bytesMatch(u, 0, [0x1a, 0x45, 0xdf, 0xa3]))
+            return 'video/webm';
+        if (TSFile.bytesMatch(u, 4, [0x66, 0x74, 0x79, 0x70]))
+            return 'video/mp4';
+        if (TSFile.bytesMatch(u, 0, [0xff, 0xd8, 0xff]))
+            return 'image/jpeg';
+        if (TSFile.bytesMatch(u, 0, [0x89, 0x50, 0x4e, 0x47]))
+            return 'image/png';
+        if (TSFile.bytesMatch(u, 0, [0x47, 0x49, 0x46]))
+            return 'image/gif';
+        return undefined;
+    }
+    static headerSuggestsBinaryVideo(header) {
+        return TSFile.isImageMime(header) || header === OCTET_STREAM || !header;
+    }
+    static normalizeVideoExtMime(extMime) {
+        if (extMime === 'video/webm')
+            return 'video/webm';
+        if (extMime === 'video/quicktime')
+            return 'video/quicktime';
+        return 'video/mp4';
+    }
+    static resolveMediaContentType(filename, headerContentType, buffer) {
+        const header = (headerContentType?.split(';')[0] ?? '').trim().toLowerCase();
+        const extMime = TSFile.mimeFromFilename(filename)?.toLowerCase();
+        const sniff = TSFile.sniffMimeFromMagicBytes(buffer);
+        if (sniff && TSFile.isVideoMime(sniff) && (TSFile.isVideoMime(extMime) || TSFile.headerSuggestsBinaryVideo(header))) {
+            return sniff;
+        }
+        if (sniff && TSFile.isImageMime(sniff) && TSFile.isImageMime(extMime)) {
+            return sniff;
+        }
+        if (TSFile.isVideoMime(extMime) && TSFile.headerSuggestsBinaryVideo(header)) {
+            return TSFile.normalizeVideoExtMime(extMime);
+        }
+        if (TSFile.isImageMime(extMime) && TSFile.isVideoMime(header)) {
+            return extMime;
+        }
+        return header || extMime || OCTET_STREAM;
+    }
+    static async readJson(filename) {
+        const content = await (0, promises_1.readFile)(filename, 'utf-8');
+        return JSON.parse(content);
+    }
+    static async writeJson(filename, data, space = 2) {
+        await (0, promises_1.writeFile)(filename, `${JSON.stringify(data, null, space)}\n`, 'utf-8');
+    }
+}
+exports.TSFile = TSFile;

package/utils/TSHash.d.ts

@@ -0,0 +1,19 @@
+export type HmacEncoding = 'hex' | 'base64url' | 'base64';
+export declare class TSHash {
+    static id: (length?: number) => string;
+    static md5: (text: string) => string;
+    static sha256: (text: string, encoding?: 'hex' | 'base64' | 'base64url') => string;
+    static hmacSha256: (data: string, secret: string | Buffer, encoding?: HmacEncoding) => string;
+    static verifyHmacSha256: (data: string, signature: string, secret: string | Buffer, encoding?: HmacEncoding) => boolean;
+    static encryptAesGcm: (plaintext: string, secret: string) => string;
+    static decryptAesGcm: (token: string, secret: string) => string | null;
+    static hashToken: (token: string) => Promise<string>;
+    /**
+     * Random bytes encoded as the requested format.
+     * - `'base64url'` (default) — opaque tokens, refresh-token-size secrets
+     * - `'hex'` — webhook signatures, short IDs, stub identifiers
+     * - `'base64'` — legacy base64 encoding
+     */
+    static randomId: (byteLength?: number, encoding?: 'base64url' | 'hex' | 'base64') => string;
+    static generateBackupCodes: (count?: number) => string[];
+}

package/utils/TSHash.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TSHash = void 0;
+const node_crypto_1 = require("node:crypto");
+function idChar(e) {
+    const x = e & 63;
+    if (x < 36)
+        return x.toString(36);
+    if (x < 62)
+        return (x - 26).toString(36).toUpperCase();
+    return x > 62 ? '-' : '_';
+}
+const AES_ALGORITHM = 'aes-256-gcm';
+const IV_LENGTH = 12;
+const AUTH_TAG_LENGTH = 16;
+class TSHash {
+    static id = (length = 9) => (0, node_crypto_1.getRandomValues)(new Uint8Array(length)).reduce((t, e) => t + idChar(e), '');
+    static md5 = (text) => (0, node_crypto_1.createHash)('md5').update(text).digest('hex');
+    static sha256 = (text, encoding = 'hex') => (0, node_crypto_1.createHash)('sha256').update(text).digest(encoding);
+    static hmacSha256 = (data, secret, encoding = 'hex') => (0, node_crypto_1.createHmac)('sha256', secret).update(data).digest(encoding);
+    static verifyHmacSha256 = (data, signature, secret, encoding = 'hex') => {
+        const expected = TSHash.hmacSha256(data, secret, encoding);
+        const sigBuf = Buffer.from(signature, encoding);
+        const expBuf = Buffer.from(expected, encoding);
+        if (sigBuf.length !== expBuf.length)
+            return false;
+        return (0, node_crypto_1.timingSafeEqual)(sigBuf, expBuf);
+    };
+    static encryptAesGcm = (plaintext, secret) => {
+        const key = (0, node_crypto_1.createHash)('sha256').update(secret).digest();
+        const iv = (0, node_crypto_1.randomBytes)(IV_LENGTH);
+        const cipher = (0, node_crypto_1.createCipheriv)(AES_ALGORITHM, key, iv, { authTagLength: AUTH_TAG_LENGTH });
+        const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]);
+        const authTag = cipher.getAuthTag();
+        return Buffer.concat([iv, encrypted, authTag]).toString('base64url');
+    };
+    static decryptAesGcm = (token, secret) => {
+        try {
+            const buf = Buffer.from(token, 'base64url');
+            if (buf.length < IV_LENGTH + AUTH_TAG_LENGTH + 1)
+                return null;
+            const iv = buf.subarray(0, IV_LENGTH);
+            const authTag = buf.subarray(buf.length - AUTH_TAG_LENGTH);
+            const encrypted = buf.subarray(IV_LENGTH, buf.length - AUTH_TAG_LENGTH);
+            const key = (0, node_crypto_1.createHash)('sha256').update(secret).digest();
+            const decipher = (0, node_crypto_1.createDecipheriv)(AES_ALGORITHM, key, iv, { authTagLength: AUTH_TAG_LENGTH });
+            decipher.setAuthTag(authTag);
+            return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8');
+        }
+        catch {
+            return null;
+        }
+    };
+    static hashToken = (token) => Promise.resolve(TSHash.sha256(token, 'hex'));
+    /**
+     * Random bytes encoded as the requested format.
+     * - `'base64url'` (default) — opaque tokens, refresh-token-size secrets
+     * - `'hex'` — webhook signatures, short IDs, stub identifiers
+     * - `'base64'` — legacy base64 encoding
+     */
+    static randomId = (byteLength = 32, encoding = 'base64url') => (0, node_crypto_1.randomBytes)(byteLength).toString(encoding);
+    static generateBackupCodes = (count = 10) => {
+        const codes = [];
+        for (let i = 0; i < count; i++) {
+            const code = (0, node_crypto_1.randomBytes)(4).toString('hex').toUpperCase();
+            codes.push(`${code.slice(0, 4)}-${code.slice(4)}`);
+        }
+        return codes;
+    };
+}
+exports.TSHash = TSHash;