ts-server-lib 0.0.48

package/utils/TSRequest.d.ts

@@ -0,0 +1,248 @@
+export type ITSResponse = {
+    data?: any;
+    kind: string;
+    status: number;
+    message: string;
+} | void;
+/** Options for {@link TSRequest.raw} / form / json / xml — `signal` aborts the in-flight HTTP request. */
+export type TSRequestHttpOptions = Record<string, unknown> & {
+    signal?: AbortSignal;
+};
+/**
+ * Body size at or above which XML parsing is moved off the main thread.
+ *
+ * `fast-xml-parser` is synchronous and un-interruptible, so a large REST response blocks the event
+ * loop for the whole parse. Measured in a sports feed replay: a `matches` schedule sweep (~30 day
+ * schedules plus tournament schedules, one 16-tournament chunk holding 14,266 events) produced
+ * event-loop stalls of ~364 ms and periodic >1 s AMQP dispatch breaches on every `matches` cron minute.
+ * The AMQP path never had this problem because it already parses inside a TSRQW worker; only REST
+ * parsed inline.
+ *
+ * Below this size, offloading is a net loss: the structured-clone round-trip to a worker costs more
+ * than parsing a small document. Tuned to sit above ordinary API replies and below the multi-hundred-KB
+ * schedule documents that actually stall the loop.
+ *
+ * ─────────────────────────────────────────────────────────────────────────────────────────────────────────────
+ * 🔴 **128 KB -> 64 KB, 2026-08-14. The schedule documents were missing the worker by 2.8 KB.**
+ *
+ * The paragraph above predicted "multi-hundred-KB schedule documents". Measured, they are not: the largest
+ * inline document is **125.2 KB**, i.e. **2.8 KB under the old 128 KB cutoff**, so the exact population this
+ * threshold exists to offload was taking the inline path every time. The same near-miss was recorded once
+ * before at 130,676 bytes against 131,072 (396 bytes under) — twice is a pattern, not a coincidence: UOF
+ * schedule responses cluster just below 128 KB.
+ *
+ * Measured on the sports feed, 6,124 inline parses over one catalog window:
+ *
+ *     mean inline document     5.4 KB  ->  1.14 ms      <- thousands of small entity fetches
+ *     MAX  inline document   125.2 KB  ->    42 ms      <- the schedule documents
+ *     aggregate inline time    6,995 ms (~2 % duty)
+ *
+ * ⚠ **Do NOT size this from the mean.** 5.4 KB / 1.14 ms makes inline parsing look free, and that average is
+ * what hid the problem: it mixes two populations — many tiny documents that dominate the COUNT and a few large
+ * ones that dominate the COST. The relevant statistic is `inlineMaxMs` x caller concurrency.
+ *
+ * Why it mattered: the REST callers fetch N documents through one `Promise.all`, so N parses resolve
+ * back-to-back with no yield between them. At N=8 that is 8 x 42 ms = **336 ms** of un-yieldable main-thread
+ * time; observed `lagPeakMs` on those stages was 338 / 368 / 548 / 676 / 1,301 / 1,640 ms **with dispatch at
+ * zero**. The consuming service's background contract is a 4 ms cooperative slice, and a monolithic
+ * `XMLParser().parse()` cannot be sliced at all — so the only way background XML honours that contract is to
+ * leave this thread.
+ *
+ * 64 KB is chosen to move that population and nothing else: it is ~12x the 5.4 KB mean, so small documents keep
+ * the inline path where the round trip genuinely costs more than the parse. `TSRequestXml.spec.ts` already
+ * bounds this constant to [64 KB, 512 KB]; this sits at that floor deliberately.
+ *
+ * 🟢 **Safe for the money path by construction:** AMQP messages never reach this function — they parse inside a
+ * TSRQW worker pool (see the note above, and `threadFactor` in the atfeed betradar driver). Only REST does, and
+ * in the sports feed REST is exclusively the catalog/schedule sync, i.e. background work.
+ *
+ * Watch `offThread` count after this change: the pool is ONE worker, so if the 64-128 KB band turns out to hold
+ * thousands of documents rather than dozens they will serialise behind each other. That is still the right place
+ * for them — a queued worker does not block the loop — but it makes the catalog sweep slower, which is the
+ * trade being accepted here.
+ */
+/**
+ * DEFAULT threshold. The authority for this number is this library's own measurement,
+ * `test/utils/TSRequestXmlCrossover.spec.ts`, which measures the main-thread BLOCK per mode across a size
+ * ladder. Change it against that spec's output, not against reasoning: it has been wrong twice before, in
+ * opposite directions, both times from reasoning alone.
+ *
+ * Two specs constrain it and the effective bound is their INTERSECTION, [64 KB, 128 KB] —
+ * `TSRequestXml.spec.ts` asserts [64 KB, 512 KB], the crossover spec asserts [32 KB, 128 KB]. 64 KB therefore
+ * sits exactly on the floor: it cannot be lowered by editing this line alone, which is deliberate.
+ *
+ * A consumer that wants a different value sets it at startup with {@link configureXmlParsing} instead. That
+ * accepts a wider range on purpose — probing is a runtime question, while this constant is a shipped default —
+ * and it keeps the workload that justifies a change, and the record of it, in the service that measured it.
+ */
+export declare const XML_OFFTHREAD_MIN_BYTES: number;
+/**
+ * Default ceiling for one off-thread parse. Generous next to a ~1 MB document (~250 ms measured), because
+ * firing it also retires the worker — see `TSRQWPoolOptions.taskTimeoutMs`.
+ */
+export declare const XML_WORKER_TASK_TIMEOUT_MS = 30000;
+/**
+ * Override XML parsing behaviour for THIS process.
+ *
+ * The defaults above stay the measured-neutral values — 64 KB is the only threshold with evidence behind it
+ * (see {@link XML_OFFTHREAD_MIN_BYTES}), so a service that configures nothing gets exactly what was measured.
+ * This exists so a service can probe a different value from its OWN startup, without a library edit, a
+ * version bump across every consumer, or a hand-patched `node_modules` copy — all three of which were done
+ * on 2026-08-16 to test a single number.
+ *
+ * ⚠ Call this at startup, BEFORE the first large document is parsed. The worker pool is created lazily on
+ * first use and reads `workerTaskTimeoutMs` at construction, so a later change to the timeout applies only
+ * if the pool is rebuilt. `offThreadMinBytes` is read per call and takes effect immediately.
+ *
+ * Throws on an out-of-range or non-integer threshold rather than clamping: this value decides which parses
+ * hold the event loop, and a silently corrected one would misreport what a measurement actually ran.
+ */
+export declare function configureXmlParsing(opts: {
+    offThreadMinBytes?: number;
+    workerTaskTimeoutMs?: number;
+}): void;
+/** The values actually in force — report these with a measurement, never the defaults. */
+export declare function getXmlParsingConfig(): {
+    offThreadMinBytes: number;
+    workerTaskTimeoutMs: number;
+};
+/**
+ * Old-generation ceiling for the single XML parse worker.
+ *
+ * Bounds a cost that was previously unbounded in two directions at once: an unbounded number of workers
+ * (see the pool construction) and an unbounded heap per worker. V8 does not return old-generation pages
+ * to the OS, so a long-lived worker's high-water mark IS the process's RSS for its lifetime.
+ *
+ * 192 MB is ~25x the parsed graph of the largest document observed (~1 MB of XML), so it is a runaway
+ * guard rather than a working limit. Raising it should be justified by `offThreadBytes / offThread`, not
+ * by a worker failure — a failure here degrades to inline parsing, it does not drop the document.
+ */
+export declare const XML_WORKER_MAX_OLD_GEN_MB = 192;
+/**
+ * THE single definition of the feed `isArray` predicate: array-ify every non-attribute nested node.
+ *
+ * Everything else derives from this — including the worker copies, via
+ * {@link FEED_XML_PARSER_WORKER_CONFIG}. Do not restate this logic anywhere.
+ */
+export declare const feedXmlIsArray: (_n: any, jPath: any, _l: any, isAttr: any) => boolean;
+/**
+ * Data-only fast-xml-parser options (structured-cloneable — no functions).
+ *
+ * These are the values the atfeed Betradar AMQP worker had been carrying locally, promoted here so
+ * every feed parse — REST and AMQP — shares one configuration and atfeed no longer overrides anything.
+ *
+ * `ignoreDeclaration: true` was the one genuine divergence: the AMQP worker set it, while
+ * `TSRequest.xml` and atfeed's own `recovery-request.ts` parser did not. Without it, a document with an
+ * `<?xml …?>` prolog yields an extra `?xml` key in the parsed object, so REST and AMQP produced
+ * different shapes for the same markup. Adopting the AMQP value drops that key from REST parses too —
+ * a deliberate behaviour change that makes the two paths agree. Nothing reads `?xml`; it is prolog
+ * metadata, not content.
+ */
+export declare const FEED_XML_PARSER_DATA_OPTIONS: {
+    readonly ignoreDeclaration: true;
+    readonly ignoreAttributes: false;
+    readonly attributeNamePrefix: '';
+    readonly attributesGroupName: '$';
+    readonly textNodeName: '_';
+};
+/**
+ * Canonical fast-xml-parser options for provider feed XML (REST **and** AMQP).
+ *
+ * Use this everywhere a feed document is parsed. Divergence between two parser configurations does not
+ * fail loudly — it silently changes the parsed shape, which for odds/settlement payloads means wrong
+ * market or outcome data rather than an error.
+ */
+export declare function feedXmlParserOptions(): Record<string, unknown>;
+/**
+ * Options for a worker that must rebuild the parser itself.
+ *
+ * A TSRQW callback is stringified into an eval worker, so it closes over nothing, and `isArray` is a
+ * function — which structured-clone cannot transfer. Passing the predicate's SOURCE keeps a single
+ * definition: the worker reconstructs it with {@link buildFeedXmlParserOptions} rather than restating
+ * the logic. Derived via `toString()` so the source can never drift from the function above.
+ *
+ * Pass this as TSRQW's `workerData` (or merge it into an existing `workerData`).
+ */
+export declare const FEED_XML_PARSER_WORKER_CONFIG: {
+    readonly ignoreDeclaration: true;
+    readonly ignoreAttributes: false;
+    readonly attributeNamePrefix: '';
+    readonly attributesGroupName: '$';
+    readonly textNodeName: '_';
+    readonly isArraySource: string;
+};
+/**
+ * Rebuild the canonical parser options inside a worker from {@link FEED_XML_PARSER_WORKER_CONFIG}.
+ *
+ * Safe to call from stringified worker code: it only needs the plain config object. `new Function` is
+ * used to revive the predicate — no additional risk in a context that is already an eval worker, and
+ * TSRQW's wrapper predefines `__name` for the same reason (bundlers inject it into `toString()` output).
+ */
+export declare function buildFeedXmlParserOptions(cfg: {
+    isArraySource?: string;
+} & Record<string, unknown>): Record<string, unknown>;
+/**
+ * Counters for how XML documents were parsed.
+ *
+ * `fallback` is the one to watch: a non-zero and growing value means large documents are being parsed
+ * on the main thread despite the threshold, so the event-loop protection is not actually in effect.
+ */
+export declare function getXmlParseStats(): {
+    inline: number;
+    offThread: number;
+    fallback: number;
+    inlineBytes: number;
+    inlineMaxBytes: number;
+    offThreadBytes: number;
+    /** Cumulative wall time of inline parses — see {@link xmlParseStats}. */
+    inlineMs: number;
+    /** Single worst inline parse. This is the event loop held in one uninterruptible go. */
+    inlineMaxMs: number;
+    /** Documents handed to the pool. `submitted - offThread - fallback` is the backlog. */
+    submitted: number;
+    /** Documents currently awaiting a worker reply. Sustained non-zero at idle means a stuck task. */
+    inFlight: number;
+};
+export declare function resetXmlParseStatsForTests(): void;
+/** Shut the XML parse pool down (tests, graceful shutdown). Safe to call when none exists. */
+export declare function closeXmlWorkerPool(): void;
+/**
+ * Parse an XML document, off the main thread when it is large enough to matter.
+ *
+ * Falls back to inline parsing when the document is small, when no worker pool can be created, or when
+ * the worker fails for any reason — the result must be identical either way, so the only difference a
+ * caller can observe is whether the event loop was blocked.
+ */
+export declare function parseXmlDocument(body: string): Promise<any>;
+export declare class TSRequest {
+    private static responseBody;
+    private static isNonXmlBody;
+    private static debugTrace;
+    static form(url: string, options?: TSRequestHttpOptions, debug?: boolean): Promise<any>;
+    static xml(url: string, options?: TSRequestHttpOptions, debug?: boolean): Promise<any>;
+    static json(url: string, options?: TSRequestHttpOptions, debug?: boolean): Promise<any>;
+    static raw(url: string, options?: TSRequestHttpOptions): Promise<ITSResponse>;
+    static error(kind: string, status: number, message: any): Error;
+    static url(url: string): {
+        href: string;
+        protocol: string;
+        host: string;
+        hostname: string;
+        port: string;
+        path: string;
+        pathname: string;
+        search: string;
+        hash: string;
+    } | {
+        href?: undefined;
+        protocol?: undefined;
+        host?: undefined;
+        hostname?: undefined;
+        port?: undefined;
+        path?: undefined;
+        pathname?: undefined;
+        search?: undefined;
+        hash?: undefined;
+    };
+}
+export declare function request(url: string, { body, qs, debug, ...options }?: Record<string, unknown>, kind?: 'raw' | 'xml' | 'json' | 'form'): Promise<any>;