xtrawl 0.1.1 → 0.1.3

package/dist/manifest/destinations.js

@@ -0,0 +1,47 @@
+import { ManifestError } from "../domain/errors.js";
+export const DEFAULT_CREDENTIAL_HOSTS = [
+    "x.com",
+    "www.x.com",
+    "api.x.com",
+    "twitter.com",
+    "www.twitter.com",
+    "api.twitter.com",
+];
+const GRAPHQL_PATH = /^\/(?:i\/api\/)?graphql\/[A-Za-z0-9_-]+\/[A-Za-z0-9_]+$/u;
+export function allowedCredentialHosts(extraOrigins = []) {
+    const hosts = new Set(DEFAULT_CREDENTIAL_HOSTS);
+    for (const origin of extraOrigins) {
+        const host = originHost(origin);
+        if (host)
+            hosts.add(host);
+    }
+    return hosts;
+}
+export function assertCredentialDestination(value, extraOrigins = []) {
+    let parsed;
+    try {
+        parsed = new URL(value);
+    }
+    catch {
+        throw new ManifestError("Manifest endpoint is not a valid URL.");
+    }
+    if (parsed.protocol !== "https:")
+        throw new ManifestError(`Refusing non-HTTPS credential destination: ${parsed.origin}`);
+    if (!allowedCredentialHosts(extraOrigins).has(parsed.hostname))
+        throw new ManifestError(`Refusing untrusted credential destination: ${parsed.origin}`);
+    if (!GRAPHQL_PATH.test(parsed.pathname))
+        throw new ManifestError(`Refusing untrusted credential path: ${parsed.pathname}`);
+    return parsed.toString();
+}
+function originHost(value) {
+    const trimmed = value.trim();
+    if (!trimmed)
+        return undefined;
+    try {
+        return new URL(trimmed.includes("://") ? trimmed : `https://${trimmed}`).hostname;
+    }
+    catch {
+        return undefined;
+    }
+}
+//# sourceMappingURL=destinations.js.map

package/dist/manifest/destinations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"destinations.js","sourceRoot":"","sources":["../../src/manifest/destinations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,OAAO;IACP,WAAW;IACX,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,iBAAiB;CACT,CAAC;AAEX,MAAM,YAAY,GAAG,0DAA0D,CAAC;AAEhF,MAAM,UAAU,sBAAsB,CAAC,eAAkC,EAAE;IACzE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,wBAAwB,CAAC,CAAC;IACxD,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,IAAI;YAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,KAAa,EAAE,eAAkC,EAAE;IAC7F,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,aAAa,CAAC,uCAAuC,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAC9B,MAAM,IAAI,aAAa,CAAC,8CAA8C,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACzF,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC5D,MAAM,IAAI,aAAa,CAAC,8CAA8C,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACzF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QACrC,MAAM,IAAI,aAAa,CAAC,uCAAuC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACpF,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}

package/dist/manifest/model.d.ts

@@ -1,3 +1,4 @@
-import type { Manifest, ManifestPayload } from "../domain/manifest.js";
-export declare function createManifest(payload: ManifestPayload): Manifest;
+import type { Manifest, ManifestCreateOptions, ManifestPayload } from "../domain/manifest.js";
+export type { ManifestCreateOptions } from "../domain/manifest.js";
+export declare function createManifest(payload: ManifestPayload, options?: ManifestCreateOptions): Manifest;
 //# sourceMappingURL=model.d.ts.map

package/dist/manifest/model.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../src/manifest/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA8B,QAAQ,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAInG,wBAAgB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,QAAQ,CA0BjE"}
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../src/manifest/model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,QAAQ,EACR,qBAAqB,EACrB,eAAe,EAChB,MAAM,uBAAuB,CAAC;AAK/B,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAEnE,wBAAgB,cAAc,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,GAAE,qBAA0B,GAAG,QAAQ,CAiCtG"}

package/dist/manifest/model.js

@@ -1,6 +1,7 @@
 import { ManifestError } from "../domain/errors.js";
 import { asInteger, asString, isRecord } from "../utils/guards.js";
-export function createManifest(payload) {
+import { assertCredentialDestination } from "./destinations.js";
+export function createManifest(payload, options = {}) {
     if (!payload || !isRecord(payload.queryIds) || !isRecord(payload.endpoints)) {
         throw new ManifestError("Manifest requires queryIds and endpoints objects.");
     }
@@ -8,6 +9,10 @@ export function createManifest(payload) {
     const endpoints = mapStrings(payload.endpoints);
     if (!queryIds.search_timeline || !endpoints.search_timeline)
         throw new ManifestError("Manifest requires a search endpoint.");
+    for (const [operation, endpoint] of Object.entries(endpoints)) {
+        const queryId = queryIds[operation] ?? "validation";
+        assertCredentialDestination(endpoint.includes("{query_id}") ? endpoint.replace("{query_id}", queryId) : endpoint, options.allowedOrigins ?? []);
+    }
     const features = mapBooleans(payload.features);
     const operationFeatures = mapBooleanMap(payload.operationFeatures);
     const operationFieldToggles = mapBooleanMap(payload.operationFieldToggles);

package/dist/manifest/model.js.map

@@ -1 +1 @@
-{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/manifest/model.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnE,MAAM,UAAU,cAAc,CAAC,OAAwB;IACrD,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,aAAa,CAAC,mDAAmD,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,eAAe;QACzD,MAAM,IAAI,aAAa,CAAC,sCAAsC,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,iBAAiB,GAAG,aAAa,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnE,MAAM,qBAAqB,GAAG,aAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC3E,OAAO;QACL,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,SAAS;QAC/C,QAAQ;QACR,SAAS;QACT,QAAQ;QACR,iBAAiB;QACjB,qBAAqB;QACrB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAClE,WAAW,CAAC,SAAiB;YAC3B,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAClE,CAAC;QACD,eAAe,CAAC,SAAiB;YAC/B,OAAO,qBAAqB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAChD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAA8B,EAAE,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CACnG,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC"}
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/manifest/model.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAIhE,MAAM,UAAU,cAAc,CAAC,OAAwB,EAAE,UAAiC,EAAE;IAC1F,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,aAAa,CAAC,mDAAmD,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,eAAe;QACzD,MAAM,IAAI,aAAa,CAAC,sCAAsC,CAAC,CAAC;IAClE,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9D,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC;QACpD,2BAA2B,CACzB,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,EACpF,OAAO,CAAC,cAAc,IAAI,EAAE,CAC7B,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,iBAAiB,GAAG,aAAa,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnE,MAAM,qBAAqB,GAAG,aAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC3E,OAAO;QACL,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,SAAS;QAC/C,QAAQ;QACR,SAAS;QACT,QAAQ;QACR,iBAAiB;QACjB,qBAAqB;QACrB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAClE,WAAW,CAAC,SAAiB;YAC3B,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAClE,CAAC;QACD,eAAe,CAAC,SAAiB;YAC/B,OAAO,qBAAqB,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAChD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAA8B,EAAE,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CACnG,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACpG,CAAC"}

package/dist/manifest/provider.d.ts

@@ -1,19 +1,26 @@
 import type { ClientConfig } from "../config/types.js";
-import type { Manifest, ManifestPayload } from "../domain/manifest.js";
+import type { Manifest, ManifestPayload, ManifestScrapeOptions } from "../domain/manifest.js";
 import type { ManifestRepository } from "../storage/manifest-repository.js";
-import type { ManifestScrapeOptions } from "./scraper.js";
+export declare const LIVE_MANIFEST_CACHE_KEY = "xtrawl:live-web";
 export declare class ManifestProvider {
     private readonly config;
     private readonly repository;
     private readonly remoteFetch;
     private readonly liveAuthToken?;
     private readonly liveScrape;
-    private liveManifest?;
+    private readonly local;
+    private active?;
+    private inFlightRefresh?;
+    private refreshFailedUntil;
     private remoteRefreshAttempted;
     constructor(config: ClientConfig, repository: ManifestRepository, remoteFetch?: (url: string) => Promise<unknown>, liveAuthToken?: string | undefined, liveScrape?: (base: ManifestPayload, options?: ManifestScrapeOptions) => Promise<ManifestPayload>);
     getManifest(): Promise<Manifest>;
     refreshLive(authToken?: string): Promise<Manifest>;
+    private loadRemote;
     private fetchRemote;
     private live;
+    private runLive;
+    private readCached;
+    private manifestFrom;
 }
 //# sourceMappingURL=provider.d.ts.map

package/dist/manifest/provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/manifest/provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAEvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAI5E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAE1D,qBAAa,gBAAgB;IAKzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAR7B,OAAO,CAAC,YAAY,CAAC,CAAW;IAChC,OAAO,CAAC,sBAAsB,CAAS;gBAGpB,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,kBAAkB,EAC9B,WAAW,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAoB,EACjE,aAAa,CAAC,EAAE,MAAM,YAAA,EACtB,UAAU,GAAE,CAC3B,IAAI,EAAE,eAAe,EACrB,OAAO,CAAC,EAAE,qBAAqB,KAC5B,OAAO,CAAC,eAAe,CAAyB;IAG1C,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IA4BhC,WAAW,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;YAIjD,WAAW;YAOX,IAAI;CAenB"}
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../src/manifest/provider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAE9F,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAK5E,eAAO,MAAM,uBAAuB,oBAAoB,CAAC;AAGzD,qBAAa,gBAAgB;IAQzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAX7B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAW;IACjC,OAAO,CAAC,MAAM,CAAC,CAAW;IAC1B,OAAO,CAAC,eAAe,CAAC,CAAoB;IAC5C,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,sBAAsB,CAAS;gBAGpB,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,kBAAkB,EAC9B,WAAW,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAoB,EACjE,aAAa,CAAC,EAAE,MAAM,YAAA,EACtB,UAAU,GAAE,CAC3B,IAAI,EAAE,eAAe,EACrB,OAAO,CAAC,EAAE,qBAAqB,KAC5B,OAAO,CAAC,eAAe,CAAyB;IAK1C,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAmBhC,WAAW,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;YAIjD,UAAU;YAqBV,WAAW;YAQX,IAAI;YAaJ,OAAO;IAuBrB,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,YAAY;CAGrB"}

package/dist/manifest/provider.js

@@ -2,13 +2,18 @@ import { ManifestError } from "../domain/errors.js";
 import { DEFAULT_MANIFEST } from "./default-manifest.js";
 import { createManifest } from "./model.js";
 import { scrapeManifestFromWeb } from "./scraper.js";
+export const LIVE_MANIFEST_CACHE_KEY = "xtrawl:live-web";
+const REFRESH_BACKOFF_MS = 30_000;
 export class ManifestProvider {
     config;
     repository;
     remoteFetch;
     liveAuthToken;
     liveScrape;
-    liveManifest;
+    local;
+    active;
+    inFlightRefresh;
+    refreshFailedUntil = 0;
     remoteRefreshAttempted = false;
     constructor(config, repository, remoteFetch = defaultFetchJson, liveAuthToken, liveScrape = scrapeManifestFromWeb) {
         this.config = config;
@@ -16,66 +21,113 @@ export class ManifestProvider {
         this.remoteFetch = remoteFetch;
         this.liveAuthToken = liveAuthToken;
         this.liveScrape = liveScrape;
+        this.local = this.manifestFrom(DEFAULT_MANIFEST);
     }
     async getManifest() {
-        const local = createManifest(DEFAULT_MANIFEST);
-        if (this.config.manifestScrapeOnInit) {
-            if (this.liveManifest)
-                return this.liveManifest;
-            return this.live(local, false);
-        }
+        if (this.active)
+            return this.active;
+        if (this.config.manifestScrapeOnInit)
+            return this.live(false);
         if (this.config.manifestUrl) {
-            if (this.config.manifestUpdateOnInit && !this.remoteRefreshAttempted) {
-                this.remoteRefreshAttempted = true;
-                try {
-                    return await this.fetchRemote(this.config.manifestUrl);
-                }
-                catch {
-                    const stale = this.repository.get(this.config.manifestUrl, true);
-                    if (stale)
-                        return createManifest(stale);
-                }
+            const remote = await this.loadRemote(this.config.manifestUrl);
+            if (remote) {
+                this.active = remote;
+                return remote;
             }
-            const cached = this.repository.get(this.config.manifestUrl);
-            if (cached)
-                return createManifest(cached);
+        }
+        const cachedLive = this.readCached(LIVE_MANIFEST_CACHE_KEY);
+        if (cachedLive) {
+            this.active = cachedLive;
+            return cachedLive;
+        }
+        this.active = this.local;
+        return this.local;
+    }
+    async refreshLive(authToken) {
+        return this.live(true, authToken);
+    }
+    async loadRemote(url) {
+        if (this.config.manifestUpdateOnInit && !this.remoteRefreshAttempted) {
+            this.remoteRefreshAttempted = true;
             try {
-                return await this.fetchRemote(this.config.manifestUrl);
+                return await this.fetchRemote(url);
             }
-            catch {
-                const stale = this.repository.get(this.config.manifestUrl, true);
+            catch (error) {
+                if (error instanceof ManifestError)
+                    throw error;
+                const stale = this.readCached(url, true);
                 if (stale)
-                    return createManifest(stale);
+                    return stale;
             }
         }
-        return local;
-    }
-    async refreshLive(authToken) {
-        return this.live(createManifest(DEFAULT_MANIFEST), true, authToken);
+        const cached = this.readCached(url);
+        if (cached)
+            return cached;
+        try {
+            return await this.fetchRemote(url);
+        }
+        catch (error) {
+            if (error instanceof ManifestError)
+                throw error;
+            return this.readCached(url, true);
+        }
     }
     async fetchRemote(url) {
         const remote = await this.remoteFetch(url);
         const payload = normalizePayload(remote);
+        const manifest = this.manifestFrom(payload);
         this.repository.set(url, payload, this.config.manifestTtlMs);
-        return createManifest(payload);
+        return manifest;
+    }
+    async live(strict, authToken) {
+        if (this.inFlightRefresh)
+            return this.inFlightRefresh;
+        if (Date.now() < this.refreshFailedUntil) {
+            if (strict)
+                throw new ManifestError("Live manifest refresh is in backoff.");
+            return this.active ?? this.local;
+        }
+        const pending = this.runLive(strict, authToken).finally(() => {
+            if (this.inFlightRefresh === pending)
+                this.inFlightRefresh = undefined;
+        });
+        this.inFlightRefresh = pending;
+        return pending;
     }
-    async live(local, strict, authToken) {
+    async runLive(strict, authToken) {
         try {
             const token = authToken ?? this.liveAuthToken;
             const payload = await this.liveScrape(DEFAULT_MANIFEST, {
                 ...(token ? { authToken: token } : {}),
             });
+            this.active = this.manifestFrom(payload);
+            this.repository.set(this.config.manifestUrl ?? LIVE_MANIFEST_CACHE_KEY, payload, this.config.manifestTtlMs);
             if (this.config.manifestUrl)
-                this.repository.set(this.config.manifestUrl, payload, this.config.manifestTtlMs);
-            this.liveManifest = createManifest(payload);
-            return this.liveManifest;
+                this.repository.set(LIVE_MANIFEST_CACHE_KEY, payload, this.config.manifestTtlMs);
+            this.refreshFailedUntil = 0;
+            return this.active;
         }
         catch (error) {
+            this.refreshFailedUntil = Date.now() + REFRESH_BACKOFF_MS;
             if (strict)
                 throw new ManifestError(`Live manifest refresh failed: ${String(error)}`);
-            return local;
+            return this.active ?? this.local;
         }
     }
+    readCached(key, allowExpired = false) {
+        const payload = this.repository.get(key, allowExpired);
+        if (!payload)
+            return undefined;
+        try {
+            return this.manifestFrom(payload);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    manifestFrom(payload) {
+        return createManifest(payload, { allowedOrigins: this.config.allowedManifestOrigins });
+    }
 }
 async function defaultFetchJson(url) {
     const response = await fetch(url);

package/dist/manifest/provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/manifest/provider.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAGrD,MAAM,OAAO,gBAAgB;IAKR;IACA;IACA;IACA;IACA;IARX,YAAY,CAAY;IACxB,sBAAsB,GAAG,KAAK,CAAC;IAEvC,YACmB,MAAoB,EACpB,UAA8B,EAC9B,cAAiD,gBAAgB,EACjE,aAAsB,EACtB,aAGe,qBAAqB;QAPpC,WAAM,GAAN,MAAM,CAAc;QACpB,eAAU,GAAV,UAAU,CAAoB;QAC9B,gBAAW,GAAX,WAAW,CAAsD;QACjE,kBAAa,GAAb,aAAa,CAAS;QACtB,eAAU,GAAV,UAAU,CAG0B;IACpD,CAAC;IAEG,KAAK,CAAC,WAAW;QACtB,MAAM,KAAK,GAAG,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC/C,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBACrE,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;gBACnC,IAAI,CAAC;oBACH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzD,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;oBACjE,IAAI,KAAK;wBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5D,IAAI,MAAM;gBAAE,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACzD,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;gBACjE,IAAI,KAAK;oBAAE,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,SAAkB;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;IACtE,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC7D,OAAO,cAAc,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,KAAe,EAAE,MAAe,EAAE,SAAkB;QACrE,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;YAC9C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE;gBACtD,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvC,CAAC,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW;gBACzB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnF,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;YAC5C,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM;gBAAE,MAAM,IAAI,aAAa,CAAC,iCAAiC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACtF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1F,OAAO,QAAQ,CAAC,IAAI,EAAsB,CAAC;AAC7C,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7D,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,KAAiC,CAAC;IACpD,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,SAAS;QAC7C,MAAM,IAAI,aAAa,CAAC,mDAAmD,CAAC,CAAC;IAC/E,OAAO,SAA4B,CAAC;AACtC,CAAC"}
+{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/manifest/provider.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD,MAAM,CAAC,MAAM,uBAAuB,GAAG,iBAAiB,CAAC;AACzD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,OAAO,gBAAgB;IAQR;IACA;IACA;IACA;IACA;IAXF,KAAK,CAAW;IACzB,MAAM,CAAY;IAClB,eAAe,CAAqB;IACpC,kBAAkB,GAAG,CAAC,CAAC;IACvB,sBAAsB,GAAG,KAAK,CAAC;IAEvC,YACmB,MAAoB,EACpB,UAA8B,EAC9B,cAAiD,gBAAgB,EACjE,aAAsB,EACtB,aAGe,qBAAqB;QAPpC,WAAM,GAAN,MAAM,CAAc;QACpB,eAAU,GAAV,UAAU,CAAoB;QAC9B,gBAAW,GAAX,WAAW,CAAsD;QACjE,kBAAa,GAAb,aAAa,CAAS;QACtB,eAAU,GAAV,UAAU,CAG0B;QAErD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAC;IACnD,CAAC;IAEM,KAAK,CAAC,WAAW;QACtB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACpC,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAC9D,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACrB,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAC;QAC5D,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC;YACzB,OAAO,UAAU,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,SAAkB;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACrE,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;YACnC,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACrC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,aAAa;oBAAE,MAAM,KAAK,CAAC;gBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACzC,IAAI,KAAK;oBAAE,OAAO,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,aAAa;gBAAE,MAAM,KAAK,CAAC;YAChD,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAC5C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC7D,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,MAAe,EAAE,SAAkB;QACpD,IAAI,IAAI,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC,eAAe,CAAC;QACtD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzC,IAAI,MAAM;gBAAE,MAAM,IAAI,aAAa,CAAC,sCAAsC,CAAC,CAAC;YAC5E,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC;QACnC,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;YAC3D,IAAI,IAAI,CAAC,eAAe,KAAK,OAAO;gBAAE,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACzE,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAC/B,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,MAAe,EAAE,SAAkB;QACvD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;YAC9C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE;gBACtD,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,uBAAuB,EAClD,OAAO,EACP,IAAI,CAAC,MAAM,CAAC,aAAa,CAC1B,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW;gBACzB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,uBAAuB,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACnF,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;YAC5B,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;YAC1D,IAAI,MAAM;gBAAE,MAAM,IAAI,aAAa,CAAC,iCAAiC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACtF,OAAO,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC;QACnC,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAW,EAAE,YAAY,GAAG,KAAK;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,OAAO;YAAE,OAAO,SAAS,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,OAAwB;QAC3C,OAAO,cAAc,CAAC,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC,CAAC;IACzF,CAAC;CACF;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1F,OAAO,QAAQ,CAAC,IAAI,EAAsB,CAAC;AAC7C,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7D,MAAM,IAAI,aAAa,CAAC,mCAAmC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,KAAiC,CAAC;IACpD,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,SAAS;QAC7C,MAAM,IAAI,aAAa,CAAC,mDAAmD,CAAC,CAAC;IAC/E,OAAO,SAA4B,CAAC;AACtC,CAAC"}

package/dist/manifest/scraper.d.ts

@@ -1,9 +1,5 @@
-import type { ManifestPayload } from "../domain/manifest.js";
-export interface ManifestScrapeOptions {
-    readonly authToken?: string;
-    readonly fetcher?: typeof fetch;
-    readonly maxScripts?: number;
-}
+import type { ManifestPayload, ManifestScrapeOptions } from "../domain/manifest.js";
+export type { ManifestScrapeOptions } from "../domain/manifest.js";
 export declare function extractManifestFromJavascript(source: string, fallback: ManifestPayload): ManifestPayload;
 export declare function extractOperationQueryIdsFromJavascript(source: string): Record<string, string>;
 export declare function extractOperationFeaturesFromJavascript(source: string): Record<string, Record<string, boolean>>;

package/dist/manifest/scraper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/manifest/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAa7D,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IAChC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,GAAG,eAAe,CAWxG;AAED,wBAAgB,sCAAsC,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAe7F;AAED,wBAAgB,sCAAsC,CACpD,MAAM,EAAE,MAAM,GACb,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAmBzC;AAED,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,eAAe,CAAC,CAsC1B"}
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/manifest/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAmBpF,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAEnE,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,GAAG,eAAe,CAWxG;AAED,wBAAgB,sCAAsC,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAe7F;AAED,wBAAgB,sCAAsC,CACpD,MAAM,EAAE,MAAM,GACb,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAmBzC;AAED,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,eAAe,CAAC,CA+C1B"}

package/dist/manifest/scraper.js

@@ -1,3 +1,4 @@
+import { AUXILIARY_REQUEST_TIMEOUT_MS, cancelBody, DEFAULT_MAX_RESPONSE_BYTES, readResponseText, } from "../transport/body.js";
 import { asString } from "../utils/guards.js";
 const OPERATION_NAMES = {
     SearchTimeline: "search_timeline",
@@ -64,24 +65,34 @@ export async function scrapeManifestFromWeb(base, options = {}) {
         ...scriptHeaders,
         ...(options.authToken ? { Cookie: `auth_token=${options.authToken}` } : {}),
     };
-    const response = await fetcher("https://x.com/home", { headers: pageHeaders, redirect: "follow" });
-    if (!response.ok)
-        throw new Error(`Manifest page failed with status ${response.status}`);
-    const html = await response.text();
-    const scripts = [...new Set([...html.matchAll(/<script[^>]+src=["']([^"']+)["']/giu)])]
-        .map((match) => match[1])
-        .filter((item) => Boolean(item))
-        .map((source) => safeScriptUrl(source))
-        .filter((source) => Boolean(source))
-        .sort((left, right) => scriptPriority(left) - scriptPriority(right))
-        .slice(0, options.maxScripts ?? 20);
+    const html = await fetchBoundedText(fetcher, "https://x.com/home", {
+        headers: pageHeaders,
+        redirect: "error",
+    });
+    const seen = new Set();
+    const scripts = [];
+    for (const match of html.matchAll(/<script[^>]+src=["']([^"']+)["']/giu)) {
+        const raw = match[1];
+        if (!raw)
+            continue;
+        const url = safeScriptUrl(raw);
+        if (!url || seen.has(url))
+            continue;
+        seen.add(url);
+        scripts.push(url);
+    }
+    scripts.sort((left, right) => scriptPriority(left) - scriptPriority(right));
+    const selected = scripts.slice(0, options.maxScripts ?? 20);
     const discovered = {};
     const discoveredFeatures = {};
-    for (const url of scripts) {
-        const bundle = await fetcher(url, { headers: scriptHeaders, redirect: "follow" });
-        if (!bundle.ok)
+    for (const url of selected) {
+        let source;
+        try {
+            source = await fetchBoundedText(fetcher, url, { headers: scriptHeaders, redirect: "follow" });
+        }
+        catch {
             continue;
-        const source = await bundle.text();
+        }
         Object.assign(discovered, extractOperationQueryIdsFromJavascript(source));
         Object.assign(discoveredFeatures, extractOperationFeaturesFromJavascript(source));
         if (Object.keys(discovered).length === Object.keys(OPERATION_NAMES).length)
@@ -97,6 +108,26 @@ export async function scrapeManifestFromWeb(base, options = {}) {
         operationFeatures: mergeOperationFeatures(base.operationFeatures, discoveredFeatures),
     };
 }
+async function fetchBoundedText(fetcher, url, init) {
+    const timeout = new AbortController();
+    const timer = setTimeout(() => timeout.abort(), AUXILIARY_REQUEST_TIMEOUT_MS);
+    const signal = init.signal ? AbortSignal.any([init.signal, timeout.signal]) : timeout.signal;
+    try {
+        const response = await fetcher(url, { ...init, signal, redirect: init.redirect ?? "follow" });
+        try {
+            if (!response.ok)
+                throw new Error(`Manifest fetch failed with status ${response.status}`);
+            return await readResponseText(response, { signal, maxBytes: DEFAULT_MAX_RESPONSE_BYTES });
+        }
+        catch (error) {
+            await cancelBody(response);
+            throw error;
+        }
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
 function mergeOperationFeatures(base, discovered) {
     const out = {};
     if (base)

package/dist/manifest/scraper.js.map

@@ -1 +1 @@
-{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/manifest/scraper.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,eAAe,GAA2B;IAC9C,cAAc,EAAE,iBAAiB;IACjC,gBAAgB,EAAE,yBAAyB;IAC3C,UAAU,EAAE,kBAAkB;IAC9B,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,qBAAqB,EAAE,oBAAoB;IAC3C,mBAAmB,EAAE,cAAc;CACpC,CAAC;AAQF,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,QAAyB;IACrF,MAAM,iBAAiB,GAAG,sCAAsC,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,QAAQ;QACX,QAAQ,EAAE;YACR,GAAG,QAAQ,CAAC,QAAQ;YACpB,GAAG,sCAAsC,CAAC,MAAM,CAAC;SAClD;QACD,iBAAiB,EAAE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;QACxF,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,MAAc;IACnE,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,UAAU,GAAG,IAAI,MAAM,CAC3B,sEAAsE,aAAa,MAAM,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,MAAM,CAC/B,6BAA6B,aAAa,+CAA+C,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,IAAI,EAAE;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,MAAc;IAEd,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,6BAA6B,aAAa,MAAM,EAAE,IAAI,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACnG,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,YAAY,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrD,IAAI,UAAU,GAAG,CAAC;YAAE,SAAS;QAC7B,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;aACrF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAqB,EACrB,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IACzC,MAAM,aAAa,GAAG;QACpB,YAAY,EAAE,aAAa;KAC5B,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,GAAG,aAAa;QAChB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IACnG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC;SACpF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC/C,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;SACtC,MAAM,CAAC,CAAC,MAAM,EAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;SACnE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,kBAAkB,GAA4C,EAAE,CAAC;IACvE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,SAAS;QACzB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM;YAAE,MAAM;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO;QACL,GAAG,IAAI;QACP,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,UAAU,EAAE;QAC7C,iBAAiB,EAAE,sBAAsB,CAAC,IAAI,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KACtF,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,IAA0C,EAC1C,UAAuE;IAEvE,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,IAAI,IAAI;QACN,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;YACtD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAI,QAAoC,EAAE,CAAC;IAClE,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;QAC5D,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAChD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,eAAe;YAAE,OAAO,SAAS,CAAC;QACnF,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,MAAc;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;IACtC,IAAI,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC"}
+{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/manifest/scraper.ts"],"names":[],"mappings":"AACA,OAAO,EACL,4BAA4B,EAC5B,UAAU,EACV,0BAA0B,EAC1B,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,eAAe,GAA2B;IAC9C,cAAc,EAAE,iBAAiB;IACjC,gBAAgB,EAAE,yBAAyB;IAC3C,UAAU,EAAE,kBAAkB;IAC9B,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,qBAAqB,EAAE,oBAAoB;IAC3C,mBAAmB,EAAE,cAAc;CACpC,CAAC;AAIF,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,QAAyB;IACrF,MAAM,iBAAiB,GAAG,sCAAsC,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,QAAQ;QACX,QAAQ,EAAE;YACR,GAAG,QAAQ,CAAC,QAAQ;YACpB,GAAG,sCAAsC,CAAC,MAAM,CAAC;SAClD;QACD,iBAAiB,EAAE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;QACxF,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,MAAc;IACnE,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,UAAU,GAAG,IAAI,MAAM,CAC3B,sEAAsE,aAAa,MAAM,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,MAAM,CAC/B,6BAA6B,aAAa,+CAA+C,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,IAAI,EAAE;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,MAAc;IAEd,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,6BAA6B,aAAa,MAAM,EAAE,IAAI,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACnG,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,YAAY,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrD,IAAI,UAAU,GAAG,CAAC;YAAE,SAAS;QAC7B,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;aACrF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAqB,EACrB,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IACzC,MAAM,aAAa,GAAG;QACpB,YAAY,EAAE,aAAa;KAC5B,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,GAAG,aAAa;QAChB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,oBAAoB,EAAE;QACjE,OAAO,EAAE,WAAW;QACpB,QAAQ,EAAE,OAAO;KAClB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qCAAqC,CAAC,EAAE,CAAC;QACzE,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,kBAAkB,GAA4C,EAAE,CAAC;IACvE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChG,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM;YAAE,MAAM;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO;QACL,GAAG,IAAI;QACP,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,UAAU,EAAE;QAC7C,iBAAiB,EAAE,sBAAsB,CAAC,IAAI,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KACtF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,OAAqB,EAAE,GAAW,EAAE,IAAiB;IACnF,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,4BAA4B,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7F,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1F,OAAO,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAC5F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC3B,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAC7B,IAA0C,EAC1C,UAAuE;IAEvE,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,IAAI,IAAI;QACN,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;YACtD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAI,QAAoC,EAAE,CAAC;IAClE,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;QAC5D,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAChD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,eAAe;YAAE,OAAO,SAAS,CAAC;QACnF,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,MAAc;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;IACtC,IAAI,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/output/csv-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAGA,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EACxC,MAAM,UAAQ,GACb,OAAO,CAAC,IAAI,CAAC,CAiBf"}
+{"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAMA,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EACxC,MAAM,UAAQ,GACb,OAAO,CAAC,IAAI,CAAC,CAwBf"}

package/dist/output/csv-writer.js

@@ -1,18 +1,40 @@
-import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { access, appendFile, mkdir, readFile, rename, writeFile } from "node:fs/promises";
 import { dirname } from "node:path";
+import { randomUUID } from "node:crypto";
+import { EngineError } from "../domain/errors.js";
+import { withDestinationLock } from "./lock.js";
 export async function writeCsv(path, rows, append = false) {
     if (rows.length === 0)
         return;
-    const existing = append && (await exists(path)) ? await readFile(path, "utf8") : "";
-    const parsedExisting = existing ? parseCsv(existing) : [];
-    const currentHeaders = parsedExisting[0] ?? [];
-    const headers = mergeHeaders(currentHeaders, rows.flatMap((row) => Object.keys(row)));
-    const existingRows = parsedExisting
-        .slice(1)
-        .map((values) => Object.fromEntries(currentHeaders.map((header, index) => [header, values[index] ?? ""])));
-    const content = renderCsv(headers, [...existingRows, ...rows]);
+    await withDestinationLock(path, async () => {
+        const incomingKeys = rows.flatMap((row) => Object.keys(row));
+        if (append && (await exists(path))) {
+            const existing = await readFile(path, "utf8");
+            const parsedExisting = existing ? parseCsv(existing) : [];
+            const currentHeaders = parsedExisting[0] ?? [];
+            if (currentHeaders.length === 0)
+                throw new EngineError(`Refusing to append to a CSV file without a header: ${path}`);
+            const nextHeaders = mergeHeaders(currentHeaders, incomingKeys);
+            if (!sameHeaders(currentHeaders, nextHeaders))
+                throw new EngineError(`CSV schema changed for ${path}; write a new snapshot instead of appending mixed columns.`);
+            const body = rows
+                .map((row) => currentHeaders.map((header) => escapeCsv(flattenValue(row[header]))).join(","))
+                .join("\n");
+            await appendFile(path, `${body}\n`, "utf8");
+            return;
+        }
+        const headers = mergeHeaders([], incomingKeys);
+        await atomicWrite(path, renderCsv(headers, rows));
+    });
+}
+function sameHeaders(left, right) {
+    return left.length === right.length && left.every((header, index) => header === right[index]);
+}
+async function atomicWrite(path, content) {
     await mkdir(dirname(path), { recursive: true });
-    await writeFile(path, content, "utf8");
+    const temporary = `${path}.${randomUUID()}.tmp`;
+    await writeFile(temporary, content, "utf8");
+    await rename(temporary, path);
 }
 function renderCsv(headers, rows) {
     const body = rows.map((row) => headers.map((header) => escapeCsv(flattenValue(row[header]))).join(","));

package/dist/output/csv-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv-writer.js","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,IAAwC,EACxC,MAAM,GAAG,KAAK;IAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,YAAY,CAC1B,cAAc,EACd,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CACxC,CAAC;IACF,MAAM,YAAY,GAAG,cAAc;SAChC,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACd,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CACzF,CAAC;IACJ,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,GAAG,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,SAAS,CAAC,OAA0B,EAAE,IAAwC;IACrF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACxG,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACrE,CAAC;AAED,SAAS,YAAY,CAAC,QAA2B,EAAE,QAA2B;IAC5E,MAAM,SAAS,GAAG;QAChB,SAAS;QACT,WAAW;QACX,UAAU;QACV,MAAM;QACN,MAAM;QACN,OAAO;QACP,UAAU;QACV,UAAU;QACV,UAAU;KACX,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACtB,IAAI,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAC3B,KAAK,IAAI,GAAG,CAAC;gBACb,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;;gBAAM,MAAM,GAAG,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;aAAM,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjE,IAAI,SAAS,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;gBAAE,KAAK,IAAI,CAAC,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,GAAG,GAAG,EAAE,CAAC;QACX,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,SAAS,CAAC;QACrB,CAAC;IACH,CAAC;IACD,IAAI,KAAK,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
+{"version":3,"file":"csv-writer.js","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,IAAwC,EACxC,MAAM,GAAG,KAAK;IAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,mBAAmB,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7D,IAAI,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/C,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAC7B,MAAM,IAAI,WAAW,CAAC,sDAAsD,IAAI,EAAE,CAAC,CAAC;YACtF,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,WAAW,CAAC;gBAC3C,MAAM,IAAI,WAAW,CACnB,0BAA0B,IAAI,4DAA4D,CAC3F,CAAC;YACJ,MAAM,IAAI,GAAG,IAAI;iBACd,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBAC5F,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,UAAU,CAAC,IAAI,EAAE,GAAG,IAAI,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5C,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAC/C,MAAM,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,IAAuB,EAAE,KAAwB;IACpE,OAAO,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAAe;IACtD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,UAAU,EAAE,MAAM,CAAC;IAChD,MAAM,SAAS,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,SAAS,CAAC,OAA0B,EAAE,IAAwC;IACrF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACxG,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACrE,CAAC;AAED,SAAS,YAAY,CAAC,QAA2B,EAAE,QAA2B;IAC5E,MAAM,SAAS,GAAG;QAChB,SAAS;QACT,WAAW;QACX,UAAU;QACV,MAAM;QACN,MAAM;QACN,OAAO;QACP,UAAU;QACV,UAAU;QACV,UAAU;KACX,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACtB,IAAI,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAC3B,KAAK,IAAI,GAAG,CAAC;gBACb,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;;gBAAM,MAAM,GAAG,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;aAAM,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjE,IAAI,SAAS,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;gBAAE,KAAK,IAAI,CAAC,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,GAAG,GAAG,EAAE,CAAC;QACX,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,SAAS,CAAC;QACrB,CAAC;IACH,CAAC;IACD,IAAI,KAAK,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/output/json-writer.d.ts

@@ -1,2 +1,3 @@
 export declare function writeJson(path: string, rows: readonly unknown[], append?: boolean): Promise<void>;
+export declare function writeNdjson(path: string, rows: readonly unknown[], append?: boolean): Promise<void>;
 //# sourceMappingURL=json-writer.d.ts.map

package/dist/output/json-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json-writer.d.ts","sourceRoot":"","sources":["../../src/output/json-writer.ts"],"names":[],"mappings":"AAGA,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAarG"}
+{"version":3,"file":"json-writer.d.ts","sourceRoot":"","sources":["../../src/output/json-writer.ts"],"names":[],"mappings":"AAMA,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBrG;AAED,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAWvG"}

package/dist/output/json-writer.js

@@ -1,21 +1,46 @@
-import { access, mkdir, readFile, writeFile } from "node:fs/promises";
+import { access, appendFile, mkdir, readFile, rename, writeFile } from "node:fs/promises";
 import { dirname } from "node:path";
+import { randomUUID } from "node:crypto";
+import { EngineError } from "../domain/errors.js";
+import { withDestinationLock } from "./lock.js";
 export async function writeJson(path, rows, append = false) {
     if (rows.length === 0)
         return;
-    let output = [...rows];
-    if (append && (await exists(path))) {
-        try {
-            const current = JSON.parse(await readFile(path, "utf8"));
-            if (Array.isArray(current))
-                output = [...current, ...rows];
+    await withDestinationLock(path, async () => {
+        let output = [...rows];
+        if (append && (await exists(path))) {
+            let current;
+            try {
+                current = JSON.parse(await readFile(path, "utf8"));
+            }
+            catch {
+                throw new EngineError(`Refusing to append to malformed JSON file: ${path}`);
+            }
+            if (!Array.isArray(current))
+                throw new EngineError(`Refusing to append to a JSON file that is not an array: ${path}`);
+            output = [...current, ...rows];
         }
-        catch {
-            output = [...rows];
+        await atomicWrite(path, `${JSON.stringify(output, null, 2)}\n`);
+    });
+}
+export async function writeNdjson(path, rows, append = false) {
+    if (rows.length === 0)
+        return;
+    await withDestinationLock(path, async () => {
+        const chunk = `${rows.map((row) => JSON.stringify(row)).join("\n")}\n`;
+        if (!append || !(await exists(path))) {
+            await atomicWrite(path, chunk);
+            return;
         }
-    }
+        await mkdir(dirname(path), { recursive: true });
+        await appendFile(path, chunk, "utf8");
+    });
+}
+async function atomicWrite(path, content) {
     await mkdir(dirname(path), { recursive: true });
-    await writeFile(path, `${JSON.stringify(output, null, 2)}\n`, "utf8");
+    const temporary = `${path}.${randomUUID()}.tmp`;
+    await writeFile(temporary, content, "utf8");
+    await rename(temporary, path);
 }
 async function exists(path) {
     try {