xtrawl 0.1.0

package/dist/client/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/client/client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAUzD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAsB,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAA0B,MAAM,iBAAiB,CAAC;AAC/F,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,OAAO,MAAM;IACD,MAAM,CAAe;IACrB,OAAO,CAAgB;IACvB,EAAE,CAAiB;IAClB,IAAI,CAAc;IAClB,MAAM,CAAY;IAEnC,YAAmB,UAAyB,EAAE;QAC5C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAC7C,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;YAClD,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;YAC9C,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;SACnC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACxB,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QACnG,MAAM,SAAS,GAAG,IAAI,gBAAgB,CACpC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,SAAS,EACT,eAAe,EAAE,SAAS,CAC3B,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,qBAAqB,CAAC,OAAO,CAAC,mBAAmB,EAAE;YAC1E,OAAO,EACL,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;YACvG,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;SACtC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;YACpC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;YACjC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1F,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvE,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YAC1F,IAAI,CAAC,OAAO,CAAC,SAAS;gBAAE,OAAO,KAAK,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,6BAA6B,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACjG,IAAI,CAAC,OAAO,EAAE,GAAG;gBAAE,OAAO,KAAK,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC3B,GAAG,OAAO;gBACV,MAAM,EAAE,CAAC;gBACT,cAAc,EAAE,CAAC;gBACjB,SAAS,EAAE,OAAO,CAAC,GAAG;gBACtB,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE;aAC5C,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;IAC1F,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAyB,EAAE;QACpD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,MAAM,CAAC,wBAAwB,EAAE,CAAC;QACxC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,EAAE,EAAE,UAAyB,EAAE;QACzD,OAAO,aAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACjE,CAAC;IAEM,KAAK,CAAC,WAAW,CACtB,OAA0C,EAC1C,UAA2B,EAAE;QAE7B,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;QACrD,OAAO,eAAe,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxE,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAAc;QAClC,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,WAAW,CAAC,yDAAyD,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChG,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAC3B,OAA0C,EAC1C,UAAmD,EAAE;QAErD,OAAO,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpG,CAAC;IAEM,KAAK,CAAC,YAAY,CACvB,OAA0C,EAC1C,UAA0D,EAAE;QAE5D,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;QACrD,OAAO,cAAc,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,UAAU,EAAE;YAC1D,OAAO,EAAE,UAAU;YACnB,GAAG,OAAO;YACV,UAAU,EAAE,WAAW;SACxB,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,YAAY,CACvB,OAA0C,EAC1C,UAA0D,EAAE;QAE5D,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;QACrD,OAAO,cAAc,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,UAAU,EAAE;YAC1D,OAAO,EAAE,UAAU;YACnB,GAAG,OAAO;YACV,UAAU,EAAE,WAAW;SACxB,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAC/B,OAA0C,EAC1C,UAA0D,EAAE;QAE5D,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC;QACrD,OAAO,cAAc,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,UAAU,EAAE;YAC1D,OAAO,EAAE,UAAU;YACnB,GAAG,OAAO;YACV,UAAU,EAAE,oBAAoB;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,OAAO;QACZ,OAAO;YACL,MAAM,EAAE;gBACN,GAAG,IAAI,CAAC,MAAM;gBACd,WAAW,EAAE,YAAY;gBACzB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE;YACD,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;IAC3B,CAAC;IAEM,KAAK;QACV,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IAEO,iBAAiB;QACvB,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;IAC9F,CAAC;IAEO,SAAS,CAAC,OAAsB;QACtC,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK;YAAE,OAAO;QACxC,MAAM,OAAO,GAAoB,EAAE,CAAC;QACpC,IAAI,OAAO,CAAC,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAClF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACxF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;QAChE,IAAI,SAAS;YACX,OAAO,CAAC,IAAI,CACV,oBAAoB,CAAC;gBACnB,SAAS;gBACT,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpC,CAAC,CACH,CAAC;QACJ,IAAI,OAAO,CAAC,WAAW;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;QACpF,IAAI,OAAO,CAAC,YAAY;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACtF,IAAI,OAAO,CAAC,OAAO;YACjB,OAAO,CAAC,IAAI,CAAC,GAAG,8BAA8B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC7F,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACnD,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS;gBAAE,SAAS;YACtD,MAAM,OAAO,GAAG,MAAM,6BAA6B,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACjG,IAAI,OAAO,EAAE,GAAG;gBACd,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC3B,GAAG,OAAO;oBACV,SAAS,EAAE,OAAO,CAAC,GAAG;oBACtB,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE;iBAC5C,CAAC,CAAC;QACP,CAAC;IACH,CAAC;CACF;AAED,SAAS,WAAW,CAAC,KAA6B;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAC3B,OAAO;YACL,GAAG,KAAK;YACR,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC;IACJ,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC;QAClD,IAAI,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC;QAClD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,YAAY,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IACzC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CACxB,mFAAmF,CACpF,CAAC;IACF,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC"}

package/dist/client/collectors.d.ts

@@ -0,0 +1,15 @@
+import type { ClientConfig } from "../config/types.js";
+import type { FollowRecord, SearchResult } from "../domain/records.js";
+import type { FollowsRequest, ProfileTimelineRequest, TargetInput } from "../domain/requests.js";
+import type { ApiEngine } from "../engine/api-engine.js";
+import type { AccountPool } from "../pool/account-pool.js";
+import type { StorageBundle } from "../storage/index.js";
+export interface CollectionContext {
+    readonly config: ClientConfig;
+    readonly pool: AccountPool;
+    readonly engine: ApiEngine;
+    readonly storage: StorageBundle;
+}
+export declare function collectProfileTweets(context: CollectionContext, targets: readonly TargetInput[], options: Omit<ProfileTimelineRequest, "targets">): Promise<SearchResult>;
+export declare function collectFollows(context: CollectionContext, targets: readonly TargetInput[], options: FollowsRequest): Promise<readonly FollowRecord[]>;
+//# sourceMappingURL=collectors.d.ts.map

package/dist/client/collectors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collectors.d.ts","sourceRoot":"","sources":["../../src/client/collectors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAe,MAAM,sBAAsB,CAAC;AACpF,OAAO,KAAK,EAAE,cAAc,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEjG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAI3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIzD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;CACjC;AAED,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,CAAC,GAC/C,OAAO,CAAC,YAAY,CAAC,CAmGvB;AAED,wBAAsB,cAAc,CAClC,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,SAAS,YAAY,EAAE,CAAC,CA0ElC"}

package/dist/client/collectors.js

@@ -0,0 +1,185 @@
+import { mapFollow } from "../engine/extractors.js";
+import { RunFailed } from "../domain/errors.js";
+import { queryHash } from "../query/hash.js";
+import { ExecutionRunner } from "../runner/runner.js";
+import { saveRows } from "../output/writer.js";
+import { targetOutputName } from "../output/names.js";
+export async function collectProfileTweets(context, targets, options) {
+    const request = { targets, ...options };
+    const out = [];
+    const seenByTarget = new Map();
+    const cursors = new Map();
+    const counts = new Map();
+    let limitReached = false;
+    let poolRetries = 0;
+    const runner = new ExecutionRunner({
+        concurrency: workerCount(context, targets.length),
+        maxAttempts: 1,
+    });
+    const outcome = await runner.run(targets, async (target) => {
+        const resolved = await resolveTarget(context, target, () => {
+            poolRetries += 1;
+        }, options.maxAccountSwitches);
+        const checkpointHash = queryHash({ operation: "profile_tweets", target, options });
+        const seen = seenByTarget.get(checkpointHash) ?? new Set();
+        seenByTarget.set(checkpointHash, seen);
+        const saved = options.resume ? context.storage.checkpoints.get(checkpointHash) : undefined;
+        let cursor = cursors.get(checkpointHash) ??
+            options.initialCursors?.[targetKey(target, resolved.userId)] ??
+            saved?.root;
+        let empty = 0;
+        let pages = 0;
+        let profileCount = counts.get(checkpointHash) ?? 0;
+        const maxPages = options.maxPagesPerProfile ?? Number.POSITIVE_INFINITY;
+        while (pages < maxPages && !limitReached) {
+            pages += 1;
+            const page = await context.pool.execute("profile-tweets", ({ session }) => context.engine.profilePage(session, resolved.userId, request, cursor), {
+                countTweets: (value) => value.tweets.length,
+                onRetry: () => {
+                    poolRetries += 1;
+                },
+                maxAccountSwitches: options.maxAccountSwitches,
+            });
+            const globalRemaining = options.limit === undefined ? page.tweets.length : Math.max(0, options.limit - out.length);
+            const profileRemaining = options.perProfileLimit === undefined
+                ? page.tweets.length
+                : Math.max(0, options.perProfileLimit - profileCount);
+            const remaining = page.tweets.slice(0, Math.min(globalRemaining, profileRemaining));
+            let added = 0;
+            for (const tweet of remaining) {
+                if (seen.has(tweet.tweetId))
+                    continue;
+                seen.add(tweet.tweetId);
+                out.push(tweet);
+                added += 1;
+                profileCount += 1;
+                counts.set(checkpointHash, profileCount);
+                if (options.limit !== undefined && out.length >= options.limit) {
+                    limitReached = true;
+                    break;
+                }
+            }
+            empty = added === 0 ? empty + 1 : 0;
+            if (options.resume && page.cursor)
+                context.storage.checkpoints.save(checkpointHash, { root: page.cursor });
+            if (page.cursor)
+                cursors.set(checkpointHash, page.cursor);
+            if (stopPaging(options, out.length, profileCount, page.cursor, empty, options.maxEmptyPages ?? context.config.maxEmptyPages))
+                break;
+            cursor = page.cursor;
+        }
+        if (!limitReached)
+            context.storage.checkpoints.clear(checkpointHash);
+    });
+    if (outcome.failed.length > 0 && (context.config.strict || out.length === 0))
+        throw failureFor("Profile timeline", outcome.failed);
+    const tweets = options.limit === undefined ? out : out.slice(0, options.limit);
+    await saveCollection(context, "profile_tweets", tweets, options, targets);
+    return {
+        tweets,
+        stats: {
+            tweetsCount: tweets.length,
+            tasksTotal: targets.length,
+            tasksDone: outcome.complete.length,
+            tasksFailed: outcome.failed.length,
+            retries: outcome.retries + poolRetries,
+        },
+    };
+}
+export async function collectFollows(context, targets, options) {
+    const out = [];
+    const seenByTarget = new Map();
+    const cursors = new Map();
+    let limitReached = false;
+    const runner = new ExecutionRunner({
+        concurrency: workerCount(context, targets.length),
+        maxAttempts: 1,
+    });
+    const outcome = await runner.run(targets, async (target) => {
+        const resolved = await resolveTarget(context, target, undefined, options.maxAccountSwitches);
+        const checkpointHash = queryHash({ operation: options.followType, target, options });
+        const saved = options.resume ? context.storage.checkpoints.get(checkpointHash) : undefined;
+        let cursor = cursors.get(checkpointHash) ??
+            options.initialCursors?.[targetKey(target, resolved.userId)] ??
+            saved?.root;
+        let empty = 0;
+        let pages = 0;
+        const seen = seenByTarget.get(checkpointHash) ?? new Set();
+        seenByTarget.set(checkpointHash, seen);
+        let targetCount = seen.size;
+        const maxPages = options.maxPagesPerProfile ?? Number.POSITIVE_INFINITY;
+        while (pages < maxPages && !limitReached) {
+            pages += 1;
+            const page = await context.pool.execute(options.followType, ({ session }) => context.engine.followsPage(session, resolved.userId, options.followType, cursor), { maxAccountSwitches: options.maxAccountSwitches });
+            let added = 0;
+            for (const user of page.users) {
+                const mapped = mapFollow(user, { ...target, userId: resolved.userId }, options.followType, options.rawJson);
+                const key = mapped.userId ?? mapped.username;
+                if (key && !seen.has(key) && withinProfileLimit(options.perProfileLimit, targetCount)) {
+                    seen.add(key);
+                    out.push(mapped);
+                    added += 1;
+                    targetCount += 1;
+                    if (options.limit !== undefined && out.length >= options.limit) {
+                        limitReached = true;
+                        break;
+                    }
+                }
+            }
+            empty = added === 0 ? empty + 1 : 0;
+            if (options.resume && page.cursor)
+                context.storage.checkpoints.save(checkpointHash, { root: page.cursor });
+            if (page.cursor)
+                cursors.set(checkpointHash, page.cursor);
+            if (stopPaging(options, out.length, targetCount, page.cursor, empty, options.maxEmptyPages ?? context.config.maxEmptyPages))
+                break;
+            cursor = page.cursor;
+        }
+        if (!limitReached)
+            context.storage.checkpoints.clear(checkpointHash);
+    });
+    if (outcome.failed.length > 0 && (context.config.strict || out.length === 0))
+        throw failureFor("Relationship collection", outcome.failed);
+    const follows = options.limit === undefined ? out : out.slice(0, options.limit);
+    await saveCollection(context, options.followType, follows, options, targets);
+    return follows;
+}
+async function saveCollection(context, name, rows, options, targets) {
+    if (!options.save)
+        return;
+    await saveRows(options.saveName ?? targetOutputName(name, targets), rows, {
+        directory: options.saveDir ?? context.config.saveDir,
+        format: options.saveFormat ?? context.config.saveFormat,
+        append: true,
+    });
+}
+async function resolveTarget(context, target, onRetry, maxAccountSwitches) {
+    if (target.userId && !target.username)
+        return { username: target.userId, userId: target.userId, raw: {} };
+    return context.pool.execute("user-lookup", ({ session }) => context.engine.resolveTarget(session, target), {
+        onRetry,
+        maxAccountSwitches,
+    });
+}
+function workerCount(context, tasks) {
+    return Math.max(1, Math.min(context.config.concurrency, context.pool.summary.eligible, tasks));
+}
+function failureFor(label, failed) {
+    const first = failed[0]?.error;
+    return first instanceof Error
+        ? first
+        : new RunFailed(`${label} failed for ${failed.length} target(s): ${String(first)}`);
+}
+function stopPaging(options, total, profileTotal, cursor, empty, maxEmptyPages) {
+    return Boolean((options.limit !== undefined && total >= options.limit) ||
+        (options.perProfileLimit !== undefined && profileTotal >= options.perProfileLimit) ||
+        !cursor ||
+        empty >= maxEmptyPages);
+}
+function withinProfileLimit(limit, count) {
+    return limit === undefined || count < limit;
+}
+function targetKey(target, resolvedUserId) {
+    return target.username ?? target.userId ?? target.profileUrl ?? target.raw ?? resolvedUserId;
+}
+//# sourceMappingURL=collectors.js.map

package/dist/client/collectors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collectors.js","sourceRoot":"","sources":["../../src/client/collectors.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAStD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAA0B,EAC1B,OAA+B,EAC/B,OAAgD;IAEhD,MAAM,OAAO,GAA2B,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IAChE,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAc;QAC9C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;QACjD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAClC,OAAO,EACP,MAAM,EACN,GAAG,EAAE;YACH,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC,EACD,OAAO,CAAC,kBAAkB,CAC3B,CAAC;QACF,MAAM,cAAc,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACnF,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QACnE,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3F,IAAI,MAAM,GACR,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC3B,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,EAAE,IAAI,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACxE,OAAO,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,gBAAgB,EAChB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,EACtF;gBACE,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;gBAC3C,OAAO,EAAE,GAAG,EAAE;oBACZ,WAAW,IAAI,CAAC,CAAC;gBACnB,CAAC;gBACD,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;aAC/C,CACF,CAAC;YACF,MAAM,eAAe,GACnB,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7F,MAAM,gBAAgB,GACpB,OAAO,CAAC,eAAe,KAAK,SAAS;gBACnC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;gBACpB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAAC;YAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;oBAAE,SAAS;gBACtC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAChB,KAAK,IAAI,CAAC,CAAC;gBACX,YAAY,IAAI,CAAC,CAAC;gBAClB,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;gBACzC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAC/D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM;gBACR,CAAC;YACH,CAAC;YACD,KAAK,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBAC/B,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,IACE,UAAU,CACR,OAAO,EACP,GAAG,CAAC,MAAM,EACV,YAAY,EACZ,IAAI,CAAC,MAAM,EACX,KAAK,EACL,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CACtD;gBAED,MAAM;YACR,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1E,MAAM,UAAU,CAAC,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/E,MAAM,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;YAClC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;YAClC,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,WAAW;SACvC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA0B,EAC1B,OAA+B,EAC/B,OAAuB;IAEvB,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAc;QAC9C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;QACjD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC7F,MAAM,cAAc,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3F,IAAI,MAAM,GACR,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC3B,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,EAAE,IAAI,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QACnE,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACxE,OAAO,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,OAAO,CAAC,UAAU,EAClB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,EACjG,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CACnD,CAAC;YACF,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,SAAS,CACtB,IAAI,EACJ,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,EACtC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,OAAO,CAChB,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC;gBAC7C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,kBAAkB,CAAC,OAAO,CAAC,eAAe,EAAE,WAAW,CAAC,EAAE,CAAC;oBACtF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACd,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACjB,KAAK,IAAI,CAAC,CAAC;oBACX,WAAW,IAAI,CAAC,CAAC;oBACjB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;wBAC/D,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YACD,KAAK,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBAC/B,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,IACE,UAAU,CACR,OAAO,EACP,GAAG,CAAC,MAAM,EACV,WAAW,EACX,IAAI,CAAC,MAAM,EACX,KAAK,EACL,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CACtD;gBAED,MAAM;YACR,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1E,MAAM,UAAU,CAAC,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChF,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAA0B,EAC1B,IAAY,EACZ,IAAwB,EACxB,OAAoB,EACpB,OAA+B;IAE/B,IAAI,CAAC,OAAO,CAAC,IAAI;QAAE,OAAO;IAC1B,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE;QACxE,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;QACpD,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;QACvD,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,OAA0B,EAC1B,MAAmB,EACnB,OAAoB,EACpB,kBAA2B;IAE3B,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAC1G,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;QACzG,OAAO;QACP,kBAAkB;KACnB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,OAA0B,EAAE,KAAa;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,MAA8C;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;IAC/B,OAAO,KAAK,YAAY,KAAK;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,eAAe,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,UAAU,CACjB,OAAuE,EACvE,KAAa,EACb,YAAoB,EACpB,MAA0B,EAC1B,KAAa,EACb,aAAqB;IAErB,OAAO,OAAO,CACZ,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;QACvD,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,IAAI,YAAY,IAAI,OAAO,CAAC,eAAe,CAAC;QAClF,CAAC,MAAM;QACP,KAAK,IAAI,aAAa,CACvB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB,EAAE,KAAa;IAClE,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK,CAAC;AAC9C,CAAC;AAED,SAAS,SAAS,CAAC,MAAmB,EAAE,cAAsB;IAC5D,OAAO,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,GAAG,IAAI,cAAc,CAAC;AAC/F,CAAC"}

package/dist/client/database.d.ts

@@ -0,0 +1,56 @@
+import type { AccountInput, ProxySettings } from "../domain/accounts.js";
+import type { StorageBundle } from "../storage/index.js";
+export interface AccountListOptions {
+    readonly eligibleOnly?: boolean;
+    readonly unusableOnly?: boolean;
+    readonly includeCookies?: boolean;
+    readonly revealSecrets?: boolean;
+}
+export interface AccountImportOptions {
+    readonly accountsFile?: string;
+    readonly cookiesFile?: string;
+    readonly envFile?: string;
+    readonly cookies?: unknown;
+    readonly accounts?: readonly AccountInput[];
+    readonly proxy?: string | ProxySettings;
+}
+export declare class XTrawlDatabase {
+    private readonly storage;
+    constructor(storage: StorageBundle);
+    accountsSummary(): import("../domain/accounts.js").AccountSummary;
+    listAccounts(options?: AccountListOptions): readonly Record<string, unknown>[];
+    getAccount(username: string, options?: Pick<AccountListOptions, "includeCookies" | "revealSecrets">): Record<string, unknown> | undefined;
+    deleteAccount(username: string): boolean;
+    setAccountProxy(username: string, proxy?: string | ProxySettings): boolean;
+    markAccountUnusable(username: string, reason?: string): boolean;
+    repairAccount(username: string, forceRefresh?: boolean): Promise<boolean>;
+    resetAccountCooldowns(usernames?: readonly string[], includeUnusable?: boolean): number;
+    clearLeases(expiredOnly?: boolean): number;
+    resetDailyCounters(usernames?: readonly string[]): number;
+    importAccounts(options: AccountImportOptions): {
+        readonly processed: number;
+        readonly eligible: number;
+    };
+    collapseDuplicateAccounts(dryRun?: boolean): {
+        removed: number;
+        merged: number;
+        dryRun: false;
+        wouldRemove?: undefined;
+    } | {
+        dryRun: true;
+        removed: number;
+        merged: number;
+        wouldRemove: number;
+    };
+    getCheckpoint(queryHash: string): Readonly<Record<string, string>> | undefined;
+    clearCheckpoint(queryHash: string): boolean;
+    clearAllCheckpoints(): number;
+    listRuns(limit?: number): import("../storage/run-repository.js").RunRecord[];
+    lastRun(): import("../storage/run-repository.js").RunRecord | undefined;
+    runsSummary(limit?: number): {
+        readonly totalRuns: number;
+        readonly byStatus: Readonly<Record<string, number>>;
+        readonly lastRun?: import("../storage/run-repository.js").RunRecord;
+    };
+}
+//# sourceMappingURL=database.d.ts.map

package/dist/client/database.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/client/database.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAiB,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACxF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAClC,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;CACzC;AAED,qBAAa,cAAc;IACN,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,aAAa;IAEnD,eAAe;IAIf,YAAY,CAAC,OAAO,GAAE,kBAAuB,GAAG,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;IAQlF,UAAU,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,GAAG,eAAe,CAAM,GACzE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAK/B,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,OAAO;IAI1E,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAW,GAAG,OAAO;IAI3D,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,UAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAmB7E,qBAAqB,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,eAAe,UAAQ,GAAG,MAAM;IAIrF,WAAW,CAAC,WAAW,UAAO,GAAG,MAAM;IAIvC,kBAAkB,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM;IAIzD,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG;QACpD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B;IAaM,yBAAyB,CAAC,MAAM,UAAO;;;;;;;;;;;IASvC,aAAa,CAAC,SAAS,EAAE,MAAM;IAI/B,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAI3C,mBAAmB,IAAI,MAAM;IAI7B,QAAQ,CAAC,KAAK,SAAK;IAInB,OAAO;IAIP,WAAW,CAAC,KAAK,SAAM;;;;;CAG/B"}

package/dist/client/database.js

@@ -0,0 +1,146 @@
+import { bootstrapCookiesFromAuthToken } from "../auth/bootstrap.js";
+import { loadAccountsFileSync, loadInlineAccounts } from "../auth/loaders.js";
+import { accountInputToRecord } from "../auth/records.js";
+import { loadAccountFromEnvironmentSync } from "../config/environment.js";
+import { tokenFingerprint } from "../utils/fingerprint.js";
+export class XTrawlDatabase {
+    storage;
+    constructor(storage) {
+        this.storage = storage;
+    }
+    accountsSummary() {
+        return this.storage.accounts.summary();
+    }
+    listAccounts(options = {}) {
+        return this.storage.accounts
+            .list()
+            .filter((account) => !options.eligibleOnly || this.storage.accounts.eligible(account))
+            .filter((account) => !options.unusableOnly || account.status === 0)
+            .map((account) => redactAccount(account, options));
+    }
+    getAccount(username, options = {}) {
+        const account = this.storage.accounts.findByUsername(username);
+        return account ? redactAccount(account, options) : undefined;
+    }
+    deleteAccount(username) {
+        return this.storage.accounts.delete(username);
+    }
+    setAccountProxy(username, proxy) {
+        return this.storage.accounts.setProxy(username, proxy);
+    }
+    markAccountUnusable(username, reason = "manual") {
+        return this.storage.accounts.markUnusable(username, 401, `unusable:${reason}`);
+    }
+    async repairAccount(username, forceRefresh = false) {
+        const account = this.storage.accounts.findByUsername(username);
+        if (!account?.authToken)
+            return false;
+        if (!forceRefresh && account.csrfToken) {
+            this.storage.accounts.resetCooldowns([username], true);
+            return true;
+        }
+        const cookies = await bootstrapCookiesFromAuthToken(account.authToken, undefined, account.proxy);
+        if (!cookies?.ct0)
+            return false;
+        this.storage.accounts.upsert({
+            ...account,
+            status: 1,
+            availableUntil: 0,
+            csrfToken: cookies.ct0,
+            cookies: { ...account.cookies, ...cookies },
+        });
+        return true;
+    }
+    resetAccountCooldowns(usernames, includeUnusable = false) {
+        return this.storage.accounts.resetCooldowns(usernames, includeUnusable);
+    }
+    clearLeases(expiredOnly = true) {
+        return this.storage.accounts.clearLeases(expiredOnly);
+    }
+    resetDailyCounters(usernames) {
+        return this.storage.accounts.resetDailyUsage(usernames);
+    }
+    importAccounts(options) {
+        const accounts = [];
+        if (options.accounts)
+            accounts.push(...options.accounts.map(accountInputToRecord));
+        if (options.cookies !== undefined)
+            accounts.push(...loadInlineAccounts(options.cookies));
+        if (options.accountsFile)
+            accounts.push(...loadAccountsFileSync(options.accountsFile));
+        if (options.cookiesFile)
+            accounts.push(...loadAccountsFileSync(options.cookiesFile));
+        if (options.envFile)
+            accounts.push(...loadAccountFromEnvironmentSync(options.envFile).map(accountInputToRecord));
+        for (const account of accounts)
+            this.storage.accounts.upsert({ ...account, proxy: account.proxy ?? options.proxy });
+        return { processed: accounts.length, eligible: this.storage.accounts.summary().eligible };
+    }
+    collapseDuplicateAccounts(dryRun = true) {
+        if (!dryRun)
+            return { dryRun, ...this.storage.accounts.collapseDuplicatesByAuthToken() };
+        const tokens = this.storage.accounts
+            .list()
+            .flatMap((account) => (account.authToken ? [account.authToken] : []));
+        const duplicates = tokens.length - new Set(tokens).size;
+        return { dryRun, removed: 0, merged: 0, wouldRemove: duplicates };
+    }
+    getCheckpoint(queryHash) {
+        return this.storage.checkpoints.get(queryHash);
+    }
+    clearCheckpoint(queryHash) {
+        return this.storage.checkpoints.clear(queryHash);
+    }
+    clearAllCheckpoints() {
+        return this.storage.checkpoints.clearAll();
+    }
+    listRuns(limit = 50) {
+        return this.storage.runs.list(limit);
+    }
+    lastRun() {
+        return this.storage.runs.last();
+    }
+    runsSummary(limit = 500) {
+        return this.storage.runs.summary(limit);
+    }
+}
+function redactAccount(account, options) {
+    const safe = {
+        ...account,
+        authToken: account.authToken ? tokenFingerprint(account.authToken) : undefined,
+        csrfToken: account.csrfToken ? "[redacted]" : undefined,
+        bearerToken: account.bearerToken ? "[redacted]" : undefined,
+        password: account.password ? "[redacted]" : undefined,
+        email: account.email ? "[redacted]" : undefined,
+        emailPassword: account.emailPassword ? "[redacted]" : undefined,
+        twoFactorSecret: account.twoFactorSecret ? "[redacted]" : undefined,
+        proxy: account.proxy ? redactProxy(account.proxy) : undefined,
+        cookies: options.includeCookies ? redactCookies(account.cookies) : undefined,
+    };
+    if (options.revealSecrets)
+        return { ...account, cookies: options.includeCookies ? account.cookies : undefined };
+    return safe;
+}
+function redactCookies(cookies) {
+    return Object.fromEntries(Object.keys(cookies).map((name) => [name, "[redacted]"]));
+}
+function redactProxy(proxy) {
+    if (typeof proxy !== "string")
+        return {
+            ...proxy,
+            ...(proxy.username ? { username: "[redacted]" } : {}),
+            ...(proxy.password ? { password: "[redacted]" } : {}),
+        };
+    try {
+        const value = new URL(proxy.includes("://") ? proxy : `http://${proxy}`);
+        if (value.username)
+            value.username = "[redacted]";
+        if (value.password)
+            value.password = "[redacted]";
+        return value.toString();
+    }
+    catch {
+        return "[redacted]";
+    }
+}
+//# sourceMappingURL=database.js.map

package/dist/client/database.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/client/database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAG1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAkB3D,MAAM,OAAO,cAAc;IACW;IAApC,YAAoC,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAEvD,eAAe;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;IAEM,YAAY,CAAC,UAA8B,EAAE;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ;aACzB,IAAI,EAAE;aACN,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACrF,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;aAClE,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAEM,UAAU,CACf,QAAgB,EAChB,UAAwE,EAAE;QAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/D,CAAC;IAEM,aAAa,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAEM,eAAe,CAAC,QAAgB,EAAE,KAA8B;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzD,CAAC;IAEM,mBAAmB,CAAC,QAAgB,EAAE,MAAM,GAAG,QAAQ;QAC5D,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,YAAY,GAAG,KAAK;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO,EAAE,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,6BAA6B,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjG,IAAI,CAAC,OAAO,EAAE,GAAG;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3B,GAAG,OAAO;YACV,MAAM,EAAE,CAAC;YACT,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,GAAG;YACtB,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE;SAC5C,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,qBAAqB,CAAC,SAA6B,EAAE,eAAe,GAAG,KAAK;QACjF,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC1E,CAAC;IAEM,WAAW,CAAC,WAAW,GAAG,IAAI;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC;IAEM,kBAAkB,CAAC,SAA6B;QACrD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAC1D,CAAC;IAEM,cAAc,CAAC,OAA6B;QAIjD,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACnF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACzF,IAAI,OAAO,CAAC,YAAY;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,WAAW;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;QACrF,IAAI,OAAO,CAAC,OAAO;YACjB,QAAQ,CAAC,IAAI,CAAC,GAAG,8BAA8B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC9F,KAAK,MAAM,OAAO,IAAI,QAAQ;YAC5B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACtF,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC5F,CAAC;IAEM,yBAAyB,CAAC,MAAM,GAAG,IAAI;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,EAAE,EAAE,CAAC;QACzF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ;aACjC,IAAI,EAAE;aACN,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;QACxD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;IACpE,CAAC;IAEM,aAAa,CAAC,SAAiB;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAEM,eAAe,CAAC,SAAiB;QACtC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACnD,CAAC;IAEM,mBAAmB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;IAC7C,CAAC;IAEM,QAAQ,CAAC,KAAK,GAAG,EAAE;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IAEM,WAAW,CAAC,KAAK,GAAG,GAAG;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,SAAS,aAAa,CACpB,OAAsB,EACtB,OAAqE;IAErE,MAAM,IAAI,GAA4B;QACpC,GAAG,OAAO;QACV,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QAC9E,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACvD,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACrD,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QAC/C,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QAC/D,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACnE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QAC7D,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAC7E,CAAC;IACF,IAAI,OAAO,CAAC,aAAa;QACvB,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACvF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,OAAyC;IAC9D,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,WAAW,CAAC,KAA6B;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAC3B,OAAO;YACL,GAAG,KAAK;YACR,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC;IACJ,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC;QAClD,IAAI,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC;QAClD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,YAAY,CAAC;IACtB,CAAC;AACH,CAAC"}

package/dist/client/profiles.d.ts

@@ -0,0 +1,5 @@
+import type { ProfileRecord } from "../domain/records.js";
+import type { TargetInput, UserInfoRequest } from "../domain/requests.js";
+import type { CollectionContext } from "./collectors.js";
+export declare function collectProfiles(context: CollectionContext, targets: readonly TargetInput[], options: UserInfoRequest): Promise<readonly ProfileRecord[]>;
+//# sourceMappingURL=profiles.d.ts.map

package/dist/client/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../src/client/profiles.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAO1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,wBAAsB,eAAe,CACnC,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC,CAgCnC"}

package/dist/client/profiles.js

@@ -0,0 +1,36 @@
+import { RunFailed } from "../domain/errors.js";
+import { mapProfile } from "../engine/extractors.js";
+import { saveRows } from "../output/writer.js";
+import { targetOutputName } from "../output/names.js";
+import { targetUsername } from "../query/builder.js";
+import { ExecutionRunner } from "../runner/runner.js";
+export async function collectProfiles(context, targets, options) {
+    const records = new Map();
+    const tasks = targets.flatMap((target, index) => (targetUsername(target) ? [{ index, target }] : []));
+    const runner = new ExecutionRunner({
+        concurrency: Math.max(1, Math.min(context.config.concurrency, context.pool.summary.eligible, tasks.length)),
+        maxAttempts: 1,
+    });
+    const outcome = await runner.run(tasks, async ({ index, target }) => {
+        const username = targetUsername(target);
+        if (!username)
+            return;
+        const user = await context.pool.execute("user-info", ({ session }) => context.engine.lookupUser(session, username));
+        records.set(index, mapProfile(user, target, username));
+    });
+    if (outcome.failed.length > 0 && (context.config.strict || records.size === 0)) {
+        const first = outcome.failed[0]?.error;
+        throw first instanceof RunFailed
+            ? first
+            : new RunFailed(`Profile lookup failed for ${outcome.failed.length} target(s): ${String(first)}`);
+    }
+    const result = [...records.entries()].sort(([left], [right]) => left - right).map(([, value]) => value);
+    if (options.save)
+        await saveRows(options.saveName ?? targetOutputName("user_info", targets), result, {
+            directory: options.saveDir ?? context.config.saveDir,
+            format: options.saveFormat ?? context.config.saveFormat,
+            append: true,
+        });
+    return result;
+}
+//# sourceMappingURL=profiles.js.map

package/dist/client/profiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../src/client/profiles.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAA0B,EAC1B,OAA+B,EAC/B,OAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtG,MAAM,MAAM,GAAG,IAAI,eAAe,CAAyB;QACzD,WAAW,EAAE,IAAI,CAAC,GAAG,CACnB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAClF;QACD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAClE,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CACnE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAC7C,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;QACvC,MAAM,KAAK,YAAY,SAAS;YAC9B,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,IAAI,SAAS,CAAC,6BAA6B,OAAO,CAAC,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IACxG,IAAI,OAAO,CAAC,IAAI;QACd,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE;YACjF,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;YACpD,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;YACvD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/client/search.d.ts

@@ -0,0 +1,21 @@
+import type { ClientConfig } from "../config/types.js";
+import type { SearchResult } from "../domain/records.js";
+import type { SearchRequest } from "../domain/requests.js";
+import type { ApiEngine } from "../engine/api-engine.js";
+import type { AccountPool } from "../pool/account-pool.js";
+import type { StorageBundle } from "../storage/index.js";
+export interface SearchContext {
+    readonly config: ClientConfig;
+    readonly pool: AccountPool;
+    readonly engine: ApiEngine;
+    readonly storage: StorageBundle;
+}
+interface SearchTask {
+    readonly id: string;
+    readonly request: SearchRequest;
+}
+export declare function collectSearch(context: SearchContext, query: string, options: SearchRequest): Promise<SearchResult>;
+export declare function withDefaultBounds(request: SearchRequest, now?: Date): SearchRequest;
+export declare function splitSearchTasks(request: SearchRequest, requestedSplits: number, minIntervalMs: number): readonly SearchTask[];
+export {};
+//# sourceMappingURL=search.d.ts.map

package/dist/client/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/client/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,OAAO,KAAK,EAAE,YAAY,EAAe,MAAM,sBAAsB,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAG3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIzD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;CACjC;AAED,UAAU,UAAU;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;CACjC;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAiGvB;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,OAAa,GAAG,aAAa,CAIzF;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,EACtB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,GACpB,SAAS,UAAU,EAAE,CAavB"}