xtrawl 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DOCUMENTATION.md +51 -0
- package/README.md +34 -6
- package/dist/auth/bootstrap.d.ts +1 -1
- package/dist/auth/bootstrap.d.ts.map +1 -1
- package/dist/auth/bootstrap.js +27 -14
- package/dist/auth/bootstrap.js.map +1 -1
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +13 -12
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/parser.d.ts +2 -14
- package/dist/cli/parser.d.ts.map +1 -1
- package/dist/cli/parser.js +26 -30
- package/dist/cli/parser.js.map +1 -1
- package/dist/cli/types.d.ts +17 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +2 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/client/accounts.d.ts.map +1 -1
- package/dist/client/accounts.js +13 -6
- package/dist/client/accounts.js.map +1 -1
- package/dist/client/client.d.ts +16 -3
- package/dist/client/client.d.ts.map +1 -1
- package/dist/client/client.js +96 -38
- package/dist/client/client.js.map +1 -1
- package/dist/client/collectors.d.ts +2 -10
- package/dist/client/collectors.d.ts.map +1 -1
- package/dist/client/collectors.js +122 -57
- package/dist/client/collectors.js.map +1 -1
- package/dist/client/database.d.ts +6 -18
- package/dist/client/database.d.ts.map +1 -1
- package/dist/client/database.js +4 -21
- package/dist/client/database.js.map +1 -1
- package/dist/client/page-commit.d.ts +15 -0
- package/dist/client/page-commit.d.ts.map +1 -0
- package/dist/client/page-commit.js +19 -0
- package/dist/client/page-commit.js.map +1 -0
- package/dist/client/profiles.d.ts +1 -1
- package/dist/client/profiles.d.ts.map +1 -1
- package/dist/client/profiles.js +10 -3
- package/dist/client/profiles.js.map +1 -1
- package/dist/client/search.d.ts +5 -17
- package/dist/client/search.d.ts.map +1 -1
- package/dist/client/search.js +81 -26
- package/dist/client/search.js.map +1 -1
- package/dist/client/types.d.ts +40 -2
- package/dist/client/types.d.ts.map +1 -1
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +5 -2
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/types.d.ts +5 -2
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/validation.d.ts +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +27 -11
- package/dist/config/validation.js.map +1 -1
- package/dist/constants/accounts.d.ts +16 -0
- package/dist/constants/accounts.d.ts.map +1 -0
- package/dist/constants/accounts.js +16 -0
- package/dist/constants/accounts.js.map +1 -0
- package/dist/constants/cli.d.ts +10 -0
- package/dist/constants/cli.d.ts.map +1 -0
- package/dist/constants/cli.js +10 -0
- package/dist/constants/cli.js.map +1 -0
- package/dist/constants/collection.d.ts +9 -0
- package/dist/constants/collection.d.ts.map +1 -0
- package/dist/constants/collection.js +9 -0
- package/dist/constants/collection.js.map +1 -0
- package/dist/constants/config.d.ts +6 -0
- package/dist/constants/config.d.ts.map +1 -0
- package/dist/constants/config.js +6 -0
- package/dist/constants/config.js.map +1 -0
- package/dist/constants/errors.d.ts +9 -0
- package/dist/constants/errors.d.ts.map +1 -0
- package/dist/constants/errors.js +9 -0
- package/dist/constants/errors.js.map +1 -0
- package/dist/constants/output.d.ts +7 -0
- package/dist/constants/output.d.ts.map +1 -0
- package/dist/constants/output.js +7 -0
- package/dist/constants/output.js.map +1 -0
- package/dist/constants/pages.d.ts +5 -0
- package/dist/constants/pages.d.ts.map +1 -0
- package/dist/constants/pages.js +5 -0
- package/dist/constants/pages.js.map +1 -0
- package/dist/constants/requests.d.ts +18 -0
- package/dist/constants/requests.d.ts.map +1 -0
- package/dist/constants/requests.js +18 -0
- package/dist/constants/requests.js.map +1 -0
- package/dist/constants/runner.d.ts +8 -0
- package/dist/constants/runner.d.ts.map +1 -0
- package/dist/constants/runner.js +8 -0
- package/dist/constants/runner.js.map +1 -0
- package/dist/constants/runs.d.ts +8 -0
- package/dist/constants/runs.d.ts.map +1 -0
- package/dist/constants/runs.js +8 -0
- package/dist/constants/runs.js.map +1 -0
- package/dist/domain/accounts.d.ts +4 -2
- package/dist/domain/accounts.d.ts.map +1 -1
- package/dist/domain/collection.d.ts +27 -0
- package/dist/domain/collection.d.ts.map +1 -0
- package/dist/domain/collection.js +2 -0
- package/dist/domain/collection.js.map +1 -0
- package/dist/domain/diagnostics.d.ts +11 -0
- package/dist/domain/diagnostics.d.ts.map +1 -0
- package/dist/domain/diagnostics.js +2 -0
- package/dist/domain/diagnostics.js.map +1 -0
- package/dist/domain/errors.d.ts +2 -1
- package/dist/domain/errors.d.ts.map +1 -1
- package/dist/domain/errors.js +10 -9
- package/dist/domain/errors.js.map +1 -1
- package/dist/domain/http.d.ts +7 -1
- package/dist/domain/http.d.ts.map +1 -1
- package/dist/domain/manifest.d.ts +8 -0
- package/dist/domain/manifest.d.ts.map +1 -1
- package/dist/domain/output.d.ts +3 -0
- package/dist/domain/output.d.ts.map +1 -0
- package/dist/domain/output.js +2 -0
- package/dist/domain/output.js.map +1 -0
- package/dist/domain/pages.d.ts +20 -0
- package/dist/domain/pages.d.ts.map +1 -0
- package/dist/domain/pages.js +2 -0
- package/dist/domain/pages.js.map +1 -0
- package/dist/domain/pool.d.ts +22 -0
- package/dist/domain/pool.d.ts.map +1 -0
- package/dist/domain/pool.js +2 -0
- package/dist/domain/pool.js.map +1 -0
- package/dist/domain/records.d.ts +6 -1
- package/dist/domain/records.d.ts.map +1 -1
- package/dist/domain/requests.d.ts +19 -10
- package/dist/domain/requests.d.ts.map +1 -1
- package/dist/domain/runner.d.ts +30 -0
- package/dist/domain/runner.d.ts.map +1 -0
- package/dist/domain/runner.js +2 -0
- package/dist/domain/runner.js.map +1 -0
- package/dist/domain/runs.d.ts +12 -0
- package/dist/domain/runs.d.ts.map +1 -0
- package/dist/domain/runs.js +2 -0
- package/dist/domain/runs.js.map +1 -0
- package/dist/engine/api-engine.d.ts +7 -7
- package/dist/engine/api-engine.d.ts.map +1 -1
- package/dist/engine/api-engine.js +42 -19
- package/dist/engine/api-engine.js.map +1 -1
- package/dist/engine/extractors.d.ts +2 -8
- package/dist/engine/extractors.d.ts.map +1 -1
- package/dist/engine/extractors.js +30 -11
- package/dist/engine/extractors.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/manifest/destinations.d.ts +4 -0
- package/dist/manifest/destinations.d.ts.map +1 -0
- package/dist/manifest/destinations.js +47 -0
- package/dist/manifest/destinations.js.map +1 -0
- package/dist/manifest/model.d.ts +3 -2
- package/dist/manifest/model.d.ts.map +1 -1
- package/dist/manifest/model.js +6 -1
- package/dist/manifest/model.js.map +1 -1
- package/dist/manifest/provider.d.ts +10 -3
- package/dist/manifest/provider.d.ts.map +1 -1
- package/dist/manifest/provider.js +86 -34
- package/dist/manifest/provider.js.map +1 -1
- package/dist/manifest/scraper.d.ts +2 -6
- package/dist/manifest/scraper.d.ts.map +1 -1
- package/dist/manifest/scraper.js +46 -15
- package/dist/manifest/scraper.js.map +1 -1
- package/dist/output/csv-writer.d.ts.map +1 -1
- package/dist/output/csv-writer.js +32 -10
- package/dist/output/csv-writer.js.map +1 -1
- package/dist/output/json-writer.d.ts +1 -0
- package/dist/output/json-writer.d.ts.map +1 -1
- package/dist/output/json-writer.js +36 -11
- package/dist/output/json-writer.js.map +1 -1
- package/dist/output/lock.d.ts +2 -0
- package/dist/output/lock.d.ts.map +1 -0
- package/dist/output/lock.js +18 -0
- package/dist/output/lock.js.map +1 -0
- package/dist/output/writer.d.ts +2 -1
- package/dist/output/writer.d.ts.map +1 -1
- package/dist/output/writer.js +6 -3
- package/dist/output/writer.js.map +1 -1
- package/dist/pool/account-pool.d.ts +3 -12
- package/dist/pool/account-pool.d.ts.map +1 -1
- package/dist/pool/account-pool.js +54 -16
- package/dist/pool/account-pool.js.map +1 -1
- package/dist/pool/cooldown.d.ts +3 -7
- package/dist/pool/cooldown.d.ts.map +1 -1
- package/dist/pool/cooldown.js +10 -7
- package/dist/pool/cooldown.js.map +1 -1
- package/dist/pool/limiter.d.ts +3 -3
- package/dist/pool/limiter.d.ts.map +1 -1
- package/dist/pool/limiter.js +5 -6
- package/dist/pool/limiter.js.map +1 -1
- package/dist/query/builder.d.ts +1 -1
- package/dist/query/builder.d.ts.map +1 -1
- package/dist/query/builder.js +10 -4
- package/dist/query/builder.js.map +1 -1
- package/dist/query/collection-id.d.ts +3 -0
- package/dist/query/collection-id.d.ts.map +1 -0
- package/dist/query/collection-id.js +21 -0
- package/dist/query/collection-id.js.map +1 -0
- package/dist/query/normalize.d.ts +3 -32
- package/dist/query/normalize.d.ts.map +1 -1
- package/dist/query/normalize.js +10 -16
- package/dist/query/normalize.js.map +1 -1
- package/dist/query/targets.d.ts +2 -7
- package/dist/query/targets.d.ts.map +1 -1
- package/dist/query/targets.js.map +1 -1
- package/dist/query/types.d.ts +40 -0
- package/dist/query/types.d.ts.map +1 -0
- package/dist/query/types.js +2 -0
- package/dist/query/types.js.map +1 -0
- package/dist/runner/index.d.ts +3 -0
- package/dist/runner/index.d.ts.map +1 -1
- package/dist/runner/index.js.map +1 -1
- package/dist/runner/retry.d.ts +3 -6
- package/dist/runner/retry.d.ts.map +1 -1
- package/dist/runner/retry.js +5 -3
- package/dist/runner/retry.js.map +1 -1
- package/dist/runner/runner.d.ts +2 -14
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +11 -7
- package/dist/runner/runner.js.map +1 -1
- package/dist/runner/task-queue.d.ts +9 -11
- package/dist/runner/task-queue.d.ts.map +1 -1
- package/dist/runner/task-queue.js +79 -32
- package/dist/runner/task-queue.js.map +1 -1
- package/dist/storage/account-repository.d.ts +2 -15
- package/dist/storage/account-repository.d.ts.map +1 -1
- package/dist/storage/account-repository.js +65 -35
- package/dist/storage/account-repository.js.map +1 -1
- package/dist/storage/database.d.ts +3 -1
- package/dist/storage/database.d.ts.map +1 -1
- package/dist/storage/database.js +10 -4
- package/dist/storage/database.js.map +1 -1
- package/dist/storage/index.d.ts +3 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/progress-repository.d.ts +12 -0
- package/dist/storage/progress-repository.d.ts.map +1 -0
- package/dist/storage/progress-repository.js +76 -0
- package/dist/storage/progress-repository.js.map +1 -0
- package/dist/storage/run-repository.d.ts +4 -10
- package/dist/storage/run-repository.d.ts.map +1 -1
- package/dist/storage/run-repository.js +4 -2
- package/dist/storage/run-repository.js.map +1 -1
- package/dist/storage/schema.d.ts.map +1 -1
- package/dist/storage/schema.js +19 -0
- package/dist/storage/schema.js.map +1 -1
- package/dist/storage/types.d.ts +15 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +2 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/transport/body.d.ts +15 -0
- package/dist/transport/body.d.ts.map +1 -0
- package/dist/transport/body.js +61 -0
- package/dist/transport/body.js.map +1 -0
- package/dist/transport/dispatcher-pool.d.ts +13 -0
- package/dist/transport/dispatcher-pool.d.ts.map +1 -0
- package/dist/transport/dispatcher-pool.js +47 -0
- package/dist/transport/dispatcher-pool.js.map +1 -0
- package/dist/transport/graphql.d.ts +1 -1
- package/dist/transport/graphql.d.ts.map +1 -1
- package/dist/transport/graphql.js +36 -19
- package/dist/transport/graphql.js.map +1 -1
- package/dist/transport/session.d.ts +8 -11
- package/dist/transport/session.d.ts.map +1 -1
- package/dist/transport/session.js +71 -26
- package/dist/transport/session.js.map +1 -1
- package/dist/transport/transaction-id.d.ts +2 -7
- package/dist/transport/transaction-id.d.ts.map +1 -1
- package/dist/transport/transaction-id.js +20 -16
- package/dist/transport/transaction-id.js.map +1 -1
- package/dist/transport/types.d.ts +28 -0
- package/dist/transport/types.d.ts.map +1 -0
- package/dist/transport/types.js +2 -0
- package/dist/transport/types.js.map +1 -0
- package/dist/utils/abort.d.ts +6 -0
- package/dist/utils/abort.d.ts.map +1 -0
- package/dist/utils/abort.js +45 -0
- package/dist/utils/abort.js.map +1 -0
- package/dist/utils/redact.d.ts +5 -0
- package/dist/utils/redact.d.ts.map +1 -0
- package/dist/utils/redact.js +42 -0
- package/dist/utils/redact.js.map +1 -0
- package/docs/architecture/overview.md +8 -4
- package/docs/product/specification.md +18 -8
- package/docs/security/data-boundary.md +3 -0
- package/package.json +5 -4
|
@@ -1,11 +1,16 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { PROGRESS_STATE } from "../constants/collection.js";
|
|
1
3
|
import { mapFollow } from "../engine/extractors.js";
|
|
2
4
|
import { RunFailed } from "../domain/errors.js";
|
|
3
|
-
import {
|
|
5
|
+
import { collectionIdentity } from "../query/collection-id.js";
|
|
4
6
|
import { ExecutionRunner } from "../runner/runner.js";
|
|
5
7
|
import { saveRows } from "../output/writer.js";
|
|
6
8
|
import { targetOutputName } from "../output/names.js";
|
|
9
|
+
import { commitAcceptedPage } from "./page-commit.js";
|
|
10
|
+
import { isRecord } from "../utils/guards.js";
|
|
7
11
|
export async function collectProfileTweets(context, targets, options) {
|
|
8
12
|
const request = { targets, ...options };
|
|
13
|
+
const runScope = options.resume ? {} : { runId: randomUUID() };
|
|
9
14
|
const out = [];
|
|
10
15
|
const seenByTarget = new Map();
|
|
11
16
|
const cursors = new Map();
|
|
@@ -20,56 +25,79 @@ export async function collectProfileTweets(context, targets, options) {
|
|
|
20
25
|
const resolved = await resolveTarget(context, target, () => {
|
|
21
26
|
poolRetries += 1;
|
|
22
27
|
}, options.maxAccountSwitches);
|
|
23
|
-
const
|
|
24
|
-
const seen = seenByTarget.get(
|
|
25
|
-
seenByTarget.set(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
+
const collectionId = collectionIdentity("profile_tweets", { target, ...options }, runScope);
|
|
29
|
+
const seen = seenByTarget.get(collectionId) ?? new Set();
|
|
30
|
+
seenByTarget.set(collectionId, seen);
|
|
31
|
+
if (options.resume) {
|
|
32
|
+
for (const record of context.storage.progress.accepted(collectionId)) {
|
|
33
|
+
if (typeof record.id === "string")
|
|
34
|
+
seen.add(record.id);
|
|
35
|
+
if (isTweetRecord(record.payload)) {
|
|
36
|
+
out.push(record.payload);
|
|
37
|
+
counts.set(collectionId, (counts.get(collectionId) ?? 0) + 1);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const saved = context.storage.progress.task(collectionId, collectionId);
|
|
42
|
+
let cursor = cursors.get(collectionId) ??
|
|
28
43
|
options.initialCursors?.[targetKey(target, resolved.userId)] ??
|
|
29
|
-
saved?.
|
|
44
|
+
saved?.cursor ??
|
|
45
|
+
(options.resume ? context.storage.checkpoints.get(collectionId)?.root : undefined);
|
|
30
46
|
let empty = 0;
|
|
31
47
|
let pages = 0;
|
|
32
|
-
let profileCount = counts.get(
|
|
48
|
+
let profileCount = counts.get(collectionId) ?? 0;
|
|
33
49
|
const maxPages = options.maxPagesPerProfile ?? Number.POSITIVE_INFINITY;
|
|
34
|
-
while (pages < maxPages && !limitReached) {
|
|
50
|
+
while (pages < maxPages && !limitReached && saved?.state !== PROGRESS_STATE.EXHAUSTED) {
|
|
35
51
|
pages += 1;
|
|
36
|
-
const
|
|
52
|
+
const inputCursor = cursor;
|
|
53
|
+
const page = await context.pool.execute("profile-tweets", ({ session, signal, chargeRequest }) => context.engine.profilePage(session, resolved.userId, request, inputCursor, signal ?? context.signal, chargeRequest), {
|
|
37
54
|
countTweets: (value) => value.tweets.length,
|
|
38
55
|
onRetry: () => {
|
|
39
56
|
poolRetries += 1;
|
|
40
57
|
},
|
|
41
58
|
maxAccountSwitches: options.maxAccountSwitches,
|
|
59
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
42
60
|
});
|
|
43
|
-
const
|
|
44
|
-
const profileRemaining = options.perProfileLimit === undefined
|
|
45
|
-
? page.tweets.length
|
|
46
|
-
: Math.max(0, options.perProfileLimit - profileCount);
|
|
47
|
-
const remaining = page.tweets.slice(0, Math.min(globalRemaining, profileRemaining));
|
|
61
|
+
const batch = [];
|
|
48
62
|
let added = 0;
|
|
49
|
-
|
|
63
|
+
let capped = false;
|
|
64
|
+
for (const tweet of page.tweets) {
|
|
50
65
|
if (seen.has(tweet.tweetId))
|
|
51
66
|
continue;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
if (options.limit !== undefined && out.length >= options.limit) {
|
|
58
|
-
limitReached = true;
|
|
67
|
+
if ((options.limit !== undefined && out.length + batch.length >= options.limit) ||
|
|
68
|
+
(options.perProfileLimit !== undefined && profileCount + batch.length >= options.perProfileLimit)) {
|
|
69
|
+
capped = true;
|
|
70
|
+
if (options.limit !== undefined && out.length + batch.length >= options.limit)
|
|
71
|
+
limitReached = true;
|
|
59
72
|
break;
|
|
60
73
|
}
|
|
74
|
+
batch.push({ id: tweet.tweetId, payload: tweet });
|
|
75
|
+
added += 1;
|
|
76
|
+
}
|
|
77
|
+
for (const item of batch) {
|
|
78
|
+
seen.add(item.id);
|
|
79
|
+
out.push(item.payload);
|
|
80
|
+
profileCount += 1;
|
|
61
81
|
}
|
|
62
|
-
|
|
63
|
-
if (options.
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
82
|
+
counts.set(collectionId, profileCount);
|
|
83
|
+
if (options.limit !== undefined && out.length >= options.limit)
|
|
84
|
+
limitReached = true;
|
|
85
|
+
empty = added === 0 && !capped ? empty + 1 : 0;
|
|
86
|
+
const exhausted = !capped &&
|
|
87
|
+
stopPaging(options, out.length, profileCount, page.cursor, empty, options.maxEmptyPages ?? context.config.maxEmptyPages);
|
|
88
|
+
commitAcceptedPage(context.storage, {
|
|
89
|
+
collectionId,
|
|
90
|
+
taskId: collectionId,
|
|
91
|
+
state: capped ? PROGRESS_STATE.CAPPED : exhausted ? PROGRESS_STATE.EXHAUSTED : PROGRESS_STATE.ACTIVE,
|
|
92
|
+
...(inputCursor === undefined ? {} : { inputCursor }),
|
|
93
|
+
...(page.cursor === undefined ? {} : { nextCursor: page.cursor }),
|
|
94
|
+
records: batch,
|
|
95
|
+
persistLegacyCheckpoint: Boolean(options.resume),
|
|
96
|
+
});
|
|
97
|
+
if (capped || limitReached || exhausted)
|
|
68
98
|
break;
|
|
69
99
|
cursor = page.cursor;
|
|
70
100
|
}
|
|
71
|
-
if (!limitReached)
|
|
72
|
-
context.storage.checkpoints.clear(checkpointHash);
|
|
73
101
|
});
|
|
74
102
|
if (outcome.failed.length > 0 && (context.config.strict || out.length === 0))
|
|
75
103
|
throw failureFor("Profile timeline", outcome.failed);
|
|
@@ -87,6 +115,7 @@ export async function collectProfileTweets(context, targets, options) {
|
|
|
87
115
|
};
|
|
88
116
|
}
|
|
89
117
|
export async function collectFollows(context, targets, options) {
|
|
118
|
+
const runScope = options.resume ? {} : { runId: randomUUID() };
|
|
90
119
|
const out = [];
|
|
91
120
|
const seenByTarget = new Map();
|
|
92
121
|
const cursors = new Map();
|
|
@@ -97,46 +126,75 @@ export async function collectFollows(context, targets, options) {
|
|
|
97
126
|
});
|
|
98
127
|
const outcome = await runner.run(targets, async (target) => {
|
|
99
128
|
const resolved = await resolveTarget(context, target, undefined, options.maxAccountSwitches);
|
|
100
|
-
const
|
|
101
|
-
const
|
|
102
|
-
|
|
129
|
+
const collectionId = collectionIdentity(options.followType, { target, ...options }, runScope);
|
|
130
|
+
const seen = seenByTarget.get(collectionId) ?? new Set();
|
|
131
|
+
seenByTarget.set(collectionId, seen);
|
|
132
|
+
if (options.resume) {
|
|
133
|
+
for (const record of context.storage.progress.accepted(collectionId)) {
|
|
134
|
+
seen.add(record.id);
|
|
135
|
+
if (isFollowRecord(record.payload))
|
|
136
|
+
out.push(record.payload);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
const saved = context.storage.progress.task(collectionId, collectionId);
|
|
140
|
+
let cursor = cursors.get(collectionId) ??
|
|
103
141
|
options.initialCursors?.[targetKey(target, resolved.userId)] ??
|
|
104
|
-
saved?.
|
|
142
|
+
saved?.cursor ??
|
|
143
|
+
(options.resume ? context.storage.checkpoints.get(collectionId)?.root : undefined);
|
|
105
144
|
let empty = 0;
|
|
106
145
|
let pages = 0;
|
|
107
|
-
const seen = seenByTarget.get(checkpointHash) ?? new Set();
|
|
108
|
-
seenByTarget.set(checkpointHash, seen);
|
|
109
146
|
let targetCount = seen.size;
|
|
110
147
|
const maxPages = options.maxPagesPerProfile ?? Number.POSITIVE_INFINITY;
|
|
111
|
-
while (pages < maxPages && !limitReached) {
|
|
148
|
+
while (pages < maxPages && !limitReached && saved?.state !== PROGRESS_STATE.EXHAUSTED) {
|
|
112
149
|
pages += 1;
|
|
113
|
-
const
|
|
150
|
+
const inputCursor = cursor;
|
|
151
|
+
const page = await context.pool.execute(options.followType, ({ session, signal, chargeRequest }) => context.engine.followsPage(session, resolved.userId, options.followType, inputCursor, signal ?? context.signal, chargeRequest), {
|
|
152
|
+
maxAccountSwitches: options.maxAccountSwitches,
|
|
153
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
154
|
+
});
|
|
155
|
+
const batch = [];
|
|
114
156
|
let added = 0;
|
|
157
|
+
let capped = false;
|
|
115
158
|
for (const user of page.users) {
|
|
116
159
|
const mapped = mapFollow(user, { ...target, userId: resolved.userId }, options.followType, options.rawJson);
|
|
117
160
|
const key = mapped.userId ?? mapped.username;
|
|
118
|
-
if (key
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if (options.limit !== undefined && out.length >= options.limit) {
|
|
124
|
-
limitReached = true;
|
|
125
|
-
break;
|
|
126
|
-
}
|
|
161
|
+
if (!key || seen.has(key) || batch.some((item) => item.id === key))
|
|
162
|
+
continue;
|
|
163
|
+
if (!withinProfileLimit(options.perProfileLimit, targetCount + batch.length)) {
|
|
164
|
+
capped = true;
|
|
165
|
+
break;
|
|
127
166
|
}
|
|
167
|
+
if (options.limit !== undefined && out.length + batch.length >= options.limit) {
|
|
168
|
+
capped = true;
|
|
169
|
+
limitReached = true;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
batch.push({ id: key, payload: mapped });
|
|
173
|
+
added += 1;
|
|
174
|
+
}
|
|
175
|
+
for (const item of batch) {
|
|
176
|
+
seen.add(item.id);
|
|
177
|
+
out.push(item.payload);
|
|
178
|
+
targetCount += 1;
|
|
128
179
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
180
|
+
if (options.limit !== undefined && out.length >= options.limit)
|
|
181
|
+
limitReached = true;
|
|
182
|
+
empty = added === 0 && !capped ? empty + 1 : 0;
|
|
183
|
+
const exhausted = !capped &&
|
|
184
|
+
stopPaging(options, out.length, targetCount, page.cursor, empty, options.maxEmptyPages ?? context.config.maxEmptyPages);
|
|
185
|
+
commitAcceptedPage(context.storage, {
|
|
186
|
+
collectionId,
|
|
187
|
+
taskId: collectionId,
|
|
188
|
+
state: capped ? PROGRESS_STATE.CAPPED : exhausted ? PROGRESS_STATE.EXHAUSTED : PROGRESS_STATE.ACTIVE,
|
|
189
|
+
...(inputCursor === undefined ? {} : { inputCursor }),
|
|
190
|
+
...(page.cursor === undefined ? {} : { nextCursor: page.cursor }),
|
|
191
|
+
records: batch,
|
|
192
|
+
persistLegacyCheckpoint: Boolean(options.resume),
|
|
193
|
+
});
|
|
194
|
+
if (capped || limitReached || exhausted)
|
|
135
195
|
break;
|
|
136
196
|
cursor = page.cursor;
|
|
137
197
|
}
|
|
138
|
-
if (!limitReached)
|
|
139
|
-
context.storage.checkpoints.clear(checkpointHash);
|
|
140
198
|
});
|
|
141
199
|
if (outcome.failed.length > 0 && (context.config.strict || out.length === 0))
|
|
142
200
|
throw failureFor("Relationship collection", outcome.failed);
|
|
@@ -156,9 +214,10 @@ async function saveCollection(context, name, rows, options, targets) {
|
|
|
156
214
|
async function resolveTarget(context, target, onRetry, maxAccountSwitches) {
|
|
157
215
|
if (target.userId && !target.username)
|
|
158
216
|
return { username: target.userId, userId: target.userId, raw: {} };
|
|
159
|
-
return context.pool.execute("user-lookup", ({ session }) => context.engine.resolveTarget(session, target), {
|
|
217
|
+
return context.pool.execute("user-lookup", ({ session, signal, chargeRequest }) => context.engine.resolveTarget(session, target, signal ?? context.signal, chargeRequest), {
|
|
160
218
|
onRetry,
|
|
161
219
|
maxAccountSwitches,
|
|
220
|
+
...(context.signal ? { signal: context.signal } : {}),
|
|
162
221
|
});
|
|
163
222
|
}
|
|
164
223
|
function workerCount(context, tasks) {
|
|
@@ -182,4 +241,10 @@ function withinProfileLimit(limit, count) {
|
|
|
182
241
|
function targetKey(target, resolvedUserId) {
|
|
183
242
|
return target.username ?? target.userId ?? target.profileUrl ?? target.raw ?? resolvedUserId;
|
|
184
243
|
}
|
|
244
|
+
function isTweetRecord(value) {
|
|
245
|
+
return isRecord(value) && typeof value.tweetId === "string";
|
|
246
|
+
}
|
|
247
|
+
function isFollowRecord(value) {
|
|
248
|
+
return isRecord(value) && typeof value.type === "string";
|
|
249
|
+
}
|
|
185
250
|
//# sourceMappingURL=collectors.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collectors.js","sourceRoot":"","sources":["../../src/client/collectors.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAGpD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAStD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAA0B,EAC1B,OAA+B,EAC/B,OAAgD;IAEhD,MAAM,OAAO,GAA2B,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IAChE,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAc;QAC9C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;QACjD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAClC,OAAO,EACP,MAAM,EACN,GAAG,EAAE;YACH,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC,EACD,OAAO,CAAC,kBAAkB,CAC3B,CAAC;QACF,MAAM,cAAc,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACnF,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QACnE,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3F,IAAI,MAAM,GACR,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC3B,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,EAAE,IAAI,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACxE,OAAO,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,gBAAgB,EAChB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,EACtF;gBACE,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;gBAC3C,OAAO,EAAE,GAAG,EAAE;oBACZ,WAAW,IAAI,CAAC,CAAC;gBACnB,CAAC;gBACD,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;aAC/C,CACF,CAAC;YACF,MAAM,eAAe,GACnB,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;YAC7F,MAAM,gBAAgB,GACpB,OAAO,CAAC,eAAe,KAAK,SAAS;gBACnC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;gBACpB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAAC;YAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;oBAAE,SAAS;gBACtC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACxB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAChB,KAAK,IAAI,CAAC,CAAC;gBACX,YAAY,IAAI,CAAC,CAAC;gBAClB,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;gBACzC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAC/D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM;gBACR,CAAC;YACH,CAAC;YACD,KAAK,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBAC/B,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,IACE,UAAU,CACR,OAAO,EACP,GAAG,CAAC,MAAM,EACV,YAAY,EACZ,IAAI,CAAC,MAAM,EACX,KAAK,EACL,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CACtD;gBAED,MAAM;YACR,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1E,MAAM,UAAU,CAAC,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/E,MAAM,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;YAClC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;YAClC,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,WAAW;SACvC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA0B,EAC1B,OAA+B,EAC/B,OAAuB;IAEvB,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAc;QAC9C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;QACjD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC7F,MAAM,cAAc,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACrF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3F,IAAI,MAAM,GACR,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC3B,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,EAAE,IAAI,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QACnE,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACxE,OAAO,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,OAAO,CAAC,UAAU,EAClB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,EACjG,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CACnD,CAAC;YACF,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,SAAS,CACtB,IAAI,EACJ,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,EACtC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,OAAO,CAChB,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC;gBAC7C,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,kBAAkB,CAAC,OAAO,CAAC,eAAe,EAAE,WAAW,CAAC,EAAE,CAAC;oBACtF,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACd,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACjB,KAAK,IAAI,CAAC,CAAC;oBACX,WAAW,IAAI,CAAC,CAAC;oBACjB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;wBAC/D,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YACD,KAAK,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBAC/B,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,IACE,UAAU,CACR,OAAO,EACP,GAAG,CAAC,MAAM,EACV,WAAW,EACX,IAAI,CAAC,MAAM,EACX,KAAK,EACL,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CACtD;gBAED,MAAM;YACR,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1E,MAAM,UAAU,CAAC,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChF,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAA0B,EAC1B,IAAY,EACZ,IAAwB,EACxB,OAAoB,EACpB,OAA+B;IAE/B,IAAI,CAAC,OAAO,CAAC,IAAI;QAAE,OAAO;IAC1B,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE;QACxE,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;QACpD,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;QACvD,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,OAA0B,EAC1B,MAAmB,EACnB,OAAoB,EACpB,kBAA2B;IAE3B,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAC1G,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;QACzG,OAAO;QACP,kBAAkB;KACnB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,OAA0B,EAAE,KAAa;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,MAA8C;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;IAC/B,OAAO,KAAK,YAAY,KAAK;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,eAAe,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,UAAU,CACjB,OAAuE,EACvE,KAAa,EACb,YAAoB,EACpB,MAA0B,EAC1B,KAAa,EACb,aAAqB;IAErB,OAAO,OAAO,CACZ,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;QACvD,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,IAAI,YAAY,IAAI,OAAO,CAAC,eAAe,CAAC;QAClF,CAAC,MAAM;QACP,KAAK,IAAI,aAAa,CACvB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB,EAAE,KAAa;IAClE,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK,CAAC;AAC9C,CAAC;AAED,SAAS,SAAS,CAAC,MAAmB,EAAE,cAAsB;IAC5D,OAAO,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,GAAG,IAAI,cAAc,CAAC;AAC/F,CAAC"}
|
|
1
|
+
{"version":3,"file":"collectors.js","sourceRoot":"","sources":["../../src/client/collectors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAG5D,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAI9C,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAA0B,EAC1B,OAA+B,EAC/B,OAAgD;IAEhD,MAAM,OAAO,GAA2B,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;IAC/D,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAc;QAC9C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;QACjD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAClC,OAAO,EACP,MAAM,EACN,GAAG,EAAE;YACH,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC,EACD,OAAO,CAAC,kBAAkB,CAC3B,CAAC;QACF,MAAM,YAAY,GAAG,kBAAkB,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC5F,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QACjE,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACrE,IAAI,OAAO,MAAM,CAAC,EAAE,KAAK,QAAQ;oBAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACvD,IAAI,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;oBAClC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBACzB,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QACxE,IAAI,MAAM,GACR,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;YACzB,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,EAAE,MAAM;YACb,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACrF,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACxE,OAAO,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,IAAI,KAAK,EAAE,KAAK,KAAK,cAAc,CAAC,SAAS,EAAE,CAAC;YACtF,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,WAAW,GAAG,MAAM,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,gBAAgB,EAChB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,MAAM,CAAC,WAAW,CACxB,OAAO,EACP,QAAQ,CAAC,MAAM,EACf,OAAO,EACP,WAAW,EACX,MAAM,IAAI,OAAO,CAAC,MAAM,EACxB,aAAa,CACd,EACH;gBACE,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;gBAC3C,OAAO,EAAE,GAAG,EAAE;oBACZ,WAAW,IAAI,CAAC,CAAC;gBACnB,CAAC;gBACD,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;gBAC9C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CACF,CAAC;YACF,MAAM,KAAK,GAAgD,EAAE,CAAC;YAC9D,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;oBAAE,SAAS;gBACtC,IACE,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC;oBAC3E,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,IAAI,YAAY,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,eAAe,CAAC,EACjG,CAAC;oBACD,MAAM,GAAG,IAAI,CAAC;oBACd,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK;wBAAE,YAAY,GAAG,IAAI,CAAC;oBACnG,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,YAAY,IAAI,CAAC,CAAC;YACpB,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;YACvC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK;gBAAE,YAAY,GAAG,IAAI,CAAC;YACpF,KAAK,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,SAAS,GACb,CAAC,MAAM;gBACP,UAAU,CACR,OAAO,EACP,GAAG,CAAC,MAAM,EACV,YAAY,EACZ,IAAI,CAAC,MAAM,EACX,KAAK,EACL,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CACtD,CAAC;YACJ,kBAAkB,CAAC,OAAO,CAAC,OAAO,EAAE;gBAClC,YAAY;gBACZ,MAAM,EAAE,YAAY;gBACpB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM;gBACpG,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjE,OAAO,EAAE,KAAK;gBACd,uBAAuB,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;aACjD,CAAC,CAAC;YACH,IAAI,MAAM,IAAI,YAAY,IAAI,SAAS;gBAAE,MAAM;YAC/C,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1E,MAAM,UAAU,CAAC,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/E,MAAM,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;YAClC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;YAClC,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,WAAW;SACvC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAA0B,EAC1B,OAA+B,EAC/B,OAAuB;IAEvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC;IAC/D,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,MAAM,GAAG,IAAI,eAAe,CAAc;QAC9C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;QACjD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACzD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC7F,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC9F,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QACjE,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACrE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACpB,IAAI,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC;oBAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QACxE,IAAI,MAAM,GACR,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;YACzB,OAAO,CAAC,cAAc,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,EAAE,MAAM;YACb,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACrF,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC;QACxE,OAAO,KAAK,GAAG,QAAQ,IAAI,CAAC,YAAY,IAAI,KAAK,EAAE,KAAK,KAAK,cAAc,CAAC,SAAS,EAAE,CAAC;YACtF,KAAK,IAAI,CAAC,CAAC;YACX,MAAM,WAAW,GAAG,MAAM,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,OAAO,CAAC,UAAU,EAClB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,MAAM,CAAC,WAAW,CACxB,OAAO,EACP,QAAQ,CAAC,MAAM,EACf,OAAO,CAAC,UAAU,EAClB,WAAW,EACX,MAAM,IAAI,OAAO,CAAC,MAAM,EACxB,aAAa,CACd,EACH;gBACE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;gBAC9C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CACF,CAAC;YACF,MAAM,KAAK,GAAiD,EAAE,CAAC;YAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,MAAM,GAAG,KAAK,CAAC;YACnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,SAAS,CACtB,IAAI,EACJ,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,EACtC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,OAAO,CAChB,CAAC;gBACF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC;gBAC7C,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,GAAG,CAAC;oBAAE,SAAS;gBAC7E,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,eAAe,EAAE,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC7E,MAAM,GAAG,IAAI,CAAC;oBACd,MAAM;gBACR,CAAC;gBACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAC9E,MAAM,GAAG,IAAI,CAAC;oBACd,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzC,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,WAAW,IAAI,CAAC,CAAC;YACnB,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK;gBAAE,YAAY,GAAG,IAAI,CAAC;YACpF,KAAK,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,SAAS,GACb,CAAC,MAAM;gBACP,UAAU,CACR,OAAO,EACP,GAAG,CAAC,MAAM,EACV,WAAW,EACX,IAAI,CAAC,MAAM,EACX,KAAK,EACL,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CACtD,CAAC;YACJ,kBAAkB,CAAC,OAAO,CAAC,OAAO,EAAE;gBAClC,YAAY;gBACZ,MAAM,EAAE,YAAY;gBACpB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM;gBACpG,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;gBACrD,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjE,OAAO,EAAE,KAAK;gBACd,uBAAuB,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;aACjD,CAAC,CAAC;YACH,IAAI,MAAM,IAAI,YAAY,IAAI,SAAS;gBAAE,MAAM;YAC/C,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC;QAC1E,MAAM,UAAU,CAAC,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAChF,MAAM,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAA0B,EAC1B,IAAY,EACZ,IAAwB,EACxB,OAAoB,EACpB,OAA+B;IAE/B,IAAI,CAAC,OAAO,CAAC,IAAI;QAAE,OAAO;IAC1B,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE;QACxE,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;QACpD,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;QACvD,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,OAA0B,EAC1B,MAAmB,EACnB,OAAoB,EACpB,kBAA2B;IAE3B,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ;QAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAC1G,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CACzB,aAAa,EACb,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,EACxF;QACE,OAAO;QACP,kBAAkB;QAClB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,OAA0B,EAAE,KAAa;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,MAA8C;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;IAC/B,OAAO,KAAK,YAAY,KAAK;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,eAAe,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,UAAU,CACjB,OAAuE,EACvE,KAAa,EACb,YAAoB,EACpB,MAA0B,EAC1B,KAAa,EACb,aAAqB;IAErB,OAAO,OAAO,CACZ,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;QACvD,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,IAAI,YAAY,IAAI,OAAO,CAAC,eAAe,CAAC;QAClF,CAAC,MAAM;QACP,KAAK,IAAI,aAAa,CACvB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB,EAAE,KAAa;IAClE,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK,CAAC;AAC9C,CAAC;AAED,SAAS,SAAS,CAAC,MAAmB,EAAE,cAAsB;IAC5D,OAAO,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,GAAG,IAAI,cAAc,CAAC;AAC/F,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC;AAC9D,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC3D,CAAC"}
|
|
@@ -1,19 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AccountRecord, ProxySettings } from "../domain/accounts.js";
|
|
2
2
|
import type { StorageBundle } from "../storage/index.js";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
readonly unusableOnly?: boolean;
|
|
6
|
-
readonly includeCookies?: boolean;
|
|
7
|
-
readonly revealSecrets?: boolean;
|
|
8
|
-
}
|
|
9
|
-
export interface AccountImportOptions {
|
|
10
|
-
readonly accountsFile?: string;
|
|
11
|
-
readonly cookiesFile?: string;
|
|
12
|
-
readonly envFile?: string;
|
|
13
|
-
readonly cookies?: unknown;
|
|
14
|
-
readonly accounts?: readonly AccountInput[];
|
|
15
|
-
readonly proxy?: string | ProxySettings;
|
|
16
|
-
}
|
|
3
|
+
import type { AccountImportOptions, AccountListOptions } from "./types.js";
|
|
4
|
+
export type { AccountImportOptions, AccountListOptions } from "./types.js";
|
|
17
5
|
export declare class XTrawlDatabase {
|
|
18
6
|
private readonly storage;
|
|
19
7
|
constructor(storage: StorageBundle);
|
|
@@ -45,12 +33,12 @@ export declare class XTrawlDatabase {
|
|
|
45
33
|
getCheckpoint(queryHash: string): Readonly<Record<string, string>> | undefined;
|
|
46
34
|
clearCheckpoint(queryHash: string): boolean;
|
|
47
35
|
clearAllCheckpoints(): number;
|
|
48
|
-
listRuns(limit?: number): import("../
|
|
49
|
-
lastRun(): import("../
|
|
36
|
+
listRuns(limit?: number): import("../domain/runs.js").RunRecord[];
|
|
37
|
+
lastRun(): import("../domain/runs.js").RunRecord | undefined;
|
|
50
38
|
runsSummary(limit?: number): {
|
|
51
39
|
readonly totalRuns: number;
|
|
52
40
|
readonly byStatus: Readonly<Record<string, number>>;
|
|
53
|
-
readonly lastRun?: import("../
|
|
41
|
+
readonly lastRun?: import("../domain/runs.js").RunRecord;
|
|
54
42
|
};
|
|
55
43
|
}
|
|
56
44
|
export declare function redactAccount(account: AccountRecord, options: Pick<AccountListOptions, "includeCookies" | "revealSecrets">): Record<string, unknown>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/client/database.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/client/database.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGzD,OAAO,KAAK,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE3E,YAAY,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE3E,qBAAa,cAAc;IACN,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,aAAa;IAEnD,eAAe;IAIf,YAAY,CAAC,OAAO,GAAE,kBAAuB,GAAG,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;IAQlF,UAAU,CACf,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,GAAG,eAAe,CAAM,GACzE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAK/B,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,OAAO;IAI1E,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAW,GAAG,OAAO;IAI3D,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,UAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAmB7E,qBAAqB,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,eAAe,UAAQ,GAAG,MAAM;IAIrF,WAAW,CAAC,WAAW,UAAO,GAAG,MAAM;IAIvC,kBAAkB,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM;IAIzD,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG;QACpD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B;IAaM,yBAAyB,CAAC,MAAM,UAAO;;;;;;;;;;;IASvC,aAAa,CAAC,SAAS,EAAE,MAAM;IAI/B,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAI3C,mBAAmB,IAAI,MAAM;IAI7B,QAAQ,CAAC,KAAK,SAAK;IAInB,OAAO;IAIP,WAAW,CAAC,KAAK,SAAM;;;;;CAG/B;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,GAAG,eAAe,CAAC,GACpE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAgBzB"}
|
package/dist/client/database.js
CHANGED
|
@@ -2,7 +2,9 @@ import { bootstrapCookiesFromAuthToken } from "../auth/bootstrap.js";
|
|
|
2
2
|
import { loadAccountsFileSync, loadInlineAccounts } from "../auth/loaders.js";
|
|
3
3
|
import { accountInputToRecord } from "../auth/records.js";
|
|
4
4
|
import { loadAccountFromEnvironmentSync } from "../config/environment.js";
|
|
5
|
+
import { ACCOUNT_STATUS_CODE } from "../constants/accounts.js";
|
|
5
6
|
import { tokenFingerprint } from "../utils/fingerprint.js";
|
|
7
|
+
import { redactProxy } from "../utils/redact.js";
|
|
6
8
|
export class XTrawlDatabase {
|
|
7
9
|
storage;
|
|
8
10
|
constructor(storage) {
|
|
@@ -15,7 +17,7 @@ export class XTrawlDatabase {
|
|
|
15
17
|
return this.storage.accounts
|
|
16
18
|
.list()
|
|
17
19
|
.filter((account) => !options.eligibleOnly || this.storage.accounts.eligible(account))
|
|
18
|
-
.filter((account) => !options.unusableOnly || account.status ===
|
|
20
|
+
.filter((account) => !options.unusableOnly || account.status === ACCOUNT_STATUS_CODE.UNUSABLE)
|
|
19
21
|
.map((account) => redactAccount(account, options));
|
|
20
22
|
}
|
|
21
23
|
getAccount(username, options = {}) {
|
|
@@ -44,7 +46,7 @@ export class XTrawlDatabase {
|
|
|
44
46
|
return false;
|
|
45
47
|
this.storage.accounts.upsert({
|
|
46
48
|
...account,
|
|
47
|
-
status:
|
|
49
|
+
status: ACCOUNT_STATUS_CODE.HEALTHY,
|
|
48
50
|
availableUntil: 0,
|
|
49
51
|
csrfToken: cookies.ct0,
|
|
50
52
|
cookies: { ...account.cookies, ...cookies },
|
|
@@ -124,23 +126,4 @@ export function redactAccount(account, options) {
|
|
|
124
126
|
function redactCookies(cookies) {
|
|
125
127
|
return Object.fromEntries(Object.keys(cookies).map((name) => [name, "[redacted]"]));
|
|
126
128
|
}
|
|
127
|
-
function redactProxy(proxy) {
|
|
128
|
-
if (typeof proxy !== "string")
|
|
129
|
-
return {
|
|
130
|
-
...proxy,
|
|
131
|
-
...(proxy.username ? { username: "[redacted]" } : {}),
|
|
132
|
-
...(proxy.password ? { password: "[redacted]" } : {}),
|
|
133
|
-
};
|
|
134
|
-
try {
|
|
135
|
-
const value = new URL(proxy.includes("://") ? proxy : `http://${proxy}`);
|
|
136
|
-
if (value.username)
|
|
137
|
-
value.username = "[redacted]";
|
|
138
|
-
if (value.password)
|
|
139
|
-
value.password = "[redacted]";
|
|
140
|
-
return value.toString();
|
|
141
|
-
}
|
|
142
|
-
catch {
|
|
143
|
-
return "[redacted]";
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
129
|
//# sourceMappingURL=database.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/client/database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/client/database.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAC1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKjD,MAAM,OAAO,cAAc;IACW;IAApC,YAAoC,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAEvD,eAAe;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IACzC,CAAC;IAEM,YAAY,CAAC,UAA8B,EAAE;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ;aACzB,IAAI,EAAE;aACN,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aACrF,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,MAAM,KAAK,mBAAmB,CAAC,QAAQ,CAAC;aAC7F,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAEM,UAAU,CACf,QAAgB,EAChB,UAAwE,EAAE;QAE1E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/D,CAAC;IAEM,aAAa,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAEM,eAAe,CAAC,QAAgB,EAAE,KAA8B;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzD,CAAC;IAEM,mBAAmB,CAAC,QAAgB,EAAE,MAAM,GAAG,QAAQ;QAC5D,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,MAAM,EAAE,CAAC,CAAC;IACjF,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,YAAY,GAAG,KAAK;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO,EAAE,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,6BAA6B,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjG,IAAI,CAAC,OAAO,EAAE,GAAG;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3B,GAAG,OAAO;YACV,MAAM,EAAE,mBAAmB,CAAC,OAAO;YACnC,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,GAAG;YACtB,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE;SAC5C,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,qBAAqB,CAAC,SAA6B,EAAE,eAAe,GAAG,KAAK;QACjF,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC1E,CAAC;IAEM,WAAW,CAAC,WAAW,GAAG,IAAI;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC;IAEM,kBAAkB,CAAC,SAA6B;QACrD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAC1D,CAAC;IAEM,cAAc,CAAC,OAA6B;QAIjD,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACnF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACzF,IAAI,OAAO,CAAC,YAAY;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,WAAW;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;QACrF,IAAI,OAAO,CAAC,OAAO;YACjB,QAAQ,CAAC,IAAI,CAAC,GAAG,8BAA8B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC9F,KAAK,MAAM,OAAO,IAAI,QAAQ;YAC5B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACtF,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;IAC5F,CAAC;IAEM,yBAAyB,CAAC,MAAM,GAAG,IAAI;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,6BAA6B,EAAE,EAAE,CAAC;QACzF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ;aACjC,IAAI,EAAE;aACN,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;QACxD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;IACpE,CAAC;IAEM,aAAa,CAAC,SAAiB;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAEM,eAAe,CAAC,SAAiB;QACtC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACnD,CAAC;IAEM,mBAAmB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;IAC7C,CAAC;IAEM,QAAQ,CAAC,KAAK,GAAG,EAAE;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAClC,CAAC;IAEM,WAAW,CAAC,KAAK,GAAG,GAAG;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;CACF;AAED,MAAM,UAAU,aAAa,CAC3B,OAAsB,EACtB,OAAqE;IAErE,MAAM,IAAI,GAA4B;QACpC,GAAG,OAAO;QACV,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QAC9E,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACvD,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACrD,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QAC/C,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QAC/D,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;QACnE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;QAC7D,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAC7E,CAAC;IACF,IAAI,OAAO,CAAC,aAAa;QACvB,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IACvF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,OAAyC;IAC9D,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;AACtF,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ProgressState } from "../domain/collection.js";
|
|
2
|
+
import type { StorageBundle } from "../storage/index.js";
|
|
3
|
+
export declare function commitAcceptedPage(storage: StorageBundle, input: {
|
|
4
|
+
readonly collectionId: string;
|
|
5
|
+
readonly taskId: string;
|
|
6
|
+
readonly state: ProgressState;
|
|
7
|
+
readonly inputCursor?: string;
|
|
8
|
+
readonly nextCursor?: string;
|
|
9
|
+
readonly records: readonly {
|
|
10
|
+
readonly id: string;
|
|
11
|
+
readonly payload: unknown;
|
|
12
|
+
}[];
|
|
13
|
+
readonly persistLegacyCheckpoint?: boolean;
|
|
14
|
+
}): void;
|
|
15
|
+
//# sourceMappingURL=page-commit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"page-commit.d.ts","sourceRoot":"","sources":["../../src/client/page-commit.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE;IACL,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;IAChF,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;CAC5C,GACA,IAAI,CAcN"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { PROGRESS_STATE } from "../constants/collection.js";
|
|
2
|
+
export function commitAcceptedPage(storage, input) {
|
|
3
|
+
storage.progress.commitPage({
|
|
4
|
+
collectionId: input.collectionId,
|
|
5
|
+
taskId: input.taskId,
|
|
6
|
+
state: input.state,
|
|
7
|
+
...(input.inputCursor === undefined ? {} : { inputCursor: input.inputCursor }),
|
|
8
|
+
...(input.nextCursor === undefined ? {} : { nextCursor: input.nextCursor }),
|
|
9
|
+
records: input.records,
|
|
10
|
+
});
|
|
11
|
+
if (!input.persistLegacyCheckpoint)
|
|
12
|
+
return;
|
|
13
|
+
const cursor = input.state === PROGRESS_STATE.CAPPED ? input.inputCursor : input.nextCursor;
|
|
14
|
+
if (cursor)
|
|
15
|
+
storage.checkpoints.save(input.taskId, { root: cursor });
|
|
16
|
+
else if (input.state === PROGRESS_STATE.EXHAUSTED || input.state === PROGRESS_STATE.CAPPED)
|
|
17
|
+
storage.checkpoints.clear(input.taskId);
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=page-commit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"page-commit.js","sourceRoot":"","sources":["../../src/client/page-commit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAI5D,MAAM,UAAU,kBAAkB,CAChC,OAAsB,EACtB,KAQC;IAED,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9E,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC;QAC3E,OAAO,EAAE,KAAK,CAAC,OAAO;KACvB,CAAC,CAAC;IACH,IAAI,CAAC,KAAK,CAAC,uBAAuB;QAAE,OAAO;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,KAAK,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;IAC5F,IAAI,MAAM;QAAE,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;SAChE,IAAI,KAAK,CAAC,KAAK,KAAK,cAAc,CAAC,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,cAAc,CAAC,MAAM;QACxF,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ProfileRecord } from "../domain/records.js";
|
|
2
2
|
import type { TargetInput, UserInfoRequest } from "../domain/requests.js";
|
|
3
|
-
import type { CollectionContext } from "./
|
|
3
|
+
import type { CollectionContext } from "./types.js";
|
|
4
4
|
export declare function collectProfiles(context: CollectionContext, targets: readonly TargetInput[], options: UserInfoRequest): Promise<readonly ProfileRecord[]>;
|
|
5
5
|
//# sourceMappingURL=profiles.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../src/client/profiles.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAO1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../src/client/profiles.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAO1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,wBAAsB,eAAe,CACnC,OAAO,EAAE,iBAAiB,EAC1B,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC,CA2CnC"}
|
package/dist/client/profiles.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RunFailed } from "../domain/errors.js";
|
|
1
|
+
import { ConfigError, RunFailed } from "../domain/errors.js";
|
|
2
2
|
import { mapProfile } from "../engine/extractors.js";
|
|
3
3
|
import { saveRows } from "../output/writer.js";
|
|
4
4
|
import { targetOutputName } from "../output/names.js";
|
|
@@ -6,7 +6,14 @@ import { targetUsername } from "../query/builder.js";
|
|
|
6
6
|
import { ExecutionRunner } from "../runner/runner.js";
|
|
7
7
|
export async function collectProfiles(context, targets, options) {
|
|
8
8
|
const records = new Map();
|
|
9
|
-
const
|
|
9
|
+
for (const target of targets) {
|
|
10
|
+
if (targetUsername(target))
|
|
11
|
+
continue;
|
|
12
|
+
if (target.userId)
|
|
13
|
+
throw new ConfigError(`Profile lookup requires a username; numeric id ${target.userId} is not supported.`);
|
|
14
|
+
throw new ConfigError("Profile lookup requires a username.");
|
|
15
|
+
}
|
|
16
|
+
const tasks = targets.map((target, index) => ({ index, target }));
|
|
10
17
|
const runner = new ExecutionRunner({
|
|
11
18
|
concurrency: Math.max(1, Math.min(context.config.concurrency, context.pool.summary.eligible, tasks.length)),
|
|
12
19
|
maxAttempts: 1,
|
|
@@ -15,7 +22,7 @@ export async function collectProfiles(context, targets, options) {
|
|
|
15
22
|
const username = targetUsername(target);
|
|
16
23
|
if (!username)
|
|
17
24
|
return;
|
|
18
|
-
const user = await context.pool.execute("user-info", ({ session }) => context.engine.lookupUser(session, username));
|
|
25
|
+
const user = await context.pool.execute("user-info", ({ session, signal, chargeRequest }) => context.engine.lookupUser(session, username, signal ?? context.signal, chargeRequest), context.signal ? { signal: context.signal } : {});
|
|
19
26
|
records.set(index, mapProfile(user, target, username));
|
|
20
27
|
});
|
|
21
28
|
if (outcome.failed.length > 0 && (context.config.strict || records.size === 0)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../src/client/profiles.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../src/client/profiles.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAGtD,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAA0B,EAC1B,OAA+B,EAC/B,OAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,cAAc,CAAC,MAAM,CAAC;YAAE,SAAS;QACrC,IAAI,MAAM,CAAC,MAAM;YACf,MAAM,IAAI,WAAW,CACnB,kDAAkD,MAAM,CAAC,MAAM,oBAAoB,CACpF,CAAC;QACJ,MAAM,IAAI,WAAW,CAAC,qCAAqC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,IAAI,eAAe,CAAyB;QACzD,WAAW,EAAE,IAAI,CAAC,GAAG,CACnB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAClF;QACD,WAAW,EAAE,CAAC;KACf,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAClE,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,WAAW,EACX,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE,CACrC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,EACvF,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CACjD,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;QACvC,MAAM,KAAK,YAAY,SAAS;YAC9B,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,IAAI,SAAS,CAAC,6BAA6B,OAAO,CAAC,MAAM,CAAC,MAAM,eAAe,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IACxG,IAAI,OAAO,CAAC,IAAI;QACd,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE;YACjF,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;YACpD,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;YACvD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/client/search.d.ts
CHANGED
|
@@ -1,21 +1,9 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import type { StorageBundle } from "../storage/index.js";
|
|
7
|
-
export interface SearchContext {
|
|
8
|
-
readonly config: ClientConfig;
|
|
9
|
-
readonly pool: AccountPool;
|
|
10
|
-
readonly engine: ApiEngine;
|
|
11
|
-
readonly storage: StorageBundle;
|
|
12
|
-
}
|
|
13
|
-
interface SearchTask {
|
|
14
|
-
readonly id: string;
|
|
15
|
-
readonly request: SearchRequest;
|
|
16
|
-
}
|
|
1
|
+
import type { SearchPageResult, SearchResult } from "../domain/records.js";
|
|
2
|
+
import type { SearchPageRequest, SearchRequest } from "../domain/requests.js";
|
|
3
|
+
import type { SearchContext, SearchTask } from "./types.js";
|
|
4
|
+
export type { SearchContext } from "./types.js";
|
|
5
|
+
export declare function collectSearchPage(context: SearchContext, query: string, options: SearchPageRequest): Promise<SearchPageResult>;
|
|
17
6
|
export declare function collectSearch(context: SearchContext, query: string, options: SearchRequest): Promise<SearchResult>;
|
|
18
7
|
export declare function withDefaultBounds(request: SearchRequest, now?: Date): SearchRequest;
|
|
19
8
|
export declare function splitSearchTasks(request: SearchRequest, requestedSplits: number, minIntervalMs: number): readonly SearchTask[];
|
|
20
|
-
export {};
|
|
21
9
|
//# sourceMappingURL=search.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/client/search.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/client/search.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAe,MAAM,sBAAsB,CAAC;AACxF,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAO9E,OAAO,KAAK,EAAwB,aAAa,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGlF,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAO3B;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAsIvB;AAqBD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,OAAa,GAAG,aAAa,CAIzF;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,aAAa,EACtB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,GACpB,SAAS,UAAU,EAAE,CAavB"}
|