xtrawl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DOCUMENTATION.md +774 -0
- package/README.md +232 -0
- package/dist/auth/bootstrap.d.ts +4 -0
- package/dist/auth/bootstrap.d.ts.map +1 -0
- package/dist/auth/bootstrap.js +29 -0
- package/dist/auth/bootstrap.js.map +1 -0
- package/dist/auth/cookies.d.ts +7 -0
- package/dist/auth/cookies.d.ts.map +1 -0
- package/dist/auth/cookies.js +83 -0
- package/dist/auth/cookies.js.map +1 -0
- package/dist/auth/loaders.d.ts +6 -0
- package/dist/auth/loaders.d.ts.map +1 -0
- package/dist/auth/loaders.js +69 -0
- package/dist/auth/loaders.js.map +1 -0
- package/dist/auth/material.d.ts +3 -0
- package/dist/auth/material.d.ts.map +1 -0
- package/dist/auth/material.js +20 -0
- package/dist/auth/material.js.map +1 -0
- package/dist/auth/records.d.ts +4 -0
- package/dist/auth/records.d.ts.map +1 -0
- package/dist/auth/records.js +38 -0
- package/dist/auth/records.js.map +1 -0
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +30 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +85 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/parser.d.ts +22 -0
- package/dist/cli/parser.d.ts.map +1 -0
- package/dist/cli/parser.js +275 -0
- package/dist/cli/parser.js.map +1 -0
- package/dist/client/client.d.ts +29 -0
- package/dist/client/client.d.ts.map +1 -0
- package/dist/client/client.js +192 -0
- package/dist/client/client.js.map +1 -0
- package/dist/client/collectors.d.ts +15 -0
- package/dist/client/collectors.d.ts.map +1 -0
- package/dist/client/collectors.js +185 -0
- package/dist/client/collectors.js.map +1 -0
- package/dist/client/database.d.ts +56 -0
- package/dist/client/database.d.ts.map +1 -0
- package/dist/client/database.js +146 -0
- package/dist/client/database.js.map +1 -0
- package/dist/client/profiles.d.ts +5 -0
- package/dist/client/profiles.d.ts.map +1 -0
- package/dist/client/profiles.js +36 -0
- package/dist/client/profiles.js.map +1 -0
- package/dist/client/search.d.ts +21 -0
- package/dist/client/search.d.ts.map +1 -0
- package/dist/client/search.js +139 -0
- package/dist/client/search.js.map +1 -0
- package/dist/client/types.d.ts +21 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/client/types.js +2 -0
- package/dist/client/types.js.map +1 -0
- package/dist/config/defaults.d.ts +3 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +38 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/environment.d.ts +5 -0
- package/dist/config/environment.d.ts.map +1 -0
- package/dist/config/environment.js +62 -0
- package/dist/config/environment.js.map +1 -0
- package/dist/config/types.d.ts +47 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/config/validation.d.ts +6 -0
- package/dist/config/validation.d.ts.map +1 -0
- package/dist/config/validation.js +147 -0
- package/dist/config/validation.js.map +1 -0
- package/dist/domain/accounts.d.ts +64 -0
- package/dist/domain/accounts.d.ts.map +1 -0
- package/dist/domain/accounts.js +2 -0
- package/dist/domain/accounts.js.map +1 -0
- package/dist/domain/errors.d.ts +61 -0
- package/dist/domain/errors.d.ts.map +1 -0
- package/dist/domain/errors.js +89 -0
- package/dist/domain/errors.js.map +1 -0
- package/dist/domain/http.d.ts +36 -0
- package/dist/domain/http.d.ts.map +1 -0
- package/dist/domain/http.js +2 -0
- package/dist/domain/http.js.map +1 -0
- package/dist/domain/manifest.d.ts +23 -0
- package/dist/domain/manifest.d.ts.map +1 -0
- package/dist/domain/manifest.js +2 -0
- package/dist/domain/manifest.js.map +1 -0
- package/dist/domain/records.d.ts +67 -0
- package/dist/domain/records.d.ts.map +1 -0
- package/dist/domain/records.js +2 -0
- package/dist/domain/records.js.map +1 -0
- package/dist/domain/requests.d.ts +87 -0
- package/dist/domain/requests.d.ts.map +1 -0
- package/dist/domain/requests.js +2 -0
- package/dist/domain/requests.js.map +1 -0
- package/dist/engine/api-engine.d.ts +25 -0
- package/dist/engine/api-engine.d.ts.map +1 -0
- package/dist/engine/api-engine.js +98 -0
- package/dist/engine/api-engine.js.map +1 -0
- package/dist/engine/extractors.d.ts +27 -0
- package/dist/engine/extractors.d.ts.map +1 -0
- package/dist/engine/extractors.js +248 -0
- package/dist/engine/extractors.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/manifest/default-manifest.d.ts +3 -0
- package/dist/manifest/default-manifest.d.ts.map +1 -0
- package/dist/manifest/default-manifest.js +46 -0
- package/dist/manifest/default-manifest.js.map +1 -0
- package/dist/manifest/model.d.ts +3 -0
- package/dist/manifest/model.d.ts.map +1 -0
- package/dist/manifest/model.js +48 -0
- package/dist/manifest/model.js.map +1 -0
- package/dist/manifest/provider.d.ts +19 -0
- package/dist/manifest/provider.d.ts.map +1 -0
- package/dist/manifest/provider.js +94 -0
- package/dist/manifest/provider.js.map +1 -0
- package/dist/manifest/scraper.d.ts +11 -0
- package/dist/manifest/scraper.d.ts.map +1 -0
- package/dist/manifest/scraper.js +130 -0
- package/dist/manifest/scraper.js.map +1 -0
- package/dist/output/csv-writer.d.ts +2 -0
- package/dist/output/csv-writer.d.ts.map +1 -0
- package/dist/output/csv-writer.js +94 -0
- package/dist/output/csv-writer.js.map +1 -0
- package/dist/output/json-writer.d.ts +2 -0
- package/dist/output/json-writer.d.ts.map +1 -0
- package/dist/output/json-writer.js +29 -0
- package/dist/output/json-writer.js.map +1 -0
- package/dist/output/names.d.ts +4 -0
- package/dist/output/names.d.ts.map +1 -0
- package/dist/output/names.js +22 -0
- package/dist/output/names.js.map +1 -0
- package/dist/output/writer.d.ts +7 -0
- package/dist/output/writer.d.ts.map +1 -0
- package/dist/output/writer.js +33 -0
- package/dist/output/writer.js.map +1 -0
- package/dist/pool/account-pool.d.ts +29 -0
- package/dist/pool/account-pool.d.ts.map +1 -0
- package/dist/pool/account-pool.js +145 -0
- package/dist/pool/account-pool.js.map +1 -0
- package/dist/pool/cooldown.d.ts +18 -0
- package/dist/pool/cooldown.d.ts.map +1 -0
- package/dist/pool/cooldown.js +38 -0
- package/dist/pool/cooldown.js.map +1 -0
- package/dist/pool/limiter.d.ts +11 -0
- package/dist/pool/limiter.d.ts.map +1 -0
- package/dist/pool/limiter.js +27 -0
- package/dist/pool/limiter.js.map +1 -0
- package/dist/query/builder.d.ts +19 -0
- package/dist/query/builder.d.ts.map +1 -0
- package/dist/query/builder.js +110 -0
- package/dist/query/builder.js.map +1 -0
- package/dist/query/hash.d.ts +2 -0
- package/dist/query/hash.d.ts.map +1 -0
- package/dist/query/hash.js +6 -0
- package/dist/query/hash.js.map +1 -0
- package/dist/query/normalize.d.ts +39 -0
- package/dist/query/normalize.d.ts.map +1 -0
- package/dist/query/normalize.js +176 -0
- package/dist/query/normalize.js.map +1 -0
- package/dist/query/targets.d.ts +11 -0
- package/dist/query/targets.d.ts.map +1 -0
- package/dist/query/targets.js +96 -0
- package/dist/query/targets.js.map +1 -0
- package/dist/runner/index.d.ts +4 -0
- package/dist/runner/index.d.ts.map +1 -0
- package/dist/runner/index.js +4 -0
- package/dist/runner/index.js.map +1 -0
- package/dist/runner/retry.d.ts +7 -0
- package/dist/runner/retry.d.ts.map +1 -0
- package/dist/runner/retry.js +26 -0
- package/dist/runner/retry.js.map +1 -0
- package/dist/runner/runner.d.ts +20 -0
- package/dist/runner/runner.d.ts.map +1 -0
- package/dist/runner/runner.js +37 -0
- package/dist/runner/runner.js.map +1 -0
- package/dist/runner/task-queue.d.ts +19 -0
- package/dist/runner/task-queue.d.ts.map +1 -0
- package/dist/runner/task-queue.js +58 -0
- package/dist/runner/task-queue.js.map +1 -0
- package/dist/storage/account-repository.d.ts +50 -0
- package/dist/storage/account-repository.d.ts.map +1 -0
- package/dist/storage/account-repository.js +171 -0
- package/dist/storage/account-repository.js.map +1 -0
- package/dist/storage/account-row.d.ts +4 -0
- package/dist/storage/account-row.d.ts.map +1 -0
- package/dist/storage/account-row.js +46 -0
- package/dist/storage/account-row.js.map +1 -0
- package/dist/storage/checkpoint-repository.d.ts +10 -0
- package/dist/storage/checkpoint-repository.d.ts.map +1 -0
- package/dist/storage/checkpoint-repository.js +30 -0
- package/dist/storage/checkpoint-repository.js.map +1 -0
- package/dist/storage/database.d.ts +18 -0
- package/dist/storage/database.d.ts.map +1 -0
- package/dist/storage/database.js +42 -0
- package/dist/storage/database.js.map +1 -0
- package/dist/storage/index.d.ts +15 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +18 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/manifest-repository.d.ts +9 -0
- package/dist/storage/manifest-repository.d.ts.map +1 -0
- package/dist/storage/manifest-repository.js +22 -0
- package/dist/storage/manifest-repository.js.map +1 -0
- package/dist/storage/run-repository.d.ts +24 -0
- package/dist/storage/run-repository.d.ts.map +1 -0
- package/dist/storage/run-repository.js +51 -0
- package/dist/storage/run-repository.js.map +1 -0
- package/dist/storage/schema.d.ts +3 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +53 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/transport/graphql.d.ts +8 -0
- package/dist/transport/graphql.d.ts.map +1 -0
- package/dist/transport/graphql.js +104 -0
- package/dist/transport/graphql.js.map +1 -0
- package/dist/transport/proxy.d.ts +5 -0
- package/dist/transport/proxy.d.ts.map +1 -0
- package/dist/transport/proxy.js +36 -0
- package/dist/transport/proxy.js.map +1 -0
- package/dist/transport/session.d.ts +25 -0
- package/dist/transport/session.d.ts.map +1 -0
- package/dist/transport/session.js +138 -0
- package/dist/transport/session.js.map +1 -0
- package/dist/transport/transaction-id.d.ts +20 -0
- package/dist/transport/transaction-id.d.ts.map +1 -0
- package/dist/transport/transaction-id.js +74 -0
- package/dist/transport/transaction-id.js.map +1 -0
- package/dist/utils/fingerprint.d.ts +3 -0
- package/dist/utils/fingerprint.d.ts.map +1 -0
- package/dist/utils/fingerprint.js +20 -0
- package/dist/utils/fingerprint.js.map +1 -0
- package/dist/utils/guards.d.ts +8 -0
- package/dist/utils/guards.d.ts.map +1 -0
- package/dist/utils/guards.js +38 -0
- package/dist/utils/guards.js.map +1 -0
- package/docs/architecture/overview.md +59 -0
- package/docs/decisions/0001-runtime-and-state.md +21 -0
- package/docs/decisions/0002-modular-boundaries.md +16 -0
- package/docs/plans/initial-build.md +18 -0
- package/docs/product/specification.md +87 -0
- package/docs/security/data-boundary.md +27 -0
- package/package.json +89 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { RunFailed, XTrawlError } from "../domain/errors.js";
|
|
2
|
+
import { queryHash } from "../query/hash.js";
|
|
3
|
+
import { ExecutionRunner } from "../runner/runner.js";
|
|
4
|
+
import { saveRows } from "../output/writer.js";
|
|
5
|
+
import { searchOutputName } from "../output/names.js";
|
|
6
|
+
export async function collectSearch(context, query, options) {
|
|
7
|
+
const request = withDefaultBounds({ ...options, ...(query ? { searchQuery: query } : {}) });
|
|
8
|
+
const hash = queryHash({ operation: "search", request });
|
|
9
|
+
const run = context.storage.runs.create("search", hash);
|
|
10
|
+
const usefulSplits = request.limit === undefined
|
|
11
|
+
? context.config.searchSplits
|
|
12
|
+
: Math.min(context.config.searchSplits, Math.max(1, Math.ceil(request.limit / context.config.apiPageSize)));
|
|
13
|
+
const tasks = splitSearchTasks(request, usefulSplits, context.config.schedulerMinIntervalMs);
|
|
14
|
+
const tweets = new Map();
|
|
15
|
+
const cursors = new Map();
|
|
16
|
+
let poolRetries = 0;
|
|
17
|
+
let limitReached = false;
|
|
18
|
+
try {
|
|
19
|
+
for (const task of tasks) {
|
|
20
|
+
if (!request.resume)
|
|
21
|
+
continue;
|
|
22
|
+
const saved = context.storage.checkpoints.get(task.id);
|
|
23
|
+
if (saved?.root)
|
|
24
|
+
cursors.set(task.id, saved.root);
|
|
25
|
+
}
|
|
26
|
+
const runner = new ExecutionRunner({
|
|
27
|
+
concurrency: workerCount(context, tasks.length),
|
|
28
|
+
maxAttempts: 1,
|
|
29
|
+
});
|
|
30
|
+
const outcome = await runner.run(tasks, async (task) => {
|
|
31
|
+
let cursor = cursors.get(task.id);
|
|
32
|
+
let emptyPages = 0;
|
|
33
|
+
while (!limitReached) {
|
|
34
|
+
const page = await context.pool.execute("search", ({ session }) => context.engine.search(session, task.request, cursor), {
|
|
35
|
+
countTweets: (value) => value.tweets.length,
|
|
36
|
+
onRetry: () => {
|
|
37
|
+
poolRetries += 1;
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
let added = 0;
|
|
41
|
+
for (const tweet of page.tweets) {
|
|
42
|
+
const previousSize = tweets.size;
|
|
43
|
+
if (!tweets.has(tweet.tweetId))
|
|
44
|
+
tweets.set(tweet.tweetId, tweet);
|
|
45
|
+
if (tweets.size > previousSize)
|
|
46
|
+
added += 1;
|
|
47
|
+
if (request.limit !== undefined && tweets.size >= request.limit) {
|
|
48
|
+
limitReached = true;
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
emptyPages = added === 0 ? emptyPages + 1 : 0;
|
|
53
|
+
if (page.cursor) {
|
|
54
|
+
cursors.set(task.id, page.cursor);
|
|
55
|
+
if (request.resume)
|
|
56
|
+
context.storage.checkpoints.save(task.id, { root: page.cursor });
|
|
57
|
+
}
|
|
58
|
+
if (!page.cursor || emptyPages >= (request.maxEmptyPages ?? context.config.maxEmptyPages))
|
|
59
|
+
break;
|
|
60
|
+
cursor = page.cursor;
|
|
61
|
+
}
|
|
62
|
+
if (!limitReached)
|
|
63
|
+
context.storage.checkpoints.clear(task.id);
|
|
64
|
+
});
|
|
65
|
+
if (outcome.failed.length > 0 && (context.config.strict || tweets.size === 0))
|
|
66
|
+
throw failureFor("Search", outcome.failed);
|
|
67
|
+
const collected = request.limit === undefined ? [...tweets.values()] : [...tweets.values()].slice(0, request.limit);
|
|
68
|
+
if (request.save)
|
|
69
|
+
await saveRows(request.saveName ??
|
|
70
|
+
searchOutputName(request.searchQuery, request.since, request.until, request.fromUsers), collected, {
|
|
71
|
+
directory: request.saveDir ?? context.config.saveDir,
|
|
72
|
+
format: request.saveFormat ?? context.config.saveFormat,
|
|
73
|
+
append: true,
|
|
74
|
+
});
|
|
75
|
+
const result = {
|
|
76
|
+
tweets: collected,
|
|
77
|
+
stats: {
|
|
78
|
+
tweetsCount: collected.length,
|
|
79
|
+
tasksTotal: tasks.length,
|
|
80
|
+
tasksDone: outcome.complete.length,
|
|
81
|
+
tasksFailed: outcome.failed.length,
|
|
82
|
+
retries: outcome.retries + poolRetries,
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
context.storage.runs.finalize(run.id, "complete");
|
|
86
|
+
return result;
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
context.storage.runs.finalize(run.id, "failed", error instanceof Error ? { name: error.name, message: error.message } : error);
|
|
90
|
+
throw error instanceof XTrawlError
|
|
91
|
+
? error
|
|
92
|
+
: new RunFailed(error instanceof Error ? error.message : String(error));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
export function withDefaultBounds(request, now = new Date()) {
|
|
96
|
+
const until = request.until ?? dateOnly(now);
|
|
97
|
+
const since = request.since ?? dateOnly(new Date(now.getTime() - 30 * 24 * 60 * 60 * 1_000));
|
|
98
|
+
return { ...request, since, until };
|
|
99
|
+
}
|
|
100
|
+
export function splitSearchTasks(request, requestedSplits, minIntervalMs) {
|
|
101
|
+
const start = parseBound(request.since);
|
|
102
|
+
const end = parseBound(request.until);
|
|
103
|
+
if (start === undefined || end === undefined || end <= start)
|
|
104
|
+
return [{ id: queryHash({ operation: "search_task", request }), request }];
|
|
105
|
+
const duration = end - start;
|
|
106
|
+
const count = Math.max(1, Math.min(requestedSplits, Math.floor(duration / minIntervalMs)));
|
|
107
|
+
return Array.from({ length: count }, (_, index) => {
|
|
108
|
+
const since = start + Math.floor((duration * index) / count);
|
|
109
|
+
const until = start + Math.floor((duration * (index + 1)) / count);
|
|
110
|
+
const taskRequest = { ...request, since: searchDate(since), until: searchDate(until) };
|
|
111
|
+
return { id: queryHash({ operation: "search_task", request: taskRequest }), request: taskRequest };
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
function workerCount(context, taskCount) {
|
|
115
|
+
return Math.max(1, Math.min(context.config.concurrency, context.pool.summary.eligible, taskCount));
|
|
116
|
+
}
|
|
117
|
+
function failureFor(label, failed) {
|
|
118
|
+
const first = failed[0]?.error;
|
|
119
|
+
return first instanceof Error
|
|
120
|
+
? first
|
|
121
|
+
: new RunFailed(`${label} failed for ${failed.length} task(s): ${String(first)}`);
|
|
122
|
+
}
|
|
123
|
+
function parseBound(value) {
|
|
124
|
+
if (!value)
|
|
125
|
+
return undefined;
|
|
126
|
+
const normalized = value.endsWith("_UTC") ? value.replace("_UTC", "Z").replace("_", "T") : value;
|
|
127
|
+
const parsed = Date.parse(normalized);
|
|
128
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
129
|
+
}
|
|
130
|
+
function searchDate(value) {
|
|
131
|
+
return new Date(value)
|
|
132
|
+
.toISOString()
|
|
133
|
+
.replace("T", "_")
|
|
134
|
+
.replace(/\.000Z$/u, "_UTC");
|
|
135
|
+
}
|
|
136
|
+
function dateOnly(value) {
|
|
137
|
+
return value.toISOString().slice(0, 10);
|
|
138
|
+
}
|
|
139
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/client/search.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAK7D,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AActD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAsB,EACtB,KAAa,EACb,OAAsB;IAEtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,GAAG,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5F,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACxD,MAAM,YAAY,GAChB,OAAO,CAAC,KAAK,KAAK,SAAS;QACzB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY;QAC7B,CAAC,CAAC,IAAI,CAAC,GAAG,CACN,OAAO,CAAC,MAAM,CAAC,YAAY,EAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CACnE,CAAC;IACR,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,SAAS;YAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACvD,IAAI,KAAK,EAAE,IAAI;gBAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAa;YAC7C,WAAW,EAAE,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC;YAC/C,WAAW,EAAE,CAAC;SACf,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrD,IAAI,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClC,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,YAAY,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CACrC,QAAQ,EACR,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,EACrE;oBACE,WAAW,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3C,OAAO,EAAE,GAAG,EAAE;wBACZ,WAAW,IAAI,CAAC,CAAC;oBACnB,CAAC;iBACF,CACF,CAAC;gBACF,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChC,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC;oBACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;wBAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBACjE,IAAI,MAAM,CAAC,IAAI,GAAG,YAAY;wBAAE,KAAK,IAAI,CAAC,CAAC;oBAC3C,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;wBAChE,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM;oBACR,CAAC;gBACH,CAAC;gBACD,UAAU,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBAClC,IAAI,OAAO,CAAC,MAAM;wBAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBACvF,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,UAAU,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC;oBAAE,MAAM;gBACjG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACvB,CAAC;YACD,IAAI,CAAC,YAAY;gBAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC;YAC3E,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,SAAS,GACb,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACpG,IAAI,OAAO,CAAC,IAAI;YACd,MAAM,QAAQ,CACZ,OAAO,CAAC,QAAQ;gBACd,gBAAgB,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,EACxF,SAAS,EACT;gBACE,SAAS,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO;gBACpD,MAAM,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU;gBACvD,MAAM,EAAE,IAAI;aACb,CACF,CAAC;QACJ,MAAM,MAAM,GAAiB;YAC3B,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE;gBACL,WAAW,EAAE,SAAS,CAAC,MAAM;gBAC7B,UAAU,EAAE,KAAK,CAAC,MAAM;gBACxB,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAClC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM;gBAClC,OAAO,EAAE,OAAO,CAAC,OAAO,GAAG,WAAW;aACvC;SACF,CAAC;QACF,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QAClD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAC3B,GAAG,CAAC,EAAE,EACN,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAC9E,CAAC;QACF,MAAM,KAAK,YAAY,WAAW;YAChC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,IAAI,SAAS,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAsB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;IAC7F,OAAO,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAsB,EACtB,eAAuB,EACvB,aAAqB;IAErB,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,KAAK,KAAK,SAAS,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,IAAI,KAAK;QAC1D,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,GAAG,GAAG,KAAK,CAAC;IAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3F,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QAChD,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QACnE,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACvF,OAAO,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IACrG,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,OAAsB,EAAE,SAAiB;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;AACrG,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,MAA8C;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC;IAC/B,OAAO,KAAK,YAAY,KAAK;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,KAAK,eAAe,MAAM,CAAC,MAAM,aAAa,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,UAAU,CAAC,KAAyB;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC;SACnB,WAAW,EAAE;SACb,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;SACjB,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAW;IAC3B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ConfigInput, ClientConfig } from "../config/types.js";
|
|
2
|
+
import type { AccountInput } from "../domain/accounts.js";
|
|
3
|
+
import type { TransactionIdSource } from "../transport/transaction-id.js";
|
|
4
|
+
import type { SessionFactory } from "../domain/http.js";
|
|
5
|
+
export interface ClientOptions extends ConfigInput {
|
|
6
|
+
readonly cookiesFile?: string;
|
|
7
|
+
readonly accountsFile?: string;
|
|
8
|
+
readonly envFile?: string;
|
|
9
|
+
readonly authToken?: string;
|
|
10
|
+
readonly csrfToken?: string;
|
|
11
|
+
readonly cookies?: unknown;
|
|
12
|
+
readonly accounts?: readonly AccountInput[];
|
|
13
|
+
readonly provision?: boolean;
|
|
14
|
+
readonly sessionFactory?: SessionFactory;
|
|
15
|
+
readonly transactionIdSource?: TransactionIdSource;
|
|
16
|
+
}
|
|
17
|
+
export interface ClientInspection {
|
|
18
|
+
readonly config: ClientConfig;
|
|
19
|
+
readonly accounts: readonly Record<string, unknown>[];
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/client/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,WAAW,aAAc,SAAQ,WAAW;IAChD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC5C,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,CAAC;IACzC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CACpD;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;CACvD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/client/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"defaults.d.ts","sourceRoot":"","sources":["../../src/config/defaults.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,eAAO,MAAM,cAAc,EAAE,YAqC5B,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export const DEFAULT_CONFIG = {
|
|
2
|
+
dbPath: "xtrawl_state.db",
|
|
3
|
+
concurrency: 5,
|
|
4
|
+
saveDir: "outputs",
|
|
5
|
+
saveFormat: "csv",
|
|
6
|
+
apiHttpMode: "auto",
|
|
7
|
+
dailyRequestsLimit: 30,
|
|
8
|
+
dailyTweetsLimit: 600,
|
|
9
|
+
maxEmptyPages: 1,
|
|
10
|
+
apiPageSize: 20,
|
|
11
|
+
searchSplits: 5,
|
|
12
|
+
schedulerMinIntervalMs: 300_000,
|
|
13
|
+
minDelayMs: 2_000,
|
|
14
|
+
leaseTtlMs: 120_000,
|
|
15
|
+
leaseHeartbeatMs: 30_000,
|
|
16
|
+
cooldownDefaultMs: 120_000,
|
|
17
|
+
transientCooldownMs: 120_000,
|
|
18
|
+
authCooldownMs: 30 * 24 * 60 * 60 * 1_000,
|
|
19
|
+
cooldownJitterMs: 10_000,
|
|
20
|
+
requestsPerMinute: 30,
|
|
21
|
+
retryBaseMs: 1_000,
|
|
22
|
+
retryMaxMs: 30_000,
|
|
23
|
+
maxTaskAttempts: 3,
|
|
24
|
+
maxFallbackAttempts: 3,
|
|
25
|
+
maxAccountSwitches: 2,
|
|
26
|
+
proxyCheckOnLease: true,
|
|
27
|
+
proxyCheckUrl: "https://x.com/robots.txt",
|
|
28
|
+
proxyCheckTimeoutMs: 10_000,
|
|
29
|
+
profileTimelineAllowAnonymous: false,
|
|
30
|
+
manifestTtlMs: 3_600_000,
|
|
31
|
+
manifestUpdateOnInit: false,
|
|
32
|
+
manifestScrapeOnInit: false,
|
|
33
|
+
transactionIdEnabled: true,
|
|
34
|
+
transactionIdTtlMs: 6 * 60 * 60 * 1_000,
|
|
35
|
+
strict: false,
|
|
36
|
+
bearerToken: "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
|
|
37
|
+
};
|
|
38
|
+
//# sourceMappingURL=defaults.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"defaults.js","sourceRoot":"","sources":["../../src/config/defaults.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,cAAc,GAAiB;IAC1C,MAAM,EAAE,iBAAiB;IACzB,WAAW,EAAE,CAAC;IACd,OAAO,EAAE,SAAS;IAClB,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,MAAM;IACnB,kBAAkB,EAAE,EAAE;IACtB,gBAAgB,EAAE,GAAG;IACrB,aAAa,EAAE,CAAC;IAChB,WAAW,EAAE,EAAE;IACf,YAAY,EAAE,CAAC;IACf,sBAAsB,EAAE,OAAO;IAC/B,UAAU,EAAE,KAAK;IACjB,UAAU,EAAE,OAAO;IACnB,gBAAgB,EAAE,MAAM;IACxB,iBAAiB,EAAE,OAAO;IAC1B,mBAAmB,EAAE,OAAO;IAC5B,cAAc,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK;IACzC,gBAAgB,EAAE,MAAM;IACxB,iBAAiB,EAAE,EAAE;IACrB,WAAW,EAAE,KAAK;IAClB,UAAU,EAAE,MAAM;IAClB,eAAe,EAAE,CAAC;IAClB,mBAAmB,EAAE,CAAC;IACtB,kBAAkB,EAAE,CAAC;IACrB,iBAAiB,EAAE,IAAI;IACvB,aAAa,EAAE,0BAA0B;IACzC,mBAAmB,EAAE,MAAM;IAC3B,6BAA6B,EAAE,KAAK;IACpC,aAAa,EAAE,SAAS;IACxB,oBAAoB,EAAE,KAAK;IAC3B,oBAAoB,EAAE,KAAK;IAC3B,oBAAoB,EAAE,IAAI;IAC1B,kBAAkB,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK;IACvC,MAAM,EAAE,KAAK;IACb,WAAW,EACT,0GAA0G;CAC7G,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { AccountInput } from "../domain/accounts.js";
|
|
2
|
+
export declare function readDotenv(path: string): Promise<Record<string, string>>;
|
|
3
|
+
export declare function loadAccountFromEnvironment(path: string): Promise<AccountInput[]>;
|
|
4
|
+
export declare function loadAccountFromEnvironmentSync(path: string): AccountInput[];
|
|
5
|
+
//# sourceMappingURL=environment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../src/config/environment.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAE1D,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAa9E;AAED,wBAAsB,0BAA0B,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAGtF;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,EAAE,CAE3E"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { asString } from "../utils/guards.js";
|
|
4
|
+
export async function readDotenv(path) {
|
|
5
|
+
const text = await readFile(path, "utf8");
|
|
6
|
+
const values = {};
|
|
7
|
+
for (const raw of text.split(/\r?\n/u)) {
|
|
8
|
+
const line = raw.trim();
|
|
9
|
+
if (!line || line.startsWith("#"))
|
|
10
|
+
continue;
|
|
11
|
+
const separator = line.indexOf("=");
|
|
12
|
+
if (separator <= 0)
|
|
13
|
+
continue;
|
|
14
|
+
const key = line.slice(0, separator).trim();
|
|
15
|
+
const rawValue = line.slice(separator + 1).trim();
|
|
16
|
+
values[key] = rawValue.replace(/^['"]|['"]$/gu, "");
|
|
17
|
+
}
|
|
18
|
+
return values;
|
|
19
|
+
}
|
|
20
|
+
export async function loadAccountFromEnvironment(path) {
|
|
21
|
+
const values = await readDotenv(path);
|
|
22
|
+
return accountFromValues(values);
|
|
23
|
+
}
|
|
24
|
+
export function loadAccountFromEnvironmentSync(path) {
|
|
25
|
+
return accountFromValues(parseDotenv(readFileSync(path, "utf8")));
|
|
26
|
+
}
|
|
27
|
+
function accountFromValues(values) {
|
|
28
|
+
const username = asString(values.USERNAME) ?? asString(values.USER);
|
|
29
|
+
const email = asString(values.EMAIL);
|
|
30
|
+
const authToken = asString(values.AUTH_TOKEN) ?? asString(values.X_AUTH_TOKEN);
|
|
31
|
+
const csrfToken = asString(values.CT0) ?? asString(values.CSRF) ?? asString(values.X_CSRF_TOKEN);
|
|
32
|
+
if (!username && !email && !authToken)
|
|
33
|
+
return [];
|
|
34
|
+
return [
|
|
35
|
+
{
|
|
36
|
+
...(username ? { username } : {}),
|
|
37
|
+
...(email ? { email } : {}),
|
|
38
|
+
...(asString(values.PASSWORD) ? { password: asString(values.PASSWORD) } : {}),
|
|
39
|
+
...(asString(values.EMAIL_PASSWORD) ? { emailPassword: asString(values.EMAIL_PASSWORD) } : {}),
|
|
40
|
+
...(asString(values.TWO_FA) ? { twoFactorSecret: asString(values.TWO_FA) } : {}),
|
|
41
|
+
...(authToken ? { authToken } : {}),
|
|
42
|
+
...(csrfToken ? { csrfToken } : {}),
|
|
43
|
+
},
|
|
44
|
+
];
|
|
45
|
+
}
|
|
46
|
+
function parseDotenv(text) {
|
|
47
|
+
const values = {};
|
|
48
|
+
for (const raw of text.split(/\r?\n/u)) {
|
|
49
|
+
const line = raw.trim();
|
|
50
|
+
if (!line || line.startsWith("#"))
|
|
51
|
+
continue;
|
|
52
|
+
const separator = line.indexOf("=");
|
|
53
|
+
if (separator <= 0)
|
|
54
|
+
continue;
|
|
55
|
+
values[line.slice(0, separator).trim()] = line
|
|
56
|
+
.slice(separator + 1)
|
|
57
|
+
.trim()
|
|
58
|
+
.replace(/^['"]|['"]$/gu, "");
|
|
59
|
+
}
|
|
60
|
+
return values;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=environment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"environment.js","sourceRoot":"","sources":["../../src/config/environment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAG9C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,IAAI,CAAC;YAAE,SAAS;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,IAAY;IAC3D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IACtC,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,IAAY;IACzD,OAAO,iBAAiB,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA8B;IACvD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACjG,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IACjD,OAAO;QACL;YACE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7E,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9F,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChF,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,IAAI,CAAC;YAAE,SAAS;QAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;aAC3C,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;aACpB,IAAI,EAAE;aACN,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { ProxySettings } from "../domain/accounts.js";
|
|
2
|
+
import type { ApiHttpMode } from "./validation.js";
|
|
3
|
+
export interface ClientConfig {
|
|
4
|
+
readonly dbPath: string;
|
|
5
|
+
readonly proxy?: string | ProxySettings;
|
|
6
|
+
readonly concurrency: number;
|
|
7
|
+
readonly saveDir: string;
|
|
8
|
+
readonly saveFormat: "csv" | "json" | "both";
|
|
9
|
+
readonly apiHttpMode: ApiHttpMode;
|
|
10
|
+
readonly apiHttpImpersonate?: string;
|
|
11
|
+
readonly apiUserAgent?: string;
|
|
12
|
+
readonly dailyRequestsLimit: number;
|
|
13
|
+
readonly dailyTweetsLimit: number;
|
|
14
|
+
readonly maxEmptyPages: number;
|
|
15
|
+
readonly apiPageSize: number;
|
|
16
|
+
readonly searchSplits: number;
|
|
17
|
+
readonly schedulerMinIntervalMs: number;
|
|
18
|
+
readonly minDelayMs: number;
|
|
19
|
+
readonly leaseTtlMs: number;
|
|
20
|
+
readonly leaseHeartbeatMs: number;
|
|
21
|
+
readonly cooldownDefaultMs: number;
|
|
22
|
+
readonly transientCooldownMs: number;
|
|
23
|
+
readonly authCooldownMs: number;
|
|
24
|
+
readonly cooldownJitterMs: number;
|
|
25
|
+
readonly requestsPerMinute: number;
|
|
26
|
+
readonly retryBaseMs: number;
|
|
27
|
+
readonly retryMaxMs: number;
|
|
28
|
+
readonly maxTaskAttempts: number;
|
|
29
|
+
readonly maxFallbackAttempts: number;
|
|
30
|
+
readonly maxAccountSwitches: number;
|
|
31
|
+
readonly proxyCheckOnLease: boolean;
|
|
32
|
+
readonly proxyCheckUrl: string;
|
|
33
|
+
readonly proxyCheckTimeoutMs: number;
|
|
34
|
+
readonly profileTimelineAllowAnonymous: boolean;
|
|
35
|
+
readonly manifestUrl?: string;
|
|
36
|
+
readonly manifestTtlMs: number;
|
|
37
|
+
readonly manifestUpdateOnInit: boolean;
|
|
38
|
+
readonly manifestScrapeOnInit: boolean;
|
|
39
|
+
readonly transactionIdEnabled: boolean;
|
|
40
|
+
readonly transactionIdTtlMs: number;
|
|
41
|
+
readonly strict: boolean;
|
|
42
|
+
readonly bearerToken: string;
|
|
43
|
+
}
|
|
44
|
+
export type ConfigInput = Partial<Omit<ClientConfig, "proxy">> & {
|
|
45
|
+
readonly proxy?: string | ProxySettings;
|
|
46
|
+
};
|
|
47
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IACxC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;IAC7C,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IACpC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,6BAA6B,EAAE,OAAO,CAAC;IAChD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,GAAG;IAC/D,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;CACzC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/config/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ProxySettings } from "../domain/accounts.js";
|
|
2
|
+
import type { ClientConfig, ConfigInput } from "./types.js";
|
|
3
|
+
export type ApiHttpMode = "auto" | "async" | "sync";
|
|
4
|
+
export declare function normalizeProxyPayload(value: unknown): string | ProxySettings | undefined;
|
|
5
|
+
export declare function validateConfig(input?: ConfigInput): ClientConfig;
|
|
6
|
+
//# sourceMappingURL=validation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/config/validation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG3D,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AA4CpD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,SAAS,CA4BxF;AAED,wBAAgB,cAAc,CAAC,KAAK,GAAE,WAAgB,GAAG,YAAY,CA8DpE"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { ConfigError } from "../domain/errors.js";
|
|
2
|
+
import { asBoolean, asInteger, asString, isRecord } from "../utils/guards.js";
|
|
3
|
+
import { DEFAULT_CONFIG } from "./defaults.js";
|
|
4
|
+
const CONFIG_KEYS = [
|
|
5
|
+
"dbPath",
|
|
6
|
+
"proxy",
|
|
7
|
+
"concurrency",
|
|
8
|
+
"saveDir",
|
|
9
|
+
"saveFormat",
|
|
10
|
+
"apiHttpMode",
|
|
11
|
+
"apiHttpImpersonate",
|
|
12
|
+
"apiUserAgent",
|
|
13
|
+
"dailyRequestsLimit",
|
|
14
|
+
"dailyTweetsLimit",
|
|
15
|
+
"maxEmptyPages",
|
|
16
|
+
"apiPageSize",
|
|
17
|
+
"searchSplits",
|
|
18
|
+
"schedulerMinIntervalMs",
|
|
19
|
+
"minDelayMs",
|
|
20
|
+
"leaseTtlMs",
|
|
21
|
+
"leaseHeartbeatMs",
|
|
22
|
+
"cooldownDefaultMs",
|
|
23
|
+
"transientCooldownMs",
|
|
24
|
+
"authCooldownMs",
|
|
25
|
+
"cooldownJitterMs",
|
|
26
|
+
"requestsPerMinute",
|
|
27
|
+
"retryBaseMs",
|
|
28
|
+
"retryMaxMs",
|
|
29
|
+
"maxTaskAttempts",
|
|
30
|
+
"maxFallbackAttempts",
|
|
31
|
+
"maxAccountSwitches",
|
|
32
|
+
"proxyCheckOnLease",
|
|
33
|
+
"proxyCheckUrl",
|
|
34
|
+
"proxyCheckTimeoutMs",
|
|
35
|
+
"profileTimelineAllowAnonymous",
|
|
36
|
+
"manifestUrl",
|
|
37
|
+
"manifestTtlMs",
|
|
38
|
+
"manifestUpdateOnInit",
|
|
39
|
+
"manifestScrapeOnInit",
|
|
40
|
+
"transactionIdEnabled",
|
|
41
|
+
"transactionIdTtlMs",
|
|
42
|
+
"strict",
|
|
43
|
+
"bearerToken",
|
|
44
|
+
];
|
|
45
|
+
export function normalizeProxyPayload(value) {
|
|
46
|
+
if (value === null || value === undefined || value === "")
|
|
47
|
+
return undefined;
|
|
48
|
+
if (typeof value === "string") {
|
|
49
|
+
const text = value.trim();
|
|
50
|
+
if (!text)
|
|
51
|
+
return undefined;
|
|
52
|
+
if (text.startsWith("{") || text.startsWith("[")) {
|
|
53
|
+
try {
|
|
54
|
+
return normalizeProxyPayload(JSON.parse(text));
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return text;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return text.includes("://") || text.includes(":") ? text : undefined;
|
|
61
|
+
}
|
|
62
|
+
if (!isRecord(value))
|
|
63
|
+
return undefined;
|
|
64
|
+
const proxy = {
|
|
65
|
+
...(asString(value.http) ? { http: asString(value.http) } : {}),
|
|
66
|
+
...(asString(value.https) ? { https: asString(value.https) } : {}),
|
|
67
|
+
...(asString(value.host) ? { host: asString(value.host) } : {}),
|
|
68
|
+
...(asInteger(value.port) > 0 ? { port: asInteger(value.port) } : {}),
|
|
69
|
+
...(value.scheme === "http" || value.scheme === "https" || value.scheme === "socks5"
|
|
70
|
+
? { scheme: value.scheme }
|
|
71
|
+
: {}),
|
|
72
|
+
...(asString(value.username) ? { username: asString(value.username) } : {}),
|
|
73
|
+
...(asString(value.password) ? { password: asString(value.password) } : {}),
|
|
74
|
+
};
|
|
75
|
+
if (proxy.http || proxy.https || proxy.host)
|
|
76
|
+
return proxy;
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
export function validateConfig(input = {}) {
|
|
80
|
+
const merged = { ...DEFAULT_CONFIG, ...configOverrides(input) };
|
|
81
|
+
const mode = merged.apiHttpMode ?? DEFAULT_CONFIG.apiHttpMode;
|
|
82
|
+
if (mode !== "auto" && mode !== "async" && mode !== "sync") {
|
|
83
|
+
throw new ConfigError(`Unsupported HTTP mode: ${String(mode)}`);
|
|
84
|
+
}
|
|
85
|
+
const proxy = normalizeProxyPayload(merged.proxy);
|
|
86
|
+
if (merged.proxy !== undefined && proxy === undefined) {
|
|
87
|
+
throw new ConfigError("Invalid proxy. Use a URL or a host/port object.");
|
|
88
|
+
}
|
|
89
|
+
const positiveFields = [
|
|
90
|
+
"concurrency",
|
|
91
|
+
"dailyRequestsLimit",
|
|
92
|
+
"dailyTweetsLimit",
|
|
93
|
+
"maxEmptyPages",
|
|
94
|
+
"apiPageSize",
|
|
95
|
+
"searchSplits",
|
|
96
|
+
"schedulerMinIntervalMs",
|
|
97
|
+
"leaseTtlMs",
|
|
98
|
+
"manifestTtlMs",
|
|
99
|
+
"requestsPerMinute",
|
|
100
|
+
"maxTaskAttempts",
|
|
101
|
+
"maxFallbackAttempts",
|
|
102
|
+
"proxyCheckTimeoutMs",
|
|
103
|
+
"transactionIdTtlMs",
|
|
104
|
+
];
|
|
105
|
+
for (const field of positiveFields) {
|
|
106
|
+
if (!Number.isInteger(merged[field]) || Number(merged[field]) < 1) {
|
|
107
|
+
throw new ConfigError(`${field} must be a positive integer`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
if (!Number.isInteger(merged.maxAccountSwitches) || merged.maxAccountSwitches < 0) {
|
|
111
|
+
throw new ConfigError("maxAccountSwitches must be a non-negative integer");
|
|
112
|
+
}
|
|
113
|
+
if (merged.apiPageSize > 100)
|
|
114
|
+
throw new ConfigError("apiPageSize must be at most 100");
|
|
115
|
+
try {
|
|
116
|
+
const proxyCheckUrl = new URL(merged.proxyCheckUrl);
|
|
117
|
+
if (proxyCheckUrl.protocol !== "http:" && proxyCheckUrl.protocol !== "https:")
|
|
118
|
+
throw new Error();
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
throw new ConfigError("proxyCheckUrl must be an HTTP(S) URL");
|
|
122
|
+
}
|
|
123
|
+
if (merged.minDelayMs < 0 ||
|
|
124
|
+
merged.leaseHeartbeatMs < 0 ||
|
|
125
|
+
merged.cooldownDefaultMs < 0 ||
|
|
126
|
+
merged.transientCooldownMs < 0 ||
|
|
127
|
+
merged.authCooldownMs < 0 ||
|
|
128
|
+
merged.cooldownJitterMs < 0 ||
|
|
129
|
+
merged.retryBaseMs < 0 ||
|
|
130
|
+
merged.retryMaxMs < 0) {
|
|
131
|
+
throw new ConfigError("delay and cooldown values cannot be negative");
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
...merged,
|
|
135
|
+
proxy,
|
|
136
|
+
apiHttpMode: mode,
|
|
137
|
+
proxyCheckOnLease: asBoolean(merged.proxyCheckOnLease),
|
|
138
|
+
transactionIdEnabled: asBoolean(merged.transactionIdEnabled),
|
|
139
|
+
strict: asBoolean(merged.strict),
|
|
140
|
+
bearerToken: asString(merged.bearerToken) ?? DEFAULT_CONFIG.bearerToken,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
function configOverrides(input) {
|
|
144
|
+
const entries = CONFIG_KEYS.flatMap((key) => input[key] === undefined ? [] : [[key, input[key]]]);
|
|
145
|
+
return Object.fromEntries(entries);
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=validation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validation.js","sourceRoot":"","sources":["../../src/config/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAK/C,MAAM,WAAW,GAAG;IAClB,QAAQ;IACR,OAAO;IACP,aAAa;IACb,SAAS;IACT,YAAY;IACZ,aAAa;IACb,oBAAoB;IACpB,cAAc;IACd,oBAAoB;IACpB,kBAAkB;IAClB,eAAe;IACf,aAAa;IACb,cAAc;IACd,wBAAwB;IACxB,YAAY;IACZ,YAAY;IACZ,kBAAkB;IAClB,mBAAmB;IACnB,qBAAqB;IACrB,gBAAgB;IAChB,kBAAkB;IAClB,mBAAmB;IACnB,aAAa;IACb,YAAY;IACZ,iBAAiB;IACjB,qBAAqB;IACrB,oBAAoB;IACpB,mBAAmB;IACnB,eAAe;IACf,qBAAqB;IACrB,+BAA+B;IAC/B,aAAa;IACb,eAAe;IACf,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,oBAAoB;IACpB,QAAQ;IACR,aAAa;CACqC,CAAC;AAErD,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAC5E,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO,SAAS,CAAC;QAC5B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,OAAO,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACvE,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACvC,MAAM,KAAK,GAAkB;QAC3B,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;YAClF,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE;YAC1B,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAC;IACF,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAAqB,EAAE;IACpD,MAAM,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;IAChE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,IAAI,cAAc,CAAC,WAAW,CAAC;IAC9D,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3D,MAAM,IAAI,WAAW,CAAC,0BAA0B,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,MAAM,KAAK,GAAG,qBAAqB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtD,MAAM,IAAI,WAAW,CAAC,iDAAiD,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,cAAc,GAA8B;QAChD,aAAa;QACb,oBAAoB;QACpB,kBAAkB;QAClB,eAAe;QACf,aAAa;QACb,cAAc;QACd,wBAAwB;QACxB,YAAY;QACZ,eAAe;QACf,mBAAmB;QACnB,iBAAiB;QACjB,qBAAqB;QACrB,qBAAqB;QACrB,oBAAoB;KACrB,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,WAAW,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,kBAAkB,CAAC,IAAI,MAAM,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;QAClF,MAAM,IAAI,WAAW,CAAC,mDAAmD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,GAAG,GAAG;QAAE,MAAM,IAAI,WAAW,CAAC,iCAAiC,CAAC,CAAC;IACvF,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACpD,IAAI,aAAa,CAAC,QAAQ,KAAK,OAAO,IAAI,aAAa,CAAC,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,EAAE,CAAC;IACnG,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,WAAW,CAAC,sCAAsC,CAAC,CAAC;IAChE,CAAC;IACD,IACE,MAAM,CAAC,UAAU,GAAG,CAAC;QACrB,MAAM,CAAC,gBAAgB,GAAG,CAAC;QAC3B,MAAM,CAAC,iBAAiB,GAAG,CAAC;QAC5B,MAAM,CAAC,mBAAmB,GAAG,CAAC;QAC9B,MAAM,CAAC,cAAc,GAAG,CAAC;QACzB,MAAM,CAAC,gBAAgB,GAAG,CAAC;QAC3B,MAAM,CAAC,WAAW,GAAG,CAAC;QACtB,MAAM,CAAC,UAAU,GAAG,CAAC,EACrB,CAAC;QACD,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;IACxE,CAAC;IACD,OAAO;QACL,GAAG,MAAM;QACT,KAAK;QACL,WAAW,EAAE,IAAI;QACjB,iBAAiB,EAAE,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;QACtD,oBAAoB,EAAE,SAAS,CAAC,MAAM,CAAC,oBAAoB,CAAC;QAC5D,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;QAChC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,WAAW;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAkB;IACzC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1C,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAW,CAC/D,CAAC;IACF,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAgB,CAAC;AACpD,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
export type CookieMap = Readonly<Record<string, string>>;
|
|
2
|
+
export interface AuthMaterial {
|
|
3
|
+
readonly authToken: string;
|
|
4
|
+
readonly csrfToken: string;
|
|
5
|
+
readonly bearerToken: string;
|
|
6
|
+
readonly cookies: CookieMap;
|
|
7
|
+
}
|
|
8
|
+
export interface AccountInput {
|
|
9
|
+
readonly username?: string;
|
|
10
|
+
readonly password?: string;
|
|
11
|
+
readonly email?: string;
|
|
12
|
+
readonly emailPassword?: string;
|
|
13
|
+
readonly twoFactorSecret?: string;
|
|
14
|
+
readonly authToken?: string;
|
|
15
|
+
readonly csrfToken?: string;
|
|
16
|
+
readonly cookies?: CookieMap;
|
|
17
|
+
readonly proxy?: string | ProxySettings;
|
|
18
|
+
}
|
|
19
|
+
export interface ProxySettings {
|
|
20
|
+
readonly http?: string;
|
|
21
|
+
readonly https?: string;
|
|
22
|
+
readonly host?: string;
|
|
23
|
+
readonly port?: number;
|
|
24
|
+
readonly scheme?: "http" | "https" | "socks5";
|
|
25
|
+
readonly username?: string;
|
|
26
|
+
readonly password?: string;
|
|
27
|
+
}
|
|
28
|
+
export type AccountStatus = "healthy" | "cooling_down" | "unusable";
|
|
29
|
+
export interface AccountRecord {
|
|
30
|
+
readonly id?: number;
|
|
31
|
+
readonly username: string;
|
|
32
|
+
readonly password?: string;
|
|
33
|
+
readonly email?: string;
|
|
34
|
+
readonly emailPassword?: string;
|
|
35
|
+
readonly twoFactorSecret?: string;
|
|
36
|
+
readonly authToken?: string;
|
|
37
|
+
readonly csrfToken?: string;
|
|
38
|
+
readonly cookies: CookieMap;
|
|
39
|
+
readonly bearerToken?: string;
|
|
40
|
+
readonly proxy?: string | ProxySettings;
|
|
41
|
+
readonly status?: number;
|
|
42
|
+
readonly availableUntil?: number;
|
|
43
|
+
readonly dailyRequests?: number;
|
|
44
|
+
readonly dailyTweets?: number;
|
|
45
|
+
readonly totalTweets?: number;
|
|
46
|
+
readonly lastResetDate?: string;
|
|
47
|
+
readonly lastUsed?: number;
|
|
48
|
+
readonly leaseId?: string;
|
|
49
|
+
readonly leaseExpiresAt?: number;
|
|
50
|
+
readonly lastErrorCode?: number;
|
|
51
|
+
readonly cooldownReason?: string;
|
|
52
|
+
}
|
|
53
|
+
export interface AccountLease extends AccountRecord {
|
|
54
|
+
readonly leaseId: string;
|
|
55
|
+
readonly leaseExpiresAt: number;
|
|
56
|
+
}
|
|
57
|
+
export interface AccountSummary {
|
|
58
|
+
readonly dbPath: string;
|
|
59
|
+
readonly total: number;
|
|
60
|
+
readonly eligible: number;
|
|
61
|
+
readonly unusable: number;
|
|
62
|
+
readonly coolingDown: number;
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=accounts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"accounts.d.ts","sourceRoot":"","sources":["../../src/domain/accounts.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAEzD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC;IAC7B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;CACzC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,cAAc,GAAG,UAAU,CAAC;AAEpE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IACxC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,YAAa,SAAQ,aAAa;IACjD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"accounts.js","sourceRoot":"","sources":["../../src/domain/accounts.ts"],"names":[],"mappings":""}
|