xtrawl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DOCUMENTATION.md +774 -0
- package/README.md +232 -0
- package/dist/auth/bootstrap.d.ts +4 -0
- package/dist/auth/bootstrap.d.ts.map +1 -0
- package/dist/auth/bootstrap.js +29 -0
- package/dist/auth/bootstrap.js.map +1 -0
- package/dist/auth/cookies.d.ts +7 -0
- package/dist/auth/cookies.d.ts.map +1 -0
- package/dist/auth/cookies.js +83 -0
- package/dist/auth/cookies.js.map +1 -0
- package/dist/auth/loaders.d.ts +6 -0
- package/dist/auth/loaders.d.ts.map +1 -0
- package/dist/auth/loaders.js +69 -0
- package/dist/auth/loaders.js.map +1 -0
- package/dist/auth/material.d.ts +3 -0
- package/dist/auth/material.d.ts.map +1 -0
- package/dist/auth/material.js +20 -0
- package/dist/auth/material.js.map +1 -0
- package/dist/auth/records.d.ts +4 -0
- package/dist/auth/records.d.ts.map +1 -0
- package/dist/auth/records.js +38 -0
- package/dist/auth/records.js.map +1 -0
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +30 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +85 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/parser.d.ts +22 -0
- package/dist/cli/parser.d.ts.map +1 -0
- package/dist/cli/parser.js +275 -0
- package/dist/cli/parser.js.map +1 -0
- package/dist/client/client.d.ts +29 -0
- package/dist/client/client.d.ts.map +1 -0
- package/dist/client/client.js +192 -0
- package/dist/client/client.js.map +1 -0
- package/dist/client/collectors.d.ts +15 -0
- package/dist/client/collectors.d.ts.map +1 -0
- package/dist/client/collectors.js +185 -0
- package/dist/client/collectors.js.map +1 -0
- package/dist/client/database.d.ts +56 -0
- package/dist/client/database.d.ts.map +1 -0
- package/dist/client/database.js +146 -0
- package/dist/client/database.js.map +1 -0
- package/dist/client/profiles.d.ts +5 -0
- package/dist/client/profiles.d.ts.map +1 -0
- package/dist/client/profiles.js +36 -0
- package/dist/client/profiles.js.map +1 -0
- package/dist/client/search.d.ts +21 -0
- package/dist/client/search.d.ts.map +1 -0
- package/dist/client/search.js +139 -0
- package/dist/client/search.js.map +1 -0
- package/dist/client/types.d.ts +21 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/client/types.js +2 -0
- package/dist/client/types.js.map +1 -0
- package/dist/config/defaults.d.ts +3 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +38 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/environment.d.ts +5 -0
- package/dist/config/environment.d.ts.map +1 -0
- package/dist/config/environment.js +62 -0
- package/dist/config/environment.js.map +1 -0
- package/dist/config/types.d.ts +47 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/config/validation.d.ts +6 -0
- package/dist/config/validation.d.ts.map +1 -0
- package/dist/config/validation.js +147 -0
- package/dist/config/validation.js.map +1 -0
- package/dist/domain/accounts.d.ts +64 -0
- package/dist/domain/accounts.d.ts.map +1 -0
- package/dist/domain/accounts.js +2 -0
- package/dist/domain/accounts.js.map +1 -0
- package/dist/domain/errors.d.ts +61 -0
- package/dist/domain/errors.d.ts.map +1 -0
- package/dist/domain/errors.js +89 -0
- package/dist/domain/errors.js.map +1 -0
- package/dist/domain/http.d.ts +36 -0
- package/dist/domain/http.d.ts.map +1 -0
- package/dist/domain/http.js +2 -0
- package/dist/domain/http.js.map +1 -0
- package/dist/domain/manifest.d.ts +23 -0
- package/dist/domain/manifest.d.ts.map +1 -0
- package/dist/domain/manifest.js +2 -0
- package/dist/domain/manifest.js.map +1 -0
- package/dist/domain/records.d.ts +67 -0
- package/dist/domain/records.d.ts.map +1 -0
- package/dist/domain/records.js +2 -0
- package/dist/domain/records.js.map +1 -0
- package/dist/domain/requests.d.ts +87 -0
- package/dist/domain/requests.d.ts.map +1 -0
- package/dist/domain/requests.js +2 -0
- package/dist/domain/requests.js.map +1 -0
- package/dist/engine/api-engine.d.ts +25 -0
- package/dist/engine/api-engine.d.ts.map +1 -0
- package/dist/engine/api-engine.js +98 -0
- package/dist/engine/api-engine.js.map +1 -0
- package/dist/engine/extractors.d.ts +27 -0
- package/dist/engine/extractors.d.ts.map +1 -0
- package/dist/engine/extractors.js +248 -0
- package/dist/engine/extractors.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/manifest/default-manifest.d.ts +3 -0
- package/dist/manifest/default-manifest.d.ts.map +1 -0
- package/dist/manifest/default-manifest.js +46 -0
- package/dist/manifest/default-manifest.js.map +1 -0
- package/dist/manifest/model.d.ts +3 -0
- package/dist/manifest/model.d.ts.map +1 -0
- package/dist/manifest/model.js +48 -0
- package/dist/manifest/model.js.map +1 -0
- package/dist/manifest/provider.d.ts +19 -0
- package/dist/manifest/provider.d.ts.map +1 -0
- package/dist/manifest/provider.js +94 -0
- package/dist/manifest/provider.js.map +1 -0
- package/dist/manifest/scraper.d.ts +11 -0
- package/dist/manifest/scraper.d.ts.map +1 -0
- package/dist/manifest/scraper.js +130 -0
- package/dist/manifest/scraper.js.map +1 -0
- package/dist/output/csv-writer.d.ts +2 -0
- package/dist/output/csv-writer.d.ts.map +1 -0
- package/dist/output/csv-writer.js +94 -0
- package/dist/output/csv-writer.js.map +1 -0
- package/dist/output/json-writer.d.ts +2 -0
- package/dist/output/json-writer.d.ts.map +1 -0
- package/dist/output/json-writer.js +29 -0
- package/dist/output/json-writer.js.map +1 -0
- package/dist/output/names.d.ts +4 -0
- package/dist/output/names.d.ts.map +1 -0
- package/dist/output/names.js +22 -0
- package/dist/output/names.js.map +1 -0
- package/dist/output/writer.d.ts +7 -0
- package/dist/output/writer.d.ts.map +1 -0
- package/dist/output/writer.js +33 -0
- package/dist/output/writer.js.map +1 -0
- package/dist/pool/account-pool.d.ts +29 -0
- package/dist/pool/account-pool.d.ts.map +1 -0
- package/dist/pool/account-pool.js +145 -0
- package/dist/pool/account-pool.js.map +1 -0
- package/dist/pool/cooldown.d.ts +18 -0
- package/dist/pool/cooldown.d.ts.map +1 -0
- package/dist/pool/cooldown.js +38 -0
- package/dist/pool/cooldown.js.map +1 -0
- package/dist/pool/limiter.d.ts +11 -0
- package/dist/pool/limiter.d.ts.map +1 -0
- package/dist/pool/limiter.js +27 -0
- package/dist/pool/limiter.js.map +1 -0
- package/dist/query/builder.d.ts +19 -0
- package/dist/query/builder.d.ts.map +1 -0
- package/dist/query/builder.js +110 -0
- package/dist/query/builder.js.map +1 -0
- package/dist/query/hash.d.ts +2 -0
- package/dist/query/hash.d.ts.map +1 -0
- package/dist/query/hash.js +6 -0
- package/dist/query/hash.js.map +1 -0
- package/dist/query/normalize.d.ts +39 -0
- package/dist/query/normalize.d.ts.map +1 -0
- package/dist/query/normalize.js +176 -0
- package/dist/query/normalize.js.map +1 -0
- package/dist/query/targets.d.ts +11 -0
- package/dist/query/targets.d.ts.map +1 -0
- package/dist/query/targets.js +96 -0
- package/dist/query/targets.js.map +1 -0
- package/dist/runner/index.d.ts +4 -0
- package/dist/runner/index.d.ts.map +1 -0
- package/dist/runner/index.js +4 -0
- package/dist/runner/index.js.map +1 -0
- package/dist/runner/retry.d.ts +7 -0
- package/dist/runner/retry.d.ts.map +1 -0
- package/dist/runner/retry.js +26 -0
- package/dist/runner/retry.js.map +1 -0
- package/dist/runner/runner.d.ts +20 -0
- package/dist/runner/runner.d.ts.map +1 -0
- package/dist/runner/runner.js +37 -0
- package/dist/runner/runner.js.map +1 -0
- package/dist/runner/task-queue.d.ts +19 -0
- package/dist/runner/task-queue.d.ts.map +1 -0
- package/dist/runner/task-queue.js +58 -0
- package/dist/runner/task-queue.js.map +1 -0
- package/dist/storage/account-repository.d.ts +50 -0
- package/dist/storage/account-repository.d.ts.map +1 -0
- package/dist/storage/account-repository.js +171 -0
- package/dist/storage/account-repository.js.map +1 -0
- package/dist/storage/account-row.d.ts +4 -0
- package/dist/storage/account-row.d.ts.map +1 -0
- package/dist/storage/account-row.js +46 -0
- package/dist/storage/account-row.js.map +1 -0
- package/dist/storage/checkpoint-repository.d.ts +10 -0
- package/dist/storage/checkpoint-repository.d.ts.map +1 -0
- package/dist/storage/checkpoint-repository.js +30 -0
- package/dist/storage/checkpoint-repository.js.map +1 -0
- package/dist/storage/database.d.ts +18 -0
- package/dist/storage/database.d.ts.map +1 -0
- package/dist/storage/database.js +42 -0
- package/dist/storage/database.js.map +1 -0
- package/dist/storage/index.d.ts +15 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +18 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/manifest-repository.d.ts +9 -0
- package/dist/storage/manifest-repository.d.ts.map +1 -0
- package/dist/storage/manifest-repository.js +22 -0
- package/dist/storage/manifest-repository.js.map +1 -0
- package/dist/storage/run-repository.d.ts +24 -0
- package/dist/storage/run-repository.d.ts.map +1 -0
- package/dist/storage/run-repository.js +51 -0
- package/dist/storage/run-repository.js.map +1 -0
- package/dist/storage/schema.d.ts +3 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +53 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/transport/graphql.d.ts +8 -0
- package/dist/transport/graphql.d.ts.map +1 -0
- package/dist/transport/graphql.js +104 -0
- package/dist/transport/graphql.js.map +1 -0
- package/dist/transport/proxy.d.ts +5 -0
- package/dist/transport/proxy.d.ts.map +1 -0
- package/dist/transport/proxy.js +36 -0
- package/dist/transport/proxy.js.map +1 -0
- package/dist/transport/session.d.ts +25 -0
- package/dist/transport/session.d.ts.map +1 -0
- package/dist/transport/session.js +138 -0
- package/dist/transport/session.js.map +1 -0
- package/dist/transport/transaction-id.d.ts +20 -0
- package/dist/transport/transaction-id.d.ts.map +1 -0
- package/dist/transport/transaction-id.js +74 -0
- package/dist/transport/transaction-id.js.map +1 -0
- package/dist/utils/fingerprint.d.ts +3 -0
- package/dist/utils/fingerprint.d.ts.map +1 -0
- package/dist/utils/fingerprint.js +20 -0
- package/dist/utils/fingerprint.js.map +1 -0
- package/dist/utils/guards.d.ts +8 -0
- package/dist/utils/guards.d.ts.map +1 -0
- package/dist/utils/guards.js +38 -0
- package/dist/utils/guards.js.map +1 -0
- package/docs/architecture/overview.md +59 -0
- package/docs/decisions/0001-runtime-and-state.md +21 -0
- package/docs/decisions/0002-modular-boundaries.md +16 -0
- package/docs/plans/initial-build.md +18 -0
- package/docs/product/specification.md +87 -0
- package/docs/security/data-boundary.md +27 -0
- package/package.json +89 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { asString } from "../utils/guards.js";
|
|
2
|
+
const OPERATION_NAMES = {
|
|
3
|
+
SearchTimeline: "search_timeline",
|
|
4
|
+
UserByScreenName: "user_lookup_screen_name",
|
|
5
|
+
UserTweets: "profile_timeline",
|
|
6
|
+
Followers: "followers",
|
|
7
|
+
Following: "following",
|
|
8
|
+
BlueVerifiedFollowers: "verified_followers",
|
|
9
|
+
TweetResultByRestId: "tweet_result",
|
|
10
|
+
};
|
|
11
|
+
export function extractManifestFromJavascript(source, fallback) {
|
|
12
|
+
const operationFeatures = extractOperationFeaturesFromJavascript(source);
|
|
13
|
+
return {
|
|
14
|
+
...fallback,
|
|
15
|
+
queryIds: {
|
|
16
|
+
...fallback.queryIds,
|
|
17
|
+
...extractOperationQueryIdsFromJavascript(source),
|
|
18
|
+
},
|
|
19
|
+
operationFeatures: mergeOperationFeatures(fallback.operationFeatures, operationFeatures),
|
|
20
|
+
version: asString(fallback.version) ?? "live",
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export function extractOperationQueryIdsFromJavascript(source) {
|
|
24
|
+
const queryIds = {};
|
|
25
|
+
for (const [operationName, key] of Object.entries(OPERATION_NAMES)) {
|
|
26
|
+
const queryFirst = new RegExp(`queryId\\s*:\\s*["']([^"']+)["']\\s*,\\s*operationName\\s*:\\s*["']${operationName}["']`, "u");
|
|
27
|
+
const operationFirst = new RegExp(`operationName\\s*:\\s*["']${operationName}["']\\s*,\\s*queryId\\s*:\\s*["']([^"']+)["']`, "u");
|
|
28
|
+
const id = source.match(queryFirst)?.[1] ?? source.match(operationFirst)?.[1];
|
|
29
|
+
if (id)
|
|
30
|
+
queryIds[key] = id;
|
|
31
|
+
}
|
|
32
|
+
return queryIds;
|
|
33
|
+
}
|
|
34
|
+
export function extractOperationFeaturesFromJavascript(source) {
|
|
35
|
+
const out = {};
|
|
36
|
+
for (const [operationName, key] of Object.entries(OPERATION_NAMES)) {
|
|
37
|
+
const marker = new RegExp(`operationName\\s*:\\s*["']${operationName}["']`, "gu");
|
|
38
|
+
const match = marker.exec(source);
|
|
39
|
+
if (!match)
|
|
40
|
+
continue;
|
|
41
|
+
const start = match.index;
|
|
42
|
+
const next = source.indexOf("operationName:", start + match[0].length);
|
|
43
|
+
const window = source.slice(start, next > start ? Math.min(next, start + 50_000) : start + 50_000);
|
|
44
|
+
const featureStart = window.indexOf("featureSwitches:[");
|
|
45
|
+
if (featureStart < 0)
|
|
46
|
+
continue;
|
|
47
|
+
const featureEnd = window.indexOf("]", featureStart);
|
|
48
|
+
if (featureEnd < 0)
|
|
49
|
+
continue;
|
|
50
|
+
const names = [...window.slice(featureStart, featureEnd).matchAll(/["']([^"']+)["']/gu)]
|
|
51
|
+
.map((item) => item[1])
|
|
52
|
+
.filter((item) => Boolean(item));
|
|
53
|
+
if (names.length > 0)
|
|
54
|
+
out[key] = Object.fromEntries(names.map((name) => [name, false]));
|
|
55
|
+
}
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
export async function scrapeManifestFromWeb(base, options = {}) {
|
|
59
|
+
const fetcher = options.fetcher ?? fetch;
|
|
60
|
+
const scriptHeaders = {
|
|
61
|
+
"User-Agent": "Mozilla/5.0",
|
|
62
|
+
};
|
|
63
|
+
const pageHeaders = {
|
|
64
|
+
...scriptHeaders,
|
|
65
|
+
...(options.authToken ? { Cookie: `auth_token=${options.authToken}` } : {}),
|
|
66
|
+
};
|
|
67
|
+
const response = await fetcher("https://x.com/home", { headers: pageHeaders, redirect: "follow" });
|
|
68
|
+
if (!response.ok)
|
|
69
|
+
throw new Error(`Manifest page failed with status ${response.status}`);
|
|
70
|
+
const html = await response.text();
|
|
71
|
+
const scripts = [...new Set([...html.matchAll(/<script[^>]+src=["']([^"']+)["']/giu)])]
|
|
72
|
+
.map((match) => match[1])
|
|
73
|
+
.filter((item) => Boolean(item))
|
|
74
|
+
.map((source) => safeScriptUrl(source))
|
|
75
|
+
.filter((source) => Boolean(source))
|
|
76
|
+
.sort((left, right) => scriptPriority(left) - scriptPriority(right))
|
|
77
|
+
.slice(0, options.maxScripts ?? 20);
|
|
78
|
+
const discovered = {};
|
|
79
|
+
const discoveredFeatures = {};
|
|
80
|
+
for (const url of scripts) {
|
|
81
|
+
const bundle = await fetcher(url, { headers: scriptHeaders, redirect: "follow" });
|
|
82
|
+
if (!bundle.ok)
|
|
83
|
+
continue;
|
|
84
|
+
const source = await bundle.text();
|
|
85
|
+
Object.assign(discovered, extractOperationQueryIdsFromJavascript(source));
|
|
86
|
+
Object.assign(discoveredFeatures, extractOperationFeaturesFromJavascript(source));
|
|
87
|
+
if (Object.keys(discovered).length === Object.keys(OPERATION_NAMES).length)
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
if (Object.keys(discovered).length === 0) {
|
|
91
|
+
throw new Error("Manifest bundles did not contain any supported operation identifiers");
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
...base,
|
|
95
|
+
version: "web-live",
|
|
96
|
+
queryIds: { ...base.queryIds, ...discovered },
|
|
97
|
+
operationFeatures: mergeOperationFeatures(base.operationFeatures, discoveredFeatures),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function mergeOperationFeatures(base, discovered) {
|
|
101
|
+
const out = {};
|
|
102
|
+
if (base)
|
|
103
|
+
for (const [operation, features] of Object.entries(base))
|
|
104
|
+
out[operation] = { ...features };
|
|
105
|
+
for (const [operation, features] of Object.entries(discovered))
|
|
106
|
+
out[operation] = { ...(out[operation] ?? {}), ...features };
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
function safeScriptUrl(source) {
|
|
110
|
+
try {
|
|
111
|
+
const url = new URL(source, "https://x.com");
|
|
112
|
+
if (url.protocol !== "https:")
|
|
113
|
+
return undefined;
|
|
114
|
+
if (url.hostname !== "x.com" && url.hostname !== "abs.twimg.com")
|
|
115
|
+
return undefined;
|
|
116
|
+
return url.toString();
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function scriptPriority(source) {
|
|
123
|
+
const path = new URL(source).pathname;
|
|
124
|
+
if (/\/responsive-web\/client-web\/main\.[^/]+\.js$/u.test(path))
|
|
125
|
+
return 0;
|
|
126
|
+
if (/\/main\.[^/]+\.js$/u.test(path))
|
|
127
|
+
return 1;
|
|
128
|
+
return 2;
|
|
129
|
+
}
|
|
130
|
+
//# sourceMappingURL=scraper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/manifest/scraper.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,eAAe,GAA2B;IAC9C,cAAc,EAAE,iBAAiB;IACjC,gBAAgB,EAAE,yBAAyB;IAC3C,UAAU,EAAE,kBAAkB;IAC9B,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,qBAAqB,EAAE,oBAAoB;IAC3C,mBAAmB,EAAE,cAAc;CACpC,CAAC;AAQF,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,QAAyB;IACrF,MAAM,iBAAiB,GAAG,sCAAsC,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,QAAQ;QACX,QAAQ,EAAE;YACR,GAAG,QAAQ,CAAC,QAAQ;YACpB,GAAG,sCAAsC,CAAC,MAAM,CAAC;SAClD;QACD,iBAAiB,EAAE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;QACxF,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,MAAc;IACnE,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,UAAU,GAAG,IAAI,MAAM,CAC3B,sEAAsE,aAAa,MAAM,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,MAAM,CAC/B,6BAA6B,aAAa,+CAA+C,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,IAAI,EAAE;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,MAAc;IAEd,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,6BAA6B,aAAa,MAAM,EAAE,IAAI,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACnG,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,YAAY,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrD,IAAI,UAAU,GAAG,CAAC;YAAE,SAAS;QAC7B,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;aACrF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAqB,EACrB,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IACzC,MAAM,aAAa,GAAG;QACpB,YAAY,EAAE,aAAa;KAC5B,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,GAAG,aAAa;QAChB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IACnG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC;SACpF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC/C,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;SACtC,MAAM,CAAC,CAAC,MAAM,EAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;SACnE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,kBAAkB,GAA4C,EAAE,CAAC;IACvE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,SAAS;QACzB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM;YAAE,MAAM;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO;QACL,GAAG,IAAI;QACP,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,UAAU,EAAE;QAC7C,iBAAiB,EAAE,sBAAsB,CAAC,IAAI,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KACtF,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,IAA0C,EAC1C,UAAuE;IAEvE,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,IAAI,IAAI;QACN,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;YACtD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAI,QAAoC,EAAE,CAAC;IAClE,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;QAC5D,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAChD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,eAAe;YAAE,OAAO,SAAS,CAAC;QACnF,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,MAAc;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;IACtC,IAAI,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAGA,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EACxC,MAAM,UAAQ,GACb,OAAO,CAAC,IAAI,CAAC,CAiBf"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
export async function writeCsv(path, rows, append = false) {
|
|
4
|
+
if (rows.length === 0)
|
|
5
|
+
return;
|
|
6
|
+
const existing = append && (await exists(path)) ? await readFile(path, "utf8") : "";
|
|
7
|
+
const parsedExisting = existing ? parseCsv(existing) : [];
|
|
8
|
+
const currentHeaders = parsedExisting[0] ?? [];
|
|
9
|
+
const headers = mergeHeaders(currentHeaders, rows.flatMap((row) => Object.keys(row)));
|
|
10
|
+
const existingRows = parsedExisting
|
|
11
|
+
.slice(1)
|
|
12
|
+
.map((values) => Object.fromEntries(currentHeaders.map((header, index) => [header, values[index] ?? ""])));
|
|
13
|
+
const content = renderCsv(headers, [...existingRows, ...rows]);
|
|
14
|
+
await mkdir(dirname(path), { recursive: true });
|
|
15
|
+
await writeFile(path, content, "utf8");
|
|
16
|
+
}
|
|
17
|
+
function renderCsv(headers, rows) {
|
|
18
|
+
const body = rows.map((row) => headers.map((header) => escapeCsv(flattenValue(row[header]))).join(","));
|
|
19
|
+
return `${headers.map(escapeCsv).join(",")}\n${body.join("\n")}\n`;
|
|
20
|
+
}
|
|
21
|
+
function mergeHeaders(existing, incoming) {
|
|
22
|
+
const preferred = [
|
|
23
|
+
"tweetId",
|
|
24
|
+
"timestamp",
|
|
25
|
+
"username",
|
|
26
|
+
"name",
|
|
27
|
+
"text",
|
|
28
|
+
"likes",
|
|
29
|
+
"retweets",
|
|
30
|
+
"comments",
|
|
31
|
+
"tweetUrl",
|
|
32
|
+
];
|
|
33
|
+
const all = [...new Set([...existing, ...incoming])];
|
|
34
|
+
return [...preferred.filter((key) => all.includes(key)), ...all.filter((key) => !preferred.includes(key))];
|
|
35
|
+
}
|
|
36
|
+
function parseCsv(value) {
|
|
37
|
+
const rows = [];
|
|
38
|
+
let row = [];
|
|
39
|
+
let field = "";
|
|
40
|
+
let quoted = false;
|
|
41
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
42
|
+
const character = value[index];
|
|
43
|
+
const next = value[index + 1];
|
|
44
|
+
if (character === '"') {
|
|
45
|
+
if (quoted && next === '"') {
|
|
46
|
+
field += '"';
|
|
47
|
+
index += 1;
|
|
48
|
+
}
|
|
49
|
+
else
|
|
50
|
+
quoted = !quoted;
|
|
51
|
+
}
|
|
52
|
+
else if (character === "," && !quoted) {
|
|
53
|
+
row.push(field);
|
|
54
|
+
field = "";
|
|
55
|
+
}
|
|
56
|
+
else if ((character === "\n" || character === "\r") && !quoted) {
|
|
57
|
+
if (character === "\r" && next === "\n")
|
|
58
|
+
index += 1;
|
|
59
|
+
row.push(field);
|
|
60
|
+
field = "";
|
|
61
|
+
if (row.some((item) => item.length > 0))
|
|
62
|
+
rows.push(row);
|
|
63
|
+
row = [];
|
|
64
|
+
}
|
|
65
|
+
else {
|
|
66
|
+
field += character;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (field || row.length > 0) {
|
|
70
|
+
row.push(field);
|
|
71
|
+
rows.push(row);
|
|
72
|
+
}
|
|
73
|
+
return rows;
|
|
74
|
+
}
|
|
75
|
+
function flattenValue(value) {
|
|
76
|
+
if (value === undefined || value === null)
|
|
77
|
+
return "";
|
|
78
|
+
if (typeof value === "string")
|
|
79
|
+
return value;
|
|
80
|
+
return JSON.stringify(value);
|
|
81
|
+
}
|
|
82
|
+
function escapeCsv(value) {
|
|
83
|
+
return /[",\n\r]/u.test(value) ? `"${value.replaceAll('"', '""')}"` : value;
|
|
84
|
+
}
|
|
85
|
+
async function exists(path) {
|
|
86
|
+
try {
|
|
87
|
+
await access(path);
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=csv-writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-writer.js","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,IAAwC,EACxC,MAAM,GAAG,KAAK;IAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,YAAY,CAC1B,cAAc,EACd,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CACxC,CAAC;IACF,MAAM,YAAY,GAAG,cAAc;SAChC,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACd,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CACzF,CAAC;IACJ,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,GAAG,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,SAAS,CAAC,OAA0B,EAAE,IAAwC;IACrF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACxG,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACrE,CAAC;AAED,SAAS,YAAY,CAAC,QAA2B,EAAE,QAA2B;IAC5E,MAAM,SAAS,GAAG;QAChB,SAAS;QACT,WAAW;QACX,UAAU;QACV,MAAM;QACN,MAAM;QACN,OAAO;QACP,UAAU;QACV,UAAU;QACV,UAAU;KACX,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACtB,IAAI,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAC3B,KAAK,IAAI,GAAG,CAAC;gBACb,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;;gBAAM,MAAM,GAAG,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;aAAM,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjE,IAAI,SAAS,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;gBAAE,KAAK,IAAI,CAAC,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,GAAG,GAAG,EAAE,CAAC;QACX,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,SAAS,CAAC;QACrB,CAAC;IACH,CAAC;IACD,IAAI,KAAK,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-writer.d.ts","sourceRoot":"","sources":["../../src/output/json-writer.ts"],"names":[],"mappings":"AAGA,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAarG"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
export async function writeJson(path, rows, append = false) {
|
|
4
|
+
if (rows.length === 0)
|
|
5
|
+
return;
|
|
6
|
+
let output = [...rows];
|
|
7
|
+
if (append && (await exists(path))) {
|
|
8
|
+
try {
|
|
9
|
+
const current = JSON.parse(await readFile(path, "utf8"));
|
|
10
|
+
if (Array.isArray(current))
|
|
11
|
+
output = [...current, ...rows];
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
output = [...rows];
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
await mkdir(dirname(path), { recursive: true });
|
|
18
|
+
await writeFile(path, `${JSON.stringify(output, null, 2)}\n`, "utf8");
|
|
19
|
+
}
|
|
20
|
+
async function exists(path) {
|
|
21
|
+
try {
|
|
22
|
+
await access(path);
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=json-writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-writer.js","sourceRoot":"","sources":["../../src/output/json-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,IAAwB,EAAE,MAAM,GAAG,KAAK;IACpF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,IAAI,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACvB,IAAI,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;YACpE,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;gBAAE,MAAM,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { TargetInput } from "../domain/requests.js";
|
|
2
|
+
export declare function searchOutputName(query: string | undefined, since: string | undefined, until: string | undefined, fromUsers: readonly string[] | undefined): string;
|
|
3
|
+
export declare function targetOutputName(operation: string, targets: readonly TargetInput[]): string;
|
|
4
|
+
//# sourceMappingURL=names.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/output/names.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GACvC,MAAM,CAIR;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,MAAM,CAM3F"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export function searchOutputName(query, since, until, fromUsers) {
|
|
2
|
+
const source = query?.trim() || fromUsers?.slice(0, 3).join("_") || "search";
|
|
3
|
+
const parts = [safePart(source), since?.slice(0, 10), until?.slice(0, 10)].filter(Boolean);
|
|
4
|
+
return parts.join("_");
|
|
5
|
+
}
|
|
6
|
+
export function targetOutputName(operation, targets) {
|
|
7
|
+
const names = targets
|
|
8
|
+
.slice(0, 3)
|
|
9
|
+
.map((target) => target.username ?? target.userId)
|
|
10
|
+
.filter((value) => Boolean(value));
|
|
11
|
+
return safePart(names.length > 0 ? `${operation}_${names.join("_")}` : operation);
|
|
12
|
+
}
|
|
13
|
+
function safePart(value) {
|
|
14
|
+
const cleaned = value
|
|
15
|
+
.split(/\s+/u)
|
|
16
|
+
.slice(0, 3)
|
|
17
|
+
.join("_")
|
|
18
|
+
.replace(/[^A-Za-z0-9_.-]+/gu, "_")
|
|
19
|
+
.replace(/^_+|_+$/gu, "");
|
|
20
|
+
return cleaned || "output";
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=names.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"names.js","sourceRoot":"","sources":["../../src/output/names.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAC9B,KAAyB,EACzB,KAAyB,EACzB,KAAyB,EACzB,SAAwC;IAExC,MAAM,MAAM,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,SAAS,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;IAC7E,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3F,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,OAA+B;IACjF,MAAM,KAAK,GAAG,OAAO;SAClB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC;SACjD,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,OAAO,GAAG,KAAK;SAClB,KAAK,CAAC,MAAM,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,IAAI,CAAC,GAAG,CAAC;SACT,OAAO,CAAC,oBAAoB,EAAE,GAAG,CAAC;SAClC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC5B,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export type OutputFormat = "csv" | "json" | "both";
|
|
2
|
+
export declare function saveRows(name: string, rows: readonly unknown[], options: {
|
|
3
|
+
readonly directory: string;
|
|
4
|
+
readonly format: OutputFormat;
|
|
5
|
+
readonly append?: boolean;
|
|
6
|
+
}): Promise<void>;
|
|
7
|
+
//# sourceMappingURL=writer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/output/writer.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;AAEnD,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,OAAO,EAAE,EACxB,OAAO,EAAE;IAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAChG,OAAO,CAAC,IAAI,CAAC,CAMf"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { writeCsv } from "./csv-writer.js";
|
|
4
|
+
import { writeJson } from "./json-writer.js";
|
|
5
|
+
export async function saveRows(name, rows, options) {
|
|
6
|
+
await mkdir(options.directory, { recursive: true });
|
|
7
|
+
if (options.format === "json" || options.format === "both")
|
|
8
|
+
await writeJson(join(options.directory, `${name}.json`), rows, options.append);
|
|
9
|
+
if (options.format === "csv" || options.format === "both")
|
|
10
|
+
await writeCsv(join(options.directory, `${name}.csv`), rows.map(toFlatRow), options.append);
|
|
11
|
+
}
|
|
12
|
+
function toFlatRow(value) {
|
|
13
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
14
|
+
return { value };
|
|
15
|
+
const row = value;
|
|
16
|
+
if ("tweetId" in row) {
|
|
17
|
+
const tweet = row;
|
|
18
|
+
return {
|
|
19
|
+
tweetId: tweet.tweetId,
|
|
20
|
+
timestamp: tweet.timestamp,
|
|
21
|
+
username: tweet.user.screenName,
|
|
22
|
+
name: tweet.user.name,
|
|
23
|
+
text: tweet.text,
|
|
24
|
+
likes: tweet.likes,
|
|
25
|
+
retweets: tweet.retweets,
|
|
26
|
+
comments: tweet.comments,
|
|
27
|
+
tweetUrl: tweet.tweetUrl,
|
|
28
|
+
media: tweet.media.imageLinks,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
return Object.fromEntries(Object.entries(row).filter(([key]) => key !== "raw"));
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../src/output/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7C,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,IAAwB,EACxB,OAAiG;IAEjG,MAAM,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;QACxD,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;QACvD,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAChG,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAClF,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,GAA6B,CAAC;QAC5C,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI;YACrB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC;AAClF,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { AccountLease } from "../domain/accounts.js";
|
|
2
|
+
import type { ClientConfig } from "../config/types.js";
|
|
3
|
+
import type { HttpSession } from "../domain/http.js";
|
|
4
|
+
import type { AccountRepository } from "../storage/account-repository.js";
|
|
5
|
+
import type { SessionBuilder } from "../transport/session.js";
|
|
6
|
+
export interface PoolExecutionContext {
|
|
7
|
+
readonly account: AccountLease;
|
|
8
|
+
readonly session: HttpSession;
|
|
9
|
+
}
|
|
10
|
+
export interface PoolExecutionOptions<T> {
|
|
11
|
+
readonly countTweets?: (value: T) => number;
|
|
12
|
+
readonly onRetry?: (error: unknown, attempt: number) => void;
|
|
13
|
+
readonly maxAccountSwitches?: number;
|
|
14
|
+
}
|
|
15
|
+
export type AccountRepair = (account: AccountLease) => Promise<boolean>;
|
|
16
|
+
export declare class AccountPool {
|
|
17
|
+
private readonly repository;
|
|
18
|
+
private readonly sessions;
|
|
19
|
+
private readonly config;
|
|
20
|
+
private readonly repairAccount?;
|
|
21
|
+
private readonly limiters;
|
|
22
|
+
constructor(repository: AccountRepository, sessions: SessionBuilder, config: ClientConfig, repairAccount?: AccountRepair | undefined);
|
|
23
|
+
execute<T>(label: string, operation: (context: PoolExecutionContext) => Promise<T>, options?: PoolExecutionOptions<T>): Promise<T>;
|
|
24
|
+
get summary(): import("../domain/accounts.js").AccountSummary;
|
|
25
|
+
private releaseFailure;
|
|
26
|
+
private startHeartbeat;
|
|
27
|
+
private limiterFor;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=account-pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"account-pool.d.ts","sourceRoot":"","sources":["../../src/pool/account-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAU1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAE1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAI9D,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,CAAC;IAC5C,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExE,qBAAa,WAAW;IAIpB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IANjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyC;gBAG/C,UAAU,EAAE,iBAAiB,EAC7B,QAAQ,EAAE,cAAc,EACxB,MAAM,EAAE,YAAY,EACpB,aAAa,CAAC,EAAE,aAAa,YAAA;IAGnC,OAAO,CAAC,CAAC,EACpB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,CAAC,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAC,CAAC,CAAC,EACxD,OAAO,GAAE,oBAAoB,CAAC,CAAC,CAAM,GACpC,OAAO,CAAC,CAAC,CAAC;IA4Db,IAAW,OAAO,mDAEjB;IAED,OAAO,CAAC,cAAc;IA+BtB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,UAAU;CAQnB"}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { AccountPoolExhausted, AccountSessionBuildError, AuthError, XTrawlError, NetworkError, RateLimitError, ProxyError, } from "../domain/errors.js";
|
|
2
|
+
import { computeCooldown } from "./cooldown.js";
|
|
3
|
+
import { isRecord } from "../utils/guards.js";
|
|
4
|
+
import { TokenBucketLimiter, sleep } from "./limiter.js";
|
|
5
|
+
export class AccountPool {
|
|
6
|
+
repository;
|
|
7
|
+
sessions;
|
|
8
|
+
config;
|
|
9
|
+
repairAccount;
|
|
10
|
+
limiters = new Map();
|
|
11
|
+
constructor(repository, sessions, config, repairAccount) {
|
|
12
|
+
this.repository = repository;
|
|
13
|
+
this.sessions = sessions;
|
|
14
|
+
this.config = config;
|
|
15
|
+
this.repairAccount = repairAccount;
|
|
16
|
+
}
|
|
17
|
+
async execute(label, operation, options = {}) {
|
|
18
|
+
const maxAttempts = this.config.maxTaskAttempts;
|
|
19
|
+
const seenAccounts = new Set();
|
|
20
|
+
let lastError;
|
|
21
|
+
let repairs = 0;
|
|
22
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
23
|
+
const account = this.repository.lease({ requireAuthMaterial: true });
|
|
24
|
+
if (!account) {
|
|
25
|
+
if (lastError)
|
|
26
|
+
throw lastError;
|
|
27
|
+
throw new AccountPoolExhausted(`No eligible account for ${label}.`);
|
|
28
|
+
}
|
|
29
|
+
const accountKey = String(account.id ?? account.username);
|
|
30
|
+
seenAccounts.add(accountKey);
|
|
31
|
+
const switchLimit = options.maxAccountSwitches ?? this.config.maxAccountSwitches;
|
|
32
|
+
if (seenAccounts.size > switchLimit + 1) {
|
|
33
|
+
this.repository.release(account.leaseId, { status: "healthy" });
|
|
34
|
+
if (lastError)
|
|
35
|
+
throw lastError;
|
|
36
|
+
throw new AccountPoolExhausted(`Account-switch limit reached for ${label}.`);
|
|
37
|
+
}
|
|
38
|
+
let session;
|
|
39
|
+
let operationStarted = false;
|
|
40
|
+
const heartbeat = this.startHeartbeat(account);
|
|
41
|
+
try {
|
|
42
|
+
if (this.config.proxyCheckOnLease)
|
|
43
|
+
await this.sessions.assertProxyHealthy(account, {
|
|
44
|
+
url: this.config.proxyCheckUrl,
|
|
45
|
+
timeoutMs: this.config.proxyCheckTimeoutMs,
|
|
46
|
+
});
|
|
47
|
+
session = this.sessions.forAccount(account);
|
|
48
|
+
await this.limiterFor(account).acquire();
|
|
49
|
+
operationStarted = true;
|
|
50
|
+
const value = await operation({ account, session });
|
|
51
|
+
this.repository.recordUsage(account.leaseId, 1, Math.max(0, options.countTweets?.(value) ?? countTweets(value)));
|
|
52
|
+
this.repository.release(account.leaseId, { status: "healthy" });
|
|
53
|
+
return value;
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
lastError = error;
|
|
57
|
+
if (operationStarted)
|
|
58
|
+
this.repository.recordUsage(account.leaseId, 1, 0);
|
|
59
|
+
this.releaseFailure(account, error);
|
|
60
|
+
if (error instanceof AuthError && this.repairAccount && repairs < this.config.maxFallbackAttempts) {
|
|
61
|
+
repairs += 1;
|
|
62
|
+
await this.repairAccount(account).catch(() => false);
|
|
63
|
+
}
|
|
64
|
+
if (attempt >= maxAttempts || !isRetryable(error))
|
|
65
|
+
throw error;
|
|
66
|
+
options.onRetry?.(error, attempt);
|
|
67
|
+
await sleep(Math.min(this.config.retryMaxMs, this.config.retryBaseMs * 2 ** (attempt - 1)));
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
if (heartbeat)
|
|
71
|
+
clearInterval(heartbeat);
|
|
72
|
+
await session?.close();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
throw lastError instanceof Error
|
|
76
|
+
? lastError
|
|
77
|
+
: new AccountPoolExhausted(`No eligible account for ${label}.`);
|
|
78
|
+
}
|
|
79
|
+
get summary() {
|
|
80
|
+
return this.repository.summary();
|
|
81
|
+
}
|
|
82
|
+
releaseFailure(account, error) {
|
|
83
|
+
const status = error instanceof XTrawlError && typeof error.diagnostics.statusCode === "number"
|
|
84
|
+
? error.diagnostics.statusCode
|
|
85
|
+
: error instanceof AccountSessionBuildError
|
|
86
|
+
? error.statusCode
|
|
87
|
+
: 599;
|
|
88
|
+
const category = error instanceof AuthError
|
|
89
|
+
? "auth"
|
|
90
|
+
: error instanceof RateLimitError
|
|
91
|
+
? "rate_limit"
|
|
92
|
+
: error instanceof ProxyError
|
|
93
|
+
? "proxy"
|
|
94
|
+
: error instanceof NetworkError
|
|
95
|
+
? "network"
|
|
96
|
+
: error instanceof AccountSessionBuildError
|
|
97
|
+
? error.category
|
|
98
|
+
: "transient";
|
|
99
|
+
const decision = computeCooldown(category, status, Date.now(), {
|
|
100
|
+
defaultMs: this.config.cooldownDefaultMs,
|
|
101
|
+
transientMs: this.config.transientCooldownMs,
|
|
102
|
+
authMs: this.config.authCooldownMs,
|
|
103
|
+
...(error instanceof XTrawlError && typeof error.diagnostics.resetAt === "number"
|
|
104
|
+
? { resetAt: error.diagnostics.resetAt }
|
|
105
|
+
: {}),
|
|
106
|
+
jitterMs: this.config.cooldownJitterMs,
|
|
107
|
+
});
|
|
108
|
+
this.repository.release(account.leaseId, decision);
|
|
109
|
+
}
|
|
110
|
+
startHeartbeat(account) {
|
|
111
|
+
if (this.config.leaseHeartbeatMs <= 0)
|
|
112
|
+
return undefined;
|
|
113
|
+
const timer = setInterval(() => this.repository.heartbeat(account.leaseId, this.config.leaseTtlMs), this.config.leaseHeartbeatMs);
|
|
114
|
+
timer.unref();
|
|
115
|
+
return timer;
|
|
116
|
+
}
|
|
117
|
+
limiterFor(account) {
|
|
118
|
+
const key = String(account.id ?? account.username);
|
|
119
|
+
const current = this.limiters.get(key);
|
|
120
|
+
if (current)
|
|
121
|
+
return current;
|
|
122
|
+
const limiter = new TokenBucketLimiter(this.config.requestsPerMinute, this.config.minDelayMs);
|
|
123
|
+
this.limiters.set(key, limiter);
|
|
124
|
+
return limiter;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function countTweets(value) {
|
|
128
|
+
if (Array.isArray(value))
|
|
129
|
+
return value.filter((item) => isRecord(item) && "tweetId" in item).length;
|
|
130
|
+
if (!isRecord(value))
|
|
131
|
+
return 0;
|
|
132
|
+
if (Array.isArray(value.tweets))
|
|
133
|
+
return value.tweets.length;
|
|
134
|
+
if (isRecord(value.result) && Array.isArray(value.result.tweets))
|
|
135
|
+
return value.result.tweets.length;
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
function isRetryable(error) {
|
|
139
|
+
if (error instanceof AuthError || error instanceof RateLimitError || error instanceof NetworkError)
|
|
140
|
+
return true;
|
|
141
|
+
if (error instanceof ProxyError || error instanceof AccountSessionBuildError)
|
|
142
|
+
return true;
|
|
143
|
+
return !(error instanceof XTrawlError);
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=account-pool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"account-pool.js","sourceRoot":"","sources":["../../src/pool/account-pool.ts"],"names":[],"mappings":"AACA,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,SAAS,EACT,WAAW,EACX,YAAY,EACZ,cAAc,EACd,UAAU,GACX,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAezD,MAAM,OAAO,WAAW;IAIH;IACA;IACA;IACA;IANF,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,YACmB,UAA6B,EAC7B,QAAwB,EACxB,MAAoB,EACpB,aAA6B;QAH7B,eAAU,GAAV,UAAU,CAAmB;QAC7B,aAAQ,GAAR,QAAQ,CAAgB;QACxB,WAAM,GAAN,MAAM,CAAc;QACpB,kBAAa,GAAb,aAAa,CAAgB;IAC7C,CAAC;IAEG,KAAK,CAAC,OAAO,CAClB,KAAa,EACb,SAAwD,EACxD,UAAmC,EAAE;QAErC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;QAChD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,IAAI,SAAkB,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,IAAI,SAAS;oBAAE,MAAM,SAAS,CAAC;gBAC/B,MAAM,IAAI,oBAAoB,CAAC,2BAA2B,KAAK,GAAG,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC1D,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7B,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;YACjF,IAAI,YAAY,CAAC,IAAI,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAChE,IAAI,SAAS;oBAAE,MAAM,SAAS,CAAC;gBAC/B,MAAM,IAAI,oBAAoB,CAAC,oCAAoC,KAAK,GAAG,CAAC,CAAC;YAC/E,CAAC;YACD,IAAI,OAAgC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,OAAO,EAAE;wBAC9C,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;wBAC9B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB;qBAC3C,CAAC,CAAC;gBACL,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;gBACzC,gBAAgB,GAAG,IAAI,CAAC;gBACxB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACpD,IAAI,CAAC,UAAU,CAAC,WAAW,CACzB,OAAO,CAAC,OAAO,EACf,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAChE,CAAC;gBACF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAChE,OAAO,KAAK,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,gBAAgB;oBAAE,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzE,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACpC,IAAI,KAAK,YAAY,SAAS,IAAI,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;oBAClG,OAAO,IAAI,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBACvD,CAAC;gBACD,IAAI,OAAO,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;oBAAE,MAAM,KAAK,CAAC;gBAC/D,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBAClC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9F,CAAC;oBAAS,CAAC;gBACT,IAAI,SAAS;oBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBACxC,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;QACD,MAAM,SAAS,YAAY,KAAK;YAC9B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,oBAAoB,CAAC,2BAA2B,KAAK,GAAG,CAAC,CAAC;IACpE,CAAC;IAED,IAAW,OAAO;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAEO,cAAc,CAAC,OAAqB,EAAE,KAAc;QAC1D,MAAM,MAAM,GACV,KAAK,YAAY,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,CAAC,UAAU,KAAK,QAAQ;YAC9E,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU;YAC9B,CAAC,CAAC,KAAK,YAAY,wBAAwB;gBACzC,CAAC,CAAC,KAAK,CAAC,UAAU;gBAClB,CAAC,CAAC,GAAG,CAAC;QACZ,MAAM,QAAQ,GACZ,KAAK,YAAY,SAAS;YACxB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,KAAK,YAAY,cAAc;gBAC/B,CAAC,CAAC,YAAY;gBACd,CAAC,CAAC,KAAK,YAAY,UAAU;oBAC3B,CAAC,CAAC,OAAO;oBACT,CAAC,CAAC,KAAK,YAAY,YAAY;wBAC7B,CAAC,CAAC,SAAS;wBACX,CAAC,CAAC,KAAK,YAAY,wBAAwB;4BACzC,CAAC,CAAC,KAAK,CAAC,QAAQ;4BAChB,CAAC,CAAC,WAAW,CAAC;QAC1B,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE;YAC7D,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;YACxC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB;YAC5C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAClC,GAAG,CAAC,KAAK,YAAY,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,CAAC,OAAO,KAAK,QAAQ;gBAC/E,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE;gBACxC,CAAC,CAAC,EAAE,CAAC;YACP,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;SACvC,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACrD,CAAC;IAEO,cAAc,CAAC,OAAqB;QAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QACxD,MAAM,KAAK,GAAG,WAAW,CACvB,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EACxE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAC7B,CAAC;QACF,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,UAAU,CAAC,OAAqB;QACtC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9F,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;IACpG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5D,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;IACpG,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,YAAY,cAAc,IAAI,KAAK,YAAY,YAAY;QAChG,OAAO,IAAI,CAAC;IACd,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,YAAY,wBAAwB;QAAE,OAAO,IAAI,CAAC;IAC1F,OAAO,CAAC,CAAC,KAAK,YAAY,WAAW,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ErrorCategory } from "../domain/errors.js";
|
|
2
|
+
export interface CooldownDecision {
|
|
3
|
+
readonly status: "healthy" | "cooling_down" | "unusable";
|
|
4
|
+
readonly availableUntil: number;
|
|
5
|
+
readonly lastErrorCode?: number;
|
|
6
|
+
readonly reason?: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function parseRateLimitReset(headers: Readonly<Record<string, string>>): number | undefined;
|
|
9
|
+
export declare function parseRateLimitRemaining(headers: Readonly<Record<string, string>>): number | undefined;
|
|
10
|
+
export declare function effectiveStatus(status: number, headers: Readonly<Record<string, string>>): number;
|
|
11
|
+
export declare function computeCooldown(category: ErrorCategory | undefined, status: number, now: number, options: {
|
|
12
|
+
readonly defaultMs: number;
|
|
13
|
+
readonly transientMs: number;
|
|
14
|
+
readonly authMs: number;
|
|
15
|
+
readonly resetAt?: number;
|
|
16
|
+
readonly jitterMs?: number;
|
|
17
|
+
}): CooldownDecision;
|
|
18
|
+
//# sourceMappingURL=cooldown.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cooldown.d.ts","sourceRoot":"","sources":["../../src/pool/cooldown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,UAAU,CAAC;IACzD,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,SAAS,CAKjG;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,SAAS,CAKrG;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,CAGjG;AAED,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,aAAa,GAAG,SAAS,EACnC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,gBAAgB,CAiBlB"}
|