xtrawl 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/DOCUMENTATION.md +774 -0
  2. package/README.md +232 -0
  3. package/dist/auth/bootstrap.d.ts +4 -0
  4. package/dist/auth/bootstrap.d.ts.map +1 -0
  5. package/dist/auth/bootstrap.js +29 -0
  6. package/dist/auth/bootstrap.js.map +1 -0
  7. package/dist/auth/cookies.d.ts +7 -0
  8. package/dist/auth/cookies.d.ts.map +1 -0
  9. package/dist/auth/cookies.js +83 -0
  10. package/dist/auth/cookies.js.map +1 -0
  11. package/dist/auth/loaders.d.ts +6 -0
  12. package/dist/auth/loaders.d.ts.map +1 -0
  13. package/dist/auth/loaders.js +69 -0
  14. package/dist/auth/loaders.js.map +1 -0
  15. package/dist/auth/material.d.ts +3 -0
  16. package/dist/auth/material.d.ts.map +1 -0
  17. package/dist/auth/material.js +20 -0
  18. package/dist/auth/material.js.map +1 -0
  19. package/dist/auth/records.d.ts +4 -0
  20. package/dist/auth/records.d.ts.map +1 -0
  21. package/dist/auth/records.js +38 -0
  22. package/dist/auth/records.js.map +1 -0
  23. package/dist/cli/help.d.ts +2 -0
  24. package/dist/cli/help.d.ts.map +1 -0
  25. package/dist/cli/help.js +30 -0
  26. package/dist/cli/help.js.map +1 -0
  27. package/dist/cli/main.d.ts +3 -0
  28. package/dist/cli/main.d.ts.map +1 -0
  29. package/dist/cli/main.js +85 -0
  30. package/dist/cli/main.js.map +1 -0
  31. package/dist/cli/parser.d.ts +22 -0
  32. package/dist/cli/parser.d.ts.map +1 -0
  33. package/dist/cli/parser.js +275 -0
  34. package/dist/cli/parser.js.map +1 -0
  35. package/dist/client/client.d.ts +29 -0
  36. package/dist/client/client.d.ts.map +1 -0
  37. package/dist/client/client.js +192 -0
  38. package/dist/client/client.js.map +1 -0
  39. package/dist/client/collectors.d.ts +15 -0
  40. package/dist/client/collectors.d.ts.map +1 -0
  41. package/dist/client/collectors.js +185 -0
  42. package/dist/client/collectors.js.map +1 -0
  43. package/dist/client/database.d.ts +56 -0
  44. package/dist/client/database.d.ts.map +1 -0
  45. package/dist/client/database.js +146 -0
  46. package/dist/client/database.js.map +1 -0
  47. package/dist/client/profiles.d.ts +5 -0
  48. package/dist/client/profiles.d.ts.map +1 -0
  49. package/dist/client/profiles.js +36 -0
  50. package/dist/client/profiles.js.map +1 -0
  51. package/dist/client/search.d.ts +21 -0
  52. package/dist/client/search.d.ts.map +1 -0
  53. package/dist/client/search.js +139 -0
  54. package/dist/client/search.js.map +1 -0
  55. package/dist/client/types.d.ts +21 -0
  56. package/dist/client/types.d.ts.map +1 -0
  57. package/dist/client/types.js +2 -0
  58. package/dist/client/types.js.map +1 -0
  59. package/dist/config/defaults.d.ts +3 -0
  60. package/dist/config/defaults.d.ts.map +1 -0
  61. package/dist/config/defaults.js +38 -0
  62. package/dist/config/defaults.js.map +1 -0
  63. package/dist/config/environment.d.ts +5 -0
  64. package/dist/config/environment.d.ts.map +1 -0
  65. package/dist/config/environment.js +62 -0
  66. package/dist/config/environment.js.map +1 -0
  67. package/dist/config/types.d.ts +47 -0
  68. package/dist/config/types.d.ts.map +1 -0
  69. package/dist/config/types.js +2 -0
  70. package/dist/config/types.js.map +1 -0
  71. package/dist/config/validation.d.ts +6 -0
  72. package/dist/config/validation.d.ts.map +1 -0
  73. package/dist/config/validation.js +147 -0
  74. package/dist/config/validation.js.map +1 -0
  75. package/dist/domain/accounts.d.ts +64 -0
  76. package/dist/domain/accounts.d.ts.map +1 -0
  77. package/dist/domain/accounts.js +2 -0
  78. package/dist/domain/accounts.js.map +1 -0
  79. package/dist/domain/errors.d.ts +61 -0
  80. package/dist/domain/errors.d.ts.map +1 -0
  81. package/dist/domain/errors.js +89 -0
  82. package/dist/domain/errors.js.map +1 -0
  83. package/dist/domain/http.d.ts +36 -0
  84. package/dist/domain/http.d.ts.map +1 -0
  85. package/dist/domain/http.js +2 -0
  86. package/dist/domain/http.js.map +1 -0
  87. package/dist/domain/manifest.d.ts +23 -0
  88. package/dist/domain/manifest.d.ts.map +1 -0
  89. package/dist/domain/manifest.js +2 -0
  90. package/dist/domain/manifest.js.map +1 -0
  91. package/dist/domain/records.d.ts +67 -0
  92. package/dist/domain/records.d.ts.map +1 -0
  93. package/dist/domain/records.js +2 -0
  94. package/dist/domain/records.js.map +1 -0
  95. package/dist/domain/requests.d.ts +87 -0
  96. package/dist/domain/requests.d.ts.map +1 -0
  97. package/dist/domain/requests.js +2 -0
  98. package/dist/domain/requests.js.map +1 -0
  99. package/dist/engine/api-engine.d.ts +25 -0
  100. package/dist/engine/api-engine.d.ts.map +1 -0
  101. package/dist/engine/api-engine.js +98 -0
  102. package/dist/engine/api-engine.js.map +1 -0
  103. package/dist/engine/extractors.d.ts +27 -0
  104. package/dist/engine/extractors.d.ts.map +1 -0
  105. package/dist/engine/extractors.js +248 -0
  106. package/dist/engine/extractors.js.map +1 -0
  107. package/dist/index.d.ts +21 -0
  108. package/dist/index.d.ts.map +1 -0
  109. package/dist/index.js +15 -0
  110. package/dist/index.js.map +1 -0
  111. package/dist/manifest/default-manifest.d.ts +3 -0
  112. package/dist/manifest/default-manifest.d.ts.map +1 -0
  113. package/dist/manifest/default-manifest.js +46 -0
  114. package/dist/manifest/default-manifest.js.map +1 -0
  115. package/dist/manifest/model.d.ts +3 -0
  116. package/dist/manifest/model.d.ts.map +1 -0
  117. package/dist/manifest/model.js +48 -0
  118. package/dist/manifest/model.js.map +1 -0
  119. package/dist/manifest/provider.d.ts +19 -0
  120. package/dist/manifest/provider.d.ts.map +1 -0
  121. package/dist/manifest/provider.js +94 -0
  122. package/dist/manifest/provider.js.map +1 -0
  123. package/dist/manifest/scraper.d.ts +11 -0
  124. package/dist/manifest/scraper.d.ts.map +1 -0
  125. package/dist/manifest/scraper.js +130 -0
  126. package/dist/manifest/scraper.js.map +1 -0
  127. package/dist/output/csv-writer.d.ts +2 -0
  128. package/dist/output/csv-writer.d.ts.map +1 -0
  129. package/dist/output/csv-writer.js +94 -0
  130. package/dist/output/csv-writer.js.map +1 -0
  131. package/dist/output/json-writer.d.ts +2 -0
  132. package/dist/output/json-writer.d.ts.map +1 -0
  133. package/dist/output/json-writer.js +29 -0
  134. package/dist/output/json-writer.js.map +1 -0
  135. package/dist/output/names.d.ts +4 -0
  136. package/dist/output/names.d.ts.map +1 -0
  137. package/dist/output/names.js +22 -0
  138. package/dist/output/names.js.map +1 -0
  139. package/dist/output/writer.d.ts +7 -0
  140. package/dist/output/writer.d.ts.map +1 -0
  141. package/dist/output/writer.js +33 -0
  142. package/dist/output/writer.js.map +1 -0
  143. package/dist/pool/account-pool.d.ts +29 -0
  144. package/dist/pool/account-pool.d.ts.map +1 -0
  145. package/dist/pool/account-pool.js +145 -0
  146. package/dist/pool/account-pool.js.map +1 -0
  147. package/dist/pool/cooldown.d.ts +18 -0
  148. package/dist/pool/cooldown.d.ts.map +1 -0
  149. package/dist/pool/cooldown.js +38 -0
  150. package/dist/pool/cooldown.js.map +1 -0
  151. package/dist/pool/limiter.d.ts +11 -0
  152. package/dist/pool/limiter.d.ts.map +1 -0
  153. package/dist/pool/limiter.js +27 -0
  154. package/dist/pool/limiter.js.map +1 -0
  155. package/dist/query/builder.d.ts +19 -0
  156. package/dist/query/builder.d.ts.map +1 -0
  157. package/dist/query/builder.js +110 -0
  158. package/dist/query/builder.js.map +1 -0
  159. package/dist/query/hash.d.ts +2 -0
  160. package/dist/query/hash.d.ts.map +1 -0
  161. package/dist/query/hash.js +6 -0
  162. package/dist/query/hash.js.map +1 -0
  163. package/dist/query/normalize.d.ts +39 -0
  164. package/dist/query/normalize.d.ts.map +1 -0
  165. package/dist/query/normalize.js +176 -0
  166. package/dist/query/normalize.js.map +1 -0
  167. package/dist/query/targets.d.ts +11 -0
  168. package/dist/query/targets.d.ts.map +1 -0
  169. package/dist/query/targets.js +96 -0
  170. package/dist/query/targets.js.map +1 -0
  171. package/dist/runner/index.d.ts +4 -0
  172. package/dist/runner/index.d.ts.map +1 -0
  173. package/dist/runner/index.js +4 -0
  174. package/dist/runner/index.js.map +1 -0
  175. package/dist/runner/retry.d.ts +7 -0
  176. package/dist/runner/retry.d.ts.map +1 -0
  177. package/dist/runner/retry.js +26 -0
  178. package/dist/runner/retry.js.map +1 -0
  179. package/dist/runner/runner.d.ts +20 -0
  180. package/dist/runner/runner.d.ts.map +1 -0
  181. package/dist/runner/runner.js +37 -0
  182. package/dist/runner/runner.js.map +1 -0
  183. package/dist/runner/task-queue.d.ts +19 -0
  184. package/dist/runner/task-queue.d.ts.map +1 -0
  185. package/dist/runner/task-queue.js +58 -0
  186. package/dist/runner/task-queue.js.map +1 -0
  187. package/dist/storage/account-repository.d.ts +50 -0
  188. package/dist/storage/account-repository.d.ts.map +1 -0
  189. package/dist/storage/account-repository.js +171 -0
  190. package/dist/storage/account-repository.js.map +1 -0
  191. package/dist/storage/account-row.d.ts +4 -0
  192. package/dist/storage/account-row.d.ts.map +1 -0
  193. package/dist/storage/account-row.js +46 -0
  194. package/dist/storage/account-row.js.map +1 -0
  195. package/dist/storage/checkpoint-repository.d.ts +10 -0
  196. package/dist/storage/checkpoint-repository.d.ts.map +1 -0
  197. package/dist/storage/checkpoint-repository.js +30 -0
  198. package/dist/storage/checkpoint-repository.js.map +1 -0
  199. package/dist/storage/database.d.ts +18 -0
  200. package/dist/storage/database.d.ts.map +1 -0
  201. package/dist/storage/database.js +42 -0
  202. package/dist/storage/database.js.map +1 -0
  203. package/dist/storage/index.d.ts +15 -0
  204. package/dist/storage/index.d.ts.map +1 -0
  205. package/dist/storage/index.js +18 -0
  206. package/dist/storage/index.js.map +1 -0
  207. package/dist/storage/manifest-repository.d.ts +9 -0
  208. package/dist/storage/manifest-repository.d.ts.map +1 -0
  209. package/dist/storage/manifest-repository.js +22 -0
  210. package/dist/storage/manifest-repository.js.map +1 -0
  211. package/dist/storage/run-repository.d.ts +24 -0
  212. package/dist/storage/run-repository.d.ts.map +1 -0
  213. package/dist/storage/run-repository.js +51 -0
  214. package/dist/storage/run-repository.js.map +1 -0
  215. package/dist/storage/schema.d.ts +3 -0
  216. package/dist/storage/schema.d.ts.map +1 -0
  217. package/dist/storage/schema.js +53 -0
  218. package/dist/storage/schema.js.map +1 -0
  219. package/dist/transport/graphql.d.ts +8 -0
  220. package/dist/transport/graphql.d.ts.map +1 -0
  221. package/dist/transport/graphql.js +104 -0
  222. package/dist/transport/graphql.js.map +1 -0
  223. package/dist/transport/proxy.d.ts +5 -0
  224. package/dist/transport/proxy.d.ts.map +1 -0
  225. package/dist/transport/proxy.js +36 -0
  226. package/dist/transport/proxy.js.map +1 -0
  227. package/dist/transport/session.d.ts +25 -0
  228. package/dist/transport/session.d.ts.map +1 -0
  229. package/dist/transport/session.js +138 -0
  230. package/dist/transport/session.js.map +1 -0
  231. package/dist/transport/transaction-id.d.ts +20 -0
  232. package/dist/transport/transaction-id.d.ts.map +1 -0
  233. package/dist/transport/transaction-id.js +74 -0
  234. package/dist/transport/transaction-id.js.map +1 -0
  235. package/dist/utils/fingerprint.d.ts +3 -0
  236. package/dist/utils/fingerprint.d.ts.map +1 -0
  237. package/dist/utils/fingerprint.js +20 -0
  238. package/dist/utils/fingerprint.js.map +1 -0
  239. package/dist/utils/guards.d.ts +8 -0
  240. package/dist/utils/guards.d.ts.map +1 -0
  241. package/dist/utils/guards.js +38 -0
  242. package/dist/utils/guards.js.map +1 -0
  243. package/docs/architecture/overview.md +59 -0
  244. package/docs/decisions/0001-runtime-and-state.md +21 -0
  245. package/docs/decisions/0002-modular-boundaries.md +16 -0
  246. package/docs/plans/initial-build.md +18 -0
  247. package/docs/product/specification.md +87 -0
  248. package/docs/security/data-boundary.md +27 -0
  249. package/package.json +89 -0
@@ -0,0 +1,130 @@
1
+ import { asString } from "../utils/guards.js";
2
+ const OPERATION_NAMES = {
3
+ SearchTimeline: "search_timeline",
4
+ UserByScreenName: "user_lookup_screen_name",
5
+ UserTweets: "profile_timeline",
6
+ Followers: "followers",
7
+ Following: "following",
8
+ BlueVerifiedFollowers: "verified_followers",
9
+ TweetResultByRestId: "tweet_result",
10
+ };
11
+ export function extractManifestFromJavascript(source, fallback) {
12
+ const operationFeatures = extractOperationFeaturesFromJavascript(source);
13
+ return {
14
+ ...fallback,
15
+ queryIds: {
16
+ ...fallback.queryIds,
17
+ ...extractOperationQueryIdsFromJavascript(source),
18
+ },
19
+ operationFeatures: mergeOperationFeatures(fallback.operationFeatures, operationFeatures),
20
+ version: asString(fallback.version) ?? "live",
21
+ };
22
+ }
23
+ export function extractOperationQueryIdsFromJavascript(source) {
24
+ const queryIds = {};
25
+ for (const [operationName, key] of Object.entries(OPERATION_NAMES)) {
26
+ const queryFirst = new RegExp(`queryId\\s*:\\s*["']([^"']+)["']\\s*,\\s*operationName\\s*:\\s*["']${operationName}["']`, "u");
27
+ const operationFirst = new RegExp(`operationName\\s*:\\s*["']${operationName}["']\\s*,\\s*queryId\\s*:\\s*["']([^"']+)["']`, "u");
28
+ const id = source.match(queryFirst)?.[1] ?? source.match(operationFirst)?.[1];
29
+ if (id)
30
+ queryIds[key] = id;
31
+ }
32
+ return queryIds;
33
+ }
34
+ export function extractOperationFeaturesFromJavascript(source) {
35
+ const out = {};
36
+ for (const [operationName, key] of Object.entries(OPERATION_NAMES)) {
37
+ const marker = new RegExp(`operationName\\s*:\\s*["']${operationName}["']`, "gu");
38
+ const match = marker.exec(source);
39
+ if (!match)
40
+ continue;
41
+ const start = match.index;
42
+ const next = source.indexOf("operationName:", start + match[0].length);
43
+ const window = source.slice(start, next > start ? Math.min(next, start + 50_000) : start + 50_000);
44
+ const featureStart = window.indexOf("featureSwitches:[");
45
+ if (featureStart < 0)
46
+ continue;
47
+ const featureEnd = window.indexOf("]", featureStart);
48
+ if (featureEnd < 0)
49
+ continue;
50
+ const names = [...window.slice(featureStart, featureEnd).matchAll(/["']([^"']+)["']/gu)]
51
+ .map((item) => item[1])
52
+ .filter((item) => Boolean(item));
53
+ if (names.length > 0)
54
+ out[key] = Object.fromEntries(names.map((name) => [name, false]));
55
+ }
56
+ return out;
57
+ }
58
+ export async function scrapeManifestFromWeb(base, options = {}) {
59
+ const fetcher = options.fetcher ?? fetch;
60
+ const scriptHeaders = {
61
+ "User-Agent": "Mozilla/5.0",
62
+ };
63
+ const pageHeaders = {
64
+ ...scriptHeaders,
65
+ ...(options.authToken ? { Cookie: `auth_token=${options.authToken}` } : {}),
66
+ };
67
+ const response = await fetcher("https://x.com/home", { headers: pageHeaders, redirect: "follow" });
68
+ if (!response.ok)
69
+ throw new Error(`Manifest page failed with status ${response.status}`);
70
+ const html = await response.text();
71
+ const scripts = [...new Set([...html.matchAll(/<script[^>]+src=["']([^"']+)["']/giu)])]
72
+ .map((match) => match[1])
73
+ .filter((item) => Boolean(item))
74
+ .map((source) => safeScriptUrl(source))
75
+ .filter((source) => Boolean(source))
76
+ .sort((left, right) => scriptPriority(left) - scriptPriority(right))
77
+ .slice(0, options.maxScripts ?? 20);
78
+ const discovered = {};
79
+ const discoveredFeatures = {};
80
+ for (const url of scripts) {
81
+ const bundle = await fetcher(url, { headers: scriptHeaders, redirect: "follow" });
82
+ if (!bundle.ok)
83
+ continue;
84
+ const source = await bundle.text();
85
+ Object.assign(discovered, extractOperationQueryIdsFromJavascript(source));
86
+ Object.assign(discoveredFeatures, extractOperationFeaturesFromJavascript(source));
87
+ if (Object.keys(discovered).length === Object.keys(OPERATION_NAMES).length)
88
+ break;
89
+ }
90
+ if (Object.keys(discovered).length === 0) {
91
+ throw new Error("Manifest bundles did not contain any supported operation identifiers");
92
+ }
93
+ return {
94
+ ...base,
95
+ version: "web-live",
96
+ queryIds: { ...base.queryIds, ...discovered },
97
+ operationFeatures: mergeOperationFeatures(base.operationFeatures, discoveredFeatures),
98
+ };
99
+ }
100
+ function mergeOperationFeatures(base, discovered) {
101
+ const out = {};
102
+ if (base)
103
+ for (const [operation, features] of Object.entries(base))
104
+ out[operation] = { ...features };
105
+ for (const [operation, features] of Object.entries(discovered))
106
+ out[operation] = { ...(out[operation] ?? {}), ...features };
107
+ return out;
108
+ }
109
+ function safeScriptUrl(source) {
110
+ try {
111
+ const url = new URL(source, "https://x.com");
112
+ if (url.protocol !== "https:")
113
+ return undefined;
114
+ if (url.hostname !== "x.com" && url.hostname !== "abs.twimg.com")
115
+ return undefined;
116
+ return url.toString();
117
+ }
118
+ catch {
119
+ return undefined;
120
+ }
121
+ }
122
+ function scriptPriority(source) {
123
+ const path = new URL(source).pathname;
124
+ if (/\/responsive-web\/client-web\/main\.[^/]+\.js$/u.test(path))
125
+ return 0;
126
+ if (/\/main\.[^/]+\.js$/u.test(path))
127
+ return 1;
128
+ return 2;
129
+ }
130
+ //# sourceMappingURL=scraper.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/manifest/scraper.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,eAAe,GAA2B;IAC9C,cAAc,EAAE,iBAAiB;IACjC,gBAAgB,EAAE,yBAAyB;IAC3C,UAAU,EAAE,kBAAkB;IAC9B,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,qBAAqB,EAAE,oBAAoB;IAC3C,mBAAmB,EAAE,cAAc;CACpC,CAAC;AAQF,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,QAAyB;IACrF,MAAM,iBAAiB,GAAG,sCAAsC,CAAC,MAAM,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,QAAQ;QACX,QAAQ,EAAE;YACR,GAAG,QAAQ,CAAC,QAAQ;YACpB,GAAG,sCAAsC,CAAC,MAAM,CAAC;SAClD;QACD,iBAAiB,EAAE,sBAAsB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;QACxF,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,MAAc;IACnE,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,UAAU,GAAG,IAAI,MAAM,CAC3B,sEAAsE,aAAa,MAAM,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,MAAM,CAC/B,6BAA6B,aAAa,+CAA+C,EACzF,GAAG,CACJ,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9E,IAAI,EAAE;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,MAAc;IAEd,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACnE,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,6BAA6B,aAAa,MAAM,EAAE,IAAI,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACnG,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACzD,IAAI,YAAY,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACrD,IAAI,UAAU,GAAG,CAAC;YAAE,SAAS;QAC7B,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;aACrF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACtB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,IAAqB,EACrB,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IACzC,MAAM,aAAa,GAAG;QACpB,YAAY,EAAE,aAAa;KAC5B,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,GAAG,aAAa;QAChB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IACnG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC;SACpF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACxB,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC/C,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;SACtC,MAAM,CAAC,CAAC,MAAM,EAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACrD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;SACnE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IACtC,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,MAAM,kBAAkB,GAA4C,EAAE,CAAC;IACvE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClF,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,SAAS;QACzB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sCAAsC,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM;YAAE,MAAM;IACpF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO;QACL,GAAG,IAAI;QACP,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,UAAU,EAAE;QAC7C,iBAAiB,EAAE,sBAAsB,CAAC,IAAI,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KACtF,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,IAA0C,EAC1C,UAAuE;IAEvE,MAAM,GAAG,GAA4C,EAAE,CAAC;IACxD,IAAI,IAAI;QACN,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;YACtD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAI,QAAoC,EAAE,CAAC;IAClE,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;QAC5D,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAChD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,eAAe;YAAE,OAAO,SAAS,CAAC;QACnF,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,MAAc;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;IACtC,IAAI,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3E,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function writeCsv(path: string, rows: readonly Record<string, unknown>[], append?: boolean): Promise<void>;
2
+ //# sourceMappingURL=csv-writer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAGA,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EACxC,MAAM,UAAQ,GACb,OAAO,CAAC,IAAI,CAAC,CAiBf"}
@@ -0,0 +1,94 @@
1
+ import { access, mkdir, readFile, writeFile } from "node:fs/promises";
2
+ import { dirname } from "node:path";
3
+ export async function writeCsv(path, rows, append = false) {
4
+ if (rows.length === 0)
5
+ return;
6
+ const existing = append && (await exists(path)) ? await readFile(path, "utf8") : "";
7
+ const parsedExisting = existing ? parseCsv(existing) : [];
8
+ const currentHeaders = parsedExisting[0] ?? [];
9
+ const headers = mergeHeaders(currentHeaders, rows.flatMap((row) => Object.keys(row)));
10
+ const existingRows = parsedExisting
11
+ .slice(1)
12
+ .map((values) => Object.fromEntries(currentHeaders.map((header, index) => [header, values[index] ?? ""])));
13
+ const content = renderCsv(headers, [...existingRows, ...rows]);
14
+ await mkdir(dirname(path), { recursive: true });
15
+ await writeFile(path, content, "utf8");
16
+ }
17
+ function renderCsv(headers, rows) {
18
+ const body = rows.map((row) => headers.map((header) => escapeCsv(flattenValue(row[header]))).join(","));
19
+ return `${headers.map(escapeCsv).join(",")}\n${body.join("\n")}\n`;
20
+ }
21
+ function mergeHeaders(existing, incoming) {
22
+ const preferred = [
23
+ "tweetId",
24
+ "timestamp",
25
+ "username",
26
+ "name",
27
+ "text",
28
+ "likes",
29
+ "retweets",
30
+ "comments",
31
+ "tweetUrl",
32
+ ];
33
+ const all = [...new Set([...existing, ...incoming])];
34
+ return [...preferred.filter((key) => all.includes(key)), ...all.filter((key) => !preferred.includes(key))];
35
+ }
36
+ function parseCsv(value) {
37
+ const rows = [];
38
+ let row = [];
39
+ let field = "";
40
+ let quoted = false;
41
+ for (let index = 0; index < value.length; index += 1) {
42
+ const character = value[index];
43
+ const next = value[index + 1];
44
+ if (character === '"') {
45
+ if (quoted && next === '"') {
46
+ field += '"';
47
+ index += 1;
48
+ }
49
+ else
50
+ quoted = !quoted;
51
+ }
52
+ else if (character === "," && !quoted) {
53
+ row.push(field);
54
+ field = "";
55
+ }
56
+ else if ((character === "\n" || character === "\r") && !quoted) {
57
+ if (character === "\r" && next === "\n")
58
+ index += 1;
59
+ row.push(field);
60
+ field = "";
61
+ if (row.some((item) => item.length > 0))
62
+ rows.push(row);
63
+ row = [];
64
+ }
65
+ else {
66
+ field += character;
67
+ }
68
+ }
69
+ if (field || row.length > 0) {
70
+ row.push(field);
71
+ rows.push(row);
72
+ }
73
+ return rows;
74
+ }
75
+ function flattenValue(value) {
76
+ if (value === undefined || value === null)
77
+ return "";
78
+ if (typeof value === "string")
79
+ return value;
80
+ return JSON.stringify(value);
81
+ }
82
+ function escapeCsv(value) {
83
+ return /[",\n\r]/u.test(value) ? `"${value.replaceAll('"', '""')}"` : value;
84
+ }
85
+ async function exists(path) {
86
+ try {
87
+ await access(path);
88
+ return true;
89
+ }
90
+ catch {
91
+ return false;
92
+ }
93
+ }
94
+ //# sourceMappingURL=csv-writer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"csv-writer.js","sourceRoot":"","sources":["../../src/output/csv-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,IAAwC,EACxC,MAAM,GAAG,KAAK;IAEd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,MAAM,cAAc,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,YAAY,CAC1B,cAAc,EACd,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CACxC,CAAC;IACF,MAAM,YAAY,GAAG,cAAc;SAChC,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACd,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CACzF,CAAC;IACJ,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,GAAG,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,SAAS,CAAC,OAA0B,EAAE,IAAwC;IACrF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACxG,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACrE,CAAC;AAED,SAAS,YAAY,CAAC,QAA2B,EAAE,QAA2B;IAC5E,MAAM,SAAS,GAAG;QAChB,SAAS;QACT,WAAW;QACX,UAAU;QACV,MAAM;QACN,MAAM;QACN,OAAO;QACP,UAAU;QACV,UAAU;QACV,UAAU;KACX,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACtB,IAAI,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAC3B,KAAK,IAAI,GAAG,CAAC;gBACb,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;;gBAAM,MAAM,GAAG,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,IAAI,SAAS,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;aAAM,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,SAAS,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjE,IAAI,SAAS,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;gBAAE,KAAK,IAAI,CAAC,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,GAAG,GAAG,EAAE,CAAC;QACX,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,SAAS,CAAC;QACrB,CAAC;IACH,CAAC;IACD,IAAI,KAAK,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function writeJson(path: string, rows: readonly unknown[], append?: boolean): Promise<void>;
2
+ //# sourceMappingURL=json-writer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json-writer.d.ts","sourceRoot":"","sources":["../../src/output/json-writer.ts"],"names":[],"mappings":"AAGA,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAarG"}
@@ -0,0 +1,29 @@
1
+ import { access, mkdir, readFile, writeFile } from "node:fs/promises";
2
+ import { dirname } from "node:path";
3
+ export async function writeJson(path, rows, append = false) {
4
+ if (rows.length === 0)
5
+ return;
6
+ let output = [...rows];
7
+ if (append && (await exists(path))) {
8
+ try {
9
+ const current = JSON.parse(await readFile(path, "utf8"));
10
+ if (Array.isArray(current))
11
+ output = [...current, ...rows];
12
+ }
13
+ catch {
14
+ output = [...rows];
15
+ }
16
+ }
17
+ await mkdir(dirname(path), { recursive: true });
18
+ await writeFile(path, `${JSON.stringify(output, null, 2)}\n`, "utf8");
19
+ }
20
+ async function exists(path) {
21
+ try {
22
+ await access(path);
23
+ return true;
24
+ }
25
+ catch {
26
+ return false;
27
+ }
28
+ }
29
+ //# sourceMappingURL=json-writer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json-writer.js","sourceRoot":"","sources":["../../src/output/json-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,IAAwB,EAAE,MAAM,GAAG,KAAK;IACpF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC9B,IAAI,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACvB,IAAI,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;YACpE,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;gBAAE,MAAM,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
@@ -0,0 +1,4 @@
1
+ import type { TargetInput } from "../domain/requests.js";
2
+ export declare function searchOutputName(query: string | undefined, since: string | undefined, until: string | undefined, fromUsers: readonly string[] | undefined): string;
3
+ export declare function targetOutputName(operation: string, targets: readonly TargetInput[]): string;
4
+ //# sourceMappingURL=names.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"names.d.ts","sourceRoot":"","sources":["../../src/output/names.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GACvC,MAAM,CAIR;AAED,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,MAAM,CAM3F"}
@@ -0,0 +1,22 @@
1
+ export function searchOutputName(query, since, until, fromUsers) {
2
+ const source = query?.trim() || fromUsers?.slice(0, 3).join("_") || "search";
3
+ const parts = [safePart(source), since?.slice(0, 10), until?.slice(0, 10)].filter(Boolean);
4
+ return parts.join("_");
5
+ }
6
+ export function targetOutputName(operation, targets) {
7
+ const names = targets
8
+ .slice(0, 3)
9
+ .map((target) => target.username ?? target.userId)
10
+ .filter((value) => Boolean(value));
11
+ return safePart(names.length > 0 ? `${operation}_${names.join("_")}` : operation);
12
+ }
13
+ function safePart(value) {
14
+ const cleaned = value
15
+ .split(/\s+/u)
16
+ .slice(0, 3)
17
+ .join("_")
18
+ .replace(/[^A-Za-z0-9_.-]+/gu, "_")
19
+ .replace(/^_+|_+$/gu, "");
20
+ return cleaned || "output";
21
+ }
22
+ //# sourceMappingURL=names.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"names.js","sourceRoot":"","sources":["../../src/output/names.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAC9B,KAAyB,EACzB,KAAyB,EACzB,KAAyB,EACzB,SAAwC;IAExC,MAAM,MAAM,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,SAAS,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;IAC7E,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3F,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,OAA+B;IACjF,MAAM,KAAK,GAAG,OAAO;SAClB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC;SACjD,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,OAAO,GAAG,KAAK;SAClB,KAAK,CAAC,MAAM,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,IAAI,CAAC,GAAG,CAAC;SACT,OAAO,CAAC,oBAAoB,EAAE,GAAG,CAAC;SAClC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC5B,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC"}
@@ -0,0 +1,7 @@
1
+ export type OutputFormat = "csv" | "json" | "both";
2
+ export declare function saveRows(name: string, rows: readonly unknown[], options: {
3
+ readonly directory: string;
4
+ readonly format: OutputFormat;
5
+ readonly append?: boolean;
6
+ }): Promise<void>;
7
+ //# sourceMappingURL=writer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/output/writer.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;AAEnD,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,OAAO,EAAE,EACxB,OAAO,EAAE;IAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAE,GAChG,OAAO,CAAC,IAAI,CAAC,CAMf"}
@@ -0,0 +1,33 @@
1
+ import { mkdir } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ import { writeCsv } from "./csv-writer.js";
4
+ import { writeJson } from "./json-writer.js";
5
+ export async function saveRows(name, rows, options) {
6
+ await mkdir(options.directory, { recursive: true });
7
+ if (options.format === "json" || options.format === "both")
8
+ await writeJson(join(options.directory, `${name}.json`), rows, options.append);
9
+ if (options.format === "csv" || options.format === "both")
10
+ await writeCsv(join(options.directory, `${name}.csv`), rows.map(toFlatRow), options.append);
11
+ }
12
+ function toFlatRow(value) {
13
+ if (!value || typeof value !== "object" || Array.isArray(value))
14
+ return { value };
15
+ const row = value;
16
+ if ("tweetId" in row) {
17
+ const tweet = row;
18
+ return {
19
+ tweetId: tweet.tweetId,
20
+ timestamp: tweet.timestamp,
21
+ username: tweet.user.screenName,
22
+ name: tweet.user.name,
23
+ text: tweet.text,
24
+ likes: tweet.likes,
25
+ retweets: tweet.retweets,
26
+ comments: tweet.comments,
27
+ tweetUrl: tweet.tweetUrl,
28
+ media: tweet.media.imageLinks,
29
+ };
30
+ }
31
+ return Object.fromEntries(Object.entries(row).filter(([key]) => key !== "raw"));
32
+ }
33
+ //# sourceMappingURL=writer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"writer.js","sourceRoot":"","sources":["../../src/output/writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAI7C,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,IAAwB,EACxB,OAAiG;IAEjG,MAAM,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;QACxD,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;QACvD,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAChG,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAClF,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,GAA6B,CAAC;QAC5C,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU;YAC/B,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI;YACrB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC;AAClF,CAAC"}
@@ -0,0 +1,29 @@
1
+ import type { AccountLease } from "../domain/accounts.js";
2
+ import type { ClientConfig } from "../config/types.js";
3
+ import type { HttpSession } from "../domain/http.js";
4
+ import type { AccountRepository } from "../storage/account-repository.js";
5
+ import type { SessionBuilder } from "../transport/session.js";
6
+ export interface PoolExecutionContext {
7
+ readonly account: AccountLease;
8
+ readonly session: HttpSession;
9
+ }
10
+ export interface PoolExecutionOptions<T> {
11
+ readonly countTweets?: (value: T) => number;
12
+ readonly onRetry?: (error: unknown, attempt: number) => void;
13
+ readonly maxAccountSwitches?: number;
14
+ }
15
+ export type AccountRepair = (account: AccountLease) => Promise<boolean>;
16
+ export declare class AccountPool {
17
+ private readonly repository;
18
+ private readonly sessions;
19
+ private readonly config;
20
+ private readonly repairAccount?;
21
+ private readonly limiters;
22
+ constructor(repository: AccountRepository, sessions: SessionBuilder, config: ClientConfig, repairAccount?: AccountRepair | undefined);
23
+ execute<T>(label: string, operation: (context: PoolExecutionContext) => Promise<T>, options?: PoolExecutionOptions<T>): Promise<T>;
24
+ get summary(): import("../domain/accounts.js").AccountSummary;
25
+ private releaseFailure;
26
+ private startHeartbeat;
27
+ private limiterFor;
28
+ }
29
+ //# sourceMappingURL=account-pool.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"account-pool.d.ts","sourceRoot":"","sources":["../../src/pool/account-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAU1D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAE1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAI9D,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,CAAC;IAC5C,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExE,qBAAa,WAAW;IAIpB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IANjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyC;gBAG/C,UAAU,EAAE,iBAAiB,EAC7B,QAAQ,EAAE,cAAc,EACxB,MAAM,EAAE,YAAY,EACpB,aAAa,CAAC,EAAE,aAAa,YAAA;IAGnC,OAAO,CAAC,CAAC,EACpB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,CAAC,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAC,CAAC,CAAC,EACxD,OAAO,GAAE,oBAAoB,CAAC,CAAC,CAAM,GACpC,OAAO,CAAC,CAAC,CAAC;IA4Db,IAAW,OAAO,mDAEjB;IAED,OAAO,CAAC,cAAc;IA+BtB,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,UAAU;CAQnB"}
@@ -0,0 +1,145 @@
1
+ import { AccountPoolExhausted, AccountSessionBuildError, AuthError, XTrawlError, NetworkError, RateLimitError, ProxyError, } from "../domain/errors.js";
2
+ import { computeCooldown } from "./cooldown.js";
3
+ import { isRecord } from "../utils/guards.js";
4
+ import { TokenBucketLimiter, sleep } from "./limiter.js";
5
+ export class AccountPool {
6
+ repository;
7
+ sessions;
8
+ config;
9
+ repairAccount;
10
+ limiters = new Map();
11
+ constructor(repository, sessions, config, repairAccount) {
12
+ this.repository = repository;
13
+ this.sessions = sessions;
14
+ this.config = config;
15
+ this.repairAccount = repairAccount;
16
+ }
17
+ async execute(label, operation, options = {}) {
18
+ const maxAttempts = this.config.maxTaskAttempts;
19
+ const seenAccounts = new Set();
20
+ let lastError;
21
+ let repairs = 0;
22
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
23
+ const account = this.repository.lease({ requireAuthMaterial: true });
24
+ if (!account) {
25
+ if (lastError)
26
+ throw lastError;
27
+ throw new AccountPoolExhausted(`No eligible account for ${label}.`);
28
+ }
29
+ const accountKey = String(account.id ?? account.username);
30
+ seenAccounts.add(accountKey);
31
+ const switchLimit = options.maxAccountSwitches ?? this.config.maxAccountSwitches;
32
+ if (seenAccounts.size > switchLimit + 1) {
33
+ this.repository.release(account.leaseId, { status: "healthy" });
34
+ if (lastError)
35
+ throw lastError;
36
+ throw new AccountPoolExhausted(`Account-switch limit reached for ${label}.`);
37
+ }
38
+ let session;
39
+ let operationStarted = false;
40
+ const heartbeat = this.startHeartbeat(account);
41
+ try {
42
+ if (this.config.proxyCheckOnLease)
43
+ await this.sessions.assertProxyHealthy(account, {
44
+ url: this.config.proxyCheckUrl,
45
+ timeoutMs: this.config.proxyCheckTimeoutMs,
46
+ });
47
+ session = this.sessions.forAccount(account);
48
+ await this.limiterFor(account).acquire();
49
+ operationStarted = true;
50
+ const value = await operation({ account, session });
51
+ this.repository.recordUsage(account.leaseId, 1, Math.max(0, options.countTweets?.(value) ?? countTweets(value)));
52
+ this.repository.release(account.leaseId, { status: "healthy" });
53
+ return value;
54
+ }
55
+ catch (error) {
56
+ lastError = error;
57
+ if (operationStarted)
58
+ this.repository.recordUsage(account.leaseId, 1, 0);
59
+ this.releaseFailure(account, error);
60
+ if (error instanceof AuthError && this.repairAccount && repairs < this.config.maxFallbackAttempts) {
61
+ repairs += 1;
62
+ await this.repairAccount(account).catch(() => false);
63
+ }
64
+ if (attempt >= maxAttempts || !isRetryable(error))
65
+ throw error;
66
+ options.onRetry?.(error, attempt);
67
+ await sleep(Math.min(this.config.retryMaxMs, this.config.retryBaseMs * 2 ** (attempt - 1)));
68
+ }
69
+ finally {
70
+ if (heartbeat)
71
+ clearInterval(heartbeat);
72
+ await session?.close();
73
+ }
74
+ }
75
+ throw lastError instanceof Error
76
+ ? lastError
77
+ : new AccountPoolExhausted(`No eligible account for ${label}.`);
78
+ }
79
+ get summary() {
80
+ return this.repository.summary();
81
+ }
82
+ releaseFailure(account, error) {
83
+ const status = error instanceof XTrawlError && typeof error.diagnostics.statusCode === "number"
84
+ ? error.diagnostics.statusCode
85
+ : error instanceof AccountSessionBuildError
86
+ ? error.statusCode
87
+ : 599;
88
+ const category = error instanceof AuthError
89
+ ? "auth"
90
+ : error instanceof RateLimitError
91
+ ? "rate_limit"
92
+ : error instanceof ProxyError
93
+ ? "proxy"
94
+ : error instanceof NetworkError
95
+ ? "network"
96
+ : error instanceof AccountSessionBuildError
97
+ ? error.category
98
+ : "transient";
99
+ const decision = computeCooldown(category, status, Date.now(), {
100
+ defaultMs: this.config.cooldownDefaultMs,
101
+ transientMs: this.config.transientCooldownMs,
102
+ authMs: this.config.authCooldownMs,
103
+ ...(error instanceof XTrawlError && typeof error.diagnostics.resetAt === "number"
104
+ ? { resetAt: error.diagnostics.resetAt }
105
+ : {}),
106
+ jitterMs: this.config.cooldownJitterMs,
107
+ });
108
+ this.repository.release(account.leaseId, decision);
109
+ }
110
+ startHeartbeat(account) {
111
+ if (this.config.leaseHeartbeatMs <= 0)
112
+ return undefined;
113
+ const timer = setInterval(() => this.repository.heartbeat(account.leaseId, this.config.leaseTtlMs), this.config.leaseHeartbeatMs);
114
+ timer.unref();
115
+ return timer;
116
+ }
117
+ limiterFor(account) {
118
+ const key = String(account.id ?? account.username);
119
+ const current = this.limiters.get(key);
120
+ if (current)
121
+ return current;
122
+ const limiter = new TokenBucketLimiter(this.config.requestsPerMinute, this.config.minDelayMs);
123
+ this.limiters.set(key, limiter);
124
+ return limiter;
125
+ }
126
+ }
127
+ function countTweets(value) {
128
+ if (Array.isArray(value))
129
+ return value.filter((item) => isRecord(item) && "tweetId" in item).length;
130
+ if (!isRecord(value))
131
+ return 0;
132
+ if (Array.isArray(value.tweets))
133
+ return value.tweets.length;
134
+ if (isRecord(value.result) && Array.isArray(value.result.tweets))
135
+ return value.result.tweets.length;
136
+ return 0;
137
+ }
138
+ function isRetryable(error) {
139
+ if (error instanceof AuthError || error instanceof RateLimitError || error instanceof NetworkError)
140
+ return true;
141
+ if (error instanceof ProxyError || error instanceof AccountSessionBuildError)
142
+ return true;
143
+ return !(error instanceof XTrawlError);
144
+ }
145
+ //# sourceMappingURL=account-pool.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"account-pool.js","sourceRoot":"","sources":["../../src/pool/account-pool.ts"],"names":[],"mappings":"AACA,OAAO,EACL,oBAAoB,EACpB,wBAAwB,EACxB,SAAS,EACT,WAAW,EACX,YAAY,EACZ,cAAc,EACd,UAAU,GACX,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAezD,MAAM,OAAO,WAAW;IAIH;IACA;IACA;IACA;IANF,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,YACmB,UAA6B,EAC7B,QAAwB,EACxB,MAAoB,EACpB,aAA6B;QAH7B,eAAU,GAAV,UAAU,CAAmB;QAC7B,aAAQ,GAAR,QAAQ,CAAgB;QACxB,WAAM,GAAN,MAAM,CAAc;QACpB,kBAAa,GAAb,aAAa,CAAgB;IAC7C,CAAC;IAEG,KAAK,CAAC,OAAO,CAClB,KAAa,EACb,SAAwD,EACxD,UAAmC,EAAE;QAErC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;QAChD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,IAAI,SAAkB,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;YACrE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,IAAI,SAAS;oBAAE,MAAM,SAAS,CAAC;gBAC/B,MAAM,IAAI,oBAAoB,CAAC,2BAA2B,KAAK,GAAG,CAAC,CAAC;YACtE,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC1D,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7B,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;YACjF,IAAI,YAAY,CAAC,IAAI,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAChE,IAAI,SAAS;oBAAE,MAAM,SAAS,CAAC;gBAC/B,MAAM,IAAI,oBAAoB,CAAC,oCAAoC,KAAK,GAAG,CAAC,CAAC;YAC/E,CAAC;YACD,IAAI,OAAgC,CAAC;YACrC,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,OAAO,EAAE;wBAC9C,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa;wBAC9B,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB;qBAC3C,CAAC,CAAC;gBACL,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;gBACzC,gBAAgB,GAAG,IAAI,CAAC;gBACxB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACpD,IAAI,CAAC,UAAU,CAAC,WAAW,CACzB,OAAO,CAAC,OAAO,EACf,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAChE,CAAC;gBACF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAChE,OAAO,KAAK,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,gBAAgB;oBAAE,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzE,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACpC,IAAI,KAAK,YAAY,SAAS,IAAI,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;oBAClG,OAAO,IAAI,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBACvD,CAAC;gBACD,IAAI,OAAO,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;oBAAE,MAAM,KAAK,CAAC;gBAC/D,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBAClC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9F,CAAC;oBAAS,CAAC;gBACT,IAAI,SAAS;oBAAE,aAAa,CAAC,SAAS,CAAC,CAAC;gBACxC,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;QACD,MAAM,SAAS,YAAY,KAAK;YAC9B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,oBAAoB,CAAC,2BAA2B,KAAK,GAAG,CAAC,CAAC;IACpE,CAAC;IAED,IAAW,OAAO;QAChB,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAEO,cAAc,CAAC,OAAqB,EAAE,KAAc;QAC1D,MAAM,MAAM,GACV,KAAK,YAAY,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,CAAC,UAAU,KAAK,QAAQ;YAC9E,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU;YAC9B,CAAC,CAAC,KAAK,YAAY,wBAAwB;gBACzC,CAAC,CAAC,KAAK,CAAC,UAAU;gBAClB,CAAC,CAAC,GAAG,CAAC;QACZ,MAAM,QAAQ,GACZ,KAAK,YAAY,SAAS;YACxB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,KAAK,YAAY,cAAc;gBAC/B,CAAC,CAAC,YAAY;gBACd,CAAC,CAAC,KAAK,YAAY,UAAU;oBAC3B,CAAC,CAAC,OAAO;oBACT,CAAC,CAAC,KAAK,YAAY,YAAY;wBAC7B,CAAC,CAAC,SAAS;wBACX,CAAC,CAAC,KAAK,YAAY,wBAAwB;4BACzC,CAAC,CAAC,KAAK,CAAC,QAAQ;4BAChB,CAAC,CAAC,WAAW,CAAC;QAC1B,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE;YAC7D,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;YACxC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB;YAC5C,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAClC,GAAG,CAAC,KAAK,YAAY,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,CAAC,OAAO,KAAK,QAAQ;gBAC/E,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE;gBACxC,CAAC,CAAC,EAAE,CAAC;YACP,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;SACvC,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACrD,CAAC;IAEO,cAAc,CAAC,OAAqB;QAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QACxD,MAAM,KAAK,GAAG,WAAW,CACvB,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EACxE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAC7B,CAAC;QACF,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,UAAU,CAAC,OAAqB;QACtC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9F,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;IACpG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5D,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;IACpG,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,YAAY,cAAc,IAAI,KAAK,YAAY,YAAY;QAChG,OAAO,IAAI,CAAC;IACd,IAAI,KAAK,YAAY,UAAU,IAAI,KAAK,YAAY,wBAAwB;QAAE,OAAO,IAAI,CAAC;IAC1F,OAAO,CAAC,CAAC,KAAK,YAAY,WAAW,CAAC,CAAC;AACzC,CAAC"}
@@ -0,0 +1,18 @@
1
+ import type { ErrorCategory } from "../domain/errors.js";
2
+ export interface CooldownDecision {
3
+ readonly status: "healthy" | "cooling_down" | "unusable";
4
+ readonly availableUntil: number;
5
+ readonly lastErrorCode?: number;
6
+ readonly reason?: string;
7
+ }
8
+ export declare function parseRateLimitReset(headers: Readonly<Record<string, string>>): number | undefined;
9
+ export declare function parseRateLimitRemaining(headers: Readonly<Record<string, string>>): number | undefined;
10
+ export declare function effectiveStatus(status: number, headers: Readonly<Record<string, string>>): number;
11
+ export declare function computeCooldown(category: ErrorCategory | undefined, status: number, now: number, options: {
12
+ readonly defaultMs: number;
13
+ readonly transientMs: number;
14
+ readonly authMs: number;
15
+ readonly resetAt?: number;
16
+ readonly jitterMs?: number;
17
+ }): CooldownDecision;
18
+ //# sourceMappingURL=cooldown.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cooldown.d.ts","sourceRoot":"","sources":["../../src/pool/cooldown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,cAAc,GAAG,UAAU,CAAC;IACzD,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,SAAS,CAKjG;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,GAAG,SAAS,CAKrG;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,CAGjG;AAED,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,aAAa,GAAG,SAAS,EACnC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;IACP,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B,GACA,gBAAgB,CAiBlB"}