xtrawl 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/DOCUMENTATION.md +127 -4
  2. package/LICENSE +21 -0
  3. package/README.md +170 -19
  4. package/dist/auth/account-state.d.ts +3 -0
  5. package/dist/auth/account-state.d.ts.map +1 -0
  6. package/dist/auth/account-state.js +85 -0
  7. package/dist/auth/account-state.js.map +1 -0
  8. package/dist/client/accounts.d.ts +31 -0
  9. package/dist/client/accounts.d.ts.map +1 -0
  10. package/dist/client/accounts.js +163 -0
  11. package/dist/client/accounts.js.map +1 -0
  12. package/dist/client/client.d.ts +16 -5
  13. package/dist/client/client.d.ts.map +1 -1
  14. package/dist/client/client.js +58 -19
  15. package/dist/client/client.js.map +1 -1
  16. package/dist/client/database.d.ts +2 -1
  17. package/dist/client/database.d.ts.map +1 -1
  18. package/dist/client/database.js +1 -1
  19. package/dist/client/database.js.map +1 -1
  20. package/dist/client/search.d.ts +3 -2
  21. package/dist/client/search.d.ts.map +1 -1
  22. package/dist/client/search.js +23 -7
  23. package/dist/client/search.js.map +1 -1
  24. package/dist/client/types.d.ts +2 -0
  25. package/dist/client/types.d.ts.map +1 -1
  26. package/dist/domain/account-state.d.ts +68 -0
  27. package/dist/domain/account-state.d.ts.map +1 -0
  28. package/dist/domain/account-state.js +2 -0
  29. package/dist/domain/account-state.js.map +1 -0
  30. package/dist/domain/errors.d.ts +3 -0
  31. package/dist/domain/errors.d.ts.map +1 -1
  32. package/dist/domain/errors.js +5 -0
  33. package/dist/domain/errors.js.map +1 -1
  34. package/dist/domain/records.d.ts +4 -0
  35. package/dist/domain/records.d.ts.map +1 -1
  36. package/dist/domain/requests.d.ts +4 -0
  37. package/dist/domain/requests.d.ts.map +1 -1
  38. package/dist/index.d.ts +7 -4
  39. package/dist/index.d.ts.map +1 -1
  40. package/dist/index.js +3 -1
  41. package/dist/index.js.map +1 -1
  42. package/dist/pool/account-pool.d.ts +12 -5
  43. package/dist/pool/account-pool.d.ts.map +1 -1
  44. package/dist/pool/account-pool.js +102 -13
  45. package/dist/pool/account-pool.js.map +1 -1
  46. package/dist/storage/account-repository.d.ts +8 -1
  47. package/dist/storage/account-repository.d.ts.map +1 -1
  48. package/dist/storage/account-repository.js +63 -26
  49. package/dist/storage/account-repository.js.map +1 -1
  50. package/docs/architecture/overview.md +16 -11
  51. package/docs/decisions/0003-caller-owned-account-state.md +33 -0
  52. package/docs/product/specification.md +19 -2
  53. package/docs/security/data-boundary.md +9 -2
  54. package/package.json +4 -3
package/DOCUMENTATION.md CHANGED
@@ -8,6 +8,7 @@ the command line. XTrawl is an authenticated, read-only collector for public X d
8
8
  - [Install](#install)
9
9
  - [Authenticate](#authenticate)
10
10
  - [Use multiple accounts](#use-multiple-accounts)
11
+ - [Own account state](#own-account-state)
11
12
  - [Use a proxy](#use-a-proxy)
12
13
  - [Use the TypeScript API](#use-the-typescript-api)
13
14
  - [Search filters](#search-filters)
@@ -161,6 +162,64 @@ usable `auth_token` and `ct0` values. XTrawl does not perform interactive userna
161
162
  If you want to reuse accounts already stored in the configured SQLite database without provisioning
162
163
  new input, create the client with `provision: false`.
163
164
 
165
+ ## Own account state
166
+
167
+ SQLite is the default account store. A TypeScript application can instead implement the exported
168
+ `AccountStateStore` interface and keep account state in its own database, vault, or service:
169
+
170
+ ```ts
171
+ import { XTrawl, type AccountStateStore } from "xtrawl";
172
+
173
+ const accountStore: AccountStateStore = createMyAccountStore();
174
+
175
+ const client = await XTrawl.create({
176
+ accountStore,
177
+ dbPath: "./state/runs-and-cursors.db",
178
+ });
179
+ ```
180
+
181
+ A custom store must be passed to `await XTrawl.create()`. The synchronous constructor rejects it
182
+ because account provisioning and initialization may be asynchronous. `dbPath` remains active for
183
+ run records, pagination checkpoints, and manifest caching; only account state moves to the adapter.
184
+
185
+ The store owns these operations:
186
+
187
+ | Method | Required behavior |
188
+ | --- | --- |
189
+ | `list`, `findByUsername`, `upsert`, `delete` | Persist records; `upsert` merges fields and cookie keys while preserving omitted fields |
190
+ | `replaceAll` | Atomically replace all account records |
191
+ | `acquireLease` | Atomically choose and lease one eligible account |
192
+ | `renewLease` | Extend only the matching active lease |
193
+ | `completeLease` | Atomically clear the lease, record usage, and apply health/cooldown state |
194
+
195
+ Every method may return its result directly or in a promise. Lease operations must be atomic across
196
+ all workers or processes sharing the store. The request passed to `acquireLease` includes the current
197
+ time, generated lease ID, expiry, UTC date, authentication requirement, and configured daily limits.
198
+
199
+ ### Export and restore sessions
200
+
201
+ Use `client.accounts` when the application needs to persist or transfer reusable session state:
202
+
203
+ ```ts
204
+ const state = await client.accounts.exportState({ includeSecrets: true });
205
+ await mySecretStore.set("xtrawl/accounts", state);
206
+
207
+ const saved: unknown = await mySecretStore.get("xtrawl/accounts");
208
+ await client.accounts.restoreState(saved, { mode: "merge" });
209
+ ```
210
+
211
+ Snapshots are versioned and validated at restore time. They include cookies, authentication and
212
+ CSRF tokens, optional bearer tokens and proxies, usage counters, cooldown state, and last-error
213
+ metadata. They exclude storage IDs, active lease ownership, passwords, email credentials, and
214
+ two-factor secrets. `merge` upserts by username. `replace` calls the store's atomic `replaceAll`.
215
+
216
+ `includeSecrets: true` is intentionally required: exported state can authenticate as the supplied
217
+ accounts. Keep snapshots in an encrypted caller-owned secret store and never print, log, or commit
218
+ them.
219
+
220
+ Other async account operations include `summary()`, `list()`, `get()`, `import()`, `setProxy()`,
221
+ `repair()`, and `delete()`. Listings are redacted unless `revealSecrets` is explicitly requested.
222
+
164
223
  ## Use a proxy
165
224
 
166
225
  Set one proxy for all accounts:
@@ -258,6 +317,39 @@ operators and combined with that query. If neither date bound is supplied, XTraw
258
317
  previous 30 days. A bounded interval is split into up to `searchSplits` tasks and processed with the
259
318
  available account concurrency.
260
319
 
320
+ ### Request one search page
321
+
322
+ Use `searchPage()` when the caller needs to own cursor persistence and pagination:
323
+
324
+ ```ts
325
+ import type { SearchPageRequest } from "xtrawl";
326
+
327
+ const filters = {
328
+ since: "2026-08-01",
329
+ until: "2026-08-12",
330
+ fromUsers: ["OpenAI"],
331
+ minLikes: 10,
332
+ displayType: "Latest",
333
+ } satisfies SearchPageRequest;
334
+
335
+ let cursor: string | undefined;
336
+ do {
337
+ const page = await client.searchPage("typescript", { ...filters, cursor });
338
+ await saveTweets(page.tweets);
339
+ cursor = page.nextCursor;
340
+ } while (cursor);
341
+ ```
342
+
343
+ Each call requests one logical page through the normal account pool, including bounded retries,
344
+ account switching, cooldowns, and usage accounting. The returned `nextCursor` is opaque: store it
345
+ without modifying it and send it back with the same query, filters, display type, and date bounds.
346
+
347
+ `SearchPageRequest` supports all search filters plus `cursor` and `maxAccountSwitches`. It excludes
348
+ high-level controls such as `limit`, `resume`, `save`, and `maxEmptyPages`. `searchPage()` does not
349
+ split the date interval, write files, create a run record, or read and update SQLite checkpoints.
350
+ Use `search()` for automatic multi-page scheduling, deduplication, limits, resume state, output, and
351
+ run statistics.
352
+
261
353
  ### Read profile information
262
354
 
263
355
  ```ts
@@ -490,6 +582,10 @@ updates it as pagination advances. Checkpoint identity includes the operation an
490
582
  so a materially different request starts from its own checkpoint. A successfully completed operation
491
583
  clears its checkpoint.
492
584
 
585
+ For caller-managed search pagination, use `searchPage()`. Save its `nextCursor` in your own state and
586
+ pass it to the next call. This page-level API never reads or writes XTrawl checkpoints; the caller is
587
+ responsible for stopping when `nextCursor` is `undefined` and for avoiding repeated cursors.
588
+
493
589
  For profile and relationship methods, `initialCursors` can provide an explicit cursor keyed by the
494
590
  target identity. An explicit initial cursor takes precedence over a stored checkpoint.
495
591
 
@@ -574,6 +670,20 @@ interface SearchResult {
574
670
  and optional raw source data when available. `RunStats` reports collected count, task counts,
575
671
  failures, and retries.
576
672
 
673
+ ### `SearchPageResult`
674
+
675
+ `searchPage()` returns one normalized page and its opaque continuation cursor:
676
+
677
+ ```ts
678
+ interface SearchPageResult {
679
+ readonly tweets: readonly TweetRecord[];
680
+ readonly nextCursor: string | undefined;
681
+ }
682
+ ```
683
+
684
+ The number of posts requested per page comes from `apiPageSize`. The remote endpoint may return
685
+ fewer records.
686
+
577
687
  ### `ProfileRecord`
578
688
 
579
689
  Profile lookup returns normalized identity, biography, location, account creation time, public
@@ -652,7 +762,8 @@ XTrawl caches it in SQLite and can use a stale cached value when a refresh fails
652
762
 
653
763
  ## Understand storage and account health
654
764
 
655
- XTrawl uses SQLite for operational state:
765
+ XTrawl uses SQLite for run, checkpoint, and manifest state. SQLite is also the default account-state
766
+ adapter:
656
767
 
657
768
  - Provisioned accounts and their health status
658
769
  - Exclusive account leases and lease expiry
@@ -669,12 +780,23 @@ leased, is not cooling down, and remains within configured local limits. After t
669
780
  - A rate-limit, network, proxy, or transient failure applies the corresponding cooldown.
670
781
  - An authentication rejection marks the account unusable so it is not selected again.
671
782
 
672
- SQLite coordinates account leases so separate work does not intentionally use the same stored
673
- account at the same time.
783
+ The selected account store coordinates leases so separate work does not intentionally use the same
784
+ account at the same time. A custom implementation must provide the same atomic lease guarantees.
674
785
 
675
786
  ## Manage local state
676
787
 
677
- `client.db` provides scoped operational maintenance without exposing the storage implementation:
788
+ `client.accounts` is the storage-independent async account facade:
789
+
790
+ ```ts
791
+ console.log(await client.accounts.summary());
792
+ console.log(await client.accounts.list({ eligibleOnly: true }));
793
+
794
+ await client.accounts.setProxy("collector-one", "socks5://127.0.0.1:1080");
795
+ await client.accounts.repair("collector-one", true);
796
+ await client.accounts.delete("old-account");
797
+ ```
798
+
799
+ `client.db` provides SQLite-specific account maintenance and the run/checkpoint APIs:
678
800
 
679
801
  ```ts
680
802
  console.log(client.db.accountsSummary());
@@ -703,6 +825,7 @@ Treat the following as sensitive:
703
825
  - `auth_token`, `ct0`, bearer overrides, and complete cookie jars
704
826
  - SQLite state files containing provisioned account records
705
827
  - Proxy URLs containing usernames or passwords
828
+ - Account-state snapshots returned by `client.accounts.exportState()`
706
829
  - Collected output that may contain personal data
707
830
 
708
831
  Follow these rules:
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 XTrawl contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -4,11 +4,24 @@
4
4
  Scrape public X posts, profiles, followers, and following.
5
5
  </p>
6
6
 
7
+ <p align="center">
8
+ <a href="https://www.npmjs.com/package/xtrawl"><img alt="npm version" src="https://img.shields.io/npm/v/xtrawl"></a>
9
+ <a href="https://github.com/ensp1re/xtrawl/actions/workflows/ci.yml"><img alt="CI status" src="https://github.com/ensp1re/xtrawl/actions/workflows/ci.yml/badge.svg"></a>
10
+ <a href="LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-blue.svg"></a>
11
+ </p>
12
+
7
13
  XTrawl collects public data from X without an official API key. Search posts, read profiles and
8
14
  timelines, inspect individual posts, collect follower and following lists, and save results as CSV or
9
15
  JSON. It splits long searches, rotates authorized accounts when requests fail, and saves resumable
10
16
  progress in SQLite. Use it from TypeScript or the command line.
11
17
 
18
+ XTrawl is published on npm as [`xtrawl`](https://www.npmjs.com/package/xtrawl).
19
+
20
+ > [!WARNING]
21
+ > XTrawl is provided for educational and research purposes only. Automated scraping may violate
22
+ > X's rules and can cause any account used with XTrawl to be restricted, locked, or permanently
23
+ > banned. Use only accounts you own or are authorized to use, and proceed at your own risk.
24
+
12
25
  ## What XTrawl collects
13
26
 
14
27
  - Search results with date, account, phrase, hashtag, language, location, media, and engagement filters
@@ -17,6 +30,7 @@ progress in SQLite. Use it from TypeScript or the command line.
17
30
  - Posts from public profile timelines
18
31
  - Public followers, following, and verified-follower relationships
19
32
  - Normalized TypeScript records with optional CSV and JSON output
33
+ - Caller-controlled search pagination with opaque cursors for application-owned state
20
34
 
21
35
  XTrawl never posts, replies, likes, follows, messages, or changes account settings.
22
36
 
@@ -107,6 +121,31 @@ const verified = await client.getVerifiedFollowers(["OpenAI"], { limit: 500 });
107
121
  Targets may be usernames, `@user` handles, X or Twitter profile URLs, or typed target objects.
108
122
  Profile-timeline and relationship methods also accept numeric user IDs and `/i/user/ID` URLs.
109
123
 
124
+ ## Control search pages yourself
125
+
126
+ Use `searchPage()` when your application owns cursor storage and pagination. Each call returns one
127
+ page and an opaque `nextCursor`:
128
+
129
+ ```ts
130
+ const filters = {
131
+ since: "2026-08-01",
132
+ until: "2026-08-12",
133
+ fromUsers: ["OpenAI"],
134
+ displayType: "Latest" as const,
135
+ };
136
+
137
+ let cursor: string | undefined;
138
+ do {
139
+ const page = await client.searchPage("typescript", { ...filters, cursor });
140
+ await saveTweets(page.tweets);
141
+ cursor = page.nextCursor;
142
+ } while (cursor);
143
+ ```
144
+
145
+ Reuse the same query, filters, and explicit date bounds with every cursor. `searchPage()` supports
146
+ the advanced search filters, but does not split date intervals, save output, or read and write run
147
+ history or checkpoints. Use `search()` when XTrawl should manage those concerns automatically.
148
+
110
149
  ## Use the CLI
111
150
 
112
151
  Global options must appear before the command. Command options come after it.
@@ -151,33 +190,130 @@ flowchart LR
151
190
  Parse --> Result["Typed records"]
152
191
  Result --> Output["stdout / CSV / JSON"]
153
192
 
154
- State[("SQLite state")] <--> Client
155
- State <--> Pool
156
- State <--> Query
193
+ Accounts[("Account state store")] <--> Pool
194
+ SQLite[("SQLite runs and checkpoints")] <--> Client
195
+ SQLite <--> Query
196
+ Accounts -.->|SQLite by default| SQLite
157
197
  ```
158
198
 
159
- For each operation, XTrawl leases an eligible account from SQLite, verifies its proxy when configured,
199
+ For each operation, XTrawl leases an eligible account from the configured account store, verifies its proxy when configured,
160
200
  creates an authenticated session, builds a read-only web request, and paginates until the requested
161
201
  limit or another stop condition is reached. Searches default to the previous 30 days and divide that
162
202
  interval into concurrent tasks. Failed requests use bounded backoff, account switching, and session
163
203
  repair. Responses enter the application as unknown data and are narrowed into typed records at the
164
204
  engine boundary.
165
205
 
166
- SQLite tracks account health, request usage, leases, run history, resumable cursors, and cached
167
- operation manifests. It does not store collected posts or profiles unless you explicitly enable file
206
+ SQLite tracks run history, resumable cursors, cached operation manifests, and—by default—account
207
+ health, request usage, and leases. Applications may replace only the account-state boundary with
208
+ their own store. XTrawl does not store collected posts or profiles unless you explicitly enable file
168
209
  output.
169
210
 
170
- ## Accounts and proxies
211
+ ## Use multiple accounts
212
+
213
+ Create a local `accounts.json` file containing one object per account. Each live account needs an
214
+ `auth_token` and `ct0` value from the same authorized browser session. You can provide them as named
215
+ fields or inside `cookies`:
216
+
217
+ ```json
218
+ [
219
+ {
220
+ "username": "collector-one",
221
+ "authToken": "replace-with-auth-token",
222
+ "csrfToken": "replace-with-ct0"
223
+ },
224
+ {
225
+ "username": "collector-two",
226
+ "cookies": {
227
+ "auth_token": "replace-with-auth-token",
228
+ "ct0": "replace-with-ct0"
229
+ },
230
+ "proxy": "socks5://proxy-user:proxy-password@127.0.0.1:1080"
231
+ }
232
+ ]
233
+ ```
234
+
235
+ Keep this file outside version control. Load the complete pool with the CLI:
236
+
237
+ ```bash
238
+ npx xtrawl \
239
+ --cookies-file ./accounts.json \
240
+ --db-path ./state/xtrawl.db \
241
+ --concurrency 5 \
242
+ search "typescript" \
243
+ --limit 100
244
+ ```
245
+
246
+ Or load it from the TypeScript API:
247
+
248
+ ```ts
249
+ const client = await XTrawl.create({
250
+ accountsFile: "./accounts.json",
251
+ dbPath: "./state/xtrawl.db",
252
+ concurrency: 5,
253
+ });
254
+
255
+ console.log(client.poolSummary);
256
+ ```
257
+
258
+ XTrawl imports accounts into the configured account store and leases eligible accounts as work is
259
+ scheduled. SQLite is the default; applications that provide `accountStore` keep these records in
260
+ their own adapter. The default concurrency is five; loading more accounts does not make every
261
+ account run at once. There is no configured account-count limit, but very large pools have not been
262
+ load-tested. A failed page is retried up to three times and may switch accounts twice by default.
263
+ Rate-limit, network, proxy, and transient failures place the affected account into cooldown. A
264
+ rejected session marks the account unusable and attempts a CSRF-cookie repair before another account
265
+ is selected.
266
+
267
+ An account-level `proxy` takes precedence over the global `--proxy`. HTTP, HTTPS, and SOCKS5 proxies
268
+ are supported. XTrawl does not currently accept a separate proxy list or automatically assign and
269
+ reassign proxies; attach a proxy to each account when you need one-to-one account/proxy routing.
270
+
271
+ ### Cookie lifecycle
272
+
273
+ XTrawl does not refresh cookies after every request and does not persist `Set-Cookie` response
274
+ updates. It reuses each stored cookie jar until the session fails. `XTrawl.create()` tries to obtain a
275
+ missing `ct0` cookie when an `auth_token` is available, and an authentication failure triggers the
276
+ same repair path. If the `auth_token` has expired or been revoked, replace the stored cookies before
277
+ using that account again.
278
+
279
+ XTrawl does not perform username/password, email, or two-factor login. Supplying those fields without
280
+ valid session cookies does not make an account eligible for requests.
281
+
282
+ ## Own account state
171
283
 
172
- For one account, pass `X_AUTH_TOKEN` and `X_CSRF_TOKEN` directly. For an account pool, provide a JSON,
173
- Netscape-cookie, or delimited account file with `--cookies-file`, or use the equivalent library
174
- options. A global proxy can be set with `--proxy`; library account records may also define their own
175
- HTTP, HTTPS, or SOCKS5 proxy.
284
+ TypeScript applications can keep accounts in their own database or secret store by implementing
285
+ `AccountStateStore`. The adapter owns account records, atomic lease acquisition and completion, and
286
+ atomic replacement. Its methods may be synchronous or asynchronous; custom stores must be supplied
287
+ through `XTrawl.create()`:
176
288
 
177
- XTrawl selects only accounts that have usable authentication, are outside cooldown, are not already
178
- leased, and remain within configured daily limits. Rate-limit and transient failures trigger a
179
- cooldown and can switch the current task to another account. Authentication failures attempt a CSRF
180
- cookie repair before the account remains unusable.
289
+ ```ts
290
+ import { XTrawl, type AccountStateStore } from "xtrawl";
291
+
292
+ const accountStore: AccountStateStore = createMyAccountStore();
293
+
294
+ const client = await XTrawl.create({
295
+ accountStore,
296
+ dbPath: "./state/runs-and-cursors.db",
297
+ });
298
+ ```
299
+
300
+ `dbPath` still holds run history, checkpoints, and manifest cache. Account credentials, health,
301
+ limits, cooldowns, and leases use the custom store.
302
+
303
+ You can also move reusable session state between stores:
304
+
305
+ ```ts
306
+ const state = await client.accounts.exportState({ includeSecrets: true });
307
+ await mySecretStore.set("xtrawl/accounts", state);
308
+
309
+ const saved = await mySecretStore.get("xtrawl/accounts");
310
+ await client.accounts.restoreState(saved, { mode: "merge" });
311
+ ```
312
+
313
+ Export requires the explicit `includeSecrets: true` acknowledgement because the snapshot contains
314
+ authentication tokens, cookies, bearer overrides, and proxy credentials. It excludes account IDs,
315
+ active leases, passwords, email credentials, and two-factor secrets. `merge` updates matching
316
+ usernames; `replace` atomically replaces all accounts. Never log or commit a snapshot.
181
317
 
182
318
  ## Resume and save output
183
319
 
@@ -190,10 +326,11 @@ append records, and generated filenames include the query/date range or collecti
190
326
 
191
327
  ## Manage local state
192
328
 
193
- The library exposes safe account and run maintenance through `client.db`. It can list redacted
194
- accounts, import accounts, assign proxies, repair or disable an account, clear expired leases and
195
- checkpoints, reset local counters, inspect run history, and remove a named account. Destructive
196
- duplicate cleanup is a dry run unless explicitly enabled.
329
+ Use the async `client.accounts` facade for redacted account listing, import, proxy assignment,
330
+ repair, deletion, and explicit state export/restore. It works with SQLite and custom stores.
331
+
332
+ `client.db` provides SQLite-specific account maintenance plus run and checkpoint inspection.
333
+ Destructive duplicate cleanup is a dry run unless explicitly enabled.
197
334
 
198
335
  ## Safety and limitations
199
336
 
@@ -230,3 +367,17 @@ npm run verify
230
367
  The full gate checks formatting, linting, strict TypeScript compilation, unit tests, package and CLI
231
368
  smoke behavior, harness integrity, and context-budget limits. Live integration tests require
232
369
  caller-supplied credentials and are disabled unless explicitly enabled.
370
+
371
+ ## Contributing and security
372
+
373
+ Bug reports, documentation fixes, and read-only collection improvements are welcome. Read
374
+ [CONTRIBUTING.md](https://github.com/ensp1re/xtrawl/blob/main/CONTRIBUTING.md) before opening a pull
375
+ request. Please do not propose posting, liking, following, messaging, or other account mutation
376
+ features.
377
+
378
+ Do not report vulnerabilities or credential leaks in a public issue. Follow the private disclosure
379
+ process in [SECURITY.md](https://github.com/ensp1re/xtrawl/blob/main/SECURITY.md). By participating in
380
+ the project, you agree to follow the
381
+ [Code of Conduct](https://github.com/ensp1re/xtrawl/blob/main/CODE_OF_CONDUCT.md).
382
+
383
+ XTrawl is available under the [MIT License](LICENSE).
@@ -0,0 +1,3 @@
1
+ import type { AccountStateSnapshot } from "../domain/account-state.js";
2
+ export declare function parseAccountStateSnapshot(value: unknown): AccountStateSnapshot;
3
+ //# sourceMappingURL=account-state.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"account-state.d.ts","sourceRoot":"","sources":["../../src/auth/account-state.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,oBAAoB,EAA8B,MAAM,4BAA4B,CAAC;AAInG,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,oBAAoB,CAqB9E"}
@@ -0,0 +1,85 @@
1
+ import { normalizeProxyPayload } from "../config/validation.js";
2
+ import { AccountStateError } from "../domain/errors.js";
3
+ import { isRecord } from "../utils/guards.js";
4
+ export function parseAccountStateSnapshot(value) {
5
+ if (!isRecord(value) || value.schemaVersion !== 1 || !Array.isArray(value.accounts))
6
+ throw new AccountStateError("Account state must use schemaVersion 1 and contain accounts.");
7
+ if (typeof value.exportedAt !== "string" ||
8
+ !Number.isFinite(Date.parse(value.exportedAt)) ||
9
+ new Date(value.exportedAt).toISOString() !== value.exportedAt)
10
+ throw new AccountStateError("Account state exportedAt must be an ISO date string.");
11
+ const accounts = value.accounts.map((account, index) => parseSnapshotAccount(account, index));
12
+ const usernames = new Set();
13
+ for (const account of accounts) {
14
+ if (usernames.has(account.username))
15
+ throw new AccountStateError(`Account state contains duplicate username: ${account.username}.`);
16
+ usernames.add(account.username);
17
+ }
18
+ return {
19
+ schemaVersion: 1,
20
+ exportedAt: value.exportedAt,
21
+ accounts,
22
+ };
23
+ }
24
+ function parseSnapshotAccount(value, index) {
25
+ if (!isRecord(value) || typeof value.username !== "string" || !value.username.trim())
26
+ throw invalidAccount(index, "username must be a non-empty string");
27
+ if (!isRecord(value.cookies))
28
+ throw invalidAccount(index, "cookies must be an object");
29
+ const cookies = parseCookies(value.cookies, index);
30
+ const proxy = normalizeProxyPayload(value.proxy);
31
+ if (value.proxy !== undefined && proxy === undefined)
32
+ throw invalidAccount(index, "proxy must be a URL or host/port object");
33
+ const status = optionalStatus(value.status, index);
34
+ return {
35
+ username: value.username.trim(),
36
+ ...optionalString("authToken", value.authToken, index),
37
+ ...optionalString("csrfToken", value.csrfToken, index),
38
+ cookies,
39
+ ...optionalString("bearerToken", value.bearerToken, index),
40
+ ...(proxy === undefined ? {} : { proxy }),
41
+ ...(status === undefined ? {} : { status }),
42
+ ...optionalNonNegativeNumber("availableUntil", value.availableUntil, index),
43
+ ...optionalNonNegativeNumber("dailyRequests", value.dailyRequests, index),
44
+ ...optionalNonNegativeNumber("dailyTweets", value.dailyTweets, index),
45
+ ...optionalNonNegativeNumber("totalTweets", value.totalTweets, index),
46
+ ...optionalString("lastResetDate", value.lastResetDate, index),
47
+ ...optionalNonNegativeNumber("lastUsed", value.lastUsed, index),
48
+ ...optionalNonNegativeNumber("lastErrorCode", value.lastErrorCode, index),
49
+ ...optionalString("cooldownReason", value.cooldownReason, index),
50
+ };
51
+ }
52
+ function parseCookies(value, index) {
53
+ const cookies = {};
54
+ for (const [name, cookie] of Object.entries(value)) {
55
+ if (typeof cookie !== "string")
56
+ throw invalidAccount(index, "cookie values must be strings");
57
+ cookies[name] = cookie;
58
+ }
59
+ return cookies;
60
+ }
61
+ function optionalString(key, value, index) {
62
+ if (value === undefined)
63
+ return {};
64
+ if (typeof value !== "string")
65
+ throw invalidAccount(index, `${key} must be a string`);
66
+ return { [key]: value };
67
+ }
68
+ function optionalNonNegativeNumber(key, value, index) {
69
+ if (value === undefined)
70
+ return {};
71
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
72
+ throw invalidAccount(index, `${key} must be a non-negative number`);
73
+ return { [key]: value };
74
+ }
75
+ function optionalStatus(value, index) {
76
+ if (value === undefined)
77
+ return undefined;
78
+ if (value !== 0 && value !== 1 && value !== 2)
79
+ throw invalidAccount(index, "status must be 0, 1, or 2");
80
+ return value;
81
+ }
82
+ function invalidAccount(index, reason) {
83
+ return new AccountStateError(`Invalid account state at accounts[${index}]: ${reason}.`);
84
+ }
85
+ //# sourceMappingURL=account-state.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"account-state.js","sourceRoot":"","sources":["../../src/auth/account-state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEhE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,UAAU,yBAAyB,CAAC,KAAc;IACtD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjF,MAAM,IAAI,iBAAiB,CAAC,8DAA8D,CAAC,CAAC;IAC9F,IACE,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ;QACpC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,UAAU;QAE7D,MAAM,IAAI,iBAAiB,CAAC,sDAAsD,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,oBAAoB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC9F,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;YACjC,MAAM,IAAI,iBAAiB,CAAC,8CAA8C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACjG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc,EAAE,KAAa;IACzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;QAClF,MAAM,cAAc,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC;IACrE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,MAAM,cAAc,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClD,MAAM,cAAc,CAAC,KAAK,EAAE,yCAAyC,CAAC,CAAC;IACzE,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACnD,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE;QAC/B,GAAG,cAAc,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC;QACtD,GAAG,cAAc,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC;QACtD,OAAO;QACP,GAAG,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC;QAC1D,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3C,GAAG,yBAAyB,CAAC,gBAAgB,EAAE,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC;QAC3E,GAAG,yBAAyB,CAAC,eAAe,EAAE,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC;QACzE,GAAG,yBAAyB,CAAC,aAAa,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC;QACrE,GAAG,yBAAyB,CAAC,aAAa,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC;QACrE,GAAG,cAAc,CAAC,eAAe,EAAE,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC;QAC9D,GAAG,yBAAyB,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC;QAC/D,GAAG,yBAAyB,CAAC,eAAe,EAAE,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC;QACzE,GAAG,cAAc,CAAC,gBAAgB,EAAE,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC;KACjE,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAA8B,EAAE,KAAa;IACjE,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,IAAI,OAAO,MAAM,KAAK,QAAQ;YAAE,MAAM,cAAc,CAAC,KAAK,EAAE,+BAA+B,CAAC,CAAC;QAC7F,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAmB,GAAM,EAAE,KAAc,EAAE,KAAa;IAC7E,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,GAAG,mBAAmB,CAAC,CAAC;IACtF,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAgC,CAAC;AACxD,CAAC;AAED,SAAS,yBAAyB,CAChC,GAAM,EACN,KAAc,EACd,KAAa;IAEb,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QACnE,MAAM,cAAc,CAAC,KAAK,EAAE,GAAG,GAAG,gCAAgC,CAAC,CAAC;IACtE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAgC,CAAC;AACxD,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,MAAM,cAAc,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;IACxG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAa,EAAE,MAAc;IACnD,OAAO,IAAI,iBAAiB,CAAC,qCAAqC,KAAK,MAAM,MAAM,GAAG,CAAC,CAAC;AAC1F,CAAC"}
@@ -0,0 +1,31 @@
1
+ import type { ClientConfig } from "../config/types.js";
2
+ import type { AccountStateExportOptions, AccountStateRestoreOptions, AccountStateSnapshot, AccountStateStore } from "../domain/account-state.js";
3
+ import type { AccountRecord, AccountSummary, ProxySettings } from "../domain/accounts.js";
4
+ import { type AccountImportOptions, type AccountListOptions } from "./database.js";
5
+ export declare class XTrawlAccounts {
6
+ private readonly store;
7
+ private readonly config;
8
+ private readonly location;
9
+ private readonly onAccountsChanged?;
10
+ private cachedAccounts;
11
+ constructor(store: AccountStateStore, config: ClientConfig, location: string, onAccountsChanged?: ((accounts: readonly AccountRecord[]) => void) | undefined);
12
+ refresh(): Promise<void>;
13
+ initialize(accounts: readonly AccountRecord[]): void;
14
+ inspectCached(): readonly Record<string, unknown>[];
15
+ summary(): Promise<AccountSummary>;
16
+ list(options?: AccountListOptions): Promise<readonly Record<string, unknown>[]>;
17
+ get(username: string, options?: Pick<AccountListOptions, "includeCookies" | "revealSecrets">): Promise<Record<string, unknown> | undefined>;
18
+ import(options: AccountImportOptions): Promise<{
19
+ readonly processed: number;
20
+ readonly eligible: number;
21
+ }>;
22
+ delete(username: string): Promise<boolean>;
23
+ setProxy(username: string, proxy: string | ProxySettings): Promise<boolean>;
24
+ repair(username: string, forceRefresh?: boolean): Promise<boolean>;
25
+ exportState(options: AccountStateExportOptions): Promise<AccountStateSnapshot>;
26
+ restoreState(state: unknown, options?: AccountStateRestoreOptions): Promise<{
27
+ readonly restored: number;
28
+ readonly mode: "merge" | "replace";
29
+ }>;
30
+ }
31
+ //# sourceMappingURL=accounts.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"accounts.d.ts","sourceRoot":"","sources":["../../src/client/accounts.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,oBAAoB,EAEpB,iBAAiB,EAClB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAG1F,OAAO,EAAiB,KAAK,oBAAoB,EAAE,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAElG,qBAAa,cAAc;IAIvB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IANrC,OAAO,CAAC,cAAc,CAAgC;gBAGnC,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE,YAAY,EACpB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,CAAC,GAAE,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,KAAK,IAAI,aAAA;IAGtE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9B,UAAU,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;IAI7C,OAAO,IAAI,OAAO,CAAC,cAAc,CAAC;IAKlC,IAAI,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IASnF,GAAG,CACd,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,GAAG,eAAe,CAAM,GACzE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAKlC,MAAM,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC;QAC1D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B,CAAC;IAOW,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAM1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IAQ3E,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,UAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAqBhE,WAAW,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAW9E,YAAY,CACvB,KAAK,EAAE,OAAO,EACd,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC;QAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,SAAS,CAAA;KAAE,CAAC;CAW9E"}