xtrawl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DOCUMENTATION.md +774 -0
- package/README.md +232 -0
- package/dist/auth/bootstrap.d.ts +4 -0
- package/dist/auth/bootstrap.d.ts.map +1 -0
- package/dist/auth/bootstrap.js +29 -0
- package/dist/auth/bootstrap.js.map +1 -0
- package/dist/auth/cookies.d.ts +7 -0
- package/dist/auth/cookies.d.ts.map +1 -0
- package/dist/auth/cookies.js +83 -0
- package/dist/auth/cookies.js.map +1 -0
- package/dist/auth/loaders.d.ts +6 -0
- package/dist/auth/loaders.d.ts.map +1 -0
- package/dist/auth/loaders.js +69 -0
- package/dist/auth/loaders.js.map +1 -0
- package/dist/auth/material.d.ts +3 -0
- package/dist/auth/material.d.ts.map +1 -0
- package/dist/auth/material.js +20 -0
- package/dist/auth/material.js.map +1 -0
- package/dist/auth/records.d.ts +4 -0
- package/dist/auth/records.d.ts.map +1 -0
- package/dist/auth/records.js +38 -0
- package/dist/auth/records.js.map +1 -0
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +30 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +85 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/parser.d.ts +22 -0
- package/dist/cli/parser.d.ts.map +1 -0
- package/dist/cli/parser.js +275 -0
- package/dist/cli/parser.js.map +1 -0
- package/dist/client/client.d.ts +29 -0
- package/dist/client/client.d.ts.map +1 -0
- package/dist/client/client.js +192 -0
- package/dist/client/client.js.map +1 -0
- package/dist/client/collectors.d.ts +15 -0
- package/dist/client/collectors.d.ts.map +1 -0
- package/dist/client/collectors.js +185 -0
- package/dist/client/collectors.js.map +1 -0
- package/dist/client/database.d.ts +56 -0
- package/dist/client/database.d.ts.map +1 -0
- package/dist/client/database.js +146 -0
- package/dist/client/database.js.map +1 -0
- package/dist/client/profiles.d.ts +5 -0
- package/dist/client/profiles.d.ts.map +1 -0
- package/dist/client/profiles.js +36 -0
- package/dist/client/profiles.js.map +1 -0
- package/dist/client/search.d.ts +21 -0
- package/dist/client/search.d.ts.map +1 -0
- package/dist/client/search.js +139 -0
- package/dist/client/search.js.map +1 -0
- package/dist/client/types.d.ts +21 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/client/types.js +2 -0
- package/dist/client/types.js.map +1 -0
- package/dist/config/defaults.d.ts +3 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +38 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/environment.d.ts +5 -0
- package/dist/config/environment.d.ts.map +1 -0
- package/dist/config/environment.js +62 -0
- package/dist/config/environment.js.map +1 -0
- package/dist/config/types.d.ts +47 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/config/validation.d.ts +6 -0
- package/dist/config/validation.d.ts.map +1 -0
- package/dist/config/validation.js +147 -0
- package/dist/config/validation.js.map +1 -0
- package/dist/domain/accounts.d.ts +64 -0
- package/dist/domain/accounts.d.ts.map +1 -0
- package/dist/domain/accounts.js +2 -0
- package/dist/domain/accounts.js.map +1 -0
- package/dist/domain/errors.d.ts +61 -0
- package/dist/domain/errors.d.ts.map +1 -0
- package/dist/domain/errors.js +89 -0
- package/dist/domain/errors.js.map +1 -0
- package/dist/domain/http.d.ts +36 -0
- package/dist/domain/http.d.ts.map +1 -0
- package/dist/domain/http.js +2 -0
- package/dist/domain/http.js.map +1 -0
- package/dist/domain/manifest.d.ts +23 -0
- package/dist/domain/manifest.d.ts.map +1 -0
- package/dist/domain/manifest.js +2 -0
- package/dist/domain/manifest.js.map +1 -0
- package/dist/domain/records.d.ts +67 -0
- package/dist/domain/records.d.ts.map +1 -0
- package/dist/domain/records.js +2 -0
- package/dist/domain/records.js.map +1 -0
- package/dist/domain/requests.d.ts +87 -0
- package/dist/domain/requests.d.ts.map +1 -0
- package/dist/domain/requests.js +2 -0
- package/dist/domain/requests.js.map +1 -0
- package/dist/engine/api-engine.d.ts +25 -0
- package/dist/engine/api-engine.d.ts.map +1 -0
- package/dist/engine/api-engine.js +98 -0
- package/dist/engine/api-engine.js.map +1 -0
- package/dist/engine/extractors.d.ts +27 -0
- package/dist/engine/extractors.d.ts.map +1 -0
- package/dist/engine/extractors.js +248 -0
- package/dist/engine/extractors.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/manifest/default-manifest.d.ts +3 -0
- package/dist/manifest/default-manifest.d.ts.map +1 -0
- package/dist/manifest/default-manifest.js +46 -0
- package/dist/manifest/default-manifest.js.map +1 -0
- package/dist/manifest/model.d.ts +3 -0
- package/dist/manifest/model.d.ts.map +1 -0
- package/dist/manifest/model.js +48 -0
- package/dist/manifest/model.js.map +1 -0
- package/dist/manifest/provider.d.ts +19 -0
- package/dist/manifest/provider.d.ts.map +1 -0
- package/dist/manifest/provider.js +94 -0
- package/dist/manifest/provider.js.map +1 -0
- package/dist/manifest/scraper.d.ts +11 -0
- package/dist/manifest/scraper.d.ts.map +1 -0
- package/dist/manifest/scraper.js +130 -0
- package/dist/manifest/scraper.js.map +1 -0
- package/dist/output/csv-writer.d.ts +2 -0
- package/dist/output/csv-writer.d.ts.map +1 -0
- package/dist/output/csv-writer.js +94 -0
- package/dist/output/csv-writer.js.map +1 -0
- package/dist/output/json-writer.d.ts +2 -0
- package/dist/output/json-writer.d.ts.map +1 -0
- package/dist/output/json-writer.js +29 -0
- package/dist/output/json-writer.js.map +1 -0
- package/dist/output/names.d.ts +4 -0
- package/dist/output/names.d.ts.map +1 -0
- package/dist/output/names.js +22 -0
- package/dist/output/names.js.map +1 -0
- package/dist/output/writer.d.ts +7 -0
- package/dist/output/writer.d.ts.map +1 -0
- package/dist/output/writer.js +33 -0
- package/dist/output/writer.js.map +1 -0
- package/dist/pool/account-pool.d.ts +29 -0
- package/dist/pool/account-pool.d.ts.map +1 -0
- package/dist/pool/account-pool.js +145 -0
- package/dist/pool/account-pool.js.map +1 -0
- package/dist/pool/cooldown.d.ts +18 -0
- package/dist/pool/cooldown.d.ts.map +1 -0
- package/dist/pool/cooldown.js +38 -0
- package/dist/pool/cooldown.js.map +1 -0
- package/dist/pool/limiter.d.ts +11 -0
- package/dist/pool/limiter.d.ts.map +1 -0
- package/dist/pool/limiter.js +27 -0
- package/dist/pool/limiter.js.map +1 -0
- package/dist/query/builder.d.ts +19 -0
- package/dist/query/builder.d.ts.map +1 -0
- package/dist/query/builder.js +110 -0
- package/dist/query/builder.js.map +1 -0
- package/dist/query/hash.d.ts +2 -0
- package/dist/query/hash.d.ts.map +1 -0
- package/dist/query/hash.js +6 -0
- package/dist/query/hash.js.map +1 -0
- package/dist/query/normalize.d.ts +39 -0
- package/dist/query/normalize.d.ts.map +1 -0
- package/dist/query/normalize.js +176 -0
- package/dist/query/normalize.js.map +1 -0
- package/dist/query/targets.d.ts +11 -0
- package/dist/query/targets.d.ts.map +1 -0
- package/dist/query/targets.js +96 -0
- package/dist/query/targets.js.map +1 -0
- package/dist/runner/index.d.ts +4 -0
- package/dist/runner/index.d.ts.map +1 -0
- package/dist/runner/index.js +4 -0
- package/dist/runner/index.js.map +1 -0
- package/dist/runner/retry.d.ts +7 -0
- package/dist/runner/retry.d.ts.map +1 -0
- package/dist/runner/retry.js +26 -0
- package/dist/runner/retry.js.map +1 -0
- package/dist/runner/runner.d.ts +20 -0
- package/dist/runner/runner.d.ts.map +1 -0
- package/dist/runner/runner.js +37 -0
- package/dist/runner/runner.js.map +1 -0
- package/dist/runner/task-queue.d.ts +19 -0
- package/dist/runner/task-queue.d.ts.map +1 -0
- package/dist/runner/task-queue.js +58 -0
- package/dist/runner/task-queue.js.map +1 -0
- package/dist/storage/account-repository.d.ts +50 -0
- package/dist/storage/account-repository.d.ts.map +1 -0
- package/dist/storage/account-repository.js +171 -0
- package/dist/storage/account-repository.js.map +1 -0
- package/dist/storage/account-row.d.ts +4 -0
- package/dist/storage/account-row.d.ts.map +1 -0
- package/dist/storage/account-row.js +46 -0
- package/dist/storage/account-row.js.map +1 -0
- package/dist/storage/checkpoint-repository.d.ts +10 -0
- package/dist/storage/checkpoint-repository.d.ts.map +1 -0
- package/dist/storage/checkpoint-repository.js +30 -0
- package/dist/storage/checkpoint-repository.js.map +1 -0
- package/dist/storage/database.d.ts +18 -0
- package/dist/storage/database.d.ts.map +1 -0
- package/dist/storage/database.js +42 -0
- package/dist/storage/database.js.map +1 -0
- package/dist/storage/index.d.ts +15 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +18 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/manifest-repository.d.ts +9 -0
- package/dist/storage/manifest-repository.d.ts.map +1 -0
- package/dist/storage/manifest-repository.js +22 -0
- package/dist/storage/manifest-repository.js.map +1 -0
- package/dist/storage/run-repository.d.ts +24 -0
- package/dist/storage/run-repository.d.ts.map +1 -0
- package/dist/storage/run-repository.js +51 -0
- package/dist/storage/run-repository.js.map +1 -0
- package/dist/storage/schema.d.ts +3 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +53 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/transport/graphql.d.ts +8 -0
- package/dist/transport/graphql.d.ts.map +1 -0
- package/dist/transport/graphql.js +104 -0
- package/dist/transport/graphql.js.map +1 -0
- package/dist/transport/proxy.d.ts +5 -0
- package/dist/transport/proxy.d.ts.map +1 -0
- package/dist/transport/proxy.js +36 -0
- package/dist/transport/proxy.js.map +1 -0
- package/dist/transport/session.d.ts +25 -0
- package/dist/transport/session.d.ts.map +1 -0
- package/dist/transport/session.js +138 -0
- package/dist/transport/session.js.map +1 -0
- package/dist/transport/transaction-id.d.ts +20 -0
- package/dist/transport/transaction-id.d.ts.map +1 -0
- package/dist/transport/transaction-id.js +74 -0
- package/dist/transport/transaction-id.js.map +1 -0
- package/dist/utils/fingerprint.d.ts +3 -0
- package/dist/utils/fingerprint.d.ts.map +1 -0
- package/dist/utils/fingerprint.js +20 -0
- package/dist/utils/fingerprint.js.map +1 -0
- package/dist/utils/guards.d.ts +8 -0
- package/dist/utils/guards.d.ts.map +1 -0
- package/dist/utils/guards.js +38 -0
- package/dist/utils/guards.js.map +1 -0
- package/docs/architecture/overview.md +59 -0
- package/docs/decisions/0001-runtime-and-state.md +21 -0
- package/docs/decisions/0002-modular-boundaries.md +16 -0
- package/docs/plans/initial-build.md +18 -0
- package/docs/product/specification.md +87 -0
- package/docs/security/data-boundary.md +27 -0
- package/package.json +89 -0
package/README.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
<h1 align="center">XTrawl</h1>
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
Scrape public X posts, profiles, followers, and following.
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
XTrawl collects public data from X without an official API key. Search posts, read profiles and
|
|
8
|
+
timelines, inspect individual posts, collect follower and following lists, and save results as CSV or
|
|
9
|
+
JSON. It splits long searches, rotates authorized accounts when requests fail, and saves resumable
|
|
10
|
+
progress in SQLite. Use it from TypeScript or the command line.
|
|
11
|
+
|
|
12
|
+
## What XTrawl collects
|
|
13
|
+
|
|
14
|
+
- Search results with date, account, phrase, hashtag, language, location, media, and engagement filters
|
|
15
|
+
- Individual posts by ID or status URL
|
|
16
|
+
- Public profile details for one or more usernames
|
|
17
|
+
- Posts from public profile timelines
|
|
18
|
+
- Public followers, following, and verified-follower relationships
|
|
19
|
+
- Normalized TypeScript records with optional CSV and JSON output
|
|
20
|
+
|
|
21
|
+
XTrawl never posts, replies, likes, follows, messages, or changes account settings.
|
|
22
|
+
|
|
23
|
+
## Quick start
|
|
24
|
+
|
|
25
|
+
XTrawl requires Node.js 22.5 or newer and npm.
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npm install xtrawl
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Provide session cookies from an X account you own or are authorized to use. Keep them in environment
|
|
32
|
+
variables; never commit them.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
export X_AUTH_TOKEN="your-auth-token"
|
|
36
|
+
export X_CSRF_TOKEN="your-ct0-token"
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Run a search:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
npx xtrawl search "typescript" \
|
|
43
|
+
--since 2026-01-01 \
|
|
44
|
+
--display-type Latest \
|
|
45
|
+
--limit 100 \
|
|
46
|
+
--save \
|
|
47
|
+
--save-format json \
|
|
48
|
+
--pretty
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`--pretty` prints indented JSON to stdout. With `--save`, XTrawl also writes records to `outputs/`
|
|
52
|
+
by default.
|
|
53
|
+
|
|
54
|
+
## Use the TypeScript API
|
|
55
|
+
|
|
56
|
+
Import the package from TypeScript or JavaScript:
|
|
57
|
+
|
|
58
|
+
```ts
|
|
59
|
+
import { XTrawl } from "xtrawl";
|
|
60
|
+
|
|
61
|
+
const client = await XTrawl.create({
|
|
62
|
+
authToken: process.env.X_AUTH_TOKEN,
|
|
63
|
+
csrfToken: process.env.X_CSRF_TOKEN,
|
|
64
|
+
dbPath: "xtrawl.db",
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
const result = await client.search("typescript", {
|
|
69
|
+
since: "2026-01-01",
|
|
70
|
+
fromUsers: ["OpenAI"],
|
|
71
|
+
minLikes: 10,
|
|
72
|
+
displayType: "Latest",
|
|
73
|
+
limit: 100,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
console.log(result.tweets);
|
|
77
|
+
console.log(result.stats);
|
|
78
|
+
} finally {
|
|
79
|
+
client.close();
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Use `XTrawl.create()` for normal startup. If an account has `auth_token` but no `ct0` value, this
|
|
84
|
+
factory attempts to bootstrap the missing CSRF cookie before the first request.
|
|
85
|
+
|
|
86
|
+
## Common tasks
|
|
87
|
+
|
|
88
|
+
```ts
|
|
89
|
+
// Resolve public profiles.
|
|
90
|
+
const profiles = await client.getUserInfo(["OpenAI", "github"]);
|
|
91
|
+
|
|
92
|
+
// Inspect one public post.
|
|
93
|
+
const tweet = await client.getTweet("https://x.com/OpenAI/status/1234567890");
|
|
94
|
+
|
|
95
|
+
// Collect posts from public profile timelines.
|
|
96
|
+
const timeline = await client.getProfileTweets(["OpenAI"], {
|
|
97
|
+
perProfileLimit: 100,
|
|
98
|
+
resume: true,
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// Collect public relationships.
|
|
102
|
+
const followers = await client.getFollowers(["OpenAI"], { limit: 500 });
|
|
103
|
+
const following = await client.getFollowing(["OpenAI"], { limit: 500 });
|
|
104
|
+
const verified = await client.getVerifiedFollowers(["OpenAI"], { limit: 500 });
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Targets may be usernames, `@user` handles, X or Twitter profile URLs, or typed target objects.
|
|
108
|
+
Profile-timeline and relationship methods also accept numeric user IDs and `/i/user/ID` URLs.
|
|
109
|
+
|
|
110
|
+
## Use the CLI
|
|
111
|
+
|
|
112
|
+
Global options must appear before the command. Command options come after it.
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
# Inspect profiles
|
|
116
|
+
npx xtrawl user-info OpenAI github --pretty
|
|
117
|
+
|
|
118
|
+
# Inspect a post
|
|
119
|
+
npx xtrawl tweet 1234567890 --pretty
|
|
120
|
+
|
|
121
|
+
# Save posts from two profiles as CSV and JSON
|
|
122
|
+
npx xtrawl profile-tweets OpenAI github \
|
|
123
|
+
--per-profile-limit 100 \
|
|
124
|
+
--resume \
|
|
125
|
+
--save \
|
|
126
|
+
--save-format both
|
|
127
|
+
|
|
128
|
+
# Use an account file, a proxy, and a separate state database
|
|
129
|
+
npx xtrawl \
|
|
130
|
+
--cookies-file ./accounts.json \
|
|
131
|
+
--proxy http://127.0.0.1:8080 \
|
|
132
|
+
--db-path ./state/xtrawl.db \
|
|
133
|
+
followers OpenAI \
|
|
134
|
+
--limit 500
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Available commands are `search`, `tweet`, `profile-tweets`, `followers`, `following`,
|
|
138
|
+
`verified-followers`, and `user-info`.
|
|
139
|
+
|
|
140
|
+
## How XTrawl works
|
|
141
|
+
|
|
142
|
+
```mermaid
|
|
143
|
+
flowchart LR
|
|
144
|
+
API["TypeScript API"] --> Client["XTrawl client"]
|
|
145
|
+
CLI["Command-line interface"] --> Client
|
|
146
|
+
Client --> Query["Query builder and interval scheduler"]
|
|
147
|
+
Query --> Pool["Account pool and retries"]
|
|
148
|
+
Pool --> Session["Authenticated read-only session"]
|
|
149
|
+
Session --> X["X web GraphQL endpoints"]
|
|
150
|
+
X --> Parse["Runtime guards and typed extractors"]
|
|
151
|
+
Parse --> Result["Typed records"]
|
|
152
|
+
Result --> Output["stdout / CSV / JSON"]
|
|
153
|
+
|
|
154
|
+
State[("SQLite state")] <--> Client
|
|
155
|
+
State <--> Pool
|
|
156
|
+
State <--> Query
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
For each operation, XTrawl leases an eligible account from SQLite, verifies its proxy when configured,
|
|
160
|
+
creates an authenticated session, builds a read-only web request, and paginates until the requested
|
|
161
|
+
limit or another stop condition is reached. Searches default to the previous 30 days and divide that
|
|
162
|
+
interval into concurrent tasks. Failed requests use bounded backoff, account switching, and session
|
|
163
|
+
repair. Responses enter the application as unknown data and are narrowed into typed records at the
|
|
164
|
+
engine boundary.
|
|
165
|
+
|
|
166
|
+
SQLite tracks account health, request usage, leases, run history, resumable cursors, and cached
|
|
167
|
+
operation manifests. It does not store collected posts or profiles unless you explicitly enable file
|
|
168
|
+
output.
|
|
169
|
+
|
|
170
|
+
## Accounts and proxies
|
|
171
|
+
|
|
172
|
+
For one account, pass `X_AUTH_TOKEN` and `X_CSRF_TOKEN` directly. For an account pool, provide a JSON,
|
|
173
|
+
Netscape-cookie, or delimited account file with `--cookies-file`, or use the equivalent library
|
|
174
|
+
options. A global proxy can be set with `--proxy`; library account records may also define their own
|
|
175
|
+
HTTP, HTTPS, or SOCKS5 proxy.
|
|
176
|
+
|
|
177
|
+
XTrawl selects only accounts that have usable authentication, are outside cooldown, are not already
|
|
178
|
+
leased, and remain within configured daily limits. Rate-limit and transient failures trigger a
|
|
179
|
+
cooldown and can switch the current task to another account. Authentication failures attempt a CSRF
|
|
180
|
+
cookie repair before the account remains unusable.
|
|
181
|
+
|
|
182
|
+
## Resume and save output
|
|
183
|
+
|
|
184
|
+
Set `resume: true` or pass `--resume` to persist pagination cursors. XTrawl clears a checkpoint after
|
|
185
|
+
that operation finishes successfully and retains it when a run is interrupted.
|
|
186
|
+
|
|
187
|
+
Set `save: true` or pass `--save` to write records. Supported formats are `csv`, `json`, and `both`.
|
|
188
|
+
The default output directory is `outputs/`; use `saveDir` or `--save-dir` to change it. Repeated saves
|
|
189
|
+
append records, and generated filenames include the query/date range or collection targets.
|
|
190
|
+
|
|
191
|
+
## Manage local state
|
|
192
|
+
|
|
193
|
+
The library exposes safe account and run maintenance through `client.db`. It can list redacted
|
|
194
|
+
accounts, import accounts, assign proxies, repair or disable an account, clear expired leases and
|
|
195
|
+
checkpoints, reset local counters, inspect run history, and remove a named account. Destructive
|
|
196
|
+
duplicate cleanup is a dry run unless explicitly enabled.
|
|
197
|
+
|
|
198
|
+
## Safety and limitations
|
|
199
|
+
|
|
200
|
+
- Collect only public data with accounts you own or are authorized to use.
|
|
201
|
+
- Keep cookies, tokens, databases, and output files out of version control and secret-sharing paths.
|
|
202
|
+
- XTrawl cannot access private or protected content that the active session cannot already view.
|
|
203
|
+
- X's undocumented web endpoints and operation identifiers can change without notice.
|
|
204
|
+
- Session cookies expire, and account availability or platform rate limits can interrupt a run.
|
|
205
|
+
- Conservative local limits reduce pressure on accounts but do not guarantee uninterrupted access.
|
|
206
|
+
- You are responsible for following platform terms and applicable privacy, data, and automation laws.
|
|
207
|
+
|
|
208
|
+
## Documentation
|
|
209
|
+
|
|
210
|
+
- [Complete usage and API guide](DOCUMENTATION.md)
|
|
211
|
+
- [Product specification](docs/product/specification.md)
|
|
212
|
+
- [Architecture overview](docs/architecture/overview.md)
|
|
213
|
+
- [Security and data boundary](docs/security/data-boundary.md)
|
|
214
|
+
- [Architecture decisions](docs/decisions/)
|
|
215
|
+
|
|
216
|
+
## Development
|
|
217
|
+
|
|
218
|
+
Run the fast local checks while working:
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
npm run check
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Run the complete delivery gate before publishing a change:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
npm run verify
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
The full gate checks formatting, linting, strict TypeScript compilation, unit tests, package and CLI
|
|
231
|
+
smoke behavior, harness integrity, and context-budget limits. Live integration tests require
|
|
232
|
+
caller-supplied credentials and are disabled unless explicitly enabled.
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { fetch as undiciFetch } from "undici";
|
|
2
|
+
import type { CookieMap, ProxySettings } from "../domain/accounts.js";
|
|
3
|
+
export declare function bootstrapCookiesFromAuthToken(authToken: string, fetcher?: typeof undiciFetch, proxy?: string | ProxySettings): Promise<CookieMap | undefined>;
|
|
4
|
+
//# sourceMappingURL=bootstrap.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bootstrap.d.ts","sourceRoot":"","sources":["../../src/auth/bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtE,wBAAsB,6BAA6B,CACjD,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAO,WAAyB,EACzC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,GAC7B,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAsBhC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { fetch as undiciFetch } from "undici";
|
|
2
|
+
import { proxyDispatcher } from "../transport/proxy.js";
|
|
3
|
+
export async function bootstrapCookiesFromAuthToken(authToken, fetcher = undiciFetch, proxy) {
|
|
4
|
+
const dispatcher = proxyDispatcher(proxy);
|
|
5
|
+
try {
|
|
6
|
+
const response = await fetcher("https://x.com/home", {
|
|
7
|
+
headers: { Cookie: `auth_token=${authToken}`, "User-Agent": "Mozilla/5.0" },
|
|
8
|
+
redirect: "follow",
|
|
9
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
10
|
+
});
|
|
11
|
+
if (response.status >= 400)
|
|
12
|
+
return undefined;
|
|
13
|
+
const cookies = { auth_token: authToken };
|
|
14
|
+
const setCookies = typeof response.headers.getSetCookie === "function" ? response.headers.getSetCookie() : [];
|
|
15
|
+
for (const header of setCookies) {
|
|
16
|
+
const first = header.split(";", 1)[0];
|
|
17
|
+
if (!first)
|
|
18
|
+
continue;
|
|
19
|
+
const separator = first.indexOf("=");
|
|
20
|
+
if (separator > 0)
|
|
21
|
+
cookies[first.slice(0, separator)] = first.slice(separator + 1);
|
|
22
|
+
}
|
|
23
|
+
return cookies.ct0 ? cookies : undefined;
|
|
24
|
+
}
|
|
25
|
+
finally {
|
|
26
|
+
await dispatcher?.close();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=bootstrap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bootstrap.js","sourceRoot":"","sources":["../../src/auth/bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,SAAiB,EACjB,UAA8B,WAAW,EACzC,KAA8B;IAE9B,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,oBAAoB,EAAE;YACnD,OAAO,EAAE,EAAE,MAAM,EAAE,cAAc,SAAS,EAAE,EAAE,YAAY,EAAE,aAAa,EAAE;YAC3E,QAAQ,EAAE,QAAQ;YAClB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtC,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,SAAS,CAAC;QAC7C,MAAM,OAAO,GAA2B,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;QAClE,MAAM,UAAU,GACd,OAAO,QAAQ,CAAC,OAAO,CAAC,YAAY,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,SAAS,GAAG,CAAC;gBAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3C,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { CookieMap } from "../domain/accounts.js";
|
|
2
|
+
export declare function cookiesToMap(value: unknown): CookieMap;
|
|
3
|
+
export declare function parseCookieHeader(value: string): CookieMap;
|
|
4
|
+
export declare function parseNetscapeCookies(value: string): CookieMap;
|
|
5
|
+
export declare function normalizeCookiesPayload(value: unknown): CookieMap;
|
|
6
|
+
export declare function deriveUsername(username: string | undefined, email: string | undefined, authToken: string | undefined, cookies: CookieMap): string | undefined;
|
|
7
|
+
//# sourceMappingURL=cookies.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookies.d.ts","sourceRoot":"","sources":["../../src/auth/cookies.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CActD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAW1D;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAa7D;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CAmBjE;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,EAC5B,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,OAAO,EAAE,SAAS,GACjB,MAAM,GAAG,SAAS,CAQpB"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { asString, isRecord } from "../utils/guards.js";
|
|
3
|
+
export function cookiesToMap(value) {
|
|
4
|
+
if (isRecord(value)) {
|
|
5
|
+
return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, String(child ?? "")]));
|
|
6
|
+
}
|
|
7
|
+
if (Array.isArray(value)) {
|
|
8
|
+
const pairs = value.flatMap((item) => {
|
|
9
|
+
if (!isRecord(item))
|
|
10
|
+
return [];
|
|
11
|
+
const name = asString(item.name);
|
|
12
|
+
const child = asString(item.value);
|
|
13
|
+
return name && child !== undefined ? [[name, child]] : [];
|
|
14
|
+
});
|
|
15
|
+
return Object.fromEntries(pairs);
|
|
16
|
+
}
|
|
17
|
+
return {};
|
|
18
|
+
}
|
|
19
|
+
export function parseCookieHeader(value) {
|
|
20
|
+
const raw = value.trim().replace(/^cookie:\s*/iu, "");
|
|
21
|
+
const entries = [];
|
|
22
|
+
for (const part of raw.split(";")) {
|
|
23
|
+
const separator = part.indexOf("=");
|
|
24
|
+
if (separator <= 0)
|
|
25
|
+
continue;
|
|
26
|
+
const name = part.slice(0, separator).trim();
|
|
27
|
+
const cookie = part.slice(separator + 1).trim();
|
|
28
|
+
if (name && cookie)
|
|
29
|
+
entries.push([name, cookie]);
|
|
30
|
+
}
|
|
31
|
+
return Object.fromEntries(entries);
|
|
32
|
+
}
|
|
33
|
+
export function parseNetscapeCookies(value) {
|
|
34
|
+
const out = {};
|
|
35
|
+
for (const raw of value.split(/\r?\n/u)) {
|
|
36
|
+
let line = raw.trim();
|
|
37
|
+
if (!line || (line.startsWith("#") && !line.startsWith("#HttpOnly_")))
|
|
38
|
+
continue;
|
|
39
|
+
line = line.replace(/^#HttpOnly_/u, "");
|
|
40
|
+
const parts = line.split(/\s+/u);
|
|
41
|
+
if (parts.length < 7)
|
|
42
|
+
continue;
|
|
43
|
+
const name = parts[5];
|
|
44
|
+
const cookie = parts.slice(6).join(" ");
|
|
45
|
+
if (name && cookie)
|
|
46
|
+
out[name] = cookie;
|
|
47
|
+
}
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
export function normalizeCookiesPayload(value) {
|
|
51
|
+
if (value === undefined || value === null)
|
|
52
|
+
return {};
|
|
53
|
+
if (typeof value === "string") {
|
|
54
|
+
const text = value.trim();
|
|
55
|
+
if (!text)
|
|
56
|
+
return {};
|
|
57
|
+
if (text.includes("\t") ||
|
|
58
|
+
text.includes("#HttpOnly_") ||
|
|
59
|
+
text.startsWith("# Netscape") ||
|
|
60
|
+
text.split(/\r?\n/u).some((line) => line.trim().split(/\s+/u).length >= 7))
|
|
61
|
+
return parseNetscapeCookies(text);
|
|
62
|
+
try {
|
|
63
|
+
return normalizeCookiesPayload(JSON.parse(text));
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return text.includes("=") ? parseCookieHeader(text) : { auth_token: text };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return cookiesToMap(value);
|
|
70
|
+
}
|
|
71
|
+
export function deriveUsername(username, email, authToken, cookies) {
|
|
72
|
+
if (username?.trim())
|
|
73
|
+
return username.trim();
|
|
74
|
+
if (email?.includes("@"))
|
|
75
|
+
return email.split("@", 1)[0]?.trim() || undefined;
|
|
76
|
+
if (authToken?.trim())
|
|
77
|
+
return `auth_${createHash("sha1").update(authToken).digest("hex").slice(0, 12)}`;
|
|
78
|
+
const serialized = JSON.stringify(cookies, Object.keys(cookies).sort());
|
|
79
|
+
return Object.keys(cookies).length > 0
|
|
80
|
+
? `cookie_${createHash("sha1").update(serialized).digest("hex").slice(0, 12)}`
|
|
81
|
+
: undefined;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=cookies.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookies.js","sourceRoot":"","sources":["../../src/auth/cookies.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAExD,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACnC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,OAAO,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,IAAI,CAAC;YAAE,SAAS;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,IAAI,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAAE,SAAS;QAChF,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,IAAI,IAAI,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;IACzC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,IACE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACnB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;YAE1E,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC;YACH,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,QAA4B,EAC5B,KAAyB,EACzB,SAA6B,EAC7B,OAAkB;IAElB,IAAI,QAAQ,EAAE,IAAI,EAAE;QAAE,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC7C,IAAI,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;IAC7E,IAAI,SAAS,EAAE,IAAI,EAAE;QAAE,OAAO,QAAQ,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACxG,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;QACpC,CAAC,CAAC,UAAU,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;QAC9E,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { AccountRecord } from "../domain/accounts.js";
|
|
2
|
+
export declare function loadAccountsFile(path: string): Promise<AccountRecord[]>;
|
|
3
|
+
export declare function loadAccountsFileSync(path: string): AccountRecord[];
|
|
4
|
+
export declare function loadAccountsPayload(payload: unknown): AccountRecord[];
|
|
5
|
+
export declare function loadInlineAccounts(payload: unknown): AccountRecord[];
|
|
6
|
+
//# sourceMappingURL=loaders.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loaders.d.ts","sourceRoot":"","sources":["../../src/auth/loaders.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAG7E;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,EAAE,CAElE;AAqCD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,EAAE,CAerE;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,aAAa,EAAE,CAIpE"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { isRecord } from "../utils/guards.js";
|
|
4
|
+
import { normalizeCookiesPayload } from "./cookies.js";
|
|
5
|
+
import { accountInputToRecord } from "./records.js";
|
|
6
|
+
export async function loadAccountsFile(path) {
|
|
7
|
+
const text = await readFile(path, "utf8");
|
|
8
|
+
return parseAccountsFileText(path, text);
|
|
9
|
+
}
|
|
10
|
+
export function loadAccountsFileSync(path) {
|
|
11
|
+
return parseAccountsFileText(path, readFileSync(path, "utf8"));
|
|
12
|
+
}
|
|
13
|
+
function parseAccountsFileText(path, text) {
|
|
14
|
+
if (path.toLowerCase().endsWith(".json")) {
|
|
15
|
+
return loadAccountsPayload(JSON.parse(text));
|
|
16
|
+
}
|
|
17
|
+
if (looksLikeCookieFile(text)) {
|
|
18
|
+
const cookies = normalizeCookiesPayload(text);
|
|
19
|
+
return Object.keys(cookies).length > 0 ? [accountInputToRecord({ cookies })] : [];
|
|
20
|
+
}
|
|
21
|
+
const records = [];
|
|
22
|
+
for (const raw of text.split(/\r?\n/u)) {
|
|
23
|
+
const line = raw.trim();
|
|
24
|
+
if (!line || line.startsWith("#"))
|
|
25
|
+
continue;
|
|
26
|
+
const parts = line.includes("|") ? line.split("|").map((item) => item.trim()) : line.split(":");
|
|
27
|
+
if (parts.length < 2)
|
|
28
|
+
continue;
|
|
29
|
+
const [username, password, email, emailPassword, twoFactorSecret, authToken, csrfToken] = parts;
|
|
30
|
+
records.push(accountInputToRecord({
|
|
31
|
+
username,
|
|
32
|
+
password,
|
|
33
|
+
email,
|
|
34
|
+
emailPassword,
|
|
35
|
+
twoFactorSecret,
|
|
36
|
+
csrfToken,
|
|
37
|
+
authToken,
|
|
38
|
+
}));
|
|
39
|
+
}
|
|
40
|
+
return records;
|
|
41
|
+
}
|
|
42
|
+
function looksLikeCookieFile(text) {
|
|
43
|
+
if (text.includes("\t") || text.includes("#HttpOnly_") || /^#\s*Netscape/imu.test(text))
|
|
44
|
+
return true;
|
|
45
|
+
return text.split(/\r?\n/u).some((line) => line.trim().split(/\s+/u).length >= 7);
|
|
46
|
+
}
|
|
47
|
+
export function loadAccountsPayload(payload) {
|
|
48
|
+
const rows = Array.isArray(payload) ? payload : [payload];
|
|
49
|
+
return rows.flatMap((row) => {
|
|
50
|
+
if (!isRecord(row))
|
|
51
|
+
return [];
|
|
52
|
+
const cookieValue = row.cookies ?? row.cookies_json ?? row.cookieJar;
|
|
53
|
+
if (cookieValue !== undefined) {
|
|
54
|
+
return [
|
|
55
|
+
accountInputToRecord({
|
|
56
|
+
...row,
|
|
57
|
+
cookies: normalizeCookiesPayload(cookieValue),
|
|
58
|
+
}),
|
|
59
|
+
];
|
|
60
|
+
}
|
|
61
|
+
return [accountInputToRecord(row)];
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
export function loadInlineAccounts(payload) {
|
|
65
|
+
if (typeof payload === "string")
|
|
66
|
+
return [accountInputToRecord({ cookies: normalizeCookiesPayload(payload) })];
|
|
67
|
+
return loadAccountsPayload(payload);
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=loaders.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loaders.js","sourceRoot":"","sources":["../../src/auth/loaders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGpD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY;IACjD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,OAAO,qBAAqB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,OAAO,qBAAqB,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,IAAY;IACvD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACzC,OAAO,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,CAAC;IACD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChG,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC;QAChG,OAAO,CAAC,IAAI,CACV,oBAAoB,CAAC;YACnB,QAAQ;YACR,QAAQ;YACR,KAAK;YACL,aAAa;YACb,eAAe;YACf,SAAS;YACT,SAAS;SACV,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACrG,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAgB;IAClD,MAAM,IAAI,GAAc,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9B,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,SAAS,CAAC;QACrE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO;gBACL,oBAAoB,CAAC;oBACnB,GAAI,GAAoB;oBACxB,OAAO,EAAE,uBAAuB,CAAC,WAAW,CAAC;iBAC9C,CAAC;aACH,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,oBAAoB,CAAC,GAAmB,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,IAAI,OAAO,OAAO,KAAK,QAAQ;QAC7B,OAAO,CAAC,oBAAoB,CAAC,EAAE,OAAO,EAAE,uBAAuB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/E,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"material.d.ts","sourceRoot":"","sources":["../../src/auth/material.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAIzE,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,GAAG,YAAY,CAapG"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { AccountSessionAuthError } from "../domain/errors.js";
|
|
2
|
+
import { asString } from "../utils/guards.js";
|
|
3
|
+
export function prepareAuthMaterial(account, defaultBearerToken) {
|
|
4
|
+
const authToken = asString(account.authToken) ?? asString(account.cookies.auth_token);
|
|
5
|
+
const csrfToken = asString(account.csrfToken) ?? asString(account.cookies.ct0);
|
|
6
|
+
const bearerToken = asString(account.bearerToken) ?? defaultBearerToken;
|
|
7
|
+
if (!authToken)
|
|
8
|
+
throw new AccountSessionAuthError("missing_auth_token", "auth token is missing");
|
|
9
|
+
if (!csrfToken)
|
|
10
|
+
throw new AccountSessionAuthError("missing_csrf", "CSRF cookie is missing");
|
|
11
|
+
if (!bearerToken)
|
|
12
|
+
throw new AccountSessionAuthError("missing_bearer", "web bearer token is missing");
|
|
13
|
+
return {
|
|
14
|
+
authToken,
|
|
15
|
+
csrfToken,
|
|
16
|
+
bearerToken: bearerToken.replace(/^Bearer\s+/iu, ""),
|
|
17
|
+
cookies: { ...account.cookies, auth_token: authToken, ct0: csrfToken },
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=material.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"material.js","sourceRoot":"","sources":["../../src/auth/material.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,MAAM,UAAU,mBAAmB,CAAC,OAAsB,EAAE,kBAA0B;IACpF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/E,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,kBAAkB,CAAC;IACxE,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,uBAAuB,CAAC,oBAAoB,EAAE,uBAAuB,CAAC,CAAC;IACjG,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,uBAAuB,CAAC,cAAc,EAAE,wBAAwB,CAAC,CAAC;IAC5F,IAAI,CAAC,WAAW;QAAE,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,EAAE,6BAA6B,CAAC,CAAC;IACrG,OAAO;QACL,SAAS;QACT,SAAS;QACT,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;QACpD,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE;KACvE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { AccountInput, AccountRecord } from "../domain/accounts.js";
|
|
2
|
+
export declare function normalizeAccountRecord(input: Record<string, unknown> | AccountInput): AccountRecord;
|
|
3
|
+
export declare function accountInputToRecord(input: AccountInput): AccountRecord;
|
|
4
|
+
//# sourceMappingURL=records.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../../src/auth/records.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAa,MAAM,uBAAuB,CAAC;AAIpF,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,YAAY,GAAG,aAAa,CAoCnG;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,YAAY,GAAG,aAAa,CAEvE"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { asString, isRecord } from "../utils/guards.js";
|
|
2
|
+
import { deriveUsername, normalizeCookiesPayload } from "./cookies.js";
|
|
3
|
+
export function normalizeAccountRecord(input) {
|
|
4
|
+
const value = input;
|
|
5
|
+
const cookies = normalizeCookiesPayload(value.cookies ?? value.cookiesJson ?? value.cookieJar);
|
|
6
|
+
const authToken = asString(value.authToken ?? value.auth_token ?? value.token) ?? asString(cookies.auth_token);
|
|
7
|
+
const csrfToken = asString(value.csrfToken ?? value.csrf ?? value.ct0) ?? asString(cookies.ct0);
|
|
8
|
+
const username = deriveUsername(asString(value.username ?? value.user ?? value.handle), asString(value.email), authToken, cookies);
|
|
9
|
+
if (!username)
|
|
10
|
+
throw new Error("Account requires username, email, auth token, or cookies.");
|
|
11
|
+
const mergedCookies = {
|
|
12
|
+
...cookies,
|
|
13
|
+
...(authToken ? { auth_token: authToken } : {}),
|
|
14
|
+
...(csrfToken ? { ct0: csrfToken } : {}),
|
|
15
|
+
};
|
|
16
|
+
return {
|
|
17
|
+
username,
|
|
18
|
+
...(asString(value.password) ? { password: asString(value.password) } : {}),
|
|
19
|
+
...(asString(value.email) ? { email: asString(value.email) } : {}),
|
|
20
|
+
...(asString(value.emailPassword ?? value.email_password)
|
|
21
|
+
? { emailPassword: asString(value.emailPassword ?? value.email_password) }
|
|
22
|
+
: {}),
|
|
23
|
+
...(asString(value.twoFactorSecret ?? value.two_fa ?? value.otp)
|
|
24
|
+
? { twoFactorSecret: asString(value.twoFactorSecret ?? value.two_fa ?? value.otp) }
|
|
25
|
+
: {}),
|
|
26
|
+
...(authToken ? { authToken } : {}),
|
|
27
|
+
...(csrfToken ? { csrfToken } : {}),
|
|
28
|
+
cookies: mergedCookies,
|
|
29
|
+
...(isRecord(value.proxy) || typeof value.proxy === "string"
|
|
30
|
+
? { proxy: value.proxy }
|
|
31
|
+
: {}),
|
|
32
|
+
...(asString(value.bearer) ? { bearerToken: asString(value.bearer) } : {}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export function accountInputToRecord(input) {
|
|
36
|
+
return normalizeAccountRecord(input);
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=records.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"records.js","sourceRoot":"","sources":["../../src/auth/records.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEvE,MAAM,UAAU,sBAAsB,CAAC,KAA6C;IAClF,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,MAAM,OAAO,GAAG,uBAAuB,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAC/F,MAAM,SAAS,GACb,QAAQ,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC/F,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,cAAc,CAC7B,QAAQ,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,EACtD,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EACrB,SAAS,EACT,OAAO,CACR,CAAC;IACF,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC5F,MAAM,aAAa,GAAc;QAC/B,GAAG,OAAO;QACV,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzC,CAAC;IACF,OAAO;QACL,QAAQ;QACR,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,cAAc,CAAC;YACvD,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,cAAc,CAAC,EAAE;YAC1E,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC;YAC9D,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE;YACnF,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,aAAa;QACtB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;YAC1D,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAA+B,EAAE;YAClD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAmB;IACtD,OAAO,sBAAsB,CAAC,KAA2C,CAAC,CAAC;AAC7E,CAAC"}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export declare const HELP = "xtrawl \u2014 read-only X scraper and crawler\n\nUsage:\n xtrawl [global options] search [query] [options]\n xtrawl [global options] tweet TWEET_ID [TWEET_ID ...] [options]\n xtrawl [global options] profile-tweets USER [USER ...] [options]\n xtrawl [global options] followers USER [USER ...] [options]\n xtrawl [global options] following USER [USER ...] [options]\n xtrawl [global options] verified-followers USER [USER ...] [options]\n xtrawl [global options] user-info USER [USER ...] [options]\n\nGlobal options:\n --auth-token TOKEN session auth cookie\n --csrf-token TOKEN session CSRF cookie\n --cookies-file PATH JSON or delimited account source\n --env-file PATH dotenv account source\n --db-path PATH SQLite state path\n --proxy URL HTTP(S) or SOCKS5 proxy\n --concurrency N configured worker count\n --manifest-scrape-on-init refresh live operation identifiers\n --verbose, -v enable redacted diagnostics\n\nCommon options:\n --limit N cap returned records\n --since DATE lower search bound\n --until DATE upper search bound\n --save --save-format csv|json|both --save-dir PATH\n --pretty print indented JSON\n";
|
|
2
|
+
//# sourceMappingURL=help.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"help.d.ts","sourceRoot":"","sources":["../../src/cli/help.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,IAAI,svCA4BhB,CAAC"}
|
package/dist/cli/help.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export const HELP = `xtrawl — read-only X scraper and crawler
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
xtrawl [global options] search [query] [options]
|
|
5
|
+
xtrawl [global options] tweet TWEET_ID [TWEET_ID ...] [options]
|
|
6
|
+
xtrawl [global options] profile-tweets USER [USER ...] [options]
|
|
7
|
+
xtrawl [global options] followers USER [USER ...] [options]
|
|
8
|
+
xtrawl [global options] following USER [USER ...] [options]
|
|
9
|
+
xtrawl [global options] verified-followers USER [USER ...] [options]
|
|
10
|
+
xtrawl [global options] user-info USER [USER ...] [options]
|
|
11
|
+
|
|
12
|
+
Global options:
|
|
13
|
+
--auth-token TOKEN session auth cookie
|
|
14
|
+
--csrf-token TOKEN session CSRF cookie
|
|
15
|
+
--cookies-file PATH JSON or delimited account source
|
|
16
|
+
--env-file PATH dotenv account source
|
|
17
|
+
--db-path PATH SQLite state path
|
|
18
|
+
--proxy URL HTTP(S) or SOCKS5 proxy
|
|
19
|
+
--concurrency N configured worker count
|
|
20
|
+
--manifest-scrape-on-init refresh live operation identifiers
|
|
21
|
+
--verbose, -v enable redacted diagnostics
|
|
22
|
+
|
|
23
|
+
Common options:
|
|
24
|
+
--limit N cap returned records
|
|
25
|
+
--since DATE lower search bound
|
|
26
|
+
--until DATE upper search bound
|
|
27
|
+
--save --save-format csv|json|both --save-dir PATH
|
|
28
|
+
--pretty print indented JSON
|
|
29
|
+
`;
|
|
30
|
+
//# sourceMappingURL=help.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"help.js","sourceRoot":"","sources":["../../src/cli/help.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BnB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":";AAOA,wBAAsB,MAAM,CAAC,IAAI,GAAE,SAAS,MAAM,EAA0B,GAAG,OAAO,CAAC,MAAM,CAAC,CA+D7F"}
|