x2llm 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- x2llm-0.1.0/.gitignore +7 -0
- x2llm-0.1.0/Makefile +35 -0
- x2llm-0.1.0/PKG-INFO +8 -0
- x2llm-0.1.0/README.md +401 -0
- x2llm-0.1.0/pyproject.toml +26 -0
- x2llm-0.1.0/scripts/har_to_endpoints.py +152 -0
- x2llm-0.1.0/src/x2llm/__init__.py +5 -0
- x2llm-0.1.0/src/x2llm/__main__.py +3 -0
- x2llm-0.1.0/src/x2llm/cli.py +145 -0
- x2llm-0.1.0/src/x2llm/client.py +133 -0
- x2llm-0.1.0/src/x2llm/config.py +73 -0
- x2llm-0.1.0/src/x2llm/cores.py +218 -0
- x2llm-0.1.0/src/x2llm/endpoints.py +6 -0
- x2llm-0.1.0/src/x2llm/mcp.py +173 -0
- x2llm-0.1.0/src/x2llm/parse.py +120 -0
- x2llm-0.1.0/tests/test_x2llm.py +467 -0
- x2llm-0.1.0/uv.lock +824 -0
x2llm-0.1.0/.gitignore
ADDED
x2llm-0.1.0/Makefile
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
.PHONY: help setup install uninstall test lint fmt check refresh clean
|
|
2
|
+
|
|
3
|
+
-include .env
|
|
4
|
+
export
|
|
5
|
+
|
|
6
|
+
UV ?= uv
|
|
7
|
+
|
|
8
|
+
help: ## Show available targets
|
|
9
|
+
@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-10s\033[0m %s\n", $$1, $$2}'
|
|
10
|
+
|
|
11
|
+
setup: ## Install dependencies into .venv (uv sync)
|
|
12
|
+
$(UV) sync
|
|
13
|
+
|
|
14
|
+
install: ## Install x2llm system-wide via uv tool
|
|
15
|
+
$(UV) tool install --upgrade .
|
|
16
|
+
|
|
17
|
+
uninstall: ## Remove system-wide x2llm
|
|
18
|
+
$(UV) tool uninstall x2llm
|
|
19
|
+
|
|
20
|
+
test: ## Run test suite
|
|
21
|
+
$(UV) run pytest -q
|
|
22
|
+
|
|
23
|
+
lint: ## Run ruff checks
|
|
24
|
+
$(UV) run ruff check src tests scripts
|
|
25
|
+
|
|
26
|
+
fmt: ## Ruff autofix
|
|
27
|
+
$(UV) run ruff check --fix src tests scripts
|
|
28
|
+
|
|
29
|
+
check: lint test ## Everything a change must pass
|
|
30
|
+
|
|
31
|
+
refresh: ## Regenerate endpoints.py + .env.fresh (txids/cookies) from a HAR: make refresh HAR=…
|
|
32
|
+
$(UV) run python scripts/har_to_endpoints.py '$(HAR)'
|
|
33
|
+
|
|
34
|
+
clean: ## Remove caches
|
|
35
|
+
rm -rf .pytest_cache .ruff_cache __pycache__ src/x2llm/__pycache__
|
x2llm-0.1.0/PKG-INFO
ADDED
x2llm-0.1.0/README.md
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
# x2llm
|
|
2
|
+
|
|
3
|
+
x.com (Twitter) client for LLM agents. One credential set from the
|
|
4
|
+
environment, three faces: a Python library, a `click` CLI, and a stdio MCP
|
|
5
|
+
server. Built for machines first: every command prints exactly one JSON
|
|
6
|
+
document, every error prints `{"error": "..."}` and exits non-zero. Everything
|
|
7
|
+
is read-only **except one deliberate write: `like`** — no posting, replying,
|
|
8
|
+
following, or DMing.
|
|
9
|
+
|
|
10
|
+
## Key Features
|
|
11
|
+
|
|
12
|
+
- Search (full operator support: `from:user`, `since:`, `lang:`, `min_faves:` ...)
|
|
13
|
+
- Tweet + reply-thread retrieval
|
|
14
|
+
- Notification tab reading
|
|
15
|
+
- Account analytics (impressions, likes, follows — your own statistics page)
|
|
16
|
+
- Follower/following graphs, typeahead suggestions, unread badge counts
|
|
17
|
+
- Like a tweet (the single write action, explicit everywhere)
|
|
18
|
+
- Same core functions power both CLI and MCP server
|
|
19
|
+
|
|
20
|
+
## Tech Stack
|
|
21
|
+
|
|
22
|
+
- **Language**: Python 3.12+
|
|
23
|
+
- **HTTP**: httpx (async)
|
|
24
|
+
- **CLI**: click
|
|
25
|
+
- **MCP**: official `mcp` SDK v2 (stdio transport)
|
|
26
|
+
- **Packaging**: uv + hatchling (src layout)
|
|
27
|
+
- **Tests**: pytest + pytest-asyncio, ruff for linting
|
|
28
|
+
|
|
29
|
+
## How x.com authentication works (the 30-second version)
|
|
30
|
+
|
|
31
|
+
The web app authenticates every API call with four pieces:
|
|
32
|
+
|
|
33
|
+
| Piece | What it is | Secret? |
|
|
34
|
+
|---|---|---|
|
|
35
|
+
| `authorization: Bearer AAAA…` | Public web-client key, baked into x.com's JS bundle. Same for every user. | No |
|
|
36
|
+
| Cookie `auth_token` | The session itself (httpOnly). **Whoever holds it IS the account** — read and write. | **Yes** |
|
|
37
|
+
| Cookie `ct0` | CSRF token. Mirrored verbatim into the `x-csrf-token` header. | Semi |
|
|
38
|
+
| `x-client-transaction-id` | Anti-automation header derived from page state. Only *some* endpoints validate it (search does, most others don't); validated copies are captured per endpoint and expire after a while. | No |
|
|
39
|
+
|
|
40
|
+
There is no request signing. Bearer + cookies + mirrored CSRF header is the
|
|
41
|
+
whole scheme. `x2llm` encapsulates all of this in `XClient` — nothing above
|
|
42
|
+
that layer ever touches headers.
|
|
43
|
+
|
|
44
|
+
## Prerequisites
|
|
45
|
+
|
|
46
|
+
- Python 3.12+
|
|
47
|
+
- [uv](https://docs.astral.sh/uv/) (or pip + venv, but commands below assume uv)
|
|
48
|
+
- A browser logged in to x.com (for credentials — 2 minutes, see below)
|
|
49
|
+
|
|
50
|
+
## Getting Started
|
|
51
|
+
|
|
52
|
+
### 1. Clone and install
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
cd x2llm
|
|
56
|
+
make setup # uv sync — creates .venv with all deps
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### 2. Extract credentials from your logged-in browser
|
|
60
|
+
|
|
61
|
+
You need two cookie values from a browser session that is logged in to x.com:
|
|
62
|
+
`auth_token` and `ct0`.
|
|
63
|
+
|
|
64
|
+
> [!IMPORTANT]
|
|
65
|
+
> `document.cookie` in the devtools console will **not** show `auth_token` —
|
|
66
|
+
> it is httpOnly. Use one of the methods below.
|
|
67
|
+
|
|
68
|
+
#### Method A — DevTools Application tab (fastest, cookies only)
|
|
69
|
+
|
|
70
|
+
1. Open `https://x.com` in your logged-in browser
|
|
71
|
+
2. Press `F12` → **Storage** (Firefox) or **Application** (Chrome) → **Cookies** → `https://x.com`
|
|
72
|
+
3. Find the rows `auth_token` and `ct0`, copy their **Value** columns
|
|
73
|
+
4. Put them into `.env`:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
cp .env.example .env
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
X_AUTH_TOKEN=paste auth_token value here
|
|
81
|
+
X_CT0=paste ct0 value here
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
#### Method B — Network tab (gets you everything, incl. transaction ids)
|
|
85
|
+
|
|
86
|
+
1. `F12` → **Network** tab, filter for `i/api`
|
|
87
|
+
2. Scroll your x.com home timeline once — badge-count calls appear
|
|
88
|
+
3. Click any request, then **Headers** → **Request Headers** → find the `Cookie:` header
|
|
89
|
+
4. Copy the values of `auth_token=…` and `ct0=…` from it into `.env` as above
|
|
90
|
+
|
|
91
|
+
If you also want `search` to work, export a HAR (see
|
|
92
|
+
[Refreshing captured tokens](#refreshing-captured-tokens)) — search validates
|
|
93
|
+
`x-client-transaction-id`, which the HAR provides.
|
|
94
|
+
|
|
95
|
+
> [!TIP]
|
|
96
|
+
> Cookies from the Network tab can be pasted wholesale into `X_COOKIE` —
|
|
97
|
+
> including `cf_clearance` — which helps if Cloudflare challenges your
|
|
98
|
+
> machine: `X_COOKIE=auth_token=…; ct0=…; cf_clearance=…`
|
|
99
|
+
|
|
100
|
+
#### Verifying credentials work
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
source .env && export X_AUTH_TOKEN X_CT0
|
|
104
|
+
uv run x2llm badges
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Expected output:
|
|
108
|
+
|
|
109
|
+
```json
|
|
110
|
+
{
|
|
111
|
+
"badges": {
|
|
112
|
+
"dm_unread_count": 0,
|
|
113
|
+
"ntab_unread_count": 1,
|
|
114
|
+
"total_unread_count": 1
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### 3. Install the CLI system-wide (optional)
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
make install # uv tool install — puts `x2llm` on your PATH
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Architecture
|
|
126
|
+
|
|
127
|
+
### Directory structure
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
├── src/x2llm/
|
|
131
|
+
│ ├── config.py # Config.from_env() — every knob is an env var
|
|
132
|
+
│ ├── endpoints.py # GENERATED: public wire constants (query ids, features, toggles)
|
|
133
|
+
│ ├── client.py # XClient — the ONLY module that knows the RPC protocol
|
|
134
|
+
│ ├── parse.py # response-blob walkers → flat dicts (tweet/user/cursor rows)
|
|
135
|
+
│ ├── cores.py # 8 async service cores, shared by CLI and MCP
|
|
136
|
+
│ ├── cli.py # click commands, one JSON doc per command
|
|
137
|
+
│ └── mcp.py # stdio MCP server wrapping the same cores
|
|
138
|
+
├── scripts/
|
|
139
|
+
│ └── har_to_endpoints.py # regenerate endpoints.py + .env.fresh (txids, cookies) from a HAR
|
|
140
|
+
├── tests/test_x2llm.py # offline: MockTransport, FakeClient, CliRunner, stdio roundtrip
|
|
141
|
+
├── Makefile
|
|
142
|
+
└── .env.example
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Request lifecycle
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
click command / MCP tool
|
|
149
|
+
│
|
|
150
|
+
▼
|
|
151
|
+
cores.py (search, tweet, notifications, … like — dicts in, dicts out)
|
|
152
|
+
│
|
|
153
|
+
▼
|
|
154
|
+
XClient.graphql(key, variables) — reads
|
|
155
|
+
XClient.graphql_post(key, variables) — POST GraphQL (home feed, like write)
|
|
156
|
+
XClient.rest(path, params) — legacy REST reads
|
|
157
|
+
│ builds URL: GET https://x.com/i/api/graphql/<queryId>/<Operation>
|
|
158
|
+
│ ?variables=<json>&features=<json>
|
|
159
|
+
│ attaches: bearer, cookie session, x-csrf-token (== ct0),
|
|
160
|
+
│ x-twitter-auth-type, per-endpoint x-client-transaction-id
|
|
161
|
+
▼
|
|
162
|
+
httpx.AsyncClient
|
|
163
|
+
│
|
|
164
|
+
▼
|
|
165
|
+
x.com → 200 JSON (or XApiError with a human hint)
|
|
166
|
+
│
|
|
167
|
+
▼
|
|
168
|
+
parse.py walkers (walk / collect_tweets / collect_cursors / notification_rows)
|
|
169
|
+
│
|
|
170
|
+
▼
|
|
171
|
+
flat JSON row: {id, author, text, likes, views, …}
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### Rate limiting
|
|
175
|
+
|
|
176
|
+
XClient spaces requests so at most `X_RATE_LIMIT` per minute leave the process
|
|
177
|
+
(default 30 — deliberately below what the session allows; x.com's own
|
|
178
|
+
per-endpoint budgets seen in responses are far higher, e.g. 12 000 / 15 min
|
|
179
|
+
for following-list). The limiter is a simple monotonic-clock slot allocator
|
|
180
|
+
under an asyncio lock; 0 disables. Server-side 429s surface as errors with a
|
|
181
|
+
hint instead of being retried — the right answer is to slow down.
|
|
182
|
+
|
|
183
|
+
### Design invariants
|
|
184
|
+
|
|
185
|
+
- **RPC is encapsulated.** Only `client.py` imports httpx or knows what a
|
|
186
|
+
header is. Only `endpoints.py` holds wire constants, and it is generated.
|
|
187
|
+
- **Cores are transport-agnostic.** They call `graphql(key, variables)` /
|
|
188
|
+
`graphql_mutation(key, variables)` / `rest(path, params)` — anything with
|
|
189
|
+
those three async methods works (see `FakeClient` in the tests). This is
|
|
190
|
+
what keeps the suite offline.
|
|
191
|
+
- **One code path.** CLI handlers and MCP tools bind the same core functions;
|
|
192
|
+
behavior can never drift between the two surfaces.
|
|
193
|
+
- **Writes are isolated.** Read feeds are GET, except POST-fed timelines
|
|
194
|
+
(home — the browser itself scrolls via POST). The single write,
|
|
195
|
+
`like`, shares the POST primitive but nothing else does. The MCP server
|
|
196
|
+
registers it with `read_only_hint=false` and a loud description.
|
|
197
|
+
- **One pagination model.** Every list core (`home`, `search`, `tweet`,
|
|
198
|
+
`notifications`, `following`) returns a `cursor`; feed it back for the next
|
|
199
|
+
page, `null` means end. The home timeline has two tabs served by different
|
|
200
|
+
operations: `for_you` (HomeTimeline, algorithmic) and `following`
|
|
201
|
+
(HomeLatestTimeline, chronological) — cursors are tab-specific, pass back
|
|
202
|
+
what the same tab returned.
|
|
203
|
+
|
|
204
|
+
## Environment Variables
|
|
205
|
+
|
|
206
|
+
### Required
|
|
207
|
+
|
|
208
|
+
| Variable | Description | How to get it |
|
|
209
|
+
|---|---|---|
|
|
210
|
+
| `X_AUTH_TOKEN` | Session secret (`auth_token` cookie) | DevTools → Cookies (see Getting Started) |
|
|
211
|
+
| `X_CT0` | CSRF token (`ct0` cookie) | same |
|
|
212
|
+
|
|
213
|
+
### Optional
|
|
214
|
+
|
|
215
|
+
| Variable | Description | Default |
|
|
216
|
+
|---|---|---|
|
|
217
|
+
| `X_COOKIE` | Full `Cookie` header verbatim (overrides the two vars; add `cf_clearance` here if Cloudflare challenges) | composed from the two above |
|
|
218
|
+
| `X_BEARER` | Web-client bearer token | current public one (in `endpoints.py`) |
|
|
219
|
+
| `X_USER_AGENT` | Should match the browser the cookies came from | Firefox/Linux UA |
|
|
220
|
+
| `X_API_BASE` | API root | `https://x.com/i/api` |
|
|
221
|
+
| `X_TIMEOUT` | Per-request timeout, seconds | `20` |
|
|
222
|
+
| `X_RATE_LIMIT` | Max requests per minute, client-enforced (0 = unlimited) | `30` |
|
|
223
|
+
| `X_TXIDS` | JSON `{"search": "<x-client-transaction-id>", ...}` — per-operation transaction ids; only some endpoints need them and they expire within hours. Written to `.env.fresh` by `make refresh`. | `{}` |
|
|
224
|
+
|
|
225
|
+
The `Makefile` auto-exports `.env`; for other entry points source it yourself
|
|
226
|
+
or use something like `direnv`.
|
|
227
|
+
|
|
228
|
+
## CLI Reference
|
|
229
|
+
|
|
230
|
+
| Command | Description |
|
|
231
|
+
|---|---|
|
|
232
|
+
| `x2llm home [--tab for_you\|following] [--limit N] [--cursor C]` | Home timeline; scroll with the returned cursor |
|
|
233
|
+
| `x2llm badges` | Unread notification/DM counts — cheapest auth check |
|
|
234
|
+
| `x2llm user SCREEN_NAME` | Profile lookup: bio, url, follower counts, verification |
|
|
235
|
+
| `x2llm me` | Viewer id + follower counts |
|
|
236
|
+
| `x2llm suggest QUERY` | Typeahead topics/users for a prefix |
|
|
237
|
+
| `x2llm search QUERY [--limit N] [--product Top\|Latest\|Media] [--cursor C]` | Full search with X operators |
|
|
238
|
+
| `x2llm tweet ID [--cursor C]` | Tweet + reply thread |
|
|
239
|
+
| `x2llm notifications [--limit N] [--cursor C]` | Notification tab |
|
|
240
|
+
| `x2llm analytics [--days N]` | Engagement statistics (1–28 days) |
|
|
241
|
+
| `x2llm following USER [--limit N] [--cursor C]` | Accounts USER follows (id or screen name) |
|
|
242
|
+
| `x2llm like TWEET_ID` | **Like a tweet — real, public, notifies the author** |
|
|
243
|
+
| `x2llm mcp` | Run as stdio MCP server |
|
|
244
|
+
|
|
245
|
+
All pagination works the same way: the response carries a `cursor`; feed it
|
|
246
|
+
back via `--cursor` for the next page (`null` means end).
|
|
247
|
+
|
|
248
|
+
Example — search, take the cursor, get page two:
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
uv run x2llm search "solana lang:en min_faves:100" --limit 10
|
|
252
|
+
uv run x2llm search "solana lang:en min_faves:100" --limit 10 --cursor 'DAACCgACHUB…'
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
## MCP Server
|
|
256
|
+
|
|
257
|
+
`x2llm mcp` speaks MCP over stdio with 11 tools — the same cores as the CLI.
|
|
258
|
+
Ten are annotated `read_only_hint` (`get_home_timeline`, `get_user`,
|
|
259
|
+
`get_badges`, `me`, `suggest`, `search`, `get_tweet`, `get_notifications`,
|
|
260
|
+
`get_analytics`, `get_following`); the eleventh, `like_tweet`, is the one write and is registered with `read_only_hint=false`
|
|
261
|
+
and a description that instructs agents to call it only on explicit user
|
|
262
|
+
request. Liking an already-liked tweet returns the server's idempotent
|
|
263
|
+
rejection (`{"result": "error", "detail": "... has already favorited ..."}`)
|
|
264
|
+
— no state changes.
|
|
265
|
+
|
|
266
|
+
Wire it into an MCP client config (example for Claude-style configs):
|
|
267
|
+
|
|
268
|
+
```json
|
|
269
|
+
{
|
|
270
|
+
"mcpServers": {
|
|
271
|
+
"x": {
|
|
272
|
+
"command": "/absolute/path/to/x2llm/.venv/bin/x2llm",
|
|
273
|
+
"args": ["mcp"],
|
|
274
|
+
"env": {
|
|
275
|
+
"X_AUTH_TOKEN": "…",
|
|
276
|
+
"X_CT0": "…"
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
Quick smoke test by hand:
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
printf '%s\n%s\n%s\n' \
|
|
287
|
+
'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"t","version":"0"}}}' \
|
|
288
|
+
'{"jsonrpc":"2.0","method":"notifications/initialized"}' \
|
|
289
|
+
'{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \
|
|
290
|
+
| uv run x2llm mcp
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
## Refreshing captured tokens
|
|
294
|
+
|
|
295
|
+
The package ships only **public, session-independent constants**
|
|
296
|
+
(`endpoints.py`: query ids, features blobs, fieldToggles — identical for every
|
|
297
|
+
user, taken from x.com's JS bundles; they rotate on a months scale). Everything
|
|
298
|
+
session-scoped comes from the environment: `X_AUTH_TOKEN`, `X_CT0`, `X_TXIDS`.
|
|
299
|
+
**Transaction ids expire fast (observed: tens of minutes to hours)** and only
|
|
300
|
+
some endpoints enforce them — `search` is the one that hurts. Nothing
|
|
301
|
+
user-specific ever enters the package, so it is safe to publish.
|
|
302
|
+
|
|
303
|
+
Refreshes are **incremental**: entries in the HARs you pass win; operations
|
|
304
|
+
they don't contain keep their previous query ids from the existing
|
|
305
|
+
`endpoints.py`. One small HAR that only captured a search is enough — the run
|
|
306
|
+
fails only if the merged result is still incomplete. The generator prints
|
|
307
|
+
which endpoints were refreshed vs kept, and writes `.env.fresh` with the
|
|
308
|
+
session's `X_AUTH_TOKEN`, `X_CT0` and `X_TXIDS` (quoted so both
|
|
309
|
+
`source .env` and the Makefile include parse it).
|
|
310
|
+
|
|
311
|
+
When `search` or `like` starts failing with an empty 404 (the error text says
|
|
312
|
+
so):
|
|
313
|
+
|
|
314
|
+
1. Open x.com logged in → `F12` → Network → filter `i/api`
|
|
315
|
+
2. Visit: the **home timeline** (badges), run one **search**, open one
|
|
316
|
+
**tweet**, open **notifications**, open **x.com/analytics**, open one
|
|
317
|
+
**user profile**, **like** one tweet, **scroll both home tabs** (FOR YOU
|
|
318
|
+
and FOLLOWING) two screens (that touches all nine operations)
|
|
319
|
+
3. Export HAR: Network ⚙️ gear → *Save all as HAR*
|
|
320
|
+
4. Regenerate:
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
make refresh HAR='../x.com_Archive [26-10-07 13-27-31].har'
|
|
324
|
+
make check # public constants only; CI-safe, no secrets touched
|
|
325
|
+
cp .env.fresh .env # your fresh session credentials + X_TXIDS
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
While `search`/`like` are down, everything else keeps working — tweet,
|
|
329
|
+
notifications, analytics, following, suggest, both home tabs and user lookups
|
|
330
|
+
work without any transaction id at all.
|
|
331
|
+
Features blobs and fieldToggles are captured per endpoint (x.com ships
|
|
332
|
+
several variants). The generator also extracts `X_AUTH_TOKEN`/`X_CT0` from
|
|
333
|
+
the capture's cookies and writes them to `.env.fresh` (it never touches an
|
|
334
|
+
existing `.env`; review and `cp .env.fresh .env`). Both files hold live
|
|
335
|
+
session secrets and are gitignored. The generator accepts several HAR files
|
|
336
|
+
and merges them, so captures can be split across sessions:
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
uv run python scripts/har_to_endpoints.py 'reads.har' 'like.har'
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
## Testing
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
make check # ruff + pytest — the gate every change must pass
|
|
346
|
+
uv run pytest -q tests/test_x2llm.py::test_search_core_truncates_and_returns_cursor # single test
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
The suite is offline by design: `httpx.MockTransport` exercises the real
|
|
350
|
+
`XClient` request construction, `FakeClient` exercises cores, `CliRunner`
|
|
351
|
+
exercises commands, and one subprocess test does a full MCP stdio handshake
|
|
352
|
+
(`initialize` + `tools/list`) with dummy credentials. No test needs your real
|
|
353
|
+
account or network.
|
|
354
|
+
|
|
355
|
+
## Troubleshooting
|
|
356
|
+
|
|
357
|
+
### `{"error": "…: HTTP 404 … transaction-id expired"}`
|
|
358
|
+
|
|
359
|
+
Search hit its transaction-id expiry. Refresh per the section above. Other
|
|
360
|
+
commands are unaffected.
|
|
361
|
+
|
|
362
|
+
### `{"error": "…: HTTP 401/403 …"}`
|
|
363
|
+
|
|
364
|
+
Your `auth_token`/`ct0` are stale (logged out elsewhere? password change?
|
|
365
|
+
long-lived capture?). Re-extract both from the browser. If your IP/UA changed
|
|
366
|
+
since capture, also set `X_COOKIE` with the full header including
|
|
367
|
+
`cf_clearance` and a matching `X_USER_AGENT`.
|
|
368
|
+
|
|
369
|
+
### `{"error": "missing environment variables: X_AUTH_TOKEN, X_CT0"}`
|
|
370
|
+
|
|
371
|
+
Self-explanatory — `.env` not loaded or empty. The Makefile auto-exports it;
|
|
372
|
+
elsewhere `source .env && export X_AUTH_TOKEN X_CT0`.
|
|
373
|
+
|
|
374
|
+
### Empty body with a `cf-ray` header
|
|
375
|
+
|
|
376
|
+
Cloudflare edge rejection, not x.com. Keep the same User-Agent as the browser
|
|
377
|
+
you captured from; if it persists, copy the full `Cookie` header into
|
|
378
|
+
`X_COOKIE`.
|
|
379
|
+
|
|
380
|
+
### `author: null` in results
|
|
381
|
+
|
|
382
|
+
x.com is A/B-testing a new user-object shape. `parse.py` reads both the old
|
|
383
|
+
(`legacy`) and new (`core`) paths; if both are null you are looking at a third
|
|
384
|
+
shape — file it with a response sample.
|
|
385
|
+
|
|
386
|
+
## Security Notes
|
|
387
|
+
|
|
388
|
+
> [!CAUTION]
|
|
389
|
+
> `auth_token` is a full-account bearer credential: read **and write** access
|
|
390
|
+
> until it expires. A `.env`, a shell history line, or a HAR file containing
|
|
391
|
+
> it must be treated like a password file. HAR exports additionally contain
|
|
392
|
+
> every response body the browser saw. `*.har` is gitignored for a reason —
|
|
393
|
+
> keep it that way.
|
|
394
|
+
|
|
395
|
+
This project is API-usage of your own logged-in session for personal tooling
|
|
396
|
+
(reads + likes you explicitly request). Respect x.com's terms of service and rate limits; the
|
|
397
|
+
client sends plain GETs at human-scale speeds and has no retry hammering.
|
|
398
|
+
|
|
399
|
+
## License
|
|
400
|
+
|
|
401
|
+
Internal tool, no license granted. All rights reserved.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "x2llm"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Read-only x.com (Twitter) API client for LLM agents: library, CLI, and stdio MCP server"
|
|
5
|
+
requires-python = ">=3.12"
|
|
6
|
+
dependencies = ["click>=8", "httpx>=0.27", "mcp>=2"]
|
|
7
|
+
|
|
8
|
+
[project.scripts]
|
|
9
|
+
x2llm = "x2llm.cli:main"
|
|
10
|
+
|
|
11
|
+
[build-system]
|
|
12
|
+
requires = ["hatchling"]
|
|
13
|
+
build-backend = "hatchling.build"
|
|
14
|
+
|
|
15
|
+
[tool.hatch.build.targets.wheel]
|
|
16
|
+
packages = ["src/x2llm"]
|
|
17
|
+
|
|
18
|
+
[dependency-groups]
|
|
19
|
+
dev = ["pytest>=8", "pytest-asyncio>=0.24", "ruff>=0.6"]
|
|
20
|
+
|
|
21
|
+
[tool.pytest.ini_options]
|
|
22
|
+
asyncio_mode = "auto"
|
|
23
|
+
testpaths = ["tests"]
|
|
24
|
+
|
|
25
|
+
[tool.ruff]
|
|
26
|
+
line-length = 100
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Regenerate src/x2llm/endpoints.py from browser HAR captures of x.com.
|
|
3
|
+
|
|
4
|
+
Usage: uv run python scripts/har_to_endpoints.py archive1.har [archive2.har ...]
|
|
5
|
+
|
|
6
|
+
The package ships only public, session-independent constants (query ids,
|
|
7
|
+
features blobs, fieldToggles - from x.com's JS bundles, identical for every
|
|
8
|
+
user). Session-scoped values come from the environment instead: the
|
|
9
|
+
x-client-transaction-ids the browser sent are written to .env.fresh as
|
|
10
|
+
X_TXIDS (JSON, per operation) - they expire within hours, so they could
|
|
11
|
+
never ship in the package. Same for X_AUTH_TOKEN/X_CT0 (cookie session).
|
|
12
|
+
.env.fresh never touches an existing .env - copy it over yourself. HARs and
|
|
13
|
+
.env.fresh contain live session secrets; both are gitignored.
|
|
14
|
+
|
|
15
|
+
Incremental: entries from the given HARs take precedence; operations they
|
|
16
|
+
don't contain are carried over from the existing endpoints.py. A single-HAR
|
|
17
|
+
refresh that only captured one page is fine. The run fails only if the
|
|
18
|
+
merged result is still incomplete.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import sys
|
|
24
|
+
import urllib.parse
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
READS = {
|
|
28
|
+
"SearchTimeline": "search",
|
|
29
|
+
"TweetDetail": "tweet",
|
|
30
|
+
"NotificationsTimeline": "notifications",
|
|
31
|
+
"ViewerBadgeCounts": "badges",
|
|
32
|
+
"accountOverviewDailyQuery": "analytics",
|
|
33
|
+
"UserByScreenName": "user",
|
|
34
|
+
}
|
|
35
|
+
POST_READS = {"HomeTimeline": "home", # FOR YOU tab
|
|
36
|
+
"HomeLatestTimeline": "home_following"} # FOLLOWING tab
|
|
37
|
+
MUTATIONS = {"FavoriteTweet": "like"}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_existing(endpoints: Path) -> tuple[dict, str]:
|
|
41
|
+
if not endpoints.exists():
|
|
42
|
+
return {}, ""
|
|
43
|
+
namespace: dict = {}
|
|
44
|
+
exec(endpoints.read_text(), namespace) # noqa: S102 - our own generated file
|
|
45
|
+
return dict(namespace.get("GRAPHQL") or {}), namespace.get("BEARER") or ""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def run(paths: list[str], out: Path) -> tuple[dict, str]:
|
|
49
|
+
graphql, bearer = load_existing(out)
|
|
50
|
+
previous = set(graphql)
|
|
51
|
+
fresh: set[str] = set()
|
|
52
|
+
for arg in paths:
|
|
53
|
+
har = json.loads(Path(arg).read_text())
|
|
54
|
+
for e in har["log"]["entries"]:
|
|
55
|
+
req = e["request"]
|
|
56
|
+
url = req["url"]
|
|
57
|
+
if "/i/api/graphql/" not in url:
|
|
58
|
+
continue
|
|
59
|
+
op = url.split("/graphql/", 1)[1].split("/", 1)[1].split("?", 1)[0]
|
|
60
|
+
key = READS.get(op) or POST_READS.get(op) or MUTATIONS.get(op)
|
|
61
|
+
if key is None or key in fresh:
|
|
62
|
+
continue
|
|
63
|
+
if op in POST_READS and req["method"] != "POST":
|
|
64
|
+
continue # first page may arrive as GET; capture the scroll POST
|
|
65
|
+
headers = {h["name"].lower(): h["value"] for h in req["headers"]}
|
|
66
|
+
entry = {
|
|
67
|
+
"query_id": url.split("/graphql/", 1)[1].split("/", 1)[0],
|
|
68
|
+
"name": op,
|
|
69
|
+
"method": "post" if op in MUTATIONS or op in POST_READS else "get",
|
|
70
|
+
}
|
|
71
|
+
if req["method"] == "POST" and req.get("postData", {}).get("text"):
|
|
72
|
+
body = json.loads(req["postData"]["text"])
|
|
73
|
+
if "features" in body:
|
|
74
|
+
entry["features"] = body["features"]
|
|
75
|
+
if "fieldToggles" in body:
|
|
76
|
+
entry["toggles"] = body["fieldToggles"]
|
|
77
|
+
else:
|
|
78
|
+
q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
|
79
|
+
if "features" in q:
|
|
80
|
+
entry["features"] = json.loads(q["features"][0])
|
|
81
|
+
if "fieldToggles" in q:
|
|
82
|
+
entry["toggles"] = json.loads(q["fieldToggles"][0])
|
|
83
|
+
graphql[key] = entry
|
|
84
|
+
fresh.add(key)
|
|
85
|
+
if not bearer:
|
|
86
|
+
bearer = headers.get("authorization", "")
|
|
87
|
+
wanted = set(READS.values()) | set(POST_READS.values()) | set(MUTATIONS.values())
|
|
88
|
+
missing = sorted(wanted - set(graphql))
|
|
89
|
+
if missing:
|
|
90
|
+
sys.exit(f"HARs are missing operations: {missing} - capture those pages, re-export")
|
|
91
|
+
out.write_text(
|
|
92
|
+
"# GENERATED by scripts/har_to_endpoints.py from browser HAR captures. Do not edit.\n"
|
|
93
|
+
"# Public, session-independent constants only (query ids, features, toggles).\n"
|
|
94
|
+
"# Session-scoped values (transaction ids) live in the X_TXIDS env var.\n"
|
|
95
|
+
"# Regenerate: uv run python scripts/har_to_endpoints.py <archive.har> [...]\n"
|
|
96
|
+
f'BEARER = {bearer!r}\n'
|
|
97
|
+
f"GRAPHQL = {graphql!r}\n"
|
|
98
|
+
)
|
|
99
|
+
print(f"wrote {out} ({len(graphql)} endpoints; refreshed: {sorted(fresh) or 'none'}; "
|
|
100
|
+
f"kept from previous: {sorted(previous - fresh) or 'none'})")
|
|
101
|
+
return graphql, bearer
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def write_env_fresh(paths: list[str]) -> None:
|
|
105
|
+
txids: dict[str, str] = {}
|
|
106
|
+
cookie_header = user_agent = ""
|
|
107
|
+
auth_token = ct0 = ""
|
|
108
|
+
for arg in paths:
|
|
109
|
+
har = json.loads(Path(arg).read_text())
|
|
110
|
+
for e in har["log"]["entries"]:
|
|
111
|
+
req = e["request"]
|
|
112
|
+
headers = {h["name"].lower(): h["value"] for h in req["headers"]}
|
|
113
|
+
cookies = {}
|
|
114
|
+
for part in headers.get("cookie", "").split("; "):
|
|
115
|
+
name, _, value = part.partition("=")
|
|
116
|
+
cookies[name] = value
|
|
117
|
+
if not auth_token and cookies.get("auth_token") and cookies.get("ct0"):
|
|
118
|
+
auth_token, ct0 = cookies["auth_token"], cookies["ct0"]
|
|
119
|
+
cookie_header = headers.get("cookie", "")
|
|
120
|
+
user_agent = headers.get("user-agent", "")
|
|
121
|
+
op = req["url"].split("/graphql/", 1)[1].split("/", 1)[1].split("?", 1)[0] \
|
|
122
|
+
if "/i/api/graphql/" in req["url"] else ""
|
|
123
|
+
key = READS.get(op) or POST_READS.get(op) or MUTATIONS.get(op)
|
|
124
|
+
if key and key not in txids:
|
|
125
|
+
tx = headers.get("x-client-transaction-id", "")
|
|
126
|
+
if tx:
|
|
127
|
+
txids[key] = tx
|
|
128
|
+
if not auth_token:
|
|
129
|
+
return
|
|
130
|
+
env_out = Path(__file__).resolve().parents[1] / ".env.fresh"
|
|
131
|
+
env_out.write_text(
|
|
132
|
+
"# Generated by har_to_endpoints.py - live session secrets. Keep private.\n"
|
|
133
|
+
f"X_AUTH_TOKEN={auth_token}\n"
|
|
134
|
+
f"X_CT0={ct0}\n"
|
|
135
|
+
"X_TXIDS='" + json.dumps(txids, separators=(",", ":")) + "'\n"
|
|
136
|
+
f"# X_USER_AGENT={user_agent}\n"
|
|
137
|
+
f"# X_COOKIE={cookie_header}\n"
|
|
138
|
+
)
|
|
139
|
+
print(f"wrote {env_out} ({len(txids)} txids) - review and: cp .env.fresh .env")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def main() -> int:
|
|
143
|
+
if len(sys.argv) < 2:
|
|
144
|
+
sys.exit(__doc__)
|
|
145
|
+
out = Path(__file__).resolve().parents[1] / "src" / "x2llm" / "endpoints.py"
|
|
146
|
+
run(sys.argv[1:], out)
|
|
147
|
+
write_env_fresh(sys.argv[1:])
|
|
148
|
+
return 0
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
if __name__ == "__main__":
|
|
152
|
+
main()
|