x-developer 2.6.7 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +4 -4
- package/.claude-plugin/plugin.json +2 -27
- package/.codex-plugin/plugin.json +4 -28
- package/README.de.md +505 -0
- package/README.es.md +504 -0
- package/README.fr.md +507 -0
- package/README.it.md +499 -0
- package/README.ja.md +493 -0
- package/README.ko.md +494 -0
- package/README.md +486 -123
- package/README.tr.md +502 -0
- package/README.zh-CN.md +484 -0
- package/assets/logo-400.png +0 -0
- package/commands/post.md +3 -3
- package/commands/search.md +1 -1
- package/commands/trending.md +1 -1
- package/commands/user.md +1 -1
- package/context7.json +6 -12
- package/docker-mcp-registry/xquik/readme.md +1 -0
- package/docker-mcp-registry/xquik/server.yaml +23 -0
- package/docker-mcp-registry/xquik/tools.json +1 -0
- package/glama.json +4 -0
- package/mcp.json +9 -0
- package/openclaw.plugin.json +2 -2
- package/package.json +24 -20
- package/plugin.json +22 -0
- package/registry.json +60 -0
- package/server.json +2 -2
- package/skills/x-twitter-scraper/LICENSE +21 -0
- package/skills/x-twitter-scraper/SKILL.md +175 -457
- package/skills/x-twitter-scraper/metadata.json +19 -9
- package/skills/x-twitter-scraper/references/compare-faq.md +101 -0
- package/skills/x-twitter-scraper/references/extractions.md +65 -290
- package/skills/x-twitter-scraper/references/mcp.md +24 -0
- package/skills/x-twitter-scraper/references/monitors-webhooks.md +167 -0
- package/skills/x-twitter-scraper/references/reads.md +416 -0
- package/skills/x-twitter-scraper/references/writes.md +111 -0
- package/skills/xquik-mcp/SKILL.md +75 -0
- package/skills/xquik-mcp/references/mcp-setup.md +255 -0
- package/skills/xquik-social-research/SKILL.md +61 -74
- package/skills.sh.json +6 -68
- package/start.sh +2 -2
- package/stub-server.mjs +37 -86
- package/task-guides/check-mutuals.md +3 -3
- package/task-guides/clean-followers.md +4 -4
- package/task-guides/export-tweets-csv.md +7 -7
- package/task-guides/extract-followers.md +10 -10
- package/task-guides/find-bangers.md +2 -2
- package/task-guides/find-influencers.md +4 -4
- package/task-guides/find-viral-tweets.md +9 -9
- package/task-guides/follow-unfollow.md +5 -5
- package/task-guides/for-you-feed.md +4 -4
- package/task-guides/going-viral.md +6 -6
- package/task-guides/grow-followers.md +4 -4
- package/task-guides/monitor-accounts.md +5 -5
- package/task-guides/optimize-tweets.md +3 -3
- package/task-guides/post-tweets.md +13 -13
- package/task-guides/run-giveaway.md +5 -5
- package/task-guides/search-tweets.md +7 -7
- package/task-guides/send-dms.md +11 -11
- package/task-guides/top-replies.md +7 -7
- package/task-guides/track-competitors.md +10 -10
- package/task-guides/track-hashtags.md +11 -11
- package/task-guides/track-mentions.md +7 -7
- package/task-guides/trending-news.md +4 -4
- package/task-guides/tweet-analytics.md +12 -12
- package/task-guides/tweet-ideas.md +2 -2
- package/task-guides/tweet-replies.md +8 -8
- package/task-guides/tweet-style.md +4 -4
- package/task-guides/tweet-webhooks.md +7 -7
- package/task-guides/update-x-profile.md +5 -5
- package/task-guides/user-tweets.md +10 -10
- package/task-guides/who-liked.md +4 -4
- package/task-guides/who-quoted.md +4 -4
- package/task-guides/who-retweeted.md +4 -4
- package/task-guides/write-threads.md +6 -6
- package/task-guides/write-tweets.md +8 -8
- package/task-guides/x-articles.md +3 -3
- package/task-guides/x-bookmarks.md +4 -4
- package/task-guides/x-communities.md +4 -4
- package/task-guides/x-lists.md +4 -4
- package/task-guides/x-spaces.md +4 -4
- package/task-guides/x-trends.md +6 -6
- package/x-twitter-scraper.json +10 -0
- package/CHANGELOG.md +0 -94
- package/scripts/check-reproducible.mjs +0 -48
- package/scripts/check-versions.mjs +0 -24
- package/scripts/release-guard/content-policy.mjs +0 -410
- package/scripts/release-guard/context.mjs +0 -61
- package/scripts/release-guard/frontmatter.mjs +0 -151
- package/scripts/release-guard/policy-checks.mjs +0 -134
- package/scripts/release-guard/policy.mjs +0 -226
- package/scripts/release-guard/repository-checks.mjs +0 -256
- package/skills/x-twitter-scraper/references/api-endpoints-api-keys.md +0 -14
- package/skills/x-twitter-scraper/references/api-endpoints-compose.md +0 -33
- package/skills/x-twitter-scraper/references/api-endpoints-credits.md +0 -11
- package/skills/x-twitter-scraper/references/api-endpoints-drafts.md +0 -105
- package/skills/x-twitter-scraper/references/api-endpoints-draws.md +0 -101
- package/skills/x-twitter-scraper/references/api-endpoints-error-codes.md +0 -67
- package/skills/x-twitter-scraper/references/api-endpoints-events.md +0 -50
- package/skills/x-twitter-scraper/references/api-endpoints-extractions.md +0 -113
- package/skills/x-twitter-scraper/references/api-endpoints-monitors.md +0 -111
- package/skills/x-twitter-scraper/references/api-endpoints-radar.md +0 -50
- package/skills/x-twitter-scraper/references/api-endpoints-support.md +0 -53
- package/skills/x-twitter-scraper/references/api-endpoints-trends.md +0 -48
- package/skills/x-twitter-scraper/references/api-endpoints-tweet-style-cache.md +0 -186
- package/skills/x-twitter-scraper/references/api-endpoints-webhooks.md +0 -120
- package/skills/x-twitter-scraper/references/api-endpoints-x-accounts.md +0 -54
- package/skills/x-twitter-scraper/references/api-endpoints-x-api.md +0 -267
- package/skills/x-twitter-scraper/references/api-endpoints-x-media.md +0 -50
- package/skills/x-twitter-scraper/references/api-endpoints-x-write.md +0 -270
- package/skills/x-twitter-scraper/references/api-endpoints.md +0 -30
- package/skills/x-twitter-scraper/references/automate-twitter-giveaways.md +0 -130
- package/skills/x-twitter-scraper/references/best-x-api-alternative.md +0 -198
- package/skills/x-twitter-scraper/references/compare-twitter-apis.md +0 -151
- package/skills/x-twitter-scraper/references/draws.md +0 -57
- package/skills/x-twitter-scraper/references/export-twitter-followers.md +0 -127
- package/skills/x-twitter-scraper/references/extract-x-community-data.md +0 -121
- package/skills/x-twitter-scraper/references/mcp-setup.md +0 -331
- package/skills/x-twitter-scraper/references/mcp-tools.md +0 -183
- package/skills/x-twitter-scraper/references/monitor-twitter-webhooks.md +0 -121
- package/skills/x-twitter-scraper/references/python-examples.md +0 -201
- package/skills/x-twitter-scraper/references/reliable-twitter-data-api-2026.md +0 -239
- package/skills/x-twitter-scraper/references/scrape-export-twitter-data.md +0 -173
- package/skills/x-twitter-scraper/references/security.md +0 -111
- package/skills/x-twitter-scraper/references/track-twitter-keywords-mentions.md +0 -148
- package/skills/x-twitter-scraper/references/twitter-api-alternative-faq.md +0 -128
- package/skills/x-twitter-scraper/references/twitter-api-without-x-account.md +0 -110
- package/skills/x-twitter-scraper/references/twitter-data-pipeline.md +0 -136
- package/skills/x-twitter-scraper/references/twitter-scraper-api-guide.md +0 -229
- package/skills/x-twitter-scraper/references/types-account-identity.md +0 -9
- package/skills/x-twitter-scraper/references/types-account.md +0 -26
- package/skills/x-twitter-scraper/references/types-api-keys.md +0 -22
- package/skills/x-twitter-scraper/references/types-bookmarks-and-timeline.md +0 -46
- package/skills/x-twitter-scraper/references/types-credits.md +0 -11
- package/skills/x-twitter-scraper/references/types-download-media.md +0 -22
- package/skills/x-twitter-scraper/references/types-draws.md +0 -60
- package/skills/x-twitter-scraper/references/types-error.md +0 -29
- package/skills/x-twitter-scraper/references/types-events.md +0 -23
- package/skills/x-twitter-scraper/references/types-extractions.md +0 -136
- package/skills/x-twitter-scraper/references/types-mcp-output-schemas.md +0 -13
- package/skills/x-twitter-scraper/references/types-monitors.md +0 -49
- package/skills/x-twitter-scraper/references/types-radar.md +0 -47
- package/skills/x-twitter-scraper/references/types-request-bodies.md +0 -30
- package/skills/x-twitter-scraper/references/types-rest-api-vs-mcp-field-naming.md +0 -11
- package/skills/x-twitter-scraper/references/types-support.md +0 -46
- package/skills/x-twitter-scraper/references/types-trends.md +0 -21
- package/skills/x-twitter-scraper/references/types-tweet-drafts.md +0 -20
- package/skills/x-twitter-scraper/references/types-tweet-style-cache.md +0 -50
- package/skills/x-twitter-scraper/references/types-user-activity.md +0 -35
- package/skills/x-twitter-scraper/references/types-webhooks.md +0 -46
- package/skills/x-twitter-scraper/references/types-x-accounts.md +0 -16
- package/skills/x-twitter-scraper/references/types-x-api.md +0 -144
- package/skills/x-twitter-scraper/references/types-x-articles.md +0 -36
- package/skills/x-twitter-scraper/references/types-x-write.md +0 -78
- package/skills/x-twitter-scraper/references/types.md +0 -31
- package/skills/x-twitter-scraper/references/usage.md +0 -50
- package/skills/x-twitter-scraper/references/webhooks.md +0 -321
- package/skills/x-twitter-scraper/references/workflows.md +0 -215
- package/skills/x-twitter-scraper/skill-card.md +0 -84
- package/skills/x-twitter-scraper/skillspector-report.md +0 -20
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# Monitors, events, and webhooks
|
|
2
|
+
|
|
3
|
+
A monitor watches an account or a search query and stores matching events.
|
|
4
|
+
Read events by polling, or receive them at an HTTPS webhook.
|
|
5
|
+
|
|
6
|
+
## Cost and consent
|
|
7
|
+
|
|
8
|
+
Each active monitor costs 30 credits per hour, events and webhook deliveries
|
|
9
|
+
included. That is a monitor's only charge. Reading stored events with
|
|
10
|
+
`GET /events` costs nothing. Creation needs 30 available credits and bills
|
|
11
|
+
nothing itself. The username lookup is free. Before the first create call,
|
|
12
|
+
show the whole setup. It lists the target, event types, webhook URL if any,
|
|
13
|
+
cost, and the stop calls. Get one yes for that setup, then create the monitor
|
|
14
|
+
and the webhook.
|
|
15
|
+
|
|
16
|
+
## Monitors
|
|
17
|
+
|
|
18
|
+
```http
|
|
19
|
+
POST https://xquik.com/api/v1/monitors
|
|
20
|
+
x-api-key: <XQUIK_API_KEY>
|
|
21
|
+
Content-Type: application/json
|
|
22
|
+
|
|
23
|
+
{"username": "nasa", "eventTypes": ["tweet.new"]}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
```http
|
|
27
|
+
POST https://xquik.com/api/v1/monitors/keywords
|
|
28
|
+
x-api-key: <XQUIK_API_KEY>
|
|
29
|
+
Content-Type: application/json
|
|
30
|
+
|
|
31
|
+
{"query": "#rustlang", "eventTypes": ["tweet.new"]}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The event types are `tweet.new`, `tweet.reply`, `tweet.retweet`, `tweet.quote`,
|
|
35
|
+
`tweet.media`, `tweet.link`, `tweet.poll`, `tweet.mention`, `tweet.hashtag`,
|
|
36
|
+
`tweet.longform`, and profile changes such as `profile.bio.changed`,
|
|
37
|
+
`profile.name.changed`, and `profile.avatar.changed`.
|
|
38
|
+
|
|
39
|
+
Only subscribed types are stored and delivered. `tweet.new` covers original
|
|
40
|
+
posts only, so replies, quotes, and reposts need `tweet.reply`, `tweet.quote`,
|
|
41
|
+
and `tweet.retweet`. An account monitor watches that account's own posts and
|
|
42
|
+
profile. To catch other people's posts that mention an account, use a keyword
|
|
43
|
+
monitor with a query such as `@my_brand`.
|
|
44
|
+
|
|
45
|
+
| Action | Account monitor | Keyword monitor |
|
|
46
|
+
| --- | --- | --- |
|
|
47
|
+
| List | `GET /monitors` | `GET /monitors/keywords` |
|
|
48
|
+
| Pause or resume | `PATCH /monitors/{id}` with `{"isActive": false}` | `PATCH /monitors/keywords/{id}` with `{"isActive": false}` |
|
|
49
|
+
| Delete, which also erases stored events | `DELETE /monitors/{id}` | `DELETE /monitors/keywords/{id}` |
|
|
50
|
+
|
|
51
|
+
## Poll events
|
|
52
|
+
|
|
53
|
+
`GET /events?monitorId=<id>&limit=50` or `GET /events?keywordMonitorId=<id>`.
|
|
54
|
+
Filter with `eventType`. The response holds `events`, `hasMore`, and
|
|
55
|
+
`nextCursor`. Pass `nextCursor` back as `cursor`. Each event has `id`, `type`,
|
|
56
|
+
`monitorId`, `monitorType`, `occurredAt`, and `data`, plus `keywordMonitorId`
|
|
57
|
+
and `query` for keyword monitors. Events come newest first, and `nextCursor`
|
|
58
|
+
pages toward older events. For a scheduled poll, request the first page
|
|
59
|
+
without a cursor, keep the IDs you already handled, and stop at the first
|
|
60
|
+
event whose ID you have. Follow `nextCursor` only while a page holds no
|
|
61
|
+
handled event. Save the state file before printing or forwarding events. Pause
|
|
62
|
+
a monitor instead of deleting it when its stored events still matter.
|
|
63
|
+
|
|
64
|
+
## Webhooks
|
|
65
|
+
|
|
66
|
+
`POST /webhooks` with `{"url": "https://...", "eventTypes": [...]}`. The URL
|
|
67
|
+
must use HTTPS. The response returns `secret` once. Store it in a secret
|
|
68
|
+
manager or the `XQUIK_WEBHOOK_SECRET` environment variable before leaving the
|
|
69
|
+
response. A webhook receives events from the account's monitors that match its
|
|
70
|
+
event types.
|
|
71
|
+
|
|
72
|
+
| Action | Route |
|
|
73
|
+
| --- | --- |
|
|
74
|
+
| Send a signed test and get `success` and `statusCode` in the response | `POST /webhooks/{id}/test` |
|
|
75
|
+
| Check event deliveries, which leave out tests | `GET /webhooks/{id}/deliveries` |
|
|
76
|
+
| Pause or deactivate. Deliveries stop, and the webhook is kept | `PATCH /webhooks/{id}` with `{"isActive": false}`, or `DELETE /webhooks/{id}` |
|
|
77
|
+
| Test, then resume a paused webhook | `POST /webhooks/{id}/resume` |
|
|
78
|
+
|
|
79
|
+
## Verify every delivery
|
|
80
|
+
|
|
81
|
+
Each delivery carries three headers. Show them and explain verification in
|
|
82
|
+
webhook previews.
|
|
83
|
+
|
|
84
|
+
- `X-Xquik-Timestamp` holds Unix time in milliseconds
|
|
85
|
+
- `X-Xquik-Nonce` holds 16 random bytes in hex
|
|
86
|
+
- `X-Xquik-Signature` holds `sha256=` plus the hex HMAC-SHA256 of
|
|
87
|
+
`<timestamp>.<nonce>.<raw body>`, keyed with the webhook secret
|
|
88
|
+
|
|
89
|
+
The JSON body holds `eventType`, `streamEventId` (the event ID),
|
|
90
|
+
`deliveryId`, `occurredAt`, `monitorId`, `monitorType`, and `data`, plus
|
|
91
|
+
`username` and `xUserId` for account monitors or `query` for keyword monitors.
|
|
92
|
+
A test delivery holds only `eventType: "webhook.test"`, `timestamp`, and
|
|
93
|
+
`data.message`, with no `streamEventId`. Verify it, answer `2xx`, and skip
|
|
94
|
+
deduplication and processing for it.
|
|
95
|
+
|
|
96
|
+
Verify the raw body bytes before parsing JSON. Reject timestamps more than 5
|
|
97
|
+
minutes from now. Keep each nonce until 5 minutes after the later of receipt
|
|
98
|
+
or signed time. Reject repeated nonces. Compare signatures in constant time.
|
|
99
|
+
A timestamp exactly 5 minutes away still counts as fresh. Free a nonce only
|
|
100
|
+
once its expiry is earlier than now.
|
|
101
|
+
Answer `2xx` fast and process the event later.
|
|
102
|
+
Deduplicate by `streamEventId`, because retries can repeat a delivery.
|
|
103
|
+
|
|
104
|
+
```ts
|
|
105
|
+
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
106
|
+
|
|
107
|
+
const WINDOW_MS = 5 * 60 * 1000;
|
|
108
|
+
const seenNonces = new Map<string, number>();
|
|
109
|
+
|
|
110
|
+
export function verifyXquikDelivery(
|
|
111
|
+
rawBody: Buffer,
|
|
112
|
+
headers: { timestamp?: string; nonce?: string; signature?: string },
|
|
113
|
+
secret: string,
|
|
114
|
+
now = Date.now(),
|
|
115
|
+
): boolean {
|
|
116
|
+
const { timestamp, nonce, signature } = headers;
|
|
117
|
+
if (!secret || !timestamp || !nonce || !signature) return false;
|
|
118
|
+
const sentAt = Number(timestamp);
|
|
119
|
+
if (!Number.isFinite(sentAt) || Math.abs(now - sentAt) > WINDOW_MS) return false;
|
|
120
|
+
for (const [key, expiresAt] of seenNonces) if (expiresAt < now) seenNonces.delete(key);
|
|
121
|
+
if (seenNonces.has(nonce)) return false;
|
|
122
|
+
const expected = Buffer.from(
|
|
123
|
+
"sha256=" + createHmac("sha256", secret).update(`${timestamp}.${nonce}.`).update(rawBody).digest("hex"),
|
|
124
|
+
);
|
|
125
|
+
const received = Buffer.from(signature);
|
|
126
|
+
if (expected.length !== received.length || !timingSafeEqual(expected, received)) return false;
|
|
127
|
+
seenNonces.set(nonce, Math.max(now, sentAt) + WINDOW_MS);
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Frameworks must hand the handler the unparsed body. In Express, mount the
|
|
133
|
+
route with `express.raw({ type: "application/json" })`. In Next.js route
|
|
134
|
+
handlers, read `await request.arrayBuffer()`.
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
import hashlib
|
|
138
|
+
import hmac
|
|
139
|
+
import os
|
|
140
|
+
import time
|
|
141
|
+
|
|
142
|
+
SECRET = os.environ["XQUIK_WEBHOOK_SECRET"].encode()
|
|
143
|
+
WINDOW_MS = 5 * 60 * 1000
|
|
144
|
+
seen_nonces: dict[str, float] = {}
|
|
145
|
+
|
|
146
|
+
def verify(raw: bytes, ts: str, nonce: str, sig: str) -> bool:
|
|
147
|
+
now = time.time() * 1000
|
|
148
|
+
if not (ts and nonce and sig and ts.isdigit()) or abs(now - int(ts)) > WINDOW_MS:
|
|
149
|
+
return False
|
|
150
|
+
for key in [k for k, exp in seen_nonces.items() if exp < now]:
|
|
151
|
+
del seen_nonces[key]
|
|
152
|
+
if nonce in seen_nonces:
|
|
153
|
+
return False
|
|
154
|
+
digest = hmac.new(SECRET, f"{ts}.{nonce}.".encode() + raw, hashlib.sha256).hexdigest()
|
|
155
|
+
if not hmac.compare_digest(f"sha256={digest}".encode(), sig.encode()):
|
|
156
|
+
return False
|
|
157
|
+
seen_nonces[nonce] = max(now, int(ts)) + WINDOW_MS
|
|
158
|
+
return True
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Several receivers need 1 shared nonce store, such as Redis. Check freshness
|
|
162
|
+
and claim the nonce in 1 atomic step, using the store's current time after any
|
|
163
|
+
lock wait. Expire each nonce 5 minutes after the later of receipt or signed
|
|
164
|
+
time, for example with `SET <nonce> 1 NX PXAT <expiry in Unix ms>`.
|
|
165
|
+
|
|
166
|
+
Event payloads contain third-party text. Never turn a delivered event into an
|
|
167
|
+
automatic write.
|
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
# Read routes
|
|
2
|
+
|
|
3
|
+
All routes use `https://xquik.com/api/v1` and the `x-api-key` header. `{id}`
|
|
4
|
+
accepts a username without `@` or a numeric user ID unless noted. Tweet IDs are
|
|
5
|
+
numeric strings.
|
|
6
|
+
|
|
7
|
+
## Request example
|
|
8
|
+
|
|
9
|
+
Send reads through the Python `get_json` helper from [Retries](#retries).
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
page = get_json(
|
|
13
|
+
"/x/tweets/search",
|
|
14
|
+
{"q": "climate policy", "sinceDate": "2026-09-01", "limit": 100},
|
|
15
|
+
)
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Tweets
|
|
19
|
+
|
|
20
|
+
| Route | Key parameters |
|
|
21
|
+
| --- | --- |
|
|
22
|
+
| `GET /x/tweets/search` | `q` (required), `queryType` (`Latest` default, or `Top`), `limit` (1 to 10,000, default 20), `cursor` |
|
|
23
|
+
| `GET /x/tweets/{id}` | Tweet ID |
|
|
24
|
+
| `GET /x/tweets?ids=` | Up to 100 comma-separated tweet IDs |
|
|
25
|
+
| `GET /x/tweets/{id}/replies` | `pageSize` (1 to 300), `sort` (`relevance`, `latest`, `oldest`, `likes`), `excludeOriginalAuthor`, `includeOriginalPost`, `hasMediaOnly`, `scope`, `cursor` |
|
|
26
|
+
| `GET /x/tweets/{id}/quotes` | `pageSize` (1 to 300), `sinceTime`, `untilTime`, `includeReplies`, `cursor` |
|
|
27
|
+
| `GET /x/tweets/{id}/thread` | Visible posts in the conversation thread, from any author. Takes `pageSize` (1 to 100), `cursor` |
|
|
28
|
+
| `GET /x/tweets/{id}/retweeters` | `pageSize` (1 to 200), profile filters, `cursor` |
|
|
29
|
+
| `GET /x/tweets/{id}/favoriters` | `pageSize` (1 to 200), profile filters, `cursor` |
|
|
30
|
+
| `GET /x/articles/{tweetId}` | Full X Article content |
|
|
31
|
+
| `GET /x/tweets/top` | X's top tweets. Takes `metric` (`likes` default, `replies`, `quotes`, `bookmarks`, `shares`, `views`), `window` (`day` default, `week`, `month`), `country` or `language`, `pageSize` (1 to 100), `cursor`. The list ends at 100 |
|
|
32
|
+
| `GET /x/tweets/{id}/hidden-replies` | Replies the author hid, oldest first. Takes `pageSize` (1 to 100), tweet filters, `cursor` |
|
|
33
|
+
| `GET /x/tweets/{id}/retweeter-ids` | Reposter user IDs only. Takes `pageSize` (1 to 200), `cursor` |
|
|
34
|
+
| `GET /x/tweets/{id}/retweeters/check` | `user` (required) takes an ID, username, or profile URL. Returns `isRetweeted`, `retweetedAt`, and `complete` |
|
|
35
|
+
| `GET /x/tweets/{id}/insights` | Engagements per hour, per view, and per follower of the author |
|
|
36
|
+
| `GET /x/tweets/{id}/translation` | `language` (required), such as `en` or `pt-br`. Returns X's own translation |
|
|
37
|
+
| `GET /x/tweets/{id}/subtitles` | Caption tracks of the tweet's videos, with language, timed cues, and full text |
|
|
38
|
+
| `GET /x/tweets/{id}/embed` | Embed HTML. Takes `theme` (`light` or `dark`), `hideMedia`, `hideThread`, `omitScript` |
|
|
39
|
+
|
|
40
|
+
Search, replies, quotes, thread, and user timelines accept the same tweet
|
|
41
|
+
filters as named query parameters. Send only the ones the user asked for.
|
|
42
|
+
|
|
43
|
+
- Authors and targets use `fromUser`, `toUser`, `mentioning`, `verifiedOnly`,
|
|
44
|
+
`blueVerifiedOnly`
|
|
45
|
+
- Text uses `exactPhrase`, `anyWords`, `excludeWords`, `hashtags`, `cashtags`, `url`
|
|
46
|
+
- Time uses `sinceDate` and `untilDate` (`YYYY-MM-DD`), `sinceTime` (ISO,
|
|
47
|
+
inclusive) and `untilTime` (ISO, exclusive, so end 2023 with
|
|
48
|
+
`2024-01-01T00:00:00Z`), `withinTime` (such as `7d`)
|
|
49
|
+
- Language uses `language` (such as `en`)
|
|
50
|
+
- Media uses `mediaType` (`images`, `videos`, `gifs`, `media`, `links`, `none`)
|
|
51
|
+
- Engagement uses `minLikes`, `minRetweets`, `minReplies`, `minQuotes`,
|
|
52
|
+
`minViews`, `minBookmarks`, `maxFaves`, `maxRetweets`, `maxReplies`
|
|
53
|
+
- Post types use `replies`, `retweets`, `quotes`, each `include`, `exclude`, or `only`
|
|
54
|
+
- Conversation uses `conversationId`, `inReplyToTweetId`, `quotesOfTweetId`
|
|
55
|
+
- Raw operators use `advancedQuery` for a known X search operator string, on search only
|
|
56
|
+
|
|
57
|
+
Tweet responses hold `tweets`, `has_next_page`, and `next_cursor`. Each tweet
|
|
58
|
+
has `id`, `text`, `createdAt`, `lang`, `likeCount`, `retweetCount`,
|
|
59
|
+
`replyCount`, `quoteCount`, `viewCount`, `bookmarkCount`, `media`, and
|
|
60
|
+
`author` with `id`, `username`, and `name`. Optional fields are omitted when X
|
|
61
|
+
does not return them. A filtered page can be empty and still have a next page.
|
|
62
|
+
|
|
63
|
+
## Users
|
|
64
|
+
|
|
65
|
+
Use the user timeline for an account's latest posts. Its path accepts a
|
|
66
|
+
username or numeric ID.
|
|
67
|
+
|
|
68
|
+
| Route | Key parameters |
|
|
69
|
+
| --- | --- |
|
|
70
|
+
| `GET /x/users/{id}` | Profile with `followers`, `following`, `description` (bio), `verified`, `isVerified`, `isBlueVerified`, `verifiedType`, `statusesCount`, `location`, `createdAt` |
|
|
71
|
+
| `GET /x/users/search` | `q` (required), `pageSize` (1 to 100), profile filters, `cursor` |
|
|
72
|
+
| `GET /x/users/batch` | `ids` (comma-separated user IDs), profile filters |
|
|
73
|
+
| `GET /x/users/{id}/tweets` | `pageSize` (1 to 300, default 20), `sinceTime`, `untilTime`, `includeReplies` (default `false`), tweet filters, `cursor` |
|
|
74
|
+
| `GET /x/users/{id}/replies` | With Replies timeline. Takes `pageSize` (1 to 300), `sinceTime`, `untilTime`, tweet filters, `cursor` |
|
|
75
|
+
| `GET /x/users/{id}/media` | `pageSize` (1 to 100), tweet filters, `cursor` |
|
|
76
|
+
| `GET /x/users/{id}/mentions` | `pageSize` (1 to 100), `sinceTime`, `untilTime`, tweet filters, `cursor` |
|
|
77
|
+
| `GET /x/users/{id}/likes` | `pageSize` (1 to 100), tweet filters, `cursor`. Needs a connected X account for the owner's likes |
|
|
78
|
+
| `GET /x/users/batch/tweets` | Newest posts of up to 20 users. Takes `ids` or `usernames` (comma-separated, send 1 of the 2), `pageSize` (1 to 20) |
|
|
79
|
+
| `GET /x/users/{id}/articles` | Posts on the Articles tab, newest first. Takes `pageSize` (1 to 100), tweet filters, `cursor`. Read the full text with `GET /x/articles/{tweetId}` |
|
|
80
|
+
| `GET /x/users/{id}/highlights` | Posts on the Highlights tab. Takes `pageSize` (1 to 100), tweet filters, `cursor` |
|
|
81
|
+
| `GET /x/users/{id}/insights` | Engagement rate, averages per post, and posting rate, from the last 20 posts |
|
|
82
|
+
| `GET /x/users/{id}/translation` | `language` (required). Returns X's own translation of the bio |
|
|
83
|
+
| `GET /x/users/{id}/live` | Whether the user broadcasts live video now, with the latest broadcast |
|
|
84
|
+
| `GET /x/users/{id}/lists` | Public Lists the user created. Takes `pageSize` (1 to 100), `cursor` |
|
|
85
|
+
| `GET /x/users/{id}/list-memberships` | Public Lists that include the user. Takes `pageSize` (1 to 100), `cursor` |
|
|
86
|
+
|
|
87
|
+
For verification, `isBlueVerified` means X shows the blue badge. `isVerified` and
|
|
88
|
+
`verified` mean X marks the profile as verified. `verifiedType` names the kind,
|
|
89
|
+
such as `Business`. Report only the values the response returns.
|
|
90
|
+
|
|
91
|
+
## Relationships
|
|
92
|
+
|
|
93
|
+
| Route | Key parameters |
|
|
94
|
+
| --- | --- |
|
|
95
|
+
| `GET /x/users/{id}/followers` | `pageSize` (1 to 300), `cursor`, profile filters |
|
|
96
|
+
| `GET /x/users/{id}/following` | Same as followers |
|
|
97
|
+
| `GET /x/users/{id}/verified-followers` | Same as followers |
|
|
98
|
+
| `GET /x/followers/check` | `source` and `target` usernames. Returns `isFollowing` and `isFollowedBy` |
|
|
99
|
+
| `GET /x/users/{id}/follower-ids` | Follower user IDs only, newest first. Takes `pageSize` (1 to 5,000), `cursor` |
|
|
100
|
+
| `GET /x/users/{id}/following-ids` | User IDs of the accounts the user follows. Takes `pageSize` (1 to 5,000), `cursor` |
|
|
101
|
+
| `GET /x/users/{id}/similar` | Accounts X lists as similar. Takes `pageSize` (1 to 200), profile filters, `cursor` |
|
|
102
|
+
| `GET /x/users/{id}/affiliates` | Accounts an organization lists as affiliates. Takes `pageSize` (1 to 200), profile filters, `cursor` |
|
|
103
|
+
| `GET /x/users/{id}/subscriptions` | Creators the user subscribes to. Takes `pageSize` (1 to 200), profile filters, `cursor` |
|
|
104
|
+
|
|
105
|
+
Profile filters, applied before billing, are `minFollowers`, `maxFollowers`,
|
|
106
|
+
`minFollowing`, `maxFollowing`, `minStatuses`, `maxStatuses`,
|
|
107
|
+
`minAccountAgeDays`, `verifiedOnly`, `hasWebsite`, `hasLocation`,
|
|
108
|
+
`bioContains` (any comma-separated term, ignoring case), `locationContains`,
|
|
109
|
+
`usernameContains`. Profile pages return
|
|
110
|
+
`users`, `has_next_page`, and `next_cursor`. Use a `follower_explorer`
|
|
111
|
+
extraction for a complete follower list.
|
|
112
|
+
|
|
113
|
+
## Lists and communities
|
|
114
|
+
|
|
115
|
+
| Route | Notes |
|
|
116
|
+
| --- | --- |
|
|
117
|
+
| `GET /x/lists/search` | Find public Lists with `q` (required), `pageSize` (1 to 100), `cursor` |
|
|
118
|
+
| `GET /x/lists/{id}` | Name, description, counts, and owner. Takes a List ID or a URL-encoded List URL |
|
|
119
|
+
| `GET /x/lists/{id}/members`, `/followers`, `/tweets` | List ID |
|
|
120
|
+
| `GET /x/communities/find` | Find communities by keyword with `q` (required), `pageSize` (1 to 100), `cursor` |
|
|
121
|
+
| `GET /x/communities/topics` | Topics and subtopics X groups communities under. Takes `limit` |
|
|
122
|
+
| `GET /x/communities/popular` | Top communities of a topic. Takes `topicId` or `topic`, `pageSize` (1 to 100), `cursor` |
|
|
123
|
+
| `GET /x/communities/suggested` | Communities X suggests to join. Takes `limit` |
|
|
124
|
+
| `GET /x/communities/{id}/info`, `/members`, `/moderators`, `/tweets` | Community ID |
|
|
125
|
+
| `GET /x/communities/{id}/media` | Posts with photos or videos on the Media tab, newest first. Takes `pageSize` (1 to 100), tweet filters, `cursor` |
|
|
126
|
+
| `GET /x/communities/{id}/members/search` | Find members by name or username with `q` (required) |
|
|
127
|
+
| `GET /x/communities/search` | Search posts inside a community |
|
|
128
|
+
| `GET /x/communities/tweets` | Search posts in 1 community with `communityId`, or in every community without it. Takes `q` (required), `queryType`, `pageSize` (1 to 100), tweet filters, `cursor` |
|
|
129
|
+
|
|
130
|
+
## Spaces, broadcasts, and jobs
|
|
131
|
+
|
|
132
|
+
| Route | Notes |
|
|
133
|
+
| --- | --- |
|
|
134
|
+
| `GET /x/spaces/search` | `q` (required) takes words, hashtags, or operators such as `from:solana`. `queryType`, `pageSize` (1 to 100), `cursor` |
|
|
135
|
+
| `GET /x/spaces/{id}` | Title, state, times, audience, host, and speakers. Takes a Space ID or a URL-encoded Space URL |
|
|
136
|
+
| `GET /x/spaces/{id}/replay` | HLS playback URL of an ended Space. A live or scheduled Space answers `409` |
|
|
137
|
+
| `GET /x/broadcasts/{id}` | Title, state, times, watch counts, and host of a live video broadcast |
|
|
138
|
+
| `GET /x/jobs/search` | `q` (required), `location` or `locationId`, `locationType`, `seniority`, `employmentType`, `company`, `pageSize` (1 to 100), `cursor` |
|
|
139
|
+
| `GET /x/jobs/locations` | `q` (required). Returns the location IDs that job search takes |
|
|
140
|
+
| `GET /x/jobs/{id}` | 1 job listing with its description, salary, company, and apply link |
|
|
141
|
+
|
|
142
|
+
Use the `space_explorer` extraction for a Space's participants.
|
|
143
|
+
|
|
144
|
+
## Trends and search helpers
|
|
145
|
+
|
|
146
|
+
| Route | Notes |
|
|
147
|
+
| --- | --- |
|
|
148
|
+
| `GET /x/trends` | Pick the place by `woeid` (default 1, worldwide), `country`, or `location`. `count` (1 to 50, default 30) |
|
|
149
|
+
| `GET /x/trends/locations` | Places X offers trends for. Takes `country`, `placeType`, `q`, `limit`. Pass a location's `woeid` to `/x/trends` |
|
|
150
|
+
| `GET /radar` | Trending topics from curated sources. Takes `category`, `region`, `hours`, `limit` |
|
|
151
|
+
| `GET /x/search/autocomplete` | Suggestions for a partial query. Takes `q` (required), `types` (comma-separated `users`, `topics`, `hashtags`, `cashtags`), `limit` (1 to 100, default 20) |
|
|
152
|
+
| `GET /x/places/search` | `q` (required), `limit`. Pass a place `id` as `place` to tweet search |
|
|
153
|
+
| `GET /x/hashflags` | Custom emoji X shows after hashtags. Takes `q`, `activeOnly` (default `true`), `limit` |
|
|
154
|
+
| `GET /x/links/resolve` | `urls` (required) takes 1 to 100 `https://t.co/` links, comma-separated. Returns each `resolvedUrl` |
|
|
155
|
+
|
|
156
|
+
A trends request bills 1 credit, however many trends it returns. `woeid` wins
|
|
157
|
+
when a request also names a `country` or `location`. The table lists common
|
|
158
|
+
WOEIDs.
|
|
159
|
+
|
|
160
|
+
| Region | WOEID | Region | WOEID |
|
|
161
|
+
| --- | --- | --- | --- |
|
|
162
|
+
| Worldwide | 1 | France | 23424819 |
|
|
163
|
+
| United States | 23424977 | Japan | 23424856 |
|
|
164
|
+
| United Kingdom | 23424975 | India | 23424848 |
|
|
165
|
+
| Turkey | 23424969 | Brazil | 23424768 |
|
|
166
|
+
| Spain | 23424950 | Canada | 23424775 |
|
|
167
|
+
| Germany | 23424829 | Mexico | 23424900 |
|
|
168
|
+
|
|
169
|
+
`GET /x/trends/locations` lists every other place.
|
|
170
|
+
|
|
171
|
+
Lookups, checks, insights, translations, and embeds bill 1 credit per call.
|
|
172
|
+
Lists of IDs, Lists, communities, topics, Spaces, jobs, places, locations,
|
|
173
|
+
hashflags, suggestions, and resolved links bill 1 credit per returned item.
|
|
174
|
+
Subtitles bill 1 credit only when a caption track returns.
|
|
175
|
+
[Pricing](compare-faq.md#pricing-facts) lists every rate.
|
|
176
|
+
|
|
177
|
+
## Media
|
|
178
|
+
|
|
179
|
+
`POST /x/media/download` with JSON `{"tweetInput": "<tweet URL or ID>"}` for
|
|
180
|
+
one tweet, or `{"tweetIds": ["<id>", "..."]}` for up to 50 tweets.
|
|
181
|
+
|
|
182
|
+
- One tweet returns `tweetId`, `galleryUrl`, and `cacheHit`.
|
|
183
|
+
- Several tweets return `galleryUrl`, `totalTweets`, and `totalMedia`.
|
|
184
|
+
- `galleryUrl` is the page where the user saves the files.
|
|
185
|
+
- A tweet without media returns `400 no_media`.
|
|
186
|
+
|
|
187
|
+
A download grants no reuse rights. Tell the user to confirm permission before
|
|
188
|
+
using the media in their own work.
|
|
189
|
+
|
|
190
|
+
## AI analysis
|
|
191
|
+
|
|
192
|
+
These routes read posts and answer questions about each one with AI. Send a
|
|
193
|
+
JSON body with `POST`. Each analyzed post bills 2 credits, read included.
|
|
194
|
+
|
|
195
|
+
| Route | Answers |
|
|
196
|
+
| --- | --- |
|
|
197
|
+
| `POST /x/analysis/sentiment` | `sentiment`, `intensity`, `sarcasm` |
|
|
198
|
+
| `POST /x/analysis/brand` | `relevance`, `sentiment`, `experience` for the brand in `analysis.targets` |
|
|
199
|
+
| `POST /x/analysis/news` | `format`, `attribution`, `relevance` for the topic in `analysis.targets` |
|
|
200
|
+
| `POST /x/analysis/market-signals` | `stance`, `content`, `conviction`, `relevance`, plus bullish & bearish counts per cashtag |
|
|
201
|
+
| `POST /x/analysis/viral-score` | 8 wording traits, `viralScore` from 0 to 100 & `viralVerdict` |
|
|
202
|
+
| `POST /x/analysis/classify` | 1 to 8 questions you write in `analysis.questions` |
|
|
203
|
+
|
|
204
|
+
Send exactly 1 source.
|
|
205
|
+
|
|
206
|
+
- `tweetIds` takes up to 100 post IDs or URLs.
|
|
207
|
+
- `texts` takes up to 100 texts of the user's, such as drafts. Nothing is read
|
|
208
|
+
from X.
|
|
209
|
+
- A search takes `query`, `username`, `listId`, `quotesOf`, or `repliesTo`,
|
|
210
|
+
alone or together, with `sinceTime`, `untilTime`, `queryType`, `limit` (1 to
|
|
211
|
+
100, default 20), and `cursor`.
|
|
212
|
+
|
|
213
|
+
`analysis` is optional. It takes `targets` (names with `aliases`), `context`,
|
|
214
|
+
`questions`, or a `preset`. Question IDs and category names are lower case
|
|
215
|
+
with underscores.
|
|
216
|
+
|
|
217
|
+
Send each analysis once with the helpers from [Retries](#retries). A refused
|
|
218
|
+
call bills nothing. Ask the user before sending it again.
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
response = requests.post(
|
|
222
|
+
f"{BASE}/x/analysis/brand",
|
|
223
|
+
headers=HEADERS,
|
|
224
|
+
json={"username": "Sony", "limit": 50, "analysis": {"targets": [{"name": "Sony"}]}},
|
|
225
|
+
timeout=60,
|
|
226
|
+
allow_redirects=False,
|
|
227
|
+
)
|
|
228
|
+
page = read_body(response)
|
|
229
|
+
if not 200 <= response.status_code < 300 or isinstance(page, str):
|
|
230
|
+
raise XquikError(f"{response.status_code}: {page}", response.status_code, page)
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
The response holds `results` (each post with its `answers`), `unanalyzed`
|
|
234
|
+
(free, with a `reason`), `analysisSummary` (totals per question, target and
|
|
235
|
+
cashtag), `has_next_page`, and `next_cursor`. Answers describe what posts say.
|
|
236
|
+
They do not verify claims or give investment advice.
|
|
237
|
+
|
|
238
|
+
## Private reads
|
|
239
|
+
|
|
240
|
+
These need a connected X account.
|
|
241
|
+
|
|
242
|
+
- In `GET /x/dm/{userId}/history?account=<connected handle>`, `userId` is the
|
|
243
|
+
other person's numeric ID. Resolve it with `GET /x/users/{username}`. It
|
|
244
|
+
bills 1 credit per returned message and has no page size, so the last page
|
|
245
|
+
can go past a message target. Give the cost per message, not a hard cap.
|
|
246
|
+
- `GET /x/bookmarks`, `GET /x/bookmarks/folders`, `GET /x/notifications`,
|
|
247
|
+
`GET /x/timeline`
|
|
248
|
+
- `GET /x/accounts` lists connected accounts.
|
|
249
|
+
|
|
250
|
+
## Pagination and errors
|
|
251
|
+
|
|
252
|
+
- Follow `next_cursor` while `has_next_page` is true. Stop at the user's bound.
|
|
253
|
+
Never build or decode a cursor.
|
|
254
|
+
- On `409 coverage_cursor_unavailable`, wait the exact `Retry-After` seconds,
|
|
255
|
+
then retry the same cursor. The helper below does this.
|
|
256
|
+
- On `410 coverage_cursor_gone` or `400 invalid_coverage_cursor`, restart once
|
|
257
|
+
without a cursor and deduplicate by ID. The paging loop below catches
|
|
258
|
+
`XquikError` from the helper and restarts with the remaining budget.
|
|
259
|
+
- Generated paging code keeps one result budget across the entire run. Count
|
|
260
|
+
every returned row before deduplication, including refetched rows after a
|
|
261
|
+
restart. Lower later page sizes to the remaining budget. Persist IDs
|
|
262
|
+
already appended across cron runs. An empty page can still have a cursor.
|
|
263
|
+
- On `429`, `5xx`, and failed connections, retry as [Retries](#retries) shows.
|
|
264
|
+
|
|
265
|
+
## Retries
|
|
266
|
+
|
|
267
|
+
- Retry `GET` on `429`, `409 coverage_cursor_unavailable` with `Retry-After`, `5xx`, and
|
|
268
|
+
connection failures, such as a refused or reset connection or a timeout.
|
|
269
|
+
- Wait `Retry-After` when the response has it. Otherwise back off
|
|
270
|
+
exponentially from about 1 second, cap each wait at 30 seconds, and add
|
|
271
|
+
jitter. Stop after about 5 minutes and report the last error.
|
|
272
|
+
- Retry the same URL and cursor, so pages are not skipped or repeated.
|
|
273
|
+
- Set a request timeout, such as 30 seconds, so a stalled connection cannot
|
|
274
|
+
hang a run.
|
|
275
|
+
- Do not retry other `4xx` errors. Fix the request, or handle the cursor
|
|
276
|
+
errors above.
|
|
277
|
+
- Parse bodies safely. An outage can return an HTML page, even with a `2xx`
|
|
278
|
+
status, so parsing it as JSON throws. Retry that case.
|
|
279
|
+
- Never retry `POST`, `PATCH`, or `DELETE` automatically. See
|
|
280
|
+
[writes](writes.md#responses-and-retries).
|
|
281
|
+
|
|
282
|
+
In JavaScript, `fetch` rejects on a refused or reset connection, so catch that
|
|
283
|
+
as a retryable failure. Read `await response.text()` and parse it inside
|
|
284
|
+
`try`.
|
|
285
|
+
|
|
286
|
+
```python
|
|
287
|
+
import os
|
|
288
|
+
import random
|
|
289
|
+
import time
|
|
290
|
+
from email.utils import parsedate_to_datetime
|
|
291
|
+
|
|
292
|
+
import requests
|
|
293
|
+
|
|
294
|
+
BASE = "https://xquik.com/api/v1"
|
|
295
|
+
API_KEY = os.environ.get("XQUIK_API_KEY", "").strip()
|
|
296
|
+
if not API_KEY or not API_KEY.isascii() or not API_KEY.isprintable():
|
|
297
|
+
raise ValueError("Set XQUIK_API_KEY to a valid API key.")
|
|
298
|
+
HEADERS = {"x-api-key": API_KEY}
|
|
299
|
+
NETWORK_ERRORS = (
|
|
300
|
+
requests.ConnectionError,
|
|
301
|
+
requests.Timeout,
|
|
302
|
+
requests.exceptions.ChunkedEncodingError,
|
|
303
|
+
requests.exceptions.ContentDecodingError,
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
class XquikError(Exception):
|
|
308
|
+
def __init__(self, message, status=None, body=None):
|
|
309
|
+
super().__init__(message)
|
|
310
|
+
self.status, self.body = status, body
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def redact(value):
|
|
314
|
+
"""Replace the API key in text, lists, and dictionary keys and values."""
|
|
315
|
+
if isinstance(value, str):
|
|
316
|
+
return value.replace(API_KEY, "[redacted]")
|
|
317
|
+
if isinstance(value, dict):
|
|
318
|
+
return {redact(key): redact(item) for key, item in value.items()}
|
|
319
|
+
if isinstance(value, list):
|
|
320
|
+
return [redact(item) for item in value]
|
|
321
|
+
return value
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def retry_after(response):
|
|
325
|
+
value = response.headers.get("Retry-After", "")
|
|
326
|
+
if value.isdigit():
|
|
327
|
+
return float(value)
|
|
328
|
+
try:
|
|
329
|
+
return max(0.0, parsedate_to_datetime(value).timestamp() - time.time())
|
|
330
|
+
except (TypeError, ValueError):
|
|
331
|
+
return None
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def read_body(response):
|
|
335
|
+
"""Return redacted JSON, or up to 200 characters of a redacted non-JSON body."""
|
|
336
|
+
try:
|
|
337
|
+
return redact(response.json())
|
|
338
|
+
except ValueError:
|
|
339
|
+
return redact(response.text)[:200]
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def get_json(path, params=None, budget_s=300):
|
|
343
|
+
"""GET with retries for reads only. Never use it for POST, PATCH, or DELETE."""
|
|
344
|
+
deadline = time.monotonic() + budget_s
|
|
345
|
+
attempt = 0
|
|
346
|
+
while True:
|
|
347
|
+
wait, status, body = None, None, None
|
|
348
|
+
try:
|
|
349
|
+
response = requests.get(
|
|
350
|
+
f"{BASE}{path}", headers=HEADERS, params=params,
|
|
351
|
+
timeout=30, allow_redirects=False,
|
|
352
|
+
)
|
|
353
|
+
except NETWORK_ERRORS as exc:
|
|
354
|
+
problem = redact(f"connection failed: {exc}")
|
|
355
|
+
else:
|
|
356
|
+
status, body = response.status_code, read_body(response)
|
|
357
|
+
if 200 <= status < 300 and not isinstance(body, str):
|
|
358
|
+
return body
|
|
359
|
+
wait = retry_after(response)
|
|
360
|
+
problem = f"{status}: {body}"
|
|
361
|
+
busy_cursor = (
|
|
362
|
+
status == 409
|
|
363
|
+
and isinstance(body, dict)
|
|
364
|
+
and body.get("error") == "coverage_cursor_unavailable"
|
|
365
|
+
and wait is not None
|
|
366
|
+
)
|
|
367
|
+
if not (200 <= status < 300 or status >= 500 or status == 429 or busy_cursor):
|
|
368
|
+
raise XquikError(problem, status, body)
|
|
369
|
+
if wait is None:
|
|
370
|
+
wait = min(30.0, 2.0**attempt) * random.uniform(0.5, 1.0)
|
|
371
|
+
attempt += 1
|
|
372
|
+
if time.monotonic() + wait > deadline:
|
|
373
|
+
raise XquikError(
|
|
374
|
+
f"Gave up after {attempt} attempts. Last: {problem}", status, body
|
|
375
|
+
)
|
|
376
|
+
time.sleep(wait)
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
A paging loop on that helper. It restarts once on a gone or invalid cursor.
|
|
380
|
+
It counts every returned tweet toward the user's number, refetched ones
|
|
381
|
+
included, so the bill never passes that cap. Load `seen_ids` from earlier
|
|
382
|
+
output, and make `append_row` save each row before it returns.
|
|
383
|
+
|
|
384
|
+
```python
|
|
385
|
+
def append_search_tweets(q, cap, seen_ids, append_row):
|
|
386
|
+
billed, cursor, restarted, cursors = 0, None, False, set()
|
|
387
|
+
while billed < cap:
|
|
388
|
+
params = {"q": q, "limit": cap - billed}
|
|
389
|
+
if cursor:
|
|
390
|
+
params["cursor"] = cursor
|
|
391
|
+
try:
|
|
392
|
+
page = get_json("/x/tweets/search", params)
|
|
393
|
+
except XquikError as err:
|
|
394
|
+
code = err.body.get("error") if isinstance(err.body, dict) else None
|
|
395
|
+
restartable = (err.status, code) in (
|
|
396
|
+
(410, "coverage_cursor_gone"),
|
|
397
|
+
(400, "invalid_coverage_cursor"),
|
|
398
|
+
)
|
|
399
|
+
if restarted or not restartable:
|
|
400
|
+
raise
|
|
401
|
+
cursor, restarted = None, True
|
|
402
|
+
cursors.clear()
|
|
403
|
+
continue
|
|
404
|
+
billed += len(page["tweets"])
|
|
405
|
+
for tweet in page["tweets"]:
|
|
406
|
+
if tweet["id"] not in seen_ids:
|
|
407
|
+
append_row(tweet)
|
|
408
|
+
seen_ids.add(tweet["id"])
|
|
409
|
+
if not page["has_next_page"] or billed >= cap:
|
|
410
|
+
break
|
|
411
|
+
cursor = page["next_cursor"]
|
|
412
|
+
if not isinstance(cursor, str) or not cursor or cursor in cursors:
|
|
413
|
+
raise XquikError("Invalid or repeated search cursor")
|
|
414
|
+
cursors.add(cursor)
|
|
415
|
+
return billed
|
|
416
|
+
```
|