bell-feed-mcp 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bell_feed_mcp-0.1.0/.claude/skills/crypto-intel/SKILL.md +75 -0
- bell_feed_mcp-0.1.0/.gitignore +8 -0
- bell_feed_mcp-0.1.0/PKG-INFO +6 -0
- bell_feed_mcp-0.1.0/analyze_archive.py +211 -0
- bell_feed_mcp-0.1.0/export_bookmarks.py +352 -0
- bell_feed_mcp-0.1.0/fetch_notifications.py +812 -0
- bell_feed_mcp-0.1.0/generate_rss.py +103 -0
- bell_feed_mcp-0.1.0/mcp_server.py +122 -0
- bell_feed_mcp-0.1.0/pyproject.toml +17 -0
- bell_feed_mcp-0.1.0/requirements.txt +1 -0
- bell_feed_mcp-0.1.0/serve.py +46 -0
- bell_feed_mcp-0.1.0/test_extract.py +152 -0
- bell_feed_mcp-0.1.0/uv.lock +744 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: crypto-intel
|
|
3
|
+
description: Get crypto Twitter intelligence from Bell Feed — summaries, signals, trending tokens
|
|
4
|
+
user_invocable: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Crypto Intel — Bell Feed Intelligence
|
|
8
|
+
|
|
9
|
+
Provide crypto Twitter intelligence from Bell Feed data (data.degendar.com).
|
|
10
|
+
|
|
11
|
+
## Arguments
|
|
12
|
+
|
|
13
|
+
- No args or `latest` → Show latest summaries + trading signals
|
|
14
|
+
- `trending` → Show trending tokens ranked by mention frequency
|
|
15
|
+
- `search <query>` → Search summaries by keyword (e.g., `search SOL`)
|
|
16
|
+
- `tldr` → Show today's TL;DR summary
|
|
17
|
+
|
|
18
|
+
## Execution
|
|
19
|
+
|
|
20
|
+
### Step 1: Determine Mode
|
|
21
|
+
|
|
22
|
+
Parse the user's argument to determine which mode to use:
|
|
23
|
+
- Default/`latest` → latest summaries
|
|
24
|
+
- `trending` → trending tokens
|
|
25
|
+
- `search <query>` → keyword search
|
|
26
|
+
- `tldr` → daily TL;DR
|
|
27
|
+
|
|
28
|
+
### Step 2: Fetch Data
|
|
29
|
+
|
|
30
|
+
**If Bell Feed MCP tools are available** (check for `get_latest_summaries`, `get_trending_tokens`, `get_daily_tldr`, `search_summaries` tools):
|
|
31
|
+
|
|
32
|
+
- `latest` → call `get_latest_summaries(count=5)`
|
|
33
|
+
- `trending` → call `get_trending_tokens(hours=24)`
|
|
34
|
+
- `search <query>` → call `search_summaries(query="<query>", max_results=10)`
|
|
35
|
+
- `tldr` → call `get_daily_tldr()`
|
|
36
|
+
|
|
37
|
+
**If MCP tools are NOT available**, fallback to curl:
|
|
38
|
+
|
|
39
|
+
- `latest` → `curl -s https://data.degendar.com/summaries.json | python3 -c "import sys,json; d=json.load(sys.stdin)[:5]; print(json.dumps(d,ensure_ascii=False,indent=2))"`
|
|
40
|
+
- `trending` → `curl -s https://data.degendar.com/summaries.json | python3 -c "
|
|
41
|
+
import sys,json
|
|
42
|
+
from collections import Counter
|
|
43
|
+
from datetime import datetime,timezone
|
|
44
|
+
summaries=json.load(sys.stdin)
|
|
45
|
+
cutoff=datetime.now(timezone.utc).timestamp()-86400
|
|
46
|
+
counts=Counter()
|
|
47
|
+
for s in summaries:
|
|
48
|
+
try:
|
|
49
|
+
ts=datetime.fromisoformat(s['timestamp']).timestamp()
|
|
50
|
+
except: continue
|
|
51
|
+
if ts>=cutoff:
|
|
52
|
+
counts.update(s.get('tokens',[]))
|
|
53
|
+
for t,c in counts.most_common(20):
|
|
54
|
+
print(f'{t}: {c} mentions')
|
|
55
|
+
"`
|
|
56
|
+
- `search <query>` → `curl -s https://data.degendar.com/summaries.json | python3 -c "
|
|
57
|
+
import sys,json
|
|
58
|
+
q='<query>'.lower()
|
|
59
|
+
for s in json.load(sys.stdin):
|
|
60
|
+
text=(s.get('summary_text','')+' '+s.get('signals','')+' '+' '.join(s.get('tokens',[]))).lower()
|
|
61
|
+
if q in text:
|
|
62
|
+
print(f\"[{s['id']}] {s.get('summary_text','')[:150]}\")
|
|
63
|
+
if s.get('signals'): print(f' Signals: {s[\"signals\"]}')
|
|
64
|
+
print()
|
|
65
|
+
"`
|
|
66
|
+
- `tldr` → `curl -s https://data.degendar.com/daily_tldr.json`
|
|
67
|
+
|
|
68
|
+
### Step 3: Format Output
|
|
69
|
+
|
|
70
|
+
Present the results clearly in 繁體中文:
|
|
71
|
+
|
|
72
|
+
- **Latest**: Show each summary with timestamp, summary text, trading signals, and mentioned tokens
|
|
73
|
+
- **Trending**: Show a ranked table of tokens with mention counts
|
|
74
|
+
- **Search**: Show matching summaries with highlighted context
|
|
75
|
+
- **TL;DR**: Show the daily summary directly
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""分析 Twitter archive 資料"""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
from collections import Counter
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
BOOKMARKS_PATH = Path(__file__).parent / "output" / "bookmarks.json"
|
|
13
|
+
OUTPUT_PATH = Path(__file__).parent / "output" / "analysis.md"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def find_archive_dir() -> Path:
|
|
17
|
+
docs = Path.home() / "Documents"
|
|
18
|
+
matches = list(docs.glob("twitter-*"))
|
|
19
|
+
if not matches:
|
|
20
|
+
print("ERROR: 找不到 Twitter archive,請放在 ~/Documents/twitter-*")
|
|
21
|
+
sys.exit(1)
|
|
22
|
+
return matches[0] / "data"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def load_js(archive_dir: Path, filename: str) -> list:
|
|
26
|
+
path = archive_dir / filename
|
|
27
|
+
text = path.read_text(encoding="utf-8")
|
|
28
|
+
json_start = text.index("[")
|
|
29
|
+
return json.loads(text[json_start:])
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def load_bookmarks() -> set[str]:
|
|
33
|
+
if not BOOKMARKS_PATH.exists():
|
|
34
|
+
return set()
|
|
35
|
+
data = json.loads(BOOKMARKS_PATH.read_text(encoding="utf-8"))
|
|
36
|
+
return {t["tweet_id"] for t in data if t.get("tweet_id")}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def analyze_tweets(tweets_raw: list) -> str:
|
|
40
|
+
lines = [f"## Your Tweets ({len(tweets_raw)})\n"]
|
|
41
|
+
|
|
42
|
+
tweets = []
|
|
43
|
+
for item in tweets_raw:
|
|
44
|
+
t = item["tweet"]
|
|
45
|
+
created = datetime.strptime(t["created_at"], "%a %b %d %H:%M:%S %z %Y")
|
|
46
|
+
tweets.append({
|
|
47
|
+
"text": t["full_text"],
|
|
48
|
+
"likes": int(t.get("favorite_count", 0)),
|
|
49
|
+
"rts": int(t.get("retweet_count", 0)),
|
|
50
|
+
"lang": t.get("lang", ""),
|
|
51
|
+
"created_at": created,
|
|
52
|
+
"hour": created.hour,
|
|
53
|
+
"weekday": created.strftime("%A"),
|
|
54
|
+
"is_rt": t["full_text"].startswith("RT @"),
|
|
55
|
+
"is_reply": t["full_text"].startswith("@"),
|
|
56
|
+
"mentions": [m["screen_name"] for m in t.get("entities", {}).get("user_mentions", [])],
|
|
57
|
+
"hashtags": [h["text"] for h in t.get("entities", {}).get("hashtags", [])],
|
|
58
|
+
"source": re.sub(r"<[^>]+>", "", t.get("source", "")),
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
original = [t for t in tweets if not t["is_rt"]]
|
|
62
|
+
rts = [t for t in tweets if t["is_rt"]]
|
|
63
|
+
|
|
64
|
+
lines.append(f"- Original: {len(original)} | RT: {len(rts)}")
|
|
65
|
+
lines.append(f"- Date range: {min(t['created_at'] for t in tweets).strftime('%Y-%m-%d')} ~ {max(t['created_at'] for t in tweets).strftime('%Y-%m-%d')}")
|
|
66
|
+
|
|
67
|
+
lang_counter = Counter(t["lang"] for t in original)
|
|
68
|
+
lines.append(f"- Languages: {', '.join(f'{lang}({cnt})' for lang, cnt in lang_counter.most_common(5))}")
|
|
69
|
+
|
|
70
|
+
source_counter = Counter(t["source"] for t in tweets)
|
|
71
|
+
lines.append(f"- Devices: {', '.join(f'{s}({c})' for s, c in source_counter.most_common(3))}")
|
|
72
|
+
|
|
73
|
+
lines.append("\n### Top Engagement (Original Tweets)\n")
|
|
74
|
+
lines.append("| Likes | RTs | Text |")
|
|
75
|
+
lines.append("|-------|-----|------|")
|
|
76
|
+
top = sorted(original, key=lambda t: t["likes"] + t["rts"], reverse=True)[:15]
|
|
77
|
+
for t in top:
|
|
78
|
+
text = t["text"][:80].replace("\n", " ").replace("|", "\\|")
|
|
79
|
+
lines.append(f"| {t['likes']} | {t['rts']} | {text} |")
|
|
80
|
+
|
|
81
|
+
lines.append("\n### Posting Time (UTC)\n")
|
|
82
|
+
hour_counter = Counter(t["hour"] for t in tweets)
|
|
83
|
+
max_count = max(hour_counter.values()) if hour_counter else 1
|
|
84
|
+
for hour in range(24):
|
|
85
|
+
cnt = hour_counter.get(hour, 0)
|
|
86
|
+
bar = "█" * int(cnt / max_count * 20) if cnt else ""
|
|
87
|
+
lines.append(f" {hour:02d}:00 {bar} {cnt}")
|
|
88
|
+
|
|
89
|
+
lines.append("\n### Day of Week\n")
|
|
90
|
+
day_order = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
|
|
91
|
+
day_counter = Counter(t["weekday"] for t in tweets)
|
|
92
|
+
for day in day_order:
|
|
93
|
+
cnt = day_counter.get(day, 0)
|
|
94
|
+
lines.append(f" {day:<10} {'█' * int(cnt / max(day_counter.values()) * 15)} {cnt}")
|
|
95
|
+
|
|
96
|
+
lines.append("\n### Most Mentioned Users\n")
|
|
97
|
+
mention_counter = Counter()
|
|
98
|
+
for t in tweets:
|
|
99
|
+
mention_counter.update(t["mentions"])
|
|
100
|
+
for user, cnt in mention_counter.most_common(15):
|
|
101
|
+
lines.append(f" @{user} — {cnt} times")
|
|
102
|
+
|
|
103
|
+
lines.append("\n### Top Hashtags\n")
|
|
104
|
+
hashtag_counter = Counter()
|
|
105
|
+
for t in tweets:
|
|
106
|
+
hashtag_counter.update(t["hashtags"])
|
|
107
|
+
if hashtag_counter:
|
|
108
|
+
for tag, cnt in hashtag_counter.most_common(15):
|
|
109
|
+
lines.append(f" #{tag} — {cnt}")
|
|
110
|
+
else:
|
|
111
|
+
lines.append(" (none)")
|
|
112
|
+
|
|
113
|
+
return "\n".join(lines)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def analyze_likes(likes_raw: list, bookmark_ids: set[str]) -> str:
|
|
117
|
+
lines = [f"## Likes ({len(likes_raw)})\n"]
|
|
118
|
+
|
|
119
|
+
likes = []
|
|
120
|
+
for item in likes_raw:
|
|
121
|
+
li = item["like"]
|
|
122
|
+
likes.append({
|
|
123
|
+
"tweet_id": li["tweetId"],
|
|
124
|
+
"text": li.get("fullText", ""),
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
liked_ids = {l["tweet_id"] for l in likes}
|
|
128
|
+
overlap = liked_ids & bookmark_ids
|
|
129
|
+
liked_only = liked_ids - bookmark_ids
|
|
130
|
+
bookmarked_only = bookmark_ids - liked_ids
|
|
131
|
+
|
|
132
|
+
lines.append(f"- Total likes: {len(likes)}")
|
|
133
|
+
lines.append(f"- Total bookmarks: {len(bookmark_ids)}")
|
|
134
|
+
lines.append(f"- Liked AND bookmarked: {len(overlap)}")
|
|
135
|
+
lines.append(f"- Liked but NOT bookmarked: {len(liked_only)}")
|
|
136
|
+
lines.append(f"- Bookmarked but NOT liked: {len(bookmarked_only)}")
|
|
137
|
+
|
|
138
|
+
lines.append("\n### Keyword Frequency in Liked Tweets\n")
|
|
139
|
+
word_counter = Counter()
|
|
140
|
+
stop_words = {"the", "a", "an", "is", "are", "was", "were", "be", "to", "of", "and", "in",
|
|
141
|
+
"that", "it", "for", "on", "with", "as", "at", "by", "this", "from", "or",
|
|
142
|
+
"not", "but", "you", "your", "i", "my", "me", "we", "they", "he", "she",
|
|
143
|
+
"do", "does", "did", "will", "would", "can", "could", "have", "has", "had",
|
|
144
|
+
"rt", "https", "co", "t", "s", "amp", "just", "like", "so", "if", "all",
|
|
145
|
+
"about", "what", "when", "how", "who", "which", "no", "more", "than",
|
|
146
|
+
"been", "also", "其", "的", "了", "在", "是", "我", "不", "有", "這",
|
|
147
|
+
"就", "都", "也", "人", "他", "她", "們", "要", "會", "很", "你",
|
|
148
|
+
"一", "個", "上", "嗎", "吧", "啊", "呢", "把", "被", "讓", "給",
|
|
149
|
+
"而", "但", "又", "還", "到", "去", "來", "著", "過", "得", "能",
|
|
150
|
+
"可以", "因為", "所以", "如果", "那", "什麼", "沒有", "自己"}
|
|
151
|
+
|
|
152
|
+
for li in likes:
|
|
153
|
+
text = li["text"].lower()
|
|
154
|
+
text = re.sub(r"https?://\S+", "", text)
|
|
155
|
+
text = re.sub(r"@\w+", "", text)
|
|
156
|
+
words = re.findall(r"[a-z]{3,}|[\u4e00-\u9fff]{2,}", text)
|
|
157
|
+
word_counter.update(w for w in words if w not in stop_words)
|
|
158
|
+
|
|
159
|
+
for word, cnt in word_counter.most_common(30):
|
|
160
|
+
lines.append(f" {word} — {cnt}")
|
|
161
|
+
|
|
162
|
+
return "\n".join(lines)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def analyze_following(following_raw: list, follower_raw: list) -> str:
|
|
166
|
+
lines = ["## Social Graph\n"]
|
|
167
|
+
|
|
168
|
+
following_ids = {f["following"]["accountId"] for f in following_raw}
|
|
169
|
+
follower_ids = {f["follower"]["accountId"] for f in follower_raw}
|
|
170
|
+
|
|
171
|
+
mutual = following_ids & follower_ids
|
|
172
|
+
following_only = following_ids - follower_ids
|
|
173
|
+
follower_only = follower_ids - following_ids
|
|
174
|
+
|
|
175
|
+
lines.append(f"- Following: {len(following_ids)}")
|
|
176
|
+
lines.append(f"- Followers: {len(follower_ids)}")
|
|
177
|
+
lines.append(f"- Mutual: {len(mutual)}")
|
|
178
|
+
lines.append(f"- You follow but they don't follow back: {len(following_only)}")
|
|
179
|
+
lines.append(f"- They follow you but you don't follow back: {len(follower_only)}")
|
|
180
|
+
lines.append(f"- Follow-back ratio: {len(mutual) / len(following_ids) * 100:.1f}%")
|
|
181
|
+
|
|
182
|
+
return "\n".join(lines)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def main():
|
|
186
|
+
archive_dir = find_archive_dir()
|
|
187
|
+
print(f"→ Loading archive from {archive_dir.parent.name}...")
|
|
188
|
+
|
|
189
|
+
tweets_raw = load_js(archive_dir, "tweets.js")
|
|
190
|
+
likes_raw = load_js(archive_dir, "like.js")
|
|
191
|
+
following_raw = load_js(archive_dir, "following.js")
|
|
192
|
+
follower_raw = load_js(archive_dir, "follower.js")
|
|
193
|
+
bookmark_ids = load_bookmarks()
|
|
194
|
+
|
|
195
|
+
print(f" Tweets: {len(tweets_raw)}, Likes: {len(likes_raw)}, Following: {len(following_raw)}, Followers: {len(follower_raw)}, Bookmarks: {len(bookmark_ids)}")
|
|
196
|
+
|
|
197
|
+
sections = [
|
|
198
|
+
f"# Twitter Archive Analysis\n\nGenerated: {datetime.now().strftime('%Y-%m-%d %H:%M')}\n",
|
|
199
|
+
analyze_tweets(tweets_raw),
|
|
200
|
+
analyze_likes(likes_raw, bookmark_ids),
|
|
201
|
+
analyze_following(following_raw, follower_raw),
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
report = "\n\n---\n\n".join(sections)
|
|
205
|
+
OUTPUT_PATH.write_text(report, encoding="utf-8")
|
|
206
|
+
print(f"\n→ Report saved: {OUTPUT_PATH}")
|
|
207
|
+
print(report)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
if __name__ == "__main__":
|
|
211
|
+
main()
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""X Bookmarks Export — 直接呼叫 GraphQL API"""
|
|
3
|
+
|
|
4
|
+
import csv
|
|
5
|
+
import html
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import random
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
import urllib.parse
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import httpx
|
|
16
|
+
|
|
17
|
+
OUTPUT_DIR = Path(__file__).parent / "output"
|
|
18
|
+
OUTPUT_DIR.mkdir(exist_ok=True)
|
|
19
|
+
|
|
20
|
+
GRAPHQL_URL = "https://x.com/i/api/graphql/tmd4ifV8RHltzn8ymGg1aw/Bookmarks"
|
|
21
|
+
|
|
22
|
+
FEATURES = {
|
|
23
|
+
"graphql_timeline_v2_bookmark_timeline": True,
|
|
24
|
+
"rweb_tipjar_consumption_enabled": True,
|
|
25
|
+
"responsive_web_graphql_exclude_directive_enabled": True,
|
|
26
|
+
"verified_phone_label_enabled": False,
|
|
27
|
+
"creator_subscriptions_tweet_preview_api_enabled": True,
|
|
28
|
+
"responsive_web_graphql_timeline_navigation_enabled": True,
|
|
29
|
+
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
|
|
30
|
+
"communities_web_enable_tweet_community_results_fetch": True,
|
|
31
|
+
"c9s_tweet_anatomy_moderator_badge_enabled": True,
|
|
32
|
+
"articles_preview_enabled": True,
|
|
33
|
+
"responsive_web_edit_tweet_api_enabled": True,
|
|
34
|
+
"tweetypie_unmention_optimization_enabled": True,
|
|
35
|
+
"responsive_web_enhance_cards_enabled": False,
|
|
36
|
+
"responsive_web_media_download_video_enabled": False,
|
|
37
|
+
"responsive_web_twitter_article_tweet_consumption_enabled": True,
|
|
38
|
+
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True,
|
|
39
|
+
"creator_subscriptions_quote_tweet_preview_enabled": False,
|
|
40
|
+
"freedom_of_speech_not_reach_fetch_enabled": True,
|
|
41
|
+
"standardized_nudges_misinfo": True,
|
|
42
|
+
"tweet_awards_web_tipping_enabled": False,
|
|
43
|
+
"longform_notetweets_inline_media_enabled": True,
|
|
44
|
+
"longform_notetweets_rich_text_read_enabled": True,
|
|
45
|
+
"longform_notetweets_consumption_enabled": True,
|
|
46
|
+
"responsive_web_twitter_blue_verified_badge_is_enabled": True,
|
|
47
|
+
"view_counts_everywhere_api_enabled": True,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_tweet(tweet_result: dict) -> dict | None:
|
|
52
|
+
if tweet_result.get("__typename") == "TweetWithVisibilityResults":
|
|
53
|
+
tweet_result = tweet_result.get("tweet", {})
|
|
54
|
+
|
|
55
|
+
if tweet_result.get("__typename") != "Tweet":
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
core = tweet_result.get("core", {})
|
|
59
|
+
user_results = core.get("user_results", {}).get("result", {})
|
|
60
|
+
legacy_user = user_results.get("legacy", {})
|
|
61
|
+
legacy = tweet_result.get("legacy", {})
|
|
62
|
+
|
|
63
|
+
retweeted = legacy.get("retweeted_status_result", {}).get("result", {})
|
|
64
|
+
if retweeted:
|
|
65
|
+
rt_legacy = retweeted.get("legacy", {})
|
|
66
|
+
if retweeted.get("__typename") == "TweetWithVisibilityResults":
|
|
67
|
+
rt_legacy = retweeted.get("tweet", {}).get("legacy", {})
|
|
68
|
+
full_text = rt_legacy.get("full_text", legacy.get("full_text", ""))
|
|
69
|
+
else:
|
|
70
|
+
full_text = legacy.get("full_text", "")
|
|
71
|
+
|
|
72
|
+
note_tweet = tweet_result.get("note_tweet", {}).get("note_tweet_results", {}).get("result", {})
|
|
73
|
+
if note_tweet:
|
|
74
|
+
full_text = note_tweet.get("text", full_text)
|
|
75
|
+
|
|
76
|
+
full_text = html.unescape(full_text)
|
|
77
|
+
|
|
78
|
+
tweet_id = legacy.get("id_str") or tweet_result.get("rest_id", "")
|
|
79
|
+
screen_name = legacy_user.get("screen_name", "")
|
|
80
|
+
|
|
81
|
+
media_entities = legacy.get("extended_entities", {}).get("media", [])
|
|
82
|
+
if not media_entities:
|
|
83
|
+
media_entities = legacy.get("entities", {}).get("media", [])
|
|
84
|
+
media_urls = []
|
|
85
|
+
for m in media_entities:
|
|
86
|
+
if m.get("type") in ("video", "animated_gif"):
|
|
87
|
+
variants = m.get("video_info", {}).get("variants", [])
|
|
88
|
+
mp4s = [v for v in variants if v.get("content_type") == "video/mp4"]
|
|
89
|
+
if mp4s:
|
|
90
|
+
best = max(mp4s, key=lambda v: v.get("bitrate", 0))
|
|
91
|
+
media_urls.append(best["url"])
|
|
92
|
+
else:
|
|
93
|
+
media_urls.append(m.get("media_url_https", ""))
|
|
94
|
+
else:
|
|
95
|
+
media_urls.append(m.get("media_url_https", ""))
|
|
96
|
+
|
|
97
|
+
created_at = legacy.get("created_at", "")
|
|
98
|
+
if created_at:
|
|
99
|
+
try:
|
|
100
|
+
dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
|
|
101
|
+
created_at = dt.isoformat()
|
|
102
|
+
except ValueError:
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
"tweet_id": tweet_id,
|
|
107
|
+
"author_handle": screen_name,
|
|
108
|
+
"author_name": legacy_user.get("name", ""),
|
|
109
|
+
"full_text": full_text,
|
|
110
|
+
"created_at": created_at,
|
|
111
|
+
"tweet_url": f"https://x.com/{screen_name}/status/{tweet_id}" if screen_name and tweet_id else "",
|
|
112
|
+
"likes": legacy.get("favorite_count", 0),
|
|
113
|
+
"retweets": legacy.get("retweet_count", 0),
|
|
114
|
+
"replies": legacy.get("reply_count", 0),
|
|
115
|
+
"views": tweet_result.get("views", {}).get("count", ""),
|
|
116
|
+
"media_urls": media_urls,
|
|
117
|
+
"lang": legacy.get("lang", ""),
|
|
118
|
+
"in_reply_to_handle": legacy.get("in_reply_to_screen_name", ""),
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def extract_bookmarks(data: dict) -> tuple[list[dict], str | None]:
|
|
123
|
+
tweets = []
|
|
124
|
+
cursor = None
|
|
125
|
+
|
|
126
|
+
timeline = (
|
|
127
|
+
data.get("data", {})
|
|
128
|
+
.get("bookmark_timeline_v2", {})
|
|
129
|
+
.get("timeline", {})
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
for instruction in timeline.get("instructions", []):
|
|
133
|
+
if instruction.get("type") == "TimelineAddEntries":
|
|
134
|
+
entries = instruction.get("entries", [])
|
|
135
|
+
elif instruction.get("type") == "TimelineAddToModule":
|
|
136
|
+
entries = instruction.get("moduleItems", [])
|
|
137
|
+
else:
|
|
138
|
+
continue
|
|
139
|
+
|
|
140
|
+
for entry in entries:
|
|
141
|
+
entry_id = entry.get("entryId", "")
|
|
142
|
+
|
|
143
|
+
if entry_id.startswith("cursor-bottom-"):
|
|
144
|
+
cursor = entry.get("content", {}).get("value")
|
|
145
|
+
continue
|
|
146
|
+
|
|
147
|
+
item_content = entry.get("content", {})
|
|
148
|
+
if item_content.get("entryType") == "TimelineTimelineItem":
|
|
149
|
+
tweet_result = (
|
|
150
|
+
item_content.get("itemContent", {})
|
|
151
|
+
.get("tweet_results", {})
|
|
152
|
+
.get("result", {})
|
|
153
|
+
)
|
|
154
|
+
parsed = parse_tweet(tweet_result)
|
|
155
|
+
if parsed:
|
|
156
|
+
tweets.append(parsed)
|
|
157
|
+
elif item_content.get("entryType") == "TimelineTimelineModule":
|
|
158
|
+
for sub_item in item_content.get("items", []):
|
|
159
|
+
sub_content = sub_item.get("item", {}).get("itemContent", {})
|
|
160
|
+
tweet_result = sub_content.get("tweet_results", {}).get("result", {})
|
|
161
|
+
parsed = parse_tweet(tweet_result)
|
|
162
|
+
if parsed:
|
|
163
|
+
tweets.append(parsed)
|
|
164
|
+
else:
|
|
165
|
+
item_inner = entry.get("item", {}).get("itemContent", {})
|
|
166
|
+
tweet_result = item_inner.get("tweet_results", {}).get("result", {})
|
|
167
|
+
parsed = parse_tweet(tweet_result)
|
|
168
|
+
if parsed:
|
|
169
|
+
tweets.append(parsed)
|
|
170
|
+
|
|
171
|
+
return tweets, cursor
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def export_markdown(tweets: list[dict]):
|
|
175
|
+
path = OUTPUT_DIR / "bookmarks.md"
|
|
176
|
+
if not tweets:
|
|
177
|
+
print("沒有資料可匯出")
|
|
178
|
+
return
|
|
179
|
+
|
|
180
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
181
|
+
f.write(f"# X Bookmarks ({len(tweets)} 則)\n\n")
|
|
182
|
+
for tweet in tweets:
|
|
183
|
+
date = tweet["created_at"][:10] if tweet["created_at"] else "unknown"
|
|
184
|
+
f.write(f"## [{tweet['author_name']} (@{tweet['author_handle']})]({tweet['tweet_url']})\n")
|
|
185
|
+
f.write(f"*{date}* | {tweet['likes']} likes | {tweet['retweets']} RT | {tweet['views']} views\n\n")
|
|
186
|
+
f.write(f"{tweet['full_text']}\n\n")
|
|
187
|
+
for url in tweet.get("media_urls", []):
|
|
188
|
+
if url.endswith(".mp4"):
|
|
189
|
+
f.write(f"[Video]({url})\n")
|
|
190
|
+
else:
|
|
191
|
+
f.write(f"\n")
|
|
192
|
+
f.write("\n---\n\n")
|
|
193
|
+
|
|
194
|
+
print(f"✓ Markdown 匯出: {path} ({len(tweets)} 則)")
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def export_json(tweets: list[dict]):
|
|
198
|
+
path = OUTPUT_DIR / "bookmarks.json"
|
|
199
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
200
|
+
json.dump(tweets, f, ensure_ascii=False, indent=2)
|
|
201
|
+
print(f"✓ JSON 匯出: {path} ({len(tweets)} 則)")
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def export_csv(tweets: list[dict]):
|
|
205
|
+
path = OUTPUT_DIR / "bookmarks.csv"
|
|
206
|
+
if not tweets:
|
|
207
|
+
print("沒有資料可匯出")
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
fieldnames = [
|
|
211
|
+
"tweet_id", "author_handle", "author_name", "full_text",
|
|
212
|
+
"created_at", "tweet_url", "likes", "retweets", "replies",
|
|
213
|
+
"views", "media_urls", "lang", "in_reply_to_handle",
|
|
214
|
+
]
|
|
215
|
+
|
|
216
|
+
with open(path, "w", encoding="utf-8", newline="") as f:
|
|
217
|
+
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
|
218
|
+
writer.writeheader()
|
|
219
|
+
for tweet in tweets:
|
|
220
|
+
row = {**tweet, "media_urls": " | ".join(tweet.get("media_urls", []))}
|
|
221
|
+
writer.writerow(row)
|
|
222
|
+
|
|
223
|
+
print(f"✓ CSV 匯出: {path} ({len(tweets)} 則)")
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def load_env():
|
|
227
|
+
env_path = Path(__file__).parent / ".env"
|
|
228
|
+
if not env_path.exists():
|
|
229
|
+
return
|
|
230
|
+
for line in env_path.read_text().splitlines():
|
|
231
|
+
line = line.strip()
|
|
232
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
233
|
+
continue
|
|
234
|
+
key, _, value = line.partition("=")
|
|
235
|
+
os.environ.setdefault(key.strip(), value.strip())
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def main():
|
|
239
|
+
load_env()
|
|
240
|
+
|
|
241
|
+
auth_token = os.environ.get("AUTH_TOKEN") or input("auth_token: ").strip()
|
|
242
|
+
ct0 = os.environ.get("CT0") or input("ct0: ").strip()
|
|
243
|
+
|
|
244
|
+
if not auth_token or not ct0:
|
|
245
|
+
print("ERROR: auth_token 和 ct0 都必須提供")
|
|
246
|
+
sys.exit(1)
|
|
247
|
+
|
|
248
|
+
if os.environ.get("AUTH_TOKEN"):
|
|
249
|
+
print("→ 從 .env 讀取 auth credentials")
|
|
250
|
+
|
|
251
|
+
headers = {
|
|
252
|
+
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
|
|
253
|
+
"x-csrf-token": ct0,
|
|
254
|
+
"cookie": f"auth_token={auth_token}; ct0={ct0}",
|
|
255
|
+
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
|
256
|
+
"x-twitter-active-user": "yes",
|
|
257
|
+
"x-twitter-client-language": "en",
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
existing_tweets: list[dict] = []
|
|
261
|
+
existing_ids: set[str] = set()
|
|
262
|
+
json_path = OUTPUT_DIR / "bookmarks.json"
|
|
263
|
+
if json_path.exists():
|
|
264
|
+
with open(json_path, encoding="utf-8") as f:
|
|
265
|
+
existing_tweets = json.load(f)
|
|
266
|
+
existing_ids = {t["tweet_id"] for t in existing_tweets if t.get("tweet_id")}
|
|
267
|
+
print(f"→ 載入既有資料: {len(existing_tweets)} 則")
|
|
268
|
+
|
|
269
|
+
all_tweets: list[dict] = []
|
|
270
|
+
seen_ids: set[str] = set()
|
|
271
|
+
cursor: str | None = None
|
|
272
|
+
page_num = 0
|
|
273
|
+
hit_existing = False
|
|
274
|
+
|
|
275
|
+
print("→ 開始抓取新書籤...\n")
|
|
276
|
+
|
|
277
|
+
with httpx.Client(headers=headers, timeout=30) as client:
|
|
278
|
+
while True:
|
|
279
|
+
page_num += 1
|
|
280
|
+
variables: dict = {"count": 100}
|
|
281
|
+
if cursor:
|
|
282
|
+
variables["cursor"] = cursor
|
|
283
|
+
|
|
284
|
+
params = {
|
|
285
|
+
"variables": json.dumps(variables),
|
|
286
|
+
"features": json.dumps(FEATURES),
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
resp = client.get(GRAPHQL_URL, params=params)
|
|
290
|
+
|
|
291
|
+
if resp.status_code == 429:
|
|
292
|
+
print(" ⚠ Rate limited,等待 60 秒...")
|
|
293
|
+
time.sleep(60)
|
|
294
|
+
continue
|
|
295
|
+
|
|
296
|
+
if resp.status_code != 200:
|
|
297
|
+
print(f" ERROR: HTTP {resp.status_code}")
|
|
298
|
+
print(f" {resp.text[:500]}")
|
|
299
|
+
break
|
|
300
|
+
|
|
301
|
+
data = resp.json()
|
|
302
|
+
|
|
303
|
+
if "errors" in data:
|
|
304
|
+
print(f" API 錯誤: {data['errors']}")
|
|
305
|
+
break
|
|
306
|
+
|
|
307
|
+
tweets, next_cursor = extract_bookmarks(data)
|
|
308
|
+
|
|
309
|
+
new_count = 0
|
|
310
|
+
for tweet in tweets:
|
|
311
|
+
tid = tweet["tweet_id"]
|
|
312
|
+
if tid and tid in existing_ids:
|
|
313
|
+
hit_existing = True
|
|
314
|
+
continue
|
|
315
|
+
if tid and tid not in seen_ids:
|
|
316
|
+
seen_ids.add(tid)
|
|
317
|
+
all_tweets.append(tweet)
|
|
318
|
+
new_count += 1
|
|
319
|
+
|
|
320
|
+
print(f" 第 {page_num} 頁: 取得 {len(tweets)} 則(新增 {new_count}),累計: {len(all_tweets)}")
|
|
321
|
+
|
|
322
|
+
if hit_existing:
|
|
323
|
+
print(" → 遇到已存在的書籤,停止抓取")
|
|
324
|
+
break
|
|
325
|
+
|
|
326
|
+
if not next_cursor or new_count == 0:
|
|
327
|
+
break
|
|
328
|
+
|
|
329
|
+
cursor = next_cursor
|
|
330
|
+
delay = random.uniform(2.0, 5.0)
|
|
331
|
+
time.sleep(delay)
|
|
332
|
+
|
|
333
|
+
if all_tweets:
|
|
334
|
+
print(f"\n→ 新增 {len(all_tweets)} 則書籤")
|
|
335
|
+
all_tweets.extend(existing_tweets)
|
|
336
|
+
else:
|
|
337
|
+
print("\n→ 沒有新書籤")
|
|
338
|
+
all_tweets = existing_tweets
|
|
339
|
+
|
|
340
|
+
print(f"→ 總計 {len(all_tweets)} 則書籤")
|
|
341
|
+
|
|
342
|
+
all_tweets.sort(key=lambda t: t.get("created_at", ""), reverse=True)
|
|
343
|
+
|
|
344
|
+
export_json(all_tweets)
|
|
345
|
+
export_csv(all_tweets)
|
|
346
|
+
export_markdown(all_tweets)
|
|
347
|
+
|
|
348
|
+
print("\n完成 ✓")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
if __name__ == "__main__":
|
|
352
|
+
main()
|