agent-wire-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. agent_wire/__init__.py +1 -0
  2. agent_wire/__main__.py +3 -0
  3. agent_wire/assets/SKILL.md +26 -0
  4. agent_wire/assets/cursor_rule.mdc +13 -0
  5. agent_wire/assets/hermes_skill.md +28 -0
  6. agent_wire/assets/windsurf_rules.md +14 -0
  7. agent_wire/assets/wire-worker.md +20 -0
  8. agent_wire/auth/__init__.py +0 -0
  9. agent_wire/auth/browser_paths.py +143 -0
  10. agent_wire/auth/cookie_vault.py +181 -0
  11. agent_wire/auth/rate_limiter.py +26 -0
  12. agent_wire/cache/__init__.py +0 -0
  13. agent_wire/cache/ring_buffer.py +94 -0
  14. agent_wire/cache/sqlite_ledger.py +132 -0
  15. agent_wire/channels/__init__.py +41 -0
  16. agent_wire/channels/base.py +249 -0
  17. agent_wire/channels/tier1_global/__init__.py +0 -0
  18. agent_wire/channels/tier1_global/github.py +371 -0
  19. agent_wire/channels/tier1_global/hackernews.py +135 -0
  20. agent_wire/channels/tier1_global/reddit.py +371 -0
  21. agent_wire/channels/tier1_global/rss.py +129 -0
  22. agent_wire/channels/tier1_global/search.py +160 -0
  23. agent_wire/channels/tier1_global/stackoverflow.py +94 -0
  24. agent_wire/channels/tier1_global/web.py +98 -0
  25. agent_wire/channels/tier1_global/wikipedia.py +91 -0
  26. agent_wire/channels/tier1_global/x_twitter.py +380 -0
  27. agent_wire/channels/tier1_global/youtube.py +223 -0
  28. agent_wire/channels/tier2_pro/__init__.py +0 -0
  29. agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
  30. agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
  31. agent_wire/channels/tier2_pro/devto.py +114 -0
  32. agent_wire/channels/tier2_pro/fediverse.py +181 -0
  33. agent_wire/channels/tier2_pro/hubs.py +161 -0
  34. agent_wire/channels/tier2_pro/linkedin.py +146 -0
  35. agent_wire/channels/tier2_pro/markets.py +207 -0
  36. agent_wire/channels/tier2_pro/meta_social.py +104 -0
  37. agent_wire/channels/tier2_pro/news.py +173 -0
  38. agent_wire/channels/tier2_pro/polymarket.py +105 -0
  39. agent_wire/channels/tier2_pro/publishing.py +84 -0
  40. agent_wire/channels/tier2_pro/stocktwits.py +79 -0
  41. agent_wire/channels/tier3_regional/bilibili.py +171 -0
  42. agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
  43. agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
  44. agent_wire/channels/tier3_regional/v2ex.py +70 -0
  45. agent_wire/channels/tier3_regional/wechat.py +67 -0
  46. agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
  47. agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
  48. agent_wire/channels/tier3_regional/xueqiu.py +77 -0
  49. agent_wire/channels/tier3_regional/zhihu.py +57 -0
  50. agent_wire/cli.py +403 -0
  51. agent_wire/compression/__init__.py +0 -0
  52. agent_wire/compression/anchor_slicer.py +45 -0
  53. agent_wire/compression/bm25_slicer.py +151 -0
  54. agent_wire/compression/dam_formatter.py +74 -0
  55. agent_wire/compression/url_compactor.py +45 -0
  56. agent_wire/config.py +52 -0
  57. agent_wire/engine/__init__.py +0 -0
  58. agent_wire/engine/challenge.py +58 -0
  59. agent_wire/engine/http_pool.py +136 -0
  60. agent_wire/engine/stealth_browser.py +90 -0
  61. agent_wire/engine/waterfall.py +159 -0
  62. agent_wire/firewall/__init__.py +0 -0
  63. agent_wire/firewall/inbound_shield.py +116 -0
  64. agent_wire/firewall/outbound_privacy.py +120 -0
  65. agent_wire/installers/__init__.py +0 -0
  66. agent_wire/installers/client_installer.py +315 -0
  67. agent_wire/media/__init__.py +0 -0
  68. agent_wire/media/audio_transcriber.py +77 -0
  69. agent_wire/media/subtitle_cleaner.py +102 -0
  70. agent_wire/models.py +60 -0
  71. agent_wire/research/__init__.py +1 -0
  72. agent_wire/research/brief.py +227 -0
  73. agent_wire/research/briefing.py +41 -0
  74. agent_wire/research/cluster.py +128 -0
  75. agent_wire/research/engine.py +632 -0
  76. agent_wire/research/library.py +74 -0
  77. agent_wire/research/planner.py +232 -0
  78. agent_wire/research/scoring.py +89 -0
  79. agent_wire/research/store.py +149 -0
  80. agent_wire/research/watch.py +70 -0
  81. agent_wire/router.py +107 -0
  82. agent_wire/server.py +468 -0
  83. agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
  84. agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
  85. agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
  86. agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
  87. agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,94 @@
1
+ """StackOverflow / StackExchange: question, accepted answer and top-voted answers via API 2.3."""
2
+
3
+ import html
4
+ import re
5
+ from urllib.parse import quote, urlsplit
6
+
7
+ from agent_wire.channels.base import Channel, html_to_markdown, ts
8
+ from agent_wire.models import WireComment, WireSearchHit
9
+
10
+ API = "https://api.stackexchange.com/2.3"
11
+ TOP_ANSWERS = 3
12
+ _ROOT_SITES = {"stackoverflow.com": "stackoverflow", "superuser.com": "superuser",
13
+ "serverfault.com": "serverfault", "askubuntu.com": "askubuntu",
14
+ "mathoverflow.net": "mathoverflow.net"} # fmt: skip
15
+
16
+
17
+ def site_for(host: str) -> str:
18
+ host = host.removeprefix("www.")
19
+ if host in _ROOT_SITES:
20
+ return _ROOT_SITES[host]
21
+ return host.removesuffix(".stackexchange.com")
22
+
23
+
24
+ def pick_answers(answers: list[dict]) -> list[dict]:
25
+ """Accepted answer plus the top-voted others, in vote order."""
26
+ ranked = sorted(answers, key=lambda a: -(a.get("score") or 0))
27
+ accepted = [a for a in ranked if a.get("is_accepted")]
28
+ others = [a for a in ranked if not a.get("is_accepted")][:TOP_ANSWERS]
29
+ return accepted + others
30
+
31
+
32
+ def _owner(item: dict) -> str | None:
33
+ return html.unescape((item.get("owner") or {}).get("display_name") or "") or None
34
+
35
+
36
+ class StackOverflowChannel(Channel):
37
+ name = "stackoverflow"
38
+
39
+ async def read(self, url, *, query=None, section=None, include_comments=True):
40
+ parts = urlsplit(url)
41
+ m = re.search(r"/(?:questions|q)/(\d+)", parts.path)
42
+ if not m:
43
+ return await self.ctx.channel("web").read(url, query=query, section=section)
44
+ qid, site = m[1], site_for(parts.hostname or "stackoverflow.com")
45
+ q_data, f = await self.fetch_json(f"{API}/questions/{qid}?site={site}&filter=withbody")
46
+ if not q_data.get("items"):
47
+ return await self.ctx.channel("web").read(url, query=query, section=section)
48
+ q = q_data["items"][0]
49
+ comments = []
50
+ if include_comments:
51
+ a_data, _ = await self.fetch_json(
52
+ f"{API}/questions/{qid}/answers?site={site}&filter=withbody&sort=votes&pagesize=20"
53
+ )
54
+ comments = [
55
+ WireComment(
56
+ id=str(a["answer_id"]), body=html_to_markdown(a.get("body") or ""),
57
+ author_handle=_owner(a), score=a.get("score") or 0,
58
+ is_accepted=bool(a.get("is_accepted")), timestamp=ts(a.get("creation_date")),
59
+ )
60
+ for a in pick_answers(a_data.get("items", []))
61
+ ] # fmt: skip
62
+ body = html_to_markdown(q.get("body") or "")
63
+ return self.doc(
64
+ f, url, title=html.unescape(q.get("title", "")), author_handle=_owner(q),
65
+ timestamp=ts(q.get("creation_date")), score=q.get("score"), body=body,
66
+ comments=comments,
67
+ metadata={"tags": ", ".join(q.get("tags") or []), "answers": q.get("answer_count"),
68
+ "views": q.get("view_count"), "answered": q.get("is_answered")},
69
+ ) # fmt: skip
70
+
71
+ async def search(self, query, *, limit=5, sort="relevance", since=None):
72
+ order = {"top": "votes", "new": "creation"}.get(sort, "relevance")
73
+ window = f"&fromdate={int(since.timestamp())}" if since else ""
74
+ data, _ = await self.fetch_json(
75
+ f"{API}/search/advanced?q={quote(query)}&site=stackoverflow&sort={order}&order=desc"
76
+ f"&pagesize={limit}{window}"
77
+ )
78
+ return [
79
+ WireSearchHit(
80
+ url=it["link"], title=html.unescape(it.get("title", "")),
81
+ snippet=f"score {it.get('score', 0)} · {it.get('answer_count', 0)} answers"
82
+ + (" · accepted" if it.get("accepted_answer_id") else "")
83
+ + f" · {', '.join(it.get('tags', [])[:5])}",
84
+ source="stackoverflow", rank_score=float(it.get("score") or 0),
85
+ timestamp=ts(it.get("creation_date")),
86
+ author=(it.get("owner") or {}).get("display_name"),
87
+ engagement={"score": it.get("score") or 0, "comments": it.get("answer_count") or 0,
88
+ "views": it.get("view_count") or 0},
89
+ )
90
+ for it in data.get("items", [])[:limit]
91
+ ] # fmt: skip
92
+
93
+
94
+ CHANNEL = StackOverflowChannel
@@ -0,0 +1,98 @@
1
+ """Generic web reader: waterfall fetch -> inbound shield -> trafilatura Markdown -> section slice."""
2
+
3
+ import re
4
+ from datetime import UTC, datetime, timedelta
5
+ from urllib.parse import urlsplit
6
+
7
+ import trafilatura
8
+
9
+ from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown, ts
10
+ from agent_wire.compression.anchor_slicer import parse_line_anchor, slice_lines, slice_section
11
+ from agent_wire.firewall.inbound_shield import clean_html
12
+ from agent_wire.models import WireDocument
13
+
14
+ _TEXT_TYPES = ("text/plain", "text/markdown", "text/x-", "application/json", "application/x-")
15
+ _FEED_TYPES = ("application/rss+xml", "application/atom+xml", "application/feed+json")
16
+
17
+
18
+ def is_feed(content_type: str, content: bytes) -> bool:
19
+ if content_type in _FEED_TYPES:
20
+ return True
21
+ head = content[:600].lstrip().lower()
22
+ return content_type in ("text/xml", "application/xml", "") and (
23
+ b"<rss" in head or b"<feed" in head
24
+ )
25
+
26
+
27
+ _DATE_HINT = re.compile(
28
+ r"article:published_time|article:modified_time|datePublished|dateModified|<time[\s>]"
29
+ r"|name=[\"'](?:date|pubdate|publishdate|dc\.date)",
30
+ re.IGNORECASE,
31
+ )
32
+
33
+
34
+ def _real_date(date: str | None, html: str) -> str | None:
35
+ """Drop trafilatura's fallback date: with no date markup it reports the crawl day (today)."""
36
+ if not date:
37
+ return None
38
+ today = datetime.now(UTC).date()
39
+ if date[:10] in {(today + timedelta(days=d)).isoformat() for d in (-1, 0, 1)}:
40
+ return date if _DATE_HINT.search(html) else None
41
+ return date
42
+
43
+
44
+ def extract_article(html: str) -> tuple[str, dict]:
45
+ cleaned = clean_html(html)
46
+ body = trafilatura.extract(
47
+ cleaned,
48
+ output_format="markdown",
49
+ include_tables=True,
50
+ include_formatting=True,
51
+ include_comments=False,
52
+ include_links=False,
53
+ include_images=False,
54
+ favor_recall=True,
55
+ )
56
+ meta = trafilatura.extract_metadata(cleaned)
57
+ info = {
58
+ "title": getattr(meta, "title", None),
59
+ "author": getattr(meta, "author", None),
60
+ "date": _real_date(getattr(meta, "date", None), html),
61
+ "site": getattr(meta, "sitename", None),
62
+ }
63
+ body = body or html_to_markdown(cleaned, links=False)
64
+ return body.replace("¶", ""), info # drop docs permalink glyphs
65
+
66
+
67
+ class WebChannel(Channel):
68
+ name = "web"
69
+
70
+ async def read(self, url, *, query=None, section=None, include_comments=True) -> WireDocument:
71
+ f = await self.fetch(url)
72
+ ctype = f.content_type
73
+ if is_feed(ctype, f.content):
74
+ return self.ctx.channel("rss").document(f, url)
75
+ anchor = section or urlsplit(url).fragment or None
76
+ if ctype.startswith(_TEXT_TYPES) or (not ctype and not f.content.lstrip().startswith(b"<")):
77
+ body = f.text
78
+ if lines := parse_line_anchor(url):
79
+ body = slice_lines(body, *lines)
80
+ elif anchor:
81
+ body = slice_section(body, anchor) or body
82
+ return self.doc(f, url, title=urlsplit(url).path.rsplit("/", 1)[-1] or url, body=body)
83
+ if "html" not in ctype and ctype:
84
+ raise WireUnsupported(f"{url} is {ctype}, not a readable page")
85
+
86
+ body, info = extract_article(f.text)
87
+ if anchor:
88
+ body = slice_section(body, anchor) or body
89
+ meta = {"site": info["site"]} if info["site"] else {}
90
+ if f.stage == 3 and "web.archive.org" in f.url:
91
+ meta["archived"] = f.url
92
+ return self.doc(
93
+ f, url, title=info["title"], author_handle=info["author"],
94
+ timestamp=ts(info["date"]), body=body, metadata=meta,
95
+ ) # fmt: skip
96
+
97
+
98
+ CHANNEL = WebChannel
@@ -0,0 +1,91 @@
1
+ """Wikipedia: clean plain-text extract (no [1] citations, no edit links) with section slicing."""
2
+
3
+ import re
4
+ from urllib.parse import quote, unquote, urlsplit
5
+
6
+ from agent_wire.channels.base import Channel, WireUnsupported, html_text
7
+ from agent_wire.compression.anchor_slicer import slice_section
8
+ from agent_wire.models import WireSearchHit
9
+
10
+ _HEADING = re.compile(r"^(={2,6})[ \t]*(.*?)[ \t]*={2,6}[ \t]*$", re.MULTILINE)
11
+
12
+
13
+ def extract_to_markdown(extract: str) -> tuple[str, list[str]]:
14
+ headings: list[str] = []
15
+
16
+ def repl(m):
17
+ level = len(m[1])
18
+ if level == 2:
19
+ headings.append(m[2])
20
+ return "#" * level + " " + m[2]
21
+
22
+ lines = _HEADING.sub(repl, extract).split("\n")
23
+ kept = [ln for i, ln in enumerate(lines) if not _empty_section(lines, i)]
24
+ md = re.sub(r"\n{3,}", "\n\n", "\n".join(kept))
25
+ return md.strip(), [h for h in headings if f"## {h}" in md]
26
+
27
+
28
+ def _empty_section(lines: list[str], i: int) -> bool:
29
+ """A heading followed only by blank lines and then a same-or-higher heading (or the end).
30
+
31
+ MediaWiki leaves these for link-only sections like "See also"; a heading whose content is
32
+ all in subsections is kept.
33
+ """
34
+ m = re.match(r"^(#+) ", lines[i])
35
+ if not m:
36
+ return False
37
+ j = i + 1
38
+ while j < len(lines) and not lines[j].strip():
39
+ j += 1
40
+ if j == len(lines):
41
+ return True
42
+ nxt = re.match(r"^(#+) ", lines[j])
43
+ return bool(nxt and len(nxt[1]) <= len(m[1]))
44
+
45
+
46
+ def parse_wiki_url(url: str) -> tuple[str, str]:
47
+ parts = urlsplit(url)
48
+ host = (parts.hostname or "en.wikipedia.org").replace(".m.wikipedia.org", ".wikipedia.org")
49
+ if parts.path.startswith("/wiki/"):
50
+ title = unquote(parts.path[len("/wiki/") :])
51
+ elif "title=" in parts.query:
52
+ title = unquote(re.search(r"title=([^&]+)", parts.query)[1])
53
+ else:
54
+ raise WireUnsupported("wikipedia: expected /wiki/<Title>")
55
+ return host, title.replace("_", " ")
56
+
57
+
58
+ class WikipediaChannel(Channel):
59
+ name = "wikipedia"
60
+
61
+ async def read(self, url, *, query=None, section=None, include_comments=True):
62
+ host, title = parse_wiki_url(url)
63
+ api = (f"https://{host}/w/api.php?action=query&prop=extracts|info&explaintext=1"
64
+ f"&inprop=url&redirects=1&format=json&formatversion=2&titles={quote(title)}") # fmt: skip
65
+ data, f = await self.fetch_json(api)
66
+ pages = data.get("query", {}).get("pages", [])
67
+ if not pages or pages[0].get("missing"):
68
+ raise WireUnsupported(f"no Wikipedia article {title!r} on {host}")
69
+ page = pages[0]
70
+ body, headings = extract_to_markdown(page.get("extract", ""))
71
+ anchor = section or urlsplit(url).fragment
72
+ if anchor and (part := slice_section(body, unquote(anchor).replace("_", " "))):
73
+ body = part
74
+ elif headings:
75
+ body = f"Contents: {' · '.join(headings)}\n\n{body}"
76
+ return self.doc(f, page.get("fullurl") or url, title=page.get("title"), body=body,
77
+ metadata={"lang": host.split(".")[0]}) # fmt: skip
78
+
79
+ async def search(self, query, *, limit=5, sort="relevance"):
80
+ data, _ = await self.fetch_json(
81
+ "https://en.wikipedia.org/w/api.php?action=query&list=search&format=json"
82
+ f"&formatversion=2&srlimit={limit}&srsearch={quote(query)}"
83
+ )
84
+ return [
85
+ WireSearchHit(url=f"https://en.wikipedia.org/wiki/{quote(r['title'].replace(' ', '_'))}",
86
+ title=r["title"], snippet=html_text(r.get("snippet", "")), source="wikipedia")
87
+ for r in data.get("query", {}).get("search", [])
88
+ ] # fmt: skip
89
+
90
+
91
+ CHANNEL = WikipediaChannel
@@ -0,0 +1,380 @@
1
+ """X / Twitter: zero-cookie single posts (fxtwitter, vxtwitter, syndication); GraphQL for the rest."""
2
+
3
+ import math
4
+ import re
5
+ from urllib.parse import quote, unquote, urlsplit
6
+
7
+ import orjson
8
+ from selectolax.lexbor import LexborHTMLParser
9
+
10
+ from agent_wire.channels.base import Channel, WireUnsupported, listing, ts
11
+ from agent_wire.engine.waterfall import WireFetchError
12
+ from agent_wire.firewall.outbound_privacy import parse_json
13
+ from agent_wire.models import WireComment, WireSearchHit
14
+
15
+ # The public web-app bearer token embedded in X's own JavaScript (not a user secret).
16
+ BEARER = (
17
+ "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D"
18
+ "1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
19
+ )
20
+ FEATURES = {
21
+ "responsive_web_graphql_exclude_directive_enabled": True,
22
+ "verified_phone_label_enabled": False,
23
+ "creator_subscriptions_tweet_preview_api_enabled": True,
24
+ "responsive_web_graphql_timeline_navigation_enabled": True,
25
+ "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
26
+ "longform_notetweets_consumption_enabled": True,
27
+ "longform_notetweets_rich_text_read_enabled": True,
28
+ "tweet_awards_web_tipping_enabled": False,
29
+ "freedom_of_speech_not_reach_fetch_enabled": True,
30
+ "standardized_nudges_misinfo": True,
31
+ "view_counts_everywhere_api_enabled": True,
32
+ "responsive_web_enhance_cards_enabled": False,
33
+ }
34
+ _STATUS = re.compile(r"/(?:i/web/)?(?:(\w+)/)?status(?:es)?/(\d+)")
35
+ _MISSING_FEATURES = re.compile(r"features cannot be null: (.+)", re.IGNORECASE)
36
+ _MAIN_JS = re.compile(
37
+ r"https://abs\.twimg\.com/responsive-web/client-web(?:-legacy)?/main\.[\w]+\.js"
38
+ )
39
+ _QUERY_ID = re.compile(r'queryId:"([\w-]+)",operationName:"(\w+)"')
40
+
41
+
42
+ def syndication_token(tweet_id: str) -> str:
43
+ """Python port of X embed's `((id / 1e15) * Math.PI).toString(36).replace(/(0+|\\.)/g, '')`."""
44
+ x = int(tweet_id) / 1e15 * math.pi
45
+ digits = "0123456789abcdefghijklmnopqrstuvwxyz"
46
+ whole, frac = int(x), x - int(x)
47
+ head = ""
48
+ while whole:
49
+ head, whole = digits[whole % 36] + head, whole // 36
50
+ tail = ""
51
+ for _ in range(10):
52
+ frac *= 36
53
+ tail += digits[int(frac)]
54
+ frac -= int(frac)
55
+ return re.sub(r"(0+|\.)", "", f"{head or '0'}.{tail}")
56
+
57
+
58
+ def normalize(data: dict, source: str) -> dict:
59
+ """fxtwitter / vxtwitter / syndication payload -> one tweet shape."""
60
+ if source == "fxtwitter":
61
+ t = data.get("tweet") or {}
62
+ quote_t = t.get("quote")
63
+ return {
64
+ "id": t.get("id"), "text": t.get("text", ""),
65
+ "author": (t.get("author") or {}).get("screen_name"),
66
+ "likes": t.get("likes"), "retweets": t.get("retweets"), "replies": t.get("replies"),
67
+ "views": t.get("views"), "created": ts(t.get("created_timestamp")),
68
+ "media": [m.get("altText") or m.get("type") for m in (t.get("media") or {}).get("all", [])],
69
+ "quote": normalize({"tweet": quote_t}, "fxtwitter") if quote_t else None,
70
+ } # fmt: skip
71
+ if source == "vxtwitter":
72
+ return {
73
+ "id": data.get("tweetID"), "text": data.get("text", ""),
74
+ "author": data.get("user_screen_name"), "likes": data.get("likes"),
75
+ "retweets": data.get("retweets"), "replies": data.get("replies"), "views": None,
76
+ "created": ts(data.get("date_epoch")),
77
+ "media": [m.get("altText") or m.get("type") for m in data.get("media_extended") or []],
78
+ "quote": normalize(data["qrt"], "vxtwitter") if data.get("qrt") else None,
79
+ } # fmt: skip
80
+ return { # syndication
81
+ "id": data.get("id_str"), "text": data.get("text", ""),
82
+ "author": (data.get("user") or {}).get("screen_name"),
83
+ "likes": data.get("favorite_count"), "retweets": data.get("retweet_count"),
84
+ "replies": data.get("conversation_count"), "views": None,
85
+ "created": ts(data.get("created_at")),
86
+ "media": [m.get("ext_alt_text") or m.get("type") for m in data.get("mediaDetails") or []],
87
+ "quote": normalize(data["quoted_tweet"], "syndication") if data.get("quoted_tweet") else None,
88
+ } # fmt: skip
89
+
90
+
91
+ def render_tweet(t: dict) -> str:
92
+ parts = [t["text"]]
93
+ if media := [m for m in t["media"] if m]:
94
+ parts.append("Media: " + "; ".join(media))
95
+ if q := t.get("quote"):
96
+ parts.append(f"> Quoting @{q['author']}: " + q["text"].replace("\n", "\n> "))
97
+ return "\n\n".join(parts)
98
+
99
+
100
+ def graphql_tweets(data) -> list[dict]:
101
+ """Every tweet in a GraphQL timeline, in order, deduped."""
102
+ out, seen = [], set()
103
+
104
+ def walk(o):
105
+ if isinstance(o, dict):
106
+ res = o.get("tweet_results", {}).get("result") if "tweet_results" in o else None
107
+ if res:
108
+ res = res.get("tweet", res) # TweetWithVisibilityResults wrapper
109
+ legacy = res.get("legacy") or {}
110
+ if legacy.get("full_text") is not None and res.get("rest_id") not in seen:
111
+ seen.add(res.get("rest_id"))
112
+ user = ((res.get("core") or {}).get("user_results") or {}).get("result") or {}
113
+ name = (user.get("core") or {}).get("screen_name") or (
114
+ user.get("legacy") or {}
115
+ ).get("screen_name")
116
+ note = ((res.get("note_tweet") or {}).get("note_tweet_results") or {}).get(
117
+ "result"
118
+ ) or {}
119
+ out.append({
120
+ "id": res.get("rest_id"), "author": name,
121
+ "text": note.get("text") or legacy["full_text"],
122
+ "likes": legacy.get("favorite_count"), "retweets": legacy.get("retweet_count"),
123
+ "replies": legacy.get("reply_count"),
124
+ "views": (res.get("views") or {}).get("count"),
125
+ "created": ts(_twitter_date(legacy.get("created_at"))),
126
+ "reply_to": legacy.get("in_reply_to_status_id_str"),
127
+ }) # fmt: skip
128
+ for v in o.values():
129
+ walk(v)
130
+ elif isinstance(o, list):
131
+ for v in o:
132
+ walk(v)
133
+
134
+ walk(data)
135
+ return out
136
+
137
+
138
+ def _twitter_date(value):
139
+ from email.utils import parsedate_to_datetime
140
+
141
+ if not value:
142
+ return None
143
+ try:
144
+ return parsedate_to_datetime(value).isoformat()
145
+ except (TypeError, ValueError):
146
+ return value
147
+
148
+
149
+ def tweet_items(tweets: list[dict]) -> list[dict]:
150
+ return [{"title": f"@{t['author']}: {t['text'][:120]}",
151
+ "url": f"https://x.com/{t['author']}/status/{t['id']}",
152
+ "meta": [f"♥{t.get('likes') or 0}", f"↻{t.get('retweets') or 0}",
153
+ f"💬{t.get('replies') or 0}",
154
+ f"{t['views']} views" if t.get("views") else None,
155
+ t["created"].date().isoformat() if t.get("created") else None],
156
+ "snippet": t["text"] if len(t["text"]) > 120 else ""} for t in tweets] # fmt: skip
157
+
158
+
159
+ def parse_syndication_timeline(html: str) -> list[dict]:
160
+ node = LexborHTMLParser(html).css_first("script#__NEXT_DATA__")
161
+ if not node:
162
+ return []
163
+ data = orjson.loads(node.text())
164
+ entries = data.get("props", {}).get("pageProps", {}).get("timeline", {}).get("entries", [])
165
+ out = []
166
+ for e in entries:
167
+ t = (e.get("content") or {}).get("tweet")
168
+ if t:
169
+ out.append({
170
+ "id": t.get("id_str"), "author": (t.get("user") or {}).get("screen_name"),
171
+ "text": t.get("full_text") or t.get("text", ""), "likes": t.get("favorite_count"),
172
+ "retweets": t.get("retweet_count"), "replies": t.get("reply_count"), "views": None,
173
+ "created": ts(_twitter_date(t.get("created_at"))),
174
+ }) # fmt: skip
175
+ return out
176
+
177
+
178
+ class XChannel(Channel):
179
+ name = "x"
180
+
181
+ def __init__(self, ctx):
182
+ super().__init__(ctx)
183
+ self._query_ids: dict[str, str] = {}
184
+ self._guest: str | None = None
185
+
186
+ # ---- single posts (no cookies) ---------------------------------------------------------
187
+ async def read(self, url, *, query=None, section=None, include_comments=True):
188
+ m = _STATUS.search(urlsplit(url).path)
189
+ if not m:
190
+ handle = urlsplit(url).path.strip("/").split("/")[0]
191
+ if handle:
192
+ return await self.feed(f"@{handle}")
193
+ raise WireUnsupported("x: expected a /<user>/status/<id> or profile URL")
194
+ user, tid = m[1] or "i", m[2]
195
+ primary = f"https://api.fxtwitter.com/{user}/status/{tid}"
196
+ mirrors = [f"https://api.vxtwitter.com/{user}/status/{tid}",
197
+ f"https://cdn.syndication.twimg.com/tweet-result?id={tid}&token={syndication_token(tid)}"] # fmt: skip
198
+ data, f = await self.fetch_json(primary, mirrors=mirrors)
199
+ host = urlsplit(f.requested or primary).hostname or ""
200
+ source = (
201
+ "fxtwitter"
202
+ if "fxtwitter" in host
203
+ else "vxtwitter"
204
+ if "vxtwitter" in host
205
+ else "syndication"
206
+ )
207
+ t = normalize(data, source)
208
+ if not t["text"] and not t["author"]:
209
+ raise WireFetchError(f"{url}: post unavailable (deleted, private or age-gated)")
210
+ replies = await self._replies(tid, t["author"]) if include_comments else []
211
+ return self.doc(
212
+ f, f"https://x.com/{t['author'] or user}/status/{tid}",
213
+ title=f"@{t['author']} on X", author_handle=t["author"], timestamp=t["created"],
214
+ score=t["likes"], body=render_tweet(t), comments=replies,
215
+ metadata={"likes": t["likes"], "retweets": t["retweets"], "replies": t["replies"],
216
+ "views": t["views"], "via": source},
217
+ ) # fmt: skip
218
+
219
+ async def _replies(self, tid: str, author: str | None) -> list[WireComment]:
220
+ try:
221
+ data = await self.graphql("TweetDetail", {
222
+ "focalTweetId": tid, "with_rux_injections": False, "includePromotedContent": False,
223
+ "withCommunity": True, "withVoice": True, "withV2Timeline": True,
224
+ "withBirdwatchNotes": False, "withQuickPromoteEligibilityTweetFields": False,
225
+ }, cookies=self._vault_only()) # fmt: skip
226
+ except (WireFetchError, WireUnsupported, ValueError, KeyError):
227
+ return [] # replies need a guest token X may refuse; the post itself still reads
228
+ return [
229
+ WireComment(id=t["id"], body=t["text"], author_handle=t["author"],
230
+ score=t.get("likes") or 0, timestamp=t["created"], is_op=t["author"] == author)
231
+ for t in graphql_tweets(data) if t["id"] != tid and t.get("reply_to") == tid
232
+ ] # fmt: skip
233
+
234
+ def _vault_only(self) -> dict | None:
235
+ """Saved `wire auth login x.com` session, if any (the only cookies agent-wire ever uses)."""
236
+ vault = self.ctx.vault
237
+ jar = vault.get("x.com") if vault else {}
238
+ return jar if jar.get("auth_token") and jar.get("ct0") else None
239
+
240
+ # ---- GraphQL -----------------------------------------------------------------------------
241
+ async def _ids(self) -> dict[str, str]:
242
+ if not self._query_ids:
243
+ home = await self.fetch("https://x.com/", archive=False, browser=False)
244
+ m = _MAIN_JS.search(home.text)
245
+ if not m:
246
+ raise WireUnsupported("x: couldn't locate the web client bundle for GraphQL ids")
247
+ js = await self.fetch(m[0], archive=False, browser=False)
248
+ self._query_ids = {op: qid for qid, op in _QUERY_ID.findall(js.text)}
249
+ return self._query_ids
250
+
251
+ async def _guest_token(self) -> str:
252
+ if not self._guest:
253
+ f = await self.ctx.pool.get(
254
+ "https://api.x.com/1.1/guest/activate.json", method="POST", impersonate="chrome",
255
+ headers={"authorization": f"Bearer {BEARER}"},
256
+ ) # fmt: skip
257
+ self._guest = parse_json(f.content).get("guest_token")
258
+ if not self._guest:
259
+ raise WireUnsupported("x refused a guest token")
260
+ return self._guest
261
+
262
+ async def graphql(self, op: str, variables: dict, cookies: dict | None = None) -> dict:
263
+ qid = (await self._ids()).get(op)
264
+ if not qid:
265
+ raise WireUnsupported(f"x GraphQL operation {op} not found in the web bundle")
266
+ headers = {"authorization": f"Bearer {BEARER}", "x-twitter-active-user": "yes",
267
+ "content-type": "application/json"} # fmt: skip
268
+ if cookies:
269
+ headers.update({"x-csrf-token": cookies["ct0"], "x-twitter-auth-type": "OAuth2Session"})
270
+ if self.ctx.waterfall.limiter:
271
+ await self.ctx.waterfall.limiter.wait("x.com")
272
+ else:
273
+ headers["x-guest-token"] = await self._guest_token()
274
+ features = dict(FEATURES)
275
+ for _ in range(3): # X lists any newly-required feature flags in a 400; add them and retry
276
+ url = (f"https://x.com/i/api/graphql/{qid}/{op}?variables={quote(orjson.dumps(variables).decode())}"
277
+ f"&features={quote(orjson.dumps(features).decode())}") # fmt: skip
278
+ f = await self.ctx.pool.get(url, headers=headers, cookies=cookies, impersonate="chrome")
279
+ data = parse_json(f.content) if f.content.strip().startswith(b"{") else {}
280
+ missing = [
281
+ m
282
+ for e in data.get("errors", [])
283
+ if (m := _MISSING_FEATURES.search(e.get("message", "")))
284
+ ]
285
+ if f.status == 400 and missing:
286
+ for name in re.split(r",\s*", missing[0][1]):
287
+ features[name.strip()] = True
288
+ continue
289
+ if not f.ok:
290
+ raise WireFetchError(f"x GraphQL {op}: HTTP {f.status}")
291
+ return data
292
+ raise WireFetchError(f"x GraphQL {op}: feature negotiation failed")
293
+
294
+ async def _session(self, purpose: str) -> dict:
295
+ jar = await self.ctx.vault.cookies_for("x.com") if self.ctx.vault else {}
296
+ if not (jar.get("auth_token") and jar.get("ct0")):
297
+ raise WireUnsupported(
298
+ f"{purpose} needs a logged-in X session: run `wire auth login x.com`"
299
+ )
300
+ return jar
301
+
302
+ # ---- search / feed / me ----------------------------------------------------------------
303
+ async def search(self, query, *, limit=5, sort="relevance"):
304
+ jar = await self._session("X search")
305
+ data = await self.graphql("SearchTimeline", {
306
+ "rawQuery": query, "count": max(limit, 10), "querySource": "typed_query",
307
+ "product": "Latest" if sort in ("new", "latest") else "Top",
308
+ }, cookies=jar) # fmt: skip
309
+ return [
310
+ WireSearchHit(url=f"https://x.com/{t['author']}/status/{t['id']}", title=f"@{t['author']}",
311
+ snippet=t["text"], source="x", rank_score=float(t.get("likes") or 0),
312
+ timestamp=t.get("created"), author=t["author"],
313
+ engagement={"likes": t.get("likes") or 0, "reposts": t.get("retweets") or 0,
314
+ "comments": t.get("replies") or 0,
315
+ "views": int(t.get("views") or 0)})
316
+ for t in graphql_tweets(data)[:limit]
317
+ ] # fmt: skip
318
+
319
+ async def feed(self, target, *, sort="hot", limit=10):
320
+ handle = target.strip().lstrip("@").split("/")[-1]
321
+ url = f"https://syndication.twitter.com/srv/timeline-profile/screen-name/{handle}"
322
+ try:
323
+ f = await self.fetch(url, archive=False, browser=False)
324
+ tweets = parse_syndication_timeline(f.text)
325
+ except WireFetchError:
326
+ f, tweets = None, []
327
+ if not tweets: # syndication is often empty now; fall back to GraphQL with a session
328
+ jar = await self._session(f"@{handle}'s timeline")
329
+ user = await self.graphql("UserByScreenName", {"screen_name": handle}, cookies=jar)
330
+ uid = user.get("data", {}).get("user", {}).get("result", {}).get("rest_id")
331
+ data = await self.graphql("UserTweets", {"userId": uid, "count": limit,
332
+ "includePromotedContent": False, "withVoice": True,
333
+ "withV2Timeline": True}, cookies=jar) # fmt: skip
334
+ tweets = graphql_tweets(data)
335
+ return self.doc(f, f"https://x.com/{handle}", title=f"@{handle} on X",
336
+ body=listing(tweet_items(tweets[:limit]))) # fmt: skip
337
+
338
+ async def me(self, target="latest_posts", *, limit=5):
339
+ jar = await self._session("wire_me x")
340
+ uid = unquote(jar.get("twid", "")).removeprefix("u=")
341
+ if not uid.isdigit():
342
+ raise WireUnsupported("x session has no twid cookie; re-run `wire auth login x.com`")
343
+ common = {"includePromotedContent": False, "withVoice": True, "withV2Timeline": True}
344
+ if target == "notifications":
345
+ if self.ctx.waterfall.limiter:
346
+ await self.ctx.waterfall.limiter.wait("x.com")
347
+ f = await self.ctx.pool.get(
348
+ f"https://x.com/i/api/2/notifications/all.json?count={limit * 4}", cookies=jar,
349
+ headers={"authorization": f"Bearer {BEARER}", "x-csrf-token": jar["ct0"],
350
+ "x-twitter-auth-type": "OAuth2Session"},
351
+ ) # fmt: skip
352
+ notes = parse_json(f.content).get("globalObjects", {}).get("notifications", {})
353
+ items = [{"title": n.get("message", {}).get("text", ""), "url": "https://x.com/notifications",
354
+ "meta": [ts(n.get("timestampMs"))]} for n in list(notes.values())[:limit]] # fmt: skip
355
+ return self.doc(
356
+ f, "https://x.com/notifications", title="my X notifications", body=listing(items)
357
+ )
358
+ ops = {"latest_posts": ("UserTweets", {"userId": uid, "count": limit, **common}),
359
+ "likes": ("Likes", {"userId": uid, "count": limit, **common}),
360
+ "bookmarks": ("Bookmarks", {"count": limit, "includePromotedContent": False}),
361
+ "stats": ("UserByRestId", {"userId": uid})} # fmt: skip
362
+ if target not in ops:
363
+ raise WireUnsupported(f"x targets: {', '.join([*ops, 'notifications'])}")
364
+ op, variables = ops[target]
365
+ data = await self.graphql(op, variables, cookies=jar)
366
+ if target == "stats":
367
+ legacy = data.get("data", {}).get("user", {}).get("result", {}).get("legacy", {})
368
+ body = (f"{legacy.get('followers_count')} followers · {legacy.get('friends_count')} following"
369
+ f" · {legacy.get('statuses_count')} posts") # fmt: skip
370
+ else:
371
+ tweets = graphql_tweets(data)
372
+ if target == "latest_posts":
373
+ tweets = [t for t in tweets if not t.get("reply_to")] or tweets
374
+ body = listing(tweet_items(tweets[:limit]))
375
+ return self.doc(
376
+ None, "https://x.com/home", title=f"my X {target}", body=body, auth="cookie"
377
+ )
378
+
379
+
380
+ CHANNEL = XChannel