agent-wire-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. agent_wire/__init__.py +1 -0
  2. agent_wire/__main__.py +3 -0
  3. agent_wire/assets/SKILL.md +26 -0
  4. agent_wire/assets/cursor_rule.mdc +13 -0
  5. agent_wire/assets/hermes_skill.md +28 -0
  6. agent_wire/assets/windsurf_rules.md +14 -0
  7. agent_wire/assets/wire-worker.md +20 -0
  8. agent_wire/auth/__init__.py +0 -0
  9. agent_wire/auth/browser_paths.py +143 -0
  10. agent_wire/auth/cookie_vault.py +181 -0
  11. agent_wire/auth/rate_limiter.py +26 -0
  12. agent_wire/cache/__init__.py +0 -0
  13. agent_wire/cache/ring_buffer.py +94 -0
  14. agent_wire/cache/sqlite_ledger.py +132 -0
  15. agent_wire/channels/__init__.py +41 -0
  16. agent_wire/channels/base.py +249 -0
  17. agent_wire/channels/tier1_global/__init__.py +0 -0
  18. agent_wire/channels/tier1_global/github.py +371 -0
  19. agent_wire/channels/tier1_global/hackernews.py +135 -0
  20. agent_wire/channels/tier1_global/reddit.py +371 -0
  21. agent_wire/channels/tier1_global/rss.py +129 -0
  22. agent_wire/channels/tier1_global/search.py +160 -0
  23. agent_wire/channels/tier1_global/stackoverflow.py +94 -0
  24. agent_wire/channels/tier1_global/web.py +98 -0
  25. agent_wire/channels/tier1_global/wikipedia.py +91 -0
  26. agent_wire/channels/tier1_global/x_twitter.py +380 -0
  27. agent_wire/channels/tier1_global/youtube.py +223 -0
  28. agent_wire/channels/tier2_pro/__init__.py +0 -0
  29. agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
  30. agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
  31. agent_wire/channels/tier2_pro/devto.py +114 -0
  32. agent_wire/channels/tier2_pro/fediverse.py +181 -0
  33. agent_wire/channels/tier2_pro/hubs.py +161 -0
  34. agent_wire/channels/tier2_pro/linkedin.py +146 -0
  35. agent_wire/channels/tier2_pro/markets.py +207 -0
  36. agent_wire/channels/tier2_pro/meta_social.py +104 -0
  37. agent_wire/channels/tier2_pro/news.py +173 -0
  38. agent_wire/channels/tier2_pro/polymarket.py +105 -0
  39. agent_wire/channels/tier2_pro/publishing.py +84 -0
  40. agent_wire/channels/tier2_pro/stocktwits.py +79 -0
  41. agent_wire/channels/tier3_regional/bilibili.py +171 -0
  42. agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
  43. agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
  44. agent_wire/channels/tier3_regional/v2ex.py +70 -0
  45. agent_wire/channels/tier3_regional/wechat.py +67 -0
  46. agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
  47. agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
  48. agent_wire/channels/tier3_regional/xueqiu.py +77 -0
  49. agent_wire/channels/tier3_regional/zhihu.py +57 -0
  50. agent_wire/cli.py +403 -0
  51. agent_wire/compression/__init__.py +0 -0
  52. agent_wire/compression/anchor_slicer.py +45 -0
  53. agent_wire/compression/bm25_slicer.py +151 -0
  54. agent_wire/compression/dam_formatter.py +74 -0
  55. agent_wire/compression/url_compactor.py +45 -0
  56. agent_wire/config.py +52 -0
  57. agent_wire/engine/__init__.py +0 -0
  58. agent_wire/engine/challenge.py +58 -0
  59. agent_wire/engine/http_pool.py +136 -0
  60. agent_wire/engine/stealth_browser.py +90 -0
  61. agent_wire/engine/waterfall.py +159 -0
  62. agent_wire/firewall/__init__.py +0 -0
  63. agent_wire/firewall/inbound_shield.py +116 -0
  64. agent_wire/firewall/outbound_privacy.py +120 -0
  65. agent_wire/installers/__init__.py +0 -0
  66. agent_wire/installers/client_installer.py +315 -0
  67. agent_wire/media/__init__.py +0 -0
  68. agent_wire/media/audio_transcriber.py +77 -0
  69. agent_wire/media/subtitle_cleaner.py +102 -0
  70. agent_wire/models.py +60 -0
  71. agent_wire/research/__init__.py +1 -0
  72. agent_wire/research/brief.py +227 -0
  73. agent_wire/research/briefing.py +41 -0
  74. agent_wire/research/cluster.py +128 -0
  75. agent_wire/research/engine.py +632 -0
  76. agent_wire/research/library.py +74 -0
  77. agent_wire/research/planner.py +232 -0
  78. agent_wire/research/scoring.py +89 -0
  79. agent_wire/research/store.py +149 -0
  80. agent_wire/research/watch.py +70 -0
  81. agent_wire/router.py +107 -0
  82. agent_wire/server.py +468 -0
  83. agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
  84. agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
  85. agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
  86. agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
  87. agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,207 @@
1
+ """Markets: Yahoo Finance (global), NSE India, Screener.in ratios, CoinGecko crypto."""
2
+
3
+ import asyncio
4
+ from urllib.parse import parse_qs, quote, urlsplit
5
+
6
+ from curl_cffi.requests.exceptions import RequestException
7
+ from selectolax.lexbor import LexborHTMLParser
8
+
9
+ from agent_wire.channels.base import Channel, WireUnsupported, ts
10
+ from agent_wire.engine.http_pool import Fetched
11
+ from agent_wire.engine.waterfall import WireFetchError
12
+ from agent_wire.firewall.outbound_privacy import parse_json
13
+ from agent_wire.models import WireSearchHit
14
+
15
+ YF = "https://query1.finance.yahoo.com"
16
+ CG = "https://api.coingecko.com/api/v3"
17
+ ERRORS = (WireFetchError, RequestException, ValueError, KeyError, IndexError, TypeError)
18
+
19
+
20
+ def _num(v, digits=2):
21
+ if v is None:
22
+ return "—"
23
+ if isinstance(v, int | float) and abs(v) >= 1e9:
24
+ return f"{v / 1e9:.{digits}f}B"
25
+ if isinstance(v, int | float) and abs(v) >= 1e6:
26
+ return f"{v / 1e6:.{digits}f}M"
27
+ return f"{v:,.{digits}f}" if isinstance(v, int | float) else str(v)
28
+
29
+
30
+ def yahoo_row(chart: dict, summary: dict | None = None) -> dict:
31
+ meta = chart["chart"]["result"][0]["meta"]
32
+ price, prev = (
33
+ meta.get("regularMarketPrice"),
34
+ meta.get("chartPreviousClose") or meta.get("previousClose"),
35
+ )
36
+ detail = (summary or {}).get("summaryDetail", {})
37
+ raw = lambda k: (
38
+ (detail.get(k) or {}).get("raw") if isinstance(detail.get(k), dict) else detail.get(k)
39
+ )
40
+ return {
41
+ "symbol": meta.get("symbol"), "name": meta.get("longName") or meta.get("shortName"),
42
+ "price": price, "currency": meta.get("currency"),
43
+ "change_pct": (price - prev) / prev * 100 if price and prev else None,
44
+ "high52": meta.get("fiftyTwoWeekHigh"), "low52": meta.get("fiftyTwoWeekLow"),
45
+ "pe": raw("trailingPE"), "mcap": raw("marketCap"), "exchange": meta.get("fullExchangeName"),
46
+ } # fmt: skip
47
+
48
+
49
+ def nse_row(data: dict) -> dict:
50
+ info, price = data.get("info", {}), data.get("priceInfo", {})
51
+ hl = price.get("weekHighLow", {})
52
+ return {"symbol": info.get("symbol"), "name": info.get("companyName"), "price": price.get("lastPrice"),
53
+ "currency": "INR", "change_pct": price.get("pChange"), "high52": hl.get("max"), "low52": hl.get("min"),
54
+ "pe": data.get("metadata", {}).get("pdSymbolPe"), "mcap": None, "exchange": "NSE"} # fmt: skip
55
+
56
+
57
+ def coingecko_row(m: dict) -> dict:
58
+ return {"symbol": (m.get("symbol") or "").upper(), "name": m.get("name"), "price": m.get("current_price"),
59
+ "currency": "USD", "change_pct": m.get("price_change_percentage_24h"), "high52": m.get("ath"),
60
+ "low52": m.get("atl"), "pe": None, "mcap": m.get("market_cap"), "exchange": "CoinGecko (ATH/ATL)"} # fmt: skip
61
+
62
+
63
+ def screener_ratios(html: str) -> dict[str, str]:
64
+ out = {}
65
+ for li in LexborHTMLParser(html).css("#top-ratios li"):
66
+ name, value = li.css_first(".name"), li.css_first(".value")
67
+ if name and value:
68
+ out[name.text(strip=True)] = " ".join(value.text(strip=True).split())
69
+ return out
70
+
71
+
72
+ def table(rows: list[dict]) -> str:
73
+ lines = [
74
+ "| Symbol | Name | Price | Chg % | 52W H / L | P/E | Mkt cap |",
75
+ "|---|---|---|---|---|---|---|",
76
+ ]
77
+ for r in rows:
78
+ chg = f"{r['change_pct']:+.2f}%" if isinstance(r.get("change_pct"), int | float) else "—"
79
+ lines.append(f"| {r['symbol']} | {r.get('name') or ''} | {_num(r['price'])} {r.get('currency') or ''} | {chg} "
80
+ f"| {_num(r.get('high52'))} / {_num(r.get('low52'))} | {_num(r.get('pe'))} | {_num(r.get('mcap'))} |") # fmt: skip
81
+ return "\n".join(lines)
82
+
83
+
84
+ class MarketsChannel(Channel):
85
+ name = "markets"
86
+
87
+ def __init__(self, ctx):
88
+ super().__init__(ctx)
89
+ self._crumb: str | None = None
90
+
91
+ async def _chrome(self, url: str, **kw) -> Fetched:
92
+ """Anonymous shared chrome session: Yahoo/NSE need the cookies they set on a first visit."""
93
+ return await self.ctx.pool.get(url, impersonate="chrome", **kw)
94
+
95
+ async def _yahoo(self, sym: str) -> tuple[dict, Fetched]:
96
+ f = await self._chrome(f"{YF}/v8/finance/chart/{quote(sym)}?range=5d&interval=1d")
97
+ if not f.ok:
98
+ raise WireFetchError(f"yahoo {sym}: HTTP {f.status}")
99
+ chart = parse_json(f.content)
100
+ summary = None
101
+ try:
102
+ if not self._crumb:
103
+ await self._chrome("https://fc.yahoo.com/")
104
+ self._crumb = (await self._chrome(f"{YF}/v1/test/getcrumb")).text.strip() or None
105
+ if self._crumb:
106
+ s = await self._chrome(f"{YF}/v10/finance/quoteSummary/{quote(sym)}?modules=summaryDetail"
107
+ f"&crumb={quote(self._crumb)}") # fmt: skip
108
+ summary = parse_json(s.content)["quoteSummary"]["result"][0] if s.ok else None
109
+ except ERRORS:
110
+ summary = None # P/E and market cap are a bonus over the chart quote
111
+ return yahoo_row(chart, summary), f
112
+
113
+ async def _nse(self, sym: str) -> tuple[dict, Fetched]:
114
+ await self._chrome("https://www.nseindia.com/")
115
+ f = await self._chrome(f"https://www.nseindia.com/api/quote-equity?symbol={quote(sym)}",
116
+ headers={"Referer": f"https://www.nseindia.com/get-quotes/equity?symbol={sym}"}) # fmt: skip
117
+ if not f.ok:
118
+ raise WireFetchError(f"nse {sym}: HTTP {f.status}")
119
+ return nse_row(parse_json(f.content)), f
120
+
121
+ async def _coin(self, ident: str) -> tuple[dict, Fetched]:
122
+ coin_id = ident.lower()
123
+ found, _ = await self.fetch_json(f"{CG}/search?query={quote(coin_id)}")
124
+ if found.get("coins"):
125
+ coin_id = found["coins"][0]["id"]
126
+ data, f = await self.fetch_json(f"{CG}/coins/markets?vs_currency=usd&ids={quote(coin_id)}")
127
+ if not data:
128
+ raise WireUnsupported(f"no CoinGecko coin for {ident!r}")
129
+ return coingecko_row(data[0]), f
130
+
131
+ async def quote(self, token: str) -> tuple[dict, Fetched]:
132
+ t = token.strip()
133
+ prefix, _, rest = t.partition(":")
134
+ if rest and prefix.lower() in ("crypto", "cg", "coin"):
135
+ return await self._coin(rest)
136
+ if rest and prefix.lower() == "nse":
137
+ try:
138
+ return await self._nse(rest.upper())
139
+ except ERRORS: # NSE's Akamai front blocks many networks; Yahoo carries NSE listings
140
+ return await self._yahoo(f"{rest.upper()}.NS")
141
+ return await self._yahoo(t.upper())
142
+
143
+ async def _news(self, sym: str) -> str:
144
+ try:
145
+ data, _ = await self.fetch_json(
146
+ f"{YF}/v1/finance/search?q={quote(sym)}&newsCount=5&quotesCount=0"
147
+ )
148
+ except ERRORS:
149
+ return ""
150
+ return "\n".join(f"- [{n['title']}]({n['link']}) · {n.get('publisher', '')} · "
151
+ f"{ts(n.get('providerPublishTime')).date() if n.get('providerPublishTime') else ''}"
152
+ for n in data.get("news", [])) # fmt: skip
153
+
154
+ async def read(self, url, *, query=None, section=None, include_comments=True):
155
+ p = urlsplit(url)
156
+ host, segs = (p.hostname or "").removeprefix("www."), [s for s in p.path.split("/") if s]
157
+ if host == "screener.in" and segs[:1] == ["company"] and len(segs) > 1:
158
+ f = await self.fetch(url, archive=False)
159
+ ratios = screener_ratios(f.text)
160
+ body = "\n".join(f"- {k}: {v}" for k, v in ratios.items())
161
+ return self.doc(
162
+ f,
163
+ url,
164
+ title=f"{segs[1]} · Screener.in",
165
+ body=body,
166
+ metadata={"source": "screener.in"},
167
+ )
168
+ if host == "nseindia.com":
169
+ token = "nse:" + parse_qs(p.query).get("symbol", [""])[0]
170
+ elif host == "coingecko.com":
171
+ token = "crypto:" + segs[-1]
172
+ else: # finance.yahoo.com/quote/SYM
173
+ token = segs[segs.index("quote") + 1] if "quote" in segs else segs[-1]
174
+ row, f = await self.quote(token)
175
+ news = await self._news(row["symbol"]) if host == "finance.yahoo.com" else ""
176
+ body = table([row]) + (f"\n\n## News\n{news}" if news else "")
177
+ return self.doc(f, url, title=f"{row.get('name') or row['symbol']} quote", body=body,
178
+ metadata={"exchange": row.get("exchange")}) # fmt: skip
179
+
180
+ async def feed(self, target, *, sort="hot", limit=10):
181
+ tokens = [t for t in target.replace(";", ",").split(",") if t.strip()][:limit]
182
+ results = await asyncio.gather(*(self.quote(t) for t in tokens), return_exceptions=True)
183
+ rows, errors, last = [], [], None
184
+ for tok, r in zip(tokens, results, strict=True):
185
+ if isinstance(r, BaseException):
186
+ if not isinstance(r, ERRORS + (WireUnsupported,)):
187
+ raise r
188
+ errors.append(f"{tok.strip()}: {type(r).__name__}")
189
+ else:
190
+ rows.append(r[0])
191
+ last = r[1]
192
+ if not rows:
193
+ raise WireUnsupported("no quotes resolved: " + "; ".join(errors))
194
+ body = table(rows) + (f"\n\nUnavailable: {'; '.join(errors)}" if errors else "")
195
+ return self.doc(last, "https://finance.yahoo.com/", title="Watchlist", body=body)
196
+
197
+ async def search(self, query, *, limit=5, sort="relevance"):
198
+ data, _ = await self.fetch_json(
199
+ f"{YF}/v1/finance/search?q={quote(query)}&quotesCount={limit}&newsCount=0"
200
+ )
201
+ return [WireSearchHit(url=f"https://finance.yahoo.com/quote/{q['symbol']}",
202
+ title=f"{q['symbol']} · {q.get('shortname') or q.get('longname') or ''}",
203
+ snippet=f"{q.get('quoteType', '')} on {q.get('exchDisp', '')}", source="yahoo")
204
+ for q in data.get("quotes", [])[:limit] if q.get("symbol")] # fmt: skip
205
+
206
+
207
+ CHANNEL = MarketsChannel
@@ -0,0 +1,104 @@
1
+ """Instagram and Facebook: public embeds / web-profile API / OpenGraph, cookie escalation if logged in."""
2
+
3
+ import re
4
+ from urllib.parse import urlsplit
5
+
6
+ from selectolax.lexbor import LexborHTMLParser
7
+
8
+ from agent_wire.channels.base import Channel, WireUnsupported, compact_count, json_ld, meta_tags, ts
9
+ from agent_wire.engine.waterfall import WireFetchError
10
+
11
+ IG_APP_ID = "936619743392459" # public id the Instagram web app sends with every request
12
+ _IG_POST = re.compile(r"/(?:p|reel|reels|tv)/([\w-]+)")
13
+
14
+
15
+ def parse_ig_embed(html: str) -> dict:
16
+ tree = LexborHTMLParser(html)
17
+ cap = tree.css_first(".Caption")
18
+ user = tree.css_first(".CaptionUsername, .UsernameText")
19
+ text = cap.text(separator=" ", strip=True) if cap else ""
20
+ name = user.text(strip=True) if user else None
21
+ if name and text.startswith(name):
22
+ text = text[len(name) :].strip()
23
+ likes = tree.css_first(".SocialProof a, .likes")
24
+ return {"author_handle": name, "body": text,
25
+ "metadata": {"likes": compact_count(likes.text()) if likes else None}} # fmt: skip
26
+
27
+
28
+ def parse_og_social(html: str) -> dict:
29
+ """og:description like '1,234 likes, 56 comments - user on June 1, 2025: "caption"'."""
30
+ m = meta_tags(html)
31
+ desc = m.get("og:description") or m.get("description") or ""
32
+ if not desc and not m.get("og:title"):
33
+ raise WireUnsupported("login wall; run `wire auth login instagram.com` (or facebook.com)")
34
+ likes = re.search(r"([\d.,KM]+)\s+likes?", desc)
35
+ comments = re.search(r"([\d.,KM]+)\s+comments?", desc)
36
+ caption = re.search(r':\s*"(.*)"\s*\.?$', desc, re.DOTALL)
37
+ return {"title": m.get("og:title"), "body": caption[1] if caption else desc,
38
+ "metadata": {"likes": compact_count(likes[1]) if likes else None,
39
+ "comments": compact_count(comments[1]) if comments else None}} # fmt: skip
40
+
41
+
42
+ def parse_ig_profile(data: dict) -> dict:
43
+ u = data.get("data", {}).get("user") or {}
44
+ posts = [e["node"] for e in (u.get("edge_owner_to_timeline_media") or {}).get("edges", [])]
45
+ lines = [u.get("biography") or ""]
46
+ for p in posts[:12]:
47
+ caps = (p.get("edge_media_to_caption") or {}).get("edges") or [{}]
48
+ cap = (caps[0].get("node") or {}).get("text", "")
49
+ likes = (p.get("edge_liked_by") or p.get("edge_media_preview_like") or {}).get("count")
50
+ lines.append(
51
+ f"- https://www.instagram.com/p/{p.get('shortcode')}/ · ♥{likes} · {cap[:160]}"
52
+ )
53
+ return {"title": f"{u.get('full_name') or u.get('username')} (@{u.get('username')})",
54
+ "author_handle": u.get("username"), "body": "\n".join(x for x in lines if x),
55
+ "metadata": {"followers": (u.get("edge_followed_by") or {}).get("count"),
56
+ "following": (u.get("edge_follow") or {}).get("count"),
57
+ "posts": (u.get("edge_owner_to_timeline_media") or {}).get("count")}} # fmt: skip
58
+
59
+
60
+ class MetaSocialChannel(Channel):
61
+ name = "meta_social"
62
+
63
+ async def _ig(self, url: str, **kw):
64
+ # Fetch as the "instagram" channel so the outbound firewall applies its login-only rules.
65
+ return await self.fetch(url, channel="instagram", archive=False, **kw)
66
+
67
+ async def read(self, url, *, query=None, section=None, include_comments=True):
68
+ p = urlsplit(url)
69
+ host = (p.hostname or "").removeprefix("www.").removeprefix("m.")
70
+ if host == "instagram.com":
71
+ if m := _IG_POST.search(p.path):
72
+ code = m[1]
73
+ try:
74
+ f = await self._ig(f"https://www.instagram.com/p/{code}/embed/captioned/")
75
+ post = parse_ig_embed(f.text)
76
+ if post["body"]:
77
+ return self.doc(
78
+ f, url, title=f"@{post['author_handle']} on Instagram", **post
79
+ )
80
+ except WireFetchError:
81
+ pass
82
+ f = await self._ig(url)
83
+ return self.doc(f, url, **parse_og_social(f.text))
84
+ user = p.path.strip("/").split("/")[0]
85
+ if user:
86
+ try:
87
+ data, f = await self.fetch_json(
88
+ f"https://www.instagram.com/api/v1/users/web_profile_info/?username={user}",
89
+ headers={"x-ig-app-id": IG_APP_ID}, channel="instagram",
90
+ ) # fmt: skip
91
+ return self.doc(f, url, **parse_ig_profile(data))
92
+ except (WireFetchError, ValueError):
93
+ f = await self._ig(url)
94
+ return self.doc(f, url, **parse_og_social(f.text))
95
+ f = await self.fetch(url, archive=False) # facebook public pages / posts / watch
96
+ for item in json_ld(f.text):
97
+ if item.get("articleBody") or item.get("description"):
98
+ return self.doc(f, url, title=item.get("headline") or item.get("name"),
99
+ body=item.get("articleBody") or item.get("description"),
100
+ timestamp=ts(item.get("datePublished"))) # fmt: skip
101
+ return self.doc(f, url, **parse_og_social(f.text))
102
+
103
+
104
+ CHANNEL = MetaSocialChannel
@@ -0,0 +1,173 @@
1
+ """News: Google News RSS search (date-windowed) and the Techmeme river, no keys.
2
+
3
+ Google News items link to news.google.com/rss/articles/<id>, which only redirects with JavaScript.
4
+ decode() resolves the publisher URL the way that page does: read its signature + timestamp, then
5
+ ask the batchexecute endpoint for the target.
6
+ """
7
+
8
+ import asyncio
9
+ import json
10
+ import re
11
+ import xml.etree.ElementTree as ET
12
+ from datetime import UTC, datetime, timedelta
13
+ from email.utils import parsedate_to_datetime
14
+ from urllib.parse import quote
15
+
16
+ from curl_cffi.requests.exceptions import RequestException
17
+
18
+ from agent_wire.channels.base import Channel, WireUnsupported, html_text, listing
19
+ from agent_wire.engine.waterfall import WireFetchError
20
+ from agent_wire.models import WireSearchHit
21
+
22
+ GNEWS = "https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
23
+ GN_ARTICLE = re.compile(r"^https://news\.google\.com/(?:rss/)?(?:articles|read)/([\w-]+)")
24
+ BATCH = "https://news.google.com/_/DotsSplashUi/data/batchexecute"
25
+ _SG = re.compile(rb'data-n-a-sg="([^"]+)"')
26
+ _TS = re.compile(rb'data-n-a-ts="(\d+)"')
27
+ DECODE_TIMEOUT_S = 10
28
+ TECHMEME = "https://www.techmeme.com/feed.xml"
29
+ _HREF = re.compile(r'href="(https?://[^"]+)"')
30
+ _WORD = re.compile(r"[a-z0-9+#]+")
31
+ _OUTLET = re.compile(r"\s*\((?:[A-Z][\w.&' -]{1,40})\)\s*$") # "... (Bloomberg)"
32
+
33
+
34
+ def _date(text: str | None) -> datetime | None:
35
+ try:
36
+ return parsedate_to_datetime(text) if text else None
37
+ except (TypeError, ValueError):
38
+ return None
39
+
40
+
41
+ def parse_rss_items(content: bytes) -> list[dict]:
42
+ try:
43
+ root = ET.fromstring(content)
44
+ except ET.ParseError as e:
45
+ raise WireUnsupported(f"news feed unreadable ({e})") from e
46
+ out = []
47
+ for it in root.iter("item"):
48
+ src = it.find("source")
49
+ out.append({"title": (it.findtext("title") or "").strip(), "url": it.findtext("link") or "",
50
+ "date": _date(it.findtext("pubDate")), "desc": it.findtext("description") or "",
51
+ "source": src.text if src is not None else None}) # fmt: skip
52
+ return out
53
+
54
+
55
+ def gnews_hits(items: list[dict]) -> list[WireSearchHit]:
56
+ hits = []
57
+ for it in items:
58
+ title, outlet = it["title"], it["source"]
59
+ if outlet and title.endswith(f" - {outlet}"):
60
+ title = title[: -len(outlet) - 3]
61
+ hits.append(WireSearchHit(url=it["url"], title=title, snippet=outlet or "", source="news",
62
+ timestamp=it["date"], author=outlet)) # fmt: skip
63
+ return hits
64
+
65
+
66
+ def techmeme_hits(items: list[dict]) -> list[WireSearchHit]:
67
+ hits = []
68
+ for it in items:
69
+ links = [u for u in _HREF.findall(it["desc"]) if "techmeme.com" not in u]
70
+ hits.append(WireSearchHit(url=it["url"], title=_OUTLET.sub("", it["title"]), snippet=html_text(it["desc"])[:300],
71
+ source="techmeme", timestamp=it["date"],
72
+ link=links[0] if links else None)) # fmt: skip
73
+ return hits
74
+
75
+
76
+ def batch_form(gid: str, stamp: str, sig: str) -> dict:
77
+ inner = ["garturlreq", [["X", "X", ["X", "X"], None, None, 1, 1, "US:en", None, 1, None, None,
78
+ None, None, None, 0, 1], "X", "X", 1, [1, 1, 1], 1, 1, None, 0, 0, None, 0],
79
+ gid, int(stamp), sig] # fmt: skip
80
+ return {"f.req": json.dumps([[["Fbv4je", json.dumps(inner), None, "generic"]]])}
81
+
82
+
83
+ def parse_batch(text: str) -> str | None:
84
+ try:
85
+ rows = json.loads(text.split("\n\n", 1)[1])
86
+ url = json.loads(rows[0][2])[1]
87
+ except (IndexError, ValueError, TypeError):
88
+ return None
89
+ return url if isinstance(url, str) and url.startswith("http") else None
90
+
91
+
92
+ def matches(hit: WireSearchHit, query: str) -> bool:
93
+ """Every content word of the query appears in the title or snippet."""
94
+ words = [w for w in _WORD.findall(query.lower()) if len(w) > 1]
95
+ text = f"{hit.title} {hit.snippet}".lower()
96
+ return bool(words) and all(w in text for w in words)
97
+
98
+
99
+ class NewsChannel(Channel):
100
+ name = "news"
101
+
102
+ async def _items(self, url: str) -> list[dict]:
103
+ f = await self.fetch(url, archive=False, browser=False)
104
+ return parse_rss_items(f.content)
105
+
106
+ async def techmeme(self, query: str | None = None, since=None) -> list[WireSearchHit]:
107
+ hits = techmeme_hits(await self._items(TECHMEME))
108
+ if since:
109
+ hits = [h for h in hits if h.timestamp is None or h.timestamp >= since]
110
+ return [h for h in hits if matches(h, query)] if query else hits
111
+
112
+ async def search(self, query, *, limit=5, sort="relevance", since=None):
113
+ q = query
114
+ if since:
115
+ q = f"{query} when:{max(1, (datetime.now(UTC) - since).days)}d"
116
+ hits = gnews_hits(await self._items(GNEWS.format(q=quote(q))))
117
+ if sort in ("new", "date", "recent"):
118
+ hits.sort(key=lambda h: h.timestamp or datetime.min.replace(tzinfo=UTC), reverse=True)
119
+ try:
120
+ tm = await self.techmeme(query, since)
121
+ except (WireFetchError, WireUnsupported): # Techmeme is a bonus; Google News answered
122
+ tm = []
123
+ out = (tm + hits)[:limit]
124
+ real = await asyncio.gather(*(self.decode(h.url) for h in out))
125
+ return [h.model_copy(update={"url": u}) for h, u in zip(out, real, strict=True)]
126
+
127
+ async def decode(self, url: str) -> str:
128
+ """Publisher URL behind a Google News link; the link itself when it can't be resolved."""
129
+ m = GN_ARTICLE.match(url)
130
+ if not m or self.ctx.pool is None:
131
+ return url
132
+ try:
133
+ return await asyncio.wait_for(self._decode(m[1]), DECODE_TIMEOUT_S) or url
134
+ except (RequestException, TimeoutError, ValueError):
135
+ return url
136
+
137
+ async def _decode(self, gid: str) -> str | None:
138
+ page = await self.ctx.pool.get(f"https://news.google.com/rss/articles/{gid}")
139
+ sig, stamp = _SG.search(page.content), _TS.search(page.content)
140
+ if not (sig and stamp):
141
+ return None
142
+ r = await self.ctx.pool.get(
143
+ BATCH, method="POST", data=batch_form(gid, stamp[1].decode(), sig[1].decode()),
144
+ headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"},
145
+ ) # fmt: skip
146
+ return parse_batch(r.content.decode("utf-8", "replace")) if r.ok else None
147
+
148
+ async def read(self, url, *, query=None, section=None, include_comments=True):
149
+ from agent_wire.router import route_url # router imports the channel registry
150
+
151
+ target = await self.decode(url)
152
+ if target == url:
153
+ raise WireUnsupported(f"news: could not resolve the Google News link {url}")
154
+ return await self.ctx.channel(route_url(target)).read(
155
+ target, query=query, section=section, include_comments=include_comments
156
+ )
157
+
158
+ async def feed(self, target="techmeme", *, sort="hot", limit=10):
159
+ key = target.strip().lower()
160
+ if key in ("techmeme", "tech", ""):
161
+ hits = (await self.techmeme())[:limit]
162
+ title, url = "Techmeme", "https://www.techmeme.com/"
163
+ else:
164
+ hits = await self.search(
165
+ target, limit=limit, since=datetime.now(UTC) - timedelta(days=7)
166
+ )
167
+ title, url = f"News · {target}", GNEWS.format(q=quote(target))
168
+ items = [{"title": h.title, "url": h.link or h.url, "snippet": h.snippet,
169
+ "meta": [h.timestamp.date().isoformat() if h.timestamp else None]} for h in hits] # fmt: skip
170
+ return self.doc(None, url, title=title, body=listing(items))
171
+
172
+
173
+ CHANNEL = NewsChannel
@@ -0,0 +1,105 @@
1
+ """Polymarket prediction markets (public Gamma API, no key): what money says about a topic."""
2
+
3
+ import json
4
+ from urllib.parse import quote, urlsplit
5
+
6
+ from agent_wire.channels.base import Channel, WireUnsupported, listing, ts
7
+ from agent_wire.models import WireSearchHit
8
+
9
+ GAMMA = "https://gamma-api.polymarket.com"
10
+ EVENT = "https://polymarket.com/event/"
11
+
12
+
13
+ def _odds(market: dict) -> str:
14
+ """'Yes 62% · No 38%' from the JSON-in-a-string outcome fields."""
15
+ try:
16
+ names = json.loads(market.get("outcomes") or "[]")
17
+ prices = [float(p) for p in json.loads(market.get("outcomePrices") or "[]")]
18
+ except (ValueError, TypeError):
19
+ return ""
20
+ label = market.get("groupItemTitle")
21
+ odds = " · ".join(f"{n} {round(p * 100)}%" for n, p in zip(names, prices, strict=False))
22
+ return f"{label}: {odds}" if label else odds
23
+
24
+
25
+ def _movement(market: dict) -> str:
26
+ d = market.get("oneWeekPriceChange") or market.get("oneDayPriceChange")
27
+ if not isinstance(d, int | float) or not d:
28
+ return ""
29
+ return f"({'+' if d > 0 else ''}{round(d * 100)} pts)"
30
+
31
+
32
+ def event_summary(event: dict, max_markets: int = 4) -> list[str]:
33
+ markets = sorted(event.get("markets") or [], key=lambda m: -float(m.get("volume") or 0))
34
+ lines = (" ".join(filter(None, (_odds(m), _movement(m)))) for m in markets[:max_markets])
35
+ return [line for line in lines if line]
36
+
37
+
38
+ def event_hit(event: dict) -> WireSearchHit:
39
+ vol24 = int(float(event.get("volume24hr") or 0))
40
+ return WireSearchHit(
41
+ url=EVENT + event.get("slug", ""), title=event.get("title") or "",
42
+ snippet=" | ".join(event_summary(event, 3)), source="polymarket",
43
+ rank_score=float(vol24), # undated: a market is live until it resolves, so "freshness" is moot
44
+ engagement={"volume": int(float(event.get("volume") or 0)), "volume24h": vol24,
45
+ "liquidity": int(float(event.get("liquidity") or 0)),
46
+ "comments": event.get("commentCount") or 0},
47
+ ) # fmt: skip
48
+
49
+
50
+ class PolymarketChannel(Channel):
51
+ name = "polymarket"
52
+
53
+ async def search(self, query, *, limit=5, sort="relevance", since=None):
54
+ data, _ = await self.fetch_json(
55
+ f"{GAMMA}/public-search?q={quote(query)}&limit_per_type={max(limit, 5)}&events_status=active"
56
+ )
57
+ events = [e for e in data.get("events") or [] if not e.get("closed")]
58
+ if since: # markets that resolved before the window opened are history, not news
59
+ events = [e for e in events if (t := ts(e.get("endDate"))) is None or t >= since]
60
+ return [event_hit(e) for e in events[:limit]]
61
+
62
+ async def feed(self, target="trending", *, sort="hot", limit=10):
63
+ if target.strip().lower() not in ("trending", "hot", "top", "movers", ""):
64
+ hits = await self.search(target, limit=limit)
65
+ body = listing([{"title": h.title, "url": h.url, "snippet": h.snippet,
66
+ "meta": [f"${h.engagement['volume24h']:,} 24h"]} for h in hits]) # fmt: skip
67
+ return self.doc(None, f"https://polymarket.com/search?q={quote(target)}",
68
+ title=f"Polymarket · {target}", body=body) # fmt: skip
69
+ data, f = await self.fetch_json(
70
+ f"{GAMMA}/events?limit={limit}&active=true&closed=false&order=volume24hr&ascending=false"
71
+ )
72
+ items = [{"title": e.get("title"), "url": EVENT + e.get("slug", ""),
73
+ "meta": [f"${int(float(e.get('volume24hr') or 0)):,} 24h"],
74
+ "snippet": " | ".join(event_summary(e, 2))} for e in data] # fmt: skip
75
+ return self.doc(
76
+ f, "https://polymarket.com/", title="Polymarket · trending", body=listing(items)
77
+ )
78
+
79
+ async def trending_hits(self, limit=10) -> list[WireSearchHit]:
80
+ data, _ = await self.fetch_json(
81
+ f"{GAMMA}/events?limit={limit}&active=true&closed=false&order=volume24hr&ascending=false"
82
+ )
83
+ return [event_hit(e) for e in data]
84
+
85
+ async def read(self, url, *, query=None, section=None, include_comments=True):
86
+ parts = [p for p in urlsplit(url).path.split("/") if p]
87
+ if not parts: # the front page: trending markets
88
+ return await self.feed("trending")
89
+ if len(parts) < 2 or parts[0] not in ("event", "market"):
90
+ raise WireUnsupported("polymarket: only /event/<slug> URLs and the front page")
91
+ data, f = await self.fetch_json(f"{GAMMA}/events?slug={quote(parts[1])}")
92
+ if not data:
93
+ raise WireUnsupported(f"polymarket: no event {parts[1]!r}")
94
+ e = data[0]
95
+ lines = [f"- {line}" for line in event_summary(e, 20)]
96
+ body = "\n".join([e.get("description") or "", "", "Markets:", *lines]).strip()
97
+ hit = event_hit(e)
98
+ return self.doc(
99
+ f, url, title=e.get("title"), timestamp=ts(e.get("startDate")), body=body,
100
+ metadata={"volume": hit.engagement["volume"], "volume_24h": hit.engagement["volume24h"],
101
+ "ends": (e.get("endDate") or "")[:10] or None},
102
+ ) # fmt: skip
103
+
104
+
105
+ CHANNEL = PolymarketChannel