agent-wire-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_wire/__init__.py +1 -0
- agent_wire/__main__.py +3 -0
- agent_wire/assets/SKILL.md +26 -0
- agent_wire/assets/cursor_rule.mdc +13 -0
- agent_wire/assets/hermes_skill.md +28 -0
- agent_wire/assets/windsurf_rules.md +14 -0
- agent_wire/assets/wire-worker.md +20 -0
- agent_wire/auth/__init__.py +0 -0
- agent_wire/auth/browser_paths.py +143 -0
- agent_wire/auth/cookie_vault.py +181 -0
- agent_wire/auth/rate_limiter.py +26 -0
- agent_wire/cache/__init__.py +0 -0
- agent_wire/cache/ring_buffer.py +94 -0
- agent_wire/cache/sqlite_ledger.py +132 -0
- agent_wire/channels/__init__.py +41 -0
- agent_wire/channels/base.py +249 -0
- agent_wire/channels/tier1_global/__init__.py +0 -0
- agent_wire/channels/tier1_global/github.py +371 -0
- agent_wire/channels/tier1_global/hackernews.py +135 -0
- agent_wire/channels/tier1_global/reddit.py +371 -0
- agent_wire/channels/tier1_global/rss.py +129 -0
- agent_wire/channels/tier1_global/search.py +160 -0
- agent_wire/channels/tier1_global/stackoverflow.py +94 -0
- agent_wire/channels/tier1_global/web.py +98 -0
- agent_wire/channels/tier1_global/wikipedia.py +91 -0
- agent_wire/channels/tier1_global/x_twitter.py +380 -0
- agent_wire/channels/tier1_global/youtube.py +223 -0
- agent_wire/channels/tier2_pro/__init__.py +0 -0
- agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
- agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
- agent_wire/channels/tier2_pro/devto.py +114 -0
- agent_wire/channels/tier2_pro/fediverse.py +181 -0
- agent_wire/channels/tier2_pro/hubs.py +161 -0
- agent_wire/channels/tier2_pro/linkedin.py +146 -0
- agent_wire/channels/tier2_pro/markets.py +207 -0
- agent_wire/channels/tier2_pro/meta_social.py +104 -0
- agent_wire/channels/tier2_pro/news.py +173 -0
- agent_wire/channels/tier2_pro/polymarket.py +105 -0
- agent_wire/channels/tier2_pro/publishing.py +84 -0
- agent_wire/channels/tier2_pro/stocktwits.py +79 -0
- agent_wire/channels/tier3_regional/bilibili.py +171 -0
- agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
- agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
- agent_wire/channels/tier3_regional/v2ex.py +70 -0
- agent_wire/channels/tier3_regional/wechat.py +67 -0
- agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
- agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
- agent_wire/channels/tier3_regional/xueqiu.py +77 -0
- agent_wire/channels/tier3_regional/zhihu.py +57 -0
- agent_wire/cli.py +403 -0
- agent_wire/compression/__init__.py +0 -0
- agent_wire/compression/anchor_slicer.py +45 -0
- agent_wire/compression/bm25_slicer.py +151 -0
- agent_wire/compression/dam_formatter.py +74 -0
- agent_wire/compression/url_compactor.py +45 -0
- agent_wire/config.py +52 -0
- agent_wire/engine/__init__.py +0 -0
- agent_wire/engine/challenge.py +58 -0
- agent_wire/engine/http_pool.py +136 -0
- agent_wire/engine/stealth_browser.py +90 -0
- agent_wire/engine/waterfall.py +159 -0
- agent_wire/firewall/__init__.py +0 -0
- agent_wire/firewall/inbound_shield.py +116 -0
- agent_wire/firewall/outbound_privacy.py +120 -0
- agent_wire/installers/__init__.py +0 -0
- agent_wire/installers/client_installer.py +315 -0
- agent_wire/media/__init__.py +0 -0
- agent_wire/media/audio_transcriber.py +77 -0
- agent_wire/media/subtitle_cleaner.py +102 -0
- agent_wire/models.py +60 -0
- agent_wire/research/__init__.py +1 -0
- agent_wire/research/brief.py +227 -0
- agent_wire/research/briefing.py +41 -0
- agent_wire/research/cluster.py +128 -0
- agent_wire/research/engine.py +632 -0
- agent_wire/research/library.py +74 -0
- agent_wire/research/planner.py +232 -0
- agent_wire/research/scoring.py +89 -0
- agent_wire/research/store.py +149 -0
- agent_wire/research/watch.py +70 -0
- agent_wire/router.py +107 -0
- agent_wire/server.py +468 -0
- agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
- agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
- agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
- agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
- agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Markets: Yahoo Finance (global), NSE India, Screener.in ratios, CoinGecko crypto."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from urllib.parse import parse_qs, quote, urlsplit
|
|
5
|
+
|
|
6
|
+
from curl_cffi.requests.exceptions import RequestException
|
|
7
|
+
from selectolax.lexbor import LexborHTMLParser
|
|
8
|
+
|
|
9
|
+
from agent_wire.channels.base import Channel, WireUnsupported, ts
|
|
10
|
+
from agent_wire.engine.http_pool import Fetched
|
|
11
|
+
from agent_wire.engine.waterfall import WireFetchError
|
|
12
|
+
from agent_wire.firewall.outbound_privacy import parse_json
|
|
13
|
+
from agent_wire.models import WireSearchHit
|
|
14
|
+
|
|
15
|
+
YF = "https://query1.finance.yahoo.com"
|
|
16
|
+
CG = "https://api.coingecko.com/api/v3"
|
|
17
|
+
ERRORS = (WireFetchError, RequestException, ValueError, KeyError, IndexError, TypeError)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _num(v, digits=2):
|
|
21
|
+
if v is None:
|
|
22
|
+
return "—"
|
|
23
|
+
if isinstance(v, int | float) and abs(v) >= 1e9:
|
|
24
|
+
return f"{v / 1e9:.{digits}f}B"
|
|
25
|
+
if isinstance(v, int | float) and abs(v) >= 1e6:
|
|
26
|
+
return f"{v / 1e6:.{digits}f}M"
|
|
27
|
+
return f"{v:,.{digits}f}" if isinstance(v, int | float) else str(v)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def yahoo_row(chart: dict, summary: dict | None = None) -> dict:
|
|
31
|
+
meta = chart["chart"]["result"][0]["meta"]
|
|
32
|
+
price, prev = (
|
|
33
|
+
meta.get("regularMarketPrice"),
|
|
34
|
+
meta.get("chartPreviousClose") or meta.get("previousClose"),
|
|
35
|
+
)
|
|
36
|
+
detail = (summary or {}).get("summaryDetail", {})
|
|
37
|
+
raw = lambda k: (
|
|
38
|
+
(detail.get(k) or {}).get("raw") if isinstance(detail.get(k), dict) else detail.get(k)
|
|
39
|
+
)
|
|
40
|
+
return {
|
|
41
|
+
"symbol": meta.get("symbol"), "name": meta.get("longName") or meta.get("shortName"),
|
|
42
|
+
"price": price, "currency": meta.get("currency"),
|
|
43
|
+
"change_pct": (price - prev) / prev * 100 if price and prev else None,
|
|
44
|
+
"high52": meta.get("fiftyTwoWeekHigh"), "low52": meta.get("fiftyTwoWeekLow"),
|
|
45
|
+
"pe": raw("trailingPE"), "mcap": raw("marketCap"), "exchange": meta.get("fullExchangeName"),
|
|
46
|
+
} # fmt: skip
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def nse_row(data: dict) -> dict:
|
|
50
|
+
info, price = data.get("info", {}), data.get("priceInfo", {})
|
|
51
|
+
hl = price.get("weekHighLow", {})
|
|
52
|
+
return {"symbol": info.get("symbol"), "name": info.get("companyName"), "price": price.get("lastPrice"),
|
|
53
|
+
"currency": "INR", "change_pct": price.get("pChange"), "high52": hl.get("max"), "low52": hl.get("min"),
|
|
54
|
+
"pe": data.get("metadata", {}).get("pdSymbolPe"), "mcap": None, "exchange": "NSE"} # fmt: skip
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def coingecko_row(m: dict) -> dict:
|
|
58
|
+
return {"symbol": (m.get("symbol") or "").upper(), "name": m.get("name"), "price": m.get("current_price"),
|
|
59
|
+
"currency": "USD", "change_pct": m.get("price_change_percentage_24h"), "high52": m.get("ath"),
|
|
60
|
+
"low52": m.get("atl"), "pe": None, "mcap": m.get("market_cap"), "exchange": "CoinGecko (ATH/ATL)"} # fmt: skip
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def screener_ratios(html: str) -> dict[str, str]:
|
|
64
|
+
out = {}
|
|
65
|
+
for li in LexborHTMLParser(html).css("#top-ratios li"):
|
|
66
|
+
name, value = li.css_first(".name"), li.css_first(".value")
|
|
67
|
+
if name and value:
|
|
68
|
+
out[name.text(strip=True)] = " ".join(value.text(strip=True).split())
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def table(rows: list[dict]) -> str:
|
|
73
|
+
lines = [
|
|
74
|
+
"| Symbol | Name | Price | Chg % | 52W H / L | P/E | Mkt cap |",
|
|
75
|
+
"|---|---|---|---|---|---|---|",
|
|
76
|
+
]
|
|
77
|
+
for r in rows:
|
|
78
|
+
chg = f"{r['change_pct']:+.2f}%" if isinstance(r.get("change_pct"), int | float) else "—"
|
|
79
|
+
lines.append(f"| {r['symbol']} | {r.get('name') or ''} | {_num(r['price'])} {r.get('currency') or ''} | {chg} "
|
|
80
|
+
f"| {_num(r.get('high52'))} / {_num(r.get('low52'))} | {_num(r.get('pe'))} | {_num(r.get('mcap'))} |") # fmt: skip
|
|
81
|
+
return "\n".join(lines)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class MarketsChannel(Channel):
|
|
85
|
+
name = "markets"
|
|
86
|
+
|
|
87
|
+
def __init__(self, ctx):
|
|
88
|
+
super().__init__(ctx)
|
|
89
|
+
self._crumb: str | None = None
|
|
90
|
+
|
|
91
|
+
async def _chrome(self, url: str, **kw) -> Fetched:
|
|
92
|
+
"""Anonymous shared chrome session: Yahoo/NSE need the cookies they set on a first visit."""
|
|
93
|
+
return await self.ctx.pool.get(url, impersonate="chrome", **kw)
|
|
94
|
+
|
|
95
|
+
async def _yahoo(self, sym: str) -> tuple[dict, Fetched]:
|
|
96
|
+
f = await self._chrome(f"{YF}/v8/finance/chart/{quote(sym)}?range=5d&interval=1d")
|
|
97
|
+
if not f.ok:
|
|
98
|
+
raise WireFetchError(f"yahoo {sym}: HTTP {f.status}")
|
|
99
|
+
chart = parse_json(f.content)
|
|
100
|
+
summary = None
|
|
101
|
+
try:
|
|
102
|
+
if not self._crumb:
|
|
103
|
+
await self._chrome("https://fc.yahoo.com/")
|
|
104
|
+
self._crumb = (await self._chrome(f"{YF}/v1/test/getcrumb")).text.strip() or None
|
|
105
|
+
if self._crumb:
|
|
106
|
+
s = await self._chrome(f"{YF}/v10/finance/quoteSummary/{quote(sym)}?modules=summaryDetail"
|
|
107
|
+
f"&crumb={quote(self._crumb)}") # fmt: skip
|
|
108
|
+
summary = parse_json(s.content)["quoteSummary"]["result"][0] if s.ok else None
|
|
109
|
+
except ERRORS:
|
|
110
|
+
summary = None # P/E and market cap are a bonus over the chart quote
|
|
111
|
+
return yahoo_row(chart, summary), f
|
|
112
|
+
|
|
113
|
+
async def _nse(self, sym: str) -> tuple[dict, Fetched]:
|
|
114
|
+
await self._chrome("https://www.nseindia.com/")
|
|
115
|
+
f = await self._chrome(f"https://www.nseindia.com/api/quote-equity?symbol={quote(sym)}",
|
|
116
|
+
headers={"Referer": f"https://www.nseindia.com/get-quotes/equity?symbol={sym}"}) # fmt: skip
|
|
117
|
+
if not f.ok:
|
|
118
|
+
raise WireFetchError(f"nse {sym}: HTTP {f.status}")
|
|
119
|
+
return nse_row(parse_json(f.content)), f
|
|
120
|
+
|
|
121
|
+
async def _coin(self, ident: str) -> tuple[dict, Fetched]:
|
|
122
|
+
coin_id = ident.lower()
|
|
123
|
+
found, _ = await self.fetch_json(f"{CG}/search?query={quote(coin_id)}")
|
|
124
|
+
if found.get("coins"):
|
|
125
|
+
coin_id = found["coins"][0]["id"]
|
|
126
|
+
data, f = await self.fetch_json(f"{CG}/coins/markets?vs_currency=usd&ids={quote(coin_id)}")
|
|
127
|
+
if not data:
|
|
128
|
+
raise WireUnsupported(f"no CoinGecko coin for {ident!r}")
|
|
129
|
+
return coingecko_row(data[0]), f
|
|
130
|
+
|
|
131
|
+
async def quote(self, token: str) -> tuple[dict, Fetched]:
|
|
132
|
+
t = token.strip()
|
|
133
|
+
prefix, _, rest = t.partition(":")
|
|
134
|
+
if rest and prefix.lower() in ("crypto", "cg", "coin"):
|
|
135
|
+
return await self._coin(rest)
|
|
136
|
+
if rest and prefix.lower() == "nse":
|
|
137
|
+
try:
|
|
138
|
+
return await self._nse(rest.upper())
|
|
139
|
+
except ERRORS: # NSE's Akamai front blocks many networks; Yahoo carries NSE listings
|
|
140
|
+
return await self._yahoo(f"{rest.upper()}.NS")
|
|
141
|
+
return await self._yahoo(t.upper())
|
|
142
|
+
|
|
143
|
+
async def _news(self, sym: str) -> str:
|
|
144
|
+
try:
|
|
145
|
+
data, _ = await self.fetch_json(
|
|
146
|
+
f"{YF}/v1/finance/search?q={quote(sym)}&newsCount=5"esCount=0"
|
|
147
|
+
)
|
|
148
|
+
except ERRORS:
|
|
149
|
+
return ""
|
|
150
|
+
return "\n".join(f"- [{n['title']}]({n['link']}) · {n.get('publisher', '')} · "
|
|
151
|
+
f"{ts(n.get('providerPublishTime')).date() if n.get('providerPublishTime') else ''}"
|
|
152
|
+
for n in data.get("news", [])) # fmt: skip
|
|
153
|
+
|
|
154
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
155
|
+
p = urlsplit(url)
|
|
156
|
+
host, segs = (p.hostname or "").removeprefix("www."), [s for s in p.path.split("/") if s]
|
|
157
|
+
if host == "screener.in" and segs[:1] == ["company"] and len(segs) > 1:
|
|
158
|
+
f = await self.fetch(url, archive=False)
|
|
159
|
+
ratios = screener_ratios(f.text)
|
|
160
|
+
body = "\n".join(f"- {k}: {v}" for k, v in ratios.items())
|
|
161
|
+
return self.doc(
|
|
162
|
+
f,
|
|
163
|
+
url,
|
|
164
|
+
title=f"{segs[1]} · Screener.in",
|
|
165
|
+
body=body,
|
|
166
|
+
metadata={"source": "screener.in"},
|
|
167
|
+
)
|
|
168
|
+
if host == "nseindia.com":
|
|
169
|
+
token = "nse:" + parse_qs(p.query).get("symbol", [""])[0]
|
|
170
|
+
elif host == "coingecko.com":
|
|
171
|
+
token = "crypto:" + segs[-1]
|
|
172
|
+
else: # finance.yahoo.com/quote/SYM
|
|
173
|
+
token = segs[segs.index("quote") + 1] if "quote" in segs else segs[-1]
|
|
174
|
+
row, f = await self.quote(token)
|
|
175
|
+
news = await self._news(row["symbol"]) if host == "finance.yahoo.com" else ""
|
|
176
|
+
body = table([row]) + (f"\n\n## News\n{news}" if news else "")
|
|
177
|
+
return self.doc(f, url, title=f"{row.get('name') or row['symbol']} quote", body=body,
|
|
178
|
+
metadata={"exchange": row.get("exchange")}) # fmt: skip
|
|
179
|
+
|
|
180
|
+
async def feed(self, target, *, sort="hot", limit=10):
|
|
181
|
+
tokens = [t for t in target.replace(";", ",").split(",") if t.strip()][:limit]
|
|
182
|
+
results = await asyncio.gather(*(self.quote(t) for t in tokens), return_exceptions=True)
|
|
183
|
+
rows, errors, last = [], [], None
|
|
184
|
+
for tok, r in zip(tokens, results, strict=True):
|
|
185
|
+
if isinstance(r, BaseException):
|
|
186
|
+
if not isinstance(r, ERRORS + (WireUnsupported,)):
|
|
187
|
+
raise r
|
|
188
|
+
errors.append(f"{tok.strip()}: {type(r).__name__}")
|
|
189
|
+
else:
|
|
190
|
+
rows.append(r[0])
|
|
191
|
+
last = r[1]
|
|
192
|
+
if not rows:
|
|
193
|
+
raise WireUnsupported("no quotes resolved: " + "; ".join(errors))
|
|
194
|
+
body = table(rows) + (f"\n\nUnavailable: {'; '.join(errors)}" if errors else "")
|
|
195
|
+
return self.doc(last, "https://finance.yahoo.com/", title="Watchlist", body=body)
|
|
196
|
+
|
|
197
|
+
async def search(self, query, *, limit=5, sort="relevance"):
|
|
198
|
+
data, _ = await self.fetch_json(
|
|
199
|
+
f"{YF}/v1/finance/search?q={quote(query)}"esCount={limit}&newsCount=0"
|
|
200
|
+
)
|
|
201
|
+
return [WireSearchHit(url=f"https://finance.yahoo.com/quote/{q['symbol']}",
|
|
202
|
+
title=f"{q['symbol']} · {q.get('shortname') or q.get('longname') or ''}",
|
|
203
|
+
snippet=f"{q.get('quoteType', '')} on {q.get('exchDisp', '')}", source="yahoo")
|
|
204
|
+
for q in data.get("quotes", [])[:limit] if q.get("symbol")] # fmt: skip
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
CHANNEL = MarketsChannel
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Instagram and Facebook: public embeds / web-profile API / OpenGraph, cookie escalation if logged in."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from urllib.parse import urlsplit
|
|
5
|
+
|
|
6
|
+
from selectolax.lexbor import LexborHTMLParser
|
|
7
|
+
|
|
8
|
+
from agent_wire.channels.base import Channel, WireUnsupported, compact_count, json_ld, meta_tags, ts
|
|
9
|
+
from agent_wire.engine.waterfall import WireFetchError
|
|
10
|
+
|
|
11
|
+
IG_APP_ID = "936619743392459" # public id the Instagram web app sends with every request
|
|
12
|
+
_IG_POST = re.compile(r"/(?:p|reel|reels|tv)/([\w-]+)")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def parse_ig_embed(html: str) -> dict:
|
|
16
|
+
tree = LexborHTMLParser(html)
|
|
17
|
+
cap = tree.css_first(".Caption")
|
|
18
|
+
user = tree.css_first(".CaptionUsername, .UsernameText")
|
|
19
|
+
text = cap.text(separator=" ", strip=True) if cap else ""
|
|
20
|
+
name = user.text(strip=True) if user else None
|
|
21
|
+
if name and text.startswith(name):
|
|
22
|
+
text = text[len(name) :].strip()
|
|
23
|
+
likes = tree.css_first(".SocialProof a, .likes")
|
|
24
|
+
return {"author_handle": name, "body": text,
|
|
25
|
+
"metadata": {"likes": compact_count(likes.text()) if likes else None}} # fmt: skip
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def parse_og_social(html: str) -> dict:
|
|
29
|
+
"""og:description like '1,234 likes, 56 comments - user on June 1, 2025: "caption"'."""
|
|
30
|
+
m = meta_tags(html)
|
|
31
|
+
desc = m.get("og:description") or m.get("description") or ""
|
|
32
|
+
if not desc and not m.get("og:title"):
|
|
33
|
+
raise WireUnsupported("login wall; run `wire auth login instagram.com` (or facebook.com)")
|
|
34
|
+
likes = re.search(r"([\d.,KM]+)\s+likes?", desc)
|
|
35
|
+
comments = re.search(r"([\d.,KM]+)\s+comments?", desc)
|
|
36
|
+
caption = re.search(r':\s*"(.*)"\s*\.?$', desc, re.DOTALL)
|
|
37
|
+
return {"title": m.get("og:title"), "body": caption[1] if caption else desc,
|
|
38
|
+
"metadata": {"likes": compact_count(likes[1]) if likes else None,
|
|
39
|
+
"comments": compact_count(comments[1]) if comments else None}} # fmt: skip
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_ig_profile(data: dict) -> dict:
|
|
43
|
+
u = data.get("data", {}).get("user") or {}
|
|
44
|
+
posts = [e["node"] for e in (u.get("edge_owner_to_timeline_media") or {}).get("edges", [])]
|
|
45
|
+
lines = [u.get("biography") or ""]
|
|
46
|
+
for p in posts[:12]:
|
|
47
|
+
caps = (p.get("edge_media_to_caption") or {}).get("edges") or [{}]
|
|
48
|
+
cap = (caps[0].get("node") or {}).get("text", "")
|
|
49
|
+
likes = (p.get("edge_liked_by") or p.get("edge_media_preview_like") or {}).get("count")
|
|
50
|
+
lines.append(
|
|
51
|
+
f"- https://www.instagram.com/p/{p.get('shortcode')}/ · ♥{likes} · {cap[:160]}"
|
|
52
|
+
)
|
|
53
|
+
return {"title": f"{u.get('full_name') or u.get('username')} (@{u.get('username')})",
|
|
54
|
+
"author_handle": u.get("username"), "body": "\n".join(x for x in lines if x),
|
|
55
|
+
"metadata": {"followers": (u.get("edge_followed_by") or {}).get("count"),
|
|
56
|
+
"following": (u.get("edge_follow") or {}).get("count"),
|
|
57
|
+
"posts": (u.get("edge_owner_to_timeline_media") or {}).get("count")}} # fmt: skip
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class MetaSocialChannel(Channel):
|
|
61
|
+
name = "meta_social"
|
|
62
|
+
|
|
63
|
+
async def _ig(self, url: str, **kw):
|
|
64
|
+
# Fetch as the "instagram" channel so the outbound firewall applies its login-only rules.
|
|
65
|
+
return await self.fetch(url, channel="instagram", archive=False, **kw)
|
|
66
|
+
|
|
67
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
68
|
+
p = urlsplit(url)
|
|
69
|
+
host = (p.hostname or "").removeprefix("www.").removeprefix("m.")
|
|
70
|
+
if host == "instagram.com":
|
|
71
|
+
if m := _IG_POST.search(p.path):
|
|
72
|
+
code = m[1]
|
|
73
|
+
try:
|
|
74
|
+
f = await self._ig(f"https://www.instagram.com/p/{code}/embed/captioned/")
|
|
75
|
+
post = parse_ig_embed(f.text)
|
|
76
|
+
if post["body"]:
|
|
77
|
+
return self.doc(
|
|
78
|
+
f, url, title=f"@{post['author_handle']} on Instagram", **post
|
|
79
|
+
)
|
|
80
|
+
except WireFetchError:
|
|
81
|
+
pass
|
|
82
|
+
f = await self._ig(url)
|
|
83
|
+
return self.doc(f, url, **parse_og_social(f.text))
|
|
84
|
+
user = p.path.strip("/").split("/")[0]
|
|
85
|
+
if user:
|
|
86
|
+
try:
|
|
87
|
+
data, f = await self.fetch_json(
|
|
88
|
+
f"https://www.instagram.com/api/v1/users/web_profile_info/?username={user}",
|
|
89
|
+
headers={"x-ig-app-id": IG_APP_ID}, channel="instagram",
|
|
90
|
+
) # fmt: skip
|
|
91
|
+
return self.doc(f, url, **parse_ig_profile(data))
|
|
92
|
+
except (WireFetchError, ValueError):
|
|
93
|
+
f = await self._ig(url)
|
|
94
|
+
return self.doc(f, url, **parse_og_social(f.text))
|
|
95
|
+
f = await self.fetch(url, archive=False) # facebook public pages / posts / watch
|
|
96
|
+
for item in json_ld(f.text):
|
|
97
|
+
if item.get("articleBody") or item.get("description"):
|
|
98
|
+
return self.doc(f, url, title=item.get("headline") or item.get("name"),
|
|
99
|
+
body=item.get("articleBody") or item.get("description"),
|
|
100
|
+
timestamp=ts(item.get("datePublished"))) # fmt: skip
|
|
101
|
+
return self.doc(f, url, **parse_og_social(f.text))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
CHANNEL = MetaSocialChannel
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""News: Google News RSS search (date-windowed) and the Techmeme river, no keys.
|
|
2
|
+
|
|
3
|
+
Google News items link to news.google.com/rss/articles/<id>, which only redirects with JavaScript.
|
|
4
|
+
decode() resolves the publisher URL the way that page does: read its signature + timestamp, then
|
|
5
|
+
ask the batchexecute endpoint for the target.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
import xml.etree.ElementTree as ET
|
|
12
|
+
from datetime import UTC, datetime, timedelta
|
|
13
|
+
from email.utils import parsedate_to_datetime
|
|
14
|
+
from urllib.parse import quote
|
|
15
|
+
|
|
16
|
+
from curl_cffi.requests.exceptions import RequestException
|
|
17
|
+
|
|
18
|
+
from agent_wire.channels.base import Channel, WireUnsupported, html_text, listing
|
|
19
|
+
from agent_wire.engine.waterfall import WireFetchError
|
|
20
|
+
from agent_wire.models import WireSearchHit
|
|
21
|
+
|
|
22
|
+
GNEWS = "https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
|
|
23
|
+
GN_ARTICLE = re.compile(r"^https://news\.google\.com/(?:rss/)?(?:articles|read)/([\w-]+)")
|
|
24
|
+
BATCH = "https://news.google.com/_/DotsSplashUi/data/batchexecute"
|
|
25
|
+
_SG = re.compile(rb'data-n-a-sg="([^"]+)"')
|
|
26
|
+
_TS = re.compile(rb'data-n-a-ts="(\d+)"')
|
|
27
|
+
DECODE_TIMEOUT_S = 10
|
|
28
|
+
TECHMEME = "https://www.techmeme.com/feed.xml"
|
|
29
|
+
_HREF = re.compile(r'href="(https?://[^"]+)"')
|
|
30
|
+
_WORD = re.compile(r"[a-z0-9+#]+")
|
|
31
|
+
_OUTLET = re.compile(r"\s*\((?:[A-Z][\w.&' -]{1,40})\)\s*$") # "... (Bloomberg)"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _date(text: str | None) -> datetime | None:
|
|
35
|
+
try:
|
|
36
|
+
return parsedate_to_datetime(text) if text else None
|
|
37
|
+
except (TypeError, ValueError):
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def parse_rss_items(content: bytes) -> list[dict]:
|
|
42
|
+
try:
|
|
43
|
+
root = ET.fromstring(content)
|
|
44
|
+
except ET.ParseError as e:
|
|
45
|
+
raise WireUnsupported(f"news feed unreadable ({e})") from e
|
|
46
|
+
out = []
|
|
47
|
+
for it in root.iter("item"):
|
|
48
|
+
src = it.find("source")
|
|
49
|
+
out.append({"title": (it.findtext("title") or "").strip(), "url": it.findtext("link") or "",
|
|
50
|
+
"date": _date(it.findtext("pubDate")), "desc": it.findtext("description") or "",
|
|
51
|
+
"source": src.text if src is not None else None}) # fmt: skip
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def gnews_hits(items: list[dict]) -> list[WireSearchHit]:
|
|
56
|
+
hits = []
|
|
57
|
+
for it in items:
|
|
58
|
+
title, outlet = it["title"], it["source"]
|
|
59
|
+
if outlet and title.endswith(f" - {outlet}"):
|
|
60
|
+
title = title[: -len(outlet) - 3]
|
|
61
|
+
hits.append(WireSearchHit(url=it["url"], title=title, snippet=outlet or "", source="news",
|
|
62
|
+
timestamp=it["date"], author=outlet)) # fmt: skip
|
|
63
|
+
return hits
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def techmeme_hits(items: list[dict]) -> list[WireSearchHit]:
|
|
67
|
+
hits = []
|
|
68
|
+
for it in items:
|
|
69
|
+
links = [u for u in _HREF.findall(it["desc"]) if "techmeme.com" not in u]
|
|
70
|
+
hits.append(WireSearchHit(url=it["url"], title=_OUTLET.sub("", it["title"]), snippet=html_text(it["desc"])[:300],
|
|
71
|
+
source="techmeme", timestamp=it["date"],
|
|
72
|
+
link=links[0] if links else None)) # fmt: skip
|
|
73
|
+
return hits
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def batch_form(gid: str, stamp: str, sig: str) -> dict:
|
|
77
|
+
inner = ["garturlreq", [["X", "X", ["X", "X"], None, None, 1, 1, "US:en", None, 1, None, None,
|
|
78
|
+
None, None, None, 0, 1], "X", "X", 1, [1, 1, 1], 1, 1, None, 0, 0, None, 0],
|
|
79
|
+
gid, int(stamp), sig] # fmt: skip
|
|
80
|
+
return {"f.req": json.dumps([[["Fbv4je", json.dumps(inner), None, "generic"]]])}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def parse_batch(text: str) -> str | None:
|
|
84
|
+
try:
|
|
85
|
+
rows = json.loads(text.split("\n\n", 1)[1])
|
|
86
|
+
url = json.loads(rows[0][2])[1]
|
|
87
|
+
except (IndexError, ValueError, TypeError):
|
|
88
|
+
return None
|
|
89
|
+
return url if isinstance(url, str) and url.startswith("http") else None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def matches(hit: WireSearchHit, query: str) -> bool:
|
|
93
|
+
"""Every content word of the query appears in the title or snippet."""
|
|
94
|
+
words = [w for w in _WORD.findall(query.lower()) if len(w) > 1]
|
|
95
|
+
text = f"{hit.title} {hit.snippet}".lower()
|
|
96
|
+
return bool(words) and all(w in text for w in words)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class NewsChannel(Channel):
|
|
100
|
+
name = "news"
|
|
101
|
+
|
|
102
|
+
async def _items(self, url: str) -> list[dict]:
|
|
103
|
+
f = await self.fetch(url, archive=False, browser=False)
|
|
104
|
+
return parse_rss_items(f.content)
|
|
105
|
+
|
|
106
|
+
async def techmeme(self, query: str | None = None, since=None) -> list[WireSearchHit]:
|
|
107
|
+
hits = techmeme_hits(await self._items(TECHMEME))
|
|
108
|
+
if since:
|
|
109
|
+
hits = [h for h in hits if h.timestamp is None or h.timestamp >= since]
|
|
110
|
+
return [h for h in hits if matches(h, query)] if query else hits
|
|
111
|
+
|
|
112
|
+
async def search(self, query, *, limit=5, sort="relevance", since=None):
|
|
113
|
+
q = query
|
|
114
|
+
if since:
|
|
115
|
+
q = f"{query} when:{max(1, (datetime.now(UTC) - since).days)}d"
|
|
116
|
+
hits = gnews_hits(await self._items(GNEWS.format(q=quote(q))))
|
|
117
|
+
if sort in ("new", "date", "recent"):
|
|
118
|
+
hits.sort(key=lambda h: h.timestamp or datetime.min.replace(tzinfo=UTC), reverse=True)
|
|
119
|
+
try:
|
|
120
|
+
tm = await self.techmeme(query, since)
|
|
121
|
+
except (WireFetchError, WireUnsupported): # Techmeme is a bonus; Google News answered
|
|
122
|
+
tm = []
|
|
123
|
+
out = (tm + hits)[:limit]
|
|
124
|
+
real = await asyncio.gather(*(self.decode(h.url) for h in out))
|
|
125
|
+
return [h.model_copy(update={"url": u}) for h, u in zip(out, real, strict=True)]
|
|
126
|
+
|
|
127
|
+
async def decode(self, url: str) -> str:
|
|
128
|
+
"""Publisher URL behind a Google News link; the link itself when it can't be resolved."""
|
|
129
|
+
m = GN_ARTICLE.match(url)
|
|
130
|
+
if not m or self.ctx.pool is None:
|
|
131
|
+
return url
|
|
132
|
+
try:
|
|
133
|
+
return await asyncio.wait_for(self._decode(m[1]), DECODE_TIMEOUT_S) or url
|
|
134
|
+
except (RequestException, TimeoutError, ValueError):
|
|
135
|
+
return url
|
|
136
|
+
|
|
137
|
+
async def _decode(self, gid: str) -> str | None:
|
|
138
|
+
page = await self.ctx.pool.get(f"https://news.google.com/rss/articles/{gid}")
|
|
139
|
+
sig, stamp = _SG.search(page.content), _TS.search(page.content)
|
|
140
|
+
if not (sig and stamp):
|
|
141
|
+
return None
|
|
142
|
+
r = await self.ctx.pool.get(
|
|
143
|
+
BATCH, method="POST", data=batch_form(gid, stamp[1].decode(), sig[1].decode()),
|
|
144
|
+
headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"},
|
|
145
|
+
) # fmt: skip
|
|
146
|
+
return parse_batch(r.content.decode("utf-8", "replace")) if r.ok else None
|
|
147
|
+
|
|
148
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
149
|
+
from agent_wire.router import route_url # router imports the channel registry
|
|
150
|
+
|
|
151
|
+
target = await self.decode(url)
|
|
152
|
+
if target == url:
|
|
153
|
+
raise WireUnsupported(f"news: could not resolve the Google News link {url}")
|
|
154
|
+
return await self.ctx.channel(route_url(target)).read(
|
|
155
|
+
target, query=query, section=section, include_comments=include_comments
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
async def feed(self, target="techmeme", *, sort="hot", limit=10):
|
|
159
|
+
key = target.strip().lower()
|
|
160
|
+
if key in ("techmeme", "tech", ""):
|
|
161
|
+
hits = (await self.techmeme())[:limit]
|
|
162
|
+
title, url = "Techmeme", "https://www.techmeme.com/"
|
|
163
|
+
else:
|
|
164
|
+
hits = await self.search(
|
|
165
|
+
target, limit=limit, since=datetime.now(UTC) - timedelta(days=7)
|
|
166
|
+
)
|
|
167
|
+
title, url = f"News · {target}", GNEWS.format(q=quote(target))
|
|
168
|
+
items = [{"title": h.title, "url": h.link or h.url, "snippet": h.snippet,
|
|
169
|
+
"meta": [h.timestamp.date().isoformat() if h.timestamp else None]} for h in hits] # fmt: skip
|
|
170
|
+
return self.doc(None, url, title=title, body=listing(items))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
CHANNEL = NewsChannel
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Polymarket prediction markets (public Gamma API, no key): what money says about a topic."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from urllib.parse import quote, urlsplit
|
|
5
|
+
|
|
6
|
+
from agent_wire.channels.base import Channel, WireUnsupported, listing, ts
|
|
7
|
+
from agent_wire.models import WireSearchHit
|
|
8
|
+
|
|
9
|
+
GAMMA = "https://gamma-api.polymarket.com"
|
|
10
|
+
EVENT = "https://polymarket.com/event/"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _odds(market: dict) -> str:
|
|
14
|
+
"""'Yes 62% · No 38%' from the JSON-in-a-string outcome fields."""
|
|
15
|
+
try:
|
|
16
|
+
names = json.loads(market.get("outcomes") or "[]")
|
|
17
|
+
prices = [float(p) for p in json.loads(market.get("outcomePrices") or "[]")]
|
|
18
|
+
except (ValueError, TypeError):
|
|
19
|
+
return ""
|
|
20
|
+
label = market.get("groupItemTitle")
|
|
21
|
+
odds = " · ".join(f"{n} {round(p * 100)}%" for n, p in zip(names, prices, strict=False))
|
|
22
|
+
return f"{label}: {odds}" if label else odds
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _movement(market: dict) -> str:
|
|
26
|
+
d = market.get("oneWeekPriceChange") or market.get("oneDayPriceChange")
|
|
27
|
+
if not isinstance(d, int | float) or not d:
|
|
28
|
+
return ""
|
|
29
|
+
return f"({'+' if d > 0 else ''}{round(d * 100)} pts)"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def event_summary(event: dict, max_markets: int = 4) -> list[str]:
|
|
33
|
+
markets = sorted(event.get("markets") or [], key=lambda m: -float(m.get("volume") or 0))
|
|
34
|
+
lines = (" ".join(filter(None, (_odds(m), _movement(m)))) for m in markets[:max_markets])
|
|
35
|
+
return [line for line in lines if line]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def event_hit(event: dict) -> WireSearchHit:
|
|
39
|
+
vol24 = int(float(event.get("volume24hr") or 0))
|
|
40
|
+
return WireSearchHit(
|
|
41
|
+
url=EVENT + event.get("slug", ""), title=event.get("title") or "",
|
|
42
|
+
snippet=" | ".join(event_summary(event, 3)), source="polymarket",
|
|
43
|
+
rank_score=float(vol24), # undated: a market is live until it resolves, so "freshness" is moot
|
|
44
|
+
engagement={"volume": int(float(event.get("volume") or 0)), "volume24h": vol24,
|
|
45
|
+
"liquidity": int(float(event.get("liquidity") or 0)),
|
|
46
|
+
"comments": event.get("commentCount") or 0},
|
|
47
|
+
) # fmt: skip
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class PolymarketChannel(Channel):
|
|
51
|
+
name = "polymarket"
|
|
52
|
+
|
|
53
|
+
async def search(self, query, *, limit=5, sort="relevance", since=None):
|
|
54
|
+
data, _ = await self.fetch_json(
|
|
55
|
+
f"{GAMMA}/public-search?q={quote(query)}&limit_per_type={max(limit, 5)}&events_status=active"
|
|
56
|
+
)
|
|
57
|
+
events = [e for e in data.get("events") or [] if not e.get("closed")]
|
|
58
|
+
if since: # markets that resolved before the window opened are history, not news
|
|
59
|
+
events = [e for e in events if (t := ts(e.get("endDate"))) is None or t >= since]
|
|
60
|
+
return [event_hit(e) for e in events[:limit]]
|
|
61
|
+
|
|
62
|
+
async def feed(self, target="trending", *, sort="hot", limit=10):
|
|
63
|
+
if target.strip().lower() not in ("trending", "hot", "top", "movers", ""):
|
|
64
|
+
hits = await self.search(target, limit=limit)
|
|
65
|
+
body = listing([{"title": h.title, "url": h.url, "snippet": h.snippet,
|
|
66
|
+
"meta": [f"${h.engagement['volume24h']:,} 24h"]} for h in hits]) # fmt: skip
|
|
67
|
+
return self.doc(None, f"https://polymarket.com/search?q={quote(target)}",
|
|
68
|
+
title=f"Polymarket · {target}", body=body) # fmt: skip
|
|
69
|
+
data, f = await self.fetch_json(
|
|
70
|
+
f"{GAMMA}/events?limit={limit}&active=true&closed=false&order=volume24hr&ascending=false"
|
|
71
|
+
)
|
|
72
|
+
items = [{"title": e.get("title"), "url": EVENT + e.get("slug", ""),
|
|
73
|
+
"meta": [f"${int(float(e.get('volume24hr') or 0)):,} 24h"],
|
|
74
|
+
"snippet": " | ".join(event_summary(e, 2))} for e in data] # fmt: skip
|
|
75
|
+
return self.doc(
|
|
76
|
+
f, "https://polymarket.com/", title="Polymarket · trending", body=listing(items)
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
async def trending_hits(self, limit=10) -> list[WireSearchHit]:
|
|
80
|
+
data, _ = await self.fetch_json(
|
|
81
|
+
f"{GAMMA}/events?limit={limit}&active=true&closed=false&order=volume24hr&ascending=false"
|
|
82
|
+
)
|
|
83
|
+
return [event_hit(e) for e in data]
|
|
84
|
+
|
|
85
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
86
|
+
parts = [p for p in urlsplit(url).path.split("/") if p]
|
|
87
|
+
if not parts: # the front page: trending markets
|
|
88
|
+
return await self.feed("trending")
|
|
89
|
+
if len(parts) < 2 or parts[0] not in ("event", "market"):
|
|
90
|
+
raise WireUnsupported("polymarket: only /event/<slug> URLs and the front page")
|
|
91
|
+
data, f = await self.fetch_json(f"{GAMMA}/events?slug={quote(parts[1])}")
|
|
92
|
+
if not data:
|
|
93
|
+
raise WireUnsupported(f"polymarket: no event {parts[1]!r}")
|
|
94
|
+
e = data[0]
|
|
95
|
+
lines = [f"- {line}" for line in event_summary(e, 20)]
|
|
96
|
+
body = "\n".join([e.get("description") or "", "", "Markets:", *lines]).strip()
|
|
97
|
+
hit = event_hit(e)
|
|
98
|
+
return self.doc(
|
|
99
|
+
f, url, title=e.get("title"), timestamp=ts(e.get("startDate")), body=body,
|
|
100
|
+
metadata={"volume": hit.engagement["volume"], "volume_24h": hit.engagement["volume24h"],
|
|
101
|
+
"ends": (e.get("endDate") or "")[:10] or None},
|
|
102
|
+
) # fmt: skip
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
CHANNEL = PolymarketChannel
|