agent-wire-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_wire/__init__.py +1 -0
- agent_wire/__main__.py +3 -0
- agent_wire/assets/SKILL.md +26 -0
- agent_wire/assets/cursor_rule.mdc +13 -0
- agent_wire/assets/hermes_skill.md +28 -0
- agent_wire/assets/windsurf_rules.md +14 -0
- agent_wire/assets/wire-worker.md +20 -0
- agent_wire/auth/__init__.py +0 -0
- agent_wire/auth/browser_paths.py +143 -0
- agent_wire/auth/cookie_vault.py +181 -0
- agent_wire/auth/rate_limiter.py +26 -0
- agent_wire/cache/__init__.py +0 -0
- agent_wire/cache/ring_buffer.py +94 -0
- agent_wire/cache/sqlite_ledger.py +132 -0
- agent_wire/channels/__init__.py +41 -0
- agent_wire/channels/base.py +249 -0
- agent_wire/channels/tier1_global/__init__.py +0 -0
- agent_wire/channels/tier1_global/github.py +371 -0
- agent_wire/channels/tier1_global/hackernews.py +135 -0
- agent_wire/channels/tier1_global/reddit.py +371 -0
- agent_wire/channels/tier1_global/rss.py +129 -0
- agent_wire/channels/tier1_global/search.py +160 -0
- agent_wire/channels/tier1_global/stackoverflow.py +94 -0
- agent_wire/channels/tier1_global/web.py +98 -0
- agent_wire/channels/tier1_global/wikipedia.py +91 -0
- agent_wire/channels/tier1_global/x_twitter.py +380 -0
- agent_wire/channels/tier1_global/youtube.py +223 -0
- agent_wire/channels/tier2_pro/__init__.py +0 -0
- agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
- agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
- agent_wire/channels/tier2_pro/devto.py +114 -0
- agent_wire/channels/tier2_pro/fediverse.py +181 -0
- agent_wire/channels/tier2_pro/hubs.py +161 -0
- agent_wire/channels/tier2_pro/linkedin.py +146 -0
- agent_wire/channels/tier2_pro/markets.py +207 -0
- agent_wire/channels/tier2_pro/meta_social.py +104 -0
- agent_wire/channels/tier2_pro/news.py +173 -0
- agent_wire/channels/tier2_pro/polymarket.py +105 -0
- agent_wire/channels/tier2_pro/publishing.py +84 -0
- agent_wire/channels/tier2_pro/stocktwits.py +79 -0
- agent_wire/channels/tier3_regional/bilibili.py +171 -0
- agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
- agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
- agent_wire/channels/tier3_regional/v2ex.py +70 -0
- agent_wire/channels/tier3_regional/wechat.py +67 -0
- agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
- agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
- agent_wire/channels/tier3_regional/xueqiu.py +77 -0
- agent_wire/channels/tier3_regional/zhihu.py +57 -0
- agent_wire/cli.py +403 -0
- agent_wire/compression/__init__.py +0 -0
- agent_wire/compression/anchor_slicer.py +45 -0
- agent_wire/compression/bm25_slicer.py +151 -0
- agent_wire/compression/dam_formatter.py +74 -0
- agent_wire/compression/url_compactor.py +45 -0
- agent_wire/config.py +52 -0
- agent_wire/engine/__init__.py +0 -0
- agent_wire/engine/challenge.py +58 -0
- agent_wire/engine/http_pool.py +136 -0
- agent_wire/engine/stealth_browser.py +90 -0
- agent_wire/engine/waterfall.py +159 -0
- agent_wire/firewall/__init__.py +0 -0
- agent_wire/firewall/inbound_shield.py +116 -0
- agent_wire/firewall/outbound_privacy.py +120 -0
- agent_wire/installers/__init__.py +0 -0
- agent_wire/installers/client_installer.py +315 -0
- agent_wire/media/__init__.py +0 -0
- agent_wire/media/audio_transcriber.py +77 -0
- agent_wire/media/subtitle_cleaner.py +102 -0
- agent_wire/models.py +60 -0
- agent_wire/research/__init__.py +1 -0
- agent_wire/research/brief.py +227 -0
- agent_wire/research/briefing.py +41 -0
- agent_wire/research/cluster.py +128 -0
- agent_wire/research/engine.py +632 -0
- agent_wire/research/library.py +74 -0
- agent_wire/research/planner.py +232 -0
- agent_wire/research/scoring.py +89 -0
- agent_wire/research/store.py +149 -0
- agent_wire/research/watch.py +70 -0
- agent_wire/router.py +107 -0
- agent_wire/server.py +468 -0
- agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
- agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
- agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
- agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
- agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"""GitHub: repo overview, blobs with #L slicing, issues/PRs, gists, users, search, trending."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
from functools import cache
|
|
8
|
+
from urllib.parse import quote, urlsplit
|
|
9
|
+
|
|
10
|
+
from selectolax.lexbor import LexborHTMLParser
|
|
11
|
+
|
|
12
|
+
from agent_wire.channels.base import Channel, WireUnsupported, compact_count, listing, ts
|
|
13
|
+
from agent_wire.compression.anchor_slicer import parse_line_anchor, slice_lines, slice_section
|
|
14
|
+
from agent_wire.models import WireComment, WireDocument, WireSearchHit
|
|
15
|
+
|
|
16
|
+
API = "https://api.github.com"
|
|
17
|
+
RAW = "https://raw.githubusercontent.com"
|
|
18
|
+
_LANG = {"py": "python", "js": "javascript", "ts": "typescript", "rs": "rust", "go": "go",
|
|
19
|
+
"rb": "ruby", "java": "java", "kt": "kotlin", "c": "c", "h": "c", "cpp": "cpp",
|
|
20
|
+
"cs": "csharp", "sh": "bash", "md": "markdown", "toml": "toml", "yml": "yaml",
|
|
21
|
+
"yaml": "yaml", "json": "json", "tsx": "tsx", "jsx": "jsx", "swift": "swift"} # fmt: skip
|
|
22
|
+
_SKIP_PATHS = frozenset({"orgs", "settings", "notifications", "login", "marketplace", "topics"})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@cache
|
|
26
|
+
def github_token() -> str | None:
|
|
27
|
+
if token := os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN"):
|
|
28
|
+
return token
|
|
29
|
+
if gh := shutil.which("gh"):
|
|
30
|
+
try:
|
|
31
|
+
out = subprocess.run(
|
|
32
|
+
[gh, "auth", "token"], capture_output=True, text=True, timeout=3, check=False
|
|
33
|
+
)
|
|
34
|
+
return out.stdout.strip() or None
|
|
35
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
36
|
+
return None
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _repo_hit(r: dict) -> WireSearchHit:
|
|
41
|
+
return WireSearchHit(
|
|
42
|
+
url=r["html_url"], title=r["full_name"], snippet=r.get("description") or "",
|
|
43
|
+
source="github", rank_score=float(r.get("stargazers_count", 0)),
|
|
44
|
+
timestamp=ts(r.get("pushed_at")), author=(r.get("owner") or {}).get("login"),
|
|
45
|
+
engagement={"stars": r.get("stargazers_count") or 0, "forks": r.get("forks_count") or 0},
|
|
46
|
+
) # fmt: skip
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _issue_hit(i: dict) -> WireSearchHit:
|
|
50
|
+
kind = "PR" if i.get("pull_request") else "issue"
|
|
51
|
+
repo = i.get("repository_url", "").removeprefix(f"{API}/repos/")
|
|
52
|
+
return WireSearchHit(
|
|
53
|
+
url=i["html_url"], title=f"{repo}: {i.get('title', '')}",
|
|
54
|
+
snippet=f"{kind} · {i.get('state')} · {i.get('comments', 0)} comments · "
|
|
55
|
+
+ (i.get("body") or "")[:200],
|
|
56
|
+
source="github", rank_score=float(i.get("comments") or 0),
|
|
57
|
+
timestamp=ts(i.get("created_at")), author=(i.get("user") or {}).get("login"),
|
|
58
|
+
engagement={"comments": i.get("comments") or 0,
|
|
59
|
+
"reactions": (i.get("reactions") or {}).get("total_count", 0)},
|
|
60
|
+
) # fmt: skip
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class GitHubChannel(Channel):
|
|
64
|
+
name = "github"
|
|
65
|
+
|
|
66
|
+
def _headers(self, raw: bool = False) -> dict[str, str]:
|
|
67
|
+
h = {
|
|
68
|
+
"Accept": "application/vnd.github.raw" if raw else "application/vnd.github+json",
|
|
69
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
70
|
+
}
|
|
71
|
+
if token := github_token():
|
|
72
|
+
h["Authorization"] = f"Bearer {token}"
|
|
73
|
+
return h
|
|
74
|
+
|
|
75
|
+
async def api(self, path: str):
|
|
76
|
+
data, f = await self.fetch_json(f"{API}{path}", headers=self._headers())
|
|
77
|
+
return data, f
|
|
78
|
+
|
|
79
|
+
async def raw(self, url: str) -> str:
|
|
80
|
+
f = await self.fetch(url, headers=self._headers(raw=True), archive=False, browser=False)
|
|
81
|
+
return f.text
|
|
82
|
+
|
|
83
|
+
# ---- read ------------------------------------------------------------------------------
|
|
84
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
85
|
+
parts = urlsplit(url)
|
|
86
|
+
host, segs = parts.hostname or "", [s for s in parts.path.split("/") if s]
|
|
87
|
+
anchor = section or parts.fragment or None
|
|
88
|
+
if host.endswith("githubusercontent.com"):
|
|
89
|
+
return await self._blob_raw(url, segs[0], segs[1], "/".join(segs[3:]), url)
|
|
90
|
+
if host == "gist.github.com" and len(segs) >= 2:
|
|
91
|
+
return await self._gist(url, segs[-1])
|
|
92
|
+
if not segs or segs[0] in _SKIP_PATHS:
|
|
93
|
+
return await self.ctx.channel("web").read(url, query=query, section=section)
|
|
94
|
+
if len(segs) == 1:
|
|
95
|
+
return await self._user(url, segs[0])
|
|
96
|
+
owner, repo = segs[0], segs[1].removesuffix(".git")
|
|
97
|
+
kind = segs[2] if len(segs) > 2 else ""
|
|
98
|
+
if not kind:
|
|
99
|
+
return await self._repo(url, owner, repo, anchor)
|
|
100
|
+
if kind == "blob" and len(segs) > 4:
|
|
101
|
+
raw = f"{RAW}/{owner}/{repo}/{segs[3]}/{'/'.join(segs[4:])}"
|
|
102
|
+
return await self._blob_raw(raw, owner, repo, "/".join(segs[4:]), url)
|
|
103
|
+
if kind == "tree" and len(segs) > 3:
|
|
104
|
+
return await self._tree(url, owner, repo, segs[3], "/".join(segs[4:]))
|
|
105
|
+
if kind in ("issues", "pull") and len(segs) > 3 and segs[3].isdigit():
|
|
106
|
+
return await self._issue(url, owner, repo, int(segs[3]), include_comments)
|
|
107
|
+
if kind == "releases":
|
|
108
|
+
return await self._releases(url, owner, repo)
|
|
109
|
+
if kind == "commit" and len(segs) > 3:
|
|
110
|
+
return await self._commit(url, owner, repo, segs[3])
|
|
111
|
+
return await self.ctx.channel("web").read(url, query=query, section=section)
|
|
112
|
+
|
|
113
|
+
async def _repo(self, url, owner, repo, anchor):
|
|
114
|
+
(meta, f), (tree, _), readme = await asyncio.gather(
|
|
115
|
+
self.api(f"/repos/{owner}/{repo}"),
|
|
116
|
+
self.api(f"/repos/{owner}/{repo}/contents"),
|
|
117
|
+
self._readme(owner, repo),
|
|
118
|
+
)
|
|
119
|
+
files = " ".join(
|
|
120
|
+
f"`{e['name']}/`" if e.get("type") == "dir" else f"`{e['name']}`"
|
|
121
|
+
for e in (tree if isinstance(tree, list) else [])
|
|
122
|
+
)
|
|
123
|
+
if anchor and readme:
|
|
124
|
+
readme = slice_section(readme, anchor) or readme
|
|
125
|
+
stats = {
|
|
126
|
+
"stars": meta.get("stargazers_count"), "forks": meta.get("forks_count"),
|
|
127
|
+
"language": meta.get("language"), "license": (meta.get("license") or {}).get("spdx_id"),
|
|
128
|
+
"open_issues": meta.get("open_issues_count"),
|
|
129
|
+
"topics": ", ".join(meta.get("topics") or []) or None,
|
|
130
|
+
"pushed": (meta.get("pushed_at") or "")[:10] or None,
|
|
131
|
+
} # fmt: skip
|
|
132
|
+
body = "\n\n".join(
|
|
133
|
+
p for p in (meta.get("description"), f"## Files\n{files}" if files else "",
|
|
134
|
+
f"## README\n{readme}" if readme else "") if p
|
|
135
|
+
) # fmt: skip
|
|
136
|
+
return self.doc(f, url, title=meta.get("full_name", f"{owner}/{repo}"),
|
|
137
|
+
author_handle=owner, body=body, metadata=stats) # fmt: skip
|
|
138
|
+
|
|
139
|
+
async def _readme(self, owner, repo) -> str:
|
|
140
|
+
try:
|
|
141
|
+
f = await self.fetch(f"{API}/repos/{owner}/{repo}/readme",
|
|
142
|
+
headers=self._headers(raw=True), archive=False, browser=False) # fmt: skip
|
|
143
|
+
return f.text
|
|
144
|
+
except Exception: # noqa: BLE001 - a repo without a README is still a repo
|
|
145
|
+
return ""
|
|
146
|
+
|
|
147
|
+
async def _blob_raw(self, raw_url, owner, repo, path, url):
|
|
148
|
+
f = await self.fetch(raw_url, headers=self._headers(raw=True), archive=False, browser=False)
|
|
149
|
+
text = f.text
|
|
150
|
+
lines = parse_line_anchor(url)
|
|
151
|
+
if lines:
|
|
152
|
+
text = slice_lines(text, *lines)
|
|
153
|
+
lang = _LANG.get(path.rsplit(".", 1)[-1].lower(), "")
|
|
154
|
+
body = text if lang == "markdown" and not lines else f"```{lang}\n{text}\n```"
|
|
155
|
+
meta = {"lines": f"{lines[0]}-{lines[1]}"} if lines else {}
|
|
156
|
+
return self.doc(f, url, title=f"{owner}/{repo}: {path}", body=body, metadata=meta)
|
|
157
|
+
|
|
158
|
+
async def _tree(self, url, owner, repo, ref, path):
|
|
159
|
+
data, f = await self.api(f"/repos/{owner}/{repo}/contents/{quote(path)}?ref={quote(ref)}")
|
|
160
|
+
entries = data if isinstance(data, list) else [data]
|
|
161
|
+
body = "\n".join(
|
|
162
|
+
f"- `{e['name']}{'/' if e.get('type') == 'dir' else ''}`"
|
|
163
|
+
+ (f" ({e['size']} B)" if e.get("type") == "file" else "")
|
|
164
|
+
for e in entries
|
|
165
|
+
)
|
|
166
|
+
return self.doc(f, url, title=f"{owner}/{repo}/{path or ''} @ {ref}", body=body)
|
|
167
|
+
|
|
168
|
+
async def _issue(self, url, owner, repo, num, include_comments):
|
|
169
|
+
issue, f = await self.api(f"/repos/{owner}/{repo}/issues/{num}")
|
|
170
|
+
meta = {
|
|
171
|
+
"state": issue.get("state"),
|
|
172
|
+
"labels": ", ".join(lb["name"] for lb in issue.get("labels", [])) or None,
|
|
173
|
+
"comments": issue.get("comments"),
|
|
174
|
+
}
|
|
175
|
+
if "pull_request" in issue:
|
|
176
|
+
pr, _ = await self.api(f"/repos/{owner}/{repo}/pulls/{num}")
|
|
177
|
+
meta.update(merged=pr.get("merged"), additions=pr.get("additions"),
|
|
178
|
+
deletions=pr.get("deletions"), files=pr.get("changed_files")) # fmt: skip
|
|
179
|
+
comments = []
|
|
180
|
+
if include_comments and issue.get("comments"):
|
|
181
|
+
raw, _ = await self.api(f"/repos/{owner}/{repo}/issues/{num}/comments?per_page=100")
|
|
182
|
+
author = (issue.get("user") or {}).get("login")
|
|
183
|
+
comments = [
|
|
184
|
+
WireComment(
|
|
185
|
+
id=str(c["id"]), body=c.get("body") or "",
|
|
186
|
+
author_handle=(c.get("user") or {}).get("login"),
|
|
187
|
+
timestamp=ts(c.get("created_at")),
|
|
188
|
+
score=(c.get("reactions") or {}).get("total_count", 0),
|
|
189
|
+
is_op=(c.get("user") or {}).get("login") == author,
|
|
190
|
+
)
|
|
191
|
+
for c in raw
|
|
192
|
+
] # fmt: skip
|
|
193
|
+
return self.doc(
|
|
194
|
+
f, url, title=f"#{num} {issue.get('title', '')}",
|
|
195
|
+
author_handle=(issue.get("user") or {}).get("login"),
|
|
196
|
+
timestamp=ts(issue.get("created_at")), body=issue.get("body") or "",
|
|
197
|
+
score=(issue.get("reactions") or {}).get("total_count"),
|
|
198
|
+
comments=comments, metadata=meta,
|
|
199
|
+
) # fmt: skip
|
|
200
|
+
|
|
201
|
+
async def _releases(self, url, owner, repo):
|
|
202
|
+
data, f = await self.api(f"/repos/{owner}/{repo}/releases?per_page=5")
|
|
203
|
+
body = "\n\n".join(
|
|
204
|
+
f"## {r.get('name') or r.get('tag_name')} ({(r.get('published_at') or '')[:10]})\n"
|
|
205
|
+
f"{r.get('body') or ''}"
|
|
206
|
+
for r in data
|
|
207
|
+
)
|
|
208
|
+
return self.doc(f, url, title=f"{owner}/{repo} releases", body=body)
|
|
209
|
+
|
|
210
|
+
async def _commit(self, url, owner, repo, sha):
|
|
211
|
+
c, f = await self.api(f"/repos/{owner}/{repo}/commits/{sha}")
|
|
212
|
+
files = "\n".join(
|
|
213
|
+
f"- `{x['filename']}` +{x.get('additions', 0)} -{x.get('deletions', 0)}"
|
|
214
|
+
for x in c.get("files", [])[:50]
|
|
215
|
+
)
|
|
216
|
+
stats = c.get("stats") or {}
|
|
217
|
+
return self.doc(
|
|
218
|
+
f, url, title=(c.get("commit", {}).get("message") or sha).split("\n")[0],
|
|
219
|
+
author_handle=(c.get("author") or {}).get("login"),
|
|
220
|
+
timestamp=ts(c.get("commit", {}).get("author", {}).get("date")),
|
|
221
|
+
body=f"{c.get('commit', {}).get('message', '')}\n\n## Files\n{files}",
|
|
222
|
+
metadata={"additions": stats.get("additions"), "deletions": stats.get("deletions")},
|
|
223
|
+
) # fmt: skip
|
|
224
|
+
|
|
225
|
+
async def _gist(self, url, gist_id):
|
|
226
|
+
g, f = await self.api(f"/gists/{gist_id}")
|
|
227
|
+
body = "\n\n".join(
|
|
228
|
+
f"## {name}\n```{_LANG.get(name.rsplit('.', 1)[-1].lower(), '')}\n{fl.get('content', '')}\n```"
|
|
229
|
+
for name, fl in (g.get("files") or {}).items()
|
|
230
|
+
)
|
|
231
|
+
return self.doc(f, url, title=g.get("description") or gist_id,
|
|
232
|
+
author_handle=(g.get("owner") or {}).get("login"), body=body) # fmt: skip
|
|
233
|
+
|
|
234
|
+
async def _user(self, url, login):
|
|
235
|
+
(u, f), (repos, _) = await asyncio.gather(
|
|
236
|
+
self.api(f"/users/{login}"), self.api(f"/users/{login}/repos?sort=updated&per_page=15")
|
|
237
|
+
)
|
|
238
|
+
body = (u.get("bio") or "") + "\n\n## Repositories\n" + _repo_list(repos)
|
|
239
|
+
return self.doc(f, url, title=u.get("name") or login, author_handle=login, body=body,
|
|
240
|
+
metadata={"followers": u.get("followers"), "public_repos": u.get("public_repos")}) # fmt: skip
|
|
241
|
+
|
|
242
|
+
# ---- search / feed / me ----------------------------------------------------------------
|
|
243
|
+
async def search(self, query, *, limit=5, sort="relevance", since=None):
|
|
244
|
+
order = "&sort=stars" if sort in ("top", "stars") else ""
|
|
245
|
+
if since: # research window: repos active lately plus issues/discussions opened lately
|
|
246
|
+
day = since.date().isoformat()
|
|
247
|
+
repos, _ = await self.api(f"/search/repositories?q={quote(query)}+pushed:>{day}"
|
|
248
|
+
f"&per_page={limit}&sort=stars") # fmt: skip
|
|
249
|
+
issues, _ = await self.api(f"/search/issues?q={quote(query)}+created:>{day}"
|
|
250
|
+
f"&per_page={limit}&sort=comments") # fmt: skip
|
|
251
|
+
return [_repo_hit(r) for r in repos.get("items", [])[:limit]] + [
|
|
252
|
+
_issue_hit(i) for i in issues.get("items", [])[:limit]
|
|
253
|
+
]
|
|
254
|
+
data, _ = await self.api(f"/search/repositories?q={quote(query)}&per_page={limit}{order}")
|
|
255
|
+
return [_repo_hit(r) for r in data.get("items", [])[:limit]]
|
|
256
|
+
|
|
257
|
+
async def person(self, user: str, *, since=None, limit=10) -> WireDocument:
|
|
258
|
+
"""GitHub person mode: recent PR velocity, merged count, top and freshest repos."""
|
|
259
|
+
user = user.lstrip("@").strip("/")
|
|
260
|
+
q = f"author:{user}+type:pr" + (f"+created:>{since.date().isoformat()}" if since else "")
|
|
261
|
+
prs, f = await self.api(f"/search/issues?q={q}&per_page=30&sort=created")
|
|
262
|
+
repos, _ = await self.api(f"/users/{user}/repos?sort=pushed&per_page=30")
|
|
263
|
+
merged = sum(
|
|
264
|
+
1 for p in prs.get("items", []) if (p.get("pull_request") or {}).get("merged_at")
|
|
265
|
+
)
|
|
266
|
+
top = sorted(repos, key=lambda r: -r.get("stargazers_count", 0))[:limit]
|
|
267
|
+
lines = [f"PRs opened in window: {prs.get('total_count', 0)} · merged (sampled): {merged}", "",
|
|
268
|
+
"Recent PRs:"] # fmt: skip
|
|
269
|
+
lines += [f"- [{p['title']}]({p['html_url']}) · {p.get('state')} · {(p.get('created_at') or '')[:10]}"
|
|
270
|
+
for p in prs.get("items", [])[:limit]] # fmt: skip
|
|
271
|
+
lines += ["", "Top repos:"] + [
|
|
272
|
+
f"- [{r['full_name']}]({r['html_url']}) · ★{r.get('stargazers_count', 0)} · "
|
|
273
|
+
f"pushed {(r.get('pushed_at') or '')[:10]} · {r.get('description') or ''}"
|
|
274
|
+
for r in top
|
|
275
|
+
]
|
|
276
|
+
return self.doc(f, f"https://github.com/{user}", title=f"GitHub · {user}",
|
|
277
|
+
author_handle=user, body="\n".join(lines),
|
|
278
|
+
metadata={"prs": prs.get("total_count", 0), "merged_sample": merged}) # fmt: skip
|
|
279
|
+
|
|
280
|
+
async def feed(self, target, *, sort="hot", limit=10):
|
|
281
|
+
t = target.strip().strip("/").removeprefix("https://github.com/")
|
|
282
|
+
if t.startswith("trending"):
|
|
283
|
+
url = f"https://github.com/{t}"
|
|
284
|
+
f = await self.fetch(url, archive=False)
|
|
285
|
+
items = parse_trending(f.text)[:limit]
|
|
286
|
+
return self.doc(f, url, title=f"GitHub {t}", body=listing(items))
|
|
287
|
+
if "/" in t:
|
|
288
|
+
data, f = await self.api(f"/repos/{t}/issues?state=open&sort=updated&per_page={limit}")
|
|
289
|
+
items = [{"title": f"#{i['number']} {i['title']}", "url": i["html_url"],
|
|
290
|
+
"meta": [i.get("state"), f"{i.get('comments', 0)} comments"]} for i in data] # fmt: skip
|
|
291
|
+
return self.doc(
|
|
292
|
+
f, f"https://github.com/{t}/issues", title=f"{t} issues", body=listing(items)
|
|
293
|
+
)
|
|
294
|
+
login = t.lstrip("@")
|
|
295
|
+
repos, f = await self.api(f"/users/{login}/repos?sort=updated&per_page={limit}")
|
|
296
|
+
return self.doc(
|
|
297
|
+
f, f"https://github.com/{login}", title=f"@{login} repos", body=_repo_list(repos)
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
async def me(self, target="latest_posts", *, limit=5):
|
|
301
|
+
if not github_token():
|
|
302
|
+
raise WireUnsupported("wire_me github needs GITHUB_TOKEN or `gh auth login`")
|
|
303
|
+
paths = {
|
|
304
|
+
"notifications": f"/notifications?per_page={limit}",
|
|
305
|
+
"latest_posts": f"/user/repos?sort=updated&per_page={limit}",
|
|
306
|
+
"likes": f"/user/starred?per_page={limit}",
|
|
307
|
+
"bookmarks": f"/user/starred?per_page={limit}",
|
|
308
|
+
"prs": f"/search/issues?q={quote('is:pr author:@me is:open')}&per_page={limit}",
|
|
309
|
+
"stats": "/user",
|
|
310
|
+
}
|
|
311
|
+
if target not in paths:
|
|
312
|
+
raise WireUnsupported(f"github targets: {', '.join(paths)}")
|
|
313
|
+
data, f = await self.api(paths[target])
|
|
314
|
+
if target == "stats":
|
|
315
|
+
body = listing([{"title": data.get("login"), "url": data.get("html_url"),
|
|
316
|
+
"meta": [f"{data.get('followers')} followers",
|
|
317
|
+
f"{data.get('public_repos')} public repos"]}]) # fmt: skip
|
|
318
|
+
elif target == "notifications":
|
|
319
|
+
body = listing([{"title": n["subject"]["title"],
|
|
320
|
+
"url": f"https://github.com/{n['repository']['full_name']}",
|
|
321
|
+
"meta": [n["subject"]["type"], n.get("reason"), (n.get("updated_at") or "")[:10]]}
|
|
322
|
+
for n in data]) # fmt: skip
|
|
323
|
+
elif target == "prs":
|
|
324
|
+
body = listing([{"title": i["title"], "url": i["html_url"], "meta": [i.get("state")]}
|
|
325
|
+
for i in data.get("items", [])]) # fmt: skip
|
|
326
|
+
else:
|
|
327
|
+
body = _repo_list(data)
|
|
328
|
+
return self.doc(f, f"https://github.com/{target}", title=f"my github {target}", body=body)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _repo_list(repos) -> str:
|
|
332
|
+
return listing(
|
|
333
|
+
[
|
|
334
|
+
{
|
|
335
|
+
"title": r.get("full_name") or r.get("name"),
|
|
336
|
+
"url": r.get("html_url"),
|
|
337
|
+
"meta": [f"★{r.get('stargazers_count', 0)}", r.get("language")],
|
|
338
|
+
"snippet": r.get("description") or "",
|
|
339
|
+
}
|
|
340
|
+
for r in (repos if isinstance(repos, list) else [])
|
|
341
|
+
]
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def parse_trending(html: str) -> list[dict]:
|
|
346
|
+
items = []
|
|
347
|
+
for row in LexborHTMLParser(html).css("article.Box-row"):
|
|
348
|
+
a = row.css_first("h2 a")
|
|
349
|
+
if not a:
|
|
350
|
+
continue
|
|
351
|
+
path = (a.attributes.get("href") or "").strip()
|
|
352
|
+
desc = row.css_first("p")
|
|
353
|
+
stars = row.css_first('a[href$="/stargazers"]')
|
|
354
|
+
today = row.css_first("span.d-inline-block.float-sm-right")
|
|
355
|
+
lang = row.css_first('[itemprop="programmingLanguage"]')
|
|
356
|
+
items.append(
|
|
357
|
+
{
|
|
358
|
+
"title": path.strip("/"),
|
|
359
|
+
"url": f"https://github.com{path}",
|
|
360
|
+
"meta": [
|
|
361
|
+
f"★{compact_count(stars.text(strip=True))}" if stars else None,
|
|
362
|
+
lang.text(strip=True) if lang else None,
|
|
363
|
+
today.text(strip=True) if today else None,
|
|
364
|
+
],
|
|
365
|
+
"snippet": desc.text(strip=True) if desc else "",
|
|
366
|
+
}
|
|
367
|
+
)
|
|
368
|
+
return items
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
CHANNEL = GitHubChannel
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Hacker News (Algolia API) and Lobsters (.json) threads, search and front pages."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from urllib.parse import parse_qs, quote, urlsplit
|
|
5
|
+
|
|
6
|
+
from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown, listing, ts
|
|
7
|
+
from agent_wire.models import WireComment, WireSearchHit
|
|
8
|
+
|
|
9
|
+
ALGOLIA = "https://hn.algolia.com/api/v1"
|
|
10
|
+
HN_ITEM = "https://news.ycombinator.com/item?id="
|
|
11
|
+
FEEDS = {"top": "front_page", "front_page": "front_page", "ask": "ask_hn", "show": "show_hn",
|
|
12
|
+
"new": "story", "newest": "story", "jobs": "job"} # fmt: skip
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _count(c: dict) -> int:
|
|
16
|
+
return sum(1 + _count(k) for k in c.get("children") or [])
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def parse_hn_comments(children: list, op: str | None, depth: int = 0) -> list[WireComment]:
|
|
20
|
+
out = []
|
|
21
|
+
for c in children or []:
|
|
22
|
+
if not c.get("text") and not c.get("children"):
|
|
23
|
+
continue # deleted leaf
|
|
24
|
+
out.append(
|
|
25
|
+
WireComment(
|
|
26
|
+
id=str(c["id"]), body=html_to_markdown(c.get("text") or "[deleted]"),
|
|
27
|
+
author_handle=c.get("author"), timestamp=ts(c.get("created_at_i")),
|
|
28
|
+
score=_count(c), score_kind="replies", # HN hides comment points
|
|
29
|
+
depth=depth, is_op=bool(op and c.get("author") == op),
|
|
30
|
+
replies=parse_hn_comments(c.get("children"), op, depth + 1),
|
|
31
|
+
)
|
|
32
|
+
) # fmt: skip
|
|
33
|
+
return out
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def parse_lobsters(data: dict) -> dict:
|
|
37
|
+
nodes: dict[str, WireComment] = {}
|
|
38
|
+
roots: list[WireComment] = []
|
|
39
|
+
op = data.get("submitter_user") or {}
|
|
40
|
+
op = op.get("username") if isinstance(op, dict) else op
|
|
41
|
+
for c in data.get("comments", []):
|
|
42
|
+
user = c.get("commenting_user")
|
|
43
|
+
user = user.get("username") if isinstance(user, dict) else user
|
|
44
|
+
node = WireComment(
|
|
45
|
+
id=c["short_id"], body=html_to_markdown(c.get("comment") or ""), author_handle=user,
|
|
46
|
+
score=c.get("score") or 0, depth=(c.get("depth") or 1) - 1,
|
|
47
|
+
timestamp=ts(c.get("created_at")), is_op=user == op,
|
|
48
|
+
) # fmt: skip
|
|
49
|
+
nodes[c["short_id"]] = node
|
|
50
|
+
parent = nodes.get(c.get("parent_comment") or "")
|
|
51
|
+
(parent.replies if parent else roots).append(node)
|
|
52
|
+
body = html_to_markdown(data.get("description") or "")
|
|
53
|
+
if data.get("url"):
|
|
54
|
+
body = f"Link: {data['url']}\n\n{body}".strip()
|
|
55
|
+
return {
|
|
56
|
+
"title": data.get("title"), "author_handle": op, "score": data.get("score"),
|
|
57
|
+
"timestamp": ts(data.get("created_at")), "body": body, "comments": roots,
|
|
58
|
+
"metadata": {"tags": ", ".join(data.get("tags") or []) or None,
|
|
59
|
+
"comments": data.get("comment_count")},
|
|
60
|
+
} # fmt: skip
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class HackerNewsChannel(Channel):
|
|
64
|
+
name = "hackernews"
|
|
65
|
+
|
|
66
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
67
|
+
parts = urlsplit(url)
|
|
68
|
+
if (parts.hostname or "").endswith("lobste.rs"):
|
|
69
|
+
m = re.match(r"/s/(\w+)", parts.path)
|
|
70
|
+
page = re.fullmatch(r"/?(hottest|newest|active|t/[\w,]+)?/?", parts.path)
|
|
71
|
+
if not m and page: # front page, newest, active or a tag: its JSON listing
|
|
72
|
+
data, f = await self.fetch_json(f"https://lobste.rs/{page[1] or 'hottest'}.json")
|
|
73
|
+
items = [{"title": s.get("title"), "url": s.get("short_id_url") or s.get("comments_url"),
|
|
74
|
+
"meta": [f"▲{s.get('score', 0)}", f"{s.get('comment_count', 0)} comments",
|
|
75
|
+
", ".join(s.get("tags") or [])]} for s in data] # fmt: skip
|
|
76
|
+
return self.doc(
|
|
77
|
+
f, url, title=f"Lobsters · {page[1] or 'hottest'}", body=listing(items)
|
|
78
|
+
)
|
|
79
|
+
if not m:
|
|
80
|
+
return await self.ctx.channel("web").read(url, query=query, section=section)
|
|
81
|
+
data, f = await self.fetch_json(f"https://lobste.rs/s/{m[1]}.json")
|
|
82
|
+
return self.doc(f, url, **parse_lobsters(data))
|
|
83
|
+
item_id = parse_qs(parts.query).get("id", [""])[0]
|
|
84
|
+
if not item_id.isdigit():
|
|
85
|
+
return await self.ctx.channel("web").read(url, query=query, section=section)
|
|
86
|
+
item, f = await self.fetch_json(f"{ALGOLIA}/items/{item_id}")
|
|
87
|
+
body = html_to_markdown(item.get("text") or "")
|
|
88
|
+
if item.get("url"):
|
|
89
|
+
body = f"Link: {item['url']}\n\n{body}".strip()
|
|
90
|
+
comments = (
|
|
91
|
+
parse_hn_comments(item.get("children"), item.get("author")) if include_comments else []
|
|
92
|
+
)
|
|
93
|
+
return self.doc(
|
|
94
|
+
f, url, title=item.get("title") or f"HN comment {item_id}",
|
|
95
|
+
author_handle=item.get("author"), timestamp=ts(item.get("created_at_i")),
|
|
96
|
+
score=item.get("points"), body=body, comments=comments,
|
|
97
|
+
metadata={"comments": _count(item)},
|
|
98
|
+
) # fmt: skip
|
|
99
|
+
|
|
100
|
+
async def search(self, query, *, limit=5, sort="relevance", since=None):
|
|
101
|
+
endpoint = "search_by_date" if sort in ("new", "date", "recent") else "search"
|
|
102
|
+
window = f"&numericFilters=created_at_i>{int(since.timestamp())}" if since else ""
|
|
103
|
+
data, _ = await self.fetch_json(
|
|
104
|
+
f"{ALGOLIA}/{endpoint}?query={quote(query)}&tags=story&hitsPerPage={limit}{window}"
|
|
105
|
+
)
|
|
106
|
+
return [
|
|
107
|
+
WireSearchHit(
|
|
108
|
+
url=f"{HN_ITEM}{h['objectID']}", title=h.get("title") or "",
|
|
109
|
+
snippet=f"{h.get('points', 0)} points · {h.get('num_comments', 0)} comments"
|
|
110
|
+
+ (f" · {h['url']}" if h.get("url") else ""),
|
|
111
|
+
source="hn", rank_score=float(h.get("points") or 0),
|
|
112
|
+
timestamp=ts(h.get("created_at_i")), author=h.get("author"),
|
|
113
|
+
engagement={"score": h.get("points") or 0, "comments": h.get("num_comments") or 0},
|
|
114
|
+
link=h.get("url"),
|
|
115
|
+
)
|
|
116
|
+
for h in data.get("hits", [])[:limit]
|
|
117
|
+
] # fmt: skip
|
|
118
|
+
|
|
119
|
+
async def feed(self, target, *, sort="hot", limit=10):
|
|
120
|
+
key = target.strip().lower().removeprefix("hn/") or "top"
|
|
121
|
+
if key not in FEEDS:
|
|
122
|
+
raise WireUnsupported(f"hn feeds: {', '.join(FEEDS)}")
|
|
123
|
+
endpoint = "search_by_date" if key in ("new", "newest", "jobs") else "search"
|
|
124
|
+
data, f = await self.fetch_json(
|
|
125
|
+
f"{ALGOLIA}/{endpoint}?tags={FEEDS[key]}&hitsPerPage={limit}"
|
|
126
|
+
)
|
|
127
|
+
items = [{"title": h.get("title"), "url": f"{HN_ITEM}{h['objectID']}",
|
|
128
|
+
"meta": [f"{h.get('points', 0)} pts", f"{h.get('num_comments', 0)} comments",
|
|
129
|
+
h.get("url")]} for h in data.get("hits", [])] # fmt: skip
|
|
130
|
+
return self.doc(
|
|
131
|
+
f, "https://news.ycombinator.com/", title=f"Hacker News · {key}", body=listing(items)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
CHANNEL = HackerNewsChannel
|