agent-wire-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. agent_wire/__init__.py +1 -0
  2. agent_wire/__main__.py +3 -0
  3. agent_wire/assets/SKILL.md +26 -0
  4. agent_wire/assets/cursor_rule.mdc +13 -0
  5. agent_wire/assets/hermes_skill.md +28 -0
  6. agent_wire/assets/windsurf_rules.md +14 -0
  7. agent_wire/assets/wire-worker.md +20 -0
  8. agent_wire/auth/__init__.py +0 -0
  9. agent_wire/auth/browser_paths.py +143 -0
  10. agent_wire/auth/cookie_vault.py +181 -0
  11. agent_wire/auth/rate_limiter.py +26 -0
  12. agent_wire/cache/__init__.py +0 -0
  13. agent_wire/cache/ring_buffer.py +94 -0
  14. agent_wire/cache/sqlite_ledger.py +132 -0
  15. agent_wire/channels/__init__.py +41 -0
  16. agent_wire/channels/base.py +249 -0
  17. agent_wire/channels/tier1_global/__init__.py +0 -0
  18. agent_wire/channels/tier1_global/github.py +371 -0
  19. agent_wire/channels/tier1_global/hackernews.py +135 -0
  20. agent_wire/channels/tier1_global/reddit.py +371 -0
  21. agent_wire/channels/tier1_global/rss.py +129 -0
  22. agent_wire/channels/tier1_global/search.py +160 -0
  23. agent_wire/channels/tier1_global/stackoverflow.py +94 -0
  24. agent_wire/channels/tier1_global/web.py +98 -0
  25. agent_wire/channels/tier1_global/wikipedia.py +91 -0
  26. agent_wire/channels/tier1_global/x_twitter.py +380 -0
  27. agent_wire/channels/tier1_global/youtube.py +223 -0
  28. agent_wire/channels/tier2_pro/__init__.py +0 -0
  29. agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
  30. agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
  31. agent_wire/channels/tier2_pro/devto.py +114 -0
  32. agent_wire/channels/tier2_pro/fediverse.py +181 -0
  33. agent_wire/channels/tier2_pro/hubs.py +161 -0
  34. agent_wire/channels/tier2_pro/linkedin.py +146 -0
  35. agent_wire/channels/tier2_pro/markets.py +207 -0
  36. agent_wire/channels/tier2_pro/meta_social.py +104 -0
  37. agent_wire/channels/tier2_pro/news.py +173 -0
  38. agent_wire/channels/tier2_pro/polymarket.py +105 -0
  39. agent_wire/channels/tier2_pro/publishing.py +84 -0
  40. agent_wire/channels/tier2_pro/stocktwits.py +79 -0
  41. agent_wire/channels/tier3_regional/bilibili.py +171 -0
  42. agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
  43. agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
  44. agent_wire/channels/tier3_regional/v2ex.py +70 -0
  45. agent_wire/channels/tier3_regional/wechat.py +67 -0
  46. agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
  47. agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
  48. agent_wire/channels/tier3_regional/xueqiu.py +77 -0
  49. agent_wire/channels/tier3_regional/zhihu.py +57 -0
  50. agent_wire/cli.py +403 -0
  51. agent_wire/compression/__init__.py +0 -0
  52. agent_wire/compression/anchor_slicer.py +45 -0
  53. agent_wire/compression/bm25_slicer.py +151 -0
  54. agent_wire/compression/dam_formatter.py +74 -0
  55. agent_wire/compression/url_compactor.py +45 -0
  56. agent_wire/config.py +52 -0
  57. agent_wire/engine/__init__.py +0 -0
  58. agent_wire/engine/challenge.py +58 -0
  59. agent_wire/engine/http_pool.py +136 -0
  60. agent_wire/engine/stealth_browser.py +90 -0
  61. agent_wire/engine/waterfall.py +159 -0
  62. agent_wire/firewall/__init__.py +0 -0
  63. agent_wire/firewall/inbound_shield.py +116 -0
  64. agent_wire/firewall/outbound_privacy.py +120 -0
  65. agent_wire/installers/__init__.py +0 -0
  66. agent_wire/installers/client_installer.py +315 -0
  67. agent_wire/media/__init__.py +0 -0
  68. agent_wire/media/audio_transcriber.py +77 -0
  69. agent_wire/media/subtitle_cleaner.py +102 -0
  70. agent_wire/models.py +60 -0
  71. agent_wire/research/__init__.py +1 -0
  72. agent_wire/research/brief.py +227 -0
  73. agent_wire/research/briefing.py +41 -0
  74. agent_wire/research/cluster.py +128 -0
  75. agent_wire/research/engine.py +632 -0
  76. agent_wire/research/library.py +74 -0
  77. agent_wire/research/planner.py +232 -0
  78. agent_wire/research/scoring.py +89 -0
  79. agent_wire/research/store.py +149 -0
  80. agent_wire/research/watch.py +70 -0
  81. agent_wire/router.py +107 -0
  82. agent_wire/server.py +468 -0
  83. agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
  84. agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
  85. agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
  86. agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
  87. agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,371 @@
1
+ """GitHub: repo overview, blobs with #L slicing, issues/PRs, gists, users, search, trending."""
2
+
3
+ import asyncio
4
+ import os
5
+ import shutil
6
+ import subprocess
7
+ from functools import cache
8
+ from urllib.parse import quote, urlsplit
9
+
10
+ from selectolax.lexbor import LexborHTMLParser
11
+
12
+ from agent_wire.channels.base import Channel, WireUnsupported, compact_count, listing, ts
13
+ from agent_wire.compression.anchor_slicer import parse_line_anchor, slice_lines, slice_section
14
+ from agent_wire.models import WireComment, WireDocument, WireSearchHit
15
+
16
+ API = "https://api.github.com"
17
+ RAW = "https://raw.githubusercontent.com"
18
+ _LANG = {"py": "python", "js": "javascript", "ts": "typescript", "rs": "rust", "go": "go",
19
+ "rb": "ruby", "java": "java", "kt": "kotlin", "c": "c", "h": "c", "cpp": "cpp",
20
+ "cs": "csharp", "sh": "bash", "md": "markdown", "toml": "toml", "yml": "yaml",
21
+ "yaml": "yaml", "json": "json", "tsx": "tsx", "jsx": "jsx", "swift": "swift"} # fmt: skip
22
+ _SKIP_PATHS = frozenset({"orgs", "settings", "notifications", "login", "marketplace", "topics"})
23
+
24
+
25
+ @cache
26
+ def github_token() -> str | None:
27
+ if token := os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN"):
28
+ return token
29
+ if gh := shutil.which("gh"):
30
+ try:
31
+ out = subprocess.run(
32
+ [gh, "auth", "token"], capture_output=True, text=True, timeout=3, check=False
33
+ )
34
+ return out.stdout.strip() or None
35
+ except (OSError, subprocess.TimeoutExpired):
36
+ return None
37
+ return None
38
+
39
+
40
+ def _repo_hit(r: dict) -> WireSearchHit:
41
+ return WireSearchHit(
42
+ url=r["html_url"], title=r["full_name"], snippet=r.get("description") or "",
43
+ source="github", rank_score=float(r.get("stargazers_count", 0)),
44
+ timestamp=ts(r.get("pushed_at")), author=(r.get("owner") or {}).get("login"),
45
+ engagement={"stars": r.get("stargazers_count") or 0, "forks": r.get("forks_count") or 0},
46
+ ) # fmt: skip
47
+
48
+
49
+ def _issue_hit(i: dict) -> WireSearchHit:
50
+ kind = "PR" if i.get("pull_request") else "issue"
51
+ repo = i.get("repository_url", "").removeprefix(f"{API}/repos/")
52
+ return WireSearchHit(
53
+ url=i["html_url"], title=f"{repo}: {i.get('title', '')}",
54
+ snippet=f"{kind} · {i.get('state')} · {i.get('comments', 0)} comments · "
55
+ + (i.get("body") or "")[:200],
56
+ source="github", rank_score=float(i.get("comments") or 0),
57
+ timestamp=ts(i.get("created_at")), author=(i.get("user") or {}).get("login"),
58
+ engagement={"comments": i.get("comments") or 0,
59
+ "reactions": (i.get("reactions") or {}).get("total_count", 0)},
60
+ ) # fmt: skip
61
+
62
+
63
+ class GitHubChannel(Channel):
64
+ name = "github"
65
+
66
+ def _headers(self, raw: bool = False) -> dict[str, str]:
67
+ h = {
68
+ "Accept": "application/vnd.github.raw" if raw else "application/vnd.github+json",
69
+ "X-GitHub-Api-Version": "2022-11-28",
70
+ }
71
+ if token := github_token():
72
+ h["Authorization"] = f"Bearer {token}"
73
+ return h
74
+
75
+ async def api(self, path: str):
76
+ data, f = await self.fetch_json(f"{API}{path}", headers=self._headers())
77
+ return data, f
78
+
79
+ async def raw(self, url: str) -> str:
80
+ f = await self.fetch(url, headers=self._headers(raw=True), archive=False, browser=False)
81
+ return f.text
82
+
83
+ # ---- read ------------------------------------------------------------------------------
84
+ async def read(self, url, *, query=None, section=None, include_comments=True):
85
+ parts = urlsplit(url)
86
+ host, segs = parts.hostname or "", [s for s in parts.path.split("/") if s]
87
+ anchor = section or parts.fragment or None
88
+ if host.endswith("githubusercontent.com"):
89
+ return await self._blob_raw(url, segs[0], segs[1], "/".join(segs[3:]), url)
90
+ if host == "gist.github.com" and len(segs) >= 2:
91
+ return await self._gist(url, segs[-1])
92
+ if not segs or segs[0] in _SKIP_PATHS:
93
+ return await self.ctx.channel("web").read(url, query=query, section=section)
94
+ if len(segs) == 1:
95
+ return await self._user(url, segs[0])
96
+ owner, repo = segs[0], segs[1].removesuffix(".git")
97
+ kind = segs[2] if len(segs) > 2 else ""
98
+ if not kind:
99
+ return await self._repo(url, owner, repo, anchor)
100
+ if kind == "blob" and len(segs) > 4:
101
+ raw = f"{RAW}/{owner}/{repo}/{segs[3]}/{'/'.join(segs[4:])}"
102
+ return await self._blob_raw(raw, owner, repo, "/".join(segs[4:]), url)
103
+ if kind == "tree" and len(segs) > 3:
104
+ return await self._tree(url, owner, repo, segs[3], "/".join(segs[4:]))
105
+ if kind in ("issues", "pull") and len(segs) > 3 and segs[3].isdigit():
106
+ return await self._issue(url, owner, repo, int(segs[3]), include_comments)
107
+ if kind == "releases":
108
+ return await self._releases(url, owner, repo)
109
+ if kind == "commit" and len(segs) > 3:
110
+ return await self._commit(url, owner, repo, segs[3])
111
+ return await self.ctx.channel("web").read(url, query=query, section=section)
112
+
113
+ async def _repo(self, url, owner, repo, anchor):
114
+ (meta, f), (tree, _), readme = await asyncio.gather(
115
+ self.api(f"/repos/{owner}/{repo}"),
116
+ self.api(f"/repos/{owner}/{repo}/contents"),
117
+ self._readme(owner, repo),
118
+ )
119
+ files = " ".join(
120
+ f"`{e['name']}/`" if e.get("type") == "dir" else f"`{e['name']}`"
121
+ for e in (tree if isinstance(tree, list) else [])
122
+ )
123
+ if anchor and readme:
124
+ readme = slice_section(readme, anchor) or readme
125
+ stats = {
126
+ "stars": meta.get("stargazers_count"), "forks": meta.get("forks_count"),
127
+ "language": meta.get("language"), "license": (meta.get("license") or {}).get("spdx_id"),
128
+ "open_issues": meta.get("open_issues_count"),
129
+ "topics": ", ".join(meta.get("topics") or []) or None,
130
+ "pushed": (meta.get("pushed_at") or "")[:10] or None,
131
+ } # fmt: skip
132
+ body = "\n\n".join(
133
+ p for p in (meta.get("description"), f"## Files\n{files}" if files else "",
134
+ f"## README\n{readme}" if readme else "") if p
135
+ ) # fmt: skip
136
+ return self.doc(f, url, title=meta.get("full_name", f"{owner}/{repo}"),
137
+ author_handle=owner, body=body, metadata=stats) # fmt: skip
138
+
139
+ async def _readme(self, owner, repo) -> str:
140
+ try:
141
+ f = await self.fetch(f"{API}/repos/{owner}/{repo}/readme",
142
+ headers=self._headers(raw=True), archive=False, browser=False) # fmt: skip
143
+ return f.text
144
+ except Exception: # noqa: BLE001 - a repo without a README is still a repo
145
+ return ""
146
+
147
+ async def _blob_raw(self, raw_url, owner, repo, path, url):
148
+ f = await self.fetch(raw_url, headers=self._headers(raw=True), archive=False, browser=False)
149
+ text = f.text
150
+ lines = parse_line_anchor(url)
151
+ if lines:
152
+ text = slice_lines(text, *lines)
153
+ lang = _LANG.get(path.rsplit(".", 1)[-1].lower(), "")
154
+ body = text if lang == "markdown" and not lines else f"```{lang}\n{text}\n```"
155
+ meta = {"lines": f"{lines[0]}-{lines[1]}"} if lines else {}
156
+ return self.doc(f, url, title=f"{owner}/{repo}: {path}", body=body, metadata=meta)
157
+
158
+ async def _tree(self, url, owner, repo, ref, path):
159
+ data, f = await self.api(f"/repos/{owner}/{repo}/contents/{quote(path)}?ref={quote(ref)}")
160
+ entries = data if isinstance(data, list) else [data]
161
+ body = "\n".join(
162
+ f"- `{e['name']}{'/' if e.get('type') == 'dir' else ''}`"
163
+ + (f" ({e['size']} B)" if e.get("type") == "file" else "")
164
+ for e in entries
165
+ )
166
+ return self.doc(f, url, title=f"{owner}/{repo}/{path or ''} @ {ref}", body=body)
167
+
168
+ async def _issue(self, url, owner, repo, num, include_comments):
169
+ issue, f = await self.api(f"/repos/{owner}/{repo}/issues/{num}")
170
+ meta = {
171
+ "state": issue.get("state"),
172
+ "labels": ", ".join(lb["name"] for lb in issue.get("labels", [])) or None,
173
+ "comments": issue.get("comments"),
174
+ }
175
+ if "pull_request" in issue:
176
+ pr, _ = await self.api(f"/repos/{owner}/{repo}/pulls/{num}")
177
+ meta.update(merged=pr.get("merged"), additions=pr.get("additions"),
178
+ deletions=pr.get("deletions"), files=pr.get("changed_files")) # fmt: skip
179
+ comments = []
180
+ if include_comments and issue.get("comments"):
181
+ raw, _ = await self.api(f"/repos/{owner}/{repo}/issues/{num}/comments?per_page=100")
182
+ author = (issue.get("user") or {}).get("login")
183
+ comments = [
184
+ WireComment(
185
+ id=str(c["id"]), body=c.get("body") or "",
186
+ author_handle=(c.get("user") or {}).get("login"),
187
+ timestamp=ts(c.get("created_at")),
188
+ score=(c.get("reactions") or {}).get("total_count", 0),
189
+ is_op=(c.get("user") or {}).get("login") == author,
190
+ )
191
+ for c in raw
192
+ ] # fmt: skip
193
+ return self.doc(
194
+ f, url, title=f"#{num} {issue.get('title', '')}",
195
+ author_handle=(issue.get("user") or {}).get("login"),
196
+ timestamp=ts(issue.get("created_at")), body=issue.get("body") or "",
197
+ score=(issue.get("reactions") or {}).get("total_count"),
198
+ comments=comments, metadata=meta,
199
+ ) # fmt: skip
200
+
201
+ async def _releases(self, url, owner, repo):
202
+ data, f = await self.api(f"/repos/{owner}/{repo}/releases?per_page=5")
203
+ body = "\n\n".join(
204
+ f"## {r.get('name') or r.get('tag_name')} ({(r.get('published_at') or '')[:10]})\n"
205
+ f"{r.get('body') or ''}"
206
+ for r in data
207
+ )
208
+ return self.doc(f, url, title=f"{owner}/{repo} releases", body=body)
209
+
210
+ async def _commit(self, url, owner, repo, sha):
211
+ c, f = await self.api(f"/repos/{owner}/{repo}/commits/{sha}")
212
+ files = "\n".join(
213
+ f"- `{x['filename']}` +{x.get('additions', 0)} -{x.get('deletions', 0)}"
214
+ for x in c.get("files", [])[:50]
215
+ )
216
+ stats = c.get("stats") or {}
217
+ return self.doc(
218
+ f, url, title=(c.get("commit", {}).get("message") or sha).split("\n")[0],
219
+ author_handle=(c.get("author") or {}).get("login"),
220
+ timestamp=ts(c.get("commit", {}).get("author", {}).get("date")),
221
+ body=f"{c.get('commit', {}).get('message', '')}\n\n## Files\n{files}",
222
+ metadata={"additions": stats.get("additions"), "deletions": stats.get("deletions")},
223
+ ) # fmt: skip
224
+
225
+ async def _gist(self, url, gist_id):
226
+ g, f = await self.api(f"/gists/{gist_id}")
227
+ body = "\n\n".join(
228
+ f"## {name}\n```{_LANG.get(name.rsplit('.', 1)[-1].lower(), '')}\n{fl.get('content', '')}\n```"
229
+ for name, fl in (g.get("files") or {}).items()
230
+ )
231
+ return self.doc(f, url, title=g.get("description") or gist_id,
232
+ author_handle=(g.get("owner") or {}).get("login"), body=body) # fmt: skip
233
+
234
+ async def _user(self, url, login):
235
+ (u, f), (repos, _) = await asyncio.gather(
236
+ self.api(f"/users/{login}"), self.api(f"/users/{login}/repos?sort=updated&per_page=15")
237
+ )
238
+ body = (u.get("bio") or "") + "\n\n## Repositories\n" + _repo_list(repos)
239
+ return self.doc(f, url, title=u.get("name") or login, author_handle=login, body=body,
240
+ metadata={"followers": u.get("followers"), "public_repos": u.get("public_repos")}) # fmt: skip
241
+
242
+ # ---- search / feed / me ----------------------------------------------------------------
243
+ async def search(self, query, *, limit=5, sort="relevance", since=None):
244
+ order = "&sort=stars" if sort in ("top", "stars") else ""
245
+ if since: # research window: repos active lately plus issues/discussions opened lately
246
+ day = since.date().isoformat()
247
+ repos, _ = await self.api(f"/search/repositories?q={quote(query)}+pushed:>{day}"
248
+ f"&per_page={limit}&sort=stars") # fmt: skip
249
+ issues, _ = await self.api(f"/search/issues?q={quote(query)}+created:>{day}"
250
+ f"&per_page={limit}&sort=comments") # fmt: skip
251
+ return [_repo_hit(r) for r in repos.get("items", [])[:limit]] + [
252
+ _issue_hit(i) for i in issues.get("items", [])[:limit]
253
+ ]
254
+ data, _ = await self.api(f"/search/repositories?q={quote(query)}&per_page={limit}{order}")
255
+ return [_repo_hit(r) for r in data.get("items", [])[:limit]]
256
+
257
+ async def person(self, user: str, *, since=None, limit=10) -> WireDocument:
258
+ """GitHub person mode: recent PR velocity, merged count, top and freshest repos."""
259
+ user = user.lstrip("@").strip("/")
260
+ q = f"author:{user}+type:pr" + (f"+created:>{since.date().isoformat()}" if since else "")
261
+ prs, f = await self.api(f"/search/issues?q={q}&per_page=30&sort=created")
262
+ repos, _ = await self.api(f"/users/{user}/repos?sort=pushed&per_page=30")
263
+ merged = sum(
264
+ 1 for p in prs.get("items", []) if (p.get("pull_request") or {}).get("merged_at")
265
+ )
266
+ top = sorted(repos, key=lambda r: -r.get("stargazers_count", 0))[:limit]
267
+ lines = [f"PRs opened in window: {prs.get('total_count', 0)} · merged (sampled): {merged}", "",
268
+ "Recent PRs:"] # fmt: skip
269
+ lines += [f"- [{p['title']}]({p['html_url']}) · {p.get('state')} · {(p.get('created_at') or '')[:10]}"
270
+ for p in prs.get("items", [])[:limit]] # fmt: skip
271
+ lines += ["", "Top repos:"] + [
272
+ f"- [{r['full_name']}]({r['html_url']}) · ★{r.get('stargazers_count', 0)} · "
273
+ f"pushed {(r.get('pushed_at') or '')[:10]} · {r.get('description') or ''}"
274
+ for r in top
275
+ ]
276
+ return self.doc(f, f"https://github.com/{user}", title=f"GitHub · {user}",
277
+ author_handle=user, body="\n".join(lines),
278
+ metadata={"prs": prs.get("total_count", 0), "merged_sample": merged}) # fmt: skip
279
+
280
+ async def feed(self, target, *, sort="hot", limit=10):
281
+ t = target.strip().strip("/").removeprefix("https://github.com/")
282
+ if t.startswith("trending"):
283
+ url = f"https://github.com/{t}"
284
+ f = await self.fetch(url, archive=False)
285
+ items = parse_trending(f.text)[:limit]
286
+ return self.doc(f, url, title=f"GitHub {t}", body=listing(items))
287
+ if "/" in t:
288
+ data, f = await self.api(f"/repos/{t}/issues?state=open&sort=updated&per_page={limit}")
289
+ items = [{"title": f"#{i['number']} {i['title']}", "url": i["html_url"],
290
+ "meta": [i.get("state"), f"{i.get('comments', 0)} comments"]} for i in data] # fmt: skip
291
+ return self.doc(
292
+ f, f"https://github.com/{t}/issues", title=f"{t} issues", body=listing(items)
293
+ )
294
+ login = t.lstrip("@")
295
+ repos, f = await self.api(f"/users/{login}/repos?sort=updated&per_page={limit}")
296
+ return self.doc(
297
+ f, f"https://github.com/{login}", title=f"@{login} repos", body=_repo_list(repos)
298
+ )
299
+
300
+ async def me(self, target="latest_posts", *, limit=5):
301
+ if not github_token():
302
+ raise WireUnsupported("wire_me github needs GITHUB_TOKEN or `gh auth login`")
303
+ paths = {
304
+ "notifications": f"/notifications?per_page={limit}",
305
+ "latest_posts": f"/user/repos?sort=updated&per_page={limit}",
306
+ "likes": f"/user/starred?per_page={limit}",
307
+ "bookmarks": f"/user/starred?per_page={limit}",
308
+ "prs": f"/search/issues?q={quote('is:pr author:@me is:open')}&per_page={limit}",
309
+ "stats": "/user",
310
+ }
311
+ if target not in paths:
312
+ raise WireUnsupported(f"github targets: {', '.join(paths)}")
313
+ data, f = await self.api(paths[target])
314
+ if target == "stats":
315
+ body = listing([{"title": data.get("login"), "url": data.get("html_url"),
316
+ "meta": [f"{data.get('followers')} followers",
317
+ f"{data.get('public_repos')} public repos"]}]) # fmt: skip
318
+ elif target == "notifications":
319
+ body = listing([{"title": n["subject"]["title"],
320
+ "url": f"https://github.com/{n['repository']['full_name']}",
321
+ "meta": [n["subject"]["type"], n.get("reason"), (n.get("updated_at") or "")[:10]]}
322
+ for n in data]) # fmt: skip
323
+ elif target == "prs":
324
+ body = listing([{"title": i["title"], "url": i["html_url"], "meta": [i.get("state")]}
325
+ for i in data.get("items", [])]) # fmt: skip
326
+ else:
327
+ body = _repo_list(data)
328
+ return self.doc(f, f"https://github.com/{target}", title=f"my github {target}", body=body)
329
+
330
+
331
+ def _repo_list(repos) -> str:
332
+ return listing(
333
+ [
334
+ {
335
+ "title": r.get("full_name") or r.get("name"),
336
+ "url": r.get("html_url"),
337
+ "meta": [f"★{r.get('stargazers_count', 0)}", r.get("language")],
338
+ "snippet": r.get("description") or "",
339
+ }
340
+ for r in (repos if isinstance(repos, list) else [])
341
+ ]
342
+ )
343
+
344
+
345
+ def parse_trending(html: str) -> list[dict]:
346
+ items = []
347
+ for row in LexborHTMLParser(html).css("article.Box-row"):
348
+ a = row.css_first("h2 a")
349
+ if not a:
350
+ continue
351
+ path = (a.attributes.get("href") or "").strip()
352
+ desc = row.css_first("p")
353
+ stars = row.css_first('a[href$="/stargazers"]')
354
+ today = row.css_first("span.d-inline-block.float-sm-right")
355
+ lang = row.css_first('[itemprop="programmingLanguage"]')
356
+ items.append(
357
+ {
358
+ "title": path.strip("/"),
359
+ "url": f"https://github.com{path}",
360
+ "meta": [
361
+ f"★{compact_count(stars.text(strip=True))}" if stars else None,
362
+ lang.text(strip=True) if lang else None,
363
+ today.text(strip=True) if today else None,
364
+ ],
365
+ "snippet": desc.text(strip=True) if desc else "",
366
+ }
367
+ )
368
+ return items
369
+
370
+
371
+ CHANNEL = GitHubChannel
@@ -0,0 +1,135 @@
1
+ """Hacker News (Algolia API) and Lobsters (.json) threads, search and front pages."""
2
+
3
+ import re
4
+ from urllib.parse import parse_qs, quote, urlsplit
5
+
6
+ from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown, listing, ts
7
+ from agent_wire.models import WireComment, WireSearchHit
8
+
9
+ ALGOLIA = "https://hn.algolia.com/api/v1"
10
+ HN_ITEM = "https://news.ycombinator.com/item?id="
11
+ FEEDS = {"top": "front_page", "front_page": "front_page", "ask": "ask_hn", "show": "show_hn",
12
+ "new": "story", "newest": "story", "jobs": "job"} # fmt: skip
13
+
14
+
15
+ def _count(c: dict) -> int:
16
+ return sum(1 + _count(k) for k in c.get("children") or [])
17
+
18
+
19
+ def parse_hn_comments(children: list, op: str | None, depth: int = 0) -> list[WireComment]:
20
+ out = []
21
+ for c in children or []:
22
+ if not c.get("text") and not c.get("children"):
23
+ continue # deleted leaf
24
+ out.append(
25
+ WireComment(
26
+ id=str(c["id"]), body=html_to_markdown(c.get("text") or "[deleted]"),
27
+ author_handle=c.get("author"), timestamp=ts(c.get("created_at_i")),
28
+ score=_count(c), score_kind="replies", # HN hides comment points
29
+ depth=depth, is_op=bool(op and c.get("author") == op),
30
+ replies=parse_hn_comments(c.get("children"), op, depth + 1),
31
+ )
32
+ ) # fmt: skip
33
+ return out
34
+
35
+
36
+ def parse_lobsters(data: dict) -> dict:
37
+ nodes: dict[str, WireComment] = {}
38
+ roots: list[WireComment] = []
39
+ op = data.get("submitter_user") or {}
40
+ op = op.get("username") if isinstance(op, dict) else op
41
+ for c in data.get("comments", []):
42
+ user = c.get("commenting_user")
43
+ user = user.get("username") if isinstance(user, dict) else user
44
+ node = WireComment(
45
+ id=c["short_id"], body=html_to_markdown(c.get("comment") or ""), author_handle=user,
46
+ score=c.get("score") or 0, depth=(c.get("depth") or 1) - 1,
47
+ timestamp=ts(c.get("created_at")), is_op=user == op,
48
+ ) # fmt: skip
49
+ nodes[c["short_id"]] = node
50
+ parent = nodes.get(c.get("parent_comment") or "")
51
+ (parent.replies if parent else roots).append(node)
52
+ body = html_to_markdown(data.get("description") or "")
53
+ if data.get("url"):
54
+ body = f"Link: {data['url']}\n\n{body}".strip()
55
+ return {
56
+ "title": data.get("title"), "author_handle": op, "score": data.get("score"),
57
+ "timestamp": ts(data.get("created_at")), "body": body, "comments": roots,
58
+ "metadata": {"tags": ", ".join(data.get("tags") or []) or None,
59
+ "comments": data.get("comment_count")},
60
+ } # fmt: skip
61
+
62
+
63
+ class HackerNewsChannel(Channel):
64
+ name = "hackernews"
65
+
66
+ async def read(self, url, *, query=None, section=None, include_comments=True):
67
+ parts = urlsplit(url)
68
+ if (parts.hostname or "").endswith("lobste.rs"):
69
+ m = re.match(r"/s/(\w+)", parts.path)
70
+ page = re.fullmatch(r"/?(hottest|newest|active|t/[\w,]+)?/?", parts.path)
71
+ if not m and page: # front page, newest, active or a tag: its JSON listing
72
+ data, f = await self.fetch_json(f"https://lobste.rs/{page[1] or 'hottest'}.json")
73
+ items = [{"title": s.get("title"), "url": s.get("short_id_url") or s.get("comments_url"),
74
+ "meta": [f"▲{s.get('score', 0)}", f"{s.get('comment_count', 0)} comments",
75
+ ", ".join(s.get("tags") or [])]} for s in data] # fmt: skip
76
+ return self.doc(
77
+ f, url, title=f"Lobsters · {page[1] or 'hottest'}", body=listing(items)
78
+ )
79
+ if not m:
80
+ return await self.ctx.channel("web").read(url, query=query, section=section)
81
+ data, f = await self.fetch_json(f"https://lobste.rs/s/{m[1]}.json")
82
+ return self.doc(f, url, **parse_lobsters(data))
83
+ item_id = parse_qs(parts.query).get("id", [""])[0]
84
+ if not item_id.isdigit():
85
+ return await self.ctx.channel("web").read(url, query=query, section=section)
86
+ item, f = await self.fetch_json(f"{ALGOLIA}/items/{item_id}")
87
+ body = html_to_markdown(item.get("text") or "")
88
+ if item.get("url"):
89
+ body = f"Link: {item['url']}\n\n{body}".strip()
90
+ comments = (
91
+ parse_hn_comments(item.get("children"), item.get("author")) if include_comments else []
92
+ )
93
+ return self.doc(
94
+ f, url, title=item.get("title") or f"HN comment {item_id}",
95
+ author_handle=item.get("author"), timestamp=ts(item.get("created_at_i")),
96
+ score=item.get("points"), body=body, comments=comments,
97
+ metadata={"comments": _count(item)},
98
+ ) # fmt: skip
99
+
100
+ async def search(self, query, *, limit=5, sort="relevance", since=None):
101
+ endpoint = "search_by_date" if sort in ("new", "date", "recent") else "search"
102
+ window = f"&numericFilters=created_at_i>{int(since.timestamp())}" if since else ""
103
+ data, _ = await self.fetch_json(
104
+ f"{ALGOLIA}/{endpoint}?query={quote(query)}&tags=story&hitsPerPage={limit}{window}"
105
+ )
106
+ return [
107
+ WireSearchHit(
108
+ url=f"{HN_ITEM}{h['objectID']}", title=h.get("title") or "",
109
+ snippet=f"{h.get('points', 0)} points · {h.get('num_comments', 0)} comments"
110
+ + (f" · {h['url']}" if h.get("url") else ""),
111
+ source="hn", rank_score=float(h.get("points") or 0),
112
+ timestamp=ts(h.get("created_at_i")), author=h.get("author"),
113
+ engagement={"score": h.get("points") or 0, "comments": h.get("num_comments") or 0},
114
+ link=h.get("url"),
115
+ )
116
+ for h in data.get("hits", [])[:limit]
117
+ ] # fmt: skip
118
+
119
+ async def feed(self, target, *, sort="hot", limit=10):
120
+ key = target.strip().lower().removeprefix("hn/") or "top"
121
+ if key not in FEEDS:
122
+ raise WireUnsupported(f"hn feeds: {', '.join(FEEDS)}")
123
+ endpoint = "search_by_date" if key in ("new", "newest", "jobs") else "search"
124
+ data, f = await self.fetch_json(
125
+ f"{ALGOLIA}/{endpoint}?tags={FEEDS[key]}&hitsPerPage={limit}"
126
+ )
127
+ items = [{"title": h.get("title"), "url": f"{HN_ITEM}{h['objectID']}",
128
+ "meta": [f"{h.get('points', 0)} pts", f"{h.get('num_comments', 0)} comments",
129
+ h.get("url")]} for h in data.get("hits", [])] # fmt: skip
130
+ return self.doc(
131
+ f, "https://news.ycombinator.com/", title=f"Hacker News · {key}", body=listing(items)
132
+ )
133
+
134
+
135
+ CHANNEL = HackerNewsChannel