agent-wire-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. agent_wire/__init__.py +1 -0
  2. agent_wire/__main__.py +3 -0
  3. agent_wire/assets/SKILL.md +26 -0
  4. agent_wire/assets/cursor_rule.mdc +13 -0
  5. agent_wire/assets/hermes_skill.md +28 -0
  6. agent_wire/assets/windsurf_rules.md +14 -0
  7. agent_wire/assets/wire-worker.md +20 -0
  8. agent_wire/auth/__init__.py +0 -0
  9. agent_wire/auth/browser_paths.py +143 -0
  10. agent_wire/auth/cookie_vault.py +181 -0
  11. agent_wire/auth/rate_limiter.py +26 -0
  12. agent_wire/cache/__init__.py +0 -0
  13. agent_wire/cache/ring_buffer.py +94 -0
  14. agent_wire/cache/sqlite_ledger.py +132 -0
  15. agent_wire/channels/__init__.py +41 -0
  16. agent_wire/channels/base.py +249 -0
  17. agent_wire/channels/tier1_global/__init__.py +0 -0
  18. agent_wire/channels/tier1_global/github.py +371 -0
  19. agent_wire/channels/tier1_global/hackernews.py +135 -0
  20. agent_wire/channels/tier1_global/reddit.py +371 -0
  21. agent_wire/channels/tier1_global/rss.py +129 -0
  22. agent_wire/channels/tier1_global/search.py +160 -0
  23. agent_wire/channels/tier1_global/stackoverflow.py +94 -0
  24. agent_wire/channels/tier1_global/web.py +98 -0
  25. agent_wire/channels/tier1_global/wikipedia.py +91 -0
  26. agent_wire/channels/tier1_global/x_twitter.py +380 -0
  27. agent_wire/channels/tier1_global/youtube.py +223 -0
  28. agent_wire/channels/tier2_pro/__init__.py +0 -0
  29. agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
  30. agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
  31. agent_wire/channels/tier2_pro/devto.py +114 -0
  32. agent_wire/channels/tier2_pro/fediverse.py +181 -0
  33. agent_wire/channels/tier2_pro/hubs.py +161 -0
  34. agent_wire/channels/tier2_pro/linkedin.py +146 -0
  35. agent_wire/channels/tier2_pro/markets.py +207 -0
  36. agent_wire/channels/tier2_pro/meta_social.py +104 -0
  37. agent_wire/channels/tier2_pro/news.py +173 -0
  38. agent_wire/channels/tier2_pro/polymarket.py +105 -0
  39. agent_wire/channels/tier2_pro/publishing.py +84 -0
  40. agent_wire/channels/tier2_pro/stocktwits.py +79 -0
  41. agent_wire/channels/tier3_regional/bilibili.py +171 -0
  42. agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
  43. agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
  44. agent_wire/channels/tier3_regional/v2ex.py +70 -0
  45. agent_wire/channels/tier3_regional/wechat.py +67 -0
  46. agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
  47. agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
  48. agent_wire/channels/tier3_regional/xueqiu.py +77 -0
  49. agent_wire/channels/tier3_regional/zhihu.py +57 -0
  50. agent_wire/cli.py +403 -0
  51. agent_wire/compression/__init__.py +0 -0
  52. agent_wire/compression/anchor_slicer.py +45 -0
  53. agent_wire/compression/bm25_slicer.py +151 -0
  54. agent_wire/compression/dam_formatter.py +74 -0
  55. agent_wire/compression/url_compactor.py +45 -0
  56. agent_wire/config.py +52 -0
  57. agent_wire/engine/__init__.py +0 -0
  58. agent_wire/engine/challenge.py +58 -0
  59. agent_wire/engine/http_pool.py +136 -0
  60. agent_wire/engine/stealth_browser.py +90 -0
  61. agent_wire/engine/waterfall.py +159 -0
  62. agent_wire/firewall/__init__.py +0 -0
  63. agent_wire/firewall/inbound_shield.py +116 -0
  64. agent_wire/firewall/outbound_privacy.py +120 -0
  65. agent_wire/installers/__init__.py +0 -0
  66. agent_wire/installers/client_installer.py +315 -0
  67. agent_wire/media/__init__.py +0 -0
  68. agent_wire/media/audio_transcriber.py +77 -0
  69. agent_wire/media/subtitle_cleaner.py +102 -0
  70. agent_wire/models.py +60 -0
  71. agent_wire/research/__init__.py +1 -0
  72. agent_wire/research/brief.py +227 -0
  73. agent_wire/research/briefing.py +41 -0
  74. agent_wire/research/cluster.py +128 -0
  75. agent_wire/research/engine.py +632 -0
  76. agent_wire/research/library.py +74 -0
  77. agent_wire/research/planner.py +232 -0
  78. agent_wire/research/scoring.py +89 -0
  79. agent_wire/research/store.py +149 -0
  80. agent_wire/research/watch.py +70 -0
  81. agent_wire/router.py +107 -0
  82. agent_wire/server.py +468 -0
  83. agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
  84. agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
  85. agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
  86. agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
  87. agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,181 @@
1
+ """Bluesky (public AT Protocol AppView) and Mastodon (ActivityPub REST), zero auth."""
2
+
3
+ from urllib.parse import quote, urlsplit
4
+
5
+ from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown, listing, ts
6
+ from agent_wire.engine.waterfall import WireFetchError
7
+ from agent_wire.models import WireComment, WireSearchHit
8
+
9
+ BSKY = "https://api.bsky.app/xrpc" # public.api.bsky.app 403s searchPosts for anonymous callers
10
+
11
+
12
+ def _bsky_url(post: dict) -> str:
13
+ rkey = post.get("uri", "").rsplit("/", 1)[-1]
14
+ return f"https://bsky.app/profile/{post.get('author', {}).get('handle')}/post/{rkey}"
15
+
16
+
17
+ def _bsky_hit(p: dict) -> WireSearchHit:
18
+ handle = p.get("author", {}).get("handle")
19
+ return WireSearchHit(
20
+ url=_bsky_url(p), title=f"@{handle}", snippet=p.get("record", {}).get("text", ""),
21
+ source="bluesky", rank_score=float(p.get("likeCount") or 0),
22
+ timestamp=ts(p.get("record", {}).get("createdAt")), author=handle,
23
+ engagement={"likes": p.get("likeCount") or 0, "reposts": p.get("repostCount") or 0,
24
+ "comments": p.get("replyCount") or 0},
25
+ ) # fmt: skip
26
+
27
+
28
+ def parse_bsky_thread(
29
+ thread: dict, op: str | None = None, depth: int = 0
30
+ ) -> tuple[dict, list[WireComment]]:
31
+ post = thread.get("post", {})
32
+ author = post.get("author", {}).get("handle")
33
+ replies = [
34
+ WireComment(
35
+ id=r["post"]["cid"], body=r["post"].get("record", {}).get("text", ""),
36
+ author_handle=r["post"].get("author", {}).get("handle"), score=r["post"].get("likeCount") or 0,
37
+ timestamp=ts(r["post"].get("record", {}).get("createdAt")), depth=depth,
38
+ is_op=r["post"].get("author", {}).get("handle") == (op or author),
39
+ replies=parse_bsky_thread(r, op or author, depth + 1)[1],
40
+ )
41
+ for r in thread.get("replies") or [] if r.get("post")
42
+ ] # fmt: skip
43
+ return post, replies
44
+
45
+
46
+ def parse_mastodon_context(descendants: list[dict], root_id: str, op: str) -> list[WireComment]:
47
+ children: dict[str, list[dict]] = {}
48
+ for s in descendants:
49
+ children.setdefault(s.get("in_reply_to_id"), []).append(s)
50
+
51
+ def build(parent: str, depth: int) -> list[WireComment]:
52
+ return [WireComment(id=s["id"], body=html_to_markdown(s.get("content", "")),
53
+ author_handle=s.get("account", {}).get("acct"), score=s.get("favourites_count") or 0,
54
+ timestamp=ts(s.get("created_at")), depth=depth,
55
+ is_op=s.get("account", {}).get("acct") == op, replies=build(s["id"], depth + 1))
56
+ for s in children.get(parent, [])] # fmt: skip
57
+
58
+ return build(root_id, 0)
59
+
60
+
61
+ class FediverseChannel(Channel):
62
+ name = "fediverse"
63
+
64
+ async def read(self, url, *, query=None, section=None, include_comments=True):
65
+ p = urlsplit(url)
66
+ segs = [s for s in p.path.split("/") if s]
67
+ if (p.hostname or "").endswith("bsky.app"):
68
+ if len(segs) >= 4 and segs[0] == "profile" and segs[2] == "post":
69
+ return await self._bsky_post(url, segs[1], segs[3])
70
+ if len(segs) >= 2 and segs[0] == "profile":
71
+ return await self.feed(f"bsky:{segs[1]}")
72
+ raise WireUnsupported("bluesky: expected /profile/<handle>[/post/<id>]")
73
+ if segs and segs[0].startswith("@") and len(segs) > 1 and segs[1].isdigit():
74
+ return await self._mastodon(url, f"https://{p.hostname}", segs[1], include_comments)
75
+ if len(segs) == 1 and segs[0].startswith("@"): # profile; may not be a Mastodon host
76
+ who = segs[0] if "@" in segs[0][1:] else f"{segs[0]}@{p.hostname}"
77
+ try:
78
+ return await self.feed(who)
79
+ except (WireFetchError, ValueError, KeyError, TypeError):
80
+ pass
81
+ return await self.ctx.channel("web").read(url, query=query, section=section)
82
+
83
+ async def _did(self, actor: str) -> str:
84
+ if actor.startswith("did:"):
85
+ return actor
86
+ data, _ = await self.fetch_json(f"{BSKY}/com.atproto.identity.resolveHandle?handle={actor}")
87
+ return data["did"]
88
+
89
+ async def _bsky_post(self, url, actor, rkey):
90
+ did = await self._did(actor)
91
+ uri = f"at://{did}/app.bsky.feed.post/{rkey}"
92
+ data, f = await self.fetch_json(
93
+ f"{BSKY}/app.bsky.feed.getPostThread?uri={quote(uri)}&depth=6"
94
+ )
95
+ post, replies = parse_bsky_thread(data.get("thread", {}))
96
+ record = post.get("record", {})
97
+ embed = (post.get("embed") or {}).get("external") or {}
98
+ body = record.get("text", "") + (
99
+ f"\n\nLink: {embed['uri']} ({embed.get('title', '')})" if embed.get("uri") else ""
100
+ )
101
+ return self.doc(
102
+ f, url, title=f"@{post.get('author', {}).get('handle')} on Bluesky",
103
+ author_handle=post.get("author", {}).get("handle"), timestamp=ts(record.get("createdAt")),
104
+ score=post.get("likeCount"), body=body, comments=replies,
105
+ metadata={"likes": post.get("likeCount"), "reposts": post.get("repostCount"),
106
+ "replies": post.get("replyCount")},
107
+ ) # fmt: skip
108
+
109
+ async def _mastodon(self, url, base, status_id, include_comments):
110
+ status, f = await self.fetch_json(f"{base}/api/v1/statuses/{status_id}")
111
+ op = status.get("account", {}).get("acct")
112
+ comments = []
113
+ if include_comments:
114
+ ctx, _ = await self.fetch_json(f"{base}/api/v1/statuses/{status_id}/context")
115
+ comments = parse_mastodon_context(ctx.get("descendants", []), status_id, op)
116
+ return self.doc(
117
+ f, url, title=f"@{op} on {urlsplit(base).hostname}", author_handle=op,
118
+ timestamp=ts(status.get("created_at")), score=status.get("favourites_count"),
119
+ body=html_to_markdown(status.get("content", "")), comments=comments,
120
+ metadata={"boosts": status.get("reblogs_count"), "replies": status.get("replies_count")},
121
+ ) # fmt: skip
122
+
123
+ async def search(self, query, *, limit=5, sort="relevance", since=None):
124
+ order = "latest" if sort in ("new", "latest") else "top"
125
+ window = f"&since={since.strftime('%Y-%m-%dT%H:%M:%SZ')}" if since else ""
126
+ data, _ = await self.fetch_json(
127
+ f"{BSKY}/app.bsky.feed.searchPosts?q={quote(query)}&limit={limit}&sort={order}{window}"
128
+ )
129
+ return [_bsky_hit(p) for p in data.get("posts", [])]
130
+
131
+ async def mastodon_tag(self, tag: str, *, limit=10, instance="mastodon.social"):
132
+ """Public hashtag timeline (no auth on most instances)."""
133
+ tag = "".join(ch for ch in tag if ch.isalnum() or ch == "_")
134
+ data, _ = await self.fetch_json(
135
+ f"https://{instance}/api/v1/timelines/tag/{tag}?limit={limit}"
136
+ )
137
+ hits = []
138
+ for s in data:
139
+ s = s.get("reblog") or s
140
+ acct = (s.get("account") or {}).get("acct")
141
+ hits.append(WireSearchHit(
142
+ url=s.get("url") or s.get("uri", ""), title=f"@{acct}",
143
+ snippet=html_to_markdown(s.get("content", ""))[:300], source="mastodon",
144
+ rank_score=float(s.get("favourites_count") or 0), timestamp=ts(s.get("created_at")),
145
+ author=acct, engagement={"likes": s.get("favourites_count") or 0,
146
+ "reposts": s.get("reblogs_count") or 0,
147
+ "comments": s.get("replies_count") or 0},
148
+ )) # fmt: skip
149
+ return hits
150
+
151
+ async def feed(self, target, *, sort="hot", limit=10):
152
+ t = target.strip().removeprefix("bsky:")
153
+ if t.count("@") == 2 or (t.startswith("@") and "@" in t[1:]): # @user@instance -> Mastodon
154
+ user, instance = t.lstrip("@").split("@", 1)
155
+ base = f"https://{instance}"
156
+ acct, f = await self.fetch_json(f"{base}/api/v1/accounts/lookup?acct={user}")
157
+ statuses, f = await self.fetch_json(
158
+ f"{base}/api/v1/accounts/{acct['id']}/statuses?limit={limit}&exclude_replies=true"
159
+ )
160
+ statuses = [s.get("reblog") or s for s in statuses] # boosts carry their content inside
161
+ items = [{"title": html_to_markdown(s.get("content", ""))[:120] or s.get("url"), "url": s.get("url"),
162
+ "meta": [f"♥{s.get('favourites_count', 0)}", f"↻{s.get('reblogs_count', 0)}",
163
+ (s.get("created_at") or "")[:10]]} for s in statuses] # fmt: skip
164
+ return self.doc(f, f"{base}/@{user}", title=f"@{user}@{instance}", body=listing(items))
165
+ actor = t.lstrip("@")
166
+ data, f = await self.fetch_json(
167
+ f"{BSKY}/app.bsky.feed.getAuthorFeed?actor={actor}&limit={limit}&filter=posts_no_replies"
168
+ )
169
+ posts = [i["post"] for i in data.get("feed", [])]
170
+ items = [{"title": p.get("record", {}).get("text", "")[:120], "url": _bsky_url(p),
171
+ "meta": [f"♥{p.get('likeCount', 0)}", f"↻{p.get('repostCount', 0)}",
172
+ (p.get("record", {}).get("createdAt") or "")[:10]]} for p in posts] # fmt: skip
173
+ return self.doc(
174
+ f,
175
+ f"https://bsky.app/profile/{actor}",
176
+ title=f"@{actor} on Bluesky",
177
+ body=listing(items),
178
+ )
179
+
180
+
181
+ CHANNEL = FediverseChannel
@@ -0,0 +1,161 @@
1
+ """Package and model hubs: PyPI, npm, crates.io, Hugging Face (models + datasets)."""
2
+
3
+ import asyncio
4
+ from urllib.parse import quote, urlsplit
5
+
6
+ from curl_cffi.requests.exceptions import RequestException
7
+ from selectolax.lexbor import LexborHTMLParser
8
+
9
+ from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown
10
+ from agent_wire.engine.waterfall import WireFetchError
11
+ from agent_wire.models import WireSearchHit
12
+
13
+ SEARCH_ERRORS = (WireFetchError, RequestException, ValueError, KeyError)
14
+
15
+
16
+ def parse_pypi_search(html: str) -> list[WireSearchHit]:
17
+ hits = []
18
+ for a in LexborHTMLParser(html).css("a.package-snippet"):
19
+ name = a.css_first(".package-snippet__name")
20
+ desc = a.css_first(".package-snippet__description")
21
+ if name:
22
+ hits.append(WireSearchHit(url=f"https://pypi.org{a.attributes.get('href', '')}",
23
+ title=f"pypi: {name.text(strip=True)}",
24
+ snippet=desc.text(strip=True) if desc else "", source="pypi")) # fmt: skip
25
+ return hits
26
+
27
+
28
+ class HubsChannel(Channel):
29
+ name = "hubs"
30
+
31
+ async def read(self, url, *, query=None, section=None, include_comments=True):
32
+ p = urlsplit(url)
33
+ host, segs = (p.hostname or "").removeprefix("www."), [s for s in p.path.split("/") if s]
34
+ if host == "pypi.org" and segs[:1] == ["project"] and len(segs) > 1:
35
+ return await self._pypi(url, segs[1], segs[2] if len(segs) > 2 else None)
36
+ if host.endswith("npmjs.com") and segs[:1] == ["package"] and len(segs) > 1:
37
+ name = "/".join(segs[1:3]) if segs[1].startswith("@") else segs[1]
38
+ return await self._npm(url, name)
39
+ if host == "crates.io" and segs[:1] == ["crates"] and len(segs) > 1:
40
+ return await self._crate(url, segs[1])
41
+ if host == "huggingface.co" and segs:
42
+ kind = (
43
+ "datasets"
44
+ if segs[0] == "datasets"
45
+ else "spaces"
46
+ if segs[0] == "spaces"
47
+ else "models"
48
+ )
49
+ repo = "/".join(segs[1:3] if kind != "models" else segs[:2])
50
+ return await self._hf(url, kind, repo)
51
+ return await self.ctx.channel("web").read(url, query=query, section=section)
52
+
53
+ async def _pypi(self, url, name, version):
54
+ path = f"{name}/{version}" if version else name
55
+ data, f = await self.fetch_json(f"https://pypi.org/pypi/{path}/json")
56
+ info = data.get("info", {})
57
+ urls = info.get("project_urls") or {}
58
+ meta = {"version": info.get("version"), "license": (info.get("license") or "")[:60] or None,
59
+ "requires_python": info.get("requires_python"),
60
+ "homepage": urls.get("Homepage") or urls.get("Source") or info.get("home_page")} # fmt: skip
61
+ body = f"{info.get('summary') or ''}\n\n{info.get('description') or ''}".strip()
62
+ return self.doc(f, url, title=f"{info.get('name', name)} (PyPI)", author_handle=info.get("author"),
63
+ body=body, metadata=meta) # fmt: skip
64
+
65
+ async def _npm(self, url, name):
66
+ (data, f), downloads = await asyncio.gather(
67
+ self.fetch_json(f"https://registry.npmjs.org/{name.replace('/', '%2F')}"),
68
+ self._safe_json(f"https://api.npmjs.org/downloads/point/last-week/{name}"),
69
+ )
70
+ latest = (data.get("dist-tags") or {}).get("latest")
71
+ ver = (data.get("versions") or {}).get(latest, {})
72
+ repo = data.get("repository") or {}
73
+ meta = {"version": latest, "license": data.get("license") if isinstance(data.get("license"), str) else None,
74
+ "weekly_downloads": downloads.get("downloads"),
75
+ "repository": repo.get("url") if isinstance(repo, dict) else repo,
76
+ "modified": (data.get("time") or {}).get("modified", "")[:10] or None} # fmt: skip
77
+ body = f"{data.get('description') or ver.get('description') or ''}\n\n{data.get('readme') or ''}".strip()
78
+ return self.doc(f, url, title=f"{name} (npm)", body=body, metadata=meta)
79
+
80
+ async def _crate(self, url, name):
81
+ data, f = await self.fetch_json(f"https://crates.io/api/v1/crates/{name}")
82
+ c = data.get("crate", {})
83
+ readme = ""
84
+ if ver := c.get("max_stable_version") or c.get("max_version"):
85
+ try:
86
+ rf = await self.fetch(f"https://crates.io/api/v1/crates/{name}/{ver}/readme",
87
+ archive=False, browser=False) # fmt: skip
88
+ readme = html_to_markdown(rf.text)
89
+ except WireFetchError:
90
+ readme = ""
91
+ meta = {"version": ver, "downloads": c.get("downloads"), "recent_downloads": c.get("recent_downloads"),
92
+ "repository": c.get("repository")} # fmt: skip
93
+ return self.doc(f, url, title=f"{name} (crates.io)", body=f"{c.get('description') or ''}\n\n{readme}".strip(),
94
+ metadata=meta) # fmt: skip
95
+
96
+ async def _hf(self, url, kind, repo):
97
+ api = f"https://huggingface.co/api/{kind}/{repo}"
98
+ raw = f"https://huggingface.co/{'' if kind == 'models' else kind + '/'}{repo}/raw/main/README.md"
99
+ (data, f), readme = await asyncio.gather(self.fetch_json(api), self._safe_text(raw))
100
+ meta = {"downloads": data.get("downloads"), "likes": data.get("likes"),
101
+ "pipeline": data.get("pipeline_tag"), "library": data.get("library_name"),
102
+ "license": next((t.split(":", 1)[1] for t in data.get("tags", []) if t.startswith("license:")), None),
103
+ "updated": (data.get("lastModified") or "")[:10] or None} # fmt: skip
104
+ return self.doc(f, url, title=f"{repo} (Hugging Face {kind[:-1]})", author_handle=data.get("author"),
105
+ body=readme, metadata=meta) # fmt: skip
106
+
107
+ async def _safe_json(self, url) -> dict:
108
+ try:
109
+ return (await self.fetch_json(url))[0]
110
+ except SEARCH_ERRORS:
111
+ return {}
112
+
113
+ async def _safe_text(self, url) -> str:
114
+ try:
115
+ return (await self.fetch(url, archive=False, browser=False)).text
116
+ except SEARCH_ERRORS:
117
+ return ""
118
+
119
+ async def search(self, query, *, limit=5, sort="relevance"):
120
+ q = quote(query)
121
+
122
+ async def pypi():
123
+ f = await self.fetch(f"https://pypi.org/search/?q={q}", archive=False, browser=False)
124
+ return parse_pypi_search(f.text)
125
+
126
+ async def npm():
127
+ d, _ = await self.fetch_json(
128
+ f"https://registry.npmjs.org/-/v1/search?text={q}&size={limit}"
129
+ )
130
+ return [WireSearchHit(url=o["package"]["links"]["npm"], title=f"npm: {o['package']['name']}",
131
+ snippet=o["package"].get("description") or "", source="npm")
132
+ for o in d.get("objects", [])] # fmt: skip
133
+
134
+ async def crates():
135
+ d, _ = await self.fetch_json(f"https://crates.io/api/v1/crates?q={q}&per_page={limit}")
136
+ return [WireSearchHit(url=f"https://crates.io/crates/{c['name']}", title=f"crate: {c['name']}",
137
+ snippet=c.get("description") or "", source="crates")
138
+ for c in d.get("crates", [])] # fmt: skip
139
+
140
+ async def hf():
141
+ d, _ = await self.fetch_json(
142
+ f"https://huggingface.co/api/models?search={q}&limit={limit}&sort=downloads"
143
+ )
144
+ return [WireSearchHit(url=f"https://huggingface.co/{m['id']}", title=f"hf: {m['id']}",
145
+ snippet=f"{m.get('downloads', 0)} downloads · {m.get('pipeline_tag') or ''}",
146
+ source="huggingface") for m in d] # fmt: skip
147
+
148
+ async def safe(fn):
149
+ try:
150
+ return (await fn())[:limit]
151
+ except SEARCH_ERRORS:
152
+ return []
153
+
154
+ lists = await asyncio.gather(*(safe(fn) for fn in (pypi, npm, crates, hf)))
155
+ hits = [h for group in lists for h in group]
156
+ if not hits:
157
+ raise WireUnsupported("no package/model hub returned results")
158
+ return hits
159
+
160
+
161
+ CHANNEL = HubsChannel
@@ -0,0 +1,146 @@
1
+ """LinkedIn: public guest job pages and search; posts/profiles via public JSON-LD or a local session."""
2
+
3
+ import re
4
+ from urllib.parse import parse_qs, quote, urlsplit
5
+
6
+ from selectolax.lexbor import LexborHTMLParser
7
+
8
+ from agent_wire.channels.base import (
9
+ Channel,
10
+ WireUnsupported,
11
+ html_to_markdown,
12
+ json_ld,
13
+ meta_tags,
14
+ ts,
15
+ )
16
+ from agent_wire.firewall.outbound_privacy import parse_json
17
+ from agent_wire.models import WireComment, WireSearchHit
18
+
19
+ GUEST = "https://www.linkedin.com/jobs-guest/jobs/api"
20
+
21
+
22
+ def job_id(url: str) -> str | None:
23
+ p = urlsplit(url)
24
+ if m := re.search(r"/jobs/view/(?:[\w-]*?-)?(\d{6,})", p.path):
25
+ return m[1]
26
+ return parse_qs(p.query).get("currentJobId", [None])[0]
27
+
28
+
29
+ def parse_job(html: str) -> dict:
30
+ tree = LexborHTMLParser(html)
31
+
32
+ def text(sel):
33
+ n = tree.css_first(sel)
34
+ return n.text(strip=True) if n else None
35
+
36
+ desc = tree.css_first(".show-more-less-html__markup, .description__text")
37
+ criteria = {
38
+ (li.css_first("h3").text(strip=True) if li.css_first("h3") else ""): (
39
+ li.css_first("span").text(strip=True) if li.css_first("span") else "")
40
+ for li in tree.css("li.description__job-criteria-item")
41
+ } # fmt: skip
42
+ return {
43
+ "title": text("h2.top-card-layout__title, h1.top-card-layout__title, .topcard__title"),
44
+ "company": text("a.topcard__org-name-link, .topcard__flavor a"),
45
+ "location": text(".topcard__flavor--bullet"),
46
+ "body": html_to_markdown(desc.html) if desc else "",
47
+ "criteria": {k: v for k, v in criteria.items() if k},
48
+ }
49
+
50
+
51
+ def parse_job_search(html: str) -> list[WireSearchHit]:
52
+ hits = []
53
+ for card in LexborHTMLParser(f"<ul>{html}</ul>").css("li"):
54
+ a = card.css_first("a.base-card__full-link, a[href*='/jobs/view/']")
55
+ title = card.css_first(".base-search-card__title")
56
+ if not a or not title:
57
+ continue
58
+ company = card.css_first(".base-search-card__subtitle")
59
+ loc = card.css_first(".job-search-card__location")
60
+ hits.append(WireSearchHit(
61
+ url=(a.attributes.get("href") or "").split("?")[0], title=title.text(strip=True),
62
+ snippet=" · ".join(n.text(strip=True) for n in (company, loc) if n), source="linkedin")) # fmt: skip
63
+ return hits
64
+
65
+
66
+ def parse_public_page(html: str) -> dict:
67
+ """Public post / profile / company pages: JSON-LD first, OpenGraph as fallback."""
68
+ for item in json_ld(html):
69
+ kind = item.get("@type")
70
+ if kind in ("SocialMediaPosting", "Article", "DiscussionForumPosting"):
71
+ author = item.get("author") or {}
72
+ author = author[0] if isinstance(author, list) and author else author
73
+ comments = [
74
+ WireComment(id=str(i), body=c.get("text") or "", author_handle=(c.get("author") or {}).get("name"),
75
+ timestamp=ts(c.get("datePublished")))
76
+ for i, c in enumerate(item.get("comment") or []) if isinstance(c, dict)
77
+ ] # fmt: skip
78
+ stats = {s.get("interactionType", "").split("/")[-1]: s.get("userInteractionCount")
79
+ for s in item.get("interactionStatistic") or [] if isinstance(s, dict)} # fmt: skip
80
+ return {"title": item.get("headline") or (item.get("articleBody") or "")[:80],
81
+ "author_handle": author.get("name") if isinstance(author, dict) else None,
82
+ "timestamp": ts(item.get("datePublished")), "body": item.get("articleBody") or "",
83
+ "comments": comments, "metadata": {k: v for k, v in stats.items() if k}} # fmt: skip
84
+ if kind in ("Person", "Organization", "ProfilePage"):
85
+ entity = item.get("mainEntity", item)
86
+ return {"title": entity.get("name"), "body": entity.get("description") or "",
87
+ "metadata": {"followers": (entity.get("interactionStatistic") or {}).get("userInteractionCount")
88
+ if isinstance(entity.get("interactionStatistic"), dict) else None,
89
+ "location": (entity.get("address") or {}).get("addressLocality")
90
+ if isinstance(entity.get("address"), dict) else None}} # fmt: skip
91
+ m = meta_tags(html)
92
+ if not m.get("og:title"):
93
+ raise WireUnsupported("linkedin returned a login wall; run `wire auth login linkedin.com`")
94
+ return {"title": m.get("og:title"), "body": m.get("og:description", "")}
95
+
96
+
97
+ def parse_voyager_profile(data: dict) -> dict:
98
+ included = data.get("included", [])
99
+ profile = next((i for i in included if i.get("$type", "").endswith(".Profile")), {})
100
+ positions = [i for i in included if i.get("$type", "").endswith(".Position")]
101
+ schools = [i for i in included if i.get("$type", "").endswith(".Education")]
102
+ lines = [profile.get("headline") or "", profile.get("summary") or ""]
103
+ if positions:
104
+ lines.append(
105
+ "## Experience\n"
106
+ + "\n".join(f"- {p.get('title')} · {p.get('companyName')}" for p in positions)
107
+ )
108
+ if schools:
109
+ lines.append("## Education\n" + "\n".join(f"- {s.get('schoolName')}" for s in schools))
110
+ return {"title": f"{profile.get('firstName', '')} {profile.get('lastName', '')}".strip(),
111
+ "body": "\n\n".join(x for x in lines if x), "metadata": {"location": profile.get("locationName")}} # fmt: skip
112
+
113
+
114
+ class LinkedInChannel(Channel):
115
+ name = "linkedin"
116
+
117
+ async def read(self, url, *, query=None, section=None, include_comments=True):
118
+ if jid := job_id(url):
119
+ f = await self.fetch(f"{GUEST}/jobPosting/{jid}", archive=False)
120
+ job = parse_job(f.text)
121
+ if not job["title"]:
122
+ raise WireUnsupported(f"linkedin job {jid} not found or expired")
123
+ return self.doc(f, f"https://www.linkedin.com/jobs/view/{jid}", title=job["title"],
124
+ author_handle=job["company"], body=job["body"],
125
+ metadata={"location": job["location"], **job["criteria"]}) # fmt: skip
126
+ segs = [s for s in urlsplit(url).path.split("/") if s]
127
+ jar = await self.ctx.vault.cookies_for("linkedin.com") if self.ctx.vault else {}
128
+ if segs[:1] == ["in"] and len(segs) > 1 and jar.get("JSESSIONID"):
129
+ csrf = jar["JSESSIONID"].strip('"')
130
+ f = await self.fetch(
131
+ f"https://www.linkedin.com/voyager/api/identity/profiles/{segs[1]}/profileView",
132
+ headers={"csrf-token": csrf, "accept": "application/vnd.linkedin.normalized+json+2.1"},
133
+ archive=False, browser=False, cookies=jar,
134
+ ) # fmt: skip
135
+ return self.doc(f, url, **parse_voyager_profile(parse_json(f.content)))
136
+ f = await self.fetch(url, archive=False)
137
+ return self.doc(f, url, **parse_public_page(f.text))
138
+
139
+ async def search(self, query, *, limit=5, sort="relevance"):
140
+ f = await self.fetch(
141
+ f"{GUEST}/seeMoreJobPostings/search?keywords={quote(query)}&start=0", archive=False
142
+ )
143
+ return parse_job_search(f.text)[:limit]
144
+
145
+
146
+ CHANNEL = LinkedInChannel