agent-wire-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_wire/__init__.py +1 -0
- agent_wire/__main__.py +3 -0
- agent_wire/assets/SKILL.md +26 -0
- agent_wire/assets/cursor_rule.mdc +13 -0
- agent_wire/assets/hermes_skill.md +28 -0
- agent_wire/assets/windsurf_rules.md +14 -0
- agent_wire/assets/wire-worker.md +20 -0
- agent_wire/auth/__init__.py +0 -0
- agent_wire/auth/browser_paths.py +143 -0
- agent_wire/auth/cookie_vault.py +181 -0
- agent_wire/auth/rate_limiter.py +26 -0
- agent_wire/cache/__init__.py +0 -0
- agent_wire/cache/ring_buffer.py +94 -0
- agent_wire/cache/sqlite_ledger.py +132 -0
- agent_wire/channels/__init__.py +41 -0
- agent_wire/channels/base.py +249 -0
- agent_wire/channels/tier1_global/__init__.py +0 -0
- agent_wire/channels/tier1_global/github.py +371 -0
- agent_wire/channels/tier1_global/hackernews.py +135 -0
- agent_wire/channels/tier1_global/reddit.py +371 -0
- agent_wire/channels/tier1_global/rss.py +129 -0
- agent_wire/channels/tier1_global/search.py +160 -0
- agent_wire/channels/tier1_global/stackoverflow.py +94 -0
- agent_wire/channels/tier1_global/web.py +98 -0
- agent_wire/channels/tier1_global/wikipedia.py +91 -0
- agent_wire/channels/tier1_global/x_twitter.py +380 -0
- agent_wire/channels/tier1_global/youtube.py +223 -0
- agent_wire/channels/tier2_pro/__init__.py +0 -0
- agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
- agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
- agent_wire/channels/tier2_pro/devto.py +114 -0
- agent_wire/channels/tier2_pro/fediverse.py +181 -0
- agent_wire/channels/tier2_pro/hubs.py +161 -0
- agent_wire/channels/tier2_pro/linkedin.py +146 -0
- agent_wire/channels/tier2_pro/markets.py +207 -0
- agent_wire/channels/tier2_pro/meta_social.py +104 -0
- agent_wire/channels/tier2_pro/news.py +173 -0
- agent_wire/channels/tier2_pro/polymarket.py +105 -0
- agent_wire/channels/tier2_pro/publishing.py +84 -0
- agent_wire/channels/tier2_pro/stocktwits.py +79 -0
- agent_wire/channels/tier3_regional/bilibili.py +171 -0
- agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
- agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
- agent_wire/channels/tier3_regional/v2ex.py +70 -0
- agent_wire/channels/tier3_regional/wechat.py +67 -0
- agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
- agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
- agent_wire/channels/tier3_regional/xueqiu.py +77 -0
- agent_wire/channels/tier3_regional/zhihu.py +57 -0
- agent_wire/cli.py +403 -0
- agent_wire/compression/__init__.py +0 -0
- agent_wire/compression/anchor_slicer.py +45 -0
- agent_wire/compression/bm25_slicer.py +151 -0
- agent_wire/compression/dam_formatter.py +74 -0
- agent_wire/compression/url_compactor.py +45 -0
- agent_wire/config.py +52 -0
- agent_wire/engine/__init__.py +0 -0
- agent_wire/engine/challenge.py +58 -0
- agent_wire/engine/http_pool.py +136 -0
- agent_wire/engine/stealth_browser.py +90 -0
- agent_wire/engine/waterfall.py +159 -0
- agent_wire/firewall/__init__.py +0 -0
- agent_wire/firewall/inbound_shield.py +116 -0
- agent_wire/firewall/outbound_privacy.py +120 -0
- agent_wire/installers/__init__.py +0 -0
- agent_wire/installers/client_installer.py +315 -0
- agent_wire/media/__init__.py +0 -0
- agent_wire/media/audio_transcriber.py +77 -0
- agent_wire/media/subtitle_cleaner.py +102 -0
- agent_wire/models.py +60 -0
- agent_wire/research/__init__.py +1 -0
- agent_wire/research/brief.py +227 -0
- agent_wire/research/briefing.py +41 -0
- agent_wire/research/cluster.py +128 -0
- agent_wire/research/engine.py +632 -0
- agent_wire/research/library.py +74 -0
- agent_wire/research/planner.py +232 -0
- agent_wire/research/scoring.py +89 -0
- agent_wire/research/store.py +149 -0
- agent_wire/research/watch.py +70 -0
- agent_wire/router.py +107 -0
- agent_wire/server.py +468 -0
- agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
- agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
- agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
- agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
- agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Bluesky (public AT Protocol AppView) and Mastodon (ActivityPub REST), zero auth."""
|
|
2
|
+
|
|
3
|
+
from urllib.parse import quote, urlsplit
|
|
4
|
+
|
|
5
|
+
from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown, listing, ts
|
|
6
|
+
from agent_wire.engine.waterfall import WireFetchError
|
|
7
|
+
from agent_wire.models import WireComment, WireSearchHit
|
|
8
|
+
|
|
9
|
+
BSKY = "https://api.bsky.app/xrpc" # public.api.bsky.app 403s searchPosts for anonymous callers
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _bsky_url(post: dict) -> str:
|
|
13
|
+
rkey = post.get("uri", "").rsplit("/", 1)[-1]
|
|
14
|
+
return f"https://bsky.app/profile/{post.get('author', {}).get('handle')}/post/{rkey}"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _bsky_hit(p: dict) -> WireSearchHit:
|
|
18
|
+
handle = p.get("author", {}).get("handle")
|
|
19
|
+
return WireSearchHit(
|
|
20
|
+
url=_bsky_url(p), title=f"@{handle}", snippet=p.get("record", {}).get("text", ""),
|
|
21
|
+
source="bluesky", rank_score=float(p.get("likeCount") or 0),
|
|
22
|
+
timestamp=ts(p.get("record", {}).get("createdAt")), author=handle,
|
|
23
|
+
engagement={"likes": p.get("likeCount") or 0, "reposts": p.get("repostCount") or 0,
|
|
24
|
+
"comments": p.get("replyCount") or 0},
|
|
25
|
+
) # fmt: skip
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def parse_bsky_thread(
|
|
29
|
+
thread: dict, op: str | None = None, depth: int = 0
|
|
30
|
+
) -> tuple[dict, list[WireComment]]:
|
|
31
|
+
post = thread.get("post", {})
|
|
32
|
+
author = post.get("author", {}).get("handle")
|
|
33
|
+
replies = [
|
|
34
|
+
WireComment(
|
|
35
|
+
id=r["post"]["cid"], body=r["post"].get("record", {}).get("text", ""),
|
|
36
|
+
author_handle=r["post"].get("author", {}).get("handle"), score=r["post"].get("likeCount") or 0,
|
|
37
|
+
timestamp=ts(r["post"].get("record", {}).get("createdAt")), depth=depth,
|
|
38
|
+
is_op=r["post"].get("author", {}).get("handle") == (op or author),
|
|
39
|
+
replies=parse_bsky_thread(r, op or author, depth + 1)[1],
|
|
40
|
+
)
|
|
41
|
+
for r in thread.get("replies") or [] if r.get("post")
|
|
42
|
+
] # fmt: skip
|
|
43
|
+
return post, replies
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def parse_mastodon_context(descendants: list[dict], root_id: str, op: str) -> list[WireComment]:
|
|
47
|
+
children: dict[str, list[dict]] = {}
|
|
48
|
+
for s in descendants:
|
|
49
|
+
children.setdefault(s.get("in_reply_to_id"), []).append(s)
|
|
50
|
+
|
|
51
|
+
def build(parent: str, depth: int) -> list[WireComment]:
|
|
52
|
+
return [WireComment(id=s["id"], body=html_to_markdown(s.get("content", "")),
|
|
53
|
+
author_handle=s.get("account", {}).get("acct"), score=s.get("favourites_count") or 0,
|
|
54
|
+
timestamp=ts(s.get("created_at")), depth=depth,
|
|
55
|
+
is_op=s.get("account", {}).get("acct") == op, replies=build(s["id"], depth + 1))
|
|
56
|
+
for s in children.get(parent, [])] # fmt: skip
|
|
57
|
+
|
|
58
|
+
return build(root_id, 0)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class FediverseChannel(Channel):
|
|
62
|
+
name = "fediverse"
|
|
63
|
+
|
|
64
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
65
|
+
p = urlsplit(url)
|
|
66
|
+
segs = [s for s in p.path.split("/") if s]
|
|
67
|
+
if (p.hostname or "").endswith("bsky.app"):
|
|
68
|
+
if len(segs) >= 4 and segs[0] == "profile" and segs[2] == "post":
|
|
69
|
+
return await self._bsky_post(url, segs[1], segs[3])
|
|
70
|
+
if len(segs) >= 2 and segs[0] == "profile":
|
|
71
|
+
return await self.feed(f"bsky:{segs[1]}")
|
|
72
|
+
raise WireUnsupported("bluesky: expected /profile/<handle>[/post/<id>]")
|
|
73
|
+
if segs and segs[0].startswith("@") and len(segs) > 1 and segs[1].isdigit():
|
|
74
|
+
return await self._mastodon(url, f"https://{p.hostname}", segs[1], include_comments)
|
|
75
|
+
if len(segs) == 1 and segs[0].startswith("@"): # profile; may not be a Mastodon host
|
|
76
|
+
who = segs[0] if "@" in segs[0][1:] else f"{segs[0]}@{p.hostname}"
|
|
77
|
+
try:
|
|
78
|
+
return await self.feed(who)
|
|
79
|
+
except (WireFetchError, ValueError, KeyError, TypeError):
|
|
80
|
+
pass
|
|
81
|
+
return await self.ctx.channel("web").read(url, query=query, section=section)
|
|
82
|
+
|
|
83
|
+
async def _did(self, actor: str) -> str:
|
|
84
|
+
if actor.startswith("did:"):
|
|
85
|
+
return actor
|
|
86
|
+
data, _ = await self.fetch_json(f"{BSKY}/com.atproto.identity.resolveHandle?handle={actor}")
|
|
87
|
+
return data["did"]
|
|
88
|
+
|
|
89
|
+
async def _bsky_post(self, url, actor, rkey):
|
|
90
|
+
did = await self._did(actor)
|
|
91
|
+
uri = f"at://{did}/app.bsky.feed.post/{rkey}"
|
|
92
|
+
data, f = await self.fetch_json(
|
|
93
|
+
f"{BSKY}/app.bsky.feed.getPostThread?uri={quote(uri)}&depth=6"
|
|
94
|
+
)
|
|
95
|
+
post, replies = parse_bsky_thread(data.get("thread", {}))
|
|
96
|
+
record = post.get("record", {})
|
|
97
|
+
embed = (post.get("embed") or {}).get("external") or {}
|
|
98
|
+
body = record.get("text", "") + (
|
|
99
|
+
f"\n\nLink: {embed['uri']} ({embed.get('title', '')})" if embed.get("uri") else ""
|
|
100
|
+
)
|
|
101
|
+
return self.doc(
|
|
102
|
+
f, url, title=f"@{post.get('author', {}).get('handle')} on Bluesky",
|
|
103
|
+
author_handle=post.get("author", {}).get("handle"), timestamp=ts(record.get("createdAt")),
|
|
104
|
+
score=post.get("likeCount"), body=body, comments=replies,
|
|
105
|
+
metadata={"likes": post.get("likeCount"), "reposts": post.get("repostCount"),
|
|
106
|
+
"replies": post.get("replyCount")},
|
|
107
|
+
) # fmt: skip
|
|
108
|
+
|
|
109
|
+
async def _mastodon(self, url, base, status_id, include_comments):
|
|
110
|
+
status, f = await self.fetch_json(f"{base}/api/v1/statuses/{status_id}")
|
|
111
|
+
op = status.get("account", {}).get("acct")
|
|
112
|
+
comments = []
|
|
113
|
+
if include_comments:
|
|
114
|
+
ctx, _ = await self.fetch_json(f"{base}/api/v1/statuses/{status_id}/context")
|
|
115
|
+
comments = parse_mastodon_context(ctx.get("descendants", []), status_id, op)
|
|
116
|
+
return self.doc(
|
|
117
|
+
f, url, title=f"@{op} on {urlsplit(base).hostname}", author_handle=op,
|
|
118
|
+
timestamp=ts(status.get("created_at")), score=status.get("favourites_count"),
|
|
119
|
+
body=html_to_markdown(status.get("content", "")), comments=comments,
|
|
120
|
+
metadata={"boosts": status.get("reblogs_count"), "replies": status.get("replies_count")},
|
|
121
|
+
) # fmt: skip
|
|
122
|
+
|
|
123
|
+
async def search(self, query, *, limit=5, sort="relevance", since=None):
|
|
124
|
+
order = "latest" if sort in ("new", "latest") else "top"
|
|
125
|
+
window = f"&since={since.strftime('%Y-%m-%dT%H:%M:%SZ')}" if since else ""
|
|
126
|
+
data, _ = await self.fetch_json(
|
|
127
|
+
f"{BSKY}/app.bsky.feed.searchPosts?q={quote(query)}&limit={limit}&sort={order}{window}"
|
|
128
|
+
)
|
|
129
|
+
return [_bsky_hit(p) for p in data.get("posts", [])]
|
|
130
|
+
|
|
131
|
+
async def mastodon_tag(self, tag: str, *, limit=10, instance="mastodon.social"):
|
|
132
|
+
"""Public hashtag timeline (no auth on most instances)."""
|
|
133
|
+
tag = "".join(ch for ch in tag if ch.isalnum() or ch == "_")
|
|
134
|
+
data, _ = await self.fetch_json(
|
|
135
|
+
f"https://{instance}/api/v1/timelines/tag/{tag}?limit={limit}"
|
|
136
|
+
)
|
|
137
|
+
hits = []
|
|
138
|
+
for s in data:
|
|
139
|
+
s = s.get("reblog") or s
|
|
140
|
+
acct = (s.get("account") or {}).get("acct")
|
|
141
|
+
hits.append(WireSearchHit(
|
|
142
|
+
url=s.get("url") or s.get("uri", ""), title=f"@{acct}",
|
|
143
|
+
snippet=html_to_markdown(s.get("content", ""))[:300], source="mastodon",
|
|
144
|
+
rank_score=float(s.get("favourites_count") or 0), timestamp=ts(s.get("created_at")),
|
|
145
|
+
author=acct, engagement={"likes": s.get("favourites_count") or 0,
|
|
146
|
+
"reposts": s.get("reblogs_count") or 0,
|
|
147
|
+
"comments": s.get("replies_count") or 0},
|
|
148
|
+
)) # fmt: skip
|
|
149
|
+
return hits
|
|
150
|
+
|
|
151
|
+
async def feed(self, target, *, sort="hot", limit=10):
|
|
152
|
+
t = target.strip().removeprefix("bsky:")
|
|
153
|
+
if t.count("@") == 2 or (t.startswith("@") and "@" in t[1:]): # @user@instance -> Mastodon
|
|
154
|
+
user, instance = t.lstrip("@").split("@", 1)
|
|
155
|
+
base = f"https://{instance}"
|
|
156
|
+
acct, f = await self.fetch_json(f"{base}/api/v1/accounts/lookup?acct={user}")
|
|
157
|
+
statuses, f = await self.fetch_json(
|
|
158
|
+
f"{base}/api/v1/accounts/{acct['id']}/statuses?limit={limit}&exclude_replies=true"
|
|
159
|
+
)
|
|
160
|
+
statuses = [s.get("reblog") or s for s in statuses] # boosts carry their content inside
|
|
161
|
+
items = [{"title": html_to_markdown(s.get("content", ""))[:120] or s.get("url"), "url": s.get("url"),
|
|
162
|
+
"meta": [f"♥{s.get('favourites_count', 0)}", f"↻{s.get('reblogs_count', 0)}",
|
|
163
|
+
(s.get("created_at") or "")[:10]]} for s in statuses] # fmt: skip
|
|
164
|
+
return self.doc(f, f"{base}/@{user}", title=f"@{user}@{instance}", body=listing(items))
|
|
165
|
+
actor = t.lstrip("@")
|
|
166
|
+
data, f = await self.fetch_json(
|
|
167
|
+
f"{BSKY}/app.bsky.feed.getAuthorFeed?actor={actor}&limit={limit}&filter=posts_no_replies"
|
|
168
|
+
)
|
|
169
|
+
posts = [i["post"] for i in data.get("feed", [])]
|
|
170
|
+
items = [{"title": p.get("record", {}).get("text", "")[:120], "url": _bsky_url(p),
|
|
171
|
+
"meta": [f"♥{p.get('likeCount', 0)}", f"↻{p.get('repostCount', 0)}",
|
|
172
|
+
(p.get("record", {}).get("createdAt") or "")[:10]]} for p in posts] # fmt: skip
|
|
173
|
+
return self.doc(
|
|
174
|
+
f,
|
|
175
|
+
f"https://bsky.app/profile/{actor}",
|
|
176
|
+
title=f"@{actor} on Bluesky",
|
|
177
|
+
body=listing(items),
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
CHANNEL = FediverseChannel
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Package and model hubs: PyPI, npm, crates.io, Hugging Face (models + datasets)."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from urllib.parse import quote, urlsplit
|
|
5
|
+
|
|
6
|
+
from curl_cffi.requests.exceptions import RequestException
|
|
7
|
+
from selectolax.lexbor import LexborHTMLParser
|
|
8
|
+
|
|
9
|
+
from agent_wire.channels.base import Channel, WireUnsupported, html_to_markdown
|
|
10
|
+
from agent_wire.engine.waterfall import WireFetchError
|
|
11
|
+
from agent_wire.models import WireSearchHit
|
|
12
|
+
|
|
13
|
+
SEARCH_ERRORS = (WireFetchError, RequestException, ValueError, KeyError)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def parse_pypi_search(html: str) -> list[WireSearchHit]:
|
|
17
|
+
hits = []
|
|
18
|
+
for a in LexborHTMLParser(html).css("a.package-snippet"):
|
|
19
|
+
name = a.css_first(".package-snippet__name")
|
|
20
|
+
desc = a.css_first(".package-snippet__description")
|
|
21
|
+
if name:
|
|
22
|
+
hits.append(WireSearchHit(url=f"https://pypi.org{a.attributes.get('href', '')}",
|
|
23
|
+
title=f"pypi: {name.text(strip=True)}",
|
|
24
|
+
snippet=desc.text(strip=True) if desc else "", source="pypi")) # fmt: skip
|
|
25
|
+
return hits
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class HubsChannel(Channel):
|
|
29
|
+
name = "hubs"
|
|
30
|
+
|
|
31
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
32
|
+
p = urlsplit(url)
|
|
33
|
+
host, segs = (p.hostname or "").removeprefix("www."), [s for s in p.path.split("/") if s]
|
|
34
|
+
if host == "pypi.org" and segs[:1] == ["project"] and len(segs) > 1:
|
|
35
|
+
return await self._pypi(url, segs[1], segs[2] if len(segs) > 2 else None)
|
|
36
|
+
if host.endswith("npmjs.com") and segs[:1] == ["package"] and len(segs) > 1:
|
|
37
|
+
name = "/".join(segs[1:3]) if segs[1].startswith("@") else segs[1]
|
|
38
|
+
return await self._npm(url, name)
|
|
39
|
+
if host == "crates.io" and segs[:1] == ["crates"] and len(segs) > 1:
|
|
40
|
+
return await self._crate(url, segs[1])
|
|
41
|
+
if host == "huggingface.co" and segs:
|
|
42
|
+
kind = (
|
|
43
|
+
"datasets"
|
|
44
|
+
if segs[0] == "datasets"
|
|
45
|
+
else "spaces"
|
|
46
|
+
if segs[0] == "spaces"
|
|
47
|
+
else "models"
|
|
48
|
+
)
|
|
49
|
+
repo = "/".join(segs[1:3] if kind != "models" else segs[:2])
|
|
50
|
+
return await self._hf(url, kind, repo)
|
|
51
|
+
return await self.ctx.channel("web").read(url, query=query, section=section)
|
|
52
|
+
|
|
53
|
+
async def _pypi(self, url, name, version):
|
|
54
|
+
path = f"{name}/{version}" if version else name
|
|
55
|
+
data, f = await self.fetch_json(f"https://pypi.org/pypi/{path}/json")
|
|
56
|
+
info = data.get("info", {})
|
|
57
|
+
urls = info.get("project_urls") or {}
|
|
58
|
+
meta = {"version": info.get("version"), "license": (info.get("license") or "")[:60] or None,
|
|
59
|
+
"requires_python": info.get("requires_python"),
|
|
60
|
+
"homepage": urls.get("Homepage") or urls.get("Source") or info.get("home_page")} # fmt: skip
|
|
61
|
+
body = f"{info.get('summary') or ''}\n\n{info.get('description') or ''}".strip()
|
|
62
|
+
return self.doc(f, url, title=f"{info.get('name', name)} (PyPI)", author_handle=info.get("author"),
|
|
63
|
+
body=body, metadata=meta) # fmt: skip
|
|
64
|
+
|
|
65
|
+
async def _npm(self, url, name):
|
|
66
|
+
(data, f), downloads = await asyncio.gather(
|
|
67
|
+
self.fetch_json(f"https://registry.npmjs.org/{name.replace('/', '%2F')}"),
|
|
68
|
+
self._safe_json(f"https://api.npmjs.org/downloads/point/last-week/{name}"),
|
|
69
|
+
)
|
|
70
|
+
latest = (data.get("dist-tags") or {}).get("latest")
|
|
71
|
+
ver = (data.get("versions") or {}).get(latest, {})
|
|
72
|
+
repo = data.get("repository") or {}
|
|
73
|
+
meta = {"version": latest, "license": data.get("license") if isinstance(data.get("license"), str) else None,
|
|
74
|
+
"weekly_downloads": downloads.get("downloads"),
|
|
75
|
+
"repository": repo.get("url") if isinstance(repo, dict) else repo,
|
|
76
|
+
"modified": (data.get("time") or {}).get("modified", "")[:10] or None} # fmt: skip
|
|
77
|
+
body = f"{data.get('description') or ver.get('description') or ''}\n\n{data.get('readme') or ''}".strip()
|
|
78
|
+
return self.doc(f, url, title=f"{name} (npm)", body=body, metadata=meta)
|
|
79
|
+
|
|
80
|
+
async def _crate(self, url, name):
|
|
81
|
+
data, f = await self.fetch_json(f"https://crates.io/api/v1/crates/{name}")
|
|
82
|
+
c = data.get("crate", {})
|
|
83
|
+
readme = ""
|
|
84
|
+
if ver := c.get("max_stable_version") or c.get("max_version"):
|
|
85
|
+
try:
|
|
86
|
+
rf = await self.fetch(f"https://crates.io/api/v1/crates/{name}/{ver}/readme",
|
|
87
|
+
archive=False, browser=False) # fmt: skip
|
|
88
|
+
readme = html_to_markdown(rf.text)
|
|
89
|
+
except WireFetchError:
|
|
90
|
+
readme = ""
|
|
91
|
+
meta = {"version": ver, "downloads": c.get("downloads"), "recent_downloads": c.get("recent_downloads"),
|
|
92
|
+
"repository": c.get("repository")} # fmt: skip
|
|
93
|
+
return self.doc(f, url, title=f"{name} (crates.io)", body=f"{c.get('description') or ''}\n\n{readme}".strip(),
|
|
94
|
+
metadata=meta) # fmt: skip
|
|
95
|
+
|
|
96
|
+
async def _hf(self, url, kind, repo):
|
|
97
|
+
api = f"https://huggingface.co/api/{kind}/{repo}"
|
|
98
|
+
raw = f"https://huggingface.co/{'' if kind == 'models' else kind + '/'}{repo}/raw/main/README.md"
|
|
99
|
+
(data, f), readme = await asyncio.gather(self.fetch_json(api), self._safe_text(raw))
|
|
100
|
+
meta = {"downloads": data.get("downloads"), "likes": data.get("likes"),
|
|
101
|
+
"pipeline": data.get("pipeline_tag"), "library": data.get("library_name"),
|
|
102
|
+
"license": next((t.split(":", 1)[1] for t in data.get("tags", []) if t.startswith("license:")), None),
|
|
103
|
+
"updated": (data.get("lastModified") or "")[:10] or None} # fmt: skip
|
|
104
|
+
return self.doc(f, url, title=f"{repo} (Hugging Face {kind[:-1]})", author_handle=data.get("author"),
|
|
105
|
+
body=readme, metadata=meta) # fmt: skip
|
|
106
|
+
|
|
107
|
+
async def _safe_json(self, url) -> dict:
|
|
108
|
+
try:
|
|
109
|
+
return (await self.fetch_json(url))[0]
|
|
110
|
+
except SEARCH_ERRORS:
|
|
111
|
+
return {}
|
|
112
|
+
|
|
113
|
+
async def _safe_text(self, url) -> str:
|
|
114
|
+
try:
|
|
115
|
+
return (await self.fetch(url, archive=False, browser=False)).text
|
|
116
|
+
except SEARCH_ERRORS:
|
|
117
|
+
return ""
|
|
118
|
+
|
|
119
|
+
async def search(self, query, *, limit=5, sort="relevance"):
|
|
120
|
+
q = quote(query)
|
|
121
|
+
|
|
122
|
+
async def pypi():
|
|
123
|
+
f = await self.fetch(f"https://pypi.org/search/?q={q}", archive=False, browser=False)
|
|
124
|
+
return parse_pypi_search(f.text)
|
|
125
|
+
|
|
126
|
+
async def npm():
|
|
127
|
+
d, _ = await self.fetch_json(
|
|
128
|
+
f"https://registry.npmjs.org/-/v1/search?text={q}&size={limit}"
|
|
129
|
+
)
|
|
130
|
+
return [WireSearchHit(url=o["package"]["links"]["npm"], title=f"npm: {o['package']['name']}",
|
|
131
|
+
snippet=o["package"].get("description") or "", source="npm")
|
|
132
|
+
for o in d.get("objects", [])] # fmt: skip
|
|
133
|
+
|
|
134
|
+
async def crates():
|
|
135
|
+
d, _ = await self.fetch_json(f"https://crates.io/api/v1/crates?q={q}&per_page={limit}")
|
|
136
|
+
return [WireSearchHit(url=f"https://crates.io/crates/{c['name']}", title=f"crate: {c['name']}",
|
|
137
|
+
snippet=c.get("description") or "", source="crates")
|
|
138
|
+
for c in d.get("crates", [])] # fmt: skip
|
|
139
|
+
|
|
140
|
+
async def hf():
|
|
141
|
+
d, _ = await self.fetch_json(
|
|
142
|
+
f"https://huggingface.co/api/models?search={q}&limit={limit}&sort=downloads"
|
|
143
|
+
)
|
|
144
|
+
return [WireSearchHit(url=f"https://huggingface.co/{m['id']}", title=f"hf: {m['id']}",
|
|
145
|
+
snippet=f"{m.get('downloads', 0)} downloads · {m.get('pipeline_tag') or ''}",
|
|
146
|
+
source="huggingface") for m in d] # fmt: skip
|
|
147
|
+
|
|
148
|
+
async def safe(fn):
|
|
149
|
+
try:
|
|
150
|
+
return (await fn())[:limit]
|
|
151
|
+
except SEARCH_ERRORS:
|
|
152
|
+
return []
|
|
153
|
+
|
|
154
|
+
lists = await asyncio.gather(*(safe(fn) for fn in (pypi, npm, crates, hf)))
|
|
155
|
+
hits = [h for group in lists for h in group]
|
|
156
|
+
if not hits:
|
|
157
|
+
raise WireUnsupported("no package/model hub returned results")
|
|
158
|
+
return hits
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
CHANNEL = HubsChannel
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""LinkedIn: public guest job pages and search; posts/profiles via public JSON-LD or a local session."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from urllib.parse import parse_qs, quote, urlsplit
|
|
5
|
+
|
|
6
|
+
from selectolax.lexbor import LexborHTMLParser
|
|
7
|
+
|
|
8
|
+
from agent_wire.channels.base import (
|
|
9
|
+
Channel,
|
|
10
|
+
WireUnsupported,
|
|
11
|
+
html_to_markdown,
|
|
12
|
+
json_ld,
|
|
13
|
+
meta_tags,
|
|
14
|
+
ts,
|
|
15
|
+
)
|
|
16
|
+
from agent_wire.firewall.outbound_privacy import parse_json
|
|
17
|
+
from agent_wire.models import WireComment, WireSearchHit
|
|
18
|
+
|
|
19
|
+
GUEST = "https://www.linkedin.com/jobs-guest/jobs/api"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def job_id(url: str) -> str | None:
|
|
23
|
+
p = urlsplit(url)
|
|
24
|
+
if m := re.search(r"/jobs/view/(?:[\w-]*?-)?(\d{6,})", p.path):
|
|
25
|
+
return m[1]
|
|
26
|
+
return parse_qs(p.query).get("currentJobId", [None])[0]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_job(html: str) -> dict:
|
|
30
|
+
tree = LexborHTMLParser(html)
|
|
31
|
+
|
|
32
|
+
def text(sel):
|
|
33
|
+
n = tree.css_first(sel)
|
|
34
|
+
return n.text(strip=True) if n else None
|
|
35
|
+
|
|
36
|
+
desc = tree.css_first(".show-more-less-html__markup, .description__text")
|
|
37
|
+
criteria = {
|
|
38
|
+
(li.css_first("h3").text(strip=True) if li.css_first("h3") else ""): (
|
|
39
|
+
li.css_first("span").text(strip=True) if li.css_first("span") else "")
|
|
40
|
+
for li in tree.css("li.description__job-criteria-item")
|
|
41
|
+
} # fmt: skip
|
|
42
|
+
return {
|
|
43
|
+
"title": text("h2.top-card-layout__title, h1.top-card-layout__title, .topcard__title"),
|
|
44
|
+
"company": text("a.topcard__org-name-link, .topcard__flavor a"),
|
|
45
|
+
"location": text(".topcard__flavor--bullet"),
|
|
46
|
+
"body": html_to_markdown(desc.html) if desc else "",
|
|
47
|
+
"criteria": {k: v for k, v in criteria.items() if k},
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_job_search(html: str) -> list[WireSearchHit]:
|
|
52
|
+
hits = []
|
|
53
|
+
for card in LexborHTMLParser(f"<ul>{html}</ul>").css("li"):
|
|
54
|
+
a = card.css_first("a.base-card__full-link, a[href*='/jobs/view/']")
|
|
55
|
+
title = card.css_first(".base-search-card__title")
|
|
56
|
+
if not a or not title:
|
|
57
|
+
continue
|
|
58
|
+
company = card.css_first(".base-search-card__subtitle")
|
|
59
|
+
loc = card.css_first(".job-search-card__location")
|
|
60
|
+
hits.append(WireSearchHit(
|
|
61
|
+
url=(a.attributes.get("href") or "").split("?")[0], title=title.text(strip=True),
|
|
62
|
+
snippet=" · ".join(n.text(strip=True) for n in (company, loc) if n), source="linkedin")) # fmt: skip
|
|
63
|
+
return hits
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def parse_public_page(html: str) -> dict:
|
|
67
|
+
"""Public post / profile / company pages: JSON-LD first, OpenGraph as fallback."""
|
|
68
|
+
for item in json_ld(html):
|
|
69
|
+
kind = item.get("@type")
|
|
70
|
+
if kind in ("SocialMediaPosting", "Article", "DiscussionForumPosting"):
|
|
71
|
+
author = item.get("author") or {}
|
|
72
|
+
author = author[0] if isinstance(author, list) and author else author
|
|
73
|
+
comments = [
|
|
74
|
+
WireComment(id=str(i), body=c.get("text") or "", author_handle=(c.get("author") or {}).get("name"),
|
|
75
|
+
timestamp=ts(c.get("datePublished")))
|
|
76
|
+
for i, c in enumerate(item.get("comment") or []) if isinstance(c, dict)
|
|
77
|
+
] # fmt: skip
|
|
78
|
+
stats = {s.get("interactionType", "").split("/")[-1]: s.get("userInteractionCount")
|
|
79
|
+
for s in item.get("interactionStatistic") or [] if isinstance(s, dict)} # fmt: skip
|
|
80
|
+
return {"title": item.get("headline") or (item.get("articleBody") or "")[:80],
|
|
81
|
+
"author_handle": author.get("name") if isinstance(author, dict) else None,
|
|
82
|
+
"timestamp": ts(item.get("datePublished")), "body": item.get("articleBody") or "",
|
|
83
|
+
"comments": comments, "metadata": {k: v for k, v in stats.items() if k}} # fmt: skip
|
|
84
|
+
if kind in ("Person", "Organization", "ProfilePage"):
|
|
85
|
+
entity = item.get("mainEntity", item)
|
|
86
|
+
return {"title": entity.get("name"), "body": entity.get("description") or "",
|
|
87
|
+
"metadata": {"followers": (entity.get("interactionStatistic") or {}).get("userInteractionCount")
|
|
88
|
+
if isinstance(entity.get("interactionStatistic"), dict) else None,
|
|
89
|
+
"location": (entity.get("address") or {}).get("addressLocality")
|
|
90
|
+
if isinstance(entity.get("address"), dict) else None}} # fmt: skip
|
|
91
|
+
m = meta_tags(html)
|
|
92
|
+
if not m.get("og:title"):
|
|
93
|
+
raise WireUnsupported("linkedin returned a login wall; run `wire auth login linkedin.com`")
|
|
94
|
+
return {"title": m.get("og:title"), "body": m.get("og:description", "")}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def parse_voyager_profile(data: dict) -> dict:
|
|
98
|
+
included = data.get("included", [])
|
|
99
|
+
profile = next((i for i in included if i.get("$type", "").endswith(".Profile")), {})
|
|
100
|
+
positions = [i for i in included if i.get("$type", "").endswith(".Position")]
|
|
101
|
+
schools = [i for i in included if i.get("$type", "").endswith(".Education")]
|
|
102
|
+
lines = [profile.get("headline") or "", profile.get("summary") or ""]
|
|
103
|
+
if positions:
|
|
104
|
+
lines.append(
|
|
105
|
+
"## Experience\n"
|
|
106
|
+
+ "\n".join(f"- {p.get('title')} · {p.get('companyName')}" for p in positions)
|
|
107
|
+
)
|
|
108
|
+
if schools:
|
|
109
|
+
lines.append("## Education\n" + "\n".join(f"- {s.get('schoolName')}" for s in schools))
|
|
110
|
+
return {"title": f"{profile.get('firstName', '')} {profile.get('lastName', '')}".strip(),
|
|
111
|
+
"body": "\n\n".join(x for x in lines if x), "metadata": {"location": profile.get("locationName")}} # fmt: skip
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class LinkedInChannel(Channel):
|
|
115
|
+
name = "linkedin"
|
|
116
|
+
|
|
117
|
+
async def read(self, url, *, query=None, section=None, include_comments=True):
|
|
118
|
+
if jid := job_id(url):
|
|
119
|
+
f = await self.fetch(f"{GUEST}/jobPosting/{jid}", archive=False)
|
|
120
|
+
job = parse_job(f.text)
|
|
121
|
+
if not job["title"]:
|
|
122
|
+
raise WireUnsupported(f"linkedin job {jid} not found or expired")
|
|
123
|
+
return self.doc(f, f"https://www.linkedin.com/jobs/view/{jid}", title=job["title"],
|
|
124
|
+
author_handle=job["company"], body=job["body"],
|
|
125
|
+
metadata={"location": job["location"], **job["criteria"]}) # fmt: skip
|
|
126
|
+
segs = [s for s in urlsplit(url).path.split("/") if s]
|
|
127
|
+
jar = await self.ctx.vault.cookies_for("linkedin.com") if self.ctx.vault else {}
|
|
128
|
+
if segs[:1] == ["in"] and len(segs) > 1 and jar.get("JSESSIONID"):
|
|
129
|
+
csrf = jar["JSESSIONID"].strip('"')
|
|
130
|
+
f = await self.fetch(
|
|
131
|
+
f"https://www.linkedin.com/voyager/api/identity/profiles/{segs[1]}/profileView",
|
|
132
|
+
headers={"csrf-token": csrf, "accept": "application/vnd.linkedin.normalized+json+2.1"},
|
|
133
|
+
archive=False, browser=False, cookies=jar,
|
|
134
|
+
) # fmt: skip
|
|
135
|
+
return self.doc(f, url, **parse_voyager_profile(parse_json(f.content)))
|
|
136
|
+
f = await self.fetch(url, archive=False)
|
|
137
|
+
return self.doc(f, url, **parse_public_page(f.text))
|
|
138
|
+
|
|
139
|
+
async def search(self, query, *, limit=5, sort="relevance"):
|
|
140
|
+
f = await self.fetch(
|
|
141
|
+
f"{GUEST}/seeMoreJobPostings/search?keywords={quote(query)}&start=0", archive=False
|
|
142
|
+
)
|
|
143
|
+
return parse_job_search(f.text)[:limit]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
CHANNEL = LinkedInChannel
|