agent-wire-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_wire/__init__.py +1 -0
- agent_wire/__main__.py +3 -0
- agent_wire/assets/SKILL.md +26 -0
- agent_wire/assets/cursor_rule.mdc +13 -0
- agent_wire/assets/hermes_skill.md +28 -0
- agent_wire/assets/windsurf_rules.md +14 -0
- agent_wire/assets/wire-worker.md +20 -0
- agent_wire/auth/__init__.py +0 -0
- agent_wire/auth/browser_paths.py +143 -0
- agent_wire/auth/cookie_vault.py +181 -0
- agent_wire/auth/rate_limiter.py +26 -0
- agent_wire/cache/__init__.py +0 -0
- agent_wire/cache/ring_buffer.py +94 -0
- agent_wire/cache/sqlite_ledger.py +132 -0
- agent_wire/channels/__init__.py +41 -0
- agent_wire/channels/base.py +249 -0
- agent_wire/channels/tier1_global/__init__.py +0 -0
- agent_wire/channels/tier1_global/github.py +371 -0
- agent_wire/channels/tier1_global/hackernews.py +135 -0
- agent_wire/channels/tier1_global/reddit.py +371 -0
- agent_wire/channels/tier1_global/rss.py +129 -0
- agent_wire/channels/tier1_global/search.py +160 -0
- agent_wire/channels/tier1_global/stackoverflow.py +94 -0
- agent_wire/channels/tier1_global/web.py +98 -0
- agent_wire/channels/tier1_global/wikipedia.py +91 -0
- agent_wire/channels/tier1_global/x_twitter.py +380 -0
- agent_wire/channels/tier1_global/youtube.py +223 -0
- agent_wire/channels/tier2_pro/__init__.py +0 -0
- agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
- agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
- agent_wire/channels/tier2_pro/devto.py +114 -0
- agent_wire/channels/tier2_pro/fediverse.py +181 -0
- agent_wire/channels/tier2_pro/hubs.py +161 -0
- agent_wire/channels/tier2_pro/linkedin.py +146 -0
- agent_wire/channels/tier2_pro/markets.py +207 -0
- agent_wire/channels/tier2_pro/meta_social.py +104 -0
- agent_wire/channels/tier2_pro/news.py +173 -0
- agent_wire/channels/tier2_pro/polymarket.py +105 -0
- agent_wire/channels/tier2_pro/publishing.py +84 -0
- agent_wire/channels/tier2_pro/stocktwits.py +79 -0
- agent_wire/channels/tier3_regional/bilibili.py +171 -0
- agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
- agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
- agent_wire/channels/tier3_regional/v2ex.py +70 -0
- agent_wire/channels/tier3_regional/wechat.py +67 -0
- agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
- agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
- agent_wire/channels/tier3_regional/xueqiu.py +77 -0
- agent_wire/channels/tier3_regional/zhihu.py +57 -0
- agent_wire/cli.py +403 -0
- agent_wire/compression/__init__.py +0 -0
- agent_wire/compression/anchor_slicer.py +45 -0
- agent_wire/compression/bm25_slicer.py +151 -0
- agent_wire/compression/dam_formatter.py +74 -0
- agent_wire/compression/url_compactor.py +45 -0
- agent_wire/config.py +52 -0
- agent_wire/engine/__init__.py +0 -0
- agent_wire/engine/challenge.py +58 -0
- agent_wire/engine/http_pool.py +136 -0
- agent_wire/engine/stealth_browser.py +90 -0
- agent_wire/engine/waterfall.py +159 -0
- agent_wire/firewall/__init__.py +0 -0
- agent_wire/firewall/inbound_shield.py +116 -0
- agent_wire/firewall/outbound_privacy.py +120 -0
- agent_wire/installers/__init__.py +0 -0
- agent_wire/installers/client_installer.py +315 -0
- agent_wire/media/__init__.py +0 -0
- agent_wire/media/audio_transcriber.py +77 -0
- agent_wire/media/subtitle_cleaner.py +102 -0
- agent_wire/models.py +60 -0
- agent_wire/research/__init__.py +1 -0
- agent_wire/research/brief.py +227 -0
- agent_wire/research/briefing.py +41 -0
- agent_wire/research/cluster.py +128 -0
- agent_wire/research/engine.py +632 -0
- agent_wire/research/library.py +74 -0
- agent_wire/research/planner.py +232 -0
- agent_wire/research/scoring.py +89 -0
- agent_wire/research/store.py +149 -0
- agent_wire/research/watch.py +70 -0
- agent_wire/router.py +107 -0
- agent_wire/server.py +468 -0
- agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
- agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
- agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
- agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
- agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Your research library: offline search over saved briefs, plus a local site and Atom feed."""
|
|
2
|
+
|
|
3
|
+
import html
|
|
4
|
+
import xml.etree.ElementTree as ET
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from agent_wire.research.brief import page
|
|
8
|
+
from agent_wire.research.store import Store
|
|
9
|
+
|
|
10
|
+
ATOM = "http://www.w3.org/2005/Atom"
|
|
11
|
+
EPOCH = "1970-01-01T00:00:00+00:00"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Library:
|
|
15
|
+
def __init__(self, store: Store, home: Path):
|
|
16
|
+
self.store = store
|
|
17
|
+
self.dir = Path(home) / "library"
|
|
18
|
+
|
|
19
|
+
def related(self, topic: str, limit: int = 3, exclude: int | None = None) -> list[dict]:
|
|
20
|
+
return self.store.search(topic, limit=limit, exclude=exclude)
|
|
21
|
+
|
|
22
|
+
def search_markdown(self, query: str, limit: int = 10) -> str:
|
|
23
|
+
hits = self.store.search(query, limit=limit)
|
|
24
|
+
if not hits:
|
|
25
|
+
return f"no saved briefs match {query!r} (save runs with store=True / --store)"
|
|
26
|
+
return "\n".join(f"{i}. run #{h['id']} · {h['created_at'][:10]} · {h['topic']} ({h['mode']}, "
|
|
27
|
+
f"{h['days']}d) · {h['headline']}" for i, h in enumerate(hits, 1)) # fmt: skip
|
|
28
|
+
|
|
29
|
+
def build(self, limit: int = 200) -> Path:
|
|
30
|
+
"""Write index.html, feed.xml and one page per saved run under WIRE_HOME/library."""
|
|
31
|
+
runs_dir = self.dir / "runs"
|
|
32
|
+
runs_dir.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
runs = self.store.runs(limit=limit)
|
|
34
|
+
for r in runs:
|
|
35
|
+
full = self.store.run(r["id"])
|
|
36
|
+
(runs_dir / f"{r['id']}.html").write_text(
|
|
37
|
+
page(full["markdown"], f"#{r['id']} · {r['topic']}"), encoding="utf-8"
|
|
38
|
+
)
|
|
39
|
+
rows = "".join(
|
|
40
|
+
f"<tr><td>{r['created_at'][:10]}</td><td><a href='runs/{r['id']}.html'>"
|
|
41
|
+
f"{html.escape(r['topic'])}</a></td><td>{r['mode']}</td><td>{r['days']}d</td>"
|
|
42
|
+
f"<td>{html.escape(r['headline'])}</td></tr>"
|
|
43
|
+
for r in runs
|
|
44
|
+
)
|
|
45
|
+
table = ("<p><a href='feed.xml'>Atom feed</a></p><table><tr><th>date</th><th>topic</th>"
|
|
46
|
+
f"<th>mode</th><th>window</th><th>headline</th></tr>{rows}</table>") # fmt: skip
|
|
47
|
+
index = page("# Research library", "agent-wire library").replace(
|
|
48
|
+
"<h1>Research library</h1>", "<h1>Research library</h1>" + table
|
|
49
|
+
)
|
|
50
|
+
(self.dir / "index.html").write_text(index, encoding="utf-8")
|
|
51
|
+
(self.dir / "feed.xml").write_bytes(self.atom(runs))
|
|
52
|
+
return self.dir
|
|
53
|
+
|
|
54
|
+
def atom(self, runs: list[dict]) -> bytes:
|
|
55
|
+
ET.register_namespace("", ATOM)
|
|
56
|
+
|
|
57
|
+
def sub(parent, tag, text=None, **attrs):
|
|
58
|
+
el = ET.SubElement(parent, f"{{{ATOM}}}{tag}", **attrs)
|
|
59
|
+
el.text = text
|
|
60
|
+
return el
|
|
61
|
+
|
|
62
|
+
feed = ET.Element(f"{{{ATOM}}}feed")
|
|
63
|
+
sub(feed, "title", "agent-wire research library")
|
|
64
|
+
sub(feed, "id", "urn:agent-wire:library")
|
|
65
|
+
sub(feed, "updated", runs[0]["created_at"] if runs else EPOCH)
|
|
66
|
+
sub(sub(feed, "author"), "name", "agent-wire")
|
|
67
|
+
for r in runs:
|
|
68
|
+
e = sub(feed, "entry")
|
|
69
|
+
sub(e, "title", f"{r['topic']} · last {r['days']} days")
|
|
70
|
+
sub(e, "id", f"urn:agent-wire:run:{r['id']}")
|
|
71
|
+
sub(e, "updated", r["created_at"])
|
|
72
|
+
sub(e, "link", href=(self.dir / "runs" / f"{r['id']}.html").as_uri())
|
|
73
|
+
sub(e, "summary", r["headline"])
|
|
74
|
+
return ET.tostring(feed, encoding="utf-8", xml_declaration=True)
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""Pre-research: decide mode, window, sources and per-source queries before any network call."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import asdict, dataclass, field
|
|
5
|
+
from datetime import UTC, datetime, timedelta
|
|
6
|
+
|
|
7
|
+
# Key-free sources researched by default; others join when the topic calls for them.
|
|
8
|
+
CORE = (
|
|
9
|
+
"reddit",
|
|
10
|
+
"hackernews",
|
|
11
|
+
"youtube",
|
|
12
|
+
"github",
|
|
13
|
+
"news",
|
|
14
|
+
"bluesky",
|
|
15
|
+
"mastodon",
|
|
16
|
+
"polymarket",
|
|
17
|
+
"web",
|
|
18
|
+
)
|
|
19
|
+
TECH = ("devto", "stackoverflow")
|
|
20
|
+
ALL_SOURCES = (*CORE, *TECH, "arxiv", "stocktwits", "x", "lobsters")
|
|
21
|
+
ALIASES = {"hn": "hackernews", "gh": "github", "yt": "youtube", "so": "stackoverflow",
|
|
22
|
+
"bsky": "bluesky", "fediverse": "bluesky", "twitter": "x", "techmeme": "news",
|
|
23
|
+
"papers": "arxiv", "dev.to": "devto", "markets": "polymarket"} # fmt: skip
|
|
24
|
+
DEPTHS = {"quick": {"per_source": 5, "deep_reads": 0, "queries": 1},
|
|
25
|
+
"default": {"per_source": 8, "deep_reads": 3, "queries": 1},
|
|
26
|
+
"deep": {"per_source": 15, "deep_reads": 6, "queries": 2}} # fmt: skip
|
|
27
|
+
|
|
28
|
+
_TECH_WORDS = frozenset(
|
|
29
|
+
["ai", "llm", "gpt", "model", "api", "sdk", "python", "rust", "go", "golang", "typescript", "javascript", "js", "react", "vue", "svelte", "node", "deno", "bun", "docker", "kubernetes", "k8s", "linux", "git", "github", "cli", "framework", "library", "database", "sql", "postgres", "redis", "cloud", "aws", "gcp", "azure", "devops", "ml", "agent", "agents", "mcp", "claude", "openai", "anthropic", "gemini", "llama", "compiler", "wasm", "webassembly", "kernel", "gpu", "cuda", "nvidia", "programming", "code", "coding", "developer", "open-source", "opensource", "vscode", "cursor", "neovim", "vim", "terminal", "uv", "pip", "npm", "cargo", "crate", "package"]
|
|
30
|
+
) # fmt: skip
|
|
31
|
+
_ACADEMIC = frozenset(
|
|
32
|
+
[
|
|
33
|
+
"paper",
|
|
34
|
+
"papers",
|
|
35
|
+
"arxiv",
|
|
36
|
+
"research",
|
|
37
|
+
"study",
|
|
38
|
+
"benchmark",
|
|
39
|
+
"dataset",
|
|
40
|
+
"theorem",
|
|
41
|
+
"diffusion",
|
|
42
|
+
"transformer",
|
|
43
|
+
]
|
|
44
|
+
)
|
|
45
|
+
_STOP = frozenset(
|
|
46
|
+
[
|
|
47
|
+
"the",
|
|
48
|
+
"a",
|
|
49
|
+
"an",
|
|
50
|
+
"and",
|
|
51
|
+
"or",
|
|
52
|
+
"of",
|
|
53
|
+
"for",
|
|
54
|
+
"to",
|
|
55
|
+
"in",
|
|
56
|
+
"on",
|
|
57
|
+
"with",
|
|
58
|
+
"what",
|
|
59
|
+
"is",
|
|
60
|
+
"are",
|
|
61
|
+
"how",
|
|
62
|
+
"why",
|
|
63
|
+
"best",
|
|
64
|
+
"new",
|
|
65
|
+
"about",
|
|
66
|
+
"latest",
|
|
67
|
+
"news",
|
|
68
|
+
"last",
|
|
69
|
+
"days",
|
|
70
|
+
"week",
|
|
71
|
+
"month",
|
|
72
|
+
"people",
|
|
73
|
+
"think",
|
|
74
|
+
]
|
|
75
|
+
)
|
|
76
|
+
_VS = re.compile(r"\s+(?:vs\.?|versus)\s+", re.IGNORECASE)
|
|
77
|
+
_TICKER = re.compile(r"\$([A-Za-z]{1,6}(?:\.[A-Za-z]{1,2})?)\b")
|
|
78
|
+
_DISCOVER = re.compile(
|
|
79
|
+
r"^\s*(?:what'?s\s+)?(?:trending|hot|discover|what is happening)\b", re.IGNORECASE
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class Plan:
|
|
85
|
+
topic: str
|
|
86
|
+
mode: str # topic | compare | discover | person
|
|
87
|
+
days: int
|
|
88
|
+
since: datetime
|
|
89
|
+
until: datetime
|
|
90
|
+
depth: str
|
|
91
|
+
sources: list[str]
|
|
92
|
+
queries: dict[str, list[str]] = field(default_factory=dict)
|
|
93
|
+
subjects: list[str] = field(default_factory=list)
|
|
94
|
+
tickers: list[str] = field(default_factory=list)
|
|
95
|
+
github_user: str | None = None
|
|
96
|
+
notes: list[str] = field(default_factory=list)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def per_source(self) -> int:
|
|
100
|
+
return DEPTHS[self.depth]["per_source"]
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def deep_reads(self) -> int:
|
|
104
|
+
return DEPTHS[self.depth]["deep_reads"]
|
|
105
|
+
|
|
106
|
+
def to_dict(self) -> dict:
|
|
107
|
+
d = asdict(self)
|
|
108
|
+
d["since"], d["until"] = self.since.isoformat(), self.until.isoformat()
|
|
109
|
+
return d
|
|
110
|
+
|
|
111
|
+
def describe(self) -> str:
|
|
112
|
+
"""--preflight: what would run, without running it."""
|
|
113
|
+
window = f"{self.since.date()} -> {self.until.date()} ({self.days} days)"
|
|
114
|
+
lines = [f"mode: {self.mode} · depth: {self.depth} · window: {window}"]
|
|
115
|
+
if self.subjects:
|
|
116
|
+
lines.append("subjects: " + " | ".join(self.subjects))
|
|
117
|
+
if self.github_user:
|
|
118
|
+
lines.append(f"github person: {self.github_user}")
|
|
119
|
+
lines.append("reads (anonymous unless you saved a session with `wire auth login`):")
|
|
120
|
+
lines += [f"- {s}: " + " · ".join(repr(q) for q in self.queries.get(s) or [self.topic])
|
|
121
|
+
for s in self.sources] # fmt: skip
|
|
122
|
+
lines.append("writes: none unless store=True (then WIRE_HOME/research.db)")
|
|
123
|
+
lines += [f"note: {n}" for n in self.notes]
|
|
124
|
+
return "\n".join(lines)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def terms(text: str) -> list[str]:
|
|
128
|
+
words = re.findall(r"[a-z0-9][a-z0-9+#.\-]*", text.lower())
|
|
129
|
+
return [w.strip(".-") for w in words if w not in _STOP and len(w) > 1]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def is_tech(topic: str) -> bool:
|
|
133
|
+
return any(t in _TECH_WORDS for t in terms(topic))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _expand(topic: str, n: int) -> list[str]:
|
|
137
|
+
"""Query expansion: the topic, then (deep) its key terms without filler words."""
|
|
138
|
+
out = [topic.strip()]
|
|
139
|
+
core = " ".join(terms(topic))
|
|
140
|
+
if n > 1 and core and core != topic.strip().lower():
|
|
141
|
+
out.append(core)
|
|
142
|
+
return out[:n]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def normalize_sources(sources) -> list[str] | None:
|
|
146
|
+
if not sources:
|
|
147
|
+
return None
|
|
148
|
+
if isinstance(sources, str):
|
|
149
|
+
sources = [s for s in re.split(r"[,\s]+", sources) if s]
|
|
150
|
+
out = []
|
|
151
|
+
for s in sources:
|
|
152
|
+
key = ALIASES.get(s.strip().lower(), s.strip().lower())
|
|
153
|
+
if key == "all":
|
|
154
|
+
return list(ALL_SOURCES)
|
|
155
|
+
if key not in ALL_SOURCES:
|
|
156
|
+
raise ValueError(
|
|
157
|
+
f"unknown research source {s!r}; choose from: {', '.join(ALL_SOURCES)}"
|
|
158
|
+
)
|
|
159
|
+
out.append(key)
|
|
160
|
+
return list(dict.fromkeys(out))
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def make_plan(topic: str = "", *, days: int = 30, sources=None, mode: str = "auto",
|
|
164
|
+
depth: str = "default", as_of=None, github_user: str | None = None,
|
|
165
|
+
has_x_session: bool = False) -> Plan: # fmt: skip
|
|
166
|
+
if depth not in DEPTHS:
|
|
167
|
+
raise ValueError(f"depth must be one of {', '.join(DEPTHS)}")
|
|
168
|
+
days = max(1, min(int(days), 365))
|
|
169
|
+
until = _as_of(as_of)
|
|
170
|
+
since = until - timedelta(days=days)
|
|
171
|
+
topic = (topic or "").strip()
|
|
172
|
+
chosen = normalize_sources(sources)
|
|
173
|
+
|
|
174
|
+
if mode == "auto":
|
|
175
|
+
if github_user and not topic:
|
|
176
|
+
mode = "person"
|
|
177
|
+
elif not topic or _DISCOVER.match(topic):
|
|
178
|
+
mode = "discover"
|
|
179
|
+
elif len(_VS.split(topic)) > 1:
|
|
180
|
+
mode = "compare"
|
|
181
|
+
else:
|
|
182
|
+
mode = "topic"
|
|
183
|
+
if mode not in ("topic", "compare", "discover", "person"):
|
|
184
|
+
raise ValueError("mode must be auto, topic, compare, discover or person")
|
|
185
|
+
|
|
186
|
+
plan = Plan(topic=topic, mode=mode, days=days, since=since, until=until, depth=depth,
|
|
187
|
+
sources=[], github_user=github_user.lstrip("@") if github_user else None) # fmt: skip
|
|
188
|
+
plan.tickers = [t.upper() for t in _TICKER.findall(topic)]
|
|
189
|
+
if mode == "compare":
|
|
190
|
+
plan.subjects = [s.strip() for s in _VS.split(topic) if s.strip()][:4]
|
|
191
|
+
|
|
192
|
+
if chosen:
|
|
193
|
+
plan.sources = chosen
|
|
194
|
+
elif mode == "discover":
|
|
195
|
+
plan.sources = ["hackernews", "github", "news", "polymarket", "mastodon", "lobsters"]
|
|
196
|
+
elif mode == "person":
|
|
197
|
+
plan.sources = ["github"]
|
|
198
|
+
else:
|
|
199
|
+
plan.sources = list(CORE)
|
|
200
|
+
if is_tech(topic):
|
|
201
|
+
plan.sources += list(TECH)
|
|
202
|
+
if set(terms(topic)) & _ACADEMIC:
|
|
203
|
+
plan.sources.append("arxiv")
|
|
204
|
+
if plan.tickers:
|
|
205
|
+
plan.sources.append("stocktwits")
|
|
206
|
+
if "x" in plan.sources and not has_x_session:
|
|
207
|
+
plan.sources.remove("x")
|
|
208
|
+
plan.notes.append("x skipped: no saved session (`wire auth login x.com` adds it)")
|
|
209
|
+
elif has_x_session and "x" not in plan.sources and not chosen and mode in ("topic", "compare"):
|
|
210
|
+
plan.sources.append("x")
|
|
211
|
+
|
|
212
|
+
n = DEPTHS[depth]["queries"]
|
|
213
|
+
subjects = plan.subjects or ([topic] if topic else [])
|
|
214
|
+
for s in plan.sources:
|
|
215
|
+
if s == "stocktwits":
|
|
216
|
+
plan.queries[s] = [f"${t}" for t in plan.tickers] or subjects
|
|
217
|
+
else:
|
|
218
|
+
plan.queries[s] = [q for subj in subjects for q in _expand(subj, n)]
|
|
219
|
+
if plan.github_user and "github" not in plan.sources:
|
|
220
|
+
plan.sources.append("github")
|
|
221
|
+
return plan
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _as_of(value) -> datetime:
|
|
225
|
+
if value in (None, ""):
|
|
226
|
+
return datetime.now(UTC)
|
|
227
|
+
if isinstance(value, datetime):
|
|
228
|
+
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
|
229
|
+
d = datetime.fromisoformat(str(value))
|
|
230
|
+
if len(str(value)) <= 10: # a bare date means the end of that day
|
|
231
|
+
d += timedelta(days=1)
|
|
232
|
+
return d if d.tzinfo else d.replace(tzinfo=UTC)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Score research items by relevance x (engagement, freshness), with a relevance floor."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
from agent_wire.models import WireSearchHit
|
|
8
|
+
from agent_wire.research.planner import terms
|
|
9
|
+
|
|
10
|
+
# How much one unit of each public signal is worth, roughly "one upvote".
|
|
11
|
+
WEIGHTS = {"score": 1.0, "likes": 1.0, "comments": 2.0, "reposts": 2.0, "reactions": 1.0,
|
|
12
|
+
"stars": 0.1, "forks": 0.2, "views": 0.003, "volume24h": 0.001, "volume": 0.0001} # fmt: skip
|
|
13
|
+
RELEVANCE_FLOOR = 0.34 # below this an item is off-topic, however viral
|
|
14
|
+
UNDATED_FRESHNESS = 0.35
|
|
15
|
+
MIN_SOURCE_TOP = math.log1p(50)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Item:
|
|
20
|
+
hit: WireSearchHit
|
|
21
|
+
query: str = ""
|
|
22
|
+
origin: str = "" # the planned source that produced it ("news" for a Techmeme item)
|
|
23
|
+
relevance: float = 0.0
|
|
24
|
+
freshness: float = 0.0
|
|
25
|
+
engagement: float = 0.0
|
|
26
|
+
score: float = 0.0
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def source(self) -> str:
|
|
30
|
+
return self.hit.source
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def raw_engagement(hit: WireSearchHit) -> float:
|
|
34
|
+
return sum(WEIGHTS.get(k, 0.0) * max(0, v) for k, v in hit.engagement.items())
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def relevance(hit: WireSearchHit, topic: str) -> float:
|
|
38
|
+
"""Share of topic terms present (title counts fully, snippet partly) + exact-phrase bonus."""
|
|
39
|
+
want = list(dict.fromkeys(terms(topic)))
|
|
40
|
+
if not want:
|
|
41
|
+
return 1.0
|
|
42
|
+
title, body = hit.title.lower(), f"{hit.snippet} {hit.author or ''}".lower()
|
|
43
|
+
got = sum(1.0 if t in title else 0.6 if t in body else 0.0 for t in want) / len(want)
|
|
44
|
+
if topic.lower().strip() in title:
|
|
45
|
+
got += 0.25
|
|
46
|
+
return min(got, 1.25)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def freshness(ts: datetime | None, until: datetime, days: int) -> float:
|
|
50
|
+
if ts is None:
|
|
51
|
+
return UNDATED_FRESHNESS
|
|
52
|
+
age = max(0.0, (until - ts).total_seconds() / 86400)
|
|
53
|
+
return 0.5 ** (age / max(days / 2, 0.5)) # half-life: half the window
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def in_window(hit: WireSearchHit, since: datetime, until: datetime) -> bool:
|
|
57
|
+
return hit.timestamp is None or since <= hit.timestamp <= until
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def floor_for(query: str) -> float:
|
|
61
|
+
"""Two-word topics need both words ('rust programming' is not any programming post)."""
|
|
62
|
+
n = len(set(terms(query)))
|
|
63
|
+
return 0.55 if n == 2 else 0.4 if n >= 3 else RELEVANCE_FLOOR
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def score_items(items: list[Item], topic: str, since: datetime, until: datetime, days: int,
|
|
67
|
+
floor: float | None = None) -> list[Item]: # fmt: skip
|
|
68
|
+
"""Drop out-of-window and off-topic items; fill the score fields; best first.
|
|
69
|
+
|
|
70
|
+
`floor=None` picks a relevance floor from each item's query; pass a number to override.
|
|
71
|
+
"""
|
|
72
|
+
kept = [i for i in items if in_window(i.hit, since, until)]
|
|
73
|
+
for i in kept:
|
|
74
|
+
i.relevance = relevance(i.hit, i.query or topic)
|
|
75
|
+
kept = [i for i in kept
|
|
76
|
+
if i.relevance >= (floor if floor is not None else floor_for(i.query or topic))] # fmt: skip
|
|
77
|
+
logs = {id(i): math.log1p(raw_engagement(i.hit)) for i in kept}
|
|
78
|
+
top_all = max(logs.values(), default=0.0) or 1.0
|
|
79
|
+
top_src: dict[str, float] = {}
|
|
80
|
+
for i in kept:
|
|
81
|
+
top_src[i.source] = max(top_src.get(i.source, 0.0), logs[id(i)])
|
|
82
|
+
for i in kept:
|
|
83
|
+
# per-source normalisation keeps a 50-point HN story from losing to a 5k-view video
|
|
84
|
+
# ... but a source whose best item has ~no engagement must not get a free 1.0
|
|
85
|
+
src_norm = max(top_src[i.source], MIN_SOURCE_TOP)
|
|
86
|
+
i.engagement = 0.6 * logs[id(i)] / src_norm + 0.4 * logs[id(i)] / top_all
|
|
87
|
+
i.freshness = freshness(i.hit.timestamp, until, days)
|
|
88
|
+
i.score = round(min(i.relevance, 1.0) * (0.45 * i.engagement + 0.35 * i.freshness + 0.2), 4)
|
|
89
|
+
return sorted(kept, key=lambda i: -i.score)
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Research persistence (WIRE_HOME/research.db): saved runs, their items, and watched topics."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from datetime import UTC, datetime
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from agent_wire.channels.tier1_global.search import url_key
|
|
9
|
+
from agent_wire.compression.bm25_slicer import bm25_scores
|
|
10
|
+
|
|
11
|
+
SCHEMA = """
|
|
12
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
13
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
14
|
+
topic TEXT NOT NULL,
|
|
15
|
+
mode TEXT NOT NULL,
|
|
16
|
+
days INTEGER NOT NULL,
|
|
17
|
+
created_at TEXT NOT NULL,
|
|
18
|
+
headline TEXT NOT NULL DEFAULT '',
|
|
19
|
+
markdown TEXT NOT NULL,
|
|
20
|
+
json TEXT NOT NULL
|
|
21
|
+
);
|
|
22
|
+
CREATE TABLE IF NOT EXISTS items (
|
|
23
|
+
run_id INTEGER NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
|
|
24
|
+
url_key TEXT NOT NULL,
|
|
25
|
+
url TEXT NOT NULL,
|
|
26
|
+
title TEXT NOT NULL,
|
|
27
|
+
source TEXT NOT NULL,
|
|
28
|
+
ts TEXT,
|
|
29
|
+
score REAL NOT NULL DEFAULT 0
|
|
30
|
+
);
|
|
31
|
+
CREATE INDEX IF NOT EXISTS items_run ON items(run_id);
|
|
32
|
+
CREATE INDEX IF NOT EXISTS runs_topic ON runs(topic);
|
|
33
|
+
CREATE TABLE IF NOT EXISTS watches (
|
|
34
|
+
topic TEXT PRIMARY KEY,
|
|
35
|
+
days INTEGER NOT NULL DEFAULT 7,
|
|
36
|
+
sources TEXT,
|
|
37
|
+
webhook TEXT,
|
|
38
|
+
created_at TEXT NOT NULL,
|
|
39
|
+
last_run_id INTEGER
|
|
40
|
+
);
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def norm_topic(topic: str) -> str:
|
|
45
|
+
return " ".join(topic.lower().split())
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Store:
|
|
49
|
+
def __init__(self, path: Path):
|
|
50
|
+
self.path = Path(path)
|
|
51
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
self.db = sqlite3.connect(self.path)
|
|
53
|
+
self.db.row_factory = sqlite3.Row
|
|
54
|
+
self.db.execute("PRAGMA foreign_keys = ON")
|
|
55
|
+
self.db.executescript(SCHEMA)
|
|
56
|
+
|
|
57
|
+
def close(self) -> None:
|
|
58
|
+
self.db.close()
|
|
59
|
+
|
|
60
|
+
# ---- runs --------------------------------------------------------------------------------
|
|
61
|
+
def save(self, result, markdown: str, data: dict) -> int:
|
|
62
|
+
p = result.plan
|
|
63
|
+
topic = norm_topic(p.topic) or ("discover" if p.mode == "discover" else p.github_user or "")
|
|
64
|
+
with self.db:
|
|
65
|
+
cur = self.db.execute(
|
|
66
|
+
"INSERT INTO runs (topic, mode, days, created_at, headline, markdown, json)"
|
|
67
|
+
" VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
68
|
+
(topic, p.mode, p.days, result.created_at.isoformat(), data.get("headline", ""),
|
|
69
|
+
markdown, json.dumps(data, ensure_ascii=False)),
|
|
70
|
+
) # fmt: skip
|
|
71
|
+
run_id = cur.lastrowid
|
|
72
|
+
self.db.executemany(
|
|
73
|
+
"INSERT INTO items (run_id, url_key, url, title, source, ts, score)"
|
|
74
|
+
" VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
75
|
+
[(run_id, url_key(i.hit.url), i.hit.url, i.hit.title, i.source,
|
|
76
|
+
i.hit.timestamp.isoformat() if i.hit.timestamp else None, i.score)
|
|
77
|
+
for i in result.items],
|
|
78
|
+
) # fmt: skip
|
|
79
|
+
return run_id
|
|
80
|
+
|
|
81
|
+
def runs(self, limit: int = 20, topic: str | None = None) -> list[dict]:
|
|
82
|
+
q, args = "SELECT id, topic, mode, days, created_at, headline FROM runs", []
|
|
83
|
+
if topic:
|
|
84
|
+
q, args = q + " WHERE topic = ?", [norm_topic(topic)]
|
|
85
|
+
rows = self.db.execute(q + " ORDER BY id DESC LIMIT ?", [*args, limit]).fetchall()
|
|
86
|
+
return [dict(r) for r in rows]
|
|
87
|
+
|
|
88
|
+
def run(self, run_id: int) -> dict | None:
|
|
89
|
+
row = self.db.execute("SELECT * FROM runs WHERE id = ?", (run_id,)).fetchone()
|
|
90
|
+
return dict(row) if row else None
|
|
91
|
+
|
|
92
|
+
def items(self, run_id: int) -> list[dict]:
|
|
93
|
+
rows = self.db.execute(
|
|
94
|
+
"SELECT * FROM items WHERE run_id = ? ORDER BY score DESC", (run_id,)
|
|
95
|
+
)
|
|
96
|
+
return [dict(r) for r in rows]
|
|
97
|
+
|
|
98
|
+
def previous(self, topic: str, before_id: int) -> int | None:
|
|
99
|
+
row = self.db.execute(
|
|
100
|
+
"SELECT id FROM runs WHERE topic = ? AND id < ? ORDER BY id DESC LIMIT 1",
|
|
101
|
+
(norm_topic(topic), before_id),
|
|
102
|
+
).fetchone()
|
|
103
|
+
return row["id"] if row else None
|
|
104
|
+
|
|
105
|
+
def new_items(self, run_id: int, previous_id: int | None) -> list[dict]:
|
|
106
|
+
"""Items in this run whose URL never appeared in the previous run of the same topic."""
|
|
107
|
+
if previous_id is None:
|
|
108
|
+
return self.items(run_id)
|
|
109
|
+
rows = self.db.execute("SELECT url_key FROM items WHERE run_id = ?", (previous_id,))
|
|
110
|
+
seen = {r["url_key"] for r in rows}
|
|
111
|
+
return [i for i in self.items(run_id) if i["url_key"] not in seen]
|
|
112
|
+
|
|
113
|
+
def search(self, query: str, limit: int = 10, exclude: int | None = None) -> list[dict]:
|
|
114
|
+
"""Offline, deterministic BM25 over every saved brief."""
|
|
115
|
+
rows = [dict(r) for r in self.db.execute(
|
|
116
|
+
"SELECT id, topic, mode, days, created_at, headline, markdown FROM runs"
|
|
117
|
+
" ORDER BY id DESC LIMIT 2000")] # fmt: skip
|
|
118
|
+
rows = [r for r in rows if r["id"] != exclude]
|
|
119
|
+
if not rows:
|
|
120
|
+
return []
|
|
121
|
+
texts = [f"{r['topic']} {r['topic']} {r['headline']} {r['markdown']}" for r in rows]
|
|
122
|
+
scores = bm25_scores(texts, query)
|
|
123
|
+
ranked = sorted(zip(scores, rows, strict=True), key=lambda sr: (-sr[0], -sr[1]["id"]))
|
|
124
|
+
return [{k: v for k, v in r.items() if k != "markdown"} | {"score": round(s, 3)}
|
|
125
|
+
for s, r in ranked[:limit] if s > 0] # fmt: skip
|
|
126
|
+
|
|
127
|
+
# ---- watches -----------------------------------------------------------------------------
|
|
128
|
+
def add_watch(self, topic: str, days: int = 7, sources: str | None = None,
|
|
129
|
+
webhook: str | None = None) -> None: # fmt: skip
|
|
130
|
+
with self.db:
|
|
131
|
+
self.db.execute(
|
|
132
|
+
"INSERT INTO watches (topic, days, sources, webhook, created_at) VALUES (?, ?, ?, ?, ?)"
|
|
133
|
+
" ON CONFLICT(topic) DO UPDATE SET days = excluded.days,"
|
|
134
|
+
" sources = excluded.sources, webhook = excluded.webhook",
|
|
135
|
+
(norm_topic(topic), days, sources, webhook, datetime.now(UTC).isoformat()),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
def remove_watch(self, topic: str) -> bool:
|
|
139
|
+
with self.db:
|
|
140
|
+
cur = self.db.execute("DELETE FROM watches WHERE topic = ?", (norm_topic(topic),))
|
|
141
|
+
return cur.rowcount > 0
|
|
142
|
+
|
|
143
|
+
def watches(self) -> list[dict]:
|
|
144
|
+
return [dict(r) for r in self.db.execute("SELECT * FROM watches ORDER BY topic")]
|
|
145
|
+
|
|
146
|
+
def mark_watched(self, topic: str, run_id: int) -> None:
|
|
147
|
+
with self.db:
|
|
148
|
+
self.db.execute("UPDATE watches SET last_run_id = ? WHERE topic = ?",
|
|
149
|
+
(run_id, norm_topic(topic))) # fmt: skip
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Watchlist: re-run saved topics, report only what is new since the last run, optional webhook."""
|
|
2
|
+
|
|
3
|
+
from urllib.parse import urlsplit
|
|
4
|
+
|
|
5
|
+
from curl_cffi.requests.exceptions import RequestException
|
|
6
|
+
|
|
7
|
+
from agent_wire.firewall.outbound_privacy import scrub
|
|
8
|
+
from agent_wire.research.store import norm_topic
|
|
9
|
+
|
|
10
|
+
MAX_NEW = 10
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def check_webhook(url: str | None) -> str | None:
|
|
14
|
+
if not url:
|
|
15
|
+
return None
|
|
16
|
+
parts = urlsplit(url)
|
|
17
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
18
|
+
raise ValueError("webhook must be an http(s) URL")
|
|
19
|
+
return url
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def run_watch(wire, watch: dict) -> dict:
|
|
23
|
+
"""One watched topic -> fresh stored run + the items the previous run didn't have."""
|
|
24
|
+
topic = watch["topic"]
|
|
25
|
+
result, _, _ = await wire.research_result(topic, days=watch["days"],
|
|
26
|
+
platforms=watch.get("sources"), depth="quick",
|
|
27
|
+
store=True) # fmt: skip
|
|
28
|
+
store = wire.store
|
|
29
|
+
prev = store.previous(topic, result.run_id)
|
|
30
|
+
fresh = store.new_items(result.run_id, prev)[:MAX_NEW]
|
|
31
|
+
store.mark_watched(topic, result.run_id)
|
|
32
|
+
report = {"topic": topic, "run_id": result.run_id, "previous_run": prev,
|
|
33
|
+
"new": [{"title": i["title"], "url": i["url"], "source": i["source"], "ts": i["ts"]}
|
|
34
|
+
for i in fresh]} # fmt: skip
|
|
35
|
+
if hook := watch.get("webhook"):
|
|
36
|
+
report["webhook"] = await _post(wire, hook, report)
|
|
37
|
+
return report
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def _post(wire, url: str, report: dict) -> str:
|
|
41
|
+
if not report["new"]:
|
|
42
|
+
return "skipped (nothing new)"
|
|
43
|
+
try:
|
|
44
|
+
f = await wire.pool.get(url, method="POST", impersonate=None, json_body=report,
|
|
45
|
+
headers={"content-type": "application/json"}) # fmt: skip
|
|
46
|
+
except RequestException as e:
|
|
47
|
+
return f"failed: {type(e).__name__}"
|
|
48
|
+
return f"HTTP {f.status}"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
async def run_watches(wire, topic: str | None = None) -> str:
|
|
52
|
+
watches = wire.store.watches()
|
|
53
|
+
if topic:
|
|
54
|
+
watches = [w for w in watches if w["topic"] == norm_topic(topic)]
|
|
55
|
+
if not watches:
|
|
56
|
+
return "no watched topics (wire watch add <topic>)"
|
|
57
|
+
out = []
|
|
58
|
+
for w in watches:
|
|
59
|
+
r = await run_watch(wire, w)
|
|
60
|
+
head = f"## {r['topic']} · run #{r['run_id']}"
|
|
61
|
+
if r["previous_run"]:
|
|
62
|
+
head += f" · {len(r['new'])} new since run #{r['previous_run']}"
|
|
63
|
+
else:
|
|
64
|
+
head += " · first run"
|
|
65
|
+
if r.get("webhook"):
|
|
66
|
+
head += f" · webhook {r['webhook']}"
|
|
67
|
+
lines = [f"- [{i['title']}]({i['url']}) · {i['source']} · {(i['ts'] or '')[:10]}"
|
|
68
|
+
for i in r["new"]] # fmt: skip
|
|
69
|
+
out.append("\n".join([head, *(lines or ["- nothing new"])]))
|
|
70
|
+
return scrub("\n\n".join(out))
|