agent-wire-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. agent_wire/__init__.py +1 -0
  2. agent_wire/__main__.py +3 -0
  3. agent_wire/assets/SKILL.md +26 -0
  4. agent_wire/assets/cursor_rule.mdc +13 -0
  5. agent_wire/assets/hermes_skill.md +28 -0
  6. agent_wire/assets/windsurf_rules.md +14 -0
  7. agent_wire/assets/wire-worker.md +20 -0
  8. agent_wire/auth/__init__.py +0 -0
  9. agent_wire/auth/browser_paths.py +143 -0
  10. agent_wire/auth/cookie_vault.py +181 -0
  11. agent_wire/auth/rate_limiter.py +26 -0
  12. agent_wire/cache/__init__.py +0 -0
  13. agent_wire/cache/ring_buffer.py +94 -0
  14. agent_wire/cache/sqlite_ledger.py +132 -0
  15. agent_wire/channels/__init__.py +41 -0
  16. agent_wire/channels/base.py +249 -0
  17. agent_wire/channels/tier1_global/__init__.py +0 -0
  18. agent_wire/channels/tier1_global/github.py +371 -0
  19. agent_wire/channels/tier1_global/hackernews.py +135 -0
  20. agent_wire/channels/tier1_global/reddit.py +371 -0
  21. agent_wire/channels/tier1_global/rss.py +129 -0
  22. agent_wire/channels/tier1_global/search.py +160 -0
  23. agent_wire/channels/tier1_global/stackoverflow.py +94 -0
  24. agent_wire/channels/tier1_global/web.py +98 -0
  25. agent_wire/channels/tier1_global/wikipedia.py +91 -0
  26. agent_wire/channels/tier1_global/x_twitter.py +380 -0
  27. agent_wire/channels/tier1_global/youtube.py +223 -0
  28. agent_wire/channels/tier2_pro/__init__.py +0 -0
  29. agent_wire/channels/tier2_pro/archive_bypass.py +66 -0
  30. agent_wire/channels/tier2_pro/arxiv_papers.py +220 -0
  31. agent_wire/channels/tier2_pro/devto.py +114 -0
  32. agent_wire/channels/tier2_pro/fediverse.py +181 -0
  33. agent_wire/channels/tier2_pro/hubs.py +161 -0
  34. agent_wire/channels/tier2_pro/linkedin.py +146 -0
  35. agent_wire/channels/tier2_pro/markets.py +207 -0
  36. agent_wire/channels/tier2_pro/meta_social.py +104 -0
  37. agent_wire/channels/tier2_pro/news.py +173 -0
  38. agent_wire/channels/tier2_pro/polymarket.py +105 -0
  39. agent_wire/channels/tier2_pro/publishing.py +84 -0
  40. agent_wire/channels/tier2_pro/stocktwits.py +79 -0
  41. agent_wire/channels/tier3_regional/bilibili.py +171 -0
  42. agent_wire/channels/tier3_regional/boss_zhipin.py +35 -0
  43. agent_wire/channels/tier3_regional/douyin_tiktok.py +85 -0
  44. agent_wire/channels/tier3_regional/v2ex.py +70 -0
  45. agent_wire/channels/tier3_regional/wechat.py +67 -0
  46. agent_wire/channels/tier3_regional/xiaohongshu.py +58 -0
  47. agent_wire/channels/tier3_regional/xiaoyuzhou.py +70 -0
  48. agent_wire/channels/tier3_regional/xueqiu.py +77 -0
  49. agent_wire/channels/tier3_regional/zhihu.py +57 -0
  50. agent_wire/cli.py +403 -0
  51. agent_wire/compression/__init__.py +0 -0
  52. agent_wire/compression/anchor_slicer.py +45 -0
  53. agent_wire/compression/bm25_slicer.py +151 -0
  54. agent_wire/compression/dam_formatter.py +74 -0
  55. agent_wire/compression/url_compactor.py +45 -0
  56. agent_wire/config.py +52 -0
  57. agent_wire/engine/__init__.py +0 -0
  58. agent_wire/engine/challenge.py +58 -0
  59. agent_wire/engine/http_pool.py +136 -0
  60. agent_wire/engine/stealth_browser.py +90 -0
  61. agent_wire/engine/waterfall.py +159 -0
  62. agent_wire/firewall/__init__.py +0 -0
  63. agent_wire/firewall/inbound_shield.py +116 -0
  64. agent_wire/firewall/outbound_privacy.py +120 -0
  65. agent_wire/installers/__init__.py +0 -0
  66. agent_wire/installers/client_installer.py +315 -0
  67. agent_wire/media/__init__.py +0 -0
  68. agent_wire/media/audio_transcriber.py +77 -0
  69. agent_wire/media/subtitle_cleaner.py +102 -0
  70. agent_wire/models.py +60 -0
  71. agent_wire/research/__init__.py +1 -0
  72. agent_wire/research/brief.py +227 -0
  73. agent_wire/research/briefing.py +41 -0
  74. agent_wire/research/cluster.py +128 -0
  75. agent_wire/research/engine.py +632 -0
  76. agent_wire/research/library.py +74 -0
  77. agent_wire/research/planner.py +232 -0
  78. agent_wire/research/scoring.py +89 -0
  79. agent_wire/research/store.py +149 -0
  80. agent_wire/research/watch.py +70 -0
  81. agent_wire/router.py +107 -0
  82. agent_wire/server.py +468 -0
  83. agent_wire_cli-0.2.1.dist-info/METADATA +258 -0
  84. agent_wire_cli-0.2.1.dist-info/RECORD +87 -0
  85. agent_wire_cli-0.2.1.dist-info/WHEEL +4 -0
  86. agent_wire_cli-0.2.1.dist-info/entry_points.txt +4 -0
  87. agent_wire_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,74 @@
1
+ """Your research library: offline search over saved briefs, plus a local site and Atom feed."""
2
+
3
+ import html
4
+ import xml.etree.ElementTree as ET
5
+ from pathlib import Path
6
+
7
+ from agent_wire.research.brief import page
8
+ from agent_wire.research.store import Store
9
+
10
+ ATOM = "http://www.w3.org/2005/Atom"
11
+ EPOCH = "1970-01-01T00:00:00+00:00"
12
+
13
+
14
+ class Library:
15
+ def __init__(self, store: Store, home: Path):
16
+ self.store = store
17
+ self.dir = Path(home) / "library"
18
+
19
+ def related(self, topic: str, limit: int = 3, exclude: int | None = None) -> list[dict]:
20
+ return self.store.search(topic, limit=limit, exclude=exclude)
21
+
22
+ def search_markdown(self, query: str, limit: int = 10) -> str:
23
+ hits = self.store.search(query, limit=limit)
24
+ if not hits:
25
+ return f"no saved briefs match {query!r} (save runs with store=True / --store)"
26
+ return "\n".join(f"{i}. run #{h['id']} · {h['created_at'][:10]} · {h['topic']} ({h['mode']}, "
27
+ f"{h['days']}d) · {h['headline']}" for i, h in enumerate(hits, 1)) # fmt: skip
28
+
29
+ def build(self, limit: int = 200) -> Path:
30
+ """Write index.html, feed.xml and one page per saved run under WIRE_HOME/library."""
31
+ runs_dir = self.dir / "runs"
32
+ runs_dir.mkdir(parents=True, exist_ok=True)
33
+ runs = self.store.runs(limit=limit)
34
+ for r in runs:
35
+ full = self.store.run(r["id"])
36
+ (runs_dir / f"{r['id']}.html").write_text(
37
+ page(full["markdown"], f"#{r['id']} · {r['topic']}"), encoding="utf-8"
38
+ )
39
+ rows = "".join(
40
+ f"<tr><td>{r['created_at'][:10]}</td><td><a href='runs/{r['id']}.html'>"
41
+ f"{html.escape(r['topic'])}</a></td><td>{r['mode']}</td><td>{r['days']}d</td>"
42
+ f"<td>{html.escape(r['headline'])}</td></tr>"
43
+ for r in runs
44
+ )
45
+ table = ("<p><a href='feed.xml'>Atom feed</a></p><table><tr><th>date</th><th>topic</th>"
46
+ f"<th>mode</th><th>window</th><th>headline</th></tr>{rows}</table>") # fmt: skip
47
+ index = page("# Research library", "agent-wire library").replace(
48
+ "<h1>Research library</h1>", "<h1>Research library</h1>" + table
49
+ )
50
+ (self.dir / "index.html").write_text(index, encoding="utf-8")
51
+ (self.dir / "feed.xml").write_bytes(self.atom(runs))
52
+ return self.dir
53
+
54
+ def atom(self, runs: list[dict]) -> bytes:
55
+ ET.register_namespace("", ATOM)
56
+
57
+ def sub(parent, tag, text=None, **attrs):
58
+ el = ET.SubElement(parent, f"{{{ATOM}}}{tag}", **attrs)
59
+ el.text = text
60
+ return el
61
+
62
+ feed = ET.Element(f"{{{ATOM}}}feed")
63
+ sub(feed, "title", "agent-wire research library")
64
+ sub(feed, "id", "urn:agent-wire:library")
65
+ sub(feed, "updated", runs[0]["created_at"] if runs else EPOCH)
66
+ sub(sub(feed, "author"), "name", "agent-wire")
67
+ for r in runs:
68
+ e = sub(feed, "entry")
69
+ sub(e, "title", f"{r['topic']} · last {r['days']} days")
70
+ sub(e, "id", f"urn:agent-wire:run:{r['id']}")
71
+ sub(e, "updated", r["created_at"])
72
+ sub(e, "link", href=(self.dir / "runs" / f"{r['id']}.html").as_uri())
73
+ sub(e, "summary", r["headline"])
74
+ return ET.tostring(feed, encoding="utf-8", xml_declaration=True)
@@ -0,0 +1,232 @@
1
+ """Pre-research: decide mode, window, sources and per-source queries before any network call."""
2
+
3
+ import re
4
+ from dataclasses import asdict, dataclass, field
5
+ from datetime import UTC, datetime, timedelta
6
+
7
+ # Key-free sources researched by default; others join when the topic calls for them.
8
+ CORE = (
9
+ "reddit",
10
+ "hackernews",
11
+ "youtube",
12
+ "github",
13
+ "news",
14
+ "bluesky",
15
+ "mastodon",
16
+ "polymarket",
17
+ "web",
18
+ )
19
+ TECH = ("devto", "stackoverflow")
20
+ ALL_SOURCES = (*CORE, *TECH, "arxiv", "stocktwits", "x", "lobsters")
21
+ ALIASES = {"hn": "hackernews", "gh": "github", "yt": "youtube", "so": "stackoverflow",
22
+ "bsky": "bluesky", "fediverse": "bluesky", "twitter": "x", "techmeme": "news",
23
+ "papers": "arxiv", "dev.to": "devto", "markets": "polymarket"} # fmt: skip
24
+ DEPTHS = {"quick": {"per_source": 5, "deep_reads": 0, "queries": 1},
25
+ "default": {"per_source": 8, "deep_reads": 3, "queries": 1},
26
+ "deep": {"per_source": 15, "deep_reads": 6, "queries": 2}} # fmt: skip
27
+
28
+ _TECH_WORDS = frozenset(
29
+ ["ai", "llm", "gpt", "model", "api", "sdk", "python", "rust", "go", "golang", "typescript", "javascript", "js", "react", "vue", "svelte", "node", "deno", "bun", "docker", "kubernetes", "k8s", "linux", "git", "github", "cli", "framework", "library", "database", "sql", "postgres", "redis", "cloud", "aws", "gcp", "azure", "devops", "ml", "agent", "agents", "mcp", "claude", "openai", "anthropic", "gemini", "llama", "compiler", "wasm", "webassembly", "kernel", "gpu", "cuda", "nvidia", "programming", "code", "coding", "developer", "open-source", "opensource", "vscode", "cursor", "neovim", "vim", "terminal", "uv", "pip", "npm", "cargo", "crate", "package"]
30
+ ) # fmt: skip
31
+ _ACADEMIC = frozenset(
32
+ [
33
+ "paper",
34
+ "papers",
35
+ "arxiv",
36
+ "research",
37
+ "study",
38
+ "benchmark",
39
+ "dataset",
40
+ "theorem",
41
+ "diffusion",
42
+ "transformer",
43
+ ]
44
+ )
45
+ _STOP = frozenset(
46
+ [
47
+ "the",
48
+ "a",
49
+ "an",
50
+ "and",
51
+ "or",
52
+ "of",
53
+ "for",
54
+ "to",
55
+ "in",
56
+ "on",
57
+ "with",
58
+ "what",
59
+ "is",
60
+ "are",
61
+ "how",
62
+ "why",
63
+ "best",
64
+ "new",
65
+ "about",
66
+ "latest",
67
+ "news",
68
+ "last",
69
+ "days",
70
+ "week",
71
+ "month",
72
+ "people",
73
+ "think",
74
+ ]
75
+ )
76
+ _VS = re.compile(r"\s+(?:vs\.?|versus)\s+", re.IGNORECASE)
77
+ _TICKER = re.compile(r"\$([A-Za-z]{1,6}(?:\.[A-Za-z]{1,2})?)\b")
78
+ _DISCOVER = re.compile(
79
+ r"^\s*(?:what'?s\s+)?(?:trending|hot|discover|what is happening)\b", re.IGNORECASE
80
+ )
81
+
82
+
83
+ @dataclass
84
+ class Plan:
85
+ topic: str
86
+ mode: str # topic | compare | discover | person
87
+ days: int
88
+ since: datetime
89
+ until: datetime
90
+ depth: str
91
+ sources: list[str]
92
+ queries: dict[str, list[str]] = field(default_factory=dict)
93
+ subjects: list[str] = field(default_factory=list)
94
+ tickers: list[str] = field(default_factory=list)
95
+ github_user: str | None = None
96
+ notes: list[str] = field(default_factory=list)
97
+
98
+ @property
99
+ def per_source(self) -> int:
100
+ return DEPTHS[self.depth]["per_source"]
101
+
102
+ @property
103
+ def deep_reads(self) -> int:
104
+ return DEPTHS[self.depth]["deep_reads"]
105
+
106
+ def to_dict(self) -> dict:
107
+ d = asdict(self)
108
+ d["since"], d["until"] = self.since.isoformat(), self.until.isoformat()
109
+ return d
110
+
111
+ def describe(self) -> str:
112
+ """--preflight: what would run, without running it."""
113
+ window = f"{self.since.date()} -> {self.until.date()} ({self.days} days)"
114
+ lines = [f"mode: {self.mode} · depth: {self.depth} · window: {window}"]
115
+ if self.subjects:
116
+ lines.append("subjects: " + " | ".join(self.subjects))
117
+ if self.github_user:
118
+ lines.append(f"github person: {self.github_user}")
119
+ lines.append("reads (anonymous unless you saved a session with `wire auth login`):")
120
+ lines += [f"- {s}: " + " · ".join(repr(q) for q in self.queries.get(s) or [self.topic])
121
+ for s in self.sources] # fmt: skip
122
+ lines.append("writes: none unless store=True (then WIRE_HOME/research.db)")
123
+ lines += [f"note: {n}" for n in self.notes]
124
+ return "\n".join(lines)
125
+
126
+
127
+ def terms(text: str) -> list[str]:
128
+ words = re.findall(r"[a-z0-9][a-z0-9+#.\-]*", text.lower())
129
+ return [w.strip(".-") for w in words if w not in _STOP and len(w) > 1]
130
+
131
+
132
+ def is_tech(topic: str) -> bool:
133
+ return any(t in _TECH_WORDS for t in terms(topic))
134
+
135
+
136
+ def _expand(topic: str, n: int) -> list[str]:
137
+ """Query expansion: the topic, then (deep) its key terms without filler words."""
138
+ out = [topic.strip()]
139
+ core = " ".join(terms(topic))
140
+ if n > 1 and core and core != topic.strip().lower():
141
+ out.append(core)
142
+ return out[:n]
143
+
144
+
145
+ def normalize_sources(sources) -> list[str] | None:
146
+ if not sources:
147
+ return None
148
+ if isinstance(sources, str):
149
+ sources = [s for s in re.split(r"[,\s]+", sources) if s]
150
+ out = []
151
+ for s in sources:
152
+ key = ALIASES.get(s.strip().lower(), s.strip().lower())
153
+ if key == "all":
154
+ return list(ALL_SOURCES)
155
+ if key not in ALL_SOURCES:
156
+ raise ValueError(
157
+ f"unknown research source {s!r}; choose from: {', '.join(ALL_SOURCES)}"
158
+ )
159
+ out.append(key)
160
+ return list(dict.fromkeys(out))
161
+
162
+
163
+ def make_plan(topic: str = "", *, days: int = 30, sources=None, mode: str = "auto",
164
+ depth: str = "default", as_of=None, github_user: str | None = None,
165
+ has_x_session: bool = False) -> Plan: # fmt: skip
166
+ if depth not in DEPTHS:
167
+ raise ValueError(f"depth must be one of {', '.join(DEPTHS)}")
168
+ days = max(1, min(int(days), 365))
169
+ until = _as_of(as_of)
170
+ since = until - timedelta(days=days)
171
+ topic = (topic or "").strip()
172
+ chosen = normalize_sources(sources)
173
+
174
+ if mode == "auto":
175
+ if github_user and not topic:
176
+ mode = "person"
177
+ elif not topic or _DISCOVER.match(topic):
178
+ mode = "discover"
179
+ elif len(_VS.split(topic)) > 1:
180
+ mode = "compare"
181
+ else:
182
+ mode = "topic"
183
+ if mode not in ("topic", "compare", "discover", "person"):
184
+ raise ValueError("mode must be auto, topic, compare, discover or person")
185
+
186
+ plan = Plan(topic=topic, mode=mode, days=days, since=since, until=until, depth=depth,
187
+ sources=[], github_user=github_user.lstrip("@") if github_user else None) # fmt: skip
188
+ plan.tickers = [t.upper() for t in _TICKER.findall(topic)]
189
+ if mode == "compare":
190
+ plan.subjects = [s.strip() for s in _VS.split(topic) if s.strip()][:4]
191
+
192
+ if chosen:
193
+ plan.sources = chosen
194
+ elif mode == "discover":
195
+ plan.sources = ["hackernews", "github", "news", "polymarket", "mastodon", "lobsters"]
196
+ elif mode == "person":
197
+ plan.sources = ["github"]
198
+ else:
199
+ plan.sources = list(CORE)
200
+ if is_tech(topic):
201
+ plan.sources += list(TECH)
202
+ if set(terms(topic)) & _ACADEMIC:
203
+ plan.sources.append("arxiv")
204
+ if plan.tickers:
205
+ plan.sources.append("stocktwits")
206
+ if "x" in plan.sources and not has_x_session:
207
+ plan.sources.remove("x")
208
+ plan.notes.append("x skipped: no saved session (`wire auth login x.com` adds it)")
209
+ elif has_x_session and "x" not in plan.sources and not chosen and mode in ("topic", "compare"):
210
+ plan.sources.append("x")
211
+
212
+ n = DEPTHS[depth]["queries"]
213
+ subjects = plan.subjects or ([topic] if topic else [])
214
+ for s in plan.sources:
215
+ if s == "stocktwits":
216
+ plan.queries[s] = [f"${t}" for t in plan.tickers] or subjects
217
+ else:
218
+ plan.queries[s] = [q for subj in subjects for q in _expand(subj, n)]
219
+ if plan.github_user and "github" not in plan.sources:
220
+ plan.sources.append("github")
221
+ return plan
222
+
223
+
224
+ def _as_of(value) -> datetime:
225
+ if value in (None, ""):
226
+ return datetime.now(UTC)
227
+ if isinstance(value, datetime):
228
+ return value if value.tzinfo else value.replace(tzinfo=UTC)
229
+ d = datetime.fromisoformat(str(value))
230
+ if len(str(value)) <= 10: # a bare date means the end of that day
231
+ d += timedelta(days=1)
232
+ return d if d.tzinfo else d.replace(tzinfo=UTC)
@@ -0,0 +1,89 @@
1
+ """Score research items by relevance x (engagement, freshness), with a relevance floor."""
2
+
3
+ import math
4
+ from dataclasses import dataclass
5
+ from datetime import datetime
6
+
7
+ from agent_wire.models import WireSearchHit
8
+ from agent_wire.research.planner import terms
9
+
10
+ # How much one unit of each public signal is worth, roughly "one upvote".
11
+ WEIGHTS = {"score": 1.0, "likes": 1.0, "comments": 2.0, "reposts": 2.0, "reactions": 1.0,
12
+ "stars": 0.1, "forks": 0.2, "views": 0.003, "volume24h": 0.001, "volume": 0.0001} # fmt: skip
13
+ RELEVANCE_FLOOR = 0.34 # below this an item is off-topic, however viral
14
+ UNDATED_FRESHNESS = 0.35
15
+ MIN_SOURCE_TOP = math.log1p(50)
16
+
17
+
18
+ @dataclass
19
+ class Item:
20
+ hit: WireSearchHit
21
+ query: str = ""
22
+ origin: str = "" # the planned source that produced it ("news" for a Techmeme item)
23
+ relevance: float = 0.0
24
+ freshness: float = 0.0
25
+ engagement: float = 0.0
26
+ score: float = 0.0
27
+
28
+ @property
29
+ def source(self) -> str:
30
+ return self.hit.source
31
+
32
+
33
+ def raw_engagement(hit: WireSearchHit) -> float:
34
+ return sum(WEIGHTS.get(k, 0.0) * max(0, v) for k, v in hit.engagement.items())
35
+
36
+
37
+ def relevance(hit: WireSearchHit, topic: str) -> float:
38
+ """Share of topic terms present (title counts fully, snippet partly) + exact-phrase bonus."""
39
+ want = list(dict.fromkeys(terms(topic)))
40
+ if not want:
41
+ return 1.0
42
+ title, body = hit.title.lower(), f"{hit.snippet} {hit.author or ''}".lower()
43
+ got = sum(1.0 if t in title else 0.6 if t in body else 0.0 for t in want) / len(want)
44
+ if topic.lower().strip() in title:
45
+ got += 0.25
46
+ return min(got, 1.25)
47
+
48
+
49
+ def freshness(ts: datetime | None, until: datetime, days: int) -> float:
50
+ if ts is None:
51
+ return UNDATED_FRESHNESS
52
+ age = max(0.0, (until - ts).total_seconds() / 86400)
53
+ return 0.5 ** (age / max(days / 2, 0.5)) # half-life: half the window
54
+
55
+
56
+ def in_window(hit: WireSearchHit, since: datetime, until: datetime) -> bool:
57
+ return hit.timestamp is None or since <= hit.timestamp <= until
58
+
59
+
60
+ def floor_for(query: str) -> float:
61
+ """Two-word topics need both words ('rust programming' is not any programming post)."""
62
+ n = len(set(terms(query)))
63
+ return 0.55 if n == 2 else 0.4 if n >= 3 else RELEVANCE_FLOOR
64
+
65
+
66
+ def score_items(items: list[Item], topic: str, since: datetime, until: datetime, days: int,
67
+ floor: float | None = None) -> list[Item]: # fmt: skip
68
+ """Drop out-of-window and off-topic items; fill the score fields; best first.
69
+
70
+ `floor=None` picks a relevance floor from each item's query; pass a number to override.
71
+ """
72
+ kept = [i for i in items if in_window(i.hit, since, until)]
73
+ for i in kept:
74
+ i.relevance = relevance(i.hit, i.query or topic)
75
+ kept = [i for i in kept
76
+ if i.relevance >= (floor if floor is not None else floor_for(i.query or topic))] # fmt: skip
77
+ logs = {id(i): math.log1p(raw_engagement(i.hit)) for i in kept}
78
+ top_all = max(logs.values(), default=0.0) or 1.0
79
+ top_src: dict[str, float] = {}
80
+ for i in kept:
81
+ top_src[i.source] = max(top_src.get(i.source, 0.0), logs[id(i)])
82
+ for i in kept:
83
+ # per-source normalisation keeps a 50-point HN story from losing to a 5k-view video
84
+ # ... but a source whose best item has ~no engagement must not get a free 1.0
85
+ src_norm = max(top_src[i.source], MIN_SOURCE_TOP)
86
+ i.engagement = 0.6 * logs[id(i)] / src_norm + 0.4 * logs[id(i)] / top_all
87
+ i.freshness = freshness(i.hit.timestamp, until, days)
88
+ i.score = round(min(i.relevance, 1.0) * (0.45 * i.engagement + 0.35 * i.freshness + 0.2), 4)
89
+ return sorted(kept, key=lambda i: -i.score)
@@ -0,0 +1,149 @@
1
+ """Research persistence (WIRE_HOME/research.db): saved runs, their items, and watched topics."""
2
+
3
+ import json
4
+ import sqlite3
5
+ from datetime import UTC, datetime
6
+ from pathlib import Path
7
+
8
+ from agent_wire.channels.tier1_global.search import url_key
9
+ from agent_wire.compression.bm25_slicer import bm25_scores
10
+
11
+ SCHEMA = """
12
+ CREATE TABLE IF NOT EXISTS runs (
13
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
14
+ topic TEXT NOT NULL,
15
+ mode TEXT NOT NULL,
16
+ days INTEGER NOT NULL,
17
+ created_at TEXT NOT NULL,
18
+ headline TEXT NOT NULL DEFAULT '',
19
+ markdown TEXT NOT NULL,
20
+ json TEXT NOT NULL
21
+ );
22
+ CREATE TABLE IF NOT EXISTS items (
23
+ run_id INTEGER NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
24
+ url_key TEXT NOT NULL,
25
+ url TEXT NOT NULL,
26
+ title TEXT NOT NULL,
27
+ source TEXT NOT NULL,
28
+ ts TEXT,
29
+ score REAL NOT NULL DEFAULT 0
30
+ );
31
+ CREATE INDEX IF NOT EXISTS items_run ON items(run_id);
32
+ CREATE INDEX IF NOT EXISTS runs_topic ON runs(topic);
33
+ CREATE TABLE IF NOT EXISTS watches (
34
+ topic TEXT PRIMARY KEY,
35
+ days INTEGER NOT NULL DEFAULT 7,
36
+ sources TEXT,
37
+ webhook TEXT,
38
+ created_at TEXT NOT NULL,
39
+ last_run_id INTEGER
40
+ );
41
+ """
42
+
43
+
44
+ def norm_topic(topic: str) -> str:
45
+ return " ".join(topic.lower().split())
46
+
47
+
48
+ class Store:
49
+ def __init__(self, path: Path):
50
+ self.path = Path(path)
51
+ self.path.parent.mkdir(parents=True, exist_ok=True)
52
+ self.db = sqlite3.connect(self.path)
53
+ self.db.row_factory = sqlite3.Row
54
+ self.db.execute("PRAGMA foreign_keys = ON")
55
+ self.db.executescript(SCHEMA)
56
+
57
+ def close(self) -> None:
58
+ self.db.close()
59
+
60
+ # ---- runs --------------------------------------------------------------------------------
61
+ def save(self, result, markdown: str, data: dict) -> int:
62
+ p = result.plan
63
+ topic = norm_topic(p.topic) or ("discover" if p.mode == "discover" else p.github_user or "")
64
+ with self.db:
65
+ cur = self.db.execute(
66
+ "INSERT INTO runs (topic, mode, days, created_at, headline, markdown, json)"
67
+ " VALUES (?, ?, ?, ?, ?, ?, ?)",
68
+ (topic, p.mode, p.days, result.created_at.isoformat(), data.get("headline", ""),
69
+ markdown, json.dumps(data, ensure_ascii=False)),
70
+ ) # fmt: skip
71
+ run_id = cur.lastrowid
72
+ self.db.executemany(
73
+ "INSERT INTO items (run_id, url_key, url, title, source, ts, score)"
74
+ " VALUES (?, ?, ?, ?, ?, ?, ?)",
75
+ [(run_id, url_key(i.hit.url), i.hit.url, i.hit.title, i.source,
76
+ i.hit.timestamp.isoformat() if i.hit.timestamp else None, i.score)
77
+ for i in result.items],
78
+ ) # fmt: skip
79
+ return run_id
80
+
81
+ def runs(self, limit: int = 20, topic: str | None = None) -> list[dict]:
82
+ q, args = "SELECT id, topic, mode, days, created_at, headline FROM runs", []
83
+ if topic:
84
+ q, args = q + " WHERE topic = ?", [norm_topic(topic)]
85
+ rows = self.db.execute(q + " ORDER BY id DESC LIMIT ?", [*args, limit]).fetchall()
86
+ return [dict(r) for r in rows]
87
+
88
+ def run(self, run_id: int) -> dict | None:
89
+ row = self.db.execute("SELECT * FROM runs WHERE id = ?", (run_id,)).fetchone()
90
+ return dict(row) if row else None
91
+
92
+ def items(self, run_id: int) -> list[dict]:
93
+ rows = self.db.execute(
94
+ "SELECT * FROM items WHERE run_id = ? ORDER BY score DESC", (run_id,)
95
+ )
96
+ return [dict(r) for r in rows]
97
+
98
+ def previous(self, topic: str, before_id: int) -> int | None:
99
+ row = self.db.execute(
100
+ "SELECT id FROM runs WHERE topic = ? AND id < ? ORDER BY id DESC LIMIT 1",
101
+ (norm_topic(topic), before_id),
102
+ ).fetchone()
103
+ return row["id"] if row else None
104
+
105
+ def new_items(self, run_id: int, previous_id: int | None) -> list[dict]:
106
+ """Items in this run whose URL never appeared in the previous run of the same topic."""
107
+ if previous_id is None:
108
+ return self.items(run_id)
109
+ rows = self.db.execute("SELECT url_key FROM items WHERE run_id = ?", (previous_id,))
110
+ seen = {r["url_key"] for r in rows}
111
+ return [i for i in self.items(run_id) if i["url_key"] not in seen]
112
+
113
+ def search(self, query: str, limit: int = 10, exclude: int | None = None) -> list[dict]:
114
+ """Offline, deterministic BM25 over every saved brief."""
115
+ rows = [dict(r) for r in self.db.execute(
116
+ "SELECT id, topic, mode, days, created_at, headline, markdown FROM runs"
117
+ " ORDER BY id DESC LIMIT 2000")] # fmt: skip
118
+ rows = [r for r in rows if r["id"] != exclude]
119
+ if not rows:
120
+ return []
121
+ texts = [f"{r['topic']} {r['topic']} {r['headline']} {r['markdown']}" for r in rows]
122
+ scores = bm25_scores(texts, query)
123
+ ranked = sorted(zip(scores, rows, strict=True), key=lambda sr: (-sr[0], -sr[1]["id"]))
124
+ return [{k: v for k, v in r.items() if k != "markdown"} | {"score": round(s, 3)}
125
+ for s, r in ranked[:limit] if s > 0] # fmt: skip
126
+
127
+ # ---- watches -----------------------------------------------------------------------------
128
+ def add_watch(self, topic: str, days: int = 7, sources: str | None = None,
129
+ webhook: str | None = None) -> None: # fmt: skip
130
+ with self.db:
131
+ self.db.execute(
132
+ "INSERT INTO watches (topic, days, sources, webhook, created_at) VALUES (?, ?, ?, ?, ?)"
133
+ " ON CONFLICT(topic) DO UPDATE SET days = excluded.days,"
134
+ " sources = excluded.sources, webhook = excluded.webhook",
135
+ (norm_topic(topic), days, sources, webhook, datetime.now(UTC).isoformat()),
136
+ )
137
+
138
+ def remove_watch(self, topic: str) -> bool:
139
+ with self.db:
140
+ cur = self.db.execute("DELETE FROM watches WHERE topic = ?", (norm_topic(topic),))
141
+ return cur.rowcount > 0
142
+
143
+ def watches(self) -> list[dict]:
144
+ return [dict(r) for r in self.db.execute("SELECT * FROM watches ORDER BY topic")]
145
+
146
+ def mark_watched(self, topic: str, run_id: int) -> None:
147
+ with self.db:
148
+ self.db.execute("UPDATE watches SET last_run_id = ? WHERE topic = ?",
149
+ (run_id, norm_topic(topic))) # fmt: skip
@@ -0,0 +1,70 @@
1
+ """Watchlist: re-run saved topics, report only what is new since the last run, optional webhook."""
2
+
3
+ from urllib.parse import urlsplit
4
+
5
+ from curl_cffi.requests.exceptions import RequestException
6
+
7
+ from agent_wire.firewall.outbound_privacy import scrub
8
+ from agent_wire.research.store import norm_topic
9
+
10
+ MAX_NEW = 10
11
+
12
+
13
+ def check_webhook(url: str | None) -> str | None:
14
+ if not url:
15
+ return None
16
+ parts = urlsplit(url)
17
+ if parts.scheme not in ("http", "https") or not parts.hostname:
18
+ raise ValueError("webhook must be an http(s) URL")
19
+ return url
20
+
21
+
22
+ async def run_watch(wire, watch: dict) -> dict:
23
+ """One watched topic -> fresh stored run + the items the previous run didn't have."""
24
+ topic = watch["topic"]
25
+ result, _, _ = await wire.research_result(topic, days=watch["days"],
26
+ platforms=watch.get("sources"), depth="quick",
27
+ store=True) # fmt: skip
28
+ store = wire.store
29
+ prev = store.previous(topic, result.run_id)
30
+ fresh = store.new_items(result.run_id, prev)[:MAX_NEW]
31
+ store.mark_watched(topic, result.run_id)
32
+ report = {"topic": topic, "run_id": result.run_id, "previous_run": prev,
33
+ "new": [{"title": i["title"], "url": i["url"], "source": i["source"], "ts": i["ts"]}
34
+ for i in fresh]} # fmt: skip
35
+ if hook := watch.get("webhook"):
36
+ report["webhook"] = await _post(wire, hook, report)
37
+ return report
38
+
39
+
40
+ async def _post(wire, url: str, report: dict) -> str:
41
+ if not report["new"]:
42
+ return "skipped (nothing new)"
43
+ try:
44
+ f = await wire.pool.get(url, method="POST", impersonate=None, json_body=report,
45
+ headers={"content-type": "application/json"}) # fmt: skip
46
+ except RequestException as e:
47
+ return f"failed: {type(e).__name__}"
48
+ return f"HTTP {f.status}"
49
+
50
+
51
+ async def run_watches(wire, topic: str | None = None) -> str:
52
+ watches = wire.store.watches()
53
+ if topic:
54
+ watches = [w for w in watches if w["topic"] == norm_topic(topic)]
55
+ if not watches:
56
+ return "no watched topics (wire watch add <topic>)"
57
+ out = []
58
+ for w in watches:
59
+ r = await run_watch(wire, w)
60
+ head = f"## {r['topic']} · run #{r['run_id']}"
61
+ if r["previous_run"]:
62
+ head += f" · {len(r['new'])} new since run #{r['previous_run']}"
63
+ else:
64
+ head += " · first run"
65
+ if r.get("webhook"):
66
+ head += f" · webhook {r['webhook']}"
67
+ lines = [f"- [{i['title']}]({i['url']}) · {i['source']} · {(i['ts'] or '')[:10]}"
68
+ for i in r["new"]] # fmt: skip
69
+ out.append("\n".join([head, *(lines or ["- nothing new"])]))
70
+ return scrub("\n\n".join(out))