metatron-cli 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metatron/__init__.py +51 -0
- metatron/api.py +290 -0
- metatron/cli.py +221 -0
- metatron/config.py +187 -0
- metatron/db.py +357 -0
- metatron/dedup.py +210 -0
- metatron/fetcher.py +147 -0
- metatron/llm.py +270 -0
- metatron/normalize.py +141 -0
- metatron/poller.py +325 -0
- metatron_cli-0.2.1.dist-info/METADATA +174 -0
- metatron_cli-0.2.1.dist-info/RECORD +16 -0
- metatron_cli-0.2.1.dist-info/WHEEL +5 -0
- metatron_cli-0.2.1.dist-info/entry_points.txt +2 -0
- metatron_cli-0.2.1.dist-info/licenses/LICENSE +21 -0
- metatron_cli-0.2.1.dist-info/top_level.txt +1 -0
metatron/normalize.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""URL and title normalization for the first layers of dedup.
|
|
2
|
+
|
|
3
|
+
Cheap pure functions. Catches the easy duplicates (same article, different
|
|
4
|
+
tracking params; same article syndicated with different ?utm_*; "BREAKING:"
|
|
5
|
+
prefixed reprints) without any LLM calls.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
_TRACKING_PARAM_PREFIXES = (
|
|
15
|
+
"utm_",
|
|
16
|
+
"ga_",
|
|
17
|
+
"mc_",
|
|
18
|
+
"_hs",
|
|
19
|
+
"hsa_",
|
|
20
|
+
"matomo_",
|
|
21
|
+
"pk_",
|
|
22
|
+
"fb_",
|
|
23
|
+
)
|
|
24
|
+
_TRACKING_PARAM_EXACT = {
|
|
25
|
+
"fbclid",
|
|
26
|
+
"gclid",
|
|
27
|
+
"msclkid",
|
|
28
|
+
"dclid",
|
|
29
|
+
"yclid",
|
|
30
|
+
"twclid",
|
|
31
|
+
"igshid",
|
|
32
|
+
"mkt_tok",
|
|
33
|
+
"ref",
|
|
34
|
+
"ref_src",
|
|
35
|
+
"ref_url",
|
|
36
|
+
"source",
|
|
37
|
+
"src",
|
|
38
|
+
"amp",
|
|
39
|
+
"share",
|
|
40
|
+
"from",
|
|
41
|
+
"feed",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def canonicalize_url(url: str) -> str:
|
|
46
|
+
"""Return a stable canonical form of an article URL.
|
|
47
|
+
|
|
48
|
+
Operations:
|
|
49
|
+
- lowercase the scheme and host
|
|
50
|
+
- drop www. prefix from host
|
|
51
|
+
- drop default port for http/https
|
|
52
|
+
- drop fragment
|
|
53
|
+
- drop tracking query params (utm_*, fbclid, etc.)
|
|
54
|
+
- sort remaining query params for stability
|
|
55
|
+
- strip trailing slash on non-empty paths
|
|
56
|
+
"""
|
|
57
|
+
if not url:
|
|
58
|
+
return ""
|
|
59
|
+
parts = urlsplit(url.strip())
|
|
60
|
+
|
|
61
|
+
scheme = (parts.scheme or "https").lower()
|
|
62
|
+
host = (parts.hostname or "").lower()
|
|
63
|
+
if host.startswith("www."):
|
|
64
|
+
host = host[4:]
|
|
65
|
+
netloc = host
|
|
66
|
+
if parts.port and not (
|
|
67
|
+
(scheme == "http" and parts.port == 80)
|
|
68
|
+
or (scheme == "https" and parts.port == 443)
|
|
69
|
+
):
|
|
70
|
+
netloc = f"{host}:{parts.port}"
|
|
71
|
+
|
|
72
|
+
path = parts.path
|
|
73
|
+
if len(path) > 1 and path.endswith("/"):
|
|
74
|
+
path = path[:-1]
|
|
75
|
+
|
|
76
|
+
kept_params = []
|
|
77
|
+
for key, value in parse_qsl(parts.query, keep_blank_values=False):
|
|
78
|
+
k = key.lower()
|
|
79
|
+
if k in _TRACKING_PARAM_EXACT:
|
|
80
|
+
continue
|
|
81
|
+
if any(k.startswith(p) for p in _TRACKING_PARAM_PREFIXES):
|
|
82
|
+
continue
|
|
83
|
+
kept_params.append((key, value))
|
|
84
|
+
kept_params.sort()
|
|
85
|
+
query = urlencode(kept_params)
|
|
86
|
+
|
|
87
|
+
return urlunsplit((scheme, netloc, path, query, ""))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
_TITLE_PUNCT_RE = re.compile(r"[^\w\s]")
|
|
91
|
+
_TITLE_WS_RE = re.compile(r"\s+")
|
|
92
|
+
_TITLE_PREFIX_RE = re.compile(
|
|
93
|
+
r"^\s*(breaking|exclusive|update(d)?|analysis|opinion|video|live|watch)"
|
|
94
|
+
r"\s*[:\-—–]\s*",
|
|
95
|
+
re.IGNORECASE,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def normalize_title(title: str) -> str:
|
|
100
|
+
"""Return a normalized title suitable for fingerprint comparison.
|
|
101
|
+
|
|
102
|
+
Lowercase, drop punctuation, collapse whitespace, strip common newsroom
|
|
103
|
+
prefixes ("BREAKING: ", "UPDATE -", etc.).
|
|
104
|
+
"""
|
|
105
|
+
if not title:
|
|
106
|
+
return ""
|
|
107
|
+
s = title.strip()
|
|
108
|
+
# Repeatedly strip prefixes (some titles have stacked "BREAKING: UPDATE: ...")
|
|
109
|
+
while True:
|
|
110
|
+
m = _TITLE_PREFIX_RE.match(s)
|
|
111
|
+
if not m:
|
|
112
|
+
break
|
|
113
|
+
s = s[m.end() :]
|
|
114
|
+
s = s.lower()
|
|
115
|
+
s = _TITLE_PUNCT_RE.sub(" ", s)
|
|
116
|
+
s = _TITLE_WS_RE.sub(" ", s).strip()
|
|
117
|
+
return s
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_TOKEN_RE = re.compile(r"\w+")
|
|
121
|
+
_STOPWORDS = {
|
|
122
|
+
"a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
|
|
123
|
+
"to", "of", "in", "on", "at", "for", "by", "from", "with", "as",
|
|
124
|
+
"and", "or", "but", "if", "than", "then", "that", "this", "these",
|
|
125
|
+
"those", "it", "its", "into", "over", "after", "before", "about",
|
|
126
|
+
"up", "down", "out", "off", "via", "vs", "v",
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def tokenize(text: str) -> set[str]:
|
|
131
|
+
"""Return a set of meaningful tokens for cheap overlap comparison."""
|
|
132
|
+
if not text:
|
|
133
|
+
return set()
|
|
134
|
+
return {t for t in (m.group(0).lower() for m in _TOKEN_RE.finditer(text)) if t not in _STOPWORDS and len(t) > 2}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def jaccard(a: set[str], b: set[str]) -> float:
|
|
138
|
+
"""Jaccard similarity between two token sets. 0.0 if either is empty."""
|
|
139
|
+
if not a or not b:
|
|
140
|
+
return 0.0
|
|
141
|
+
return len(a & b) / len(a | b)
|
metatron/poller.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Feed poller — fetch all feeds, dedup in one batched LLM call, store.
|
|
2
|
+
|
|
3
|
+
Per refresh / poll-tick:
|
|
4
|
+
|
|
5
|
+
1. Mark every due feed "polled" (claims it against concurrent runs).
|
|
6
|
+
2. Fetch and parse each feed.
|
|
7
|
+
3. Run the cheap dedup layers (URL + normalized title) per item; either
|
|
8
|
+
drop it as an exact duplicate or attach it to an existing cluster.
|
|
9
|
+
4. For the items that survive, build one BatchPlan and call the LLM
|
|
10
|
+
once. Apply the returned groupings.
|
|
11
|
+
5. Insert candidates as either: a new article, a join to an existing
|
|
12
|
+
cluster, or a new cluster formed across candidates.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import asyncio
|
|
18
|
+
import logging
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from metatron.db import Database
|
|
23
|
+
from metatron.dedup import DedupConfig, build_batch_plan, cheap_decide, run_batch
|
|
24
|
+
from metatron.fetcher import FeedFetchError, parse_feed
|
|
25
|
+
from metatron.llm import BatchJudge
|
|
26
|
+
from metatron.normalize import canonicalize_url
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class PollStats:
|
|
33
|
+
polled: int = 0
|
|
34
|
+
new_articles: int = 0
|
|
35
|
+
duplicates: int = 0
|
|
36
|
+
cluster_joins: int = 0
|
|
37
|
+
feed_errors: int = 0
|
|
38
|
+
|
|
39
|
+
def as_dict(self) -> dict[str, int]:
|
|
40
|
+
return {
|
|
41
|
+
"polled": self.polled,
|
|
42
|
+
"new_articles": self.new_articles,
|
|
43
|
+
"duplicates": self.duplicates,
|
|
44
|
+
"cluster_joins": self.cluster_joins,
|
|
45
|
+
"feed_errors": self.feed_errors,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Poller:
|
|
50
|
+
"""Async background poller that runs alongside the FastAPI app."""
|
|
51
|
+
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
*,
|
|
55
|
+
db: Database,
|
|
56
|
+
judge: BatchJudge,
|
|
57
|
+
tick_seconds: int = 60,
|
|
58
|
+
feed_timeout_seconds: int = 30,
|
|
59
|
+
dedup_config: DedupConfig | None = None,
|
|
60
|
+
) -> None:
|
|
61
|
+
self.db = db
|
|
62
|
+
self.judge = judge
|
|
63
|
+
self.tick_seconds = tick_seconds
|
|
64
|
+
self.feed_timeout_seconds = feed_timeout_seconds
|
|
65
|
+
self.dedup_config = dedup_config or DedupConfig()
|
|
66
|
+
self._task: asyncio.Task[None] | None = None
|
|
67
|
+
self._stop_event: asyncio.Event | None = None
|
|
68
|
+
|
|
69
|
+
def start(self) -> None:
|
|
70
|
+
if self._task is not None and not self._task.done():
|
|
71
|
+
return
|
|
72
|
+
loop = asyncio.get_event_loop()
|
|
73
|
+
self._stop_event = asyncio.Event()
|
|
74
|
+
self._task = loop.create_task(self._run(), name="metatron-poller")
|
|
75
|
+
logger.info("Background poller started")
|
|
76
|
+
|
|
77
|
+
async def stop(self) -> None:
|
|
78
|
+
if self._stop_event is not None:
|
|
79
|
+
self._stop_event.set()
|
|
80
|
+
if self._task is not None:
|
|
81
|
+
try:
|
|
82
|
+
await asyncio.wait_for(self._task, timeout=10.0)
|
|
83
|
+
except asyncio.TimeoutError:
|
|
84
|
+
self._task.cancel()
|
|
85
|
+
logger.info("Background poller stopped")
|
|
86
|
+
|
|
87
|
+
async def _run(self) -> None:
|
|
88
|
+
assert self._stop_event is not None
|
|
89
|
+
while not self._stop_event.is_set():
|
|
90
|
+
try:
|
|
91
|
+
await asyncio.get_running_loop().run_in_executor(None, self._tick)
|
|
92
|
+
except Exception as e: # noqa: BLE001 - keep loop alive
|
|
93
|
+
logger.exception("Poller tick failed: %s", e)
|
|
94
|
+
try:
|
|
95
|
+
await asyncio.wait_for(self._stop_event.wait(), timeout=self.tick_seconds)
|
|
96
|
+
except asyncio.TimeoutError:
|
|
97
|
+
pass
|
|
98
|
+
|
|
99
|
+
def _tick(self) -> None:
|
|
100
|
+
due = self.db.list_due_feeds()
|
|
101
|
+
if not due:
|
|
102
|
+
return
|
|
103
|
+
by_project: dict[str, list[dict[str, Any]]] = {}
|
|
104
|
+
for feed in due:
|
|
105
|
+
by_project.setdefault(feed["project_id"], []).append(feed)
|
|
106
|
+
for project_id, feeds in by_project.items():
|
|
107
|
+
poll_feeds(
|
|
108
|
+
project_id,
|
|
109
|
+
feeds,
|
|
110
|
+
self.db,
|
|
111
|
+
self.judge,
|
|
112
|
+
self.dedup_config,
|
|
113
|
+
self.feed_timeout_seconds,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ── core poll routine ────────────────────────────────────────────────────
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def poll_feeds(
|
|
121
|
+
project_id: str,
|
|
122
|
+
feeds: list[dict[str, Any]],
|
|
123
|
+
db: Database,
|
|
124
|
+
judge: BatchJudge,
|
|
125
|
+
dedup_config: DedupConfig,
|
|
126
|
+
feed_timeout_seconds: int,
|
|
127
|
+
) -> PollStats:
|
|
128
|
+
"""Fetch and dedup a batch of feeds for one project, with one LLM call."""
|
|
129
|
+
stats = PollStats()
|
|
130
|
+
if not feeds:
|
|
131
|
+
return stats
|
|
132
|
+
|
|
133
|
+
# Claim everything immediately so concurrent polls don't pile on.
|
|
134
|
+
for feed in feeds:
|
|
135
|
+
db.mark_feed_polled(feed["id"], error=None)
|
|
136
|
+
|
|
137
|
+
# --- Phase 1: fetch all feeds, gather raw candidates ---
|
|
138
|
+
raw: list[tuple[dict[str, Any], dict[str, Any]]] = [] # (feed_row, parsed_item)
|
|
139
|
+
for feed in feeds:
|
|
140
|
+
stats.polled += 1
|
|
141
|
+
try:
|
|
142
|
+
items = parse_feed(
|
|
143
|
+
feed["url"],
|
|
144
|
+
source_name=feed.get("name") or None,
|
|
145
|
+
timeout=feed_timeout_seconds,
|
|
146
|
+
)
|
|
147
|
+
except FeedFetchError as e:
|
|
148
|
+
logger.warning("Feed %s failed: %s", feed["url"], e)
|
|
149
|
+
db.mark_feed_polled(feed["id"], error=str(e)[:500])
|
|
150
|
+
stats.feed_errors += 1
|
|
151
|
+
continue
|
|
152
|
+
for item in items:
|
|
153
|
+
raw.append((feed, item))
|
|
154
|
+
|
|
155
|
+
if not raw:
|
|
156
|
+
return stats
|
|
157
|
+
|
|
158
|
+
# --- Phase 2: cheap layers + collect survivors for LLM batching ---
|
|
159
|
+
history = db.recent_articles_for_dedup(project_id, days=7)
|
|
160
|
+
seen_canonicals: set[str] = set()
|
|
161
|
+
|
|
162
|
+
candidates: list[dict[str, Any]] = []
|
|
163
|
+
# Candidates joining clusters via cheap layers — keyed by the candidate
|
|
164
|
+
# dict's id() to avoid relying on yet-to-be-built _ref_id.
|
|
165
|
+
cheap_links: dict[int, tuple[str | None, str | None, str | None]] = {}
|
|
166
|
+
# (cluster_id, history_article_id, peer_candidate_id_marker)
|
|
167
|
+
|
|
168
|
+
for feed, item in raw:
|
|
169
|
+
canonical = canonicalize_url(item.source_url)
|
|
170
|
+
if not canonical:
|
|
171
|
+
continue
|
|
172
|
+
if canonical in seen_canonicals:
|
|
173
|
+
stats.duplicates += 1
|
|
174
|
+
continue
|
|
175
|
+
if db.get_article_by_canonical_url(project_id, canonical):
|
|
176
|
+
stats.duplicates += 1
|
|
177
|
+
seen_canonicals.add(canonical)
|
|
178
|
+
continue
|
|
179
|
+
|
|
180
|
+
candidate = {
|
|
181
|
+
"project_id": project_id,
|
|
182
|
+
"canonical_url": canonical,
|
|
183
|
+
"source_url": item.source_url,
|
|
184
|
+
"source": item.source,
|
|
185
|
+
"title": item.title,
|
|
186
|
+
"summary": item.summary,
|
|
187
|
+
"body": item.body,
|
|
188
|
+
"published": item.published.isoformat() if item.published else None,
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
# Cheap-decide against history plus already-queued candidates (so a
|
|
192
|
+
# near-identical second-feed copy of the same headline within the
|
|
193
|
+
# same poll joins the first one's cluster without needing the LLM).
|
|
194
|
+
within_batch_history = history + [
|
|
195
|
+
{"id": f"_cand_{id(c)}", **c} for c in candidates
|
|
196
|
+
]
|
|
197
|
+
decision = cheap_decide(candidate, within_batch_history)
|
|
198
|
+
|
|
199
|
+
if decision.status == "duplicate":
|
|
200
|
+
stats.duplicates += 1
|
|
201
|
+
seen_canonicals.add(canonical)
|
|
202
|
+
continue
|
|
203
|
+
|
|
204
|
+
seen_canonicals.add(canonical)
|
|
205
|
+
|
|
206
|
+
if decision.status == "cluster":
|
|
207
|
+
history_id = decision.canonical_article_id
|
|
208
|
+
if history_id and history_id.startswith("_cand_"):
|
|
209
|
+
# Matched another in-batch candidate; resolve to a real
|
|
210
|
+
# cluster after that candidate is inserted.
|
|
211
|
+
cheap_links[id(candidate)] = (None, None, history_id)
|
|
212
|
+
else:
|
|
213
|
+
cheap_links[id(candidate)] = (
|
|
214
|
+
decision.cluster_id,
|
|
215
|
+
decision.canonical_article_id,
|
|
216
|
+
None,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
candidates.append(candidate)
|
|
220
|
+
|
|
221
|
+
if not candidates:
|
|
222
|
+
return stats
|
|
223
|
+
|
|
224
|
+
# --- Phase 3: one LLM call across ALL surviving candidates + their peers ---
|
|
225
|
+
plan = build_batch_plan(candidates, history, config=dedup_config)
|
|
226
|
+
matches = run_batch(plan, judge) if judge.enabled else {}
|
|
227
|
+
|
|
228
|
+
# Map ref_id → candidate dict for easy lookup
|
|
229
|
+
by_ref = {c["_ref_id"]: c for c in candidates}
|
|
230
|
+
history_ref_to_id = {row["id"]: row for row in history}
|
|
231
|
+
|
|
232
|
+
# --- Phase 4: write articles, respecting both cheap and LLM clusters ---
|
|
233
|
+
# Track per-candidate cluster assignment by id(candidate). After insertion,
|
|
234
|
+
# also map the in-batch marker "_cand_<id>" to the real article id so
|
|
235
|
+
# later candidates with cheap_links pointing at it can resolve.
|
|
236
|
+
candidate_to_inserted: dict[int, dict[str, Any]] = {}
|
|
237
|
+
|
|
238
|
+
for cand in candidates:
|
|
239
|
+
ref = cand.pop("_ref_id", None)
|
|
240
|
+
cluster_id_to_use: str | None = None
|
|
241
|
+
|
|
242
|
+
# (a) cheap layer linkage: either existing history or another candidate
|
|
243
|
+
cheap = cheap_links.get(id(cand))
|
|
244
|
+
if cheap is not None:
|
|
245
|
+
existing_cluster, existing_article, peer_marker = cheap
|
|
246
|
+
if peer_marker is not None:
|
|
247
|
+
# peer_marker = "_cand_<id(peer_cand)>"
|
|
248
|
+
peer_id = int(peer_marker.removeprefix("_cand_"))
|
|
249
|
+
peer_inserted = candidate_to_inserted.get(peer_id)
|
|
250
|
+
if peer_inserted is not None:
|
|
251
|
+
cluster_id_to_use = peer_inserted.get(
|
|
252
|
+
"cluster_id"
|
|
253
|
+
) or _ensure_cluster(db, project_id, peer_inserted["id"])
|
|
254
|
+
# Backfill the peer's cluster_id so future siblings see it.
|
|
255
|
+
peer_inserted["cluster_id"] = cluster_id_to_use
|
|
256
|
+
elif existing_cluster:
|
|
257
|
+
cluster_id_to_use = existing_cluster
|
|
258
|
+
elif existing_article:
|
|
259
|
+
cluster_id_to_use = _ensure_cluster(db, project_id, existing_article)
|
|
260
|
+
|
|
261
|
+
# (b) LLM peers — existing history or another candidate
|
|
262
|
+
if cluster_id_to_use is None and ref is not None:
|
|
263
|
+
peers = matches.get(ref, [])
|
|
264
|
+
for peer_ref in peers:
|
|
265
|
+
if peer_ref in history_ref_to_id:
|
|
266
|
+
peer = history_ref_to_id[peer_ref]
|
|
267
|
+
cluster_id_to_use = peer.get("cluster_id") or _ensure_cluster(
|
|
268
|
+
db, project_id, peer["id"]
|
|
269
|
+
)
|
|
270
|
+
break
|
|
271
|
+
if cluster_id_to_use is None:
|
|
272
|
+
for peer_ref in peers:
|
|
273
|
+
if peer_ref in by_ref and by_ref[peer_ref].get("cluster_id"):
|
|
274
|
+
cluster_id_to_use = by_ref[peer_ref]["cluster_id"]
|
|
275
|
+
break
|
|
276
|
+
|
|
277
|
+
if cluster_id_to_use:
|
|
278
|
+
cand["cluster_id"] = cluster_id_to_use
|
|
279
|
+
|
|
280
|
+
inserted = db.insert_article(cand)
|
|
281
|
+
if inserted is None:
|
|
282
|
+
stats.duplicates += 1
|
|
283
|
+
continue
|
|
284
|
+
|
|
285
|
+
# If LLM peers exist but pointed at not-yet-inserted candidates,
|
|
286
|
+
# open a fresh cluster around this article so peers can attach.
|
|
287
|
+
if cluster_id_to_use is None and ref is not None and matches.get(ref):
|
|
288
|
+
new_cluster_id = db.create_cluster(project_id, inserted["id"])
|
|
289
|
+
db.attach_to_cluster(inserted["id"], new_cluster_id)
|
|
290
|
+
inserted["cluster_id"] = new_cluster_id
|
|
291
|
+
cand["cluster_id"] = new_cluster_id
|
|
292
|
+
|
|
293
|
+
candidate_to_inserted[id(cand)] = inserted
|
|
294
|
+
|
|
295
|
+
if cluster_id_to_use:
|
|
296
|
+
stats.cluster_joins += 1
|
|
297
|
+
else:
|
|
298
|
+
stats.new_articles += 1
|
|
299
|
+
|
|
300
|
+
return stats
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def refresh_project_now(
|
|
304
|
+
project_id: str,
|
|
305
|
+
db: Database,
|
|
306
|
+
judge: BatchJudge,
|
|
307
|
+
dedup_config: DedupConfig,
|
|
308
|
+
feed_timeout_seconds: int,
|
|
309
|
+
) -> dict[str, int]:
|
|
310
|
+
"""Synchronously poll every enabled feed in a project — single LLM call."""
|
|
311
|
+
feeds = [f for f in db.list_feeds(project_id) if f.get("enabled")]
|
|
312
|
+
stats = poll_feeds(project_id, feeds, db, judge, dedup_config, feed_timeout_seconds)
|
|
313
|
+
return stats.as_dict()
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _ensure_cluster(
|
|
317
|
+
db: Database, project_id: str, canonical_article_id: str
|
|
318
|
+
) -> str:
|
|
319
|
+
"""Find or create the cluster for ``canonical_article_id``."""
|
|
320
|
+
article = db.get_article(canonical_article_id)
|
|
321
|
+
if article and article.get("cluster_id"):
|
|
322
|
+
return article["cluster_id"]
|
|
323
|
+
cluster_id = db.create_cluster(project_id, canonical_article_id)
|
|
324
|
+
db.attach_to_cluster(canonical_article_id, cluster_id)
|
|
325
|
+
return cluster_id
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: metatron-cli
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Multi-project web feed manager with cross-outlet deduplication
|
|
5
|
+
Author-email: Matt Weber <matt.weber@beefree.io>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/mattweberio/metatron
|
|
8
|
+
Project-URL: Repository, https://github.com/mattweberio/metatron
|
|
9
|
+
Project-URL: Issues, https://github.com/mattweberio/metatron/issues
|
|
10
|
+
Keywords: rss,feeds,deduplication,news,aggregator,web,ingestion
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
17
|
+
Classifier: Topic :: Text Processing :: Markup
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: feedparser>=6.0
|
|
22
|
+
Requires-Dist: requests>=2.31
|
|
23
|
+
Requires-Dist: trafilatura>=1.12
|
|
24
|
+
Requires-Dist: fastapi>=0.110
|
|
25
|
+
Requires-Dist: uvicorn[standard]>=0.27
|
|
26
|
+
Requires-Dist: httpx>=0.27
|
|
27
|
+
Requires-Dist: pydantic>=2.0
|
|
28
|
+
Provides-Extra: test
|
|
29
|
+
Requires-Dist: pytest>=9.0; extra == "test"
|
|
30
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "test"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# Metatron
|
|
34
|
+
|
|
35
|
+
Multi-project web feed manager with cross-outlet deduplication.
|
|
36
|
+
|
|
37
|
+
You point Metatron at sources, organized by project. It polls them, dedups
|
|
38
|
+
the articles (URL canonicalization → normalized title → token overlap →
|
|
39
|
+
LLM tiebreaker), and serves you the unique articles via a small HTTP API.
|
|
40
|
+
RSS is the source it speaks today; the architecture is medium-agnostic and
|
|
41
|
+
grows to other web sources over time.
|
|
42
|
+
|
|
43
|
+
It does this one thing. It doesn't curate, it doesn't summarize, it doesn't
|
|
44
|
+
have opinions about what's interesting. That's for whatever calls it.
|
|
45
|
+
|
|
46
|
+
## Install
|
|
47
|
+
|
|
48
|
+
The CLI is `metatron`. The published package is `metatron-cli` (the bare
|
|
49
|
+
`metatron` name was already taken on PyPI):
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pipx install metatron-cli # from PyPI
|
|
53
|
+
pipx install git+https://github.com/mattweberio/metatron # from GitHub (latest main)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Configure
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
metatron config init # writes ~/.config/metatron/config.toml
|
|
60
|
+
metatron config show # prints the resolved config
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Edit `~/.config/metatron/config.toml`. The LLM tiebreaker — needed to catch
|
|
64
|
+
"same story, different outlet" duplicates — requires an Anthropic API key:
|
|
65
|
+
|
|
66
|
+
```toml
|
|
67
|
+
[llm]
|
|
68
|
+
api_key = "sk-ant-..."
|
|
69
|
+
model = "claude-sonnet-4-6"
|
|
70
|
+
|
|
71
|
+
[api]
|
|
72
|
+
api_token = "your-bearer-token"
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Without an LLM key, dedup falls back to URL + normalized-title matching only.
|
|
76
|
+
Cross-outlet duplicates from different sources will leak through.
|
|
77
|
+
|
|
78
|
+
## Run
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
metatron serve # HTTP API on 127.0.0.1:8765
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Use
|
|
85
|
+
|
|
86
|
+
Bearer-token auth on everything except `/health`.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
TOKEN="your-bearer-token"
|
|
90
|
+
BASE="http://127.0.0.1:8765"
|
|
91
|
+
|
|
92
|
+
# Create a project
|
|
93
|
+
curl -s -X POST "$BASE/projects" \
|
|
94
|
+
-H "Authorization: Bearer $TOKEN" \
|
|
95
|
+
-H "Content-Type: application/json" \
|
|
96
|
+
-d '{"name": "ai-news"}'
|
|
97
|
+
|
|
98
|
+
# Add feeds
|
|
99
|
+
PROJECT_ID=...
|
|
100
|
+
curl -s -X POST "$BASE/projects/$PROJECT_ID/feeds" \
|
|
101
|
+
-H "Authorization: Bearer $TOKEN" \
|
|
102
|
+
-H "Content-Type: application/json" \
|
|
103
|
+
-d '{"url": "https://hnrss.org/frontpage"}'
|
|
104
|
+
|
|
105
|
+
# List deduplicated articles
|
|
106
|
+
curl -s -H "Authorization: Bearer $TOKEN" \
|
|
107
|
+
"$BASE/projects/$PROJECT_ID/articles?limit=20"
|
|
108
|
+
|
|
109
|
+
# Force a synchronous refresh
|
|
110
|
+
curl -s -X POST -H "Authorization: Bearer $TOKEN" \
|
|
111
|
+
"$BASE/projects/$PROJECT_ID/refresh"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Bulk feed import
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
metatron seed-feeds my-project ./feeds.json
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Where `feeds.json` is:
|
|
121
|
+
```json
|
|
122
|
+
{
|
|
123
|
+
"feeds": [
|
|
124
|
+
{ "url": "https://hnrss.org/frontpage", "name": "Hacker News", "category": "tech" },
|
|
125
|
+
{ "url": "https://www.theverge.com/rss/index.xml", "name": "The Verge", "category": "tech" }
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## How dedup works
|
|
131
|
+
|
|
132
|
+
Cheapest-to-most-expensive, short-circuits on first match:
|
|
133
|
+
|
|
134
|
+
1. **Canonical URL.** Strip `utm_*`, `fbclid`, etc. Same canonical URL → exact duplicate, dropped.
|
|
135
|
+
2. **Normalized title.** Lowercase, drop punctuation, strip newsroom prefixes ("BREAKING: ", "UPDATE - "). Same → joins the existing article's cluster.
|
|
136
|
+
3. **Token overlap.** Jaccard similarity on title + summary tokens. ≥0.12 → shortlist candidate (top 5).
|
|
137
|
+
4. **Sonnet tiebreaker.** Claude Sonnet reads both articles and decides "same story" or "different story." Same → joins cluster.
|
|
138
|
+
|
|
139
|
+
Articles in a cluster are stored individually (you can see the alternative
|
|
140
|
+
sources via `GET /articles/{id}`) but `/projects/{id}/articles` returns
|
|
141
|
+
only the canonical from each cluster.
|
|
142
|
+
|
|
143
|
+
## Storage
|
|
144
|
+
|
|
145
|
+
Single SQLite file at `~/.local/share/metatron/metatron.db`. WAL mode, FK
|
|
146
|
+
constraints on. Override with `[database].path` in config.
|
|
147
|
+
|
|
148
|
+
## API endpoints
|
|
149
|
+
|
|
150
|
+
| Method | Path | Description |
|
|
151
|
+
|--------|---------------------------------------|----------------------------------------------|
|
|
152
|
+
| GET | `/health` | Liveness; reports LLM enablement |
|
|
153
|
+
| POST | `/projects` | Create a project |
|
|
154
|
+
| GET | `/projects` | List projects |
|
|
155
|
+
| DELETE | `/projects/{id}` | Delete a project (cascades feeds + articles) |
|
|
156
|
+
| POST | `/projects/{id}/feeds` | Add a feed |
|
|
157
|
+
| GET | `/projects/{id}/feeds` | List feeds |
|
|
158
|
+
| DELETE | `/feeds/{id}` | Remove a feed |
|
|
159
|
+
| GET | `/projects/{id}/articles?since=&limit=` | List deduped articles |
|
|
160
|
+
| POST | `/projects/{id}/refresh` | Synchronously poll project's feeds |
|
|
161
|
+
| GET | `/articles/{id}` | Article detail + cluster members |
|
|
162
|
+
|
|
163
|
+
## Development
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
python3 -m pytest -q
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Tests cover normalization, the DB layer, the dedup pipeline (with a fake
|
|
170
|
+
LLM judge), the feed poller (with mocked RSS), and the API surface.
|
|
171
|
+
|
|
172
|
+
## License
|
|
173
|
+
|
|
174
|
+
MIT. See [LICENSE](./LICENSE).
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
metatron/__init__.py,sha256=-akQUfUSR1OTezvy17U2ypfTqqiNxkesN0sp05kolQI,1108
|
|
2
|
+
metatron/api.py,sha256=86WTtuJk2BezY0A-BFWySfiuGMwETuLs0DPFJU-3XrY,10514
|
|
3
|
+
metatron/cli.py,sha256=K-356dfShXvuAKD20AOs4YCNAHvGfjTX6MNUtJvW03M,6920
|
|
4
|
+
metatron/config.py,sha256=Vm-ChMODh3k7TvGm4GTRnVhXe1FQs_rFcSQvhYW2mKs,6232
|
|
5
|
+
metatron/db.py,sha256=zmg8-ae95G0OOXWyJBPV1SCczXXKPJuOCcfrBJd5tE8,13377
|
|
6
|
+
metatron/dedup.py,sha256=E3aRa251CGCxNzsgSI8DYCd8y8gxApqtgIbM0Gijsck,7263
|
|
7
|
+
metatron/fetcher.py,sha256=HB9Am9O5SC8m2r7QHrZLkopu0uCih_wY23Jj3uSWoqc,4446
|
|
8
|
+
metatron/llm.py,sha256=D7wmpT6EIWmtJZFlr6TI0dt0tG3c0aDQAXszZpjwLLQ,8800
|
|
9
|
+
metatron/normalize.py,sha256=x0maVuzV2z2-hy-dZ1sjUh3HyuO3vni6vTIwcnQaJPQ,3787
|
|
10
|
+
metatron/poller.py,sha256=veTDqqRUdIFrNIGh57TrM-sL-U_2EkBOd8YGkO3bagg,12067
|
|
11
|
+
metatron_cli-0.2.1.dist-info/licenses/LICENSE,sha256=at8Cx79y_PI4PKS9hCrOYeN8XMhuUigTcFET3fcCWfU,1067
|
|
12
|
+
metatron_cli-0.2.1.dist-info/METADATA,sha256=DvYSqGpC6f68ZdaiOan7VT9ttsJCskSPJF0pL7aE-2w,6129
|
|
13
|
+
metatron_cli-0.2.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
14
|
+
metatron_cli-0.2.1.dist-info/entry_points.txt,sha256=tj_pczFcDHzV9UAsphXx2-APV4iyvhWH9xJBQx06KTM,47
|
|
15
|
+
metatron_cli-0.2.1.dist-info/top_level.txt,sha256=xuqvlh66zVOuc8rmX0NCSGi9HnFxMadj0aNJ1jgxYeA,9
|
|
16
|
+
metatron_cli-0.2.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Matt Weber
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
metatron
|