metatron-cli 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
metatron/normalize.py ADDED
@@ -0,0 +1,141 @@
1
+ """URL and title normalization for the first layers of dedup.
2
+
3
+ Cheap pure functions. Catches the easy duplicates (same article, different
4
+ tracking params; same article syndicated with different ?utm_*; "BREAKING:"
5
+ prefixed reprints) without any LLM calls.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
12
+
13
+
14
+ _TRACKING_PARAM_PREFIXES = (
15
+ "utm_",
16
+ "ga_",
17
+ "mc_",
18
+ "_hs",
19
+ "hsa_",
20
+ "matomo_",
21
+ "pk_",
22
+ "fb_",
23
+ )
24
+ _TRACKING_PARAM_EXACT = {
25
+ "fbclid",
26
+ "gclid",
27
+ "msclkid",
28
+ "dclid",
29
+ "yclid",
30
+ "twclid",
31
+ "igshid",
32
+ "mkt_tok",
33
+ "ref",
34
+ "ref_src",
35
+ "ref_url",
36
+ "source",
37
+ "src",
38
+ "amp",
39
+ "share",
40
+ "from",
41
+ "feed",
42
+ }
43
+
44
+
45
+ def canonicalize_url(url: str) -> str:
46
+ """Return a stable canonical form of an article URL.
47
+
48
+ Operations:
49
+ - lowercase the scheme and host
50
+ - drop www. prefix from host
51
+ - drop default port for http/https
52
+ - drop fragment
53
+ - drop tracking query params (utm_*, fbclid, etc.)
54
+ - sort remaining query params for stability
55
+ - strip trailing slash on non-empty paths
56
+ """
57
+ if not url:
58
+ return ""
59
+ parts = urlsplit(url.strip())
60
+
61
+ scheme = (parts.scheme or "https").lower()
62
+ host = (parts.hostname or "").lower()
63
+ if host.startswith("www."):
64
+ host = host[4:]
65
+ netloc = host
66
+ if parts.port and not (
67
+ (scheme == "http" and parts.port == 80)
68
+ or (scheme == "https" and parts.port == 443)
69
+ ):
70
+ netloc = f"{host}:{parts.port}"
71
+
72
+ path = parts.path
73
+ if len(path) > 1 and path.endswith("/"):
74
+ path = path[:-1]
75
+
76
+ kept_params = []
77
+ for key, value in parse_qsl(parts.query, keep_blank_values=False):
78
+ k = key.lower()
79
+ if k in _TRACKING_PARAM_EXACT:
80
+ continue
81
+ if any(k.startswith(p) for p in _TRACKING_PARAM_PREFIXES):
82
+ continue
83
+ kept_params.append((key, value))
84
+ kept_params.sort()
85
+ query = urlencode(kept_params)
86
+
87
+ return urlunsplit((scheme, netloc, path, query, ""))
88
+
89
+
90
+ _TITLE_PUNCT_RE = re.compile(r"[^\w\s]")
91
+ _TITLE_WS_RE = re.compile(r"\s+")
92
+ _TITLE_PREFIX_RE = re.compile(
93
+ r"^\s*(breaking|exclusive|update(d)?|analysis|opinion|video|live|watch)"
94
+ r"\s*[:\-—–]\s*",
95
+ re.IGNORECASE,
96
+ )
97
+
98
+
99
+ def normalize_title(title: str) -> str:
100
+ """Return a normalized title suitable for fingerprint comparison.
101
+
102
+ Lowercase, drop punctuation, collapse whitespace, strip common newsroom
103
+ prefixes ("BREAKING: ", "UPDATE -", etc.).
104
+ """
105
+ if not title:
106
+ return ""
107
+ s = title.strip()
108
+ # Repeatedly strip prefixes (some titles have stacked "BREAKING: UPDATE: ...")
109
+ while True:
110
+ m = _TITLE_PREFIX_RE.match(s)
111
+ if not m:
112
+ break
113
+ s = s[m.end() :]
114
+ s = s.lower()
115
+ s = _TITLE_PUNCT_RE.sub(" ", s)
116
+ s = _TITLE_WS_RE.sub(" ", s).strip()
117
+ return s
118
+
119
+
120
+ _TOKEN_RE = re.compile(r"\w+")
121
+ _STOPWORDS = {
122
+ "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
123
+ "to", "of", "in", "on", "at", "for", "by", "from", "with", "as",
124
+ "and", "or", "but", "if", "than", "then", "that", "this", "these",
125
+ "those", "it", "its", "into", "over", "after", "before", "about",
126
+ "up", "down", "out", "off", "via", "vs", "v",
127
+ }
128
+
129
+
130
+ def tokenize(text: str) -> set[str]:
131
+ """Return a set of meaningful tokens for cheap overlap comparison."""
132
+ if not text:
133
+ return set()
134
+ return {t for t in (m.group(0).lower() for m in _TOKEN_RE.finditer(text)) if t not in _STOPWORDS and len(t) > 2}
135
+
136
+
137
+ def jaccard(a: set[str], b: set[str]) -> float:
138
+ """Jaccard similarity between two token sets. 0.0 if either is empty."""
139
+ if not a or not b:
140
+ return 0.0
141
+ return len(a & b) / len(a | b)
metatron/poller.py ADDED
@@ -0,0 +1,325 @@
1
+ """Feed poller — fetch all feeds, dedup in one batched LLM call, store.
2
+
3
+ Per refresh / poll-tick:
4
+
5
+ 1. Mark every due feed "polled" (claims it against concurrent runs).
6
+ 2. Fetch and parse each feed.
7
+ 3. Run the cheap dedup layers (URL + normalized title) per item; either
8
+ drop it as an exact duplicate or attach it to an existing cluster.
9
+ 4. For the items that survive, build one BatchPlan and call the LLM
10
+ once. Apply the returned groupings.
11
+ 5. Insert candidates as either: a new article, a join to an existing
12
+ cluster, or a new cluster formed across candidates.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import asyncio
18
+ import logging
19
+ from dataclasses import dataclass, field
20
+ from typing import Any
21
+
22
+ from metatron.db import Database
23
+ from metatron.dedup import DedupConfig, build_batch_plan, cheap_decide, run_batch
24
+ from metatron.fetcher import FeedFetchError, parse_feed
25
+ from metatron.llm import BatchJudge
26
+ from metatron.normalize import canonicalize_url
27
+
28
+ logger = logging.getLogger(__name__)
29
+
30
+
31
+ @dataclass
32
+ class PollStats:
33
+ polled: int = 0
34
+ new_articles: int = 0
35
+ duplicates: int = 0
36
+ cluster_joins: int = 0
37
+ feed_errors: int = 0
38
+
39
+ def as_dict(self) -> dict[str, int]:
40
+ return {
41
+ "polled": self.polled,
42
+ "new_articles": self.new_articles,
43
+ "duplicates": self.duplicates,
44
+ "cluster_joins": self.cluster_joins,
45
+ "feed_errors": self.feed_errors,
46
+ }
47
+
48
+
49
+ class Poller:
50
+ """Async background poller that runs alongside the FastAPI app."""
51
+
52
+ def __init__(
53
+ self,
54
+ *,
55
+ db: Database,
56
+ judge: BatchJudge,
57
+ tick_seconds: int = 60,
58
+ feed_timeout_seconds: int = 30,
59
+ dedup_config: DedupConfig | None = None,
60
+ ) -> None:
61
+ self.db = db
62
+ self.judge = judge
63
+ self.tick_seconds = tick_seconds
64
+ self.feed_timeout_seconds = feed_timeout_seconds
65
+ self.dedup_config = dedup_config or DedupConfig()
66
+ self._task: asyncio.Task[None] | None = None
67
+ self._stop_event: asyncio.Event | None = None
68
+
69
+ def start(self) -> None:
70
+ if self._task is not None and not self._task.done():
71
+ return
72
+ loop = asyncio.get_event_loop()
73
+ self._stop_event = asyncio.Event()
74
+ self._task = loop.create_task(self._run(), name="metatron-poller")
75
+ logger.info("Background poller started")
76
+
77
+ async def stop(self) -> None:
78
+ if self._stop_event is not None:
79
+ self._stop_event.set()
80
+ if self._task is not None:
81
+ try:
82
+ await asyncio.wait_for(self._task, timeout=10.0)
83
+ except asyncio.TimeoutError:
84
+ self._task.cancel()
85
+ logger.info("Background poller stopped")
86
+
87
+ async def _run(self) -> None:
88
+ assert self._stop_event is not None
89
+ while not self._stop_event.is_set():
90
+ try:
91
+ await asyncio.get_running_loop().run_in_executor(None, self._tick)
92
+ except Exception as e: # noqa: BLE001 - keep loop alive
93
+ logger.exception("Poller tick failed: %s", e)
94
+ try:
95
+ await asyncio.wait_for(self._stop_event.wait(), timeout=self.tick_seconds)
96
+ except asyncio.TimeoutError:
97
+ pass
98
+
99
+ def _tick(self) -> None:
100
+ due = self.db.list_due_feeds()
101
+ if not due:
102
+ return
103
+ by_project: dict[str, list[dict[str, Any]]] = {}
104
+ for feed in due:
105
+ by_project.setdefault(feed["project_id"], []).append(feed)
106
+ for project_id, feeds in by_project.items():
107
+ poll_feeds(
108
+ project_id,
109
+ feeds,
110
+ self.db,
111
+ self.judge,
112
+ self.dedup_config,
113
+ self.feed_timeout_seconds,
114
+ )
115
+
116
+
117
+ # ── core poll routine ────────────────────────────────────────────────────
118
+
119
+
120
+ def poll_feeds(
121
+ project_id: str,
122
+ feeds: list[dict[str, Any]],
123
+ db: Database,
124
+ judge: BatchJudge,
125
+ dedup_config: DedupConfig,
126
+ feed_timeout_seconds: int,
127
+ ) -> PollStats:
128
+ """Fetch and dedup a batch of feeds for one project, with one LLM call."""
129
+ stats = PollStats()
130
+ if not feeds:
131
+ return stats
132
+
133
+ # Claim everything immediately so concurrent polls don't pile on.
134
+ for feed in feeds:
135
+ db.mark_feed_polled(feed["id"], error=None)
136
+
137
+ # --- Phase 1: fetch all feeds, gather raw candidates ---
138
+ raw: list[tuple[dict[str, Any], dict[str, Any]]] = [] # (feed_row, parsed_item)
139
+ for feed in feeds:
140
+ stats.polled += 1
141
+ try:
142
+ items = parse_feed(
143
+ feed["url"],
144
+ source_name=feed.get("name") or None,
145
+ timeout=feed_timeout_seconds,
146
+ )
147
+ except FeedFetchError as e:
148
+ logger.warning("Feed %s failed: %s", feed["url"], e)
149
+ db.mark_feed_polled(feed["id"], error=str(e)[:500])
150
+ stats.feed_errors += 1
151
+ continue
152
+ for item in items:
153
+ raw.append((feed, item))
154
+
155
+ if not raw:
156
+ return stats
157
+
158
+ # --- Phase 2: cheap layers + collect survivors for LLM batching ---
159
+ history = db.recent_articles_for_dedup(project_id, days=7)
160
+ seen_canonicals: set[str] = set()
161
+
162
+ candidates: list[dict[str, Any]] = []
163
+ # Candidates joining clusters via cheap layers — keyed by the candidate
164
+ # dict's id() to avoid relying on yet-to-be-built _ref_id.
165
+ cheap_links: dict[int, tuple[str | None, str | None, str | None]] = {}
166
+ # (cluster_id, history_article_id, peer_candidate_id_marker)
167
+
168
+ for feed, item in raw:
169
+ canonical = canonicalize_url(item.source_url)
170
+ if not canonical:
171
+ continue
172
+ if canonical in seen_canonicals:
173
+ stats.duplicates += 1
174
+ continue
175
+ if db.get_article_by_canonical_url(project_id, canonical):
176
+ stats.duplicates += 1
177
+ seen_canonicals.add(canonical)
178
+ continue
179
+
180
+ candidate = {
181
+ "project_id": project_id,
182
+ "canonical_url": canonical,
183
+ "source_url": item.source_url,
184
+ "source": item.source,
185
+ "title": item.title,
186
+ "summary": item.summary,
187
+ "body": item.body,
188
+ "published": item.published.isoformat() if item.published else None,
189
+ }
190
+
191
+ # Cheap-decide against history plus already-queued candidates (so a
192
+ # near-identical second-feed copy of the same headline within the
193
+ # same poll joins the first one's cluster without needing the LLM).
194
+ within_batch_history = history + [
195
+ {"id": f"_cand_{id(c)}", **c} for c in candidates
196
+ ]
197
+ decision = cheap_decide(candidate, within_batch_history)
198
+
199
+ if decision.status == "duplicate":
200
+ stats.duplicates += 1
201
+ seen_canonicals.add(canonical)
202
+ continue
203
+
204
+ seen_canonicals.add(canonical)
205
+
206
+ if decision.status == "cluster":
207
+ history_id = decision.canonical_article_id
208
+ if history_id and history_id.startswith("_cand_"):
209
+ # Matched another in-batch candidate; resolve to a real
210
+ # cluster after that candidate is inserted.
211
+ cheap_links[id(candidate)] = (None, None, history_id)
212
+ else:
213
+ cheap_links[id(candidate)] = (
214
+ decision.cluster_id,
215
+ decision.canonical_article_id,
216
+ None,
217
+ )
218
+
219
+ candidates.append(candidate)
220
+
221
+ if not candidates:
222
+ return stats
223
+
224
+ # --- Phase 3: one LLM call across ALL surviving candidates + their peers ---
225
+ plan = build_batch_plan(candidates, history, config=dedup_config)
226
+ matches = run_batch(plan, judge) if judge.enabled else {}
227
+
228
+ # Map ref_id → candidate dict for easy lookup
229
+ by_ref = {c["_ref_id"]: c for c in candidates}
230
+ history_ref_to_id = {row["id"]: row for row in history}
231
+
232
+ # --- Phase 4: write articles, respecting both cheap and LLM clusters ---
233
+ # Track per-candidate cluster assignment by id(candidate). After insertion,
234
+ # also map the in-batch marker "_cand_<id>" to the real article id so
235
+ # later candidates with cheap_links pointing at it can resolve.
236
+ candidate_to_inserted: dict[int, dict[str, Any]] = {}
237
+
238
+ for cand in candidates:
239
+ ref = cand.pop("_ref_id", None)
240
+ cluster_id_to_use: str | None = None
241
+
242
+ # (a) cheap layer linkage: either existing history or another candidate
243
+ cheap = cheap_links.get(id(cand))
244
+ if cheap is not None:
245
+ existing_cluster, existing_article, peer_marker = cheap
246
+ if peer_marker is not None:
247
+ # peer_marker = "_cand_<id(peer_cand)>"
248
+ peer_id = int(peer_marker.removeprefix("_cand_"))
249
+ peer_inserted = candidate_to_inserted.get(peer_id)
250
+ if peer_inserted is not None:
251
+ cluster_id_to_use = peer_inserted.get(
252
+ "cluster_id"
253
+ ) or _ensure_cluster(db, project_id, peer_inserted["id"])
254
+ # Backfill the peer's cluster_id so future siblings see it.
255
+ peer_inserted["cluster_id"] = cluster_id_to_use
256
+ elif existing_cluster:
257
+ cluster_id_to_use = existing_cluster
258
+ elif existing_article:
259
+ cluster_id_to_use = _ensure_cluster(db, project_id, existing_article)
260
+
261
+ # (b) LLM peers — existing history or another candidate
262
+ if cluster_id_to_use is None and ref is not None:
263
+ peers = matches.get(ref, [])
264
+ for peer_ref in peers:
265
+ if peer_ref in history_ref_to_id:
266
+ peer = history_ref_to_id[peer_ref]
267
+ cluster_id_to_use = peer.get("cluster_id") or _ensure_cluster(
268
+ db, project_id, peer["id"]
269
+ )
270
+ break
271
+ if cluster_id_to_use is None:
272
+ for peer_ref in peers:
273
+ if peer_ref in by_ref and by_ref[peer_ref].get("cluster_id"):
274
+ cluster_id_to_use = by_ref[peer_ref]["cluster_id"]
275
+ break
276
+
277
+ if cluster_id_to_use:
278
+ cand["cluster_id"] = cluster_id_to_use
279
+
280
+ inserted = db.insert_article(cand)
281
+ if inserted is None:
282
+ stats.duplicates += 1
283
+ continue
284
+
285
+ # If LLM peers exist but pointed at not-yet-inserted candidates,
286
+ # open a fresh cluster around this article so peers can attach.
287
+ if cluster_id_to_use is None and ref is not None and matches.get(ref):
288
+ new_cluster_id = db.create_cluster(project_id, inserted["id"])
289
+ db.attach_to_cluster(inserted["id"], new_cluster_id)
290
+ inserted["cluster_id"] = new_cluster_id
291
+ cand["cluster_id"] = new_cluster_id
292
+
293
+ candidate_to_inserted[id(cand)] = inserted
294
+
295
+ if cluster_id_to_use:
296
+ stats.cluster_joins += 1
297
+ else:
298
+ stats.new_articles += 1
299
+
300
+ return stats
301
+
302
+
303
+ def refresh_project_now(
304
+ project_id: str,
305
+ db: Database,
306
+ judge: BatchJudge,
307
+ dedup_config: DedupConfig,
308
+ feed_timeout_seconds: int,
309
+ ) -> dict[str, int]:
310
+ """Synchronously poll every enabled feed in a project — single LLM call."""
311
+ feeds = [f for f in db.list_feeds(project_id) if f.get("enabled")]
312
+ stats = poll_feeds(project_id, feeds, db, judge, dedup_config, feed_timeout_seconds)
313
+ return stats.as_dict()
314
+
315
+
316
+ def _ensure_cluster(
317
+ db: Database, project_id: str, canonical_article_id: str
318
+ ) -> str:
319
+ """Find or create the cluster for ``canonical_article_id``."""
320
+ article = db.get_article(canonical_article_id)
321
+ if article and article.get("cluster_id"):
322
+ return article["cluster_id"]
323
+ cluster_id = db.create_cluster(project_id, canonical_article_id)
324
+ db.attach_to_cluster(canonical_article_id, cluster_id)
325
+ return cluster_id
@@ -0,0 +1,174 @@
1
+ Metadata-Version: 2.4
2
+ Name: metatron-cli
3
+ Version: 0.2.1
4
+ Summary: Multi-project web feed manager with cross-outlet deduplication
5
+ Author-email: Matt Weber <matt.weber@beefree.io>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/mattweberio/metatron
8
+ Project-URL: Repository, https://github.com/mattweberio/metatron
9
+ Project-URL: Issues, https://github.com/mattweberio/metatron/issues
10
+ Keywords: rss,feeds,deduplication,news,aggregator,web,ingestion
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
17
+ Classifier: Topic :: Text Processing :: Markup
18
+ Requires-Python: >=3.12
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: feedparser>=6.0
22
+ Requires-Dist: requests>=2.31
23
+ Requires-Dist: trafilatura>=1.12
24
+ Requires-Dist: fastapi>=0.110
25
+ Requires-Dist: uvicorn[standard]>=0.27
26
+ Requires-Dist: httpx>=0.27
27
+ Requires-Dist: pydantic>=2.0
28
+ Provides-Extra: test
29
+ Requires-Dist: pytest>=9.0; extra == "test"
30
+ Requires-Dist: pytest-asyncio>=0.23; extra == "test"
31
+ Dynamic: license-file
32
+
33
+ # Metatron
34
+
35
+ Multi-project web feed manager with cross-outlet deduplication.
36
+
37
+ You point Metatron at sources, organized by project. It polls them, dedups
38
+ the articles (URL canonicalization → normalized title → token overlap →
39
+ LLM tiebreaker), and serves you the unique articles via a small HTTP API.
40
+ RSS is the source it speaks today; the architecture is medium-agnostic and
41
+ grows to other web sources over time.
42
+
43
+ It does this one thing. It doesn't curate, it doesn't summarize, it doesn't
44
+ have opinions about what's interesting. That's for whatever calls it.
45
+
46
+ ## Install
47
+
48
+ The CLI is `metatron`. The published package is `metatron-cli` (the bare
49
+ `metatron` name was already taken on PyPI):
50
+
51
+ ```bash
52
+ pipx install metatron-cli # from PyPI
53
+ pipx install git+https://github.com/mattweberio/metatron # from GitHub (latest main)
54
+ ```
55
+
56
+ ## Configure
57
+
58
+ ```bash
59
+ metatron config init # writes ~/.config/metatron/config.toml
60
+ metatron config show # prints the resolved config
61
+ ```
62
+
63
+ Edit `~/.config/metatron/config.toml`. The LLM tiebreaker — needed to catch
64
+ "same story, different outlet" duplicates — requires an Anthropic API key:
65
+
66
+ ```toml
67
+ [llm]
68
+ api_key = "sk-ant-..."
69
+ model = "claude-sonnet-4-6"
70
+
71
+ [api]
72
+ api_token = "your-bearer-token"
73
+ ```
74
+
75
+ Without an LLM key, dedup falls back to URL + normalized-title matching only.
76
+ Cross-outlet duplicates from different sources will leak through.
77
+
78
+ ## Run
79
+
80
+ ```bash
81
+ metatron serve # HTTP API on 127.0.0.1:8765
82
+ ```
83
+
84
+ ## Use
85
+
86
+ Bearer-token auth on everything except `/health`.
87
+
88
+ ```bash
89
+ TOKEN="your-bearer-token"
90
+ BASE="http://127.0.0.1:8765"
91
+
92
+ # Create a project
93
+ curl -s -X POST "$BASE/projects" \
94
+ -H "Authorization: Bearer $TOKEN" \
95
+ -H "Content-Type: application/json" \
96
+ -d '{"name": "ai-news"}'
97
+
98
+ # Add feeds
99
+ PROJECT_ID=...
100
+ curl -s -X POST "$BASE/projects/$PROJECT_ID/feeds" \
101
+ -H "Authorization: Bearer $TOKEN" \
102
+ -H "Content-Type: application/json" \
103
+ -d '{"url": "https://hnrss.org/frontpage"}'
104
+
105
+ # List deduplicated articles
106
+ curl -s -H "Authorization: Bearer $TOKEN" \
107
+ "$BASE/projects/$PROJECT_ID/articles?limit=20"
108
+
109
+ # Force a synchronous refresh
110
+ curl -s -X POST -H "Authorization: Bearer $TOKEN" \
111
+ "$BASE/projects/$PROJECT_ID/refresh"
112
+ ```
113
+
114
+ ## Bulk feed import
115
+
116
+ ```bash
117
+ metatron seed-feeds my-project ./feeds.json
118
+ ```
119
+
120
+ Where `feeds.json` is:
121
+ ```json
122
+ {
123
+ "feeds": [
124
+ { "url": "https://hnrss.org/frontpage", "name": "Hacker News", "category": "tech" },
125
+ { "url": "https://www.theverge.com/rss/index.xml", "name": "The Verge", "category": "tech" }
126
+ ]
127
+ }
128
+ ```
129
+
130
+ ## How dedup works
131
+
132
+ Cheapest-to-most-expensive, short-circuits on first match:
133
+
134
+ 1. **Canonical URL.** Strip `utm_*`, `fbclid`, etc. Same canonical URL → exact duplicate, dropped.
135
+ 2. **Normalized title.** Lowercase, drop punctuation, strip newsroom prefixes ("BREAKING: ", "UPDATE - "). Same → joins the existing article's cluster.
136
+ 3. **Token overlap.** Jaccard similarity on title + summary tokens. ≥0.12 → shortlist candidate (top 5).
137
+ 4. **Sonnet tiebreaker.** Claude Sonnet reads both articles and decides "same story" or "different story." Same → joins cluster.
138
+
139
+ Articles in a cluster are stored individually (you can see the alternative
140
+ sources via `GET /articles/{id}`) but `/projects/{id}/articles` returns
141
+ only the canonical from each cluster.
142
+
143
+ ## Storage
144
+
145
+ Single SQLite file at `~/.local/share/metatron/metatron.db`. WAL mode, FK
146
+ constraints on. Override with `[database].path` in config.
147
+
148
+ ## API endpoints
149
+
150
+ | Method | Path | Description |
151
+ |--------|---------------------------------------|----------------------------------------------|
152
+ | GET | `/health` | Liveness; reports LLM enablement |
153
+ | POST | `/projects` | Create a project |
154
+ | GET | `/projects` | List projects |
155
+ | DELETE | `/projects/{id}` | Delete a project (cascades feeds + articles) |
156
+ | POST | `/projects/{id}/feeds` | Add a feed |
157
+ | GET | `/projects/{id}/feeds` | List feeds |
158
+ | DELETE | `/feeds/{id}` | Remove a feed |
159
+ | GET | `/projects/{id}/articles?since=&limit=` | List deduped articles |
160
+ | POST | `/projects/{id}/refresh` | Synchronously poll project's feeds |
161
+ | GET | `/articles/{id}` | Article detail + cluster members |
162
+
163
+ ## Development
164
+
165
+ ```bash
166
+ python3 -m pytest -q
167
+ ```
168
+
169
+ Tests cover normalization, the DB layer, the dedup pipeline (with a fake
170
+ LLM judge), the feed poller (with mocked RSS), and the API surface.
171
+
172
+ ## License
173
+
174
+ MIT. See [LICENSE](./LICENSE).
@@ -0,0 +1,16 @@
1
+ metatron/__init__.py,sha256=-akQUfUSR1OTezvy17U2ypfTqqiNxkesN0sp05kolQI,1108
2
+ metatron/api.py,sha256=86WTtuJk2BezY0A-BFWySfiuGMwETuLs0DPFJU-3XrY,10514
3
+ metatron/cli.py,sha256=K-356dfShXvuAKD20AOs4YCNAHvGfjTX6MNUtJvW03M,6920
4
+ metatron/config.py,sha256=Vm-ChMODh3k7TvGm4GTRnVhXe1FQs_rFcSQvhYW2mKs,6232
5
+ metatron/db.py,sha256=zmg8-ae95G0OOXWyJBPV1SCczXXKPJuOCcfrBJd5tE8,13377
6
+ metatron/dedup.py,sha256=E3aRa251CGCxNzsgSI8DYCd8y8gxApqtgIbM0Gijsck,7263
7
+ metatron/fetcher.py,sha256=HB9Am9O5SC8m2r7QHrZLkopu0uCih_wY23Jj3uSWoqc,4446
8
+ metatron/llm.py,sha256=D7wmpT6EIWmtJZFlr6TI0dt0tG3c0aDQAXszZpjwLLQ,8800
9
+ metatron/normalize.py,sha256=x0maVuzV2z2-hy-dZ1sjUh3HyuO3vni6vTIwcnQaJPQ,3787
10
+ metatron/poller.py,sha256=veTDqqRUdIFrNIGh57TrM-sL-U_2EkBOd8YGkO3bagg,12067
11
+ metatron_cli-0.2.1.dist-info/licenses/LICENSE,sha256=at8Cx79y_PI4PKS9hCrOYeN8XMhuUigTcFET3fcCWfU,1067
12
+ metatron_cli-0.2.1.dist-info/METADATA,sha256=DvYSqGpC6f68ZdaiOan7VT9ttsJCskSPJF0pL7aE-2w,6129
13
+ metatron_cli-0.2.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
14
+ metatron_cli-0.2.1.dist-info/entry_points.txt,sha256=tj_pczFcDHzV9UAsphXx2-APV4iyvhWH9xJBQx06KTM,47
15
+ metatron_cli-0.2.1.dist-info/top_level.txt,sha256=xuqvlh66zVOuc8rmX0NCSGi9HnFxMadj0aNJ1jgxYeA,9
16
+ metatron_cli-0.2.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (82.0.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ metatron = metatron.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Matt Weber
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ metatron