metatron-cli 0.2.1__tar.gz

metatron_cli-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matt Weber
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

metatron_cli-0.2.1/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: metatron-cli
+Version: 0.2.1
+Summary: Multi-project web feed manager with cross-outlet deduplication
+Author-email: Matt Weber <matt.weber@beefree.io>
+License: MIT
+Project-URL: Homepage, https://github.com/mattweberio/metatron
+Project-URL: Repository, https://github.com/mattweberio/metatron
+Project-URL: Issues, https://github.com/mattweberio/metatron/issues
+Keywords: rss,feeds,deduplication,news,aggregator,web,ingestion
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Text Processing :: Markup
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: feedparser>=6.0
+Requires-Dist: requests>=2.31
+Requires-Dist: trafilatura>=1.12
+Requires-Dist: fastapi>=0.110
+Requires-Dist: uvicorn[standard]>=0.27
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.0
+Provides-Extra: test
+Requires-Dist: pytest>=9.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.23; extra == "test"
+Dynamic: license-file
+
+# Metatron
+
+Multi-project web feed manager with cross-outlet deduplication.
+
+You point Metatron at sources, organized by project. It polls them, dedups
+the articles (URL canonicalization → normalized title → token overlap →
+LLM tiebreaker), and serves you the unique articles via a small HTTP API.
+RSS is the source it speaks today; the architecture is medium-agnostic and
+grows to other web sources over time.
+
+It does this one thing. It doesn't curate, it doesn't summarize, it doesn't
+have opinions about what's interesting. That's for whatever calls it.
+
+## Install
+
+The CLI is `metatron`. The published package is `metatron-cli` (the bare
+`metatron` name was already taken on PyPI):
+
+```bash
+pipx install metatron-cli                                   # from PyPI
+pipx install git+https://github.com/mattweberio/metatron    # from GitHub (latest main)
+```
+
+## Configure
+
+```bash
+metatron config init       # writes ~/.config/metatron/config.toml
+metatron config show       # prints the resolved config
+```
+
+Edit `~/.config/metatron/config.toml`. The LLM tiebreaker — needed to catch
+"same story, different outlet" duplicates — requires an Anthropic API key:
+
+```toml
+[llm]
+api_key = "sk-ant-..."
+model = "claude-sonnet-4-6"
+
+[api]
+api_token = "your-bearer-token"
+```
+
+Without an LLM key, dedup falls back to URL + normalized-title matching only.
+Cross-outlet duplicates from different sources will leak through.
+
+## Run
+
+```bash
+metatron serve             # HTTP API on 127.0.0.1:8765
+```
+
+## Use
+
+Bearer-token auth on everything except `/health`.
+
+```bash
+TOKEN="your-bearer-token"
+BASE="http://127.0.0.1:8765"
+
+# Create a project
+curl -s -X POST "$BASE/projects" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"name": "ai-news"}'
+
+# Add feeds
+PROJECT_ID=...
+curl -s -X POST "$BASE/projects/$PROJECT_ID/feeds" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"url": "https://hnrss.org/frontpage"}'
+
+# List deduplicated articles
+curl -s -H "Authorization: Bearer $TOKEN" \
+  "$BASE/projects/$PROJECT_ID/articles?limit=20"
+
+# Force a synchronous refresh
+curl -s -X POST -H "Authorization: Bearer $TOKEN" \
+  "$BASE/projects/$PROJECT_ID/refresh"
+```
+
+## Bulk feed import
+
+```bash
+metatron seed-feeds my-project ./feeds.json
+```
+
+Where `feeds.json` is:
+```json
+{
+  "feeds": [
+    { "url": "https://hnrss.org/frontpage", "name": "Hacker News", "category": "tech" },
+    { "url": "https://www.theverge.com/rss/index.xml", "name": "The Verge", "category": "tech" }
+  ]
+}
+```
+
+## How dedup works
+
+Cheapest-to-most-expensive, short-circuits on first match:
+
+1. **Canonical URL.** Strip `utm_*`, `fbclid`, etc. Same canonical URL → exact duplicate, dropped.
+2. **Normalized title.** Lowercase, drop punctuation, strip newsroom prefixes ("BREAKING: ", "UPDATE - "). Same → joins the existing article's cluster.
+3. **Token overlap.** Jaccard similarity on title + summary tokens. ≥0.12 → shortlist candidate (top 5).
+4. **Sonnet tiebreaker.** Claude Sonnet reads both articles and decides "same story" or "different story." Same → joins cluster.
+
+Articles in a cluster are stored individually (you can see the alternative
+sources via `GET /articles/{id}`) but `/projects/{id}/articles` returns
+only the canonical from each cluster.
+
+## Storage
+
+Single SQLite file at `~/.local/share/metatron/metatron.db`. WAL mode, FK
+constraints on. Override with `[database].path` in config.
+
+## API endpoints
+
+| Method | Path                                  | Description                                  |
+|--------|---------------------------------------|----------------------------------------------|
+| GET    | `/health`                             | Liveness; reports LLM enablement             |
+| POST   | `/projects`                           | Create a project                             |
+| GET    | `/projects`                           | List projects                                |
+| DELETE | `/projects/{id}`                      | Delete a project (cascades feeds + articles) |
+| POST   | `/projects/{id}/feeds`                | Add a feed                                   |
+| GET    | `/projects/{id}/feeds`                | List feeds                                   |
+| DELETE | `/feeds/{id}`                         | Remove a feed                                |
+| GET    | `/projects/{id}/articles?since=&limit=` | List deduped articles                      |
+| POST   | `/projects/{id}/refresh`              | Synchronously poll project's feeds           |
+| GET    | `/articles/{id}`                      | Article detail + cluster members             |
+
+## Development
+
+```bash
+python3 -m pytest -q
+```
+
+Tests cover normalization, the DB layer, the dedup pipeline (with a fake
+LLM judge), the feed poller (with mocked RSS), and the API surface.
+
+## License
+
+MIT. See [LICENSE](./LICENSE).

metatron_cli-0.2.1/README.md

@@ -0,0 +1,142 @@
+# Metatron
+
+Multi-project web feed manager with cross-outlet deduplication.
+
+You point Metatron at sources, organized by project. It polls them, dedups
+the articles (URL canonicalization → normalized title → token overlap →
+LLM tiebreaker), and serves you the unique articles via a small HTTP API.
+RSS is the source it speaks today; the architecture is medium-agnostic and
+grows to other web sources over time.
+
+It does this one thing. It doesn't curate, it doesn't summarize, it doesn't
+have opinions about what's interesting. That's for whatever calls it.
+
+## Install
+
+The CLI is `metatron`. The published package is `metatron-cli` (the bare
+`metatron` name was already taken on PyPI):
+
+```bash
+pipx install metatron-cli                                   # from PyPI
+pipx install git+https://github.com/mattweberio/metatron    # from GitHub (latest main)
+```
+
+## Configure
+
+```bash
+metatron config init       # writes ~/.config/metatron/config.toml
+metatron config show       # prints the resolved config
+```
+
+Edit `~/.config/metatron/config.toml`. The LLM tiebreaker — needed to catch
+"same story, different outlet" duplicates — requires an Anthropic API key:
+
+```toml
+[llm]
+api_key = "sk-ant-..."
+model = "claude-sonnet-4-6"
+
+[api]
+api_token = "your-bearer-token"
+```
+
+Without an LLM key, dedup falls back to URL + normalized-title matching only.
+Cross-outlet duplicates from different sources will leak through.
+
+## Run
+
+```bash
+metatron serve             # HTTP API on 127.0.0.1:8765
+```
+
+## Use
+
+Bearer-token auth on everything except `/health`.
+
+```bash
+TOKEN="your-bearer-token"
+BASE="http://127.0.0.1:8765"
+
+# Create a project
+curl -s -X POST "$BASE/projects" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"name": "ai-news"}'
+
+# Add feeds
+PROJECT_ID=...
+curl -s -X POST "$BASE/projects/$PROJECT_ID/feeds" \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"url": "https://hnrss.org/frontpage"}'
+
+# List deduplicated articles
+curl -s -H "Authorization: Bearer $TOKEN" \
+  "$BASE/projects/$PROJECT_ID/articles?limit=20"
+
+# Force a synchronous refresh
+curl -s -X POST -H "Authorization: Bearer $TOKEN" \
+  "$BASE/projects/$PROJECT_ID/refresh"
+```
+
+## Bulk feed import
+
+```bash
+metatron seed-feeds my-project ./feeds.json
+```
+
+Where `feeds.json` is:
+```json
+{
+  "feeds": [
+    { "url": "https://hnrss.org/frontpage", "name": "Hacker News", "category": "tech" },
+    { "url": "https://www.theverge.com/rss/index.xml", "name": "The Verge", "category": "tech" }
+  ]
+}
+```
+
+## How dedup works
+
+Cheapest-to-most-expensive, short-circuits on first match:
+
+1. **Canonical URL.** Strip `utm_*`, `fbclid`, etc. Same canonical URL → exact duplicate, dropped.
+2. **Normalized title.** Lowercase, drop punctuation, strip newsroom prefixes ("BREAKING: ", "UPDATE - "). Same → joins the existing article's cluster.
+3. **Token overlap.** Jaccard similarity on title + summary tokens. ≥0.12 → shortlist candidate (top 5).
+4. **Sonnet tiebreaker.** Claude Sonnet reads both articles and decides "same story" or "different story." Same → joins cluster.
+
+Articles in a cluster are stored individually (you can see the alternative
+sources via `GET /articles/{id}`) but `/projects/{id}/articles` returns
+only the canonical from each cluster.
+
+## Storage
+
+Single SQLite file at `~/.local/share/metatron/metatron.db`. WAL mode, FK
+constraints on. Override with `[database].path` in config.
+
+## API endpoints
+
+| Method | Path                                  | Description                                  |
+|--------|---------------------------------------|----------------------------------------------|
+| GET    | `/health`                             | Liveness; reports LLM enablement             |
+| POST   | `/projects`                           | Create a project                             |
+| GET    | `/projects`                           | List projects                                |
+| DELETE | `/projects/{id}`                      | Delete a project (cascades feeds + articles) |
+| POST   | `/projects/{id}/feeds`                | Add a feed                                   |
+| GET    | `/projects/{id}/feeds`                | List feeds                                   |
+| DELETE | `/feeds/{id}`                         | Remove a feed                                |
+| GET    | `/projects/{id}/articles?since=&limit=` | List deduped articles                      |
+| POST   | `/projects/{id}/refresh`              | Synchronously poll project's feeds           |
+| GET    | `/articles/{id}`                      | Article detail + cluster members             |
+
+## Development
+
+```bash
+python3 -m pytest -q
+```
+
+Tests cover normalization, the DB layer, the dedup pipeline (with a fake
+LLM judge), the feed poller (with mocked RSS), and the API surface.
+
+## License
+
+MIT. See [LICENSE](./LICENSE).

metatron_cli-0.2.1/pyproject.toml

@@ -0,0 +1,53 @@
+[project]
+name = "metatron-cli"
+version = "0.2.1"
+description = "Multi-project web feed manager with cross-outlet deduplication"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "MIT" }
+authors = [{ name = "Matt Weber", email = "matt.weber@beefree.io" }]
+keywords = ["rss", "feeds", "deduplication", "news", "aggregator", "web", "ingestion"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Text Processing :: Markup",
+]
+
+dependencies = [
+    "feedparser>=6.0",
+    "requests>=2.31",
+    "trafilatura>=1.12",
+    "fastapi>=0.110",
+    "uvicorn[standard]>=0.27",
+    "httpx>=0.27",
+    "pydantic>=2.0",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest>=9.0",
+    "pytest-asyncio>=0.23",
+]
+
+[project.scripts]
+metatron = "metatron.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/mattweberio/metatron"
+Repository = "https://github.com/mattweberio/metatron"
+Issues = "https://github.com/mattweberio/metatron/issues"
+
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"

metatron_cli-0.2.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

metatron_cli-0.2.1/src/metatron/__init__.py

@@ -0,0 +1,51 @@
+"""Metatron — multi-project RSS feed manager with cross-outlet deduplication."""
+
+__version__ = "0.2.0"
+
+from metatron.config import (
+    ApiConfig,
+    ConfigError,
+    DatabaseConfig,
+    LlmConfig,
+    MetatronConfig,
+    PollerConfig,
+)
+from metatron.db import Database
+from metatron.dedup import (
+    BatchPlan,
+    CheapDecision,
+    DedupConfig,
+    build_batch_plan,
+    cheap_decide,
+    run_batch,
+)
+from metatron.llm import BatchJudge, ClusterGroup, ClusterItem
+from metatron.normalize import canonicalize_url, jaccard, normalize_title, tokenize
+from metatron.poller import Poller, poll_feeds, refresh_project_now
+
+__all__ = [
+    "__version__",
+    "ApiConfig",
+    "BatchJudge",
+    "BatchPlan",
+    "CheapDecision",
+    "ClusterGroup",
+    "ClusterItem",
+    "ConfigError",
+    "Database",
+    "DatabaseConfig",
+    "DedupConfig",
+    "LlmConfig",
+    "MetatronConfig",
+    "Poller",
+    "PollerConfig",
+    "build_batch_plan",
+    "canonicalize_url",
+    "cheap_decide",
+    "jaccard",
+    "normalize_title",
+    "poll_feeds",
+    "refresh_project_now",
+    "run_batch",
+    "tokenize",
+]