metatron-cli 0.2.1__py3-none-any.whl

metatron/config.py

@@ -0,0 +1,187 @@
+"""Metatron configuration — loaded from ~/.config/metatron/config.toml.
+
+XDG-style config location, stdlib tomllib parsing, no .env files, no required
+fields. Sensible defaults so the service runs out of the box; configure the
+[llm] section to enable Sonnet-powered cross-outlet dedup, and the [api]
+section to set a bearer token.
+
+Resolution order for the config file path:
+  1. Explicit path passed to ``from_file``
+  2. ``METATRON_CONFIG`` env var (operator override only)
+  3. ``$XDG_CONFIG_HOME/metatron/config.toml``
+  4. ``~/.config/metatron/config.toml``
+"""
+
+from __future__ import annotations
+
+import os
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+class ConfigError(Exception):
+    """Raised when the config file is malformed or unreadable."""
+
+
+@dataclass(frozen=True)
+class ApiConfig:
+    """HTTP API settings."""
+
+    host: str = "127.0.0.1"
+    port: int = 8765
+    api_token: str = ""
+
+
+@dataclass(frozen=True)
+class DatabaseConfig:
+    """SQLite path override."""
+
+    path: str = ""
+
+
+@dataclass(frozen=True)
+class LlmConfig:
+    """Claude CLI wrapper for the dedup tiebreaker.
+
+    Shells out to the locally-installed `claude` binary, which uses your
+    Claude subscription. No API key is needed — the CLI handles auth on
+    your behalf. ``enabled`` here flips on the tiebreaker entirely; ``model``
+    is passed as ``claude -p --model <model>``.
+    """
+
+    enabled: bool = True
+    model: str = "sonnet"
+    binary: str = "claude"
+    # Kill the CLI subprocess only if stdout/stderr have been silent for
+    # this many seconds. A productive call is allowed to run as long as
+    # it needs — wall-clock timeouts waste tokens on perfectly fine calls.
+    idle_timeout_seconds: float = 120.0
+
+
+@dataclass(frozen=True)
+class PollerConfig:
+    """Background poller settings."""
+
+    enabled: bool = True
+    tick_seconds: int = 60  # how often the loop checks for due feeds
+    default_feed_interval_seconds: int = 1800
+    feed_timeout_seconds: int = 30
+
+
+@dataclass(frozen=True)
+class MetatronConfig:
+    """Immutable configuration loaded from TOML."""
+
+    api: ApiConfig = field(default_factory=ApiConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    llm: LlmConfig = field(default_factory=LlmConfig)
+    poller: PollerConfig = field(default_factory=PollerConfig)
+    log_dir: Path = field(default_factory=lambda: Path("logs"))
+
+    @staticmethod
+    def default_config_path() -> Path:
+        xdg = os.environ.get("XDG_CONFIG_HOME")
+        base = Path(xdg) if xdg else Path.home() / ".config"
+        return base / "metatron" / "config.toml"
+
+    @classmethod
+    def from_file(cls, path: Path | None = None) -> MetatronConfig:
+        """Load config from TOML. If path is None, follow the resolution order.
+
+        Missing file is OK — returns an all-defaults config. Malformed TOML
+        raises ConfigError.
+        """
+        if path is None:
+            override = os.environ.get("METATRON_CONFIG")
+            path = Path(override) if override else cls.default_config_path()
+
+        if not path.exists():
+            return cls()
+
+        try:
+            with path.open("rb") as fh:
+                data = tomllib.load(fh)
+        except (OSError, tomllib.TOMLDecodeError) as e:
+            raise ConfigError(f"Failed to read {path}: {e}") from e
+
+        return cls._from_dict(data)
+
+    @classmethod
+    def _from_dict(cls, data: dict) -> MetatronConfig:
+        api_section = data.get("api", {}) or {}
+        db_section = data.get("database", {}) or {}
+        llm_section = data.get("llm", {}) or {}
+        poller_section = data.get("poller", {}) or {}
+        logging_section = data.get("logging", {}) or {}
+
+        return cls(
+            api=ApiConfig(
+                host=str(api_section.get("host", "127.0.0.1")),
+                port=int(api_section.get("port", 8765)),
+                api_token=str(api_section.get("api_token", "")),
+            ),
+            database=DatabaseConfig(path=str(db_section.get("path", ""))),
+            llm=LlmConfig(
+                enabled=bool(llm_section.get("enabled", True)),
+                model=str(llm_section.get("model", "sonnet")),
+                binary=str(llm_section.get("binary", "claude")),
+                idle_timeout_seconds=float(
+                    llm_section.get("idle_timeout_seconds", 120.0)
+                ),
+            ),
+            poller=PollerConfig(
+                enabled=bool(poller_section.get("enabled", True)),
+                tick_seconds=int(poller_section.get("tick_seconds", 60)),
+                default_feed_interval_seconds=int(
+                    poller_section.get("default_feed_interval_seconds", 1800)
+                ),
+                feed_timeout_seconds=int(
+                    poller_section.get("feed_timeout_seconds", 30)
+                ),
+            ),
+            log_dir=Path(str(logging_section.get("dir", "logs"))),
+        )
+
+
+DEFAULT_CONFIG_TEMPLATE = """\
+# Metatron configuration
+# Location: ~/.config/metatron/config.toml
+# Override with METATRON_CONFIG=/path/to/config.toml
+
+[api]
+# HTTP server bind. Default 127.0.0.1 (localhost-only).
+host = "127.0.0.1"
+port = 8765
+# Bearer token clients must present. Leave empty to disable auth (dev only).
+api_token = ""
+
+[database]
+# Override the default SQLite path. Empty = ~/.local/share/metatron/metatron.db
+path = ""
+
+[llm]
+# Cross-outlet dedup tiebreaker via the local `claude` CLI (Claude Code).
+# Uses your Claude subscription — no API key needed. Set enabled = false
+# to skip the tiebreaker entirely (cheaper, but cross-outlet duplicates
+# with different headlines will leak through).
+enabled = true
+model = "sonnet"
+binary = "claude"
+# Kill the CLI subprocess only if stdout has been silent this long.
+# A productive call is never killed on wall-clock alone.
+idle_timeout_seconds = 120.0
+
+[poller]
+# Background polling. Disable to fetch only on-demand via /refresh.
+enabled = true
+# How often the loop wakes up to check for due feeds.
+tick_seconds = 60
+# Default polling interval per feed (overridable per feed).
+default_feed_interval_seconds = 1800
+# Per-feed HTTP timeout when polling.
+feed_timeout_seconds = 30
+
+[logging]
+dir = "logs"
+"""

metatron/db.py

@@ -0,0 +1,357 @@
+"""SQLite schema and helpers for Metatron.
+
+Single-file database at ~/.local/share/metatron/metatron.db (XDG-state by
+default; override with [database].path in config.toml).
+
+Schema:
+  projects(id, name, created_at)
+  feeds(id, project_id, url, name, category, enabled, poll_interval_seconds, last_polled, last_error)
+  articles(id, project_id, canonical_url, source_url, source, title, summary, body, published, fetched_at, cluster_id)
+  clusters(id, project_id, canonical_article_id, created_at)
+
+A "cluster" is a deduplicated group of articles all covering the same story
+across outlets. The canonical_article_id points at the first article in the
+cluster (the one we keep showing). Members of a cluster have cluster_id set
+to that cluster's id.
+"""
+
+from __future__ import annotations
+
+import os
+import secrets
+import sqlite3
+import threading
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterator
+
+
+def default_db_path() -> Path:
+    xdg = os.environ.get("XDG_STATE_HOME") or os.environ.get("XDG_DATA_HOME")
+    base = Path(xdg) if xdg else Path.home() / ".local" / "share"
+    return base / "metatron" / "metatron.db"
+
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS projects (
+    id          TEXT PRIMARY KEY,
+    name        TEXT NOT NULL UNIQUE,
+    created_at  TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS feeds (
+    id                    TEXT PRIMARY KEY,
+    project_id            TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+    url                   TEXT NOT NULL,
+    name                  TEXT NOT NULL,
+    category              TEXT NOT NULL DEFAULT '',
+    enabled               INTEGER NOT NULL DEFAULT 1,
+    poll_interval_seconds INTEGER NOT NULL DEFAULT 1800,
+    last_polled           TEXT,
+    last_error            TEXT,
+    UNIQUE(project_id, url)
+);
+
+CREATE TABLE IF NOT EXISTS clusters (
+    id                    TEXT PRIMARY KEY,
+    project_id            TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+    canonical_article_id  TEXT NOT NULL,
+    created_at            TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS articles (
+    id             TEXT PRIMARY KEY,
+    project_id     TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+    canonical_url  TEXT NOT NULL,
+    source_url     TEXT NOT NULL,
+    source         TEXT NOT NULL,
+    title          TEXT NOT NULL,
+    summary        TEXT NOT NULL DEFAULT '',
+    body           TEXT NOT NULL DEFAULT '',
+    published      TEXT,
+    fetched_at     TEXT NOT NULL,
+    cluster_id     TEXT REFERENCES clusters(id) ON DELETE SET NULL,
+    UNIQUE(project_id, canonical_url)
+);
+
+CREATE INDEX IF NOT EXISTS idx_articles_project_fetched ON articles(project_id, fetched_at);
+CREATE INDEX IF NOT EXISTS idx_articles_cluster        ON articles(cluster_id);
+CREATE INDEX IF NOT EXISTS idx_feeds_project_enabled   ON feeds(project_id, enabled);
+"""
+
+
+class Database:
+    """Thin wrapper around sqlite3 with thread-local connections.
+
+    SQLite handles concurrent reads fine. Writes happen from the API thread
+    and the poller thread; we use SQLite's WAL mode and a brief busy timeout
+    to handle contention.
+    """
+
+    def __init__(self, path: Path | None = None) -> None:
+        self.path = path or default_db_path()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._local = threading.local()
+        with self.connect() as conn:
+            conn.executescript(_SCHEMA)
+            conn.execute("PRAGMA journal_mode = WAL")
+            conn.execute("PRAGMA foreign_keys = ON")
+
+    @contextmanager
+    def connect(self) -> Iterator[sqlite3.Connection]:
+        conn = sqlite3.connect(self.path, timeout=10.0)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA foreign_keys = ON")
+        try:
+            yield conn
+            conn.commit()
+        finally:
+            conn.close()
+
+    # ── projects ─────────────────────────────────────────────────────────
+    def create_project(self, name: str) -> dict[str, Any]:
+        pid = _new_id()
+        now = _now_iso()
+        with self.connect() as conn:
+            conn.execute(
+                "INSERT INTO projects (id, name, created_at) VALUES (?, ?, ?)",
+                (pid, name, now),
+            )
+        return {"id": pid, "name": name, "created_at": now}
+
+    def list_projects(self) -> list[dict[str, Any]]:
+        with self.connect() as conn:
+            rows = conn.execute(
+                "SELECT id, name, created_at FROM projects ORDER BY created_at"
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    def get_project(self, project_id: str) -> dict[str, Any] | None:
+        with self.connect() as conn:
+            row = conn.execute(
+                "SELECT id, name, created_at FROM projects WHERE id = ?",
+                (project_id,),
+            ).fetchone()
+        return dict(row) if row else None
+
+    def delete_project(self, project_id: str) -> bool:
+        with self.connect() as conn:
+            cur = conn.execute("DELETE FROM projects WHERE id = ?", (project_id,))
+        return cur.rowcount > 0
+
+    # ── feeds ────────────────────────────────────────────────────────────
+    def add_feed(
+        self,
+        project_id: str,
+        url: str,
+        name: str,
+        category: str = "",
+        poll_interval_seconds: int = 1800,
+    ) -> dict[str, Any]:
+        fid = _new_id()
+        with self.connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO feeds
+                (id, project_id, url, name, category, enabled, poll_interval_seconds)
+                VALUES (?, ?, ?, ?, ?, 1, ?)
+                """,
+                (fid, project_id, url, name, category, poll_interval_seconds),
+            )
+        return self.get_feed(fid)  # type: ignore[return-value]
+
+    def get_feed(self, feed_id: str) -> dict[str, Any] | None:
+        with self.connect() as conn:
+            row = conn.execute(
+                "SELECT * FROM feeds WHERE id = ?", (feed_id,)
+            ).fetchone()
+        return dict(row) if row else None
+
+    def list_feeds(self, project_id: str) -> list[dict[str, Any]]:
+        with self.connect() as conn:
+            rows = conn.execute(
+                "SELECT * FROM feeds WHERE project_id = ? ORDER BY name",
+                (project_id,),
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    def list_due_feeds(self, now: datetime | None = None) -> list[dict[str, Any]]:
+        """Feeds whose last_polled + poll_interval_seconds is in the past."""
+        now = now or datetime.now(timezone.utc)
+        now_iso = now.isoformat()
+        with self.connect() as conn:
+            rows = conn.execute(
+                """
+                SELECT * FROM feeds
+                 WHERE enabled = 1
+                   AND (last_polled IS NULL
+                        OR datetime(last_polled, '+' || poll_interval_seconds || ' seconds')
+                           <= datetime(?))
+                """,
+                (now_iso,),
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    def delete_feed(self, feed_id: str) -> bool:
+        with self.connect() as conn:
+            cur = conn.execute("DELETE FROM feeds WHERE id = ?", (feed_id,))
+        return cur.rowcount > 0
+
+    def mark_feed_polled(
+        self, feed_id: str, error: str | None = None
+    ) -> None:
+        with self.connect() as conn:
+            conn.execute(
+                "UPDATE feeds SET last_polled = ?, last_error = ? WHERE id = ?",
+                (_now_iso(), error, feed_id),
+            )
+
+    # ── articles + clusters ──────────────────────────────────────────────
+    def get_article_by_canonical_url(
+        self, project_id: str, canonical_url: str
+    ) -> dict[str, Any] | None:
+        with self.connect() as conn:
+            row = conn.execute(
+                "SELECT * FROM articles WHERE project_id = ? AND canonical_url = ?",
+                (project_id, canonical_url),
+            ).fetchone()
+        return dict(row) if row else None
+
+    def recent_articles_for_dedup(
+        self, project_id: str, days: int = 7
+    ) -> list[dict[str, Any]]:
+        with self.connect() as conn:
+            rows = conn.execute(
+                """
+                SELECT id, title, summary, body, canonical_url, cluster_id
+                  FROM articles
+                 WHERE project_id = ?
+                   AND fetched_at >= datetime('now', ?)
+                """,
+                (project_id, f"-{days} days"),
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    def insert_article(self, article: dict[str, Any]) -> dict[str, Any] | None:
+        """Insert one article. Returns the inserted row, or ``None`` if a
+        row with this ``(project_id, canonical_url)`` already exists. The
+        race-safe behavior is important: both the background poller and
+        an on-demand /refresh may attempt to insert the same item.
+        """
+        aid = article.get("id") or _new_id()
+        article["id"] = aid
+        with self.connect() as conn:
+            cur = conn.execute(
+                """
+                INSERT INTO articles
+                (id, project_id, canonical_url, source_url, source, title,
+                 summary, body, published, fetched_at, cluster_id)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(project_id, canonical_url) DO NOTHING
+                """,
+                (
+                    aid,
+                    article["project_id"],
+                    article["canonical_url"],
+                    article["source_url"],
+                    article["source"],
+                    article["title"],
+                    article.get("summary", ""),
+                    article.get("body", ""),
+                    article.get("published"),
+                    article.get("fetched_at") or _now_iso(),
+                    article.get("cluster_id"),
+                ),
+            )
+            if cur.rowcount == 0:
+                return None
+        return self.get_article(aid)
+
+    def get_article(self, article_id: str) -> dict[str, Any] | None:
+        with self.connect() as conn:
+            row = conn.execute(
+                "SELECT * FROM articles WHERE id = ?", (article_id,)
+            ).fetchone()
+        return dict(row) if row else None
+
+    def list_articles(
+        self,
+        project_id: str,
+        since: str | None = None,
+        limit: int = 100,
+        deduped: bool = True,
+    ) -> list[dict[str, Any]]:
+        """List articles for a project, newest first.
+
+        When ``deduped`` is True (default), returns only the canonical
+        article from each cluster (or articles with no cluster, which means
+        they are themselves canonical).
+        """
+        params: list[Any] = [project_id]
+        where = "WHERE a.project_id = ?"
+        if since:
+            where += " AND a.fetched_at >= ?"
+            params.append(since)
+        if deduped:
+            # Either no cluster (singleton — itself canonical) OR the row IS
+            # the canonical of its cluster.
+            where += (
+                " AND (a.cluster_id IS NULL OR a.id = ("
+                "  SELECT canonical_article_id FROM clusters WHERE id = a.cluster_id"
+                "))"
+            )
+        params.append(limit)
+        with self.connect() as conn:
+            rows = conn.execute(
+                f"""
+                SELECT a.* FROM articles a
+                 {where}
+                 ORDER BY datetime(COALESCE(a.published, a.fetched_at)) DESC
+                 LIMIT ?
+                """,
+                params,
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+    def create_cluster(
+        self, project_id: str, canonical_article_id: str
+    ) -> str:
+        cid = _new_id()
+        with self.connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO clusters (id, project_id, canonical_article_id, created_at)
+                VALUES (?, ?, ?, ?)
+                """,
+                (cid, project_id, canonical_article_id, _now_iso()),
+            )
+        return cid
+
+    def attach_to_cluster(self, article_id: str, cluster_id: str) -> None:
+        with self.connect() as conn:
+            conn.execute(
+                "UPDATE articles SET cluster_id = ? WHERE id = ?",
+                (cluster_id, article_id),
+            )
+
+    def article_cluster_members(self, cluster_id: str) -> list[dict[str, Any]]:
+        with self.connect() as conn:
+            rows = conn.execute(
+                """
+                SELECT id, source, title, source_url
+                  FROM articles
+                 WHERE cluster_id = ?
+                 ORDER BY fetched_at
+                """,
+                (cluster_id,),
+            ).fetchall()
+        return [dict(r) for r in rows]
+
+
+def _new_id() -> str:
+    return secrets.token_urlsafe(12)
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()