metatron-cli 0.2.1__py3-none-any.whl

metatron/__init__.py

@@ -0,0 +1,51 @@
+"""Metatron — multi-project RSS feed manager with cross-outlet deduplication."""
+
+__version__ = "0.2.0"
+
+from metatron.config import (
+    ApiConfig,
+    ConfigError,
+    DatabaseConfig,
+    LlmConfig,
+    MetatronConfig,
+    PollerConfig,
+)
+from metatron.db import Database
+from metatron.dedup import (
+    BatchPlan,
+    CheapDecision,
+    DedupConfig,
+    build_batch_plan,
+    cheap_decide,
+    run_batch,
+)
+from metatron.llm import BatchJudge, ClusterGroup, ClusterItem
+from metatron.normalize import canonicalize_url, jaccard, normalize_title, tokenize
+from metatron.poller import Poller, poll_feeds, refresh_project_now
+
+__all__ = [
+    "__version__",
+    "ApiConfig",
+    "BatchJudge",
+    "BatchPlan",
+    "CheapDecision",
+    "ClusterGroup",
+    "ClusterItem",
+    "ConfigError",
+    "Database",
+    "DatabaseConfig",
+    "DedupConfig",
+    "LlmConfig",
+    "MetatronConfig",
+    "Poller",
+    "PollerConfig",
+    "build_batch_plan",
+    "canonicalize_url",
+    "cheap_decide",
+    "jaccard",
+    "normalize_title",
+    "poll_feeds",
+    "refresh_project_now",
+    "run_batch",
+    "tokenize",
+]

metatron/api.py

@@ -0,0 +1,290 @@
+"""FastAPI HTTP surface for Metatron.
+
+One purpose: manage feeds, expose deduplicated articles. No curation, no
+personality, no opinions about what's interesting — that's for the caller.
+
+Endpoints:
+  POST   /projects                       — create a project
+  GET    /projects                       — list projects
+  DELETE /projects/{project_id}          — delete a project (cascades)
+  POST   /projects/{project_id}/feeds    — add a feed to a project
+  GET    /projects/{project_id}/feeds    — list feeds for a project
+  DELETE /feeds/{feed_id}                — remove a feed
+  GET    /projects/{project_id}/articles — list deduped articles (?since=, ?limit=)
+  GET    /articles/{article_id}          — fetch one article with body + cluster members
+  POST   /projects/{project_id}/refresh  — force a synchronous poll of this project's feeds
+  GET    /health                         — liveness probe (no auth)
+
+Authentication: bearer token from config [api].api_token. When the token is
+empty, the API is open (dev convenience; configure a token before exposing).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import Depends, FastAPI, Header, HTTPException, Path as PathParam, Query
+from pydantic import BaseModel, Field
+
+from metatron.config import MetatronConfig
+from metatron.db import Database
+from metatron.dedup import DedupConfig
+from metatron.llm import BatchJudge
+from metatron.poller import Poller, refresh_project_now
+
+logger = logging.getLogger("metatron.api")
+
+
+# ── request / response shapes ────────────────────────────────────────────
+
+
+class CreateProjectIn(BaseModel):
+    name: str = Field(..., min_length=1, max_length=200)
+
+
+class ProjectOut(BaseModel):
+    id: str
+    name: str
+    created_at: str
+
+
+class AddFeedIn(BaseModel):
+    url: str = Field(..., min_length=1)
+    name: str | None = None
+    category: str = ""
+    poll_interval_seconds: int = 1800
+
+
+class FeedOut(BaseModel):
+    id: str
+    project_id: str
+    url: str
+    name: str
+    category: str
+    enabled: bool
+    poll_interval_seconds: int
+    last_polled: str | None
+    last_error: str | None
+
+
+class ArticleOut(BaseModel):
+    id: str
+    project_id: str
+    canonical_url: str
+    source_url: str
+    source: str
+    title: str
+    summary: str
+    published: str | None
+    fetched_at: str
+    cluster_id: str | None
+
+
+class ArticleDetailOut(ArticleOut):
+    body: str
+    cluster_members: list[dict[str, Any]] = []
+
+
+class RefreshOut(BaseModel):
+    polled: int
+    new_articles: int
+    duplicates: int
+    cluster_joins: int
+    feed_errors: int = 0
+
+
+# ── factory ─────────────────────────────────────────────────────────────
+
+
+def create_app(
+    *,
+    config: MetatronConfig,
+    db: Database,
+    judge: BatchJudge,
+    poller: Poller | None = None,
+) -> FastAPI:
+    """Build the FastAPI app wired to the given dependencies."""
+    app = FastAPI(title="Metatron", version="0.2.0")
+
+    expected_token = config.api.api_token
+
+    def require_auth(authorization: str | None = Header(default=None)) -> None:
+        if not expected_token:
+            return
+        if not authorization or not authorization.startswith("Bearer "):
+            raise HTTPException(status_code=401, detail="Missing bearer token")
+        token = authorization.removeprefix("Bearer ").strip()
+        if token != expected_token:
+            raise HTTPException(status_code=401, detail="Invalid bearer token")
+
+    @app.on_event("startup")
+    async def _startup() -> None:
+        if poller is not None and config.poller.enabled:
+            poller.start()
+
+    @app.on_event("shutdown")
+    async def _shutdown() -> None:
+        if poller is not None:
+            await poller.stop()
+
+    @app.get("/health")
+    async def health() -> dict[str, Any]:
+        return {"status": "ok", "llm_enabled": judge.enabled}
+
+    # ── projects ─────────────────────────────────────────────────────────
+    @app.post("/projects", response_model=ProjectOut, status_code=201, dependencies=[Depends(require_auth)])
+    async def create_project(payload: CreateProjectIn) -> ProjectOut:
+        try:
+            row = db.create_project(payload.name)
+        except Exception as e:
+            raise HTTPException(status_code=409, detail=f"Project create failed: {e}") from e
+        return ProjectOut(**row)
+
+    @app.get("/projects", response_model=list[ProjectOut], dependencies=[Depends(require_auth)])
+    async def list_projects() -> list[ProjectOut]:
+        return [ProjectOut(**r) for r in db.list_projects()]
+
+    @app.delete("/projects/{project_id}", status_code=204, dependencies=[Depends(require_auth)])
+    async def delete_project(project_id: str = PathParam(...)) -> None:
+        if not db.delete_project(project_id):
+            raise HTTPException(status_code=404, detail="Project not found")
+
+    # ── feeds ────────────────────────────────────────────────────────────
+    @app.post(
+        "/projects/{project_id}/feeds",
+        response_model=FeedOut,
+        status_code=201,
+        dependencies=[Depends(require_auth)],
+    )
+    async def add_feed(project_id: str, payload: AddFeedIn) -> FeedOut:
+        if not db.get_project(project_id):
+            raise HTTPException(status_code=404, detail="Project not found")
+        name = payload.name or _host_of(payload.url)
+        try:
+            row = db.add_feed(
+                project_id=project_id,
+                url=payload.url,
+                name=name,
+                category=payload.category,
+                poll_interval_seconds=payload.poll_interval_seconds,
+            )
+        except Exception as e:
+            raise HTTPException(status_code=409, detail=f"Feed add failed: {e}") from e
+        return _feed_to_out(row)
+
+    @app.get(
+        "/projects/{project_id}/feeds",
+        response_model=list[FeedOut],
+        dependencies=[Depends(require_auth)],
+    )
+    async def list_feeds(project_id: str) -> list[FeedOut]:
+        if not db.get_project(project_id):
+            raise HTTPException(status_code=404, detail="Project not found")
+        return [_feed_to_out(r) for r in db.list_feeds(project_id)]
+
+    @app.delete("/feeds/{feed_id}", status_code=204, dependencies=[Depends(require_auth)])
+    async def delete_feed(feed_id: str) -> None:
+        if not db.delete_feed(feed_id):
+            raise HTTPException(status_code=404, detail="Feed not found")
+
+    # ── articles ─────────────────────────────────────────────────────────
+    @app.get(
+        "/projects/{project_id}/articles",
+        response_model=list[ArticleOut],
+        dependencies=[Depends(require_auth)],
+    )
+    async def list_articles(
+        project_id: str,
+        since: str | None = Query(default=None),
+        limit: int = Query(default=50, ge=1, le=500),
+    ) -> list[ArticleOut]:
+        if not db.get_project(project_id):
+            raise HTTPException(status_code=404, detail="Project not found")
+        rows = db.list_articles(project_id, since=since, limit=limit, deduped=True)
+        return [_article_to_out(r) for r in rows]
+
+    @app.get(
+        "/articles/{article_id}",
+        response_model=ArticleDetailOut,
+        dependencies=[Depends(require_auth)],
+    )
+    async def get_article(article_id: str) -> ArticleDetailOut:
+        row = db.get_article(article_id)
+        if not row:
+            raise HTTPException(status_code=404, detail="Article not found")
+        members: list[dict[str, Any]] = []
+        if row.get("cluster_id"):
+            members = db.article_cluster_members(row["cluster_id"])
+        return ArticleDetailOut(
+            **_article_to_out(row).model_dump(),
+            body=row.get("body", ""),
+            cluster_members=members,
+        )
+
+    # ── on-demand refresh ────────────────────────────────────────────────
+    @app.post(
+        "/projects/{project_id}/refresh",
+        response_model=RefreshOut,
+        dependencies=[Depends(require_auth)],
+    )
+    async def refresh(project_id: str) -> RefreshOut:
+        if not db.get_project(project_id):
+            raise HTTPException(status_code=404, detail="Project not found")
+        loop = asyncio.get_running_loop()
+        stats = await loop.run_in_executor(
+            None,
+            refresh_project_now,
+            project_id,
+            db,
+            judge,
+            DedupConfig(),
+            config.poller.feed_timeout_seconds,
+        )
+        return RefreshOut(**stats)
+
+    return app
+
+
+# ── helpers ─────────────────────────────────────────────────────────────
+
+
+def _feed_to_out(row: dict[str, Any]) -> FeedOut:
+    return FeedOut(
+        id=row["id"],
+        project_id=row["project_id"],
+        url=row["url"],
+        name=row["name"],
+        category=row.get("category", "") or "",
+        enabled=bool(row.get("enabled", 1)),
+        poll_interval_seconds=int(row.get("poll_interval_seconds", 1800)),
+        last_polled=row.get("last_polled"),
+        last_error=row.get("last_error"),
+    )
+
+
+def _article_to_out(row: dict[str, Any]) -> ArticleOut:
+    return ArticleOut(
+        id=row["id"],
+        project_id=row["project_id"],
+        canonical_url=row["canonical_url"],
+        source_url=row["source_url"],
+        source=row["source"],
+        title=row["title"],
+        summary=row.get("summary", "") or "",
+        published=row.get("published"),
+        fetched_at=row["fetched_at"],
+        cluster_id=row.get("cluster_id"),
+    )
+
+
+def _host_of(url: str) -> str:
+    from urllib.parse import urlsplit
+
+    try:
+        host = urlsplit(url).hostname or url
+        return host[4:] if host.startswith("www.") else host
+    except ValueError:
+        return url

metatron/cli.py

@@ -0,0 +1,221 @@
+"""Metatron CLI.
+
+Subcommands:
+  metatron serve                — run the HTTP API + background poller
+  metatron config init          — write a starter config to ~/.config/metatron/config.toml
+  metatron config show          — print the resolved config
+  metatron seed-feeds <project> <feeds.json>
+                                — bulk-add feeds to a project from a JSON file
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+
+from metatron.config import (
+    DEFAULT_CONFIG_TEMPLATE,
+    ConfigError,
+    MetatronConfig,
+)
+
+logger = logging.getLogger("metatron.cli")
+
+
+def _resolve_config_path() -> Path:
+    override = os.environ.get("METATRON_CONFIG")
+    return Path(override) if override else MetatronConfig.default_config_path()
+
+
+def _cmd_config_init() -> int:
+    path = _resolve_config_path()
+    if path.exists():
+        print(f"Config already exists: {path}", file=sys.stderr)
+        return 1
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(DEFAULT_CONFIG_TEMPLATE, encoding="utf-8")
+    print(f"Wrote starter config to {path}")
+    return 0
+
+
+def _cmd_config_show() -> int:
+    path = _resolve_config_path()
+    try:
+        config = MetatronConfig.from_file()
+    except ConfigError as e:
+        print(f"Configuration error: {e}", file=sys.stderr)
+        return 1
+    print(f"Config path:        {path}")
+    print(f"Exists:             {path.exists()}")
+    print(f"API host:port:      {config.api.host}:{config.api.port}")
+    print(f"API token set:      {'yes' if config.api.api_token else 'no (open)'}")
+    print(f"LLM tiebreaker:     {'enabled' if config.llm.enabled else 'disabled'}")
+    print(f"LLM model:          {config.llm.model} (via {config.llm.binary} CLI)")
+    print(f"Poller enabled:     {config.poller.enabled}")
+    print(f"Poller tick:        {config.poller.tick_seconds}s")
+    print(f"Feed interval:      {config.poller.default_feed_interval_seconds}s")
+    return 0
+
+
+def _build_judge(config: MetatronConfig) -> "BatchJudge":
+    from metatron.llm import BatchJudge as _Judge
+
+    if not config.llm.enabled:
+        class _Disabled(_Judge):
+            @property
+            def enabled(self) -> bool:  # type: ignore[override]
+                return False
+
+            def cluster(self, items):  # type: ignore[override]
+                return []
+
+        return _Disabled(model=config.llm.model, binary=config.llm.binary)
+    return _Judge(
+        model=config.llm.model,
+        binary=config.llm.binary,
+        idle_timeout=config.llm.idle_timeout_seconds,
+    )
+
+
+def _cmd_serve() -> int:
+    import uvicorn
+
+    from metatron.api import create_app
+    from metatron.db import Database, default_db_path
+    from metatron.llm import BatchJudge
+    from metatron.poller import Poller
+
+    try:
+        config = MetatronConfig.from_file()
+    except ConfigError as e:
+        print(f"Configuration error: {e}", file=sys.stderr)
+        return 1
+
+    db_path = Path(config.database.path) if config.database.path else default_db_path()
+    db = Database(path=db_path)
+    judge = _build_judge(config)
+    poller = Poller(
+        db=db,
+        judge=judge,
+        tick_seconds=config.poller.tick_seconds,
+        feed_timeout_seconds=config.poller.feed_timeout_seconds,
+    )
+    app = create_app(config=config, db=db, judge=judge, poller=poller)
+
+    uvicorn.run(
+        app,
+        host=config.api.host,
+        port=config.api.port,
+        log_level="info",
+    )
+    return 0
+
+
+def _cmd_seed_feeds(project_name: str, feeds_path: str) -> int:
+    from metatron.db import Database, default_db_path
+
+    path = Path(feeds_path)
+    if not path.exists():
+        print(f"feeds file not found: {path}", file=sys.stderr)
+        return 1
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as e:
+        print(f"feeds file not valid JSON: {e}", file=sys.stderr)
+        return 1
+    feeds = data.get("feeds", data if isinstance(data, list) else [])
+    if not feeds:
+        print("no feeds found in input", file=sys.stderr)
+        return 1
+
+    config = MetatronConfig.from_file()
+    db_path = Path(config.database.path) if config.database.path else default_db_path()
+    db = Database(path=db_path)
+
+    existing = next(
+        (p for p in db.list_projects() if p["name"] == project_name), None
+    )
+    if existing is None:
+        project = db.create_project(project_name)
+    else:
+        project = existing
+    project_id = project["id"]
+
+    added = 0
+    skipped = 0
+    for feed in feeds:
+        url = feed.get("url")
+        name = feed.get("name") or url
+        category = feed.get("category", "")
+        if not url:
+            skipped += 1
+            continue
+        try:
+            db.add_feed(
+                project_id=project_id,
+                url=url,
+                name=name,
+                category=category,
+                poll_interval_seconds=int(
+                    feed.get(
+                        "poll_interval_seconds",
+                        config.poller.default_feed_interval_seconds,
+                    )
+                ),
+            )
+            added += 1
+        except Exception as e:
+            print(f"  skip {url}: {e}", file=sys.stderr)
+            skipped += 1
+    print(f"Project: {project_name} ({project_id})")
+    print(f"Added:   {added}")
+    print(f"Skipped: {skipped}")
+    return 0
+
+
+def main(args: list[str] | None = None) -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    from metatron import __version__
+
+    parser = argparse.ArgumentParser(
+        description="Metatron — multi-project RSS feed manager with cross-outlet deduplication",
+    )
+    parser.add_argument("--version", action="version", version=f"metatron {__version__}")
+    sub = parser.add_subparsers(dest="subcommand")
+
+    sub.add_parser("serve", help="Run the HTTP API + background poller")
+
+    cfg = sub.add_parser("config", help="Config management")
+    cfg.add_argument("action", choices=["init", "show"])
+
+    seed = sub.add_parser("seed-feeds", help="Bulk-add feeds to a project from JSON")
+    seed.add_argument("project", help="Project name (created if missing)")
+    seed.add_argument("feeds_path", help="Path to a JSON file with feeds")
+
+    parsed = parser.parse_args(args)
+
+    if parsed.subcommand == "serve":
+        return _cmd_serve()
+    if parsed.subcommand == "config":
+        if parsed.action == "init":
+            return _cmd_config_init()
+        if parsed.action == "show":
+            return _cmd_config_show()
+    if parsed.subcommand == "seed-feeds":
+        return _cmd_seed_feeds(parsed.project, parsed.feeds_path)
+
+    parser.print_help()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())