metatron-cli 0.2.1__py3-none-any.whl

metatron/dedup.py

@@ -0,0 +1,210 @@
+"""Deduplication pipeline.
+
+Two phases per refresh:
+
+  Phase A (cheap, per-item):
+    - L1: exact canonical-URL match → drop as duplicate.
+    - L2: normalized-title match (lowercase, punctuation stripped, common
+      newsroom prefixes removed) → attach to that existing article's cluster.
+
+  Phase B (batched, one LLM call):
+    - For every candidate that survived A, pair it with the recent-history
+      articles whose token overlap exceeds ``llm_threshold``.
+    - Send the union of (candidates + their candidate-history peers) to
+      Claude in a single batched prompt asking "which of these report the
+      same story?".
+    - Apply the returned clusters: each candidate either joins an
+      existing cluster (matched against history) or starts a new one
+      with another candidate.
+
+This collapses dozens of per-pair CLI calls (each with ~5–10 s overhead)
+into one prompt, while still giving the model rich pairwise context to
+work with.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass
+from typing import Any, Iterable
+
+from metatron.llm import BatchJudge, ClusterItem
+from metatron.normalize import canonicalize_url, jaccard, normalize_title, tokenize
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DedupConfig:
+    llm_threshold: float = 0.12  # min token overlap for a history pairing to be sent to the LLM
+    history_pair_cap: int = 5    # cap candidate-history peers added per candidate
+
+
+@dataclass
+class CheapDecision:
+    """Result of Phase A (cheap layers)."""
+
+    status: str  # "duplicate" | "cluster" | "candidate"
+    canonical_article_id: str | None = None
+    cluster_id: str | None = None
+    reason: str = ""
+
+
+def cheap_decide(
+    candidate: dict[str, Any], history: list[dict[str, Any]]
+) -> CheapDecision:
+    """Run Phase A (URL + normalized-title layers).
+
+    Returns:
+      - status="duplicate": exact canonical URL already exists.
+      - status="cluster":   normalized title matches an existing article.
+      - status="candidate": needs LLM review in Phase B.
+    """
+    cand_url = candidate.get("canonical_url") or canonicalize_url(
+        candidate.get("source_url", "")
+    )
+    cand_title_norm = normalize_title(candidate.get("title", ""))
+
+    for row in history:
+        if row.get("canonical_url") and row["canonical_url"] == cand_url:
+            return CheapDecision(
+                status="duplicate",
+                canonical_article_id=row["id"],
+                cluster_id=row.get("cluster_id"),
+                reason="canonical url match",
+            )
+
+    for row in history:
+        if cand_title_norm and normalize_title(row.get("title", "")) == cand_title_norm:
+            return CheapDecision(
+                status="cluster",
+                canonical_article_id=row["id"],
+                cluster_id=row.get("cluster_id"),
+                reason="normalized title match",
+            )
+
+    return CheapDecision(status="candidate", reason="needs LLM review")
+
+
+@dataclass
+class BatchPlan:
+    """Inputs for the batched LLM call.
+
+    ``items`` contains every article — both new candidates and history
+    articles selected as their token-overlap peers — that the model will
+    receive. ``candidate_ids`` is the subset that are new (not yet
+    inserted). The result maps candidate ref_id → list of history ref_ids
+    the model said cover the same story.
+    """
+
+    items: list[ClusterItem]
+    candidate_ids: set[str]
+
+
+def build_batch_plan(
+    candidates: list[dict[str, Any]],
+    history: list[dict[str, Any]],
+    *,
+    config: DedupConfig | None = None,
+) -> BatchPlan:
+    """Pick the set of articles the LLM should see in one prompt.
+
+    Every candidate is included. For each candidate we add the top
+    ``history_pair_cap`` history rows whose token overlap with the
+    candidate exceeds ``llm_threshold``. Each row is included at most
+    once even if multiple candidates peer with it.
+    """
+    cfg = config or DedupConfig()
+
+    items_by_id: dict[str, ClusterItem] = {}
+    candidate_ids: set[str] = set()
+
+    for cand in candidates:
+        rid = _candidate_ref_id(cand)
+        cand["_ref_id"] = rid
+        items_by_id[rid] = _as_cluster_item(rid, cand)
+        candidate_ids.add(rid)
+
+    if not history:
+        return BatchPlan(items=list(items_by_id.values()), candidate_ids=candidate_ids)
+
+    history_token_cache: dict[str, set[str]] = {}
+
+    def hist_tokens(row: dict[str, Any]) -> set[str]:
+        rid = row["id"]
+        if rid not in history_token_cache:
+            history_token_cache[rid] = tokenize(_lead_text(row))
+        return history_token_cache[rid]
+
+    for cand in candidates:
+        cand_tokens = tokenize(_lead_text(cand))
+        if not cand_tokens:
+            continue
+        scored: list[tuple[float, dict[str, Any]]] = []
+        for row in history:
+            score = jaccard(cand_tokens, hist_tokens(row))
+            if score >= cfg.llm_threshold:
+                scored.append((score, row))
+        scored.sort(key=lambda t: t[0], reverse=True)
+        for _, row in scored[: cfg.history_pair_cap]:
+            rid = row["id"]
+            if rid not in items_by_id:
+                items_by_id[rid] = _as_cluster_item(rid, row)
+
+    return BatchPlan(items=list(items_by_id.values()), candidate_ids=candidate_ids)
+
+
+def run_batch(
+    plan: BatchPlan, judge: BatchJudge
+) -> dict[str, list[str]]:
+    """Run the single LLM call and return candidate_id → [matched_ref_ids].
+
+    Non-candidate IDs in the returned mapping are filtered out by the
+    caller (they are history rows that grouped with a candidate).
+    """
+    if not judge.enabled or len(plan.items) < 2:
+        return {}
+
+    groups = judge.cluster(plan.items)
+    if not groups:
+        return {}
+
+    out: dict[str, list[str]] = {}
+    for group in groups:
+        cand_in_group = [r for r in group.ref_ids if r in plan.candidate_ids]
+        other_in_group = [r for r in group.ref_ids if r not in plan.candidate_ids]
+        for cid in cand_in_group:
+            peers = [r for r in group.ref_ids if r != cid]
+            out.setdefault(cid, []).extend(peers)
+        _ = other_in_group  # noqa: F841 — informational only
+    return out
+
+
+# ── helpers ──────────────────────────────────────────────────────────────
+
+
+def _candidate_ref_id(candidate: dict[str, Any]) -> str:
+    """A stable, short ID for a not-yet-inserted candidate."""
+    seed = (
+        candidate.get("canonical_url", "")
+        or candidate.get("source_url", "")
+        or candidate.get("title", "")
+    )
+    return "C-" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:10]
+
+
+def _as_cluster_item(ref_id: str, article: dict[str, Any]) -> ClusterItem:
+    return ClusterItem(
+        ref_id=ref_id,
+        title=article.get("title", "") or "",
+        summary=_lead_text(article),
+        source=article.get("source", "") or "",
+    )
+
+
+def _lead_text(article: dict[str, Any]) -> str:
+    title = article.get("title", "") or ""
+    summary = article.get("summary", "") or ""
+    body = article.get("body", "") or ""
+    return f"{title}\n{summary}\n{body[:400]}".strip()

metatron/fetcher.py

@@ -0,0 +1,147 @@
+"""Fetch a single RSS/Atom feed and produce article candidates.
+
+Two-stage:
+  1. ``parse_feed(url)`` — grab the feed XML, parse with feedparser, return
+     a list of lightweight item dicts (title, summary, source url, source
+     name, published).
+  2. ``enrich_article(item)`` — fetch the article HTML and extract clean
+     body text with trafilatura. Returns the item dict with ``body`` set.
+
+Body extraction is opt-in per article (the dedup pipeline calls it only
+when the cheap layers can't decide). Bandwidth-conscious.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+import feedparser
+import requests
+import trafilatura
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FetchedItem:
+    title: str
+    source_url: str
+    summary: str
+    source: str
+    published: datetime | None
+    body: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "title": self.title,
+            "source_url": self.source_url,
+            "summary": self.summary,
+            "source": self.source,
+            "published": self.published.isoformat() if self.published else None,
+            "body": self.body,
+        }
+
+
+class FeedFetchError(Exception):
+    """Raised when a feed cannot be retrieved or parsed."""
+
+
+def parse_feed(url: str, *, source_name: str | None = None, timeout: float = 30.0) -> list[FetchedItem]:
+    """Fetch and parse one RSS/Atom feed. Raises ``FeedFetchError`` on failure."""
+    try:
+        resp = requests.get(
+            url,
+            timeout=timeout,
+            headers={"User-Agent": "metatron/0.2 (+https://github.com/anthropic/metatron)"},
+        )
+        resp.raise_for_status()
+    except requests.RequestException as e:
+        raise FeedFetchError(f"HTTP error for {url}: {e}") from e
+
+    parsed = feedparser.parse(resp.content)
+    if parsed.bozo and not parsed.entries:
+        raise FeedFetchError(f"Feed parse failed for {url}: {parsed.bozo_exception}")
+
+    feed_title = parsed.feed.get("title", "") if hasattr(parsed, "feed") else ""
+    source = source_name or feed_title or _host_of(url)
+
+    out: list[FetchedItem] = []
+    for entry in parsed.entries:
+        link = entry.get("link", "").strip()
+        if not link:
+            continue
+        title = (entry.get("title") or "").strip()
+        summary = (entry.get("summary") or entry.get("description") or "").strip()
+        published = _parse_entry_date(entry)
+        out.append(
+            FetchedItem(
+                title=title,
+                source_url=link,
+                summary=_strip_html(summary),
+                source=source,
+                published=published,
+            )
+        )
+    return out
+
+
+def enrich_article(item: FetchedItem, *, timeout: float = 15.0) -> FetchedItem:
+    """Fetch the article page and extract clean body text.
+
+    On any failure, returns ``item`` with body left as-is (likely empty).
+    We don't want one bad page to poison the whole poll loop, so the only
+    surfacing here is a log line — the dedup pipeline gracefully handles
+    missing bodies by leaning on title/summary.
+    """
+    try:
+        downloaded = trafilatura.fetch_url(item.source_url, no_ssl=False)
+        if not downloaded:
+            return item
+        text = trafilatura.extract(
+            downloaded,
+            include_comments=False,
+            include_tables=False,
+            favor_precision=True,
+            no_fallback=False,
+        )
+        if text:
+            item.body = text.strip()
+    except Exception as e:
+        logger.debug("body extraction failed for %s: %s", item.source_url, e)
+    return item
+
+
+def _parse_entry_date(entry: Any) -> datetime | None:
+    for key in ("published_parsed", "updated_parsed"):
+        tm = entry.get(key)
+        if tm:
+            try:
+                return datetime(*tm[:6], tzinfo=timezone.utc)
+            except (TypeError, ValueError):
+                continue
+    return None
+
+
+_TAG_RE = None
+
+
+def _strip_html(text: str) -> str:
+    global _TAG_RE
+    import re
+
+    if _TAG_RE is None:
+        _TAG_RE = re.compile(r"<[^>]+>")
+    return _TAG_RE.sub("", text).strip()
+
+
+def _host_of(url: str) -> str:
+    from urllib.parse import urlsplit
+
+    try:
+        host = urlsplit(url).hostname or ""
+        return host[4:] if host.startswith("www.") else host
+    except ValueError:
+        return ""

metatron/llm.py

@@ -0,0 +1,270 @@
+"""Claude CLI wrapper for the dedup tiebreaker.
+
+The CLI overhead (~5-10s per `claude -p` invocation) makes per-pair LLM
+calls infeasible at any real scale. So we batch: one prompt per refresh
+asks Claude to identify clusters across the full candidate set + recent
+history. The model returns ``{"groups": [[id, id, ...], ...]}`` where
+each inner list groups article IDs that report the same story.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import select
+import shutil
+import subprocess
+import time
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ClusterItem:
+    """One item passed into the batch judge.
+
+    ``ref_id`` is whatever the caller wants the model to echo back — usually
+    the article's database id, or a fresh hash for new candidates that
+    aren't in the DB yet.
+    """
+
+    ref_id: str
+    title: str
+    summary: str
+    source: str
+
+
+@dataclass
+class ClusterGroup:
+    ref_ids: list[str]
+
+
+class BatchJudge:
+    """Run `claude -p` once per refresh to cluster items by same-story."""
+
+    DEFAULT_MODEL = "sonnet"
+    # Idle window: kill the subprocess only if nothing has been written to
+    # stdout/stderr for this many seconds. A productive call that runs for
+    # 10 minutes is fine; a stuck call that's been silent for 2 minutes is not.
+    DEFAULT_IDLE_TIMEOUT = 120.0
+
+    def __init__(
+        self,
+        *,
+        model: str | None = None,
+        binary: str | None = None,
+        idle_timeout: float | None = None,
+    ) -> None:
+        self.model = model or self.DEFAULT_MODEL
+        self.binary = binary or "claude"
+        self.idle_timeout = idle_timeout or self.DEFAULT_IDLE_TIMEOUT
+
+    @property
+    def enabled(self) -> bool:
+        return shutil.which(self.binary) is not None
+
+    def cluster(self, items: list[ClusterItem]) -> list[ClusterGroup]:
+        """Return groups of ref_ids that belong to the same story.
+
+        Items not appearing in any returned group are singletons (their own
+        cluster). On any failure (CLI missing, idle stall, parse error)
+        returns an empty list — caller should treat that as "no LLM
+        groupings learned" and rely on the cheap layers only.
+        """
+        if not items or len(items) < 2:
+            return []
+
+        prompt = _format_batch_prompt(items)
+        try:
+            proc = subprocess.Popen(
+                [
+                    self.binary,
+                    "-p",
+                    "--model",
+                    self.model,
+                    "--output-format",
+                    "json",
+                    prompt,
+                ],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=False,  # read bytes so partial UTF-8 mid-stream is fine
+            )
+        except FileNotFoundError:
+            logger.warning("Claude CLI not found on PATH (looked for %s)", self.binary)
+            return []
+
+        stdout_chunks: list[bytes] = []
+        stderr_chunks: list[bytes] = []
+        idle_killed = False
+        last_output = time.monotonic()
+
+        # Stream the pipes, watching for inactivity. A productive call that
+        # keeps producing output (even slowly) is allowed to run as long as
+        # it needs; we only intervene when both pipes have been silent for
+        # longer than `idle_timeout`.
+        for chunk_bytes in _stream_until_idle(
+            proc, stdout_chunks, stderr_chunks, self.idle_timeout
+        ):
+            if chunk_bytes:
+                last_output = time.monotonic()
+            elapsed_idle = time.monotonic() - last_output
+            if elapsed_idle > self.idle_timeout:
+                logger.warning(
+                    "Claude CLI batch dedup idle for %.0fs — terminating",
+                    elapsed_idle,
+                )
+                _terminate(proc)
+                idle_killed = True
+                break
+
+        # Drain any remaining buffered output once the process is done.
+        try:
+            tail_out, tail_err = proc.communicate(timeout=5)
+        except subprocess.TimeoutExpired:
+            _terminate(proc)
+            tail_out, tail_err = proc.communicate()
+        if tail_out:
+            stdout_chunks.append(tail_out)
+        if tail_err:
+            stderr_chunks.append(tail_err)
+
+        if idle_killed:
+            return []
+        if proc.returncode != 0:
+            stderr_text = b"".join(stderr_chunks).decode("utf-8", "replace")
+            logger.warning(
+                "Claude CLI batch dedup exited %d: %s",
+                proc.returncode,
+                stderr_text[:300],
+            )
+            return []
+
+        stdout_text = b"".join(stdout_chunks).decode("utf-8", "replace")
+        return _parse_cli_output(stdout_text)
+
+
+def _stream_until_idle(
+    proc: subprocess.Popen,
+    stdout_chunks: list[bytes],
+    stderr_chunks: list[bytes],
+    idle_timeout: float,
+):
+    """Yield bytes-read events while the subprocess runs.
+
+    Yields ``b""`` on each tick so the caller can re-evaluate idle elapsed,
+    and yields the actual bytes when a pipe produced data. Exits when the
+    process terminates.
+    """
+    pipes = {proc.stdout: stdout_chunks, proc.stderr: stderr_chunks}
+    open_fds = [p for p in pipes if p is not None]
+    # Make non-blocking so reads can't hang past the select tick.
+    for p in open_fds:
+        os.set_blocking(p.fileno(), False)
+
+    tick = min(1.0, max(0.1, idle_timeout / 10))
+    while open_fds and proc.poll() is None:
+        ready, _, _ = select.select(open_fds, [], [], tick)
+        if not ready:
+            yield b""
+            continue
+        produced = b""
+        for p in ready:
+            try:
+                chunk = p.read()
+            except (OSError, ValueError):
+                chunk = None
+            if not chunk:
+                # EOF or transient empty — drop from poll set if truly closed.
+                if proc.poll() is not None:
+                    open_fds = [x for x in open_fds if x is not p]
+                continue
+            pipes[p].append(chunk)
+            produced += chunk
+        yield produced
+
+
+def _terminate(proc: subprocess.Popen) -> None:
+    """Best-effort shutdown of the CLI subprocess."""
+    if proc.poll() is not None:
+        return
+    proc.terminate()
+    try:
+        proc.wait(timeout=5)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+
+
+_BATCH_PROMPT_HEADER = """\
+You are a news deduplication arbiter. Below is a numbered list of news
+articles. Identify which articles report the SAME specific story — same
+event, same announcement, same incident. Articles about similar topics
+but different specific stories are NOT the same.
+
+For each group of duplicates, list the IDs of every article in that
+group. Articles not in any group are unique stories on their own; you
+do not need to list them.
+
+Respond with ONLY a JSON object on a single line. The "groups" key holds
+an array of arrays; each inner array contains the IDs (as strings) of
+articles in one same-story cluster. If you find no duplicates, return
+{"groups": []}.
+
+Example response:
+{"groups": [["A1", "B3", "C7"], ["D2", "E5"]]}
+
+Articles:
+"""
+
+
+def _format_batch_prompt(items: list[ClusterItem]) -> str:
+    lines = [_BATCH_PROMPT_HEADER]
+    for it in items:
+        summary = (it.summary or "").replace("\n", " ").strip()[:300]
+        lines.append(
+            f"\n[{it.ref_id}] ({it.source}) {it.title}"
+            + (f"\n   {summary}" if summary else "")
+        )
+    return "".join(lines)
+
+
+_JSON_OBJ_RE = re.compile(r"\{.*\}", re.DOTALL)
+
+
+def _parse_cli_output(stdout: str) -> list[ClusterGroup]:
+    if not stdout.strip():
+        return []
+    try:
+        envelope = json.loads(stdout.strip())
+    except json.JSONDecodeError:
+        return []
+    if envelope.get("is_error"):
+        logger.warning("Claude CLI returned error: %s", str(envelope.get("result", ""))[:300])
+        return []
+    result_text = envelope.get("result") or ""
+    return _parse_groups(result_text)
+
+
+def _parse_groups(text: str) -> list[ClusterGroup]:
+    if not text:
+        return []
+    match = _JSON_OBJ_RE.search(text)
+    payload = match.group(0) if match else text
+    try:
+        data = json.loads(payload)
+    except json.JSONDecodeError:
+        logger.warning("Failed to parse model response as JSON: %s", text[:200])
+        return []
+    raw_groups = data.get("groups", [])
+    if not isinstance(raw_groups, list):
+        return []
+    result: list[ClusterGroup] = []
+    for group in raw_groups:
+        if isinstance(group, list) and len(group) >= 2:
+            ids = [str(x) for x in group if isinstance(x, (str, int))]
+            if len(ids) >= 2:
+                result.append(ClusterGroup(ref_ids=ids))
+    return result