newswatcher 0.1.0__py3-none-any.whl

newswatcher/__init__.py

@@ -0,0 +1,13 @@
+"""newswatcher: watch news sources (RSS or robots-permitted crawl), match topics,
+summarize with an LLM, and mail a digest."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+__all__ = ["__version__"]
+
+try:
+    __version__ = version("newswatcher")
+except PackageNotFoundError:   # not installed (e.g. run from a source tree)
+    __version__ = "0.0.0+unknown"

newswatcher/__main__.py

@@ -0,0 +1,10 @@
+"""Enable ``python -m newswatcher`` as an alias for the console script."""
+
+from __future__ import annotations
+
+import sys
+
+from newswatcher.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

newswatcher/_atomic.py

@@ -0,0 +1,36 @@
+"""Atomic file writes: temp file in the target directory, then rename over the
+target, so a crash or a concurrent reader never sees a half-written file, and two
+overlapping writers do not share a temp path."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+
+__all__ = ["write_text_atomic", "write_bytes_atomic"]
+
+
+def write_text_atomic(path: Path, text: str, error_cls: type[Exception]) -> None:
+    """Write ``text`` to ``path`` atomically; wrap any I/O failure in ``error_cls``."""
+    write_bytes_atomic(path, text.encode("utf-8"), error_cls)
+
+
+def write_bytes_atomic(path: Path, data: bytes, error_cls: type[Exception]) -> None:
+    """Write ``data`` to ``path`` atomically; wrap any I/O failure in ``error_cls``."""
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp")
+        tmp = Path(tmp_name)
+        try:
+            with os.fdopen(fd, "wb") as handle:
+                handle.write(data)
+                handle.flush()
+                os.fsync(handle.fileno())   # flush to disk before the rename, so a crash
+                                            # after os.replace cannot leave a half-written file
+            os.replace(tmp, path)
+        except OSError:
+            tmp.unlink(missing_ok=True)
+            raise
+    except OSError as err:
+        raise error_cls(f"could not write {path}: {err}") from err

newswatcher/_llm.py

@@ -0,0 +1,82 @@
+"""Build a thinchat LLM client, letting thinchat resolve the provider's key.
+
+newswatcher's LLM tasks -- summarizing an article, repairing a broken crawl selector -- run on
+the thinchat library, which speaks to every provider behind one interface. The key is thinchat's
+to resolve: an explicit ``api_key`` override wins, then the provider's standard environment
+variable, then thinchat's own store (``thinchat set <provider>``, which ``newswatcher setup``
+drives). newswatcher keeps no key of its own -- summarizing is newswatcher's job, but the LLM
+credential belongs with thinchat, the tool that speaks to the provider. Gemini's free tier is the
+default backend for the light summary task.
+
+thinchat scrubs any provider key from its own error messages and the exception chain beneath
+them, so newswatcher interpolates a ``ThinchatError`` directly without re-redacting it."""
+
+from __future__ import annotations
+
+from thinchat import PROVIDERS, Client, get_api_key, make_client
+from thinchat.errors import ThinchatError
+from thinchat.keys import ENV_BY_PROVIDER
+
+from newswatcher.errors import LLMError
+
+__all__ = ["DEFAULT_PROVIDER", "PROVIDERS", "make_llm_client", "provider_key_name",
+           "validate_provider"]
+
+DEFAULT_PROVIDER = "gemini"
+_MAX_RETRIES = 6
+
+
+def validate_provider(provider: str) -> None:
+    """Check that ``provider`` is a backend thinchat knows -- the single home for this check, so
+    the CLI's early validation and ``make_llm_client`` give the same message.
+
+    Raises:
+        LLMError: the provider is not a known backend.
+    """
+    if provider not in PROVIDERS:
+        raise LLMError(
+            f"unknown LLM provider {provider!r}; choose one of {', '.join(sorted(PROVIDERS))}")
+
+
+def provider_key_name(provider: str) -> str | None:
+    """The environment-variable / store name that holds ``provider``'s API key
+    (``GEMINI_API_KEY`` ...), or ``None`` for a keyless provider (ollama runs locally and needs
+    none). The single home for this mapping, so the CLI's ``set-key`` and ``make_llm_client``
+    agree on the name a key is stored under."""
+    return ENV_BY_PROVIDER.get(provider)
+
+
+def make_llm_client(
+    provider: str = DEFAULT_PROVIDER, *, model: str | None = None,
+    api_key: str | None = None, max_tokens: int, action: str,
+) -> Client:
+    """Build the thinchat client for ``provider``. The key is resolved by thinchat -- ``api_key``
+    when given, else the provider's standard env var, else thinchat's own store; ``model``
+    overrides the default; ``max_tokens`` caps the reply; ``action`` names the caller in the
+    missing-key message. A keyless provider (ollama) needs no key.
+
+    Raises:
+        LLMError: unknown provider, no API key available for a keyed provider, thinchat's store
+            was unreadable, or the client could not be constructed.
+    """
+    validate_provider(provider)
+    env_name = provider_key_name(provider)
+    if env_name is None:
+        key = api_key   # keyless (ollama): honor an explicit key if given, else none
+    else:
+        # thinchat owns the key (override > env > its own store). Resolve it here only to raise a
+        # friendly, action-named error before constructing the client; the store is thinchat's.
+        try:
+            resolved_secret = get_api_key(provider, override=api_key)
+        except ThinchatError as err:
+            raise LLMError(f"{action} could not read the stored key for {provider}: {err}") from err
+        key = resolved_secret.reveal() if resolved_secret is not None else None
+        if not key:
+            raise LLMError(
+                f"{action} needs an API key for {provider}; set {env_name} "
+                f"or run 'newswatcher setup' (it stores the key with thinchat)")
+    try:
+        return make_client(provider, model=model, api_key=key,
+                           max_tokens=max_tokens, max_retries=_MAX_RETRIES)
+    except ThinchatError as err:
+        raise LLMError(f"{action} could not start {provider}: {err}") from err

newswatcher/_select.py

@@ -0,0 +1,36 @@
+"""Run a CSS selector, turning a malformed-selector error into a domain ``SourceError``.
+
+soupsieve raises two unrelated classes for a bad selector: ``SelectorSyntaxError`` for a
+syntax error (``>>bad``, ``a[``), and a bare ``NotImplementedError`` for an unsupported
+pseudo-element (``a::attr(href)``, ``a::text`` -- the Scrapy idiom an LLM healer readily
+emits). Both are caught here, in one place used by both the crawl adapter and the body
+extractor, so one bad selector -- hand-typed in ``sources.toml`` or proposed by the healer
+-- skips its source (the poll catches ``SourceError`` per source) instead of aborting the
+whole pass with a raw error the pipeline does not expect."""
+
+from __future__ import annotations
+
+from bs4 import BeautifulSoup, Tag
+from soupsieve import SelectorSyntaxError
+
+from newswatcher.errors import SourceError
+
+__all__ = ["select_all", "select_one"]
+
+
+def select_all(node: Tag | BeautifulSoup, selector: str, source_name: str) -> list[Tag]:
+    """``node.select(selector)`` with a malformed selector reported as ``SourceError``."""
+    try:
+        return node.select(selector)
+    except (SelectorSyntaxError, NotImplementedError) as err:
+        raise SourceError(
+            f"source {source_name!r}: invalid CSS selector {selector!r}: {err}") from err
+
+
+def select_one(node: Tag | BeautifulSoup, selector: str, source_name: str) -> Tag | None:
+    """``node.select_one(selector)`` with a malformed selector reported as ``SourceError``."""
+    try:
+        return node.select_one(selector)
+    except (SelectorSyntaxError, NotImplementedError) as err:
+        raise SourceError(
+            f"source {source_name!r}: invalid CSS selector {selector!r}: {err}") from err

newswatcher/_toml.py

@@ -0,0 +1,54 @@
+"""The small TOML pieces the ``topics`` and ``sources`` registries both need: encode a
+string or a list of strings as a TOML value, and read a ``[[key]]`` table array back.
+
+The read side is the mirror of ``_atomic`` on the write side -- both registries store a
+table array of records in ``config_dir()`` and were repeating the same parse-and-validate
+boilerplate. Each registry still renders its own record blocks (their fields differ); only
+these primitives, which do not, live here. Python's ``tomllib`` reads TOML but cannot write
+it, so the encoders are hand-rolled -- a TOML basic string is spelled exactly like a JSON
+string, so ``json.dumps`` produces one (with the right escaping) for free.
+"""
+
+from __future__ import annotations
+
+import json
+import tomllib
+from pathlib import Path
+
+__all__ = ["array", "quote", "read_table_array"]
+
+
+def quote(value: str) -> str:
+    """Encode ``value`` as a TOML basic string, quoted and escaped (``na"me`` ->
+    ``"na\\"me"``). Non-ASCII is kept verbatim, matching the human-edited files."""
+    return json.dumps(value, ensure_ascii=False)
+
+
+def array(words: tuple[str, ...]) -> str:
+    """Encode ``words`` as a TOML inline array of basic strings (``["a", "b"]``)."""
+    return "[" + ", ".join(quote(word) for word in words) + "]"
+
+
+def read_table_array(
+    path: Path, key: str, error_cls: type[Exception]
+) -> list[dict[str, object]]:
+    """Read ``path`` and return the entries of its ``[[key]]`` table array (each a dict);
+    empty when the key is absent. Non-dict entries are skipped.
+
+    Raises:
+        error_cls: the file is unreadable or not valid TOML, or ``key`` holds something
+            other than a table array (a scalar ``key = ...`` instead of ``[[key]]``). The
+            caller passes its own domain error type so the message names the right file.
+    """
+    try:
+        parsed = tomllib.loads(path.read_text(encoding="utf-8"))
+    except (tomllib.TOMLDecodeError, OSError, UnicodeDecodeError) as err:
+        # UnicodeDecodeError is a ValueError, not an OSError, so a non-UTF-8 file must be
+        # named explicitly or it escapes this boundary as a bare traceback.
+        raise error_cls(f"could not read {path}: {err}") from err
+    entries = parsed.get(key)
+    if entries is None:
+        return []
+    if not isinstance(entries, list):
+        raise error_cls(f"{path}: [{key}] must be a table array ([[{key}]])")
+    return [entry for entry in entries if isinstance(entry, dict)]

newswatcher/body.py

@@ -0,0 +1,47 @@
+"""Fetch an article page and extract its main text — the raw material an LLM summary
+is written from. The body is transient: it is never stored in the archive nor placed
+in the outbound email (which carry only our summary plus the link). Extraction uses
+the source's ``body_selector`` when it defines one, else the generic extractor
+(trafilatura)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import trafilatura
+from bs4 import BeautifulSoup
+
+from newswatcher._select import select_one
+from newswatcher.feed import FeedItem
+from newswatcher.http import get
+from newswatcher.robots import RobotsGate
+from newswatcher.sources import Source
+
+if TYPE_CHECKING:
+    import requests
+
+__all__ = ["extract_body", "fetch_body"]
+
+
+def fetch_body(item: FeedItem, source: Source, gate: RobotsGate, *,
+               session: requests.Session | None = None) -> str:
+    """Fetch ``item``'s article page (robots-gated) and extract its body text, or ""
+    when nothing could be extracted.
+
+    Raises:
+        FetchError: robots disallows the article URL or the fetch failed (propagated
+            from ``http.get``).
+    """
+    return extract_body(get(item.link, gate, session=session), source)
+
+
+def extract_body(html: str, source: Source) -> str:
+    """Extract the article body from ``html``. With ``source.body_selector`` set, the
+    text of the first matching node; otherwise trafilatura's main-content extraction.
+    Returns "" when neither yields text (a summary step then falls back to the feed
+    title/summary)."""
+    if source.body_selector:
+        node = select_one(BeautifulSoup(html, "lxml"), source.body_selector, source.name)
+        return node.get_text(" ", strip=True) if node is not None else ""
+    extracted = trafilatura.extract(html, include_comments=False, include_tables=False)
+    return (extracted or "").strip()