youpdated 0.1.0__py3-none-any.whl

youpdated/http.py

@@ -0,0 +1,216 @@
+"""HTTP path for the tool.
+
+All outbound go through :class:`Client`
+"""
+
+from __future__ import annotations
+
+import random
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any, Iterable, Sequence
+from urllib.parse import urlsplit
+
+import httpx
+
+from .config import PrivacyConfig
+from .state import State
+
+# Small pool of current desktop UAs. Rotating inside a plausible set
+# it blends in better, but can be changed here
+USER_AGENTS: tuple[str, ...] = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/17.5 Safari/605.1.15",
+)
+
+RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+
+
+class FetchError(Exception):
+    """Network or unexpected-status failure for one URL."""
+
+
+@dataclass
+class Fetched:
+    url: str
+    status: int
+    content: bytes
+    headers: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def text(self) -> str:
+        return self.content.decode("utf-8", errors="replace")
+
+    def json(self) -> Any:
+        import json
+
+        return json.loads(self.content)
+
+
+class Client:
+    """Paced, proxied, no cookie HTTP client with conditional-GET support."""
+
+    def __init__(
+        self,
+        privacy: PrivacyConfig | None = None,
+        state: State | None = None,
+        *,
+        test: bool = False,
+        verbose: bool = False,
+        use_conditional: bool = True,
+        transport: httpx.BaseTransport | None = None,
+        logger=None,
+    ):
+        self.privacy = privacy or PrivacyConfig()
+        self.state = state
+        self.test = test
+        self.verbose = verbose
+        # Conditional GETs save requests, but a 304 doesn't return items
+        self.use_conditional = use_conditional
+        self._log = logger
+        self.requested_urls: list[str] = []
+
+        self._host_locks: dict[str, threading.Lock] = {}
+        self._host_last: dict[str, float] = {}
+        self._registry_lock = threading.Lock()
+
+        self._client = httpx.Client(
+            proxy=self.privacy.proxy,
+            timeout=self.privacy.timeout,
+            follow_redirects=True,
+            transport=transport,
+            headers={"Accept-Language": "en-US,en;q=0.9"},
+        )
+
+    # lifecycle
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "Client":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def describe(self) -> str:
+        proxy = self.privacy.proxy or "none (direct)"
+        ua = "rotating" if self.privacy.user_agent == "rotate" else self.privacy.user_agent
+        return (
+            f"proxy={proxy} user_agent={ua} "
+            f"jitter={self.privacy.jitter[0]}-{self.privacy.jitter[1]}s "
+            f"concurrency={self.privacy.concurrency} timeout={self.privacy.timeout}s"
+        )
+
+    # internals
+
+    def _user_agent(self) -> str:
+        if self.privacy.user_agent == "rotate":
+            return random.choice(USER_AGENTS)
+        return self.privacy.user_agent
+
+    def _pace(self, host: str) -> None:
+        """Serialize per host and leave a rand gap, so traffic to one site never looks like a burst."""
+        with self._registry_lock:
+            lock = self._host_locks.setdefault(host, threading.Lock())
+        with lock:
+            low, high = self.privacy.jitter
+            last = self._host_last.get(host)
+            now = time.monotonic()
+            if last is not None:
+                wait = random.uniform(low, high) - (now - last)
+                if wait > 0:
+                    time.sleep(wait)
+            self._host_last[host] = time.monotonic()
+
+    def note(self, message: str) -> None:
+        if self.verbose and self._log:
+            self._log(message)
+
+    # request method
+
+    def get(
+        self,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        conditional: bool = False,
+        soft_statuses: Sequence[int] | Iterable[int] = (),
+        retries: int = 2,
+    ) -> Fetched | None:
+        """GET ``url``.
+
+        Returns ``None`` when the server answers 304 Not Modified, or with ``test``. 
+        Statuses listed in ``soft_statuses`` come back as a :class:`Fetched` for interpretation
+        (itch.io answers 404 for a game that simply has no devlog)
+        """
+        self.requested_urls.append(url)
+        if self.test:
+            self.note(f"[test]] GET {url}")
+            return None
+
+        soft = frozenset(soft_statuses)
+        conditional = conditional and self.use_conditional
+        request_headers = {"User-Agent": self._user_agent()}
+        if headers:
+            request_headers.update(headers)
+        if conditional and self.state is not None:
+            request_headers.update(self.state.conditional_headers(url))
+
+        host = urlsplit(url).netloc
+        last_error: Exception | None = None
+
+        for attempt in range(retries + 1):
+            self._pace(host)
+            try:
+                # Cookies dropped every request
+                self._client.cookies.clear()
+                response = self._client.get(url, headers=request_headers)
+            except httpx.HTTPError as exc:
+                last_error = exc
+                self.note(f"GET {url} -> {type(exc).__name__}: {exc}")
+                if attempt < retries:
+                    time.sleep(1.5 * (attempt + 1))
+                    continue
+                raise FetchError(f"{url}: {type(exc).__name__}: {exc}") from exc
+
+            self.note(f"GET {url} -> {response.status_code}")
+
+            if response.status_code == 304:
+                return None
+
+            if response.status_code == 200:
+                if conditional and self.state is not None:
+                    self.state.remember_validators(
+                        url,
+                        response.headers.get("etag"),
+                        response.headers.get("last-modified"),
+                    )
+                return Fetched(
+                    url=str(response.url),
+                    status=response.status_code,
+                    content=response.content,
+                    headers=dict(response.headers),
+                )
+
+            if response.status_code in soft:
+                return Fetched(
+                    url=str(response.url),
+                    status=response.status_code,
+                    content=response.content,
+                    headers=dict(response.headers),
+                )
+
+            if response.status_code in RETRY_STATUSES and attempt < retries:
+                time.sleep(1.5 * (attempt + 1))
+                continue
+
+            raise FetchError(f"{url}: HTTP {response.status_code}")
+
+        raise FetchError(f"{url}: {last_error}")

youpdated/models.py

@@ -0,0 +1,63 @@
+"""Core data types shared by sources and renderer"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class Target:
+    """Target watched, normalized from a config entry.
+
+    ``key`` scopes dedupe and labels the item in reports
+    """
+
+    source: str
+    key: str
+    label: str | None = None
+    params: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def display(self) -> str:
+        return self.label or self.key
+
+
+@dataclass(frozen=True)
+class Update:
+    source: str
+    target: str
+    uid: str
+    title: str
+    url: str
+    published: datetime | None = None
+    version: str | None = None
+    body: str | None = None
+    tags: tuple[str, ...] = ()
+
+    @property
+    def dedupe_key(self) -> tuple[str, str, str]:
+        return (self.source, self.target, self.uid)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "target": self.target,
+            "uid": self.uid,
+            "title": self.title,
+            "url": self.url,
+            "published": self.published.isoformat() if self.published else None,
+            "version": self.version,
+            "body": self.body,
+            "tags": list(self.tags),
+        }
+
+
+@dataclass
+class RunError:
+    """A source failure, collected not raised"""
+
+    source: str
+    target: str
+    message: str

youpdated/registry.py

@@ -0,0 +1,49 @@
+"""Source registry
+
+Built-in sources register with the decorator; third-parties can ship a ``youpdated.sources`` entry point and appear without touching this.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .sources.base import Source
+
+_SOURCES: dict[str, "Source"] = {}
+_ENTRY_POINTS_LOADED = False
+
+
+def register(cls):
+    """Class decorator: instantiate and register a source by ``name``."""
+    instance = cls()
+    _SOURCES[cls.name] = instance
+    return cls
+
+
+def _load_entry_points() -> None:
+    global _ENTRY_POINTS_LOADED
+    if _ENTRY_POINTS_LOADED:
+        return
+    _ENTRY_POINTS_LOADED = True
+    from importlib.metadata import entry_points
+
+    for ep in entry_points(group="youpdated.sources"):
+        try:
+            obj = ep.load()
+        except Exception:  # a broken plugin can't stop all
+            continue
+        instance = obj() if isinstance(obj, type) else obj
+        name = getattr(instance, "name", ep.name)
+        _SOURCES.setdefault(name, instance)
+
+
+def all_sources() -> dict[str, "Source"]:
+    from . import sources  # noqa: F401  (imports register built-ins)
+
+    _load_entry_points()
+    return dict(_SOURCES)
+
+
+def get_source(name: str) -> "Source | None":
+    return all_sources().get(name)

youpdated/render/__init__.py

@@ -0,0 +1,5 @@
+"""Output renderers"""
+
+from . import json_out, rss_out, terminal  # noqa: F401
+
+__all__ = ["json_out", "rss_out", "terminal"]

youpdated/render/json_out.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+
+from ..runner import RunResult
+
+
+def render(result: RunResult) -> str:
+    payload = {
+        "generated": datetime.now(timezone.utc).isoformat(),
+        "baseline": result.baseline,
+        "counts": {
+            "targets": len(result.targets),
+            "fetched": result.total_fetched,
+            "new": len(result.updates),
+            "errors": len(result.errors),
+        },
+        # Resolved display names live on targets, expose them
+        "targets": [
+            {"source": t.source, "key": t.key, "label": t.display} for t in result.targets
+        ],
+        "updates": [u.to_dict() for u in result.updates],
+        "errors": [
+            {"source": e.source, "target": e.target, "message": e.message}
+            for e in result.errors
+        ],
+    }
+    return json.dumps(payload, indent=2, ensure_ascii=False)

youpdated/render/rss_out.py

@@ -0,0 +1,58 @@
+"""Aggregated Atom feed"""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime, timezone
+from xml.etree import ElementTree as ET
+
+from ..runner import RunResult
+
+ATOM = "http://www.w3.org/2005/Atom"
+FEED_ID = "urn:youpdated:feed"
+
+
+def _stamp(value: datetime | None) -> str:
+    return (value or datetime.now(timezone.utc)).astimezone(timezone.utc).isoformat()
+
+
+def render(result: RunResult, title: str = "Youpdated") -> str:
+    ET.register_namespace("", ATOM)
+    feed = ET.Element(f"{{{ATOM}}}feed")
+
+    ET.SubElement(feed, f"{{{ATOM}}}id").text = FEED_ID
+    ET.SubElement(feed, f"{{{ATOM}}}title").text = title
+    ET.SubElement(feed, f"{{{ATOM}}}updated").text = _stamp(None)
+    ET.SubElement(feed, f"{{{ATOM}}}generator").text = "youpdated"
+    ET.SubElement(feed, f"{{{ATOM}}}link", {"rel": "self", "href": FEED_ID})
+
+    for update in result.updates:
+        entry = ET.SubElement(feed, f"{{{ATOM}}}entry")
+        digest = hashlib.sha256(
+            "|".join(update.dedupe_key).encode("utf-8")
+        ).hexdigest()[:24]
+        ET.SubElement(entry, f"{{{ATOM}}}id").text = f"urn:youpdated:{digest}"
+
+        label = f"[{update.source}] {update.target}"
+        ET.SubElement(entry, f"{{{ATOM}}}title").text = f"{label} — {update.title}"
+        ET.SubElement(entry, f"{{{ATOM}}}updated").text = _stamp(update.published)
+        if update.published:
+            ET.SubElement(entry, f"{{{ATOM}}}published").text = _stamp(update.published)
+        if update.url:
+            ET.SubElement(entry, f"{{{ATOM}}}link", {"rel": "alternate", "href": update.url})
+
+        author = ET.SubElement(entry, f"{{{ATOM}}}author")
+        ET.SubElement(author, f"{{{ATOM}}}name").text = label
+
+        for tag in update.tags:
+            ET.SubElement(entry, f"{{{ATOM}}}category", {"term": tag})
+
+        summary = update.body or update.title
+        if update.version:
+            summary = f"{update.version} — {summary}"
+        ET.SubElement(entry, f"{{{ATOM}}}summary", {"type": "text"}).text = summary
+
+    ET.indent(feed, space="  ")
+    return '<?xml version="1.0" encoding="utf-8"?>\n' + ET.tostring(
+        feed, encoding="unicode"
+    )

youpdated/render/terminal.py

@@ -0,0 +1,142 @@
+"""default terminal report"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.text import Text
+
+from ..models import Target, Update
+from ..runner import RunResult
+
+SOURCE_STYLE = {
+    "github": "bright_magenta",
+    "npm": "bright_red",
+    "steam": "bright_blue",
+    "itch": "bright_yellow",
+    "youtube": "red",
+    "browser": "bright_green",
+    "feed": "bright_cyan",
+}
+
+
+def relative_age(when: datetime | None) -> str:
+    if when is None:
+        return "undated"
+    delta = datetime.now(timezone.utc) - when
+    seconds = int(delta.total_seconds())
+    if seconds < 0:
+        return "just now"
+    for limit, div, unit in (
+        (90, 1, "s"),
+        (5400, 60, "m"),
+        (172800, 3600, "h"),
+        (1209600, 86400, "d"),
+    ):
+        if seconds < limit:
+            return f"{seconds // div}{unit} ago"
+    return f"{seconds // 604800}w ago"
+
+
+def _labels(targets: list[Target]) -> dict[tuple[str, str], str]:
+    return {(t.source, t.key): t.display for t in targets}
+
+
+def render(result: RunResult, console: Console | None = None, verbose: bool = False) -> None:
+    console = console or Console()
+
+    if result.baseline:
+        console.print(
+            Panel(
+                Text.from_markup(
+                    f"Baseline recorded for [bold]{len(result.targets)}[/] target(s), "
+                    f"[bold]{result.total_fetched}[/] existing item(s).\n"
+                    "Future runs report only what's new. Use [bold]--all[/] to see "
+                    "everything now.",
+                ),
+                title="first run",
+                border_style="cyan",
+            )
+        )
+        _render_errors(result, console)
+        return
+
+    if not result.updates:
+        console.print(
+            Text.from_markup(
+                f"[dim]No new updates across {len(result.targets)} target(s) "
+                f"({result.total_fetched} item(s) checked).[/]"
+            )
+        )
+        _render_errors(result, console)
+        return
+
+    labels = _labels(result.targets)
+    grouped: dict[str, dict[str, list[Update]]] = defaultdict(lambda: defaultdict(list))
+    for update in result.updates:
+        grouped[update.source][update.target].append(update)
+
+    for source in sorted(grouped):
+        style = SOURCE_STYLE.get(source, "cyan")
+        blocks = []
+        for target in sorted(grouped[source]):
+            label = labels.get((source, target), target)
+            heading = Text(label, style="bold")
+            if label != target:
+                heading.append(f"  ({target})", style="dim")
+            blocks.append(heading)
+
+            for update in grouped[source][target]:
+                line = Text("  • ")
+                # GitHub releases title with tag
+                if update.version and update.version != update.title:
+                    line.append(f"{update.version}  ", style="bold green")
+                    line.append(update.title)
+                elif update.version:
+                    line.append(update.version, style="bold green")
+                else:
+                    line.append(update.title)
+                line.append(f"   {relative_age(update.published)}", style="dim")
+                blocks.append(line)
+
+                if update.body and verbose:
+                    body = update.body.replace("\n", " ")
+                    blocks.append(Text(f"    {body}", style="dim italic"))
+                if update.url:
+                    blocks.append(Text(f"    {update.url}", style="dim blue"))
+            blocks.append(Text(""))
+
+        if blocks and not blocks[-1].plain:
+            blocks.pop()
+
+        console.print(
+            Panel(
+                Group(*blocks),
+                title=f"[{style}]{source}[/]",
+                title_align="left",
+                border_style=style,
+            )
+        )
+
+    console.print(
+        Text.from_markup(
+            f"[bold]{len(result.updates)}[/] new update(s) across "
+            f"{len(grouped)} source(s)."
+        )
+    )
+    _render_errors(result, console)
+
+
+def _render_errors(result: RunResult, console: Console) -> None:
+    if not result.errors:
+        return
+    lines = [
+        Text.from_markup(f"[yellow]{e.source}[/] {e.target}: {e.message}")
+        for e in result.errors
+    ]
+    console.print(
+        Panel(Group(*lines), title="[yellow]problems[/]", title_align="left", border_style="yellow")
+    )

youpdated/runner.py

@@ -0,0 +1,134 @@
+"""Turns a config into a set of fetches and updates."""
+
+from __future__ import annotations
+
+import re
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+
+from .config import Config
+from .http import Client
+from .models import RunError, Target, Update
+from .registry import all_sources
+from .state import State
+
+
+@dataclass
+class RunResult:
+    updates: list[Update] = field(default_factory=list)
+    errors: list[RunError] = field(default_factory=list)
+    targets: list[Target] = field(default_factory=list)
+    baseline: bool = False
+    total_fetched: int = 0
+
+    @property
+    def ok(self) -> bool:
+        return not self.errors
+
+
+_DURATION_RE = re.compile(r"^(\d+)\s*([smhdw])$", re.IGNORECASE)
+_UNITS = {"s": "seconds", "m": "minutes", "h": "hours", "d": "days", "w": "weeks"}
+
+
+def parse_since(value: str) -> timedelta:
+    """Parse a window like ``7d``, ``12h``, ``2w``."""
+    match = _DURATION_RE.match(value.strip())
+    if not match:
+        raise ValueError(f"`{value}` is not a duration like 30m, 12h, 7d, 2w")
+    return timedelta(**{_UNITS[match.group(2).lower()]: int(match.group(1))})
+
+
+def build_targets(config: Config) -> tuple[list[Target], list[RunError]]:
+    available = all_sources()
+    targets: list[Target] = []
+    errors: list[RunError] = []
+
+    for name, entries in config.sources.items():
+        source = available.get(name)
+        if source is None:
+            errors.append(
+                RunError(
+                    source=name,
+                    target="-",
+                    message=f"unknown source `{name}`; known sources: "
+                    + ", ".join(sorted(available)),
+                )
+            )
+            continue
+        try:
+            targets.extend(source.targets(entries))
+        except Exception as exc:
+            errors.append(RunError(source=name, target="-", message=str(exc)))
+
+    return targets, errors
+
+
+def run(
+    config: Config,
+    state: State,
+    client: Client,
+    *,
+    only_sources: list[str] | None = None,
+    show_all: bool = False,
+    since: timedelta | None = None,
+    save: bool = True,
+) -> RunResult:
+    targets, errors = build_targets(config)
+    if only_sources:
+        wanted = set(only_sources)
+        targets = [t for t in targets if t.source in wanted]
+        errors = [e for e in errors if e.source in wanted]
+
+    result = RunResult(errors=errors, targets=targets)
+    if not targets:
+        return result
+
+    sources = all_sources()
+    fetched: list[Update] = []
+
+    def work(target: Target) -> tuple[Target, list[Update] | Exception]:
+        try:
+            return target, list(sources[target.source].fetch(target, client))
+        except Exception as exc:  # collected, non fatal
+            return target, exc
+
+    workers = max(1, min(config.privacy.concurrency, len(targets)))
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        for target, outcome in pool.map(work, targets):
+            if isinstance(outcome, Exception):
+                result.errors.append(
+                    RunError(
+                        source=target.source,
+                        target=target.display,
+                        message=f"{type(outcome).__name__}: {outcome}",
+                    )
+                )
+            else:
+                fetched.extend(outcome)
+
+    result.total_fetched = len(fetched)
+
+    # The first run would dump all published. Record baseline instead and report changes
+    first_run = state.last_run() is None
+    if first_run and not show_all:
+        result.baseline = True
+        if save:
+            state.mark_seen(fetched)
+            state.set_last_run()
+        return result
+
+    new = fetched if show_all else state.filter_new(fetched)
+
+    if since is not None:
+        cutoff = datetime.now(timezone.utc) - since
+        new = [u for u in new if u.published is None or u.published >= cutoff]
+
+    new.sort(key=lambda u: u.published or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
+    result.updates = new
+
+    if save:
+        state.mark_seen(fetched)
+        state.set_last_run()
+
+    return result