watchdiff-core 0.1.2__py3-none-any.whl

watchdiff/fetcher/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.fetcher.fetcher import AsyncFetcher, FetchError, Fetcher
+
+__all__ = ["AsyncFetcher", "FetchError", "Fetcher"]

watchdiff/fetcher/fetcher.py

@@ -0,0 +1,84 @@
+"""
+Fetcher - downloads HTML from URLs.
+
+Two implementations:
+  - Fetcher: synchronous, uses httpx.Client
+  - AsyncFetcher: async, uses httpx.AsyncClient
+"""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from watchdiff.models import WatchConfig
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (compatible; WatchDiff/1.0; "
+        "+https://github.com/watchdiff/watchdiff-py)"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+
+class FetchError(Exception):
+    """Raised when an HTTP request fails."""
+
+
+class Fetcher:
+    """Synchronous HTTP fetcher."""
+
+    def fetch(self, config: WatchConfig) -> str:
+        """
+        Download the HTML for the given config.
+
+        Returns:
+            Raw HTML string.
+
+        Raises:
+            FetchError: on network error or non-2xx response.
+        """
+        headers = {**_DEFAULT_HEADERS, **config.headers}
+        try:
+            with httpx.Client(timeout=config.timeout, follow_redirects=True) as client:
+                resp = client.get(config.url, headers=headers)
+                resp.raise_for_status()
+                return resp.text
+        except httpx.HTTPStatusError as exc:
+            raise FetchError(
+                f"HTTP {exc.response.status_code} for {config.url}"
+            ) from exc
+        except httpx.RequestError as exc:
+            raise FetchError(f"Request failed for {config.url}: {exc}") from exc
+
+
+class AsyncFetcher:
+    """Asynchronous HTTP fetcher."""
+
+    async def fetch(self, config: WatchConfig) -> str:
+        """
+        Download the HTML for the given config (async).
+
+        Returns:
+            Raw HTML string.
+
+        Raises:
+            FetchError: on network error or non-2xx response.
+        """
+        headers = {**_DEFAULT_HEADERS, **config.headers}
+        try:
+            async with httpx.AsyncClient(timeout=config.timeout, follow_redirects=True) as client:
+                resp = await client.get(config.url, headers=headers)
+                resp.raise_for_status()
+                return resp.text
+        except httpx.HTTPStatusError as exc:
+            raise FetchError(
+                f"HTTP {exc.response.status_code} for {config.url}"
+            ) from exc
+        except httpx.RequestError as exc:
+            raise FetchError(f"Request failed for {config.url}: {exc}") from exc

watchdiff/models.py

@@ -0,0 +1,163 @@
+"""
+WatchDiff models - shared data structures used across all modules.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Callable
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+class ChangeType(str, Enum):
+    ADDED               = "added"
+    REMOVED             = "removed"
+    MODIFIED            = "modified"
+    UNCHANGED           = "unchanged"
+
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+@dataclass
+class WatchConfig:
+    """Configuration for a single watched URL."""
+
+    url: str
+    target: str | None              = None          # CSS selector - None means full page
+    interval: int                   = 300                # seconds between checks
+    label: str | None               = None           # human-readable name for this watch
+    headers: dict[str, str]         = field(default_factory=dict)
+    timeout: int                    = 15                  # HTTP timeout in seconds
+    ignore_selectors: list[str]     = field(default_factory=list)  # CSS to strip
+    ignore_patterns: list[str]      = field(default_factory=list)   # regex patterns to strip
+    alert: AlertConfig | None       = None
+
+    def __post_init__(self) -> None:
+        if not self.label:
+            self.label = self.url
+
+
+@dataclass
+class AlertConfig:
+    """Alert configuration attached to a WatchConfig."""
+
+    on_change: list[Callable[[DiffReport], Any]] = field(default_factory=list)
+    webhooks: list[str]     = field(default_factory=list)   # Discord / Slack / custom URLs
+    min_changes: int        = 1               # trigger only if >= N changes
+
+
+# ---------------------------------------------------------------------------
+# Snapshots
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Snapshot:
+    """A single captured version of a watched page/selector."""
+
+    url: str
+    target: str | None
+    content: str                       # cleaned text content
+    raw_html: str                      # original HTML of the extracted zone
+    captured_at: datetime   = field(default_factory=lambda: datetime.now(timezone.utc))
+    checksum: str           = ""
+
+    def __post_init__(self) -> None:
+        self.checksum = hashlib.sha256(self.content.encode()).hexdigest()
+
+    def is_identical_to(self, other: "Snapshot") -> bool:
+        return self.checksum == other.checksum
+
+
+# ---------------------------------------------------------------------------
+# Diff
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Change:
+    """A single atomic change between two snapshots."""
+
+    kind: ChangeType
+    before: str | None      = None          # old value / text
+    after: str | None       = None           # new value / text
+    context: str | None     = None         # surrounding text hint
+
+    def human(self) -> str:
+        """Return a human-readable one-liner."""
+        match self.kind:
+            case ChangeType.ADDED:
+                return f"[+] Added: {self.after!r}"
+            case ChangeType.REMOVED:
+                return f"[-] Removed: {self.before!r}"
+            case ChangeType.MODIFIED:
+                return f"[~] Changed: {self.before!r} → {self.after!r}"
+            case _:
+                return "[=] Unchanged"
+
+    def __str__(self) -> str:
+        return self.human()
+
+
+@dataclass
+class DiffReport:
+    """Result of comparing two snapshots."""
+
+    url: str
+    target: str | None
+    label: str
+    before: Snapshot
+    after: Snapshot
+    changes: list[Change] = field(default_factory=list)
+    compared_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.changes)
+
+    @property
+    def added(self) -> list[Change]:
+        return [c for c in self.changes if c.kind == ChangeType.ADDED]
+
+    @property
+    def removed(self) -> list[Change]:
+        return [c for c in self.changes if c.kind == ChangeType.REMOVED]
+
+    @property
+    def modified(self) -> list[Change]:
+        return [c for c in self.changes if c.kind == ChangeType.MODIFIED]
+
+    def summary(self) -> str:
+        if not self.has_changes:
+            return f"[{self.label}] No changes detected."
+        parts = []
+        if self.added:
+            parts.append(f"{len(self.added)} added")
+        if self.removed:
+            parts.append(f"{len(self.removed)} removed")
+        if self.modified:
+            parts.append(f"{len(self.modified)} modified")
+        return f"[{self.label}] {', '.join(parts)} - {self.compared_at.strftime('%Y-%m-%d %H:%M:%S')} UTC"
+
+    def as_dict(self) -> dict:
+        return {
+            "url": self.url,
+            "target": self.target,
+            "label": self.label,
+            "compared_at": self.compared_at.isoformat(),
+            "changes": [
+                {
+                    "kind": c.kind.value,
+                    "before": c.before,
+                    "after": c.after,
+                    "context": c.context,
+                }
+                for c in self.changes
+            ],
+        }

watchdiff/notifier/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.notifier.notifier import Notifier
+
+__all__ = ["Notifier"]

watchdiff/notifier/notifier.py

@@ -0,0 +1,81 @@
+"""
+Notifier - dispatches alerts when a DiffReport has changes.
+
+Supported channels:
+  - Python callbacks (synchronous)
+  - Webhooks (Discord, Slack, custom - via httpx POST)
+
+Adding a new channel is as simple as adding a method and calling it
+inside `notify()`.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import httpx
+
+from watchdiff.models import AlertConfig, DiffReport
+
+logger = logging.getLogger(__name__)
+
+
+class Notifier:
+    """Sends alerts for a DiffReport based on an AlertConfig."""
+
+    def notify(self, report: DiffReport, alert: AlertConfig) -> None:
+        """
+        Dispatch all configured alert channels.
+
+        Silently logs errors - a broken webhook must not crash the watcher.
+        """
+        if not report.has_changes:
+            return
+        if len(report.changes) < alert.min_changes:
+            return
+
+        # 1. Python callbacks
+        for callback in alert.on_change:
+            try:
+                callback(report)
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("Alert callback raised an error: %s", exc)
+
+        # 2. Webhooks
+        for url in alert.webhooks:
+            self._send_webhook(url, report)
+
+    # ------------------------------------------------------------------
+    # Channels
+    # ------------------------------------------------------------------
+
+    def _send_webhook(self, url: str, report: DiffReport) -> None:
+        """POST a JSON payload to a webhook URL.
+
+        Automatically adapts payload format for Discord vs Slack vs generic.
+        """
+        payload = self._build_payload(url, report)
+        try:
+            with httpx.Client(timeout=10) as client:
+                resp = client.post(url, json=payload)
+                if not resp.is_success:
+                    logger.warning(
+                        "Webhook %s returned %d", url, resp.status_code
+                    )
+        except httpx.RequestError as exc:
+            logger.warning("Webhook request error (%s): %s", url, exc)
+
+    def _build_payload(self, url: str, report: DiffReport) -> dict:
+        """Build a webhook payload adapted to the target service."""
+        summary         = report.summary()
+        change_lines    = "\n".join(c.human() for c in report.changes[:20])
+        text            = f"{summary}\n\n{change_lines}"
+
+        if "discord.com" in url:
+            return {"content": text[:2000]}  # Discord 2000-char limit
+
+        if "hooks.slack.com" in url:
+            return {"text": text[:3000]}
+
+        # Generic JSON
+        return report.as_dict()

watchdiff/parser/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.parser.parser import Parser, ParserError
+
+__all__ = ["Parser", "ParserError"]

watchdiff/parser/parser.py

@@ -0,0 +1,75 @@
+"""
+Parser - extracts the content zone that the user actually wants to monitor.
+
+If a CSS selector (target) is provided, only that zone is extracted.
+Otherwise the full body text is used.
+"""
+
+from __future__ import annotations
+
+from bs4 import BeautifulSoup
+
+from watchdiff.models import Snapshot, WatchConfig
+
+
+class Parser:
+    """Extracts the monitored zone from a cleaned BeautifulSoup tree."""
+
+    def extract(
+        self,
+        soup: BeautifulSoup,
+        config: WatchConfig,
+    ) -> Snapshot:
+        """
+        Extract a Snapshot from a cleaned BeautifulSoup document.
+
+        Args:
+            soup:   Cleaned BeautifulSoup tree (output of Cleaner.clean).
+            config: The WatchConfig describing what to monitor.
+
+        Returns:
+            A Snapshot with content and raw_html fields populated.
+
+        Raises:
+            ParserError: if the target selector matched nothing.
+        """
+        if config.target:
+            elements = soup.select(config.target)
+            if not elements:
+                raise ParserError(
+                    f"Selector {config.target!r} matched nothing on {config.url}"
+                )
+            # Combine all matched elements
+            raw_html    = "\n".join(str(el) for el in elements)
+            content     = "\n".join(
+                el.get_text(separator="\n").strip() for el in elements
+            )
+        else:
+            body        = soup.find("body") or soup
+            raw_html    = str(body)
+            content     = body.get_text(separator="\n").strip()
+
+        # Final normalisation
+        content = _collapse_whitespace(content)
+
+        return Snapshot(
+            url         = config.url,
+            target      = config.target,
+            content     = content,
+            raw_html    = raw_html,
+        )
+
+
+class ParserError(Exception):
+    """Raised when the target selector cannot be found."""
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _collapse_whitespace(text: str) -> str:
+    import re
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return "\n".join(line.strip() for line in text.splitlines() if line.strip())

watchdiff/scheduler/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.scheduler.scheduler import AsyncScheduler, SyncScheduler
+
+__all__ = ["AsyncScheduler", "SyncScheduler"]

watchdiff/scheduler/scheduler.py

@@ -0,0 +1,223 @@
+"""
+Scheduler - drives the periodic check loop.
+
+Two implementations:
+  - SyncScheduler: blocking loop using threading.Timer, suitable for scripts.
+  - AsyncScheduler: asyncio-based loop for async applications.
+
+Both call the same internal pipeline:
+  Fetcher → Cleaner → Parser → DiffEngine → Store → Notifier
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+import time
+from typing import Callable
+
+from watchdiff.cleaner import Cleaner
+from watchdiff.diff import DiffEngine
+from watchdiff.fetcher import AsyncFetcher, FetchError, Fetcher
+from watchdiff.models import DiffReport, WatchConfig
+from watchdiff.notifier import Notifier
+from watchdiff.parser import Parser, ParserError
+from watchdiff.store import Store
+
+logger = logging.getLogger(__name__)
+
+
+class SyncScheduler:
+    """
+    Blocking multi-threaded scheduler.
+
+    Each WatchConfig runs in its own daemon thread with an independent
+    sleep interval, so different URLs can have different cadences.
+    """
+
+    def __init__(self, store: Store) -> None:
+        self._store                                                 = store
+        self._fetcher                                               = Fetcher()
+        self._parser                                                = Parser()
+        self._engine                                                = DiffEngine()
+        self._notifier                                              = Notifier()
+        self._on_diff_callbacks: list[Callable[[DiffReport], None]] = []
+        self._threads: list[threading.Thread]                       = []
+        self._stop_events: list[threading.Event]                    = []
+
+    def add_global_callback(self, callback: Callable[[DiffReport], None]) -> None:
+        """Register a callback called for every DiffReport (regardless of config)."""
+        self._on_diff_callbacks.append(callback)
+
+    def start(self, configs: list[WatchConfig], block: bool = True) -> None:
+        """
+        Start monitoring all configs.
+
+        Args:
+            configs: list of WatchConfig objects.
+            block:   If True, blocks until KeyboardInterrupt. If False,
+                     returns immediately (threads run as daemons).
+        """
+        for config in configs:
+            stop_event = threading.Event()
+            self._stop_events.append(stop_event)
+            thread = threading.Thread(
+                target      = self._run_loop,
+                args        = (config, stop_event),
+                daemon      = True,
+                name        = f"watchdiff-{config.label}",
+            )
+            self._threads.append(thread)
+            thread.start()
+            logger.info("Started watcher for %s (interval=%ds)", config.label, config.interval)
+
+        if block:
+            try:
+                while True:
+                    time.sleep(1)
+            except KeyboardInterrupt:
+                self.stop()
+
+    def stop(self) -> None:
+        """Signal all watcher threads to stop."""
+        for event in self._stop_events:
+            event.set()
+        logger.info("Stopping all watchers…")
+
+    def check_once(self, config: WatchConfig) -> DiffReport | None:
+        """Run a single check for a config and return the DiffReport."""
+        return self._check(config)
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _run_loop(self, config: WatchConfig, stop_event: threading.Event) -> None:
+        """Thread target: check, sleep, repeat."""
+        while not stop_event.is_set():
+            self._check(config)
+            stop_event.wait(timeout=config.interval)
+
+    def _check(self, config: WatchConfig) -> DiffReport | None:
+        try:
+            html = self._fetcher.fetch(config)
+        except FetchError as exc:
+            logger.error("[%s] Fetch failed: %s", config.label, exc)
+            return None
+
+        cleaner = Cleaner(
+            extra_selectors=config.ignore_selectors,
+            extra_patterns=config.ignore_patterns,
+        )
+        soup = cleaner.clean(html)
+
+        try:
+            snapshot = self._parser.extract(soup, config)
+        except ParserError as exc:
+            logger.error("[%s] Parse failed: %s", config.label, exc)
+            return None
+
+        previous = self._store.load_latest(config.url, config.target)
+
+        if previous is None:
+            # First run - just save and return
+            self._store.save_snapshot(snapshot)
+            logger.info("[%s] First snapshot captured.", config.label)
+            return None
+
+        report = self._engine.compare(previous, snapshot, config)
+        self._store.save_snapshot(snapshot)
+
+        if report.has_changes:
+            self._store.save_report(report)
+            logger.info("[%s] %s", config.label, report.summary())
+
+            # Global callbacks
+            for cb in self._on_diff_callbacks:
+                try:
+                    cb(report)
+                except Exception as exc:  # noqa: BLE001
+                    logger.warning("Global callback error: %s", exc)
+
+            # Per-config alert
+            if config.alert:
+                self._notifier.notify(report, config.alert)
+        else:
+            logger.debug("[%s] No changes.", config.label)
+
+        return report
+
+
+class AsyncScheduler:
+    """
+    Asyncio-based scheduler.
+
+    Use this inside async applications (FastAPI, aiohttp, etc.).
+    """
+
+    def __init__(self, store: Store) -> None:
+        self._store                                                 = store
+        self._fetcher                                               = AsyncFetcher()
+        self._parser                                                = Parser()
+        self._engine                                                = DiffEngine()
+        self._notifier                                              = Notifier()
+        self._on_diff_callbacks: list[Callable[[DiffReport], None]] = []
+
+    def add_global_callback(self, callback: Callable[[DiffReport], None]) -> None:
+        self._on_diff_callbacks.append(callback)
+
+    async def start(self, configs: list[WatchConfig]) -> None:
+        """Start all watchers as concurrent asyncio tasks."""
+        tasks = [asyncio.create_task(self._run_loop(cfg)) for cfg in configs]
+        await asyncio.gather(*tasks)
+
+    async def check_once(self, config: WatchConfig) -> DiffReport | None:
+        """Single async check."""
+        return await self._check(config)
+
+    async def _run_loop(self, config: WatchConfig) -> None:
+        while True:
+            await self._check(config)
+            await asyncio.sleep(config.interval)
+
+    async def _check(self, config: WatchConfig) -> DiffReport | None:
+        try:
+            html = await self._fetcher.fetch(config)
+        except FetchError as exc:
+            logger.error("[%s] Fetch failed: %s", config.label, exc)
+            return None
+
+        cleaner = Cleaner(
+            extra_selectors=config.ignore_selectors,
+            extra_patterns=config.ignore_patterns,
+        )
+        soup = cleaner.clean(html)
+
+        try:
+            snapshot = self._parser.extract(soup, config)
+        except ParserError as exc:
+            logger.error("[%s] Parse failed: %s", config.label, exc)
+            return None
+
+        previous = self._store.load_latest(config.url, config.target)
+
+        if previous is None:
+            self._store.save_snapshot(snapshot)
+            logger.info("[%s] First snapshot captured.", config.label)
+            return None
+
+        report = self._engine.compare(previous, snapshot, config)
+        self._store.save_snapshot(snapshot)
+
+        if report.has_changes:
+            self._store.save_report(report)
+            for cb in self._on_diff_callbacks:
+                try:
+                    cb(report)
+                except Exception as exc:  # noqa: BLE001
+                    logger.warning("Global callback error: %s", exc)
+            if config.alert:
+                self._notifier.notify(report, config.alert)
+
+        return report

watchdiff/store/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.store.store import Store
+
+__all__ = ["Store"]