watchdiff-core 0.1.2__py3-none-any.whl

watchdiff/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.core import WatchDiff
+
+__all__ = ["WatchDiff"]

watchdiff/cleaner/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.cleaner.cleaner import Cleaner
+
+__all__ = ["Cleaner"]

watchdiff/cleaner/cleaner.py

@@ -0,0 +1,131 @@
+"""
+Cleaner - strips scripts, styles, ads, and other noise from raw HTML.
+
+The goal: keep only the human-readable content that a user actually sees.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Sequence
+
+from bs4 import BeautifulSoup, Tag
+
+# Tags that are pure noise and should always be stripped
+_NOISE_TAGS: tuple[str, ...] = (
+    "script",
+    "style",
+    "noscript",
+    "iframe",
+    "svg",
+    "canvas",
+    "video",
+    "audio",
+    "link",      # <link rel="stylesheet"> etc.
+    "meta",
+    "head",
+)
+
+# Common ad / tracking class/id fragments
+_AD_PATTERNS: tuple[str, ...] = (
+    "ad",
+    "ads",
+    "advertisement",
+    "banner",
+    "cookie",
+    "gdpr",
+    "popup",
+    "overlay",
+    "modal",
+    "newsletter",
+    "promo",
+    "tracking",
+    "analytics",
+)
+
+
+class Cleaner:
+    """Strips noise from raw HTML and returns clean text or simplified HTML."""
+
+    def __init__(
+        self,
+        extra_selectors: Sequence[str] | None   = None,
+        extra_patterns: Sequence[str] | None    = None,
+    ) -> None:
+        """
+        Args:
+            extra_selectors: Additional CSS selectors to remove (e.g. ".cookie-banner").
+            extra_patterns: Additional regex patterns applied to the final text.
+        """
+        self.extra_selectors: list[str] = list(extra_selectors or [])
+        self.extra_patterns: list[re.Pattern] = [
+            re.compile(p) for p in (extra_patterns or [])
+        ]
+
+    def clean(self, html: str) -> BeautifulSoup:
+        """
+        Parse the HTML and remove all noisy elements.
+
+        Returns:
+            A BeautifulSoup tree of the cleaned document.
+        """
+        soup = BeautifulSoup(html, "lxml")
+
+        # 1. Strip known noise tags
+        for tag in soup.find_all(_NOISE_TAGS):
+            tag.decompose()
+
+        # 2. Strip common ad / tracking containers by class / id heuristic
+        self._strip_ad_containers(soup)
+
+        # 3. Strip user-specified CSS selectors
+        for selector in self.extra_selectors:
+            for el in soup.select(selector):
+                el.decompose()
+
+        return soup
+
+    def clean_to_text(self, html: str) -> str:
+        """Clean the HTML and return normalised plain text."""
+        soup        = self.clean(html)
+        raw_text    = soup.get_text(separator="\n")
+        return self._normalise_text(raw_text)
+
+    def clean_to_html(self, html: str) -> str:
+        """Clean the HTML and return the simplified HTML string."""
+        soup = self.clean(html)
+        return str(soup)
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _strip_ad_containers(self, soup: BeautifulSoup) -> None:
+        """Remove elements whose id/class contains ad-related keywords."""
+        for tag in soup.find_all(True):  # all tags
+            if not isinstance(tag, Tag) or tag.parent is None:
+                continue
+            raw_classes     = tag.get("class") or []
+            classes         = " ".join(
+                str(c) for c in (raw_classes if isinstance(raw_classes, list) else [raw_classes])
+            ).lower()
+            tag_id      = str(tag.get("id") or "").lower()
+            combined    = f"{classes} {tag_id}"
+            if any(pat in combined for pat in _AD_PATTERNS):
+                tag.decompose()
+
+    def _normalise_text(self, text: str) -> str:
+        """Collapse whitespace and apply extra regex strips."""
+        # Collapse blank lines
+        text    = re.sub(r"\n{3,}", "\n\n", text)
+        # Collapse horizontal spaces
+        text    = re.sub(r"[ \t]+", " ", text)
+        # Strip leading/trailing space per line
+        lines   = [line.strip() for line in text.splitlines()]
+        text    = "\n".join(line for line in lines if line)
+
+        # Apply user-defined patterns
+        for pattern in self.extra_patterns:
+            text = pattern.sub("", text)
+
+        return text.strip()

watchdiff/cli/main.py

@@ -0,0 +1,217 @@
+"""
+WatchDiff CLI - monitor URLs from the terminal.
+
+Usage:
+    watchdiff run https://example.com --target .price --interval 60
+    watchdiff check https://example.com --target .price
+    watchdiff history https://example.com --target .price
+    watchdiff reports https://example.com
+    watchdiff clear https://example.com
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+from watchdiff.core import WatchDiff
+from watchdiff.models import DiffReport
+
+app = typer.Typer(
+    name="watchdiff",
+    help="WatchDiff - lightweight web change monitoring.",
+    add_completion=False,
+)
+console = Console()
+
+# ---------------------------------------------------------------------------
+# Shared options
+# ---------------------------------------------------------------------------
+
+_URL_ARG        = typer.Argument(..., help="URL to monitor.")
+_TARGET_OPT     = typer.Option(None, "--target", "-t", help="CSS selector to watch (e.g. .price).")
+_INTERVAL_OPT   = typer.Option(300, "--interval", "-i", help="Seconds between checks.")
+_STORAGE_OPT    = typer.Option(".watchdiff", "--storage", "-s", help="Storage directory.")
+_LIMIT_OPT      = typer.Option(20, "--limit", "-n", help="Number of entries to show.")
+_VERBOSE_OPT    = typer.Option(False, "--verbose", "-v", help="Enable debug logging.")
+
+
+# ---------------------------------------------------------------------------
+# Commands
+# ---------------------------------------------------------------------------
+
+@app.command("run")
+def cmd_run(
+    url: str            = _URL_ARG,
+    target: str | None  = _TARGET_OPT,
+    interval: int       = _INTERVAL_OPT,
+    storage: str        = _STORAGE_OPT,
+    webhook: list[str]  = typer.Option([], "--webhook", "-w", help="Webhook URL (repeatable)."),
+    verbose: bool       = _VERBOSE_OPT,
+) -> None:
+    """Start continuous monitoring of a URL."""
+    _setup_logging(verbose)
+
+    def _print_report(report: DiffReport) -> None:
+        _render_report(report)
+
+    wd = WatchDiff(storage_dir=storage)
+    wd.watch(url, target=target, interval=interval, webhooks=webhook or [])
+    wd.on_change(_print_report)
+
+    console.print(
+        Panel(
+            f"[bold cyan]WatchDiff[/] monitoring [green]{url}[/]\n"
+            f"Target: [yellow]{target or 'full page'}[/]  "
+            f"Interval: [yellow]{interval}s[/]\n"
+            f"Press [bold]Ctrl+C[/] to stop.",
+            title="WatchDiff",
+        )
+    )
+    wd.start(block=True)
+
+
+@app.command("check")
+def cmd_check(
+    url: str                = _URL_ARG,
+    target: str | None      = _TARGET_OPT,
+    storage: str            = _STORAGE_OPT,
+    verbose: bool           = _VERBOSE_OPT,
+    output_json: bool       = typer.Option(False, "--json", help="Output raw JSON."),
+) -> None:
+    """Run a single check and print the result."""
+    _setup_logging(verbose)
+
+    wd = WatchDiff(storage_dir=storage)
+    wd.watch(url, target=target, interval=0)
+
+    report = wd.check_once(url)
+
+    if report is None:
+        console.print("[yellow]First snapshot captured - nothing to compare yet.[/]")
+        raise typer.Exit(0)
+
+    if output_json:
+        typer.echo(json.dumps(report.as_dict(), indent=2, ensure_ascii=False))
+    else:
+        _render_report(report)
+
+
+@app.command("history")
+def cmd_history(
+    url: str                = _URL_ARG,
+    target: str | None      = _TARGET_OPT,
+    storage: str            = _STORAGE_OPT,
+    limit: int              = _LIMIT_OPT,
+) -> None:
+    """Show snapshot history for a URL."""
+    from watchdiff.store import Store
+
+    store       = Store(storage)
+    snapshots   = store.load_history(url, target, limit=limit)
+
+    if not snapshots:
+        console.print("[yellow]No snapshots found.[/]")
+        raise typer.Exit(0)
+
+    table = Table(title=f"Snapshot history - {url}", show_lines=True)
+    table.add_column("Captured at", style="cyan")
+    table.add_column("Checksum", style="dim")
+    table.add_column("Content preview")
+
+    for snap in reversed(snapshots):
+        preview = snap.content[:80].replace("\n", " ")
+        table.add_row(
+            snap.captured_at.strftime("%Y-%m-%d %H:%M:%S"),
+            snap.checksum[:8],
+            preview,
+        )
+
+    console.print(table)
+
+
+@app.command("reports")
+def cmd_reports(
+    url: str            = _URL_ARG,
+    target: str | None  = _TARGET_OPT,
+    storage: str        = _STORAGE_OPT,
+    limit: int          = _LIMIT_OPT,
+) -> None:
+    """Show diff reports for a URL."""
+    from watchdiff.store import Store
+
+    store       = Store(storage)
+    reports     = store.load_reports(url, target, limit=limit)
+
+    if not reports:
+        console.print("[yellow]No reports found.[/]")
+        raise typer.Exit(0)
+
+    for r in reversed(reports):
+        changes = r.get("changes", [])
+        console.print(
+            Panel(
+                "\n".join(
+                    f"[{c['kind']}] {c.get('before', '')} → {c.get('after', '')}"
+                    for c in changes[:10]
+                ) or "[dim]No changes[/]",
+                title=r['compared_at'],
+                subtitle=f"{len(changes)} change(s)",
+            )
+        )
+
+
+@app.command("clear")
+def cmd_clear(
+    url: str                = _URL_ARG,
+    target: str | None      = _TARGET_OPT,
+    storage: str            = _STORAGE_OPT,
+    yes: bool               = typer.Option(False, "--yes", "-y", help="Skip confirmation."),
+) -> None:
+    """Delete all stored snapshots and reports for a URL."""
+    if not yes:
+        confirmed = typer.confirm(f"Delete all history for {url!r}?")
+        if not confirmed:
+            raise typer.Abort()
+
+    from watchdiff.store import Store
+
+    store = Store(storage)
+    store.clear_history(url, target)
+    console.print(f"[green]Done.[/] History cleared for {url}")
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _render_report(report: DiffReport) -> None:
+    if not report.has_changes:
+        console.print(f"[green]OK[/] {report.summary()}")
+        return
+
+    lines = [f"[bold]{report.summary()}[/]\n"]
+    for change in report.changes:
+        match change.kind.value:
+            case "added":
+                lines.append(f"  [green][+][/] {change.after}")
+            case "removed":
+                lines.append(f"  [red][-][/] {change.before}")
+            case "modified":
+                lines.append(f"  [yellow][~][/] {change.before} [dim]→[/] {change.after}")
+
+    console.print(Panel("\n".join(lines), title="Changes detected", border_style="yellow"))
+
+
+def _setup_logging(verbose: bool) -> None:
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
+        datefmt="%H:%M:%S",
+    )

watchdiff/core.py

@@ -0,0 +1,202 @@
+"""
+WatchDiff - high-level public facade.
+
+This is what 99% of users will interact with.
+
+Usage:
+    from watchdiff import WatchDiff
+
+    wd = WatchDiff()
+    wd.watch("https://example.com/product", target=".price", interval=300)
+    wd.on_change(lambda report: print(report.summary()))
+    wd.start()
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Callable
+
+from watchdiff.models import AlertConfig, DiffReport, WatchConfig
+from watchdiff.scheduler import AsyncScheduler, SyncScheduler
+from watchdiff.store import Store
+
+logger = logging.getLogger(__name__)
+
+
+class WatchDiff:
+    """
+    Main entry point for WatchDiff.
+
+    All methods are chainable.
+    """
+
+    def __init__(self, storage_dir: str | Path = ".watchdiff") -> None:
+        self._store = Store(storage_dir)
+        self._configs: list[WatchConfig] = []
+        self._global_callbacks: list[Callable[[DiffReport], None]] = []
+
+    # ------------------------------------------------------------------
+    # Configuration API
+    # ------------------------------------------------------------------
+
+    def watch(
+        self,
+        url: str,
+        *,
+        target: str | None                                                                  = None,
+        interval: int                                                                       = 300,
+        label: str | None                                                                   = None,
+        headers: dict[str, str] | None                                                      = None,
+        timeout: int                                                                        = 15,
+        ignore_selectors: list[str] | None                                                  = None,
+        ignore_patterns: list[str] | None                                                   = None,
+        on_change: Callable[[DiffReport], None] | list[Callable[[DiffReport], None]] | None = None,
+        webhooks: list[str] | None                                                          = None,
+        min_changes: int                                                                    = 1,
+    ) -> "WatchDiff":
+        """
+        Register a URL to monitor.
+
+        Args:
+            url:              URL to watch.
+            target:           CSS selector to narrow monitoring (e.g. ".price").
+                              If None, the full page body is monitored.
+            interval:         Seconds between checks (default 300 = 5 min).
+            label:            Human-readable name shown in logs and reports.
+            headers:          Extra HTTP headers for this URL.
+            timeout:          HTTP request timeout in seconds.
+            ignore_selectors: CSS selectors to strip before diffing.
+            ignore_patterns:  Regex patterns to strip from text before diffing.
+            on_change:        Callback(s) called with a DiffReport on each change.
+            webhooks:         Webhook URLs (Discord/Slack/custom) to POST on change.
+            min_changes:      Alert only if at least N changes are detected.
+
+        Returns:
+            self (chainable)
+        """
+        callbacks = []
+        if on_change:
+            callbacks = on_change if isinstance(on_change, list) else [on_change]
+
+        alert = AlertConfig(
+            on_change           = callbacks,
+            webhooks            = webhooks or [],
+            min_changes         = min_changes,
+        ) if (callbacks or webhooks) else None
+
+        config = WatchConfig(
+            url                 = url,
+            target              = target,
+            interval            = interval,
+            label               = label,
+            headers             = headers or {},
+            timeout             = timeout,
+            ignore_selectors    = ignore_selectors or [],
+            ignore_patterns     = ignore_patterns or [],
+            alert               = alert,
+        )
+        self._configs.append(config)
+        return self
+
+    def on_change(self, callback: Callable[[DiffReport], None]) -> "WatchDiff":
+        """
+        Register a global callback called whenever ANY watched URL changes.
+
+        Args:
+            callback: Function receiving a DiffReport.
+
+        Returns:
+            self (chainable)
+        """
+        self._global_callbacks.append(callback)
+        return self
+
+    # ------------------------------------------------------------------
+    # Run API
+    # ------------------------------------------------------------------
+
+    def start(self, block: bool = True) -> None:
+        """
+        Start the synchronous scheduler.
+
+        Args:
+            block: If True (default), blocks until Ctrl+C.
+                   If False, returns immediately (threads run as daemons).
+        """
+        if not self._configs:
+            logger.warning("No URLs registered. Call .watch() first.")
+            return
+
+        scheduler = SyncScheduler(self._store)
+        for cb in self._global_callbacks:
+            scheduler.add_global_callback(cb)
+
+        scheduler.start(self._configs, block=block)
+
+    async def start_async(self) -> None:
+        """
+        Start the async scheduler.
+
+        Use with `asyncio.run(wd.start_async())` or inside an existing event loop.
+        """
+        if not self._configs:
+            logger.warning("No URLs registered. Call .watch() first.")
+            return
+
+        scheduler = AsyncScheduler(self._store)
+        for cb in self._global_callbacks:
+            scheduler.add_global_callback(cb)
+
+        await scheduler.start(self._configs)
+
+    def check_once(self, url: str) -> DiffReport | None:
+        """
+        Run a single immediate check for a registered URL.
+
+        Useful for testing or on-demand checks without the scheduler loop.
+
+        Args:
+            url: URL to check (must have been registered via .watch()).
+
+        Returns:
+            DiffReport or None if it's the first check.
+
+        Raises:
+            ValueError: if the URL is not registered.
+        """
+        config      = self._find_config(url)
+        scheduler   = SyncScheduler(self._store)
+        for cb in self._global_callbacks:
+            scheduler.add_global_callback(cb)
+        return scheduler.check_once(config)
+
+    # ------------------------------------------------------------------
+    # History / audit API
+    # ------------------------------------------------------------------
+
+    def history(self, url: str, limit: int = 20) -> list:
+        """Return stored snapshots for a URL."""
+        config = self._find_config(url)
+        return self._store.load_history(config.url, config.target, limit=limit)
+
+    def reports(self, url: str, limit: int = 20) -> list[dict]:
+        """Return stored diff reports for a URL."""
+        config = self._find_config(url)
+        return self._store.load_reports(config.url, config.target, limit=limit)
+
+    def clear(self, url: str) -> None:
+        """Delete all stored snapshots and reports for a URL."""
+        config = self._find_config(url)
+        self._store.clear_history(config.url, config.target)
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _find_config(self, url: str) -> WatchConfig:
+        for config in self._configs:
+            if config.url == url:
+                return config
+        raise ValueError(f"URL not registered: {url!r}. Call .watch() first.")

watchdiff/diff/__init__.py

@@ -0,0 +1,3 @@
+from watchdiff.diff.engine import DiffEngine
+
+__all__ = ["DiffEngine"]

watchdiff/diff/engine.py

@@ -0,0 +1,131 @@
+"""
+Diff Engine - compares two Snapshots and produces a human-readable DiffReport.
+
+Strategy:
+  1. Split content into logical lines / sentences.
+  2. Use Python's difflib.SequenceMatcher for reliable LCS-based diffing.
+  3. Map opcodes to Change objects (added / removed / modified).
+
+This produces clean, minimal diffs without HTML noise.
+"""
+
+from __future__ import annotations
+
+import difflib
+
+from watchdiff.models import Change, ChangeType, DiffReport, Snapshot, WatchConfig
+
+
+class DiffEngine:
+    """Compares two snapshots and returns a DiffReport."""
+
+    def compare(
+        self,
+        before: Snapshot,
+        after: Snapshot,
+        config: WatchConfig,
+    ) -> DiffReport:
+        """
+        Compare two snapshots.
+
+        Returns:
+            DiffReport with a list of Change objects.
+        """
+        changes: list[Change] = []
+
+        if before.is_identical_to(after):
+            return DiffReport(
+                url         = config.url,
+                target      = config.target,
+                label       = config.label or config.url,
+                before      = before,
+                after       = after,
+                changes     = [],
+            )
+
+        before_lines    = _split_content(before.content)
+        after_lines     = _split_content(after.content)
+
+        matcher = difflib.SequenceMatcher(
+            isjunk      = None,
+            a           = before_lines,
+            b           = after_lines,
+            autojunk    = False,
+        )
+
+        for opcode, a0, a1, b0, b1 in matcher.get_opcodes():
+            match opcode:
+                case "equal":
+                    pass  # no change
+
+                case "insert":
+                    for line in after_lines[b0:b1]:
+                        if line.strip():
+                            changes.append(
+                                Change(
+                                    kind    = ChangeType.ADDED,
+                                    after   = line.strip(),
+                                    context = _context(after_lines, b0),
+                                )
+                            )
+
+                case "delete":
+                    for line in before_lines[a0:a1]:
+                        if line.strip():
+                            changes.append(
+                                Change(
+                                    kind    = ChangeType.REMOVED,
+                                    before  = line.strip(),
+                                    context = _context(before_lines, a0),
+                                )
+                            )
+
+                case "replace":
+                    old_block = " ".join(before_lines[a0:a1]).strip()
+                    new_block = " ".join(after_lines[b0:b1]).strip()
+                    if old_block and new_block:
+                        changes.append(
+                            Change(
+                                kind        = ChangeType.MODIFIED,
+                                before      = old_block,
+                                after       = new_block,
+                                context     = _context(after_lines, b0),
+                            )
+                        )
+                    elif old_block:
+                        changes.append(
+                            Change(kind=ChangeType.REMOVED, before=old_block)
+                        )
+                    elif new_block:
+                        changes.append(
+                            Change(kind=ChangeType.ADDED, after=new_block)
+                        )
+
+        return DiffReport(
+            url         = config.url,
+            target      = config.target,
+            label       = config.label or config.url,
+            before      = before,
+            after       = after,
+            changes     = changes,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _split_content(text: str) -> list[str]:
+    """Split text into meaningful units for diffing.
+
+    Splits on newlines; empty lines are preserved as markers.
+    """
+    return text.splitlines()
+
+
+def _context(lines: list[str], index: int, window: int = 1) -> str | None:
+    """Return a short surrounding context for a change."""
+    start   = max(0, index - window)
+    end     = min(len(lines), index + window + 1)
+    ctx     = " | ".join(line.strip() for line in lines[start:end] if line.strip())
+    return ctx or None