watchdiff-core 0.1.2__py3-none-any.whl

watchdiff/store/store.py

@@ -0,0 +1,144 @@
+"""
+Store - persists snapshots and diff reports to disk.
+
+Format: one JSON file per watched URL, stored in a configurable directory.
+Each file contains an ordered list of Snapshot dicts (newest last).
+
+Why JSON and not SQLite?
+  → Zero external dependencies for storage.
+  → Easy to inspect, back up, or pipe to other tools.
+  → Fine for typical monitoring workloads (< thousands of entries).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime
+from pathlib import Path
+
+from watchdiff.models import DiffReport, Snapshot
+
+
+class Store:
+    """Filesystem-based snapshot store."""
+
+    def __init__(self, directory: str | Path = ".watchdiff") -> None:
+        self.directory = Path(directory)
+        self.directory.mkdir(parents=True, exist_ok=True)
+
+    # ------------------------------------------------------------------
+    # Snapshots
+    # ------------------------------------------------------------------
+
+    def save_snapshot(self, snapshot: Snapshot) -> None:
+        """Append a snapshot to the history for its URL + target combo."""
+        path    = self._snapshot_path(snapshot.url, snapshot.target)
+        history = self._load_raw(path)
+        history.append(_snapshot_to_dict(snapshot))
+        self._save_raw(path, history)
+
+    def load_latest(self, url: str, target: str | None) -> Snapshot | None:
+        """Return the most recent snapshot, or None if no history exists."""
+        path = self._snapshot_path(url, target)
+        if not path.exists():
+            return None
+        history = self._load_raw(path)
+        if not history:
+            return None
+        return _dict_to_snapshot(history[-1])
+
+    def load_history(
+        self,
+        url: str,
+        target: str | None,
+        limit: int = 50,
+    ) -> list[Snapshot]:
+        """Return up to `limit` most recent snapshots (newest last)."""
+        path = self._snapshot_path(url, target)
+        if not path.exists():
+            return []
+        history = self._load_raw(path)
+        return [_dict_to_snapshot(d) for d in history[-limit:]]
+
+    def clear_history(self, url: str, target: str | None) -> None:
+        """Delete all stored snapshots for a URL + target combo."""
+        path = self._snapshot_path(url, target)
+        if path.exists():
+            path.unlink()
+
+    # ------------------------------------------------------------------
+    # Diff reports
+    # ------------------------------------------------------------------
+
+    def save_report(self, report: DiffReport) -> None:
+        """Persist a DiffReport for audit purposes."""
+        path    = self._report_path(report.url, report.target)
+        reports = self._load_raw(path)
+        reports.append(report.as_dict())
+        self._save_raw(path, reports)
+
+    def load_reports(
+        self,
+        url: str,
+        target: str | None,
+        limit: int = 50,
+    ) -> list[dict]:
+        """Return raw report dicts (newest last)."""
+        path = self._report_path(url, target)
+        if not path.exists():
+            return []
+        return self._load_raw(path)[-limit:]
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _key(self, url: str, target: str | None) -> str:
+        """Stable short key derived from URL + target."""
+        raw = f"{url}::{target or ''}"
+        return hashlib.md5(raw.encode()).hexdigest()[:12]
+
+    def _snapshot_path(self, url: str, target: str | None) -> Path:
+        return self.directory / f"snap_{self._key(url, target)}.json"
+
+    def _report_path(self, url: str, target: str | None) -> Path:
+        return self.directory / f"report_{self._key(url, target)}.json"
+
+    def _load_raw(self, path: Path) -> list:
+        if not path.exists():
+            return []
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            return []
+
+    def _save_raw(self, path: Path, data: list) -> None:
+        path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Serialisation helpers
+# ---------------------------------------------------------------------------
+
+def _snapshot_to_dict(s: Snapshot) -> dict:
+    return {
+        "url": s.url,
+        "target": s.target,
+        "content": s.content,
+        "raw_html": s.raw_html,
+        "captured_at": s.captured_at.isoformat(),
+        "checksum": s.checksum,
+    }
+
+
+def _dict_to_snapshot(d: dict) -> Snapshot:
+    snap = Snapshot(
+        url         = d["url"],
+        target      = d.get("target"),
+        content     = d["content"],
+        raw_html    = d.get("raw_html", ""),
+    )
+    snap.captured_at    = datetime.fromisoformat(d["captured_at"])
+    snap.checksum       = d["checksum"]
+    return snap

watchdiff_core-0.1.2.dist-info/METADATA

@@ -0,0 +1,291 @@
+Metadata-Version: 2.4
+Name: watchdiff-core
+Version: 0.1.2
+Summary: Lightweight web change monitoring library - clean diffs, structured alerts, no AI required.
+Author: WatchDiff Contributors
+License: GPL-3.0-or-later
+License-File: LICENSE
+Keywords: automation,change-detection,diff,monitoring,scraping,web
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.14
+Requires-Dist: beautifulsoup4>=4.14.3
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: lxml>=6.1.0
+Requires-Dist: rich>=15.0.0
+Requires-Dist: typer>=0.25.1
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=1.3.0; extra == 'dev'
+Requires-Dist: pytest-httpx>=0.36.2; extra == 'dev'
+Requires-Dist: pytest>=9.0.3; extra == 'dev'
+Requires-Dist: ruff>=0.15.12; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# WatchDiff
+
+**Lightweight web change monitoring - clean diffs, structured alerts, no AI required.**
+
+WatchDiff watches web pages and tells you **exactly what changed**, in plain language.  
+No noisy HTML diffs. No external services. No AI black boxes.
+
+
+## Why WatchDiff?
+
+Most change detection tools compare raw HTML - which means every minor script reload or ad rotation triggers a false positive. WatchDiff strips the noise first, then diffs only the content that matters.
+
+- **Deterministic** - same input always produces the same output
+- **Human-readable diffs** - "Price changed: $19 → $24", not a wall of HTML
+- **Zero external services** - snapshots stored locally as JSON
+- **Async-ready** - sync and async schedulers included
+- **Configurable** - target any CSS selector, ignore patterns, set webhooks
+
+
+## Install
+
+```bash
+pip install watchdiff-core
+```
+
+Or with [uv](https://github.com/astral-sh/uv):
+
+```bash
+uv add watchdiff-core
+```
+
+
+## Quick start
+
+### Python API
+
+```python
+from watchdiff import WatchDiff
+
+wd = WatchDiff()
+
+wd.watch(
+    "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",
+    target=".price_color",
+    interval=60,
+    label="Book price",
+    on_change=lambda r: print(r.summary()),
+)
+
+wd.start()
+```
+
+### CLI
+
+```bash
+# One-shot check
+watchdiff check https://example.com --target .price
+
+# Continuous monitoring (Ctrl+C to stop)
+watchdiff run https://example.com --target .price --interval 60
+
+# Snapshot history
+watchdiff history https://example.com
+
+# Diff reports
+watchdiff reports https://example.com
+
+# Clear stored data
+watchdiff clear https://example.com
+```
+
+
+## How it works
+
+Every check runs through a fixed pipeline:
+
+```
+Fetcher → Cleaner → Parser → DiffEngine → Store → Notifier
+```
+
+1. **Fetcher** - downloads the page via `httpx` (sync or async)
+2. **Cleaner** - strips scripts, styles, ads, and tracking noise
+3. **Parser** - extracts the target CSS selector (or full body)
+4. **DiffEngine** - compares content using Python's `difflib.SequenceMatcher`
+5. **Store** - persists snapshots and reports as local JSON files
+6. **Notifier** - fires callbacks and/or webhooks on detected changes
+
+
+## API reference
+
+### `WatchDiff`
+
+```python
+wd = WatchDiff(storage_dir=".watchdiff")  # default storage directory
+```
+
+#### `.watch(url, *, ...)`
+
+Register a URL to monitor. All keyword arguments are optional.
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `url` | `str` | - | URL to watch |
+| `target` | `str \| None` | `None` | CSS selector (e.g. `.price`). `None` = full page |
+| `interval` | `int` | `300` | Seconds between checks |
+| `label` | `str \| None` | URL | Human-readable name shown in logs |
+| `headers` | `dict` | `{}` | Extra HTTP headers |
+| `timeout` | `int` | `15` | Request timeout in seconds |
+| `ignore_selectors` | `list[str]` | `[]` | CSS selectors to strip before diffing |
+| `ignore_patterns` | `list[str]` | `[]` | Regex patterns to strip from text |
+| `on_change` | `Callable \| list` | `None` | Callback(s) fired on each change |
+| `webhooks` | `list[str]` | `[]` | Webhook URLs to POST on change |
+| `min_changes` | `int` | `1` | Minimum number of changes to trigger alert |
+
+All methods are **chainable**:
+
+```python
+wd.watch("https://site.com/product", target=".price", interval=300) \
+  .watch("https://site.com/stock",   target=".availability") \
+  .on_change(lambda r: print(r.summary())) \
+  .start()
+```
+
+#### `.on_change(callback)`
+
+Register a global callback called whenever **any** watched URL changes.
+
+```python
+def handle(report):
+    print(report.summary())
+    for change in report.changes:
+        print(change.human())
+
+wd.on_change(handle)
+```
+
+#### `.start(block=True)`
+
+Start the synchronous scheduler. Blocks until `Ctrl+C` by default.  
+Pass `block=False` to run in the background (daemon threads).
+
+#### `await .start_async()`
+
+Async variant - use inside an existing event loop (FastAPI, aiohttp, etc.).
+
+```python
+import asyncio
+asyncio.run(wd.start_async())
+```
+
+#### `.check_once(url)`
+
+Run a single immediate check without starting the scheduler loop.
+
+```python
+report = wd.check_once("https://example.com")
+if report:
+    print(report.summary())
+```
+
+---
+
+### `DiffReport`
+
+```python
+report.url           # str
+report.target        # str | None
+report.label         # str
+report.has_changes   # bool
+report.added         # list[Change]
+report.removed       # list[Change]
+report.modified      # list[Change]
+report.changes       # list[Change]  (all changes)
+report.compared_at   # datetime
+
+report.summary()     # "[Book price] 1 modified - 2024-01-15 10:30:00 UTC"
+report.as_dict()     # JSON-serialisable dict
+```
+
+### `Change`
+
+```python
+change.kind     # ChangeType.ADDED | REMOVED | MODIFIED | UNCHANGED
+change.before   # str | None  - previous value
+change.after    # str | None  - new value
+change.context  # str | None  - surrounding text hint
+
+change.human()  # "[~] Changed: '$19.00' → '$24.00'"
+```
+
+## Webhooks
+
+WatchDiff auto-detects the target service and adapts the payload format:
+
+| Service | Detection | Payload |
+|---|---|---|
+| Discord | `discord.com` in URL | `{"content": "..."}` (2000-char limit) |
+| Slack | `hooks.slack.com` in URL | `{"text": "..."}` |
+| Custom | anything else | full `report.as_dict()` |
+
+```python
+wd.watch(
+    "https://example.com",
+    webhooks=[
+        "https://discord.com/api/webhooks/YOUR_ID/YOUR_TOKEN",
+        "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
+        "https://your-api.com/watchdiff-hook",
+    ],
+)
+```
+
+## Async usage
+
+```python
+import asyncio
+from watchdiff import WatchDiff
+
+async def main():
+    wd = WatchDiff()
+    wd.watch("https://example.com", target="h1", interval=30)
+    wd.on_change(lambda r: print(r.summary()))
+    await wd.start_async()
+
+asyncio.run(main())
+```
+
+## CLI reference
+
+```
+Usage: watchdiff [COMMAND] [OPTIONS]
+
+Commands:
+  run       Start continuous monitoring
+  check     Run a single check and print the result
+  history   Show snapshot history for a URL
+  reports   Show diff reports for a URL
+  clear     Delete all stored data for a URL
+
+Options (shared):
+  --target   -t   CSS selector to watch
+  --storage  -s   Storage directory (default: .watchdiff)
+  --interval -i   Seconds between checks (run only)
+  --limit    -n   Number of entries to show (history/reports)
+  --verbose  -v   Enable debug logging
+  --json         Output raw JSON (check only)
+  --yes      -y   Skip confirmation (clear only)
+```
+
+## Use cases
+
+- **E-commerce** - track product prices and stock availability
+- **News monitoring** - detect article updates or new publications
+- **Documentation** - alert when API docs change
+- **Public APIs** - watch JSON endpoints for schema or value changes
+- **Compliance** - audit changes on public-facing pages over time
+
+
+## License
+
+This project is licensed under the [GNU General Public License v3.0](LICENSE).
+
+You are free to use, study, modify, and distribute this software under the terms of the GPL v3.
+Any derivative work must also be distributed under the same license.

watchdiff_core-0.1.2.dist-info/RECORD

@@ -0,0 +1,24 @@
+watchdiff/__init__.py,sha256=HlBhsOpCcnYJ-f9m0zLg9gAa4TRfCwlVpq2STxSmFU8,62
+watchdiff/core.py,sha256=UMqorOkCIxr7Vwu-k1_1E9M5F3N0x5PpWmlYXOgagXU,7544
+watchdiff/models.py,sha256=Wata-ijRhZprdqnivPtQCBjrRE565VHd7wg6MkHAlY4,5446
+watchdiff/cleaner/__init__.py,sha256=00jw16wtAFiYqza2ntmP0exIbut2kG7EfZTR-omSaVM,69
+watchdiff/cleaner/cleaner.py,sha256=FYwyWb3qRH6skhQG22yjE2hO85ZVEBVQYNZtkT-Kn4Y,4010
+watchdiff/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+watchdiff/cli/main.py,sha256=DSlbeSUevjYhMWkHwpX9rjrTnIXj_tELDf2GWBFKzKo,6960
+watchdiff/diff/__init__.py,sha256=zP2rL-zhlQy117isgd4JHkpKXGqBJl3DR3WAkaW_WvY,71
+watchdiff/diff/engine.py,sha256=AIvqusPjwvSLIb74dZZKiF2D0QM7o2aZzLILYQK-Hng,4539
+watchdiff/fetcher/__init__.py,sha256=q1i_WZbjfnhB5mbOF4DE6DojY-Tzv12gR1oBqHESb_s,125
+watchdiff/fetcher/fetcher.py,sha256=z_B4gwxqfrorsVo6K9dd0HDkQE1IudVqVZEHmfq-RQc,2463
+watchdiff/notifier/__init__.py,sha256=1Q4V121FfjLFdk9XFgfWZZ_jz-QNfYCV1_PomJmKIZ4,73
+watchdiff/notifier/notifier.py,sha256=o7vAFAHEdArBg6BH2brMKtbyTSMjlYQb_jQdfbtU6tI,2580
+watchdiff/parser/__init__.py,sha256=mK2TosyMo0nDfRP6UqUNls5QAelbwbVbMUHqsZQvylU,93
+watchdiff/parser/parser.py,sha256=Ij3QyeiXxXQHwySjRaDu1P8li9xcV4jkzYaQo2hnp8A,2300
+watchdiff/scheduler/__init__.py,sha256=hgbIPr9-gZa_VOdjeHe01Ji7kEBUlfR6PUdSd9Gctfk,119
+watchdiff/scheduler/scheduler.py,sha256=8gQd0n72XH4QDCjP7o9y2sBARvU9zcLFfKdetHSvpMQ,8142
+watchdiff/store/__init__.py,sha256=hr7rwyCqdAeACffzWKC3e8mM16QYZa4iWzRU5PdreJs,61
+watchdiff/store/store.py,sha256=JTt6uQPhJ1rgz6_zmZ-Xyvrit4VSRjMi60rMgIqAOwA,4944
+watchdiff_core-0.1.2.dist-info/METADATA,sha256=eLOgkzboib63dnJkDv1XhetYDpJ7FzU22aFG52cXjdY,8197
+watchdiff_core-0.1.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+watchdiff_core-0.1.2.dist-info/entry_points.txt,sha256=qlFOiDMDG_QyzMD_-uu1YfpGKA_xHm23pIqGpiYZxWk,53
+watchdiff_core-0.1.2.dist-info/licenses/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
+watchdiff_core-0.1.2.dist-info/RECORD,,

watchdiff_core-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

watchdiff_core-0.1.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+watchdiff = watchdiff.cli.main:app