diffgrab 0.1.0__py3-none-any.whl

diffgrab/mcp_server.py

@@ -0,0 +1,138 @@
+"""MCP server for diffgrab — 5 tools for web page change tracking."""
+
+from __future__ import annotations
+
+import json
+import sys
+
+try:
+    from fastmcp import FastMCP
+except ImportError:
+    print("MCP dependencies not installed. Run: pip install 'diffgrab[mcp]'", file=sys.stderr)
+    sys.exit(1)
+
+from diffgrab.differ import DiffResult
+from diffgrab.tracker import DiffTracker
+
+mcp = FastMCP("diffgrab", instructions="Web page change tracking with structured diffs.")
+
+# Shared tracker instance (lazily initialized)
+_tracker: DiffTracker | None = None
+
+
+def _get_tracker() -> DiffTracker:
+    global _tracker
+    if _tracker is None:
+        _tracker = DiffTracker()
+    return _tracker
+
+
+def _diff_result_to_json(result: DiffResult) -> str:
+    """Serialize a DiffResult to JSON string."""
+    return json.dumps(
+        {
+            "url": result.url,
+            "changed": result.changed,
+            "added_lines": result.added_lines,
+            "removed_lines": result.removed_lines,
+            "changed_sections": result.changed_sections,
+            "unified_diff": result.unified_diff[:2000] if result.unified_diff else "",
+            "summary": result.summary,
+            "before_snapshot_id": result.before_snapshot_id,
+            "after_snapshot_id": result.after_snapshot_id,
+            "before_timestamp": result.before_timestamp,
+            "after_timestamp": result.after_timestamp,
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+
+
+@mcp.tool()
+async def track_url(url: str, interval_hours: int = 24) -> str:
+    """Register a URL for change tracking. Takes an initial snapshot.
+
+    Args:
+        url: The URL to track for changes.
+        interval_hours: How often to check for changes (default: 24 hours).
+
+    Returns:
+        Status message confirming tracking registration.
+    """
+    tracker = _get_tracker()
+    return await tracker.track(url, interval_hours)
+
+
+@mcp.tool()
+async def check_changes(url: str | None = None) -> str:
+    """Check tracked URLs for changes. Compares current content with last snapshot.
+
+    Args:
+        url: Specific URL to check, or None to check all tracked URLs.
+
+    Returns:
+        JSON array of change results for each checked URL.
+    """
+    tracker = _get_tracker()
+    results = await tracker.check(url)
+    return json.dumps(
+        [json.loads(_diff_result_to_json(r)) for r in results],
+        ensure_ascii=False,
+        indent=2,
+    )
+
+
+@mcp.tool()
+async def get_diff(url: str, before_id: int | None = None, after_id: int | None = None) -> str:
+    """Get structured diff between two snapshots of a URL.
+
+    Args:
+        url: The URL to get diff for.
+        before_id: Snapshot ID for the older version (optional, uses latest pair if omitted).
+        after_id: Snapshot ID for the newer version (optional, uses latest pair if omitted).
+
+    Returns:
+        JSON object with diff details including unified diff and changed sections.
+    """
+    tracker = _get_tracker()
+    result = await tracker.diff(url, before_id, after_id)
+    return _diff_result_to_json(result)
+
+
+@mcp.tool()
+async def get_history(url: str, count: int = 10) -> str:
+    """Get snapshot history for a tracked URL.
+
+    Args:
+        url: The URL to get history for.
+        count: Maximum number of snapshots to return (default: 10).
+
+    Returns:
+        JSON array of snapshot metadata (id, title, word_count, hash, timestamp).
+    """
+    tracker = _get_tracker()
+    snapshots = await tracker.history(url, count)
+    return json.dumps(snapshots, ensure_ascii=False, indent=2)
+
+
+@mcp.tool()
+async def untrack_url(url: str) -> str:
+    """Remove a URL from change tracking. Deletes all stored snapshots.
+
+    Args:
+        url: The URL to stop tracking.
+
+    Returns:
+        Status message confirming removal.
+    """
+    tracker = _get_tracker()
+    return await tracker.untrack(url)
+
+
+def main() -> None:
+    """Run the MCP server."""
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

diffgrab/tracker.py

@@ -0,0 +1,241 @@
+"""DiffTracker — main orchestrator for web page change tracking."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+
+from markgrab import extract as mg_extract
+
+from diffgrab.db import DEFAULT_DB_PATH, Database
+from diffgrab.differ import DiffResult, compute_diff
+
+logger = logging.getLogger(__name__)
+
+
+def _content_hash(text: str) -> str:
+    """Compute SHA-256 hash of text content."""
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class DiffTracker:
+    """Main orchestrator for tracking web page changes.
+
+    Usage::
+
+        tracker = DiffTracker()
+        await tracker.track("https://example.com")
+        changes = await tracker.check()
+        await tracker.close()
+    """
+
+    def __init__(self, db_path: str = DEFAULT_DB_PATH) -> None:
+        self._db = Database(db_path)
+
+    async def track(self, url: str, interval_hours: int = 24) -> str:
+        """Register URL for tracking and take initial snapshot.
+
+        Args:
+            url: The URL to track.
+            interval_hours: How often to check for changes (default: 24h).
+
+        Returns:
+            Status message.
+        """
+        existing = self._db.get_tracked_url(url)
+        if existing:
+            return f"Already tracking: {url}"
+
+        self._db.add_tracked_url(url, interval_hours)
+
+        # Take initial snapshot
+        try:
+            result = await mg_extract(url)
+            markdown = result.markdown
+            title = result.title
+            word_count = result.word_count
+        except Exception as exc:
+            logger.error("Failed to fetch initial snapshot for %s: %s", url, exc)
+            return f"Tracking registered but initial snapshot failed: {exc}"
+
+        content = _content_hash(markdown)
+        self._db.add_snapshot(url, content, markdown, title, word_count)
+        self._db.update_last_checked(url)
+
+        return f"Now tracking: {url} (interval: {interval_hours}h, initial snapshot: {word_count} words)"
+
+    async def check(self, url: str | None = None) -> list[DiffResult]:
+        """Check tracked URLs for changes.
+
+        If url is None, checks all tracked URLs.
+        Fetches new content, compares hash with latest snapshot.
+        If different, stores new snapshot and returns DiffResult with changed=True.
+
+        Args:
+            url: Specific URL to check, or None for all tracked URLs.
+
+        Returns:
+            List of DiffResult objects (one per checked URL).
+        """
+        if url is not None:
+            tracked = self._db.get_tracked_url(url)
+            if tracked is None:
+                return [DiffResult(url=url, changed=False, summary=f"URL not tracked: {url}")]
+            urls_to_check = [tracked]
+        else:
+            urls_to_check = self._db.get_all_tracked_urls()
+
+        results: list[DiffResult] = []
+
+        for tracked_info in urls_to_check:
+            target_url = tracked_info["url"]
+            result = await self._check_single(target_url)
+            results.append(result)
+
+        return results
+
+    async def _check_single(self, url: str) -> DiffResult:
+        """Check a single URL for changes."""
+        try:
+            result = await mg_extract(url)
+            new_markdown = result.markdown
+            new_title = result.title
+            new_word_count = result.word_count
+        except Exception as exc:
+            logger.error("Failed to fetch %s: %s", url, exc)
+            return DiffResult(url=url, changed=False, summary=f"Fetch failed: {exc}")
+
+        new_hash = _content_hash(new_markdown)
+        latest = self._db.get_latest_snapshot(url)
+
+        if latest is None:
+            # No previous snapshot — store this one
+            self._db.add_snapshot(url, new_hash, new_markdown, new_title, new_word_count)
+            self._db.update_last_checked(url)
+            return DiffResult(url=url, changed=False, summary=f"First snapshot captured for {url}.")
+
+        if new_hash == latest["content_hash"]:
+            self._db.update_last_checked(url)
+            return DiffResult(
+                url=url,
+                changed=False,
+                summary=f"No changes detected for {url}.",
+                after_snapshot_id=latest["id"],
+                after_timestamp=latest["captured_at"],
+            )
+
+        # Content changed — store new snapshot and compute diff
+        new_id = self._db.add_snapshot(url, new_hash, new_markdown, new_title, new_word_count)
+        self._db.update_last_checked(url)
+
+        new_snapshot = self._db.get_snapshot_by_id(new_id)
+        after_timestamp = new_snapshot["captured_at"] if new_snapshot else ""
+
+        diff_result = compute_diff(
+            before_text=latest["markdown"],
+            after_text=new_markdown,
+            url=url,
+            before_snapshot_id=latest["id"],
+            after_snapshot_id=new_id,
+            before_timestamp=latest["captured_at"],
+            after_timestamp=after_timestamp,
+        )
+
+        return diff_result
+
+    async def diff(
+        self,
+        url: str,
+        before_id: int | None = None,
+        after_id: int | None = None,
+    ) -> DiffResult:
+        """Get structured diff between two snapshots.
+
+        If before_id/after_id are not provided, uses the two most recent snapshots.
+
+        Args:
+            url: The URL to diff.
+            before_id: Database ID of the older snapshot (optional).
+            after_id: Database ID of the newer snapshot (optional).
+
+        Returns:
+            DiffResult with structured diff details.
+        """
+        if before_id is not None and after_id is not None:
+            before = self._db.get_snapshot_by_id(before_id)
+            after = self._db.get_snapshot_by_id(after_id)
+        else:
+            before, after = self._db.get_snapshot_pair(url)
+
+        if before is None and after is None:
+            return DiffResult(url=url, changed=False, summary=f"No snapshots found for {url}.")
+
+        if before is None:
+            return DiffResult(
+                url=url,
+                changed=False,
+                summary=f"Only one snapshot exists for {url}. Need at least two for diff.",
+                after_snapshot_id=after["id"] if after else None,
+                after_timestamp=after["captured_at"] if after else "",
+            )
+
+        if after is None:
+            return DiffResult(
+                url=url,
+                changed=False,
+                summary=f"After snapshot not found for {url}.",
+                before_snapshot_id=before["id"],
+                before_timestamp=before["captured_at"],
+            )
+
+        return compute_diff(
+            before_text=before["markdown"],
+            after_text=after["markdown"],
+            url=url,
+            before_snapshot_id=before["id"],
+            after_snapshot_id=after["id"],
+            before_timestamp=before["captured_at"],
+            after_timestamp=after["captured_at"],
+        )
+
+    async def history(self, url: str, count: int = 10) -> list[dict]:
+        """Get snapshot history for a URL.
+
+        Args:
+            url: The URL to get history for.
+            count: Maximum number of snapshots to return (default: 10).
+
+        Returns:
+            List of snapshot dicts (without full markdown content), newest first.
+        """
+        snapshots = self._db.get_snapshots(url, count)
+        # Return metadata only, exclude full markdown for brevity
+        return [
+            {
+                "id": s["id"],
+                "url": s["url"],
+                "content_hash": s["content_hash"],
+                "title": s["title"],
+                "word_count": s["word_count"],
+                "captured_at": s["captured_at"],
+            }
+            for s in snapshots
+        ]
+
+    async def untrack(self, url: str) -> str:
+        """Remove URL from tracking and delete all its snapshots.
+
+        Args:
+            url: The URL to untrack.
+
+        Returns:
+            Status message.
+        """
+        removed = self._db.remove_tracked_url(url)
+        if removed:
+            return f"Untracked: {url}"
+        return f"URL was not being tracked: {url}"
+
+    async def close(self) -> None:
+        """Close the database connection."""
+        self._db.close()

diffgrab/visual.py

@@ -0,0 +1,181 @@
+"""Visual diff — optional snapgrab integration for screenshot comparison."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+try:
+    from snapgrab import capture as sg_capture
+
+    _SNAPGRAB_AVAILABLE = True
+except ImportError:
+    _SNAPGRAB_AVAILABLE = False
+
+try:
+    from PIL import Image, ImageChops
+
+    _PILLOW_AVAILABLE = True
+except ImportError:
+    _PILLOW_AVAILABLE = False
+
+
+@dataclass
+class VisualDiffResult:
+    """Result of visual comparison between two screenshots."""
+
+    url: str
+    changed: bool
+    before_path: str = ""
+    after_path: str = ""
+    diff_path: str = ""
+    pixel_change_ratio: float = 0.0
+    error: str = ""
+
+
+def is_available() -> bool:
+    """Check if visual diff dependencies are installed."""
+    return _SNAPGRAB_AVAILABLE
+
+
+async def capture_screenshot(url: str, output_dir: str = "/tmp/diffgrab") -> str:
+    """Capture a screenshot of a URL using snapgrab.
+
+    Args:
+        url: The URL to capture.
+        output_dir: Directory to store the screenshot.
+
+    Returns:
+        Path to the screenshot file.
+
+    Raises:
+        ImportError: If snapgrab is not installed.
+    """
+    if not _SNAPGRAB_AVAILABLE:
+        raise ImportError("snapgrab is not installed. Run: pip install 'diffgrab[visual]'")
+
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+    result = await sg_capture(url, output_dir=output_dir, full_page=True)
+    return result.path
+
+
+def compute_pixel_diff(before_path: str, after_path: str, output_path: str = "") -> tuple[bool, float, str]:
+    """Compare two screenshots pixel-by-pixel.
+
+    Args:
+        before_path: Path to the before screenshot.
+        after_path: Path to the after screenshot.
+        output_path: Optional path to save the diff image.
+
+    Returns:
+        Tuple of (changed, pixel_change_ratio, diff_path).
+
+    Raises:
+        ImportError: If Pillow is not installed.
+    """
+    if not _PILLOW_AVAILABLE:
+        raise ImportError("Pillow is not installed. Run: pip install Pillow")
+
+    img_before = Image.open(before_path).convert("RGB")
+    img_after = Image.open(after_path).convert("RGB")
+
+    # Resize to same dimensions if needed
+    if img_before.size != img_after.size:
+        target_w = max(img_before.width, img_after.width)
+        target_h = max(img_before.height, img_after.height)
+        img_before = img_before.resize((target_w, target_h))
+        img_after = img_after.resize((target_w, target_h))
+
+    diff_img = ImageChops.difference(img_before, img_after)
+
+    # Calculate change ratio
+    total_pixels = diff_img.width * diff_img.height
+    if total_pixels == 0:
+        return False, 0.0, ""
+
+    # Count non-zero pixels (changed pixels)
+    changed_pixels = 0
+    for pixel in diff_img.getdata():
+        if pixel != (0, 0, 0):
+            changed_pixels += 1
+
+    ratio = changed_pixels / total_pixels
+    changed = ratio > 0.001  # threshold: 0.1% pixels changed
+
+    diff_path = ""
+    if output_path and changed:
+        diff_img.save(output_path)
+        diff_path = output_path
+
+    return changed, ratio, diff_path
+
+
+async def visual_diff(
+    url: str,
+    before_path: str | None = None,
+    after_path: str | None = None,
+    output_dir: str = "/tmp/diffgrab",
+) -> VisualDiffResult:
+    """Perform visual diff between two screenshots of a URL.
+
+    If before_path or after_path are not provided, captures new screenshot(s).
+
+    Args:
+        url: The URL being compared.
+        before_path: Path to the before screenshot (or None to skip).
+        after_path: Path to the after screenshot (or None to capture new).
+        output_dir: Directory for screenshots and diff images.
+
+    Returns:
+        VisualDiffResult with comparison details.
+    """
+    if not _SNAPGRAB_AVAILABLE:
+        return VisualDiffResult(
+            url=url,
+            changed=False,
+            error="snapgrab is not installed. Run: pip install 'diffgrab[visual]'",
+        )
+
+    try:
+        if after_path is None:
+            after_path = await capture_screenshot(url, output_dir=output_dir)
+
+        if before_path is None:
+            return VisualDiffResult(
+                url=url,
+                changed=False,
+                after_path=after_path,
+                error="No before screenshot available for comparison.",
+            )
+
+        if not _PILLOW_AVAILABLE:
+            return VisualDiffResult(
+                url=url,
+                changed=False,
+                before_path=before_path,
+                after_path=after_path,
+                error="Pillow is not installed for pixel comparison.",
+            )
+
+        diff_output = str(Path(output_dir) / "diff.png")
+        changed, ratio, diff_path = compute_pixel_diff(before_path, after_path, diff_output)
+
+        return VisualDiffResult(
+            url=url,
+            changed=changed,
+            before_path=before_path,
+            after_path=after_path,
+            diff_path=diff_path,
+            pixel_change_ratio=ratio,
+        )
+
+    except Exception as exc:
+        logger.error("Visual diff failed for %s: %s", url, exc)
+        return VisualDiffResult(
+            url=url,
+            changed=False,
+            error=str(exc),
+        )