diffgrab 0.1.0__py3-none-any.whl

diffgrab/__init__.py

@@ -0,0 +1,109 @@
+"""diffgrab — Web page change tracking with structured diffs."""
+
+from __future__ import annotations
+
+from diffgrab.differ import DiffResult
+from diffgrab.tracker import DiffTracker
+
+__all__ = ["DiffTracker", "DiffResult", "track", "check", "diff", "history", "untrack"]
+__version__ = "0.1.0"
+
+
+async def track(url: str, interval_hours: int = 24, *, db_path: str = "") -> str:
+    """Register a URL for change tracking.
+
+    Args:
+        url: The URL to track.
+        interval_hours: Check interval in hours (default: 24).
+        db_path: Custom database path (optional).
+
+    Returns:
+        Status message.
+    """
+    kwargs = {"db_path": db_path} if db_path else {}
+    tracker = DiffTracker(**kwargs)
+    try:
+        return await tracker.track(url, interval_hours)
+    finally:
+        await tracker.close()
+
+
+async def check(url: str | None = None, *, db_path: str = "") -> list[DiffResult]:
+    """Check tracked URLs for changes.
+
+    Args:
+        url: Specific URL to check, or None for all.
+        db_path: Custom database path (optional).
+
+    Returns:
+        List of DiffResult objects.
+    """
+    kwargs = {"db_path": db_path} if db_path else {}
+    tracker = DiffTracker(**kwargs)
+    try:
+        return await tracker.check(url)
+    finally:
+        await tracker.close()
+
+
+async def diff(
+    url: str,
+    before_id: int | None = None,
+    after_id: int | None = None,
+    *,
+    db_path: str = "",
+) -> DiffResult:
+    """Get structured diff between two snapshots of a URL.
+
+    Args:
+        url: The URL to diff.
+        before_id: Database ID of the older snapshot.
+        after_id: Database ID of the newer snapshot.
+        db_path: Custom database path (optional).
+
+    Returns:
+        DiffResult with structured diff.
+    """
+    kwargs = {"db_path": db_path} if db_path else {}
+    tracker = DiffTracker(**kwargs)
+    try:
+        return await tracker.diff(url, before_id, after_id)
+    finally:
+        await tracker.close()
+
+
+async def history(url: str, count: int = 10, *, db_path: str = "") -> list[dict]:
+    """Get snapshot history for a URL.
+
+    Args:
+        url: The URL to get history for.
+        count: Maximum number of snapshots (default: 10).
+        db_path: Custom database path (optional).
+
+    Returns:
+        List of snapshot metadata dicts.
+    """
+    kwargs = {"db_path": db_path} if db_path else {}
+    tracker = DiffTracker(**kwargs)
+    try:
+        return await tracker.history(url, count)
+    finally:
+        await tracker.close()
+
+
+async def untrack(url: str, *, db_path: str = "") -> str:
+    """Remove a URL from tracking.
+
+    Args:
+        url: The URL to untrack.
+        db_path: Custom database path (optional).
+
+    Returns:
+        Status message.
+    """
+    kwargs = {"db_path": db_path} if db_path else {}
+    tracker = DiffTracker(**kwargs)
+    try:
+        return await tracker.untrack(url)
+    finally:
+        await tracker.close()

diffgrab/__main__.py

@@ -0,0 +1,132 @@
+"""CLI interface for diffgrab — web page change tracking."""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+
+
+def main() -> None:
+    """Entry point for the diffgrab CLI."""
+    try:
+        from click import argument, group, option
+        from rich.console import Console
+        from rich.table import Table
+    except ImportError:
+        print("CLI dependencies not installed. Run: pip install 'diffgrab[cli]'", file=sys.stderr)
+        sys.exit(1)
+
+    console = Console()
+
+    @group()
+    def cli() -> None:
+        """diffgrab — Web page change tracking with structured diffs."""
+
+    @cli.command()
+    @argument("url")
+    @option("--interval", default=24, type=int, help="Check interval in hours (default: 24).")
+    @option("--db", default="", help="Custom database path.")
+    def track(url: str, interval: int, db: str) -> None:
+        """Register a URL for change tracking."""
+        from diffgrab import track as _track
+
+        kwargs = {"db_path": db} if db else {}
+        result = asyncio.run(_track(url, interval, **kwargs))
+        console.print(result)
+
+    @cli.command()
+    @argument("url", required=False, default=None)
+    @option("--db", default="", help="Custom database path.")
+    def check(url: str | None, db: str) -> None:
+        """Check tracked URLs for changes."""
+        from diffgrab import check as _check
+
+        kwargs = {"db_path": db} if db else {}
+        results = asyncio.run(_check(url, **kwargs))
+
+        if not results:
+            console.print("[dim]No tracked URLs found.[/dim]")
+            return
+
+        for r in results:
+            if r.changed:
+                console.print(f"[bold red]CHANGED[/bold red] {r.url}")
+                console.print(f"  +{r.added_lines} / -{r.removed_lines} lines")
+                if r.changed_sections:
+                    console.print(f"  Sections: {', '.join(r.changed_sections[:5])}")
+                console.print(f"  {r.summary}")
+            else:
+                console.print(f"[green]OK[/green] {r.url} — {r.summary}")
+
+    @cli.command()
+    @argument("url")
+    @option("--before", "before_id", default=None, type=int, help="Snapshot ID for before.")
+    @option("--after", "after_id", default=None, type=int, help="Snapshot ID for after.")
+    @option("--db", default="", help="Custom database path.")
+    def diff(url: str, before_id: int | None, after_id: int | None, db: str) -> None:
+        """Show structured diff between two snapshots."""
+        from diffgrab import diff as _diff
+
+        kwargs = {"db_path": db} if db else {}
+        result = asyncio.run(_diff(url, before_id, after_id, **kwargs))
+
+        if not result.changed:
+            console.print(f"[green]{result.summary}[/green]")
+            return
+
+        console.print(f"[bold]Diff for {result.url}[/bold]")
+        console.print(f"+{result.added_lines} / -{result.removed_lines} lines")
+        if result.changed_sections:
+            console.print(f"Sections: {', '.join(result.changed_sections)}")
+        console.print()
+        console.print(result.unified_diff)
+
+    @cli.command()
+    @argument("url")
+    @option("--count", default=10, type=int, help="Number of snapshots to show (default: 10).")
+    @option("--db", default="", help="Custom database path.")
+    def history(url: str, count: int, db: str) -> None:
+        """Show snapshot history for a URL."""
+        from diffgrab import history as _history
+
+        kwargs = {"db_path": db} if db else {}
+        snapshots = asyncio.run(_history(url, count, **kwargs))
+
+        if not snapshots:
+            console.print(f"[dim]No snapshots for {url}[/dim]")
+            return
+
+        table = Table(title=f"Snapshots for {url}")
+        table.add_column("ID", style="cyan")
+        table.add_column("Title")
+        table.add_column("Words", justify="right")
+        table.add_column("Hash", max_width=12)
+        table.add_column("Captured At")
+
+        for s in snapshots:
+            table.add_row(
+                str(s["id"]),
+                s.get("title", ""),
+                str(s.get("word_count", 0)),
+                s["content_hash"][:12],
+                s["captured_at"],
+            )
+
+        console.print(table)
+
+    @cli.command()
+    @argument("url")
+    @option("--db", default="", help="Custom database path.")
+    def untrack(url: str, db: str) -> None:
+        """Remove a URL from tracking."""
+        from diffgrab import untrack as _untrack
+
+        kwargs = {"db_path": db} if db else {}
+        result = asyncio.run(_untrack(url, **kwargs))
+        console.print(result)
+
+    cli()
+
+
+if __name__ == "__main__":
+    main()

diffgrab/db.py

@@ -0,0 +1,167 @@
+"""SQLite storage for tracked URLs and snapshots."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_DB_PATH = "~/.local/share/diffgrab/diffgrab.db"
+
+_SCHEMA = """\
+CREATE TABLE IF NOT EXISTS tracked_urls (
+    id INTEGER PRIMARY KEY,
+    url TEXT UNIQUE NOT NULL,
+    interval_hours INTEGER DEFAULT 24,
+    created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+    last_checked_at TEXT
+);
+
+CREATE TABLE IF NOT EXISTS snapshots (
+    id INTEGER PRIMARY KEY,
+    url TEXT NOT NULL,
+    content_hash TEXT NOT NULL,
+    markdown TEXT NOT NULL,
+    title TEXT DEFAULT '',
+    word_count INTEGER DEFAULT 0,
+    captured_at TEXT DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (url) REFERENCES tracked_urls(url)
+);
+"""
+
+
+class Database:
+    """SQLite database for diffgrab storage."""
+
+    def __init__(self, db_path: str = DEFAULT_DB_PATH) -> None:
+        resolved = Path(db_path).expanduser()
+        resolved.parent.mkdir(parents=True, exist_ok=True)
+        self._path = str(resolved)
+        self._conn: sqlite3.Connection | None = None
+
+    def _get_conn(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self._conn = sqlite3.connect(self._path)
+            self._conn.row_factory = sqlite3.Row
+            self._conn.execute("PRAGMA journal_mode=WAL")
+            self._conn.execute("PRAGMA foreign_keys=ON")
+            self._init_schema()
+        return self._conn
+
+    def _init_schema(self) -> None:
+        conn = self._conn
+        if conn is None:
+            return
+        conn.executescript(_SCHEMA)
+        conn.commit()
+
+    # ── tracked_urls ──────────────────────────────────────────
+
+    def add_tracked_url(self, url: str, interval_hours: int = 24) -> int:
+        """Add a URL to tracking. Returns row id."""
+        conn = self._get_conn()
+        cur = conn.execute(
+            "INSERT OR IGNORE INTO tracked_urls (url, interval_hours) VALUES (?, ?)",
+            (url, interval_hours),
+        )
+        conn.commit()
+        if cur.lastrowid and cur.rowcount > 0:
+            return cur.lastrowid
+        # Already existed — fetch id
+        row = conn.execute("SELECT id FROM tracked_urls WHERE url = ?", (url,)).fetchone()
+        return row["id"] if row else 0
+
+    def get_tracked_url(self, url: str) -> dict | None:
+        """Get a single tracked URL record."""
+        conn = self._get_conn()
+        row = conn.execute("SELECT * FROM tracked_urls WHERE url = ?", (url,)).fetchone()
+        return dict(row) if row else None
+
+    def get_all_tracked_urls(self) -> list[dict]:
+        """Get all tracked URLs."""
+        conn = self._get_conn()
+        rows = conn.execute("SELECT * FROM tracked_urls ORDER BY created_at").fetchall()
+        return [dict(r) for r in rows]
+
+    def update_last_checked(self, url: str) -> None:
+        """Update the last_checked_at timestamp for a URL."""
+        conn = self._get_conn()
+        conn.execute(
+            "UPDATE tracked_urls SET last_checked_at = CURRENT_TIMESTAMP WHERE url = ?",
+            (url,),
+        )
+        conn.commit()
+
+    def remove_tracked_url(self, url: str) -> bool:
+        """Remove a URL from tracking. Returns True if removed."""
+        conn = self._get_conn()
+        conn.execute("DELETE FROM snapshots WHERE url = ?", (url,))
+        cur = conn.execute("DELETE FROM tracked_urls WHERE url = ?", (url,))
+        conn.commit()
+        return cur.rowcount > 0
+
+    # ── snapshots ─────────────────────────────────────────────
+
+    def add_snapshot(
+        self,
+        url: str,
+        content_hash: str,
+        markdown: str,
+        title: str = "",
+        word_count: int = 0,
+    ) -> int:
+        """Store a new snapshot. Returns row id."""
+        conn = self._get_conn()
+        cur = conn.execute(
+            "INSERT INTO snapshots (url, content_hash, markdown, title, word_count) VALUES (?, ?, ?, ?, ?)",
+            (url, content_hash, markdown, title, word_count),
+        )
+        conn.commit()
+        return cur.lastrowid or 0
+
+    def get_latest_snapshot(self, url: str) -> dict | None:
+        """Get the most recent snapshot for a URL."""
+        conn = self._get_conn()
+        row = conn.execute(
+            "SELECT * FROM snapshots WHERE url = ? ORDER BY captured_at DESC LIMIT 1",
+            (url,),
+        ).fetchone()
+        return dict(row) if row else None
+
+    def get_snapshot_by_id(self, snapshot_id: int) -> dict | None:
+        """Get a snapshot by its ID."""
+        conn = self._get_conn()
+        row = conn.execute("SELECT * FROM snapshots WHERE id = ?", (snapshot_id,)).fetchone()
+        return dict(row) if row else None
+
+    def get_snapshots(self, url: str, count: int = 10) -> list[dict]:
+        """Get recent snapshots for a URL, newest first."""
+        conn = self._get_conn()
+        rows = conn.execute(
+            "SELECT * FROM snapshots WHERE url = ? ORDER BY captured_at DESC LIMIT ?",
+            (url, count),
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    def get_snapshot_pair(self, url: str) -> tuple[dict | None, dict | None]:
+        """Get the two most recent snapshots (before, after). Returns (older, newer)."""
+        conn = self._get_conn()
+        rows = conn.execute(
+            "SELECT * FROM snapshots WHERE url = ? ORDER BY captured_at DESC LIMIT 2",
+            (url,),
+        ).fetchall()
+        if len(rows) == 0:
+            return None, None
+        if len(rows) == 1:
+            return None, dict(rows[0])
+        return dict(rows[1]), dict(rows[0])
+
+    # ── lifecycle ─────────────────────────────────────────────
+
+    def close(self) -> None:
+        """Close the database connection."""
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None

diffgrab/differ.py

@@ -0,0 +1,179 @@
+"""Text diff engine — unified diff + section-level analysis."""
+
+from __future__ import annotations
+
+import difflib
+import re
+from dataclasses import dataclass, field
+
+# Markdown heading pattern (ATX style: # Heading)
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+)$", re.MULTILINE)
+
+
+@dataclass
+class DiffResult:
+    """Structured result of comparing two snapshots."""
+
+    url: str
+    changed: bool
+    added_lines: int = 0
+    removed_lines: int = 0
+    changed_sections: list[str] = field(default_factory=list)
+    unified_diff: str = ""
+    summary: str = ""
+    before_snapshot_id: int | None = None
+    after_snapshot_id: int | None = None
+    before_timestamp: str = ""
+    after_timestamp: str = ""
+
+
+def _count_diff_lines(diff_text: str) -> tuple[int, int]:
+    """Count added and removed lines from unified diff output."""
+    added = 0
+    removed = 0
+    for line in diff_text.splitlines():
+        if line.startswith("+") and not line.startswith("+++"):
+            added += 1
+        elif line.startswith("-") and not line.startswith("---"):
+            removed += 1
+    return added, removed
+
+
+def _find_changed_sections(before_text: str, after_text: str) -> list[str]:
+    """Identify markdown headings whose sections contain changes.
+
+    Strategy: split each document by headings, compare section content.
+    """
+    before_sections = _split_by_headings(before_text)
+    after_sections = _split_by_headings(after_text)
+
+    changed: list[str] = []
+    all_headings = set(before_sections.keys()) | set(after_sections.keys())
+
+    for heading in sorted(all_headings):
+        b_content = before_sections.get(heading, "")
+        a_content = after_sections.get(heading, "")
+        if b_content != a_content:
+            changed.append(heading)
+
+    return changed
+
+
+def _split_by_headings(text: str) -> dict[str, str]:
+    """Split markdown text into sections keyed by heading.
+
+    Text before the first heading goes under "(top)".
+    """
+    sections: dict[str, str] = {}
+    current_heading = "(top)"
+    current_lines: list[str] = []
+
+    for line in text.splitlines():
+        match = _HEADING_RE.match(line)
+        if match:
+            # Save previous section
+            sections[current_heading] = "\n".join(current_lines)
+            current_heading = match.group(2).strip()
+            current_lines = []
+        else:
+            current_lines.append(line)
+
+    sections[current_heading] = "\n".join(current_lines)
+    return sections
+
+
+def _generate_summary(
+    added: int,
+    removed: int,
+    changed_sections: list[str],
+    url: str,
+) -> str:
+    """Generate a human-readable summary of changes."""
+    if added == 0 and removed == 0:
+        return f"No changes detected for {url}."
+
+    parts: list[str] = []
+
+    if added > 0 and removed > 0:
+        parts.append(f"{added} lines added, {removed} lines removed")
+    elif added > 0:
+        parts.append(f"{added} lines added")
+    else:
+        parts.append(f"{removed} lines removed")
+
+    if changed_sections:
+        section_names = ", ".join(changed_sections[:5])
+        if len(changed_sections) > 5:
+            section_names += f" (+{len(changed_sections) - 5} more)"
+        parts.append(f"in sections: {section_names}")
+
+    return ". ".join(parts) + "."
+
+
+def compute_diff(
+    before_text: str,
+    after_text: str,
+    url: str = "",
+    before_snapshot_id: int | None = None,
+    after_snapshot_id: int | None = None,
+    before_timestamp: str = "",
+    after_timestamp: str = "",
+) -> DiffResult:
+    """Compute structured diff between two markdown texts.
+
+    Args:
+        before_text: The older markdown content.
+        after_text: The newer markdown content.
+        url: The URL being compared.
+        before_snapshot_id: Database ID of the older snapshot.
+        after_snapshot_id: Database ID of the newer snapshot.
+        before_timestamp: Timestamp of the older snapshot.
+        after_timestamp: Timestamp of the newer snapshot.
+
+    Returns:
+        DiffResult with all diff details.
+    """
+    changed = before_text != after_text
+
+    if not changed:
+        return DiffResult(
+            url=url,
+            changed=False,
+            summary=f"No changes detected for {url}.",
+            before_snapshot_id=before_snapshot_id,
+            after_snapshot_id=after_snapshot_id,
+            before_timestamp=before_timestamp,
+            after_timestamp=after_timestamp,
+        )
+
+    before_lines = before_text.splitlines(keepends=True)
+    after_lines = after_text.splitlines(keepends=True)
+
+    diff_lines = list(
+        difflib.unified_diff(
+            before_lines,
+            after_lines,
+            fromfile=f"before ({before_timestamp})" if before_timestamp else "before",
+            tofile=f"after ({after_timestamp})" if after_timestamp else "after",
+            lineterm="",
+        )
+    )
+    unified_diff = "\n".join(diff_lines)
+
+    added, removed = _count_diff_lines(unified_diff)
+    changed_sections = _find_changed_sections(before_text, after_text)
+    summary = _generate_summary(added, removed, changed_sections, url)
+
+    return DiffResult(
+        url=url,
+        changed=True,
+        added_lines=added,
+        removed_lines=removed,
+        changed_sections=changed_sections,
+        unified_diff=unified_diff,
+        summary=summary,
+        before_snapshot_id=before_snapshot_id,
+        after_snapshot_id=after_snapshot_id,
+        before_timestamp=before_timestamp,
+        after_timestamp=after_timestamp,
+    )