okb 1.0.0__py3-none-any.whl

okb/rescan.py

@@ -0,0 +1,227 @@
+"""
+Rescan indexed documents for freshness and re-ingest changed files.
+
+Checks stored file_modified_at metadata against actual file mtime,
+and re-ingests documents that have changed or been deleted.
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+import psycopg
+from psycopg.rows import dict_row
+
+
+@dataclass
+class RescanResult:
+    """Result of a rescan operation."""
+
+    updated: list[str] = field(default_factory=list)  # Re-ingested files
+    deleted: list[str] = field(default_factory=list)  # Removed (--delete flag)
+    missing: list[str] = field(default_factory=list)  # No longer exist (without --delete)
+    unchanged: int = 0
+    errors: list[tuple[str, str]] = field(default_factory=list)  # (path, error_message)
+
+
+# Virtual path prefixes that should be skipped (handled by sync or manual re-fetch)
+VIRTUAL_PREFIXES = ("claude://", "todoist://", "dropbox://", "http://", "https://")
+
+# Source types that are file-based
+FILE_SOURCE_TYPES = ("markdown", "code", "text", "org", "org-todo", "pdf", "docx")
+
+
+def _escape_like(text: str) -> str:
+    """Escape special LIKE pattern characters (%, _, \\) so they match literally."""
+    return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+class Rescanner:
+    """Checks indexed documents for freshness and re-ingests changed files."""
+
+    def __init__(self, db_url: str, use_modal: bool = True):
+        self.db_url = db_url
+        self.use_modal = use_modal
+
+    def get_indexed_files(self) -> list[dict]:
+        """
+        Query file-based documents with stored mtime.
+
+        Returns list of dicts with:
+        - base_path: The file path (without :: anchor)
+        - source_type: Document source type
+        - stored_mtime: ISO timestamp from metadata
+        - derived_count: Number of derived documents (e.g., org-todo items)
+        """
+        with psycopg.connect(self.db_url, row_factory=dict_row) as conn:
+            # Get unique file paths with their mtimes
+            # Use SPLIT_PART to get base path (before ::)
+            # Group by base path to count derived documents
+            results = conn.execute(
+                """
+                SELECT
+                    SPLIT_PART(source_path, '::', 1) as base_path,
+                    MAX(source_type) as source_type,
+                    MAX(metadata->>'file_modified_at') as stored_mtime,
+                    COUNT(*) - 1 as derived_count
+                FROM documents
+                WHERE source_type = ANY(%s)
+                GROUP BY SPLIT_PART(source_path, '::', 1)
+                ORDER BY base_path
+                """,
+                (list(FILE_SOURCE_TYPES),),
+            ).fetchall()
+
+            # Filter out virtual paths
+            return [
+                dict(r)
+                for r in results
+                if not any(r["base_path"].startswith(p) for p in VIRTUAL_PREFIXES)
+            ]
+
+    def check_file_freshness(self, doc: dict) -> tuple[str, str | None]:
+        """
+        Check if a file is fresh, stale, or missing.
+
+        Args:
+            doc: Dict with base_path and stored_mtime
+
+        Returns:
+            Tuple of (status, current_mtime_iso) where status is:
+            - 'fresh': File unchanged
+            - 'stale': File modified since indexing
+            - 'missing': File no longer exists
+        """
+        path = Path(doc["base_path"])
+
+        if not path.exists():
+            return ("missing", None)
+
+        # Get current mtime
+        current_mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=UTC)
+        current_mtime_iso = current_mtime.isoformat()
+
+        stored_mtime = doc.get("stored_mtime")
+        if not stored_mtime:
+            # No stored mtime - treat as stale
+            return ("stale", current_mtime_iso)
+
+        # Parse stored mtime for comparison
+        # Handle both timezone-aware and naive ISO formats
+        try:
+            stored_dt = datetime.fromisoformat(stored_mtime.replace("Z", "+00:00"))
+            if stored_dt.tzinfo is None:
+                stored_dt = stored_dt.replace(tzinfo=UTC)
+        except (ValueError, AttributeError):
+            return ("stale", current_mtime_iso)
+
+        # Compare timestamps (allow 1 second tolerance for filesystem precision)
+        if abs((current_mtime - stored_dt).total_seconds()) <= 1:
+            return ("fresh", current_mtime_iso)
+
+        return ("stale", current_mtime_iso)
+
+    def delete_document_and_derived(self, source_path: str) -> int:
+        """
+        Delete a document and any derived documents (e.g., org-todo items).
+
+        Args:
+            source_path: The base file path
+
+        Returns:
+            Number of documents deleted
+        """
+        with psycopg.connect(self.db_url) as conn:
+            # Delete exact match and any derived docs (path::*)
+            # Escape LIKE metacharacters (%, _) so they match literally
+            escaped_path = _escape_like(source_path)
+            result = conn.execute(
+                """
+                DELETE FROM documents
+                WHERE source_path = %s OR source_path LIKE %s ESCAPE '\\'
+                RETURNING id
+                """,
+                (source_path, escaped_path + "::%"),
+            ).fetchall()
+            conn.commit()
+            return len(result)
+
+    def rescan(
+        self,
+        dry_run: bool = False,
+        delete_missing: bool = False,
+        verbose: bool = True,
+    ) -> RescanResult:
+        """
+        Scan indexed files for freshness and re-ingest changed ones.
+
+        Args:
+            dry_run: If True, only report what would be done
+            delete_missing: If True, remove documents for missing files
+            verbose: If True, print progress to stderr
+
+        Returns:
+            RescanResult with lists of updated/deleted/missing files
+        """
+        from .ingest import Ingester, parse_document
+
+        result = RescanResult()
+        indexed_files = self.get_indexed_files()
+
+        if verbose:
+            print(f"Found {len(indexed_files)} indexed files", file=sys.stderr)
+
+        ingester = None if dry_run else Ingester(self.db_url, use_modal=self.use_modal)
+
+        for doc in indexed_files:
+            base_path = doc["base_path"]
+            derived_count = doc.get("derived_count", 0)
+            status, current_mtime = self.check_file_freshness(doc)
+
+            if status == "fresh":
+                result.unchanged += 1
+                continue
+
+            elif status == "missing":
+                derived_note = f" (+ {derived_count} derived)" if derived_count else ""
+                if delete_missing:
+                    if verbose:
+                        print(f"  [DELETE]  {base_path}{derived_note}", file=sys.stderr)
+                    if not dry_run:
+                        self.delete_document_and_derived(base_path)
+                    result.deleted.append(base_path)
+                else:
+                    if verbose:
+                        print(f"  [MISSING] {base_path}{derived_note}", file=sys.stderr)
+                    result.missing.append(base_path)
+
+            elif status == "stale":
+                derived_note = f" (+ {derived_count} derived)" if derived_count else ""
+                if verbose:
+                    print(f"  [STALE]   {base_path}{derived_note}", file=sys.stderr)
+
+                if not dry_run:
+                    try:
+                        # Delete old document(s) first
+                        self.delete_document_and_derived(base_path)
+
+                        # Re-ingest
+                        path = Path(base_path)
+                        documents = parse_document(path)
+                        if documents:
+                            # Set file_modified_at on all documents
+                            for d in documents:
+                                d.metadata.extra["file_modified_at"] = current_mtime
+                            ingester.ingest_documents(documents)
+                        result.updated.append(base_path)
+                    except Exception as e:
+                        result.errors.append((base_path, str(e)))
+                        if verbose:
+                            print(f"    ERROR: {e}", file=sys.stderr)
+                else:
+                    result.updated.append(base_path)
+
+        return result

okb/scripts/__init__.py

@@ -0,0 +1 @@
+"""Utility scripts for knowledge base management."""

okb/scripts/watch.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+Watch directories and auto-ingest changed files.
+
+Usage:
+    python scripts/watch.py ~/notes ~/docs
+    python scripts/watch.py ~/notes --local  # Use CPU embedding
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileSystemEvent
+
+from ..config import config
+from ..ingest import Ingester, parse_document, content_hash, check_file_skip, read_text_with_fallback
+
+
+class KnowledgeHandler(FileSystemEventHandler):
+    """Handle file system events for knowledge base updates."""
+
+    def __init__(self, ingester: Ingester, debounce_seconds: float = 2.0):
+        self.ingester = ingester
+        self.debounce_seconds = debounce_seconds
+        self._pending: dict[str, float] = {}  # path -> last_event_time
+        self._processed_hashes: dict[str, str] = {}  # path -> content_hash
+
+    def _should_process(self, path: Path) -> bool:
+        """Check if file should be processed."""
+        if not path.is_file():
+            return False
+
+        if config.should_skip_path(path):
+            return False
+
+        if path.suffix not in config.all_extensions:
+            return False
+
+        # Check block/skip patterns (e.g., .env, *.min.js, temp files)
+        skip_check = check_file_skip(path)
+        if skip_check.should_skip:
+            return False
+
+        return True
+
+    def _debounced_update(self, path: Path):
+        """Update document with debouncing."""
+        path_str = str(path)
+        now = time.time()
+
+        # Check debounce
+        if path_str in self._pending:
+            if now - self._pending[path_str] < self.debounce_seconds:
+                self._pending[path_str] = now
+                return
+
+        self._pending[path_str] = now
+
+        # Check if content actually changed
+        try:
+            content = read_text_with_fallback(path)
+            new_hash = content_hash(content)
+
+            if self._processed_hashes.get(path_str) == new_hash:
+                return  # No actual change
+
+            self._processed_hashes[path_str] = new_hash
+
+            # Content-based checks (secrets, minified)
+            if config.scan_content:
+                skip_check = check_file_skip(path, content)
+                if skip_check.should_skip:
+                    prefix = "BLOCKED" if skip_check.is_security else "Skipping"
+                    print(f"[watch] {prefix}: {path} ({skip_check.reason})")
+                    return
+
+        except Exception:
+            return
+
+        # Process the file
+        print(f"[watch] Updating: {path}")
+
+        try:
+            doc = parse_document(path)
+            # Capture file mtime for staleness tracking
+            mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
+            doc.metadata.extra["file_modified_at"] = mtime.isoformat()
+            self.ingester.ingest_documents([doc])
+
+        except Exception as e:
+            print(f"[watch] Error processing {path}: {e}", file=sys.stderr)
+
+    def on_modified(self, event: FileSystemEvent):
+        """Handle file modification."""
+        if event.is_directory:
+            return
+
+        path = Path(event.src_path)
+        if self._should_process(path):
+            self._debounced_update(path)
+
+    def on_created(self, event: FileSystemEvent):
+        """Handle new file creation."""
+        if event.is_directory:
+            return
+
+        path = Path(event.src_path)
+        if self._should_process(path):
+            self._debounced_update(path)
+
+    def on_deleted(self, event: FileSystemEvent):
+        """Handle file deletion."""
+        if event.is_directory:
+            return
+
+        path = Path(event.src_path)
+        path_str = str(path.resolve())
+
+        # Clean up tracking
+        self._pending.pop(str(path), None)
+        self._processed_hashes.pop(str(path), None)
+
+        # Remove from database
+        print(f"[watch] Removing: {path}")
+        try:
+            self.ingester.delete_document(path_str)
+        except Exception as e:
+            print(f"[watch] Error removing {path}: {e}", file=sys.stderr)
+
+    def on_moved(self, event: FileSystemEvent):
+        """Handle file rename/move."""
+        # Treat as delete + create
+        if hasattr(event, "src_path"):
+            src = Path(event.src_path)
+            self._pending.pop(str(src), None)
+            self._processed_hashes.pop(str(src), None)
+
+            print(f"[watch] Removing (moved): {src}")
+            try:
+                self.ingester.delete_document(str(src.resolve()))
+            except Exception:
+                pass
+
+        if hasattr(event, "dest_path"):
+            dest = Path(event.dest_path)
+            if self._should_process(dest):
+                self._debounced_update(dest)
+
+
+def watch(directories: list[Path], db_url: str, use_modal: bool = True):
+    """Watch directories for changes."""
+    ingester = Ingester(db_url, use_modal=use_modal)
+    handler = KnowledgeHandler(ingester)
+    observer = Observer()
+
+    for directory in directories:
+        if not directory.exists():
+            print(f"Warning: Directory does not exist: {directory}", file=sys.stderr)
+            continue
+
+        observer.schedule(handler, str(directory), recursive=True)
+        print(f"[watch] Watching: {directory}")
+
+    observer.start()
+    print("[watch] Press Ctrl+C to stop")
+
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        print("\n[watch] Stopping...")
+        observer.stop()
+
+    observer.join()
+    print("[watch] Done.")
+
+
+def main():
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Watch directories and auto-ingest changes")
+    parser.add_argument(
+        "directories",
+        nargs="*",
+        type=Path,
+        default=[Path.home() / "notes"],
+        help="Directories to watch (default: ~/notes)",
+    )
+    parser.add_argument("--db-url", default=config.db_url, help="Database URL")
+    parser.add_argument(
+        "--local",
+        action="store_true",
+        help="Use local CPU embedding instead of Modal",
+    )
+
+    args = parser.parse_args()
+    watch(args.directories, args.db_url, use_modal=not args.local)
+
+
+if __name__ == "__main__":
+    main()