footprinter-cli 1.0.0__py3-none-any.whl

footprinter/ingest/browser_indexer.py

@@ -0,0 +1,216 @@
+"""
+Browser parsers for Safari and Chrome.
+"""
+
+import logging
+import platform
+import shutil
+import sqlite3
+import tempfile
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Dict, Generator
+
+from footprinter.utils.time import UTC_FMT
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserParser:
+    """Base class for browser history parsing."""
+
+    def __init__(self, lookback_days: int = 14, since: datetime | None = None):
+        self.lookback_days = lookback_days
+        # Ensure cutoff is tz-aware UTC for comparison with tz-aware epoch constants
+        if since is not None:
+            self.cutoff_date = since.astimezone(timezone.utc) if since.tzinfo else since.replace(tzinfo=timezone.utc)
+        else:
+            self.cutoff_date = datetime.now(timezone.utc) - timedelta(days=lookback_days)
+
+    def parse(self) -> Generator[Dict, None, None]:
+        """Parse browser history. To be implemented by subclasses."""
+        raise NotImplementedError
+
+
+class SafariParser(BrowserParser):
+    """Parse Safari browser history."""
+
+    def __init__(self, lookback_days: int = 14, since: datetime | None = None):
+        super().__init__(lookback_days, since=since)
+        if platform.system() != "Darwin":
+            self.history_db_path = None
+        else:
+            self.history_db_path = Path.home() / "Library" / "Safari" / "History.db"
+
+    def parse(self) -> Generator[Dict, None, None]:
+        """Parse Safari history from SQLite database."""
+        if self.history_db_path is None:
+            logger.warning(
+                "Safari history parsing skipped (unsupported platform: %s)",
+                platform.system(),
+            )
+            return
+        if not self.history_db_path.exists():
+            logger.warning(f"Safari history not found at {self.history_db_path}")
+            return
+
+        # Safari's History.db may be locked, so copy it first
+        with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp_file:
+            tmp_path = tmp_file.name
+
+        conn = None
+        try:
+            shutil.copy2(self.history_db_path, tmp_path)
+
+            conn = sqlite3.connect(tmp_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            # Safari stores visit time as seconds since 2001-01-01 UTC (Core Data timestamp)
+            core_data_epoch = datetime(2001, 1, 1, tzinfo=timezone.utc)
+            cutoff_timestamp = (self.cutoff_date - core_data_epoch).total_seconds()
+
+            query = """
+                SELECT
+                    hv.visit_time,
+                    hi.url,
+                    hi.title
+                FROM history_visits hv
+                JOIN history_items hi ON hv.history_item = hi.id
+                WHERE hv.visit_time > ?
+                ORDER BY hv.visit_time DESC
+            """
+
+            cursor.execute(query, (cutoff_timestamp,))
+
+            for row in cursor:
+                # Convert Safari's Core Data timestamp to datetime
+                visit_time = core_data_epoch + timedelta(seconds=row["visit_time"])
+
+                yield {
+                    "url": row["url"],
+                    "title": row["title"],
+                    "visit_time": visit_time.strftime(UTC_FMT),
+                    "browser": "safari",
+                    "visit_count": 1,
+                }
+
+        except Exception as e:
+            logger.error(f"Error parsing Safari history: {e}")
+        finally:
+            if conn:
+                conn.close()
+            # Clean up temp file
+            try:
+                Path(tmp_path).unlink()
+            except OSError:
+                pass
+
+
+class ChromeParser(BrowserParser):
+    """Parse Chrome browser history."""
+
+    def __init__(self, lookback_days: int = 14, since: datetime | None = None):
+        super().__init__(lookback_days, since=since)
+        system = platform.system()
+        if system == "Darwin":
+            self.history_db_path = (
+                Path.home() / "Library" / "Application Support" / "Google" / "Chrome" / "Default" / "History"
+            )
+        elif system == "Linux":
+            self.history_db_path = Path.home() / ".config" / "google-chrome" / "Default" / "History"
+        else:
+            self.history_db_path = None
+
+    def parse(self) -> Generator[Dict, None, None]:
+        """Parse Chrome history from SQLite database."""
+        if self.history_db_path is None:
+            logger.warning(
+                "Chrome history parsing skipped (unsupported platform: %s)",
+                platform.system(),
+            )
+            return
+        if not self.history_db_path.exists():
+            logger.warning(f"Chrome history not found at {self.history_db_path}")
+            return
+
+        # Chrome's History may be locked, so copy it first
+        with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp_file:
+            tmp_path = tmp_file.name
+
+        conn = None
+        try:
+            shutil.copy2(self.history_db_path, tmp_path)
+
+            conn = sqlite3.connect(tmp_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            # Chrome stores time as microseconds since 1601-01-01 UTC (Windows epoch)
+            chrome_epoch = datetime(1601, 1, 1, tzinfo=timezone.utc)
+            cutoff_timestamp = int((self.cutoff_date - chrome_epoch).total_seconds() * 1_000_000)
+
+            query = """
+                SELECT
+                    urls.url,
+                    urls.title,
+                    urls.visit_count,
+                    visits.visit_time
+                FROM urls
+                LEFT JOIN visits ON urls.id = visits.url
+                WHERE visits.visit_time > ?
+                ORDER BY visits.visit_time DESC
+            """
+
+            cursor.execute(query, (cutoff_timestamp,))
+
+            for row in cursor:
+                # Convert Chrome's timestamp to datetime
+                visit_time = chrome_epoch + timedelta(microseconds=row["visit_time"])
+
+                yield {
+                    "url": row["url"],
+                    "title": row["title"],
+                    "visit_time": visit_time.strftime(UTC_FMT),
+                    "browser": "chrome",
+                    "visit_count": row["visit_count"] or 1,
+                }
+
+        except Exception as e:
+            logger.error(f"Error parsing Chrome history: {e}")
+        finally:
+            if conn:
+                conn.close()
+            # Clean up temp file
+            try:
+                Path(tmp_path).unlink()
+            except OSError:
+                pass
+
+
+class BrowserManager:
+    """Manage parsing of multiple browsers."""
+
+    def __init__(self, config: Dict, since: datetime | None = None):
+        self.config = config
+        self.lookback_days = config.get("indexing", {}).get("lookback_days", 14)
+        self.browsers = config.get("browsers", [])
+        self.since = since
+
+    def parse_all(self) -> Generator[Dict, None, None]:
+        """Parse history from all configured browsers."""
+        for browser in self.browsers:
+            browser_lower = browser.lower()
+
+            if browser_lower == "safari":
+                parser = SafariParser(self.lookback_days, since=self.since)
+                logger.info("Parsing Safari history...")
+                yield from parser.parse()
+
+            elif browser_lower == "chrome":
+                parser = ChromeParser(self.lookback_days, since=self.since)
+                logger.info("Parsing Chrome history...")
+                yield from parser.parse()
+
+            else:
+                logger.warning(f"Unknown browser: {browser}")

footprinter/ingest/chat_dedup.py

@@ -0,0 +1,156 @@
+"""Chat dedup detection and merge.
+
+Orchestrates near-duplicate chat detection via db.chats and merges
+duplicates by combining unique messages from source into target,
+marking the source as status='merged', and updating vector embeddings.
+"""
+
+import hashlib
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from footprinter.db import chats as chats_db
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DuplicateGroup:
+    """A group of chats detected as potential duplicates."""
+
+    reason: str  # 'exact_title', 'fuzzy_title', 'message_overlap'
+    confidence: str  # 'high', 'medium'
+    chats: List[Dict]  # list of chat dicts
+    detail: str = ""  # human-readable explanation
+
+
+class ChatDedup:
+    """Duplicate detection and merge for chats."""
+
+    def __init__(self, db):
+        self.db = db
+        self._hash_cache: Dict[int, List[str]] = {}
+
+    def _get_hashes(self, chat_id: int) -> List[str]:
+        """Get message content hashes, with caching."""
+        if chat_id not in self._hash_cache:
+            self._hash_cache[chat_id] = chats_db.get_chat_message_hashes(self.db.conn, chat_id)
+        return self._hash_cache[chat_id]
+
+    def detect_duplicates(self) -> List[DuplicateGroup]:
+        """Detect potential duplicate chats.
+
+        Delegates to ``footprinter.db.chats.detect_duplicates`` and
+        converts plain dicts back to ``DuplicateGroup`` dataclasses.
+        """
+        from footprinter.db.chats import detect_duplicates as _detect
+
+        raw_groups = _detect(self.db.conn)
+        return [
+            DuplicateGroup(
+                reason=g["reason"],
+                confidence=g["confidence"],
+                chats=g["chats"],
+                detail=g["detail"],
+            )
+            for g in raw_groups
+        ]
+
+    def merge(
+        self,
+        target_id: int,
+        source_id: int,
+        vector_store: Optional[Any] = None,
+    ) -> Dict:
+        """Merge source chat into target.
+
+        1. Validate both exist and aren't already merged
+        2. Hash target's messages
+        3. Identify unique messages in source
+        4. Move unique messages to target
+        5. Recount target's message_count
+        6. Mark source as merged
+        7. Update vectors if vector_store provided
+
+        Returns dict with merge stats.
+        """
+        if target_id == source_id:
+            raise ValueError("Cannot merge a chat into itself")
+
+        target = chats_db.get_chat_by_id(self.db.conn, target_id)
+        source = chats_db.get_chat_by_id(self.db.conn, source_id)
+
+        if not target:
+            raise ValueError(f"Target chat {target_id} not found")
+        if not source:
+            raise ValueError(f"Source chat {source_id} not found")
+        if target.get("status") == "merged":
+            raise ValueError(f"Target chat {target_id} is already merged")
+        if source.get("status") == "merged":
+            raise ValueError(f"Source chat {source_id} is already merged")
+
+        # Hash target's messages to identify what's already there
+        target_hashes = set(self._get_hashes(target_id))
+
+        # Find unique messages in source (not already in target)
+        source_messages = chats_db.get_chat_messages(self.db.conn, source_id)
+        unique_message_ids = []
+        duplicate_count = 0
+        for msg in source_messages:
+            content = msg["content"] or ""
+            msg_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
+            if msg_hash not in target_hashes:
+                unique_message_ids.append(msg["id"])
+            else:
+                duplicate_count += 1
+
+        # Move unique messages to target
+        moved = 0
+        if unique_message_ids:
+            moved = chats_db.move_messages_to_chat(self.db.conn, source_id, target_id, unique_message_ids)
+
+        # Recount target's messages
+        new_count = chats_db.update_chat_message_count(self.db.conn, target_id)
+
+        # Mark source as merged
+        chats_db.mark_chat_merged(self.db.conn, source_id, target_id)
+
+        # Commit the entire merge atomically (move + recount + mark)
+        self.db.conn.commit()
+
+        # Invalidate hash cache
+        self._hash_cache.pop(target_id, None)
+        self._hash_cache.pop(source_id, None)
+
+        # Update vectors if store provided
+        vectors_updated = False
+        if vector_store:
+            try:
+                # Delete source chat vectors
+                vector_store.delete_by_metadata({"chat_id": source_id})
+                # Re-index moved messages under target
+                # (Caller is responsible for full re-vectorization)
+                vectors_updated = True
+            except Exception as e:
+                logger.warning("Vector update failed (non-fatal): %s", e)
+
+        result = {
+            "target_id": target_id,
+            "source_id": source_id,
+            "target_title": target.get("title"),
+            "source_title": source.get("title"),
+            "messages_moved": moved,
+            "duplicates_skipped": duplicate_count,
+            "new_message_count": new_count,
+            "vectors_updated": vectors_updated,
+        }
+
+        logger.info(
+            "Merged chat %d into %d: %d messages moved, %d duplicates skipped",
+            source_id,
+            target_id,
+            moved,
+            duplicate_count,
+        )
+        return result