htmlgraph 0.20.1__py3-none-any.whl → 0.27.5__py3-none-any.whl

htmlgraph/archive/bloom.py

@@ -0,0 +1,234 @@
+"""
+Bloom filter implementation for archive search optimization.
+
+Uses MurmurHash3 for 22x faster hashing with hardware optimizations.
+Target: 32.8% latency reduction by skipping 70-90% of archives.
+"""
+
+import hashlib
+import json
+import math
+from pathlib import Path
+from typing import Any
+
+try:
+    import mmh3  # type: ignore
+
+    HAS_MMH3 = True
+except ImportError:
+    HAS_MMH3 = False
+
+
+class BloomFilter:
+    """
+    Space-efficient probabilistic data structure for archive filtering.
+
+    Optimized for speed with:
+    - MurmurHash3 hardware acceleration (if available)
+    - Configurable false positive rate (default 0.01)
+    - Efficient bit array storage
+    """
+
+    def __init__(
+        self, expected_items: int = 1000, false_positive_rate: float = 0.01
+    ) -> None:
+        """
+        Initialize Bloom filter.
+
+        Args:
+            expected_items: Expected number of items to add
+            false_positive_rate: Desired false positive rate (0.01 = 1%)
+        """
+        self.expected_items = expected_items
+        self.false_positive_rate = false_positive_rate
+
+        # Calculate optimal bit array size
+        self.bit_count = self._optimal_bit_count(expected_items, false_positive_rate)
+
+        # Calculate optimal number of hash functions
+        self.hash_count = self._optimal_hash_count(self.bit_count, expected_items)
+
+        # Initialize bit array (using bytearray for efficiency)
+        self.bit_array = bytearray((self.bit_count + 7) // 8)
+
+        self.items_added = 0
+
+    def _optimal_bit_count(self, n: int, p: float) -> int:
+        """
+        Calculate optimal bit array size.
+
+        Formula: m = -(n * ln(p)) / (ln(2)^2)
+        """
+        return int(-n * math.log(p) / (math.log(2) ** 2))
+
+    def _optimal_hash_count(self, m: int, n: int) -> int:
+        """
+        Calculate optimal number of hash functions.
+
+        Formula: k = (m / n) * ln(2)
+        """
+        return max(1, int((m / n) * math.log(2)))
+
+    def _hash(self, item: str, seed: int) -> int:
+        """
+        Hash item with seed using MurmurHash3 or fallback to hashlib.
+
+        Args:
+            item: Item to hash
+            seed: Hash seed for different hash functions
+
+        Returns:
+            Hash value modulo bit_count
+        """
+        if HAS_MMH3:
+            # MurmurHash3 - 22x faster with hardware optimization
+            hash_val: int = mmh3.hash(item, seed)  # type: ignore
+            return hash_val % self.bit_count
+        else:
+            # Fallback to hashlib (slower but always available)
+            hash_obj = hashlib.sha256(f"{item}{seed}".encode())
+            return int.from_bytes(hash_obj.digest()[:4], "big") % self.bit_count
+
+    def _set_bit(self, position: int) -> None:
+        """Set bit at position to 1."""
+        byte_index = position // 8
+        bit_index = position % 8
+        self.bit_array[byte_index] |= 1 << bit_index
+
+    def _get_bit(self, position: int) -> bool:
+        """Get bit value at position."""
+        byte_index = position // 8
+        bit_index = position % 8
+        return bool(self.bit_array[byte_index] & (1 << bit_index))
+
+    def add(self, item: str) -> None:
+        """
+        Add item to Bloom filter.
+
+        Args:
+            item: String to add
+        """
+        for seed in range(self.hash_count):
+            position = self._hash(item, seed)
+            self._set_bit(position)
+
+        self.items_added += 1
+
+    def might_contain(self, item: str) -> bool:
+        """
+        Check if item might be in the set.
+
+        Args:
+            item: String to check
+
+        Returns:
+            True if item might be present (or false positive)
+            False if item is definitely not present
+        """
+        for seed in range(self.hash_count):
+            position = self._hash(item, seed)
+            if not self._get_bit(position):
+                return False
+        return True
+
+    def build_for_archive(self, entities: list[dict[str, Any]]) -> None:
+        """
+        Build Bloom filter from archive entities.
+
+        Indexes:
+        - Entity IDs
+        - Titles (lowercased, tokenized)
+        - Description text (lowercased, tokenized)
+
+        Args:
+            entities: List of entity dictionaries with id, title, description
+        """
+        for entity in entities:
+            # Add entity ID
+            self.add(entity["id"])
+
+            # Add title tokens (lowercased)
+            if "title" in entity and entity["title"]:
+                for word in entity["title"].lower().split():
+                    self.add(word)
+
+            # Add description tokens (lowercased)
+            if "description" in entity and entity["description"]:
+                for word in entity["description"].lower().split():
+                    self.add(word)
+
+    def save(self, filepath: Path) -> None:
+        """
+        Save Bloom filter to disk.
+
+        Args:
+            filepath: Path to save .bloom file
+        """
+        data = {
+            "expected_items": self.expected_items,
+            "false_positive_rate": self.false_positive_rate,
+            "bit_count": self.bit_count,
+            "hash_count": self.hash_count,
+            "items_added": self.items_added,
+            "bit_array": list(self.bit_array),  # Convert bytearray to list for JSON
+        }
+
+        with open(filepath, "w") as f:
+            json.dump(data, f)
+
+    @classmethod
+    def load(cls, filepath: Path) -> "BloomFilter":
+        """
+        Load Bloom filter from disk.
+
+        Args:
+            filepath: Path to .bloom file
+
+        Returns:
+            Loaded BloomFilter instance
+        """
+        with open(filepath) as f:
+            data = json.load(f)
+
+        # Create instance with saved parameters
+        bloom = cls(
+            expected_items=data["expected_items"],
+            false_positive_rate=data["false_positive_rate"],
+        )
+
+        # Restore state
+        bloom.bit_count = data["bit_count"]
+        bloom.hash_count = data["hash_count"]
+        bloom.items_added = data["items_added"]
+        bloom.bit_array = bytearray(data["bit_array"])
+
+        return bloom
+
+    def get_stats(self) -> dict[str, Any]:
+        """
+        Get Bloom filter statistics.
+
+        Returns:
+            Dictionary with stats (size, items, FPR, etc.)
+        """
+        # Calculate actual false positive rate
+        actual_fpr = (
+            (1 - math.exp(-self.hash_count * self.items_added / self.bit_count))
+            ** self.hash_count
+            if self.items_added > 0
+            else 0
+        )
+
+        return {
+            "expected_items": self.expected_items,
+            "items_added": self.items_added,
+            "bit_count": self.bit_count,
+            "hash_count": self.hash_count,
+            "bytes_used": len(self.bit_array),
+            "target_fpr": self.false_positive_rate,
+            "actual_fpr": actual_fpr,
+            "utilization": self.items_added / self.expected_items
+            if self.expected_items > 0
+            else 0,
+            "using_mmh3": HAS_MMH3,
+        }

htmlgraph/archive/fts.py

@@ -0,0 +1,297 @@
+"""
+SQLite FTS5 full-text search index for archive content.
+
+Uses BM25 ranking for relevance scoring with O(log n) search performance.
+Provides snippet extraction with highlighting for matched terms.
+"""
+
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+
+class ArchiveFTS5Index:
+    """
+    Full-text search index using SQLite FTS5.
+
+    Features:
+    - Porter stemming for better matching
+    - Unicode61 tokenization for international text
+    - BM25 ranking for relevance scoring
+    - Snippet extraction with highlighting
+    - Metadata table for quick lookups
+    """
+
+    def __init__(self, db_path: Path) -> None:
+        """
+        Initialize FTS5 index.
+
+        Args:
+            db_path: Path to SQLite database file
+        """
+        self.db_path = db_path
+        self.conn: sqlite3.Connection | None = None
+        self._ensure_schema()
+
+    def _ensure_schema(self) -> None:
+        """Create FTS5 tables if they don't exist."""
+        conn = self._get_connection()
+
+        # Create FTS5 virtual table with porter stemming
+        conn.execute(
+            """
+            CREATE VIRTUAL TABLE IF NOT EXISTS archive_fts USING fts5(
+                entity_id UNINDEXED,
+                title,
+                description,
+                content,
+                tokenize='porter unicode61'
+            )
+            """
+        )
+
+        # Create metadata table for quick lookups
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS archive_metadata (
+                entity_id TEXT PRIMARY KEY,
+                archive_file TEXT NOT NULL,
+                entity_type TEXT,
+                status TEXT,
+                created TEXT,
+                updated TEXT
+            )
+            """
+        )
+
+        # Create index on archive_file for filtering
+        conn.execute(
+            """
+            CREATE INDEX IF NOT EXISTS idx_archive_file
+            ON archive_metadata(archive_file)
+            """
+        )
+
+        conn.commit()
+
+    def _get_connection(self) -> sqlite3.Connection:
+        """Get database connection (create if needed)."""
+        if self.conn is None:
+            self.conn = sqlite3.connect(str(self.db_path))
+            self.conn.row_factory = sqlite3.Row  # Enable dict-like access
+        return self.conn
+
+    def index_archive(self, archive_file: str, entities: list[dict[str, Any]]) -> None:
+        """
+        Index entities from an archive file.
+
+        Args:
+            archive_file: Name of archive file (e.g., '2024-Q4-completed.html')
+            entities: List of entity dictionaries
+        """
+        conn = self._get_connection()
+
+        for entity in entities:
+            entity_id = entity.get("id", "")
+            title = entity.get("title", "")
+            description = entity.get("description", "")
+            content = entity.get("content", "")
+
+            # Insert into FTS5 table
+            conn.execute(
+                """
+                INSERT INTO archive_fts (entity_id, title, description, content)
+                VALUES (?, ?, ?, ?)
+                """,
+                (entity_id, title, description, content),
+            )
+
+            # Insert into metadata table
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO archive_metadata
+                (entity_id, archive_file, entity_type, status, created, updated)
+                VALUES (?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    entity_id,
+                    archive_file,
+                    entity.get("type", ""),
+                    entity.get("status", ""),
+                    entity.get("created", ""),
+                    entity.get("updated", ""),
+                ),
+            )
+
+        conn.commit()
+
+    def search(
+        self,
+        query: str,
+        limit: int = 10,
+        archive_files: list[str] | None = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Search indexed archives with BM25 ranking.
+
+        Args:
+            query: Search query
+            limit: Maximum number of results
+            archive_files: Optional list of archive files to search
+
+        Returns:
+            List of results with entity_id, title, rank, snippet, archive_file
+        """
+        conn = self._get_connection()
+
+        # Build query with optional archive file filter
+        if archive_files:
+            placeholders = ",".join("?" * len(archive_files))
+            sql = f"""
+                SELECT
+                    fts.entity_id,
+                    meta.archive_file,
+                    meta.entity_type,
+                    meta.status,
+                    snippet(archive_fts, 1, '<mark>', '</mark>', '...', 32) as title_snippet,
+                    snippet(archive_fts, 2, '<mark>', '</mark>', '...', 64) as description_snippet,
+                    bm25(archive_fts) as rank
+                FROM archive_fts fts
+                JOIN archive_metadata meta ON fts.entity_id = meta.entity_id
+                WHERE archive_fts MATCH ?
+                  AND meta.archive_file IN ({placeholders})
+                ORDER BY rank
+                LIMIT ?
+            """
+            params = [query] + archive_files + [limit]
+        else:
+            sql = """
+                SELECT
+                    fts.entity_id,
+                    meta.archive_file,
+                    meta.entity_type,
+                    meta.status,
+                    snippet(archive_fts, 1, '<mark>', '</mark>', '...', 32) as title_snippet,
+                    snippet(archive_fts, 2, '<mark>', '</mark>', '...', 64) as description_snippet,
+                    bm25(archive_fts) as rank
+                FROM archive_fts fts
+                JOIN archive_metadata meta ON fts.entity_id = meta.entity_id
+                WHERE archive_fts MATCH ?
+                ORDER BY rank
+                LIMIT ?
+            """
+            params = [query, limit]
+
+        cursor = conn.execute(sql, params)
+
+        results = []
+        for row in cursor:
+            results.append(
+                {
+                    "entity_id": row["entity_id"],
+                    "archive_file": row["archive_file"],
+                    "entity_type": row["entity_type"],
+                    "status": row["status"],
+                    "title_snippet": row["title_snippet"],
+                    "description_snippet": row["description_snippet"],
+                    "rank": row["rank"],
+                }
+            )
+
+        return results
+
+    def get_entity_metadata(self, entity_id: str) -> dict[str, Any] | None:
+        """
+        Get metadata for an entity.
+
+        Args:
+            entity_id: Entity identifier
+
+        Returns:
+            Metadata dictionary or None if not found
+        """
+        conn = self._get_connection()
+
+        cursor = conn.execute(
+            """
+            SELECT entity_id, archive_file, entity_type, status, created, updated
+            FROM archive_metadata
+            WHERE entity_id = ?
+            """,
+            (entity_id,),
+        )
+
+        row = cursor.fetchone()
+        if row:
+            return dict(row)
+        return None
+
+    def remove_archive(self, archive_file: str) -> None:
+        """
+        Remove all entities from a specific archive file.
+
+        Args:
+            archive_file: Archive file to remove
+        """
+        conn = self._get_connection()
+
+        # Get entity IDs to remove
+        cursor = conn.execute(
+            "SELECT entity_id FROM archive_metadata WHERE archive_file = ?",
+            (archive_file,),
+        )
+        entity_ids = [row["entity_id"] for row in cursor]
+
+        # Remove from FTS5
+        for entity_id in entity_ids:
+            conn.execute("DELETE FROM archive_fts WHERE entity_id = ?", (entity_id,))
+
+        # Remove from metadata
+        conn.execute(
+            "DELETE FROM archive_metadata WHERE archive_file = ?", (archive_file,)
+        )
+
+        conn.commit()
+
+    def get_stats(self) -> dict[str, Any]:
+        """
+        Get index statistics.
+
+        Returns:
+            Dictionary with entity count, archive count, etc.
+        """
+        conn = self._get_connection()
+
+        # Count entities
+        cursor = conn.execute("SELECT COUNT(*) as count FROM archive_metadata")
+        entity_count = cursor.fetchone()["count"]
+
+        # Count archives
+        cursor = conn.execute(
+            "SELECT COUNT(DISTINCT archive_file) as count FROM archive_metadata"
+        )
+        archive_count = cursor.fetchone()["count"]
+
+        # Get database size
+        db_size = self.db_path.stat().st_size if self.db_path.exists() else 0
+
+        return {
+            "entity_count": entity_count,
+            "archive_count": archive_count,
+            "db_size_bytes": db_size,
+            "db_size_mb": db_size / (1024 * 1024),
+        }
+
+    def close(self) -> None:
+        """Close database connection."""
+        if self.conn:
+            self.conn.close()
+            self.conn = None
+
+    def __enter__(self) -> "ArchiveFTS5Index":
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        """Context manager exit."""
+        self.close()