omni-cortex 1.17.3__py3-none-any.whl → 1.17.4__py3-none-any.whl

omni_cortex/search/keyword.py

@@ -1,204 +0,0 @@
-"""Keyword search using SQLite FTS5."""
-
-import sqlite3
-from typing import Optional
-
-from ..models.memory import Memory, _row_to_memory
-
-
-def keyword_search(
-    conn: sqlite3.Connection,
-    query: str,
-    type_filter: Optional[str] = None,
-    tags_filter: Optional[list[str]] = None,
-    status_filter: Optional[str] = None,
-    min_importance: Optional[int] = None,
-    include_archived: bool = False,
-    limit: int = 10,
-) -> list[tuple[Memory, float]]:
-    """Search memories using FTS5 keyword search.
-
-    Args:
-        conn: Database connection
-        query: Search query string
-        type_filter: Filter by memory type
-        tags_filter: Filter by tags
-        status_filter: Filter by status
-        min_importance: Minimum importance score
-        include_archived: Include archived memories
-        limit: Maximum results
-
-    Returns:
-        List of (Memory, score) tuples
-    """
-    # Build the FTS query
-    # Escape special FTS5 characters
-    fts_query = _escape_fts_query(query)
-
-    # Build WHERE conditions for the join
-    where_conditions = []
-    params: list = [fts_query]
-
-    if type_filter:
-        where_conditions.append("m.type = ?")
-        params.append(type_filter)
-
-    if status_filter:
-        where_conditions.append("m.status = ?")
-        params.append(status_filter)
-    elif not include_archived:
-        where_conditions.append("m.status != 'archived'")
-
-    if min_importance is not None:
-        where_conditions.append("m.importance_score >= ?")
-        params.append(min_importance)
-
-    if tags_filter:
-        tag_conditions = []
-        for tag in tags_filter:
-            tag_conditions.append("m.tags LIKE ?")
-            params.append(f'%"{tag}"%')
-        where_conditions.append(f"({' OR '.join(tag_conditions)})")
-
-    where_sql = ""
-    if where_conditions:
-        where_sql = "AND " + " AND ".join(where_conditions)
-
-    params.append(limit)
-
-    cursor = conn.cursor()
-
-    # Use FTS5 with bm25 ranking
-    try:
-        cursor.execute(
-            f"""
-            SELECT m.*, bm25(memories_fts) as score
-            FROM memories_fts fts
-            JOIN memories m ON fts.rowid = m.rowid
-            WHERE memories_fts MATCH ?
-            {where_sql}
-            ORDER BY score
-            LIMIT ?
-            """,
-            params,
-        )
-
-        results = []
-        for row in cursor.fetchall():
-            # bm25 returns negative scores (more negative = better match)
-            # Convert to positive scores for our ranking
-            score = -row["score"]
-            memory = _row_to_memory(row)
-            results.append((memory, score))
-
-        return results
-
-    except sqlite3.OperationalError as e:
-        # If FTS query fails, fall back to LIKE search
-        if "fts5" in str(e).lower() or "match" in str(e).lower():
-            return _fallback_like_search(
-                conn, query, type_filter, tags_filter, status_filter,
-                min_importance, include_archived, limit
-            )
-        raise
-
-
-def _escape_fts_query(query: str) -> str:
-    """Escape special characters for FTS5 query.
-
-    Args:
-        query: Raw search query
-
-    Returns:
-        Escaped FTS5 query
-    """
-    # Remove FTS5 special characters that could cause syntax errors
-    special_chars = ['"', "'", "(", ")", "*", ":", "^", "-", "+"]
-    escaped = query
-    for char in special_chars:
-        escaped = escaped.replace(char, " ")
-
-    # Clean up whitespace
-    words = escaped.split()
-
-    # Handle empty query
-    if not words:
-        return '""'
-
-    # For simple queries, just use OR matching
-    if len(words) == 1:
-        return f'"{words[0]}"'
-
-    # For multi-word queries, match any word
-    return " OR ".join(f'"{word}"' for word in words)
-
-
-def _fallback_like_search(
-    conn: sqlite3.Connection,
-    query: str,
-    type_filter: Optional[str],
-    tags_filter: Optional[list[str]],
-    status_filter: Optional[str],
-    min_importance: Optional[int],
-    include_archived: bool,
-    limit: int,
-) -> list[tuple[Memory, float]]:
-    """Fallback to LIKE search if FTS5 fails."""
-    words = query.lower().split()
-    if not words:
-        return []
-
-    where_conditions = []
-    params: list = []
-
-    # Match any word in content or context
-    word_conditions = []
-    for word in words:
-        word_conditions.append("(LOWER(content) LIKE ? OR LOWER(context) LIKE ?)")
-        params.extend([f"%{word}%", f"%{word}%"])
-    where_conditions.append(f"({' OR '.join(word_conditions)})")
-
-    if type_filter:
-        where_conditions.append("type = ?")
-        params.append(type_filter)
-
-    if status_filter:
-        where_conditions.append("status = ?")
-        params.append(status_filter)
-    elif not include_archived:
-        where_conditions.append("status != 'archived'")
-
-    if min_importance is not None:
-        where_conditions.append("importance_score >= ?")
-        params.append(min_importance)
-
-    if tags_filter:
-        tag_conds = []
-        for tag in tags_filter:
-            tag_conds.append("tags LIKE ?")
-            params.append(f'%"{tag}"%')
-        where_conditions.append(f"({' OR '.join(tag_conds)})")
-
-    params.append(limit)
-
-    cursor = conn.cursor()
-    cursor.execute(
-        f"""
-        SELECT *
-        FROM memories
-        WHERE {' AND '.join(where_conditions)}
-        ORDER BY importance_score DESC, last_accessed DESC
-        LIMIT ?
-        """,
-        params,
-    )
-
-    results = []
-    for row in cursor.fetchall():
-        memory = _row_to_memory(row)
-        # Calculate a simple score based on word matches
-        content = (memory.content + " " + (memory.context or "")).lower()
-        score = sum(1 for word in words if word in content)
-        results.append((memory, float(score)))
-
-    return results

omni_cortex/search/ranking.py

@@ -1,127 +0,0 @@
-"""Multi-factor relevance ranking for search results."""
-
-import math
-from datetime import datetime, timezone
-from typing import Optional
-
-from ..models.memory import Memory
-from ..utils.timestamps import parse_iso
-
-
-def calculate_relevance_score(
-    memory: Memory,
-    keyword_score: float = 0.0,
-    semantic_score: float = 0.0,
-    query: Optional[str] = None,
-) -> float:
-    """Calculate multi-factor relevance score for a memory.
-
-    Scoring factors:
-    - Keyword match score (40%)
-    - Semantic similarity score (40%)
-    - Access frequency (log scale, max +20)
-    - Recency (exponential decay, max +15)
-    - Freshness status bonus/penalty
-    - Importance score (0-15)
-
-    Args:
-        memory: Memory object to score
-        keyword_score: Score from keyword search (0-1 normalized)
-        semantic_score: Score from semantic search (0-1 normalized)
-        query: Optional query string for additional matching
-
-    Returns:
-        Combined relevance score (higher = more relevant)
-    """
-    score = 0.0
-
-    # Base scores from search (40% each, max 80 combined)
-    # Normalize to 0-40 range
-    score += min(40.0, keyword_score * 40.0)
-    score += min(40.0, semantic_score * 40.0)
-
-    # Access frequency bonus (log scale, max +20)
-    # More frequently accessed memories are likely more useful
-    access_count = memory.access_count or 0
-    access_bonus = min(20.0, math.log1p(access_count) * 5.0)
-    score += access_bonus
-
-    # Recency bonus (exponential decay over 30 days, max +15)
-    # Recently accessed memories are more relevant
-    last_accessed = parse_iso(memory.last_accessed)
-    now = datetime.now(timezone.utc)
-    days_since_access = (now - last_accessed).days
-
-    recency_bonus = max(0.0, 15.0 * math.exp(-days_since_access / 30.0))
-    score += recency_bonus
-
-    # Freshness status bonus/penalty
-    freshness_bonus = {
-        "fresh": 10.0,
-        "needs_review": 0.0,
-        "outdated": -10.0,
-        "archived": -30.0,
-    }
-    score += freshness_bonus.get(memory.status, 0.0)
-
-    # Importance score contribution (0-100 scaled to 0-15)
-    importance = memory.importance_score or 50.0
-    score += importance * 0.15
-
-    # Exact phrase match bonus (if query provided)
-    if query and query.lower() in memory.content.lower():
-        score += 10.0
-
-    return score
-
-
-def rank_memories(
-    memories_with_scores: list[tuple[Memory, float, float]],
-    query: Optional[str] = None,
-) -> list[tuple[Memory, float]]:
-    """Rank memories by combined relevance score.
-
-    Args:
-        memories_with_scores: List of (Memory, keyword_score, semantic_score) tuples
-        query: Optional query string
-
-    Returns:
-        List of (Memory, final_score) tuples, sorted by score descending
-    """
-    results = []
-
-    for memory, keyword_score, semantic_score in memories_with_scores:
-        final_score = calculate_relevance_score(
-            memory,
-            keyword_score=keyword_score,
-            semantic_score=semantic_score,
-            query=query,
-        )
-        results.append((memory, final_score))
-
-    # Sort by score descending
-    results.sort(key=lambda x: x[1], reverse=True)
-
-    return results
-
-
-def normalize_scores(scores: list[float]) -> list[float]:
-    """Normalize a list of scores to 0-1 range.
-
-    Args:
-        scores: List of raw scores
-
-    Returns:
-        List of normalized scores
-    """
-    if not scores:
-        return []
-
-    min_score = min(scores)
-    max_score = max(scores)
-    range_score = max_score - min_score
-
-    if range_score == 0:
-        return [1.0] * len(scores)
-
-    return [(s - min_score) / range_score for s in scores]

omni_cortex/search/semantic.py

@@ -1,232 +0,0 @@
-"""Semantic search using vector embeddings."""
-
-import logging
-import sqlite3
-from typing import Optional
-
-import numpy as np
-
-from ..models.memory import Memory, _row_to_memory
-from ..embeddings.local import (
-    generate_embedding,
-    blob_to_vector,
-    DEFAULT_MODEL_NAME,
-)
-from ..config import load_config
-
-logger = logging.getLogger(__name__)
-
-
-def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
-    """Calculate cosine similarity between two vectors.
-
-    Args:
-        a: First vector
-        b: Second vector
-
-    Returns:
-        Similarity score between -1 and 1
-    """
-    dot_product = np.dot(a, b)
-    norm_a = np.linalg.norm(a)
-    norm_b = np.linalg.norm(b)
-
-    if norm_a == 0 or norm_b == 0:
-        return 0.0
-
-    return float(dot_product / (norm_a * norm_b))
-
-
-def semantic_search(
-    conn: sqlite3.Connection,
-    query: str,
-    type_filter: Optional[str] = None,
-    tags_filter: Optional[list[str]] = None,
-    status_filter: Optional[str] = None,
-    min_importance: Optional[int] = None,
-    include_archived: bool = False,
-    limit: int = 10,
-    similarity_threshold: float = 0.3,
-    model_name: str = DEFAULT_MODEL_NAME,
-) -> list[tuple[Memory, float]]:
-    """Search memories using semantic similarity.
-
-    Args:
-        conn: Database connection
-        query: Search query string
-        type_filter: Filter by memory type
-        tags_filter: Filter by tags
-        status_filter: Filter by status
-        min_importance: Minimum importance score
-        include_archived: Include archived memories
-        limit: Maximum results
-        similarity_threshold: Minimum similarity score
-        model_name: Embedding model to use
-
-    Returns:
-        List of (Memory, similarity_score) tuples
-    """
-    # Check if embeddings are enabled - skip semantic search if disabled
-    config = load_config()
-    if not config.embedding_enabled:
-        logger.debug("Embeddings disabled, skipping semantic search")
-        return []
-
-    # Generate embedding for query
-    try:
-        query_embedding = generate_embedding(query, model_name)
-    except ImportError:
-        logger.warning("sentence-transformers not available, cannot perform semantic search")
-        return []
-    except Exception as e:
-        logger.error(f"Failed to generate query embedding: {e}")
-        return []
-
-    # Build WHERE conditions for filtering
-    where_conditions = []
-    params: list = []
-
-    # Only search memories with embeddings
-    where_conditions.append("m.has_embedding = 1")
-
-    if type_filter:
-        where_conditions.append("m.type = ?")
-        params.append(type_filter)
-
-    if status_filter:
-        where_conditions.append("m.status = ?")
-        params.append(status_filter)
-    elif not include_archived:
-        where_conditions.append("m.status != 'archived'")
-
-    if min_importance is not None:
-        where_conditions.append("m.importance_score >= ?")
-        params.append(min_importance)
-
-    if tags_filter:
-        tag_conditions = []
-        for tag in tags_filter:
-            tag_conditions.append("m.tags LIKE ?")
-            params.append(f'%"{tag}"%')
-        where_conditions.append(f"({' OR '.join(tag_conditions)})")
-
-    where_sql = "WHERE " + " AND ".join(where_conditions)
-
-    # Get all matching memories with their embeddings
-    cursor = conn.cursor()
-    cursor.execute(
-        f"""
-        SELECT m.*, e.vector
-        FROM memories m
-        JOIN embeddings e ON m.id = e.memory_id
-        {where_sql}
-        """,
-        params,
-    )
-
-    # Calculate similarity scores
-    results = []
-    for row in cursor.fetchall():
-        memory_embedding = blob_to_vector(row["vector"])
-        similarity = cosine_similarity(query_embedding, memory_embedding)
-
-        # Apply threshold
-        if similarity >= similarity_threshold:
-            memory = _row_to_memory(row)
-            results.append((memory, similarity))
-
-    # Sort by similarity (highest first)
-    results.sort(key=lambda x: x[1], reverse=True)
-
-    # Limit results
-    return results[:limit]
-
-
-def find_similar_memories(
-    conn: sqlite3.Connection,
-    memory_id: str,
-    limit: int = 10,
-    similarity_threshold: float = 0.5,
-    exclude_ids: Optional[list[str]] = None,
-) -> list[tuple[Memory, float]]:
-    """Find memories similar to a given memory.
-
-    Args:
-        conn: Database connection
-        memory_id: ID of the source memory
-        limit: Maximum results
-        similarity_threshold: Minimum similarity score
-        exclude_ids: Memory IDs to exclude
-
-    Returns:
-        List of (Memory, similarity_score) tuples
-    """
-    # Get the source memory's embedding
-    cursor = conn.cursor()
-    cursor.execute(
-        "SELECT vector FROM embeddings WHERE memory_id = ?",
-        (memory_id,),
-    )
-    row = cursor.fetchone()
-
-    if not row:
-        logger.warning(f"No embedding found for memory {memory_id}")
-        return []
-
-    source_embedding = blob_to_vector(row["vector"])
-
-    # Build exclusion list
-    exclude_set = set(exclude_ids or [])
-    exclude_set.add(memory_id)  # Always exclude the source
-
-    # Get all other embeddings
-    cursor.execute(
-        """
-        SELECT m.*, e.vector
-        FROM memories m
-        JOIN embeddings e ON m.id = e.memory_id
-        WHERE m.has_embedding = 1 AND m.status != 'archived'
-        """
-    )
-
-    results = []
-    for row in cursor.fetchall():
-        if row["id"] in exclude_set:
-            continue
-
-        memory_embedding = blob_to_vector(row["vector"])
-        similarity = cosine_similarity(source_embedding, memory_embedding)
-
-        if similarity >= similarity_threshold:
-            memory = _row_to_memory(row)
-            results.append((memory, similarity))
-
-    # Sort by similarity
-    results.sort(key=lambda x: x[1], reverse=True)
-
-    return results[:limit]
-
-
-def get_embedding_coverage(conn: sqlite3.Connection) -> dict:
-    """Get statistics about embedding coverage.
-
-    Returns:
-        Dict with total_memories, with_embeddings, without_embeddings, coverage_pct
-    """
-    cursor = conn.cursor()
-
-    cursor.execute("SELECT COUNT(*) FROM memories")
-    total = cursor.fetchone()[0]
-
-    cursor.execute("SELECT COUNT(*) FROM memories WHERE has_embedding = 1")
-    with_embeddings = cursor.fetchone()[0]
-
-    without_embeddings = total - with_embeddings
-    coverage_pct = (with_embeddings / total * 100) if total > 0 else 0.0
-
-    return {
-        "total_memories": total,
-        "with_embeddings": with_embeddings,
-        "without_embeddings": without_embeddings,
-        "coverage_pct": round(coverage_pct, 1),
-    }