mem-llm 1.0.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

mem_llm/memory_db.py

@@ -5,28 +5,93 @@ Stores memory data using SQLite - Production-ready
 
 import sqlite3
 import json
+import threading
 from datetime import datetime
 from typing import Dict, List, Optional, Tuple
 from pathlib import Path
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Optional vector store support
+try:
+    from .vector_store import create_vector_store, VectorStore
+    VECTOR_STORE_AVAILABLE = True
+except ImportError:
+    VECTOR_STORE_AVAILABLE = False
+    VectorStore = None
 
 
 class SQLMemoryManager:
-    """SQLite-based memory management system"""
+    """SQLite-based memory management system with thread-safety"""
     
-    def __init__(self, db_path: str = "memories.db"):
+    def __init__(self, db_path: str = "memories/memories.db",
+                 enable_vector_search: bool = False,
+                 vector_store_type: str = "chroma",
+                 embedding_model: str = "all-MiniLM-L6-v2"):
         """
         Args:
             db_path: SQLite database file path
+            enable_vector_search: Enable vector/semantic search (optional)
+            vector_store_type: Type of vector store ('chroma', etc.)
+            embedding_model: Embedding model name (sentence-transformers)
         """
         self.db_path = Path(db_path)
+        
+        # Ensure directory exists
+        db_dir = self.db_path.parent
+        if not db_dir.exists():
+            db_dir.mkdir(parents=True, exist_ok=True)
+        
         self.conn = None
+        self._lock = threading.RLock()  # Reentrant lock for thread safety
         self._init_database()
+        
+        # Vector store (optional)
+        self.enable_vector_search = enable_vector_search
+        self.vector_store: Optional[VectorStore] = None
+        
+        if enable_vector_search:
+            if not VECTOR_STORE_AVAILABLE:
+                logger.warning(
+                    "Vector search requested but dependencies not available. "
+                    "Install with: pip install chromadb sentence-transformers"
+                )
+                self.enable_vector_search = False
+            else:
+                try:
+                    persist_dir = str(db_dir / "vector_store")
+                    self.vector_store = create_vector_store(
+                        store_type=vector_store_type,
+                        collection_name="knowledge_base",
+                        persist_directory=persist_dir,
+                        embedding_model=embedding_model
+                    )
+                    if self.vector_store:
+                        logger.info(f"Vector search enabled: {vector_store_type}")
+                    else:
+                        logger.warning("Failed to initialize vector store, falling back to keyword search")
+                        self.enable_vector_search = False
+                except Exception as e:
+                    logger.error(f"Error initializing vector store: {e}")
+                    self.enable_vector_search = False
     
     def _init_database(self) -> None:
         """Create database and tables"""
-        self.conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
+        self.conn = sqlite3.connect(
+            str(self.db_path), 
+            check_same_thread=False,
+            timeout=30.0,  # 30 second timeout for busy database
+            isolation_level=None  # Autocommit mode
+        )
         self.conn.row_factory = sqlite3.Row
         
+        # Enable WAL mode for better concurrency
+        self.conn.execute("PRAGMA journal_mode=WAL")
+        self.conn.execute("PRAGMA synchronous=NORMAL")
+        self.conn.execute("PRAGMA cache_size=-64000")  # 64MB cache
+        self.conn.execute("PRAGMA busy_timeout=30000")  # 30 second busy timeout
+        
         cursor = self.conn.cursor()
         
         # User profiles table
@@ -106,28 +171,28 @@ class SQLMemoryManager:
     def add_user(self, user_id: str, name: Optional[str] = None, 
                  metadata: Optional[Dict] = None) -> None:
         """
-        Add new user or update existing
+        Add new user or update existing (thread-safe)
         
         Args:
             user_id: User ID
             name: User name
             metadata: Additional information
         """
-        cursor = self.conn.cursor()
-        cursor.execute("""
-            INSERT INTO users (user_id, name, metadata)
-            VALUES (?, ?, ?)
-            ON CONFLICT(user_id) DO UPDATE SET
-                name = COALESCE(excluded.name, users.name),
-                metadata = COALESCE(excluded.metadata, users.metadata)
-        """, (user_id, name, json.dumps(metadata or {})))
-        self.conn.commit()
+        with self._lock:
+            cursor = self.conn.cursor()
+            cursor.execute("""
+                INSERT INTO users (user_id, name, metadata)
+                VALUES (?, ?, ?)
+                ON CONFLICT(user_id) DO UPDATE SET
+                    name = COALESCE(excluded.name, users.name),
+                    metadata = COALESCE(excluded.metadata, users.metadata)
+            """, (user_id, name, json.dumps(metadata or {})))
     
     def add_interaction(self, user_id: str, user_message: str, 
                        bot_response: str, metadata: Optional[Dict] = None,
                        resolved: bool = False) -> int:
         """
-        Record new interaction
+        Record new interaction (thread-safe)
         
         Args:
             user_id: User ID
@@ -139,30 +204,33 @@ class SQLMemoryManager:
         Returns:
             Added record ID
         """
-        cursor = self.conn.cursor()
-        
-        # Create user if not exists
-        self.add_user(user_id)
-        
-        # Record interaction
-        cursor.execute("""
-            INSERT INTO conversations 
-            (user_id, user_message, bot_response, metadata, resolved)
-            VALUES (?, ?, ?, ?, ?)
-        """, (user_id, user_message, bot_response, 
-              json.dumps(metadata or {}), resolved))
-        
-        interaction_id = cursor.lastrowid
-        
-        # Update user's last interaction time
-        cursor.execute("""
-            UPDATE users 
-            SET last_interaction = CURRENT_TIMESTAMP
-            WHERE user_id = ?
-        """, (user_id,))
+        if not user_message or not bot_response:
+            raise ValueError("user_message and bot_response cannot be None or empty")
         
-        self.conn.commit()
-        return interaction_id
+        with self._lock:
+            cursor = self.conn.cursor()
+            
+            # Create user if not exists
+            self.add_user(user_id)
+            
+            # Record interaction
+            cursor.execute("""
+                INSERT INTO conversations 
+                (user_id, user_message, bot_response, metadata, resolved)
+                VALUES (?, ?, ?, ?, ?)
+            """, (user_id, user_message, bot_response, 
+                  json.dumps(metadata or {}), resolved))
+            
+            interaction_id = cursor.lastrowid
+            
+            # Update user's last interaction time
+            cursor.execute("""
+                UPDATE users 
+                SET last_interaction = CURRENT_TIMESTAMP
+                WHERE user_id = ?
+            """, (user_id,))
+            
+            return interaction_id
     
     # Alias for compatibility
     def add_conversation(self, user_id: str, user_message: str, bot_response: str, metadata: Optional[Dict] = None) -> int:
@@ -171,7 +239,7 @@ class SQLMemoryManager:
     
     def get_recent_conversations(self, user_id: str, limit: int = 10) -> List[Dict]:
         """
-        Kullanıcının son konuşmalarını getirir
+        Kullanıcının son konuşmalarını getirir (thread-safe)
         
         Args:
             user_id: Kullanıcı kimliği
@@ -180,21 +248,22 @@ class SQLMemoryManager:
         Returns:
             Konuşmalar listesi
         """
-        cursor = self.conn.cursor()
-        cursor.execute("""
-            SELECT timestamp, user_message, bot_response, metadata, resolved
-            FROM conversations
-            WHERE user_id = ?
-            ORDER BY timestamp DESC
-            LIMIT ?
-        """, (user_id, limit))
-        
-        rows = cursor.fetchall()
-        return [dict(row) for row in rows]
+        with self._lock:
+            cursor = self.conn.cursor()
+            cursor.execute("""
+                SELECT timestamp, user_message, bot_response, metadata, resolved
+                FROM conversations
+                WHERE user_id = ?
+                ORDER BY timestamp DESC
+                LIMIT ?
+            """, (user_id, limit))
+            
+            rows = cursor.fetchall()
+            return [dict(row) for row in rows]
     
     def search_conversations(self, user_id: str, keyword: str) -> List[Dict]:
         """
-        Konuşmalarda anahtar kelime arar
+        Konuşmalarda anahtar kelime arar (thread-safe)
         
         Args:
             user_id: Kullanıcı kimliği
@@ -289,46 +358,208 @@ class SQLMemoryManager:
         """, (category, question, answer, 
               json.dumps(keywords or []), priority))
         
+        kb_id = cursor.lastrowid
         self.conn.commit()
-        return cursor.lastrowid
+        
+        # Sync to vector store if enabled
+        if self.enable_vector_search and self.vector_store:
+            try:
+                self._sync_to_vector_store(kb_id)
+            except Exception as e:
+                logger.warning(f"Failed to sync KB entry to vector store: {e}")
+        
+        return kb_id
     
     def search_knowledge(self, query: str, category: Optional[str] = None,
-                        limit: int = 5) -> List[Dict]:
+                        limit: int = 5, use_vector_search: Optional[bool] = None) -> List[Dict]:
         """
-        Bilgi bankasında arama yapar
+        Bilgi bankasında arama yapar (keyword matching veya semantic search)
         
         Args:
             query: Arama sorgusu
             category: Kategori filtresi (opsiyonel)
             limit: Maksimum sonuç sayısı
+            use_vector_search: Force vector search (None = auto-detect)
             
         Returns:
             Bulunan kayıtlar
         """
+        # Use vector search if enabled and available
+        if use_vector_search is None:
+            use_vector_search = self.enable_vector_search
+        
+        if use_vector_search and self.vector_store:
+            return self._vector_search(query, category, limit)
+        else:
+            return self._keyword_search(query, category, limit)
+    
+    def _keyword_search(self, query: str, category: Optional[str] = None,
+                       limit: int = 5) -> List[Dict]:
+        """Traditional keyword-based search"""
         cursor = self.conn.cursor()
         
+        # Extract important keywords from query (remove question words)
+        import re
+        stopwords = ['ne', 'kadar', 'nedir', 'nasıl', 'için', 'mı', 'mi', 'mu', 'mü', 
+                     'what', 'how', 'when', 'where', 'is', 'are', 'the', 'a', 'an']
+        
+        # Clean query and extract keywords
+        query_lower = query.lower()
+        words = re.findall(r'\w+', query_lower)
+        keywords = [w for w in words if w not in stopwords and len(w) > 2]
+        
+        # If no keywords, use original query
+        if not keywords:
+            keywords = [query_lower]
+        
+        # Build search conditions for each keyword
+        conditions = []
+        params = []
+        
+        for keyword in keywords[:5]:  # Max 5 keywords
+            conditions.append("(question LIKE ? OR answer LIKE ? OR keywords LIKE ?)")
+            params.extend([f"%{keyword}%", f"%{keyword}%", f"%{keyword}%"])
+        
+        where_clause = " OR ".join(conditions) if conditions else "1=1"
+        
         if category:
-            cursor.execute("""
+            sql = f"""
                 SELECT category, question, answer, priority
                 FROM knowledge_base
                 WHERE active = 1 
                 AND category = ?
-                AND (question LIKE ? OR answer LIKE ? OR keywords LIKE ?)
+                AND ({where_clause})
                 ORDER BY priority DESC, id DESC
                 LIMIT ?
-            """, (category, f"%{query}%", f"%{query}%", f"%{query}%", limit))
+            """
+            cursor.execute(sql, [category] + params + [limit])
         else:
-            cursor.execute("""
+            sql = f"""
                 SELECT category, question, answer, priority
                 FROM knowledge_base
                 WHERE active = 1 
-                AND (question LIKE ? OR answer LIKE ? OR keywords LIKE ?)
+                AND ({where_clause})
                 ORDER BY priority DESC, id DESC
                 LIMIT ?
-            """, (f"%{query}%", f"%{query}%", f"%{query}%", limit))
+            """
+            cursor.execute(sql, params + [limit])
         
         return [dict(row) for row in cursor.fetchall()]
     
+    def _vector_search(self, query: str, category: Optional[str] = None,
+                      limit: int = 5) -> List[Dict]:
+        """Vector-based semantic search"""
+        if not self.vector_store:
+            return []
+        
+        # Prepare metadata filter
+        filter_metadata = None
+        if category:
+            filter_metadata = {"category": category}
+        
+        # Search in vector store
+        vector_results = self.vector_store.search(
+            query=query,
+            limit=limit * 2,  # Get more results to filter by category if needed
+            filter_metadata=filter_metadata
+        )
+        
+        # Map vector results back to KB format
+        results = []
+        for result in vector_results[:limit]:
+            # Extract metadata
+            metadata = result.get('metadata', {})
+            
+            results.append({
+                'category': metadata.get('category', ''),
+                'question': metadata.get('question', ''),
+                'answer': result.get('text', ''),
+                'priority': metadata.get('priority', 0),
+                'score': result.get('score', 0.0),  # Similarity score
+                'vector_search': True
+            })
+        
+        return results
+    
+    def _sync_to_vector_store(self, kb_id: int) -> None:
+        """Sync a single KB entry to vector store"""
+        if not self.vector_store:
+            return
+        
+        cursor = self.conn.cursor()
+        cursor.execute("""
+            SELECT id, category, question, answer, keywords, priority
+            FROM knowledge_base
+            WHERE id = ?
+        """, (kb_id,))
+        
+        row = cursor.fetchone()
+        if row:
+            doc = {
+                'id': str(row['id']),
+                'text': f"{row['question']}\n{row['answer']}",  # Combine for better search
+                'metadata': {
+                    'category': row['category'],
+                    'question': row['question'],
+                    'answer': row['answer'],
+                    'keywords': row['keywords'],
+                    'priority': row['priority'],
+                    'kb_id': row['id']
+                }
+            }
+            self.vector_store.add_documents([doc])
+    
+    def sync_all_kb_to_vector_store(self) -> int:
+        """
+        Sync all existing KB entries to vector store
+        
+        Returns:
+            Number of entries synced
+        """
+        if not self.vector_store:
+            return 0
+        
+        cursor = self.conn.cursor()
+        cursor.execute("""
+            SELECT id, category, question, answer, keywords, priority
+            FROM knowledge_base
+            WHERE active = 1
+        """)
+        
+        rows = cursor.fetchall()
+        documents = []
+        
+        for row in rows:
+            doc = {
+                'id': str(row['id']),
+                'text': f"{row['question']}\n{row['answer']}",
+                'metadata': {
+                    'category': row['category'],
+                    'question': row['question'],
+                    'answer': row['answer'],
+                    'keywords': row['keywords'],
+                    'priority': row['priority'],
+                    'kb_id': row['id']
+                }
+            }
+            documents.append(doc)
+        
+        if documents:
+            try:
+                # Add in batches for better performance
+                batch_size = 100
+                for i in range(0, len(documents), batch_size):
+                    batch = documents[i:i + batch_size]
+                    self.vector_store.add_documents(batch)
+                    logger.debug(f"Synced {len(batch)} KB entries to vector store")
+                
+                logger.info(f"Synced {len(documents)} KB entries to vector store")
+            except Exception as e:
+                logger.error(f"Error syncing KB to vector store: {e}")
+                return 0
+        
+        return len(documents)
+    
     def get_statistics(self) -> Dict:
         """
         Genel istatistikleri döndürür

mem_llm/memory_manager.py

@@ -43,7 +43,16 @@ class MemoryManager:
             with open(user_file, 'r', encoding='utf-8') as f:
                 data = json.load(f)
                 self.conversations[user_id] = data.get('conversations', [])
-                self.user_profiles[user_id] = data.get('profile', {})
+                profile = data.get('profile', {})
+                
+                # Parse preferences if it's a JSON string (legacy format)
+                if isinstance(profile.get('preferences'), str):
+                    try:
+                        profile['preferences'] = json.loads(profile['preferences'])
+                    except:
+                        profile['preferences'] = {}
+                
+                self.user_profiles[user_id] = profile
                 return data
         else:
             # Create empty memory for new user
@@ -259,4 +268,54 @@ class MemoryManager:
             self.load_memory(user_id)
 
         return self.user_profiles.get(user_id)
+    
+    def update_user_profile(self, user_id: str, updates: Dict) -> None:
+        """
+        Update user profile (SQL-compatible alias)
+        
+        Args:
+            user_id: User ID
+            updates: Fields to update
+        """
+        return self.update_profile(user_id, updates)
+    
+    def add_user(self, user_id: str, name: Optional[str] = None, metadata: Optional[Dict] = None) -> None:
+        """
+        Add or update user (SQL-compatible method)
+        
+        Args:
+            user_id: User ID
+            name: User name (optional)
+            metadata: Additional metadata (optional)
+        """
+        self.load_memory(user_id)
+        if name and 'name' not in self.user_profiles[user_id]:
+            self.user_profiles[user_id]['name'] = name
+        if metadata:
+            self.user_profiles[user_id].update(metadata)
+        self.save_memory(user_id)
+    
+    def get_statistics(self) -> Dict:
+        """
+        Get general statistics (SQL-compatible method)
+        
+        Returns:
+            Statistics dictionary
+        """
+        all_users = list(self.memory_dir.glob("*.json"))
+        total_interactions = 0
+        
+        for user_file in all_users:
+            try:
+                with open(user_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    total_interactions += len(data.get('conversations', []))
+            except:
+                pass
+        
+        return {
+            'total_users': len(all_users),
+            'total_interactions': total_interactions,
+            'knowledge_base_entries': 0  # JSON doesn't have KB
+        }