memorisdk 2.1.1__py3-none-any.whl → 2.3.1__py3-none-any.whl

memori/__init__.py

@@ -5,7 +5,7 @@ Professional-grade memory layer with comprehensive error handling, configuration
 management, and modular architecture for production AI systems.
 """
 
-__version__ = "2.1.1"
+__version__ = "2.3.0"
 __author__ = "Harshal More"
 __email__ = "harshalmore2468@gmail.com"
 

memori/agents/conscious_agent.py

@@ -116,7 +116,7 @@ class ConsciouscAgent:
             return False
 
     async def initialize_existing_conscious_memories(
-        self, db_manager, namespace: str = "default"
+        self, db_manager, namespace: str = "default", limit: int = 10
     ) -> bool:
         """
         Initialize by copying ALL existing conscious-info memories to short-term memory
@@ -143,16 +143,17 @@ class ConsciouscAgent:
                 from sqlalchemy import text
 
                 with db_manager._get_connection() as connection:
-                    # Get ALL conscious-info labeled memories from long-term memory
+                    # Get top conscious-info labeled memories from long-term memory (limited for performance)
                     cursor = connection.execute(
                         text(
                             """SELECT memory_id, processed_data, summary, searchable_content,
                                   importance_score, created_at
                            FROM long_term_memory
                            WHERE namespace = :namespace AND classification = 'conscious-info'
-                           ORDER BY importance_score DESC, created_at DESC"""
+                           ORDER BY importance_score DESC, created_at DESC
+                           LIMIT :limit"""
                         ),
-                        {"namespace": namespace},
+                        {"namespace": namespace, "limit": limit},
                     )
                     existing_conscious_memories = cursor.fetchall()
 

memori/agents/memory_agent.py

@@ -204,6 +204,9 @@ CONVERSATION CONTEXT:
                                 "content": f"Process this conversation for enhanced memory storage:\n\n{conversation_text}\n{context_info}",
                             },
                         ],
+                        metadata=[
+                            "INTERNAL_MEMORY_PROCESSING"
+                        ],  # Internal metadata tag
                         response_format=ProcessedLongTermMemory,
                         temperature=0.1,  # Low temperature for consistent processing
                     )
@@ -237,17 +240,19 @@ CONVERSATION CONTEXT:
                 )
 
             logger.debug(
-                f"Processed conversation {chat_id}: "
-                f"classification={processed_memory.classification}, "
-                f"importance={processed_memory.importance}, "
-                f"conscious_context={processed_memory.is_user_context}, "
-                f"promotion_eligible={processed_memory.promotion_eligible}"
+                f"[AGENT] Processed conversation {chat_id[:8]}... - "
+                f"classification: {processed_memory.classification.value} | "
+                f"importance: {processed_memory.importance.value} | "
+                f"conscious_context: {processed_memory.is_user_context} | "
+                f"promotion_eligible: {processed_memory.promotion_eligible}"
             )
 
             return processed_memory
 
         except Exception as e:
-            logger.error(f"Memory agent processing failed for {chat_id}: {e}")
+            logger.error(
+                f"[AGENT] Memory processing failed for {chat_id[:8]}... - {type(e).__name__}: {e}"
+            )
             return self._create_empty_long_term_memory(
                 chat_id, f"Processing failed: {str(e)}"
             )
@@ -307,7 +312,7 @@ CONVERSATION CONTEXT:
 
             if avg_similarity >= similarity_threshold:
                 logger.info(
-                    f"Duplicate detected: {avg_similarity:.2f} similarity with {existing.conversation_id}"
+                    f"[AGENT] Duplicate detected - {avg_similarity:.2f} similarity with {existing.conversation_id[:8]}..."
                 )
                 return existing.conversation_id
 
@@ -415,6 +420,7 @@ CONVERSATION CONTEXT:
                         "content": f"Process this conversation for enhanced memory storage:\n\n{conversation_text}\n{context_info}",
                     },
                 ],
+                metadata=["INTERNAL_MEMORY_PROCESSING"],  # Internal metadata tag
                 temperature=0.1,  # Low temperature for consistent processing
                 max_tokens=2000,  # Ensure enough tokens for full response
             )

memori/agents/retrieval_agent.py

@@ -149,6 +149,9 @@ Be strategic and comprehensive in your search planning."""
                                 "content": prompt,
                             },
                         ],
+                        metadata=[
+                            "INTERNAL_MEMORY_PROCESSING"
+                        ],  # Internal metadata tag
                         response_format=MemorySearchQuery,
                         temperature=0.1,
                     )
@@ -218,15 +221,25 @@ Be strategic and comprehensive in your search planning."""
             all_results = []
             seen_memory_ids = set()
 
-            # For MongoDB and SQL, use the unified search_memories method as primary strategy
-            # This ensures we use the database's native search capabilities
-            logger.debug(f"Executing unified database search using {db_type} manager")
-            primary_results = db_manager.search_memories(
-                query=search_plan.query_text or query, namespace=namespace, limit=limit
-            )
-            logger.debug(
-                f"Primary database search returned {len(primary_results)} results"
-            )
+            # For MongoDB and SQL, use SearchService directly to avoid recursion
+            # This ensures we use the database's native search capabilities without triggering context injection
+            logger.debug(f"Executing direct SearchService search using {db_type}")
+            try:
+                from ..database.search_service import SearchService
+
+                with db_manager.SessionLocal() as session:
+                    search_service = SearchService(session, db_type)
+                    primary_results = search_service.search_memories(
+                        query=search_plan.query_text or query,
+                        namespace=namespace,
+                        limit=limit,
+                    )
+                logger.debug(
+                    f"Direct SearchService returned {len(primary_results)} results"
+                )
+            except Exception as e:
+                logger.error(f"SearchService direct access failed: {e}")
+                primary_results = []
 
             # Process primary results and add search metadata
             for result in primary_results:
@@ -383,9 +396,17 @@ Be strategic and comprehensive in your search planning."""
 
         search_terms = " ".join(keywords)
         try:
-            results = db_manager.search_memories(
-                query=search_terms, namespace=namespace, limit=limit
-            )
+            # Use SearchService directly to avoid recursion
+            from ..database.search_service import SearchService
+
+            db_type = self._detect_database_type(db_manager)
+
+            with db_manager.SessionLocal() as session:
+                search_service = SearchService(session, db_type)
+                results = search_service.search_memories(
+                    query=search_terms, namespace=namespace, limit=limit
+                )
+
             # Ensure results is a list of dictionaries
             if not isinstance(results, list):
                 logger.warning(f"Search returned non-list result: {type(results)}")
@@ -417,14 +438,24 @@ Be strategic and comprehensive in your search planning."""
         if not categories:
             return []
 
-        # This would need to be implemented in the database manager
-        # For now, get all memories and filter by category
+        # Use SearchService directly to avoid recursion
+        # Get all memories and filter by category
         logger.debug(
             f"Searching memories by categories: {categories} in namespace: {namespace}"
         )
-        all_results = db_manager.search_memories(
-            query="", namespace=namespace, limit=limit * 3
-        )
+        try:
+            from ..database.search_service import SearchService
+
+            db_type = self._detect_database_type(db_manager)
+
+            with db_manager.SessionLocal() as session:
+                search_service = SearchService(session, db_type)
+                all_results = search_service.search_memories(
+                    query="", namespace=namespace, limit=limit * 3
+                )
+        except Exception as e:
+            logger.error(f"Category search failed: {e}")
+            all_results = []
 
         logger.debug(
             f"Retrieved {len(all_results)} total results for category filtering"
@@ -628,6 +659,7 @@ Be strategic and comprehensive in your search planning."""
                         "content": prompt,
                     },
                 ],
+                metadata=["INTERNAL_MEMORY_PROCESSING"],  # Internal metadata tag
                 temperature=0.1,
                 max_tokens=1000,  # Ensure enough tokens for full response
             )

memori/core/conversation.py

@@ -207,7 +207,7 @@ class ConversationManager:
             elif mode == "auto":
                 # Auto mode: Search long-term memory database for relevant context
                 logger.debug(
-                    f"Auto-ingest: Processing user input for long-term memory search: '{user_input[:50]}...'"
+                    f"[CONTEXT] Auto-ingest processing - Query: '{user_input[:50]}...' | Session: {session_id[:8]}..."
                 )
                 context = (
                     memori_instance._get_auto_ingest_context(user_input)
@@ -217,11 +217,11 @@ class ConversationManager:
                 if context:
                     context_prompt = self._build_auto_context_prompt(context)
                     logger.debug(
-                        f"Auto-ingest: Successfully injected long-term memory context with {len(context)} items for session {session_id}"
+                        f"[CONTEXT] Long-term memory injected - {len(context)} items | Session: {session_id[:8]}..."
                     )
                 else:
                     logger.debug(
-                        f"Auto-ingest: No relevant memories found in long-term database for query '{user_input[:50]}...' in session {session_id}"
+                        f"[CONTEXT] No relevant memories found for '{user_input[:30]}...' | Session: {session_id[:8]}..."
                     )
 
             # Get conversation history
@@ -242,11 +242,17 @@ class ConversationManager:
                 if previous_messages:
                     system_content += "\n--- Conversation History ---\n"
                     for msg in previous_messages:
-                        role_label = "You" if msg["role"] == "assistant" else "User"
+                        if msg["role"] == "assistant":
+                            role_label = "Assistant"
+                        elif msg["role"] == "user":
+                            role_label = "User"
+                        else:
+                            role_label = msg["role"].capitalize()
                         system_content += f"{role_label}: {msg['content']}\n"
+
                     system_content += "--- End History ---\n"
                     logger.debug(
-                        f"Added {len(previous_messages)} history messages for session {session_id}"
+                        f"[CONTEXT] Added {len(previous_messages)} history messages | Session: {session_id[:8]}..."
                     )
 
             # Find existing system message or create new one
@@ -267,16 +273,17 @@ class ConversationManager:
                     0, {"role": "system", "content": system_content}
                 )
 
+            context_status = "yes" if context_prompt else "no"
+            history_status = "yes" if len(history_messages) > 1 else "no"
             logger.debug(
-                f"Enhanced messages for session {session_id}: context={'yes' if context_prompt else 'no'}, "
-                f"history={'yes' if len(history_messages) > 1 else 'no'}"
+                f"[CONTEXT] Enhanced messages for session {session_id[:8]}... - context: {context_status} | history: {history_status}"
             )
 
             return enhanced_messages
 
         except Exception as e:
             logger.error(
-                f"Failed to inject context with history for session {session_id}: {e}"
+                f"[CONTEXT] Failed to inject context for session {session_id[:8]}... - {type(e).__name__}: {e}"
             )
             return messages
 

memori/core/memory.py

@@ -3,6 +3,7 @@ Main Memori class - Pydantic-based memory interface v1.0
 """
 
 import asyncio
+import threading
 import time
 import uuid
 from datetime import datetime
@@ -14,6 +15,7 @@ try:
     import litellm  # noqa: F401
     from litellm import success_callback  # noqa: F401
 
+    _ = litellm  # Mark as intentionally imported
     LITELLM_AVAILABLE = True
 except ImportError:
     LITELLM_AVAILABLE = False
@@ -65,6 +67,7 @@ class Memori:
         schema_init: bool = True,  # Initialize database schema and create tables
         database_prefix: str | None = None,  # Database name prefix
         database_suffix: str | None = None,  # Database name suffix
+        conscious_memory_limit: int = 10,  # Limit for conscious memory processing
     ):
         """
         Initialize Memori memory system v1.0.
@@ -110,6 +113,20 @@ class Memori:
         self.database_prefix = database_prefix
         self.database_suffix = database_suffix
 
+        # Validate conscious_memory_limit parameter
+        if not isinstance(conscious_memory_limit, int) or isinstance(
+            conscious_memory_limit, bool
+        ):
+            raise TypeError("conscious_memory_limit must be an integer (not bool)")
+
+        if not (1 <= conscious_memory_limit <= 2000):
+            raise ValueError("conscious_memory_limit must be between 1 and 2000")
+
+        self.conscious_memory_limit = conscious_memory_limit
+
+        # Thread safety for conscious memory initialization
+        self._conscious_init_lock = threading.RLock()
+
         # Configure provider based on explicit settings ONLY - no auto-detection
         if provider_config:
             # Use provided configuration
@@ -452,7 +469,7 @@ class Memori:
                 )
                 init_success = (
                     await self.conscious_agent.initialize_existing_conscious_memories(
-                        self.db_manager, self.namespace
+                        self.db_manager, self.namespace, self.conscious_memory_limit
                     )
                 )
                 if init_success:
@@ -478,52 +495,104 @@ class Memori:
 
     def _run_synchronous_conscious_initialization(self):
         """Run conscious agent initialization synchronously (when no event loop is available)"""
-        try:
-            if not self.conscious_agent:
-                return
+        with self._conscious_init_lock:
+            try:
+                if not self.conscious_agent:
+                    return
 
-            # If both auto_ingest and conscious_ingest are enabled,
-            # initialize by copying ALL existing conscious-info memories first
-            if self.auto_ingest and self.conscious_ingest:
-                logger.info(
-                    "Conscious-ingest: Both auto_ingest and conscious_ingest enabled - initializing existing conscious memories"
-                )
+                # Check if we've already initialized in this session to avoid repeated work
+                # Use namespace-specific key to prevent conflicts between instances
+                init_key = f"_conscious_initialized_{self.namespace or 'default'}"
+                if hasattr(self, init_key) and getattr(self, init_key):
+                    logger.debug(
+                        f"[CONSCIOUS] Already initialized for namespace '{self.namespace or 'default'}', skipping"
+                    )
+                    return
+
+                # If both auto_ingest and conscious_ingest are enabled,
+                # initialize by copying the most important existing conscious-info memories first
+                if self.auto_ingest and self.conscious_ingest:
+                    logger.info(
+                        "[CONSCIOUS] Both auto_ingest and conscious_ingest enabled - initializing existing conscious memories"
+                    )
 
-                # Run synchronous initialization of existing memories
-                self._initialize_existing_conscious_memories_sync()
+                    # Run optimized synchronous initialization of existing memories
+                    import time
 
-            logger.debug(
-                "Conscious-ingest: Synchronous conscious context extraction completed"
-            )
+                    start_time = time.time()
 
-        except Exception as e:
-            logger.error(f"Synchronous conscious agent initialization failed: {e}")
+                    initialized = self._initialize_existing_conscious_memories_sync()
+
+                    elapsed = time.time() - start_time
+                    if initialized:
+                        logger.debug(
+                            f"[CONSCIOUS] Initialization completed in {elapsed:.2f}s"
+                        )
+                    else:
+                        logger.debug(
+                            f"[CONSCIOUS] Initialization skipped (no work needed) in {elapsed:.2f}s"
+                        )
+
+                # Mark as initialized to avoid repeated work for this specific namespace
+                init_key = f"_conscious_initialized_{self.namespace or 'default'}"
+                setattr(self, init_key, True)
+
+                logger.debug(
+                    "[CONSCIOUS] Synchronous conscious context extraction completed"
+                )
+
+            except Exception as e:
+                logger.error(f"Synchronous conscious agent initialization failed: {e}")
 
     def _initialize_existing_conscious_memories_sync(self):
-        """Synchronously initialize existing conscious-info memories"""
+        """Synchronously initialize existing conscious-info memories with optimization"""
         try:
             from sqlalchemy import text
 
             with self.db_manager._get_connection() as connection:
-                # Get ALL conscious-info labeled memories from long-term memory
+                # First, check if we already have conscious memories in short-term storage
+                existing_short_term = connection.execute(
+                    text(
+                        """SELECT COUNT(*) FROM short_term_memory
+                           WHERE namespace = :namespace
+                           AND (category_primary = 'conscious_context' OR memory_id LIKE 'conscious_%')"""
+                    ),
+                    {"namespace": self.namespace or "default"},
+                ).scalar()
+
+                if existing_short_term > 0:
+                    logger.debug(
+                        f"[CONSCIOUS] {existing_short_term} conscious memories already in short-term storage, skipping initialization"
+                    )
+                    return False
+
+                # Get only the most important conscious-info memories (limit to 10 for performance)
                 cursor = connection.execute(
                     text(
                         """SELECT memory_id, processed_data, summary, searchable_content,
                               importance_score, created_at
                        FROM long_term_memory
                        WHERE namespace = :namespace AND classification = 'conscious-info'
-                       ORDER BY importance_score DESC, created_at DESC"""
+                       ORDER BY importance_score DESC, created_at DESC
+                       LIMIT :limit"""
                     ),
-                    {"namespace": self.namespace or "default"},
+                    {
+                        "namespace": self.namespace or "default",
+                        "limit": self.conscious_memory_limit,
+                    },
                 )
                 existing_conscious_memories = cursor.fetchall()
 
             if not existing_conscious_memories:
                 logger.debug(
-                    "Conscious-ingest: No existing conscious-info memories found for initialization"
+                    "[CONSCIOUS] No conscious-info memories found for initialization"
                 )
                 return False
 
+            # Batch process memories for efficiency
+            logger.debug(
+                f"[CONSCIOUS] Processing {len(existing_conscious_memories)} conscious memories..."
+            )
             copied_count = 0
             for memory_row in existing_conscious_memories:
                 success = self._copy_memory_to_short_term_sync(memory_row)
@@ -532,12 +601,12 @@ class Memori:
 
             if copied_count > 0:
                 logger.info(
-                    f"Conscious-ingest: Initialized {copied_count} existing conscious-info memories to short-term memory"
+                    f"[CONSCIOUS] Initialized {copied_count} conscious memories to short-term storage"
                 )
                 return True
             else:
                 logger.debug(
-                    "Conscious-ingest: No new conscious memories to initialize (all were duplicates)"
+                    "[CONSCIOUS] No new conscious memories to initialize (all were duplicates)"
                 )
                 return False
 
@@ -564,26 +633,25 @@ class Memori:
             from sqlalchemy import text
 
             with self.db_manager._get_connection() as connection:
-                # Check if similar content already exists in short-term memory
+                # Database-agnostic duplicate check with safer pattern matching
                 existing_check = connection.execute(
                     text(
                         """SELECT COUNT(*) FROM short_term_memory
                            WHERE namespace = :namespace
-                           AND category_primary = 'conscious_context'
-                           AND (searchable_content = :searchable_content
-                                OR summary = :summary)"""
+                           AND (memory_id = :exact_id
+                               OR memory_id LIKE :conscious_pattern)"""
                     ),
                     {
                         "namespace": self.namespace or "default",
-                        "searchable_content": searchable_content,
-                        "summary": summary,
+                        "exact_id": memory_id,
+                        "conscious_pattern": f"conscious_{memory_id}_%",
                     },
                 )
 
                 existing_count = existing_check.scalar()
                 if existing_count > 0:
                     logger.debug(
-                        f"Conscious-ingest: Skipping duplicate memory {memory_id} - similar content already exists in short-term memory"
+                        f"[CONSCIOUS] Skipping duplicate memory {memory_id[:8]}... - already exists in short-term memory"
                     )
                     return False
 
@@ -1152,7 +1220,7 @@ class Memori:
                         results[:3]
                     ):  # Log first 3 results for debugging
                         logger.debug(
-                            f"Auto-ingest: Result {i+1}: {type(result)} with keys: {list(result.keys()) if isinstance(result, dict) else 'N/A'}"
+                            f"Auto-ingest: Result {i + 1}: {type(result)} with keys: {list(result.keys()) if isinstance(result, dict) else 'N/A'}"
                         )
             except Exception as db_search_e:
                 logger.error(f"Auto-ingest: Database search failed: {db_search_e}")
@@ -1892,7 +1960,7 @@ class Memori:
 
         # Debug logging for conversation recording
         logger.info(
-            f"Recording conversation - Input: '{user_input[:100]}...' Model: {model}"
+            f"[MEMORY] Recording conversation - Input: '{user_input[:60]}...' | Model: {model} | Session: {self.session_id[:8]}..."
         )
 
         # Parse response
@@ -1915,29 +1983,31 @@ class Memori:
                 namespace=self.namespace,
                 metadata=metadata or {},
             )
-            logger.debug(
-                f"Successfully stored chat history for conversation: {chat_id}"
-            )
+            logger.debug(f"[MEMORY] Chat history stored - ID: {chat_id[:8]}...")
 
             # Always process into long-term memory when memory agent is available
             if self.memory_agent:
                 self._schedule_memory_processing(
                     chat_id, user_input, response_text, response_model
                 )
-                logger.debug(f"Scheduled memory processing for conversation: {chat_id}")
+                logger.debug(f"[MEMORY] Processing scheduled - ID: {chat_id[:8]}...")
             else:
                 logger.warning(
-                    f"Memory agent not available, skipping memory processing for: {chat_id}"
+                    f"[MEMORY] Agent unavailable, skipping processing - ID: {chat_id[:8]}..."
                 )
 
-            logger.info(f"Recorded conversation successfully: {chat_id}")
+            logger.info(
+                f"[MEMORY] Conversation recorded successfully - ID: {chat_id[:8]}..."
+            )
             return chat_id
 
         except Exception as e:
-            logger.error(f"Failed to record conversation {chat_id}: {e}")
+            logger.error(
+                f"[MEMORY] Failed to record conversation {chat_id[:8]}... - {type(e).__name__}: {e}"
+            )
             import traceback
 
-            logger.error(f"Recording error details: {traceback.format_exc()}")
+            logger.debug(f"[MEMORY] Recording error details: {traceback.format_exc()}")
             raise
 
     def _schedule_memory_processing(
@@ -2571,12 +2641,14 @@ class Memori:
         Get auto-ingest context as system prompt for direct injection.
         Returns relevant memories based on user input as formatted system prompt.
         Use this for auto_ingest mode.
+
+        Note: Context retrieval is handled by _get_auto_ingest_context().
+        This function only formats pre-retrieved context.
         """
         try:
-            # For now, use recent short-term memories as a simple approach
-            # This avoids the search engine issues and still provides context
-            # TODO: Use user_input for intelligent context retrieval
-            context = self._get_conscious_context()  # Get recent short-term memories
+            # Get recent short-term memories as fallback context
+            # The actual intelligent retrieval is handled by _get_auto_ingest_context()
+            context = self._get_conscious_context()
 
             if not context:
                 return ""