npcpy 1.0.26__py3-none-any.whl → 1.2.32__py3-none-any.whl

npcpy/memory/memory_processor.py

@@ -0,0 +1,81 @@
+from dataclasses import dataclass
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+import threading
+import queue
+import time
+
+@dataclass
+class MemoryItem:
+    message_id: str
+    conversation_id: str
+    npc: str
+    team: str
+    directory_path: str
+    content: str
+    context: str
+    model: str
+    provider: str
+
+def memory_approval_ui(memories: List[Dict]) -> List[Dict]:
+    if not memories:
+        return []
+    
+    print(f"\n📝 {len(memories)} memories ready for approval:")
+    
+    approvals = []
+    for i, memory in enumerate(memories, 1):
+        print(f"\n--- Memory {i}/{len(memories)} ---")
+        print(f"NPC: {memory['npc']}")
+        content_preview = memory['content'][:200]
+        if len(memory['content']) > 200:
+            content_preview += '...'
+        print(f"Content: {content_preview}")
+        
+        while True:
+            choice = input(
+                "(a)pprove, (r)eject, (e)dit, (s)kip | "
+                "(A)ll approve, (R)all reject, (S)all skip: "
+            ).strip().lower()
+            
+            if choice == 'a':
+                approvals.append({
+                    "memory_id": memory['memory_id'],
+                    "decision": "human-approved"
+                })
+                break
+            elif choice == 'r':
+                approvals.append({
+                    "memory_id": memory['memory_id'],
+                    "decision": "human-rejected"
+                })
+                break
+            elif choice == 'e':
+                edited = input("Edit memory: ").strip()
+                if edited:
+                    approvals.append({
+                        "memory_id": memory['memory_id'],
+                        "decision": "human-edited",
+                        "final_memory": edited
+                    })
+                break
+            elif choice == 's':
+                break
+            elif choice == 'A':
+                for remaining_memory in memories[i-1:]:
+                    approvals.append({
+                        "memory_id": remaining_memory['memory_id'],
+                        "decision": "human-approved"
+                    })
+                return approvals
+            elif choice == 'R':
+                for remaining_memory in memories[i-1:]:
+                    approvals.append({
+                        "memory_id": remaining_memory['memory_id'],
+                        "decision": "human-rejected"
+                    })
+                return approvals
+            elif choice == 'S':
+                return approvals
+    
+    return approvals

npcpy/memory/search.py

@@ -35,21 +35,21 @@ def search_similar_texts(
     embedded_search_term = get_ollama_embeddings([query], embedding_model)[0]
 
     if docs_to_embed is None:
-        # Fetch from the database if no documents to embed are provided
+        
         collection_name = f"{embedding_provider}_{embedding_model}_embeddings"
         collection = chroma_client.get_collection(collection_name)
         results = collection.query(
-            query_embeddings=[embedded_search_term], n_results=top_k * 2  # Fetch more to account for filtering
+            query_embeddings=[embedded_search_term], n_results=top_k * 2  
         )
         
-        # Filter out duplicates while preserving order
+        
         seen_texts = set()
         filtered_results = []
         
         for idx, (id, distance, document) in enumerate(zip(
             results["ids"][0], results["distances"][0], results["documents"][0]
         )):
-            # Check if this is a command (starts with /) and if we've seen it before
+            
             if document not in seen_texts:
                 seen_texts.add(document)
                 filtered_results.append({
@@ -58,7 +58,7 @@ def search_similar_texts(
                     "text": document
                 })
                 
-                # Break if we have enough unique results
+                
                 if len(filtered_results) >= top_k:
                     break
                     
@@ -66,40 +66,40 @@ def search_similar_texts(
 
     print(f"\nNumber of documents to embed: {len(docs_to_embed)}")
 
-    # Get embeddings for provided documents - use np.unique to remove duplicates
-    unique_docs = list(dict.fromkeys(docs_to_embed))  # Preserves order while removing duplicates
+    
+    unique_docs = list(dict.fromkeys(docs_to_embed))  
     raw_embeddings = get_ollama_embeddings(unique_docs, embedding_model)
 
     output_embeddings = []
     unique_doc_indices = []
     
     for idx, emb in enumerate(raw_embeddings):
-        if emb:  # Exclude any empty embeddings
+        if emb:  
             output_embeddings.append(emb)
             unique_doc_indices.append(idx)
 
-    # Convert to numpy arrays for calculations
+    
     doc_embeddings = np.array(output_embeddings)
     query_embedding = np.array(embedded_search_term)
 
-    # Check for zero-length embeddings
+    
     if len(doc_embeddings) == 0:
         raise ValueError("No valid document embeddings found")
 
-    # Normalize embeddings to avoid division by zeros
+    
     doc_norms = np.linalg.norm(doc_embeddings, axis=1, keepdims=True)
     query_norm = np.linalg.norm(query_embedding)
 
-    # Ensure no zero vectors are being used in cosine similarity
+    
     if query_norm == 0:
         raise ValueError("Query embedding is zero-length")
 
-    # Calculate cosine similarities
+    
     cosine_similarities = np.dot(doc_embeddings, query_embedding) / (
         doc_norms.flatten() * query_norm
     )
 
-    # Get indices of top K documents
+    
     top_indices = np.argsort(cosine_similarities)[::-1][:top_k]
 
     return [
@@ -155,7 +155,7 @@ def execute_search_command(
     else:
         num_results = 5
 
-    # remove the -p and provider from the command string
+    
     command = command.replace(f"-p {provider}", "").replace(
         f"--provider {provider}", ""
     )
@@ -172,21 +172,127 @@ def execute_search_command(
         "messages": messages,
         "output": result[0] + f"\n\n\n Citation Links: {result[1]}",
     }
+    
+
+
+def get_facts_for_rag(
+    kuzu_db_path: str,
+    chroma_db_path: str,
+    query: str,
+    group_filters: Optional[List[str]] = None,
+    top_k: int = 10,
+) -> str:
+    """Get facts for RAG by combining vector and graph search
+
+    Args:
+        kuzu_db_path: Path to Kuzu graph database
+        chroma_db_path: Path to Chroma vector database
+        query: Search query
+        group_filters: Optional list of groups to filter by
+        top_k: Number of results to return
+        embedding_model: Model to use for embeddings
+        provider: Provider for embeddings
+
+    Returns:
+        Formatted context string with retrieved facts
+    """
+    
+    kuzu_conn = init_db(kuzu_db_path)
+    chroma_client, chroma_collection = setup_chroma_db( 
+        "knowledge_graph",
+        "Facts extracted from various sources",
+        chroma_db_path
+    )
+
+    
+    results = hybrid_search_with_chroma(
+        kuzu_conn=kuzu_conn,
+        chroma_collection=chroma_collection,
+        query=query,
+        group_filter=group_filters,
+        top_k=top_k,
+    )
+
+    
+    context = "Related facts:\n\n"
+
+    
+    context += "Most relevant facts:\n"
+    vector_matches = [r for r in results if r["source"] == "vector_search"]
+    for i, item in enumerate(vector_matches):
+        context += f"{i+1}. {item['fact']}\n"
+
+    
+    context += "\nRelated concepts:\n"
+    graph_matches = [r for r in results if r["source"] != "vector_search"]
+    for i, item in enumerate(graph_matches):
+        group = item["source"].replace("graph_relation_via_", "")
+        context += f"{i+1}. {item['fact']} (related via {group})\n"
+
+    
+    kuzu_conn.close()
+
+    return context
+def answer_with_rag(
+    query: str,
+    kuzu_db_path,
+    chroma_db_path,
+    model,
+    provider,    
+) -> str:
+    """Answer a query using RAG with facts from the knowledge base
+
+    Args:
+        query: User query
+        kuzu_db_path: Path to Kuzu graph database
+        chroma_db_path: Path to Chroma vector database
+        model: LLM model to use
+        provider: LLM provider
+        embedding_model: Model to use for embeddings
+
+    Returns:
+        Answer from the model
+    """
+    
+    context = get_facts_for_rag(
+        kuzu_db_path,
+        chroma_db_path,
+        query,
+    )
+
+    
+    prompt = f"""
+    Answer this question based on the retrieved information.
+
+    Question: {query}
+
+    {context}
+
+    Please provide a comprehensive answer based on the facts above. If the information
+    doesn't contain a direct answer, please indicate that clearly but try to synthesize
+    from the available facts.
+    """
+
+    
+    response = get_llm_response(prompt, model=model, provider=provider)
+
+    return response["response"]
+
+    
 def execute_rag_command(
     command: str,
     vector_db_path: str, 
     embedding_model: str,
     embedding_provider: str,
-    messages=None,
     top_k: int = 15,
-    file_contents=None,  # List of file content chunks
+    file_contents=None,  
     **kwargs
 ) -> dict:
     """
     Execute the RAG command with support for embedding generation.
     When file_contents is provided, it searches those instead of the database.
     """
-    # ANSI color codes for terminal output
+    
     BLUE = "\033[94m"
     GREEN = "\033[92m"
     YELLOW = "\033[93m"
@@ -194,44 +300,44 @@ def execute_rag_command(
     RESET = "\033[0m"
     BOLD = "\033[1m"
     
-    # Format header
+    
     header = f"\n{BOLD}{BLUE}RAG Query: {RESET}{GREEN}{command}{RESET}\n"
     
-    # If we have file contents, search those instead of the database
+    
     if file_contents and len(file_contents) > 0:
         similar_chunks = search_similar_texts(
             command,
             embedding_model,
             embedding_provider,
-            chroma_client=None,  # Not using ChromaDB when searching files
+            chroma_client=None,  
             
-            docs_to_embed=file_contents,  # Pass file chunks directly to embed
+            docs_to_embed=file_contents,  
             top_k=top_k
         )
         
-        # Process the results for display
+        
         file_info = f"{BOLD}{BLUE}Files Processed: {RESET}{YELLOW}{len(file_contents)}{RESET}\n"
         separator = f"{YELLOW}{'-' * 100}{RESET}\n"
         
-        # Format chunk results for display
+        
         chunk_results = []
         for i, chunk in enumerate(similar_chunks, 1):
             score = chunk['score']
             text = chunk['text']
             
-            # Truncate for display if needed
+            
             display_text = text[:150] + ("..." if len(text) > 150 else "")
             chunk_results.append(f"{BOLD}{i:2d}{RESET}. {CYAN}[{score:.2f}]{RESET} {display_text}")
         
-        # Display the file search results
+        
         file_results = header + file_info + separator + "\n".join(chunk_results)
         render_markdown(f"FILE SEARCH RESULTS:\n{file_results}")
         
-        # Prepare the chunks for the prompt (plain text version)
+        
         plain_chunks = [f"{i+1}. {chunk['text']}" for i, chunk in enumerate(similar_chunks)]
         plain_results = "\n\n".join(plain_chunks)
         
-        # Build the prompt focusing on file contents
+        
         prompt = f"""
         The user asked: {command}
         
@@ -239,28 +345,28 @@ def execute_rag_command(
         
         {plain_results}
         
-        Please respond to the user query based on these file contents.
+        Please respond to the user query based on the above information, integrating the information in an additive way, attempting to always find some possible connection
+        between the results and the initial input. do not do this haphazardly, be creative yet cautious.
         """
         
-        # Get LLM response
+        
         response = get_llm_response(
             prompt,
-            messages=messages,
             **kwargs
         )
         return response
     
     else:
-        # No file contents, search the database instead
+        
         try:
-            # Setup ChromaDB connection
+            
             chroma_client, chroma_collection = setup_chroma_db( 
                 f"{embedding_provider}_{embedding_model}_embeddings",
                 "Conversation embeddings",
                 vector_db_path
             )
             
-            # Search for similar texts in the database
+            
             similar_texts = search_similar_texts(
                 command, 
                 embedding_model,
@@ -269,16 +375,16 @@ def execute_rag_command(
                 top_k=top_k,
             )
             
-            # Process the results for display
+            
             separator = f"{YELLOW}{'-' * 100}{RESET}\n"
             
-            # Format results
+            
             processed_texts = []
             for i, similar_text in enumerate(similar_texts, 1):
                 text = similar_text['text']
                 score = similar_text['score']
                 
-                # Format timestamp if available
+                
                 timestamp_str = ""
                 try:
                     if 'id' in similar_text and '_' in similar_text['id']:
@@ -287,24 +393,24 @@ def execute_rag_command(
                 except (IndexError, ValueError, TypeError):
                     pass
                 
-                # Clean up the text
+                
                 text = text.replace('\n', ' ').strip()
                 snippet = text[:85] + ("..." if len(text) > 85 else "")
                 
-                # Format with colors
+                
                 processed_texts.append(
                     f"{BOLD}{i:2d}{RESET}. {CYAN}[{score:.2f}]{RESET} {snippet} {timestamp_str}"
                 )
             
-            # Combine for display
+            
             knowledge_results = header + separator + "\n".join(processed_texts)
             render_markdown(f"KNOWLEDGE BASE: {knowledge_results}")
             
-            # Prepare plain text for the prompt
+            
             plain_texts = [f"{i+1}. {similar_texts[i]['text']}" for i in range(len(similar_texts))]
             plain_results = "\n\n".join(plain_texts)
             
-            # Build the prompt
+            
             prompt = f"""
             The user asked: {command}
             
@@ -312,34 +418,31 @@ def execute_rag_command(
             
             {plain_results}
             
-            Please respond to the user query based on the above information.
+            Please respond to the user query based on the above information, integrating the information in an additive way, attempting to always find some possible connection
+            between the results and the initial input. do not do this haphazardly, be creative yet cautious.
             """
             
-            # Get LLM response
+            
             response = get_llm_response(
                 prompt,
-                messages=messages,
                 **kwargs
             )
             return response
             
         except Exception as e:
             traceback.print_exc()
-            return {"output": f"Error searching knowledge base: {e}", "messages": messages}
+            return {"output": f"Error searching knowledge base: {e}", "messages": kwargs.get('messages', [])}
         
         
 def execute_brainblast_command(
     command: str,
-    command_history,
-    messages=None,
-    top_k: int = 5,  # Fewer results per chunk to keep total manageable
     **kwargs
 ) -> dict:
     """
     Execute a comprehensive "brainblast" search on command history.
     Breaks the query into words and searches for combinations of those words.
     """
-    # ANSI color codes for terminal output
+    
     BLUE = "\033[94m"
     GREEN = "\033[92m"
     YELLOW = "\033[93m"
@@ -347,57 +450,61 @@ def execute_brainblast_command(
     RESET = "\033[0m"
     BOLD = "\033[1m"
     
-    print(f"\nBrainblast Query: {command}")
+
+    if not 'command_history' in kwargs:
+        raise Exception('Command history must be passed as a kwarg to this function')
+    command_history = kwargs.get('command_history')
+    top_k = kwargs.get('top_k', 10)
+    
+    
     
-    # Format header for display
     header = f"\n{BOLD}{BLUE}BRAINBLAST Query: {RESET}{GREEN}{command}{RESET}\n"
     separator = f"{YELLOW}{'-' * 100}{RESET}\n"
     
     try:
-        # Split the command into words
+        
         words = command.split()
         
         if not words:
             return {"output": "Please provide search terms to use brainblast.", "messages": messages or []}
         
-        # Generate different chunk sizes for searching
+        
         all_chunks = []
         
-        # Add individual words
+        
         all_chunks.extend(words)
         
-        # Add pairs of words
+        
         if len(words) >= 2:
             for i in range(len(words) - 1):
                 all_chunks.append(f"{words[i]} {words[i+1]}")
         
-        # Add groups of 4 words
+        
         if len(words) >= 4:
             for i in range(len(words) - 3):
                 all_chunks.append(f"{words[i]} {words[i+1]} {words[i+2]} {words[i+3]}")
         
-        # Add the entire query
+        
         if len(words) > 1:
             all_chunks.append(command)
         
-        # Remove duplicates while preserving order
+        
         unique_chunks = []
         for chunk in all_chunks:
             if chunk not in unique_chunks:
                 unique_chunks.append(chunk)
         
-        # Search for each chunk
+        
         all_results = []
         chunk_results = {}
         
         for chunk in unique_chunks:
             results = command_history.search_conversations(chunk)
-            print(results)
             if results:
-                chunk_results[chunk] = results[:top_k]  # Limit results per chunk
+                chunk_results[chunk] = results[:top_k]  
                 all_results.extend(results[:top_k])
         
-        # Remove duplicate results while preserving order
+        
         unique_results = []
         seen_ids = set()
         for result in all_results:
@@ -410,7 +517,7 @@ def execute_brainblast_command(
             result_message = f"No matches found for any combination of terms in: {command}"
             render_markdown(f"BRAINBLAST SEARCH: {header}{separator}{result_message}")
             
-            # Get LLM response
+            
             prompt = f"""
             The user asked for a brainblast search with: {command}
             
@@ -420,18 +527,18 @@ def execute_brainblast_command(
             
             response = get_llm_response(
                 prompt,
-                messages=messages,
+                messages=kwargs.get('messages'),
                 **kwargs
             )
-            return response
+            return {'output':response.get('response'), 'messages':response.get('messages') or []}
+        
         
-        # Process the results for display
         processed_chunks = []
         for chunk, results in chunk_results.items():
             if results:
                 chunk_display = f"{BOLD}{BLUE}Results for '{chunk}':{RESET}\n"
                 
-                for i, result in enumerate(results[:3], 1):  # Just show top 3 for each chunk
+                for i, result in enumerate(results[:3], 1):  
                     cmd = result.get('content', '')
                     timestamp = result.get('timestamp', '')
                     
@@ -442,29 +549,26 @@ def execute_brainblast_command(
                 
                 processed_chunks.append(chunk_display)
         
-        # Display summarized chunk results
-        chunks_output = header + separator + "\n".join(processed_chunks)
-        render_markdown(f"BRAINBLAST SEARCH: {chunks_output}")
         
-        # Prepare the consolidated results for the prompt
+        
         plain_results = []
-        for i, result in enumerate(unique_results[:15], 1):  # Limit to 15 total unique results
-            content = result.get('content', '')
+        for i, result in enumerate(unique_results[:15], 1):  
+            content = result.get('content', '')[0:250]
             timestamp = result.get('timestamp', '')
             location = result.get('directory_path', '')
             
-            # Format without ANSI colors
+            
             plain_results.append(
                 f"{i}. [{timestamp}] Command: {cmd}\n   Location: {location}\n   Output: {content[:150] + ('...' if len(content) > 150 else '')}"
             )
         
-        # Summary of which terms matched what
+        
         term_summary = []
         for chunk, results in chunk_results.items():
             if results:
                 term_summary.append(f"Term '{chunk}' matched {len(results)} commands")
         
-        # Build the prompt
+        
         f=', '.join(term_summary)
         e="\n\n".join(plain_results)
         prompt = f"""
@@ -474,27 +578,21 @@ def execute_brainblast_command(
         Here's a summary of what matched:
         {f}
         
-        Here are the top unique matching commands from the search:
+        Here are the top unique matching items from the search:
         
         {e}
         
-        Please analyze these results and provide insights to the user. Look for:
-        1. Patterns in the commands they've used
-        2. Common themes or topics
-        3. Any interesting or unusual commands that might be useful
-        4. Suggestions for how the user might better leverage their command history
-        
-        Focus on being helpful and surfacing valuable insights from their command history.
+        Please analyze these results and attempt to generate some novel insight about them in one sentence. think outside the box.
+        Provide a summary as well.        
         """
         
-        # Get LLM response
         response = get_llm_response(
             prompt,
-            messages=messages,
-            **kwargs
+            **kwargs, 
         )
-        return response
+        return  {"output": response.get('response'), 
+                 "messages": response.get('messages',[]) }
         
     except Exception as e:
         traceback.print_exc()
-        return {"output": f"Error executing brainblast command: {e}", "messages": messages or []}
+        return {"output": f"Error executing brainblast command: {e}", "messages": kwargs.get('messages',[]) }