npcpy 1.1.28__py3-none-any.whl → 1.2.32__py3-none-any.whl

npcpy/memory/command_history.py

@@ -8,7 +8,7 @@ import numpy as np
 
 try:
     import sqlalchemy
-    from sqlalchemy import create_engine, text, MetaData, Table, Column, Integer, String, Text, DateTime, LargeBinary, ForeignKey, Boolean
+    from sqlalchemy import create_engine, text, MetaData, Table, Column, Integer, String, Text, DateTime, LargeBinary, ForeignKey, Boolean, func
     from sqlalchemy.engine import Engine, Connection as SQLAlchemyConnection
     from sqlalchemy.exc import SQLAlchemyError
     from sqlalchemy.sql import select, insert, update, delete
@@ -52,7 +52,7 @@ def create_engine_from_path(db_path: str) -> Engine:
     if db_path.startswith('postgresql://') or db_path.startswith('postgres://'):
         return create_engine(db_path)
     else:
-        # Treat as SQLite file path
+        
         if db_path.startswith('~/'):
             db_path = os.path.expanduser(db_path)
         return create_engine(f'sqlite:///{db_path}')
@@ -152,7 +152,7 @@ def setup_chroma_db(collection, description='', db_path: str = ''):
 def init_kg_schema(engine: Engine):
     """Creates the multi-scoped, path-aware KG tables using SQLAlchemy"""
     
-    # Create tables using SQLAlchemy DDL
+    
     metadata = MetaData()
     
     kg_facts = Table('kg_facts', metadata,
@@ -164,7 +164,7 @@ def init_kg_schema(engine: Engine):
         Column('type', String(100)),
         Column('generation', Integer),
         Column('origin', String(100)),
-        # Composite primary key
+        
         schema=None
     )
     
@@ -197,7 +197,7 @@ def init_kg_schema(engine: Engine):
         schema=None
     )
     
-    # Create all tables
+    
     metadata.create_all(engine, checkfirst=True)
 
 def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_path: str) -> Dict[str, Any]:
@@ -213,7 +213,7 @@ def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_pat
     
     with engine.connect() as conn:
         try:
-            # Get generation
+            
             result = conn.execute(text("""
                 SELECT value FROM kg_metadata 
                 WHERE team_name = :team AND npc_name = :npc AND directory_path = :path AND key = 'generation'
@@ -223,7 +223,7 @@ def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_pat
             if row:
                 kg['generation'] = int(row.value)
 
-            # Get facts
+            
             result = conn.execute(text("""
                 SELECT statement, source_text, type, generation, origin FROM kg_facts 
                 WHERE team_name = :team AND npc_name = :npc AND directory_path = :path
@@ -240,7 +240,7 @@ def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_pat
                 for row in result
             ]
 
-            # Get concepts
+            
             result = conn.execute(text("""
                 SELECT name, generation, origin FROM kg_concepts 
                 WHERE team_name = :team AND npc_name = :npc AND directory_path = :path
@@ -251,7 +251,7 @@ def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_pat
                 for row in result
             ]
 
-            # Get links
+            
             links = {}
             result = conn.execute(text("""
                 SELECT source, target, type FROM kg_links 
@@ -271,7 +271,7 @@ def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_pat
             kg['fact_to_concept_links'] = links
             
         except SQLAlchemyError:
-            # Initialize schema if it doesn't exist
+            
             init_kg_schema(engine)
     
     return kg
@@ -280,7 +280,7 @@ def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_n
     """Saves a knowledge graph dictionary to the database, ignoring duplicates."""
     try:
         with engine.begin() as conn:
-            # Save facts
+            
             facts_to_save = [
                 {
                     "statement": fact['statement'],
@@ -294,7 +294,7 @@ def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_n
             ]
             
             if facts_to_save:
-                # Use INSERT OR IGNORE for SQLite, ON CONFLICT DO NOTHING for PostgreSQL
+                
                 if 'sqlite' in str(engine.url):
                     stmt = text("""
                         INSERT OR IGNORE INTO kg_facts 
@@ -312,7 +312,7 @@ def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_n
                 for fact in facts_to_save:
                     conn.execute(stmt, fact)
 
-            # Save concepts
+            
             concepts_to_save = [
                 {
                     "name": concept['name'],
@@ -343,7 +343,7 @@ def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_n
                 for concept in concepts_to_save:
                     conn.execute(stmt, concept)
 
-            # Update metadata (generation number)
+            
             if 'sqlite' in str(engine.url):
                 stmt = text("""
                     INSERT OR REPLACE INTO kg_metadata (key, value, team_name, npc_name, directory_path)
@@ -364,13 +364,13 @@ def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_n
                 "directory_path": directory_path
             })
 
-            # Rebuild links from scratch to ensure consistency
+            
             conn.execute(text("""
                 DELETE FROM kg_links 
                 WHERE team_name = :team_name AND npc_name = :npc_name AND directory_path = :directory_path
             """), {"team_name": team_name, "npc_name": npc_name, "directory_path": directory_path})
             
-            # Insert links
+            
             for fact, concepts in kg_data.get("fact_to_concept_links", {}).items():
                 for concept in concepts:
                     conn.execute(text("""
@@ -402,6 +402,9 @@ def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_n
     except Exception as e:
         print(f"Failed to save KG for scope '({team_name}, {npc_name}, {directory_path})': {e}")
 
+def generate_message_id() -> str:
+    return str(uuid.uuid4())
+
 class CommandHistory:
     def __init__(self, db: Union[str, Engine] = "~/npcsh_history.db"):
         
@@ -420,7 +423,7 @@ class CommandHistory:
         """Creates all necessary tables."""
         metadata = MetaData()
         
-        # Command history table
+        
         Table('command_history', metadata,
             Column('id', Integer, primary_key=True, autoincrement=True),
             Column('timestamp', String(50)),
@@ -430,7 +433,7 @@ class CommandHistory:
             Column('location', Text)
         )
         
-        # Conversation history table
+        
         Table('conversation_history', metadata,
             Column('id', Integer, primary_key=True, autoincrement=True),
             Column('message_id', String(50), unique=True, nullable=False),
@@ -445,7 +448,7 @@ class CommandHistory:
             Column('team', String(100))
         )
         
-        # Message attachments table
+        
         Table('message_attachments', metadata,
             Column('id', Integer, primary_key=True, autoincrement=True),
             Column('message_id', String(50), ForeignKey('conversation_history.message_id', ondelete='CASCADE'), nullable=False),
@@ -457,7 +460,7 @@ class CommandHistory:
             Column('file_path', Text) 
         )
         
-        # Jinx execution log table
+        
         Table('jinx_execution_log', metadata,
             Column('execution_id', Integer, primary_key=True, autoincrement=True),
             Column('triggering_message_id', String(50), ForeignKey('conversation_history.message_id', ondelete='CASCADE'), nullable=False),
@@ -474,12 +477,28 @@ class CommandHistory:
             Column('duration_ms', Integer)
         )
         
-        # Create all tables
+        Table('memory_lifecycle', metadata,
+            Column('id', Integer, primary_key=True, autoincrement=True),
+            Column('message_id', String(50), nullable=False),
+            Column('conversation_id', String(100), nullable=False),
+            Column('npc', String(100), nullable=False),
+            Column('team', String(100), nullable=False),
+            Column('directory_path', Text, nullable=False),
+            Column('timestamp', String(50), nullable=False),
+            Column('initial_memory', Text, nullable=False),
+            Column('final_memory', Text),
+            Column('status', String(50), nullable=False),
+            Column('model', String(100)),
+            Column('provider', String(100)),
+            Column('created_at', DateTime, default=func.now())
+        )
+                
+        
         metadata.create_all(self.engine, checkfirst=True)
         
-        # Create indexes for jinx table
+        
         with self.engine.begin() as conn:
-            # Check if indexes exist before creating (database-agnostic approach)
+            
             index_queries = [
                 "CREATE INDEX IF NOT EXISTS idx_jinx_log_trigger_msg ON jinx_execution_log (triggering_message_id)",
                 "CREATE INDEX IF NOT EXISTS idx_jinx_log_convo_id ON jinx_execution_log (conversation_id)",
@@ -491,10 +510,10 @@ class CommandHistory:
                 try:
                     conn.execute(text(idx_query))
                 except SQLAlchemyError:
-                    # Index might already exist or syntax might be different for PostgreSQL
+                    
                     pass
         
-        # Initialize KG schema
+        
         init_kg_schema(self.engine)
 
     def _execute_returning_id(self, stmt: str, params: Dict = None) -> Optional[int]:
@@ -533,8 +552,6 @@ class CommandHistory:
         with self.engine.begin() as conn:
             conn.execute(text(stmt), params)
 
-    def generate_message_id(self) -> str:
-        return str(uuid.uuid4())
 
     def add_conversation(
         self, 
@@ -574,11 +591,147 @@ class CommandHistory:
                     attachment_type=attachment.get("type"),
                     data=attachment.get("data"),
                     size=attachment.get("size"),
-                    file_path=attachment.get("path") # PASS THE PATH
+                    file_path=attachment.get("path") 
                 )
 
         return message_id
 
+    def add_memory_to_database(self, message_id: str, conversation_id: str, npc: str, team: str, 
+                            directory_path: str, initial_memory: str, status: str, 
+                            model: str = None, provider: str = None, final_memory: str = None):
+        """Store a memory entry in the database"""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        stmt = """
+            INSERT INTO memory_lifecycle 
+            (message_id, conversation_id, npc, team, directory_path, timestamp, 
+            initial_memory, final_memory, status, model, provider)
+            VALUES (:message_id, :conversation_id, :npc, :team, :directory_path, 
+                    :timestamp, :initial_memory, :final_memory, :status, :model, :provider)
+        """
+        
+        params = {
+            "message_id": message_id, "conversation_id": conversation_id,
+            "npc": npc, "team": team, "directory_path": directory_path,
+            "timestamp": timestamp, "initial_memory": initial_memory,
+            "final_memory": final_memory, "status": status,
+            "model": model, "provider": provider
+        }
+        
+        return self._execute_returning_id(stmt, params)
+    def get_memories_for_scope(
+        self,
+        npc: str,
+        team: str,
+        directory_path: str,
+        status: Optional[str] = None
+    ) -> List[Dict]:
+        
+        query = """
+            SELECT id, initial_memory, final_memory, 
+                status, timestamp, created_at
+            FROM memory_lifecycle
+            WHERE npc = :npc AND team = :team AND directory_path = :path
+        """
+        params = {"npc": npc, "team": team, "path": directory_path}
+        
+        if status:
+            query += " AND status = :status"
+            params["status"] = status
+        
+        query += " ORDER BY created_at DESC"
+        data =self._fetch_all(query, params)
+        return  data
+
+    def search_memory(self, query: str, npc: str = None, team: str = None, 
+                    directory_path: str = None, status_filter: str = None, limit: int = 10):
+        """Search memories with hierarchical scope"""
+        conditions = ["LOWER(initial_memory) LIKE LOWER(:query) OR LOWER(final_memory) LIKE LOWER(:query)"]
+        params = {"query": f"%{query}%"}
+        
+        if status_filter:
+            conditions.append("status = :status")
+            params["status"] = status_filter
+        
+        
+        order_parts = []
+        if npc:
+            order_parts.append(f"CASE WHEN npc = '{npc}' THEN 1 ELSE 2 END")
+        if team:
+            order_parts.append(f"CASE WHEN team = '{team}' THEN 1 ELSE 2 END")
+        if directory_path:
+            order_parts.append(f"CASE WHEN directory_path = '{directory_path}' THEN 1 ELSE 2 END")
+        
+        order_clause = ", ".join(order_parts) + ", created_at DESC" if order_parts else "created_at DESC"
+        
+        stmt = f"""
+            SELECT * FROM memory_lifecycle 
+            WHERE {' AND '.join(conditions)}
+            ORDER BY {order_clause}
+            LIMIT :limit
+        """
+        params["limit"] = limit
+        
+        return self._fetch_all(stmt, params)
+
+    def get_memory_examples_for_context(self, npc: str, team: str, directory_path: str,
+                                    n_approved: int = 10, n_rejected: int = 10):
+        """Get recent approved and rejected memories for learning context"""
+        
+        approved_stmt = """
+            SELECT initial_memory, final_memory, status FROM memory_lifecycle
+            WHERE status IN ('human-approved', 'model-approved')
+            ORDER BY 
+                CASE WHEN npc = :npc AND team = :team AND directory_path = :path THEN 1
+                    WHEN npc = :npc AND team = :team THEN 2  
+                    WHEN team = :team THEN 3
+                    ELSE 4 END,
+                created_at DESC
+            LIMIT :n_approved
+        """
+        
+        rejected_stmt = """
+            SELECT initial_memory, status FROM memory_lifecycle
+            WHERE status IN ('human-rejected', 'model-rejected')
+            ORDER BY 
+                CASE WHEN npc = :npc AND team = :team AND directory_path = :path THEN 1
+                    WHEN npc = :npc AND team = :team THEN 2
+                    WHEN team = :team THEN 3  
+                    ELSE 4 END,
+                created_at DESC
+            LIMIT :n_rejected
+        """
+        
+        params = {"npc": npc, "team": team, "path": directory_path, 
+                "n_approved": n_approved, "n_rejected": n_rejected}
+        
+        approved = self._fetch_all(approved_stmt, params)
+        rejected = self._fetch_all(rejected_stmt, params)
+        
+        return {"approved": approved, "rejected": rejected}
+
+    def get_pending_memories(self, limit: int = 50):
+        """Get memories pending human approval"""
+        stmt = """
+            SELECT * FROM memory_lifecycle 
+            WHERE status = 'pending_approval'
+            ORDER BY created_at ASC
+            LIMIT :limit
+        """
+        return self._fetch_all(stmt, {"limit": limit})
+
+    def update_memory_status(self, memory_id: int, new_status: str, final_memory: str = None):
+        """Update memory status and optionally final_memory"""
+        stmt = """
+            UPDATE memory_lifecycle 
+            SET status = :status, final_memory = :final_memory
+            WHERE id = :memory_id
+        """
+        params = {"status": new_status, "final_memory": final_memory, "memory_id": memory_id}
+        
+        with self.engine.begin() as conn:
+            conn.execute(text(stmt), params)
+            
     def add_attachment(self, message_id, name, attachment_type, data, size, file_path=None):
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         stmt = """
@@ -767,12 +920,12 @@ class CommandHistory:
             date_filter = "WHERE timestamp <= :end_date"
             params = {"end_date": end_date}
 
-        # Use different GROUP_CONCAT for different databases
+        
         if 'sqlite' in str(self.engine.url):
             group_concat_models = "GROUP_CONCAT(DISTINCT model)"
             group_concat_providers = "GROUP_CONCAT(DISTINCT provider)"
         else:
-            # PostgreSQL uses STRING_AGG
+            
             group_concat_models = "STRING_AGG(DISTINCT model, ',')"
             group_concat_providers = "STRING_AGG(DISTINCT provider, ',')"
 
@@ -806,7 +959,7 @@ class CommandHistory:
             ])
 
     def get_command_patterns(self, timeframe='day') -> pd.DataFrame:
-        # Use different date formatting for different databases
+        
         if 'sqlite' in str(self.engine.url):
             time_group_formats = {
                 'hour': "strftime('%Y-%m-%d %H', timestamp)",
@@ -815,7 +968,7 @@ class CommandHistory:
                 'month': "strftime('%Y-%m', timestamp)"
             }
         else:
-            # PostgreSQL date formatting
+            
             time_group_formats = {
                 'hour': "TO_CHAR(timestamp::timestamp, 'YYYY-MM-DD HH24')",
                 'day': "TO_CHAR(timestamp::timestamp, 'YYYY-MM-DD')",
@@ -825,7 +978,7 @@ class CommandHistory:
         
         time_group = time_group_formats.get(timeframe, time_group_formats['day'])
 
-        # Use different SUBSTR functions
+        
         if 'sqlite' in str(self.engine.url):
             substr_func = "SUBSTR"
             instr_func = "INSTR"
@@ -933,22 +1086,22 @@ def save_conversation_message(
     if wd is None:
         wd = os.getcwd()
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    if message_id is None:
+        message_id = generate_message_id()
 
 
     return command_history.add_conversation(
-        role=role,
-        timestamp=timestamp, 
-        content=content,
-        conversation_id=conversation_id,
-        directory_path=wd,
-        model=model,
-        provider=provider,
-        npc=npc,
-        team=team,
-        attachments=attachments,
-        message_id=message_id,
-    )
-
+        message_id, 
+        timestamp, 
+        role, 
+        content, 
+        conversation_id, 
+        wd, 
+        model=model, 
+        provider=provider, 
+        npc=npc, 
+        team=team, 
+        attachments=attachments)
 def retrieve_last_conversation(
     command_history: CommandHistory, conversation_id: str
     ) -> str:
@@ -957,7 +1110,7 @@ def retrieve_last_conversation(
     """
     last_message = command_history.get_last_conversation(conversation_id)
     if last_message:
-        return last_message['content']  # Use dict key access
+        return last_message['content']  
     return "No previous conversation messages found."
 
 def save_attachment_to_message(
@@ -1010,7 +1163,7 @@ def get_available_tables(db_path_or_engine: Union[str, Engine]) -> List[Tuple[st
                     "SELECT name FROM sqlite_master WHERE type='table' AND name != 'command_history'"
                 ))
             else:
-                # PostgreSQL
+                
                 result = conn.execute(text("""
                     SELECT table_name FROM information_schema.tables 
                     WHERE table_schema = 'public' AND table_name != 'command_history'