npcpy 1.0.26__py3-none-any.whl → 1.2.32__py3-none-any.whl

npcpy/memory/command_history.py

@@ -1,5 +1,4 @@
 import os
-import sqlite3
 import json
 from datetime import datetime
 import uuid
@@ -9,16 +8,29 @@ import numpy as np
 
 try:
     import sqlalchemy
-    from sqlalchemy import create_engine, text
+    from sqlalchemy import create_engine, text, MetaData, Table, Column, Integer, String, Text, DateTime, LargeBinary, ForeignKey, Boolean, func
     from sqlalchemy.engine import Engine, Connection as SQLAlchemyConnection
     from sqlalchemy.exc import SQLAlchemyError
+    from sqlalchemy.sql import select, insert, update, delete
+    from sqlalchemy.dialects import sqlite, postgresql
     _HAS_SQLALCHEMY = True
 except ImportError:
     _HAS_SQLALCHEMY = False
-    Engine = type(None) # Define dummy types if sqlalchemy not installed
-    SQLAlchemyConnection = type(None)
-    create_engine = None
-    text = None
+    print("SQLAlchemy not available - this module requires SQLAlchemy")
+    raise
+
+try:
+    import chromadb
+except ModuleNotFoundError:
+    print("chromadb not installed")
+except OSError as e:
+    print('os error importing chromadb:', e)
+except NameError as e:
+    print('name error importing chromadb:', e)
+    chromadb = None
+
+
+import logging 
 
 def flush_messages(n: int, messages: list) -> dict:
     if n <= 0:
@@ -27,45 +39,46 @@ def flush_messages(n: int, messages: list) -> dict:
             "output": "Error: 'n' must be a positive integer.",
         }
 
-    removed_count = min(n, len(messages))  # Calculate how many to remove
-    del messages[-removed_count:]  # Remove the last n messages
+    removed_count = min(n, len(messages))
+    del messages[-removed_count:]
 
     return {
         "messages": messages,
         "output": f"Flushed {removed_count} message(s). Context count is now {len(messages)} messages.",
     }
 
+def create_engine_from_path(db_path: str) -> Engine:
+    """Create SQLAlchemy engine from database path, detecting type"""
+    if db_path.startswith('postgresql://') or db_path.startswith('postgres://'):
+        return create_engine(db_path)
+    else:
+        
+        if db_path.startswith('~/'):
+            db_path = os.path.expanduser(db_path)
+        return create_engine(f'sqlite:///{db_path}')
 
-def get_db_connection():
-    conn = sqlite3.connect(db_path)
-    conn.row_factory = sqlite3.Row
-    return conn
-
-
-def fetch_messages_for_conversation(conversation_id):
-    conn = get_db_connection()
-    cursor = conn.cursor()
+def get_db_connection(db_path: str = "~/npcsh_history.db") -> Engine:
+    """Get SQLAlchemy engine"""
+    return create_engine_from_path(db_path)
 
-    query = """
+def fetch_messages_for_conversation(engine: Engine, conversation_id: str):
+    query = text("""
         SELECT role, content, timestamp
         FROM conversation_history
-        WHERE conversation_id = ?
+        WHERE conversation_id = :conversation_id
         ORDER BY timestamp ASC
-    """
-    cursor.execute(query, (conversation_id,))
-    messages = cursor.fetchall()
-    conn.close()
-
-    return [
-        {
-            "role": message["role"],
-            "content": message["content"],
-            "timestamp": message["timestamp"],
-        }
-        for message in messages
-    ]
-
-
+    """)
+    
+    with engine.connect() as conn:
+        result = conn.execute(query, {"conversation_id": conversation_id})
+        return [
+            {
+                "role": row.role,
+                "content": row.content,
+                "timestamp": row.timestamp,
+            }
+            for row in result
+        ]
 
 def deep_to_dict(obj):
     """
@@ -84,8 +97,7 @@ def deep_to_dict(obj):
     if isinstance(obj, (int, float, str, bool, type(None))):
         return obj
 
-    return None  # Drop objects that don't have a known conversion
-
+    return None
 
 class CustomJSONEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -94,59 +106,38 @@ class CustomJSONEncoder(json.JSONEncoder):
         except TypeError:
             return super().default(obj)
 
-
 def show_history(command_history, args):
     if args:
-        search_results = command_history.search(args[0])
+        search_results = command_history.search_commands(args[0])
         if search_results:
             return "\n".join(
-                [f"{item[0]}. [{item[1]}] {item[2]}" for item in search_results]
+                [f"{item['id']}. [{item['timestamp']}] {item['command']}" for item in search_results]
             )
         else:
             return f"No commands found matching '{args[0]}'"
     else:
-        all_history = command_history.get_all()
-        return "\n".join([f"{item[0]}. [{item[1]}] {item[2]}" for item in all_history])
-
+        all_history = command_history.get_all_commands()
+        return "\n".join([f"{item['id']}. [{item['timestamp']}] {item['command']}" for item in all_history])
 
 def query_history_for_llm(command_history, query):
-    results = command_history.search(query)
+    results = command_history.search_commands(query)
     formatted_results = [
-        f"Command: {r[2]}\nOutput: {r[4]}\nLocation: {r[5]}" for r in results
+        f"Command: {r['command']}\nOutput: {r['output']}\nLocation: {r['location']}" for r in results
     ]
     return "\n\n".join(formatted_results)
 
-
-try:
-    import chromadb
-except ModuleNotFoundError:
-    print("chromadb not installed")
-except OSError as e:
-    print('os error importing chromadb:', e)
-except NameError as e:
-    print('name error importing chromadb:', e)
-    chromadb = None
-import numpy as np
-import os
-from typing import Optional, Dict, List, Union, Tuple
-
-
-def setup_chroma_db(collection, description='', db_path: str= ''):
+def setup_chroma_db(collection, description='', db_path: str = ''):
     """Initialize Chroma vector database without a default embedding function"""
     if db_path == '':
         db_path = os.path.expanduser('~/npcsh_chroma_db')
         
     try:
-        # Create or connect to Chroma client with persistent storage
         client = chromadb.PersistentClient(path=db_path)
 
-        # Check if collection exists, create if not
         try:
             collection = client.get_collection(collection)
             print("Connected to existing facts collection")
         except ValueError:
-            # Create new collection without an embedding function
-            # We'll provide embeddings manually using get_embeddings
             collection = client.create_collection(
                 name=collection,
                 metadata={"description": description},
@@ -158,48 +149,271 @@ def setup_chroma_db(collection, description='', db_path: str= ''):
         print(f"Error setting up Chroma DB: {e}")
         raise
 
+def init_kg_schema(engine: Engine):
+    """Creates the multi-scoped, path-aware KG tables using SQLAlchemy"""
+    
+    
+    metadata = MetaData()
+    
+    kg_facts = Table('kg_facts', metadata,
+        Column('statement', Text, nullable=False),
+        Column('team_name', String(255), nullable=False),
+        Column('npc_name', String(255), nullable=False),
+        Column('directory_path', Text, nullable=False),
+        Column('source_text', Text),
+        Column('type', String(100)),
+        Column('generation', Integer),
+        Column('origin', String(100)),
+        
+        schema=None
+    )
+    
+    kg_concepts = Table('kg_concepts', metadata,
+        Column('name', Text, nullable=False),
+        Column('team_name', String(255), nullable=False),
+        Column('npc_name', String(255), nullable=False),
+        Column('directory_path', Text, nullable=False),
+        Column('generation', Integer),
+        Column('origin', String(100)),
+        schema=None
+    )
+    
+    kg_links = Table('kg_links', metadata,
+        Column('source', Text, nullable=False),
+        Column('target', Text, nullable=False),
+        Column('team_name', String(255), nullable=False),
+        Column('npc_name', String(255), nullable=False),
+        Column('directory_path', Text, nullable=False),
+        Column('type', String(100), nullable=False),
+        schema=None
+    )
+    
+    kg_metadata = Table('kg_metadata', metadata,
+        Column('key', String(255), nullable=False),
+        Column('team_name', String(255), nullable=False),
+        Column('npc_name', String(255), nullable=False),
+        Column('directory_path', Text, nullable=False),
+        Column('value', Text),
+        schema=None
+    )
+    
+    
+    metadata.create_all(engine, checkfirst=True)
+
+def load_kg_from_db(engine: Engine, team_name: str, npc_name: str, directory_path: str) -> Dict[str, Any]:
+    """Loads the KG for a specific scope (team, npc, path) from database."""
+    kg = {
+        "generation": 0,
+        "facts": [],
+        "concepts": [],
+        "concept_links": [],
+        "fact_to_concept_links": {},
+        "fact_to_fact_links": []
+    }
+    
+    with engine.connect() as conn:
+        try:
+            
+            result = conn.execute(text("""
+                SELECT value FROM kg_metadata 
+                WHERE team_name = :team AND npc_name = :npc AND directory_path = :path AND key = 'generation'
+            """), {"team": team_name, "npc": npc_name, "path": directory_path})
+            
+            row = result.fetchone()
+            if row:
+                kg['generation'] = int(row.value)
+
+            
+            result = conn.execute(text("""
+                SELECT statement, source_text, type, generation, origin FROM kg_facts 
+                WHERE team_name = :team AND npc_name = :npc AND directory_path = :path
+            """), {"team": team_name, "npc": npc_name, "path": directory_path})
+            
+            kg['facts'] = [
+                {
+                    "statement": row.statement,
+                    "source_text": row.source_text,
+                    "type": row.type,
+                    "generation": row.generation,
+                    "origin": row.origin
+                }
+                for row in result
+            ]
+
+            
+            result = conn.execute(text("""
+                SELECT name, generation, origin FROM kg_concepts 
+                WHERE team_name = :team AND npc_name = :npc AND directory_path = :path
+            """), {"team": team_name, "npc": npc_name, "path": directory_path})
+            
+            kg['concepts'] = [
+                {"name": row.name, "generation": row.generation, "origin": row.origin}
+                for row in result
+            ]
+
+            
+            links = {}
+            result = conn.execute(text("""
+                SELECT source, target, type FROM kg_links 
+                WHERE team_name = :team AND npc_name = :npc AND directory_path = :path
+            """), {"team": team_name, "npc": npc_name, "path": directory_path})
+            
+            for row in result:
+                if row.type == 'fact_to_concept':
+                    if row.source not in links:
+                        links[row.source] = []
+                    links[row.source].append(row.target)
+                elif row.type == 'concept_to_concept':
+                    kg['concept_links'].append((row.source, row.target))
+                elif row.type == 'fact_to_fact':
+                    kg['fact_to_fact_links'].append((row.source, row.target))
+            
+            kg['fact_to_concept_links'] = links
+            
+        except SQLAlchemyError:
+            
+            init_kg_schema(engine)
+    
+    return kg
 
-class CommandHistory:
-    def __init__(self, db: Union[str, sqlite3.Connection, Engine] = "~/npcsh_history.db"):
+def save_kg_to_db(engine: Engine, kg_data: Dict[str, Any], team_name: str, npc_name: str, directory_path: str):
+    """Saves a knowledge graph dictionary to the database, ignoring duplicates."""
+    try:
+        with engine.begin() as conn:
+            
+            facts_to_save = [
+                {
+                    "statement": fact['statement'],
+                    "team_name": team_name,
+                    "npc_name": npc_name,
+                    "directory_path": directory_path,
+                    "generation": fact.get('generation', 0),
+                    "origin": fact.get('origin', 'organic')
+                }
+                for fact in kg_data.get("facts", [])
+            ]
+            
+            if facts_to_save:
+                
+                if 'sqlite' in str(engine.url):
+                    stmt = text("""
+                        INSERT OR IGNORE INTO kg_facts 
+                        (statement, team_name, npc_name, directory_path, generation, origin)
+                        VALUES (:statement, :team_name, :npc_name, :directory_path, :generation, :origin)
+                    """)
+                else:
+                    stmt = text("""
+                        INSERT INTO kg_facts 
+                        (statement, team_name, npc_name, directory_path, generation, origin)
+                        VALUES (:statement, :team_name, :npc_name, :directory_path, :generation, :origin)
+                        ON CONFLICT (statement, team_name, npc_name, directory_path) DO NOTHING
+                    """)
+                
+                for fact in facts_to_save:
+                    conn.execute(stmt, fact)
+
+            
+            concepts_to_save = [
+                {
+                    "name": concept['name'],
+                    "team_name": team_name,
+                    "npc_name": npc_name,
+                    "directory_path": directory_path,
+                    "generation": concept.get('generation', 0),
+                    "origin": concept.get('origin', 'organic')
+                }
+                for concept in kg_data.get("concepts", [])
+            ]
+            
+            if concepts_to_save:
+                if 'sqlite' in str(engine.url):
+                    stmt = text("""
+                        INSERT OR IGNORE INTO kg_concepts 
+                        (name, team_name, npc_name, directory_path, generation, origin)
+                        VALUES (:name, :team_name, :npc_name, :directory_path, :generation, :origin)
+                    """)
+                else:
+                    stmt = text("""
+                        INSERT INTO kg_concepts 
+                        (name, team_name, npc_name, directory_path, generation, origin)
+                        VALUES (:name, :team_name, :npc_name, :directory_path, :generation, :origin)
+                        ON CONFLICT (name, team_name, npc_name, directory_path) DO NOTHING
+                    """)
+                
+                for concept in concepts_to_save:
+                    conn.execute(stmt, concept)
+
+            
+            if 'sqlite' in str(engine.url):
+                stmt = text("""
+                    INSERT OR REPLACE INTO kg_metadata (key, value, team_name, npc_name, directory_path)
+                    VALUES ('generation', :generation, :team_name, :npc_name, :directory_path)
+                """)
+            else:
+                stmt = text("""
+                    INSERT INTO kg_metadata (key, value, team_name, npc_name, directory_path)
+                    VALUES ('generation', :generation, :team_name, :npc_name, :directory_path)
+                    ON CONFLICT (key, team_name, npc_name, directory_path) 
+                    DO UPDATE SET value = EXCLUDED.value
+                """)
+            
+            conn.execute(stmt, {
+                "generation": str(kg_data.get('generation', 0)),
+                "team_name": team_name,
+                "npc_name": npc_name,
+                "directory_path": directory_path
+            })
+
+            
+            conn.execute(text("""
+                DELETE FROM kg_links 
+                WHERE team_name = :team_name AND npc_name = :npc_name AND directory_path = :directory_path
+            """), {"team_name": team_name, "npc_name": npc_name, "directory_path": directory_path})
+            
+            
+            for fact, concepts in kg_data.get("fact_to_concept_links", {}).items():
+                for concept in concepts:
+                    conn.execute(text("""
+                        INSERT INTO kg_links (source, target, type, team_name, npc_name, directory_path)
+                        VALUES (:source, :target, 'fact_to_concept', :team_name, :npc_name, :directory_path)
+                    """), {
+                        "source": fact, "target": concept,
+                        "team_name": team_name, "npc_name": npc_name, "directory_path": directory_path
+                    })
+            
+            for c1, c2 in kg_data.get("concept_links", []):
+                conn.execute(text("""
+                    INSERT INTO kg_links (source, target, type, team_name, npc_name, directory_path)
+                    VALUES (:source, :target, 'concept_to_concept', :team_name, :npc_name, :directory_path)
+                """), {
+                    "source": c1, "target": c2,
+                    "team_name": team_name, "npc_name": npc_name, "directory_path": directory_path
+                })
+            
+            for f1, f2 in kg_data.get("fact_to_fact_links", []):
+                conn.execute(text("""
+                    INSERT INTO kg_links (source, target, type, team_name, npc_name, directory_path)
+                    VALUES (:source, :target, 'fact_to_fact', :team_name, :npc_name, :directory_path)
+                """), {
+                    "source": f1, "target": f2,
+                    "team_name": team_name, "npc_name": npc_name, "directory_path": directory_path
+                })
+            
+    except Exception as e:
+        print(f"Failed to save KG for scope '({team_name}, {npc_name}, {directory_path})': {e}")
 
-        self._is_sqlalchemy = False
-        self.cursor = None
-        self.db_path = None # Store the determined path if available
+def generate_message_id() -> str:
+    return str(uuid.uuid4())
 
+class CommandHistory:
+    def __init__(self, db: Union[str, Engine] = "~/npcsh_history.db"):
+        
         if isinstance(db, str):
-            self.db_path = os.path.expanduser(db)
-            try:
-                self.conn = sqlite3.connect(self.db_path, check_same_thread=False) # Allow multithread access if needed
-                self.conn.row_factory = sqlite3.Row
-                self.cursor = self.conn.cursor()
-                self.cursor.execute("PRAGMA foreign_keys = ON")
-                self.conn.commit()
-
-            except sqlite3.Error as e:
-                print(f"FATAL: Error connecting to sqlite3 DB at {self.db_path}: {e}")
-                raise
-
-        elif isinstance(db, sqlite3.Connection):
-            self.conn = db
-            if not hasattr(self.conn, 'row_factory') or self.conn.row_factory is None:
-                 # Set row_factory if not already set on provided connection
-                 try: self.conn.row_factory = sqlite3.Row
-                 except Exception as e: print(f"Warning: Could not set row_factory on provided sqlite3 connection: {e}")
-
-            self.cursor = self.conn.cursor()
-            try:
-                self.cursor.execute("PRAGMA foreign_keys = ON")
-                self.conn.commit()
-            except sqlite3.Error as e:
-                print(f"Warning: Could not set PRAGMA foreign_keys on provided sqlite3 connection: {e}")
-
-
-        elif _HAS_SQLALCHEMY and isinstance(db, Engine):
+            self.engine = create_engine_from_path(db)
+            self.db_path = db
+        elif isinstance(db, Engine):
+            self.engine = db
             self.db_path = str(db.url)
-            self.conn = db
-            self._is_sqlalchemy = True
-
-
         else:
             raise TypeError(f"Unsupported type for CommandHistory db parameter: {type(db)}")
 
@@ -207,318 +421,468 @@ class CommandHistory:
 
     def _initialize_schema(self):
         """Creates all necessary tables."""
-        print("Initializing database schema...")
-        self.create_command_table()
-        self.create_conversation_table()
-        self.create_attachment_table()
-        self.create_jinx_call_table()
-        print("Database schema initialization complete.")
-
-    def _execute(self, sql: str, params: Optional[Union[tuple, Dict]] = None, script: bool = False, requires_fk: bool = False) -> Optional[int]:
-        """Executes SQL, handling transactions and FK pragma for SQLAlchemy. Returns lastrowid for INSERTs."""
-        last_row_id = None
-        try:
-            if self._is_sqlalchemy:
-                with self.conn.connect() as connection:
-                    with connection.begin():
-                        if requires_fk and self.conn.url.drivername == 'sqlite':
-                             try: connection.execute(text("PRAGMA foreign_keys=ON"))
-                             except SQLAlchemyError as e: print(f"Warning: SQLAlchemy PRAGMA foreign_keys=ON failed: {e}")
-
-                        if script:
-                             statements = [s.strip() for s in sql.split(';') if s.strip()]
-                             for statement in statements:
-                                  result_proxy = connection.execute(text(statement))
-                                  if result_proxy.lastrowid is not None: last_row_id = result_proxy.lastrowid
-                        else:
-                             result_proxy = connection.execute(text(sql), params or {})
-                             if result_proxy.lastrowid is not None: last_row_id = result_proxy.lastrowid
-            else:
-                 # Existing sqlite3 logic
-                 if script:
-                     self.cursor.executescript(sql)
-                 else:
-                     self.cursor.execute(sql, params or ())
-                 last_row_id = self.cursor.lastrowid # Get lastrowid for sqlite3
-                 self.conn.commit()
-
-            return last_row_id
-
-        except (sqlite3.Error, SQLAlchemyError) as e:
-             error_type = "SQLAlchemy" if self._is_sqlalchemy else "SQLite"
-             print(f"{error_type} Error executing: {sql[:100]}... Error: {e}")
-             if not self._is_sqlalchemy:
-                 try: self.conn.rollback()
-                 except Exception as rb_err: print(f"SQLite rollback failed: {rb_err}")
-             # Decide whether to raise the error or just return None/False
-             raise # Re-raise the error to indicate failure
-        except Exception as e:
-            print(f"Unexpected error in _execute: {e}")
-            raise # Re-raise unexpected errors
-
-
-    def _fetch_one(self, sql: str, params: Optional[Union[tuple, Dict]] = None) -> Optional[Dict]:
-        """Fetches a single row, adapting to connection type."""
-        try:
-            if self._is_sqlalchemy:
-                 with self.conn.connect() as connection:
-                      # No need for transaction for SELECT
-                      result = connection.execute(text(sql), params or {})
-                      row = result.fetchone()
-                      return dict(row._mapping) if row else None
-            else:
-                 self.cursor.execute(sql, params or ())
-                 row = self.cursor.fetchone()
-                 return dict(row) if row else None
-        except (sqlite3.Error, SQLAlchemyError) as e:
-             error_type = "SQLAlchemy" if self._is_sqlalchemy else "SQLite"
-             print(f"{error_type} Error fetching one: {sql[:100]}... Error: {e}")
-             return None # Return None on error
-        except Exception as e:
-            print(f"Unexpected error in _fetch_one: {e}")
-            return None
-
-    def _fetch_all(self, sql: str, params: Optional[Union[tuple, Dict]] = None) -> List[Dict]:
-        """Fetches all rows, adapting to connection type."""
-        try:
-            if self._is_sqlalchemy:
-                with self.conn.connect() as connection:
-                    # Convert tuple params to a dictionary format for SQLAlchemy
-                    if params and isinstance(params, tuple):
-                        # Extract parameter placeholders from SQL
-                        placeholders = []
-                        for i, char in enumerate(sql):
-                            if char == '?':
-                                placeholders.append(i)
-                        
-                        # Convert tuple params to dict format
-                        dict_params = {}
-                        for i, value in enumerate(params):
-                            param_name = f"param_{i}"
-                            # Replace ? with :param_name in SQL
-                            sql = sql.replace('?', f":{param_name}", 1)
-                            dict_params[param_name] = value
-                        
-                        params = dict_params
+        metadata = MetaData()
+        
+        
+        Table('command_history', metadata,
+            Column('id', Integer, primary_key=True, autoincrement=True),
+            Column('timestamp', String(50)),
+            Column('command', Text),
+            Column('subcommands', Text),
+            Column('output', Text),
+            Column('location', Text)
+        )
+        
+        
+        Table('conversation_history', metadata,
+            Column('id', Integer, primary_key=True, autoincrement=True),
+            Column('message_id', String(50), unique=True, nullable=False),
+            Column('timestamp', String(50)),
+            Column('role', String(20)),
+            Column('content', Text),
+            Column('conversation_id', String(100)),
+            Column('directory_path', Text),
+            Column('model', String(100)),
+            Column('provider', String(100)),
+            Column('npc', String(100)),
+            Column('team', String(100))
+        )
+        
+        
+        Table('message_attachments', metadata,
+            Column('id', Integer, primary_key=True, autoincrement=True),
+            Column('message_id', String(50), ForeignKey('conversation_history.message_id', ondelete='CASCADE'), nullable=False),
+            Column('attachment_name', String(255)),
+            Column('attachment_type', String(100)),
+            Column('attachment_data', LargeBinary),
+            Column('attachment_size', Integer),
+            Column('upload_timestamp', String(50)),
+            Column('file_path', Text) 
+        )
+        
+        
+        Table('jinx_execution_log', metadata,
+            Column('execution_id', Integer, primary_key=True, autoincrement=True),
+            Column('triggering_message_id', String(50), ForeignKey('conversation_history.message_id', ondelete='CASCADE'), nullable=False),
+            Column('response_message_id', String(50), ForeignKey('conversation_history.message_id', ondelete='SET NULL')),
+            Column('conversation_id', String(100), nullable=False),
+            Column('timestamp', String(50), nullable=False),
+            Column('npc_name', String(100)),
+            Column('team_name', String(100)),
+            Column('jinx_name', String(100), nullable=False),
+            Column('jinx_inputs', Text),
+            Column('jinx_output', Text),
+            Column('status', String(50), nullable=False),
+            Column('error_message', Text),
+            Column('duration_ms', Integer)
+        )
+        
+        Table('memory_lifecycle', metadata,
+            Column('id', Integer, primary_key=True, autoincrement=True),
+            Column('message_id', String(50), nullable=False),
+            Column('conversation_id', String(100), nullable=False),
+            Column('npc', String(100), nullable=False),
+            Column('team', String(100), nullable=False),
+            Column('directory_path', Text, nullable=False),
+            Column('timestamp', String(50), nullable=False),
+            Column('initial_memory', Text, nullable=False),
+            Column('final_memory', Text),
+            Column('status', String(50), nullable=False),
+            Column('model', String(100)),
+            Column('provider', String(100)),
+            Column('created_at', DateTime, default=func.now())
+        )
+                
+        
+        metadata.create_all(self.engine, checkfirst=True)
+        
+        
+        with self.engine.begin() as conn:
+            
+            index_queries = [
+                "CREATE INDEX IF NOT EXISTS idx_jinx_log_trigger_msg ON jinx_execution_log (triggering_message_id)",
+                "CREATE INDEX IF NOT EXISTS idx_jinx_log_convo_id ON jinx_execution_log (conversation_id)",
+                "CREATE INDEX IF NOT EXISTS idx_jinx_log_jinx_name ON jinx_execution_log (jinx_name)",
+                "CREATE INDEX IF NOT EXISTS idx_jinx_log_timestamp ON jinx_execution_log (timestamp)"
+            ]
+            
+            for idx_query in index_queries:
+                try:
+                    conn.execute(text(idx_query))
+                except SQLAlchemyError:
                     
-                    result = connection.execute(text(sql), params or {})
-                    rows = result.fetchall()
-                    return [dict(row._mapping) for row in rows]
-            else:
-                self.cursor.execute(sql, params or ())
-                rows = self.cursor.fetchall()
-                return [dict(row) for row in rows]
-        except (sqlite3.Error, SQLAlchemyError) as e:
-            error_type = "SQLAlchemy" if self._is_sqlalchemy else "SQLite"
-            print(f"{error_type} Error fetching all: {sql[:100]}... Error: {e}")
-            return [] # Return empty list on error
-        except Exception as e:
-            print(f"Unexpected error in _fetch_all: {e}")
-            return []
-
-    def create_command_table(self):
-        query = """
-        CREATE TABLE IF NOT EXISTS command_history (
-            id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT, command TEXT,
-            subcommands TEXT, output TEXT, location TEXT
-        )"""
-        self._execute(query)
-
-    def create_conversation_table(self):
-        query = """
-        CREATE TABLE IF NOT EXISTS conversation_history (
-            id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT UNIQUE NOT NULL,
-            timestamp TEXT, role TEXT, content TEXT, conversation_id TEXT,
-            directory_path TEXT, model TEXT, provider TEXT, npc TEXT, team TEXT
-        )"""
-        self._execute(query)
-
-    def create_attachment_table(self):
-        query = """
-        CREATE TABLE IF NOT EXISTS message_attachments (
-            id INTEGER PRIMARY KEY AUTOINCREMENT, message_id TEXT NOT NULL,
-            attachment_name TEXT, attachment_type TEXT, attachment_data BLOB,
-            attachment_size INTEGER, upload_timestamp TEXT,
-            FOREIGN KEY (message_id) REFERENCES conversation_history(message_id) ON DELETE CASCADE
-        )"""
-        self._execute(query, requires_fk=True)
-
-    def create_jinx_call_table(self):
-        table_query = '''
-        CREATE TABLE IF NOT EXISTS jinx_execution_log (
-            execution_id INTEGER PRIMARY KEY AUTOINCREMENT, triggering_message_id TEXT NOT NULL,
-            response_message_id TEXT, conversation_id TEXT NOT NULL, timestamp TEXT NOT NULL,
-            npc_name TEXT, team_name TEXT, jinx_name TEXT NOT NULL, jinx_inputs TEXT,
-            jinx_output TEXT, status TEXT NOT NULL, error_message TEXT, duration_ms INTEGER,
-            FOREIGN KEY (triggering_message_id) REFERENCES conversation_history(message_id) ON DELETE CASCADE,
-            FOREIGN KEY (response_message_id) REFERENCES conversation_history(message_id) ON DELETE SET NULL
-        );
-        '''
-        self._execute(table_query, requires_fk=True)
-
-        index_queries = [
-            "CREATE INDEX IF NOT EXISTS idx_jinx_log_trigger_msg ON jinx_execution_log (triggering_message_id);",
-            "CREATE INDEX IF NOT EXISTS idx_jinx_log_convo_id ON jinx_execution_log (conversation_id);",
-            "CREATE INDEX IF NOT EXISTS idx_jinx_log_jinx_name ON jinx_execution_log (jinx_name);",
-            "CREATE INDEX IF NOT EXISTS idx_jinx_log_timestamp ON jinx_execution_log (timestamp);"
-        ]
-        for idx_query in index_queries:
-             self._execute(idx_query)
+                    pass
+        
+        
+        init_kg_schema(self.engine)
+
+    def _execute_returning_id(self, stmt: str, params: Dict = None) -> Optional[int]:
+        """Execute INSERT and return the generated ID"""
+        with self.engine.begin() as conn:
+            result = conn.execute(text(stmt), params or {})
+            return result.lastrowid if hasattr(result, 'lastrowid') else None
+
+    def _fetch_one(self, stmt: str, params: Dict = None) -> Optional[Dict]:
+        """Fetch a single row"""
+        with self.engine.connect() as conn:
+            result = conn.execute(text(stmt), params or {})
+            row = result.fetchone()
+            return dict(row._mapping) if row else None
+
+    def _fetch_all(self, stmt: str, params: Dict = None) -> List[Dict]:
+        """Fetch all rows"""
+        with self.engine.connect() as conn:
+            result = conn.execute(text(stmt), params or {})
+            return [dict(row._mapping) for row in result]
 
     def add_command(self, command, subcommands, output, location):
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        safe_subcommands = str(subcommands)
-        safe_output = str(output)
-        sql = """
+        stmt = """
             INSERT INTO command_history (timestamp, command, subcommands, output, location)
-            VALUES (?, ?, ?, ?, ?)
-            """
-        params = (timestamp, command, safe_subcommands, safe_output, location)
-        self._execute(sql, params)
+            VALUES (:timestamp, :command, :subcommands, :output, :location)
+        """
+        params = {
+            "timestamp": timestamp,
+            "command": command,
+            "subcommands": str(subcommands),
+            "output": str(output),
+            "location": location
+        }
+        
+        with self.engine.begin() as conn:
+            conn.execute(text(stmt), params)
 
-    def generate_message_id(self) -> str:
-        return str(uuid.uuid4())
 
     def add_conversation(
-        self, role, content, conversation_id, directory_path,
-        model=None, provider=None, npc=None, team=None,
-        attachments=None, message_id=None,
+        self, 
+        message_id,
+        timestamp,
+        role, 
+        content, 
+        conversation_id, 
+        directory_path,
+        model=None, 
+        provider=None, 
+        npc=None, 
+        team=None,
+        attachments=None,
     ):
-        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        if message_id is None: message_id = self.generate_message_id()
-        if isinstance(content, dict): content = json.dumps(content, cls=CustomJSONEncoder)
-
-        existing_row = self._fetch_one(
-             "SELECT content FROM conversation_history WHERE message_id = ?", (message_id,)
-        )
+        if isinstance(content, (dict, list)):
+            content = json.dumps(content, cls=CustomJSONEncoder)
 
-        if existing_row:
-            sql = "UPDATE conversation_history SET content = ?, timestamp = ? WHERE message_id = ?"
-            params = (content, timestamp, message_id)
-            self._execute(sql, params)
-        else:
-            sql = """INSERT INTO conversation_history
-                (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
-            params = (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team,)
-            self._execute(sql, params)
+        stmt = """
+            INSERT INTO conversation_history
+            (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team)
+            VALUES (:message_id, :timestamp, :role, :content, :conversation_id, :directory_path, :model, :provider, :npc, :team)
+        """
+        params = {
+            "message_id": message_id, "timestamp": timestamp, "role": role, "content": content,
+            "conversation_id": conversation_id, "directory_path": directory_path, "model": model,
+            "provider": provider, "npc": npc, "team": team
+        }
+        with self.engine.begin() as conn:
+            conn.execute(text(stmt), params)
 
         if attachments:
             for attachment in attachments:
                 self.add_attachment(
-                    message_id, attachment["name"], attachment["type"],
-                    attachment["data"], attachment_size=attachment.get("size"),
+                    message_id=message_id,
+                    name=attachment.get("name"),
+                    attachment_type=attachment.get("type"),
+                    data=attachment.get("data"),
+                    size=attachment.get("size"),
+                    file_path=attachment.get("path") 
                 )
+
         return message_id
 
-    def add_attachment(
-        self, message_id, attachment_name, attachment_type, attachment_data, attachment_size=None,
-    ):
+    def add_memory_to_database(self, message_id: str, conversation_id: str, npc: str, team: str, 
+                            directory_path: str, initial_memory: str, status: str, 
+                            model: str = None, provider: str = None, final_memory: str = None):
+        """Store a memory entry in the database"""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        stmt = """
+            INSERT INTO memory_lifecycle 
+            (message_id, conversation_id, npc, team, directory_path, timestamp, 
+            initial_memory, final_memory, status, model, provider)
+            VALUES (:message_id, :conversation_id, :npc, :team, :directory_path, 
+                    :timestamp, :initial_memory, :final_memory, :status, :model, :provider)
+        """
+        
+        params = {
+            "message_id": message_id, "conversation_id": conversation_id,
+            "npc": npc, "team": team, "directory_path": directory_path,
+            "timestamp": timestamp, "initial_memory": initial_memory,
+            "final_memory": final_memory, "status": status,
+            "model": model, "provider": provider
+        }
+        
+        return self._execute_returning_id(stmt, params)
+    def get_memories_for_scope(
+        self,
+        npc: str,
+        team: str,
+        directory_path: str,
+        status: Optional[str] = None
+    ) -> List[Dict]:
+        
+        query = """
+            SELECT id, initial_memory, final_memory, 
+                status, timestamp, created_at
+            FROM memory_lifecycle
+            WHERE npc = :npc AND team = :team AND directory_path = :path
+        """
+        params = {"npc": npc, "team": team, "path": directory_path}
+        
+        if status:
+            query += " AND status = :status"
+            params["status"] = status
+        
+        query += " ORDER BY created_at DESC"
+        data =self._fetch_all(query, params)
+        return  data
+
+    def search_memory(self, query: str, npc: str = None, team: str = None, 
+                    directory_path: str = None, status_filter: str = None, limit: int = 10):
+        """Search memories with hierarchical scope"""
+        conditions = ["LOWER(initial_memory) LIKE LOWER(:query) OR LOWER(final_memory) LIKE LOWER(:query)"]
+        params = {"query": f"%{query}%"}
+        
+        if status_filter:
+            conditions.append("status = :status")
+            params["status"] = status_filter
+        
+        
+        order_parts = []
+        if npc:
+            order_parts.append(f"CASE WHEN npc = '{npc}' THEN 1 ELSE 2 END")
+        if team:
+            order_parts.append(f"CASE WHEN team = '{team}' THEN 1 ELSE 2 END")
+        if directory_path:
+            order_parts.append(f"CASE WHEN directory_path = '{directory_path}' THEN 1 ELSE 2 END")
+        
+        order_clause = ", ".join(order_parts) + ", created_at DESC" if order_parts else "created_at DESC"
+        
+        stmt = f"""
+            SELECT * FROM memory_lifecycle 
+            WHERE {' AND '.join(conditions)}
+            ORDER BY {order_clause}
+            LIMIT :limit
+        """
+        params["limit"] = limit
+        
+        return self._fetch_all(stmt, params)
+
+    def get_memory_examples_for_context(self, npc: str, team: str, directory_path: str,
+                                    n_approved: int = 10, n_rejected: int = 10):
+        """Get recent approved and rejected memories for learning context"""
+        
+        approved_stmt = """
+            SELECT initial_memory, final_memory, status FROM memory_lifecycle
+            WHERE status IN ('human-approved', 'model-approved')
+            ORDER BY 
+                CASE WHEN npc = :npc AND team = :team AND directory_path = :path THEN 1
+                    WHEN npc = :npc AND team = :team THEN 2  
+                    WHEN team = :team THEN 3
+                    ELSE 4 END,
+                created_at DESC
+            LIMIT :n_approved
+        """
+        
+        rejected_stmt = """
+            SELECT initial_memory, status FROM memory_lifecycle
+            WHERE status IN ('human-rejected', 'model-rejected')
+            ORDER BY 
+                CASE WHEN npc = :npc AND team = :team AND directory_path = :path THEN 1
+                    WHEN npc = :npc AND team = :team THEN 2
+                    WHEN team = :team THEN 3  
+                    ELSE 4 END,
+                created_at DESC
+            LIMIT :n_rejected
+        """
+        
+        params = {"npc": npc, "team": team, "path": directory_path, 
+                "n_approved": n_approved, "n_rejected": n_rejected}
+        
+        approved = self._fetch_all(approved_stmt, params)
+        rejected = self._fetch_all(rejected_stmt, params)
+        
+        return {"approved": approved, "rejected": rejected}
+
+    def get_pending_memories(self, limit: int = 50):
+        """Get memories pending human approval"""
+        stmt = """
+            SELECT * FROM memory_lifecycle 
+            WHERE status = 'pending_approval'
+            ORDER BY created_at ASC
+            LIMIT :limit
+        """
+        return self._fetch_all(stmt, {"limit": limit})
+
+    def update_memory_status(self, memory_id: int, new_status: str, final_memory: str = None):
+        """Update memory status and optionally final_memory"""
+        stmt = """
+            UPDATE memory_lifecycle 
+            SET status = :status, final_memory = :final_memory
+            WHERE id = :memory_id
+        """
+        params = {"status": new_status, "final_memory": final_memory, "memory_id": memory_id}
+        
+        with self.engine.begin() as conn:
+            conn.execute(text(stmt), params)
+            
+    def add_attachment(self, message_id, name, attachment_type, data, size, file_path=None):
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        if attachment_size is None and attachment_data is not None:
-            attachment_size = len(attachment_data)
-        sql = """INSERT INTO message_attachments
-            (message_id, attachment_name, attachment_type, attachment_data, attachment_size, upload_timestamp)
-            VALUES (?, ?, ?, ?, ?, ?)"""
-        params = (message_id, attachment_name, attachment_type, attachment_data, attachment_size, timestamp,)
-        self._execute(sql, params)
+        stmt = """
+            INSERT INTO message_attachments 
+            (message_id, attachment_name, attachment_type, attachment_data, attachment_size, upload_timestamp, file_path)
+            VALUES (:message_id, :name, :type, :data, :size, :timestamp, :file_path)
+        """
+        params = {
+            "message_id": message_id,
+            "name": name,
+            "type": attachment_type,
+            "data": data,
+            "size": size,
+            "timestamp": timestamp,
+            "file_path": file_path
+        }
+        with self.engine.begin() as conn:
+            conn.execute(text(stmt), params)
 
     def save_jinx_execution(
-        self, triggering_message_id: str, conversation_id: str, npc_name: Optional[str],
-        jinx_name: str, jinx_inputs: Dict, jinx_output: Any, status: str,
-        team_name: Optional[str] = None, error_message: Optional[str] = None,
-        response_message_id: Optional[str] = None, duration_ms: Optional[int] = None
+        self, 
+        triggering_message_id: str, 
+        conversation_id: str, 
+        npc_name: Optional[str],
+        jinx_name: str, 
+        jinx_inputs: Dict, 
+        jinx_output: Any, status: str,
+        team_name: Optional[str] = None, 
+        error_message: Optional[str] = None,
+        response_message_id: Optional[str] = None, 
+        duration_ms: Optional[int] = None
     ):
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        try: inputs_json = json.dumps(jinx_inputs, cls=CustomJSONEncoder)
-        except TypeError: inputs_json = json.dumps(str(jinx_inputs))
+        
+        try:
+            inputs_json = json.dumps(jinx_inputs, cls=CustomJSONEncoder)
+        except TypeError:
+            inputs_json = json.dumps(str(jinx_inputs))
+        
         try:
             if isinstance(jinx_output, (str, int, float, bool, list, dict, type(None))):
-                 outputs_json = json.dumps(jinx_output, cls=CustomJSONEncoder)
-            else: outputs_json = json.dumps(str(jinx_output))
-        except TypeError: outputs_json = json.dumps(f"Non-serializable output: {type(jinx_output)}")
+                outputs_json = json.dumps(jinx_output, cls=CustomJSONEncoder)
+            else:
+                outputs_json = json.dumps(str(jinx_output))
+        except TypeError:
+            outputs_json = json.dumps(f"Non-serializable output: {type(jinx_output)}")
 
-        sql = """INSERT INTO jinx_execution_log
+        stmt = """
+            INSERT INTO jinx_execution_log
             (triggering_message_id, conversation_id, timestamp, npc_name, team_name,
              jinx_name, jinx_inputs, jinx_output, status, error_message, response_message_id, duration_ms)
-             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
-        params = (triggering_message_id, conversation_id, timestamp, npc_name, team_name,
-                  jinx_name, inputs_json, outputs_json, status, error_message, response_message_id, duration_ms)
-        try:
-             return self._execute(sql, params) # Return lastrowid if available
-        except Exception as e:
-             print(f"CRITICAL: Failed to save tool execution via _execute: {e}")
-             return None
+             VALUES (:triggering_message_id, :conversation_id, :timestamp, :npc_name, :team_name,
+                     :jinx_name, :jinx_inputs, :jinx_output, :status, :error_message, :response_message_id, :duration_ms)
+        """
+        params = {
+            "triggering_message_id": triggering_message_id,
+            "conversation_id": conversation_id,
+            "timestamp": timestamp,
+            "npc_name": npc_name,
+            "team_name": team_name,
+            "jinx_name": jinx_name,
+            "jinx_inputs": inputs_json,
+            "jinx_output": outputs_json,
+            "status": status,
+            "error_message": error_message,
+            "response_message_id": response_message_id,
+            "duration_ms": duration_ms
+        }
+        
+        return self._execute_returning_id(stmt, params)
 
     def get_full_message_content(self, message_id):
-        sql = "SELECT content FROM conversation_history WHERE message_id = ? ORDER BY timestamp ASC"
-        rows = self._fetch_all(sql, (message_id,))
+        stmt = "SELECT content FROM conversation_history WHERE message_id = :message_id ORDER BY timestamp ASC"
+        rows = self._fetch_all(stmt, {"message_id": message_id})
         return "".join(row['content'] for row in rows)
 
     def update_message_content(self, message_id, full_content):
-        sql = "UPDATE conversation_history SET content = ? WHERE message_id = ?"
-        params = (full_content, message_id)
-        self._execute(sql, params)
+        stmt = "UPDATE conversation_history SET content = :content WHERE message_id = :message_id"
+        with self.engine.begin() as conn:
+            conn.execute(text(stmt), {"content": full_content, "message_id": message_id})
 
     def get_message_attachments(self, message_id) -> List[Dict]:
-        sql = """SELECT id, message_id, attachment_name, attachment_type, attachment_size, upload_timestamp
-                 FROM message_attachments WHERE message_id = ?"""
-        return self._fetch_all(sql, (message_id,))
+        stmt = """
+            SELECT 
+                id, 
+                message_id, 
+                attachment_name, 
+                attachment_type, 
+                attachment_size, 
+                upload_timestamp
+            FROM message_attachments WHERE message_id = :message_id
+        """
+        return self._fetch_all(stmt, {"message_id": message_id})
 
     def get_attachment_data(self, attachment_id) -> Optional[Tuple[bytes, str, str]]:
-        sql = "SELECT attachment_data, attachment_name, attachment_type FROM message_attachments WHERE id = ?"
-        row = self._fetch_one(sql, (attachment_id,))
+        stmt = "SELECT attachment_data, attachment_name, attachment_type FROM message_attachments WHERE id = :attachment_id"
+        row = self._fetch_one(stmt, {"attachment_id": attachment_id})
         if row:
             return row['attachment_data'], row['attachment_name'], row['attachment_type']
         return None, None, None
 
     def delete_attachment(self, attachment_id) -> bool:
-        sql = "DELETE FROM message_attachments WHERE id = ?"
+        stmt = "DELETE FROM message_attachments WHERE id = :attachment_id"
         try:
-            # _execute might not return rowcount reliably across drivers
-            self._execute(sql, (attachment_id,))
-            # We assume success if no exception was raised.
-            # A more robust check might involve trying to fetch the deleted row.
+            with self.engine.begin() as conn:
+                conn.execute(text(stmt), {"attachment_id": attachment_id})
             return True
         except Exception as e:
             print(f"Error deleting attachment {attachment_id}: {e}")
             return False
 
     def get_last_command(self) -> Optional[Dict]:
-        sql = "SELECT * FROM command_history ORDER BY id DESC LIMIT 1"
-        return self._fetch_one(sql)
+        stmt = "SELECT * FROM command_history ORDER BY id DESC LIMIT 1"
+        return self._fetch_one(stmt)
 
     def get_most_recent_conversation_id(self) -> Optional[Dict]:
-        sql = "SELECT conversation_id FROM conversation_history ORDER BY id DESC LIMIT 1"
-        # Returns dict like {'conversation_id': '...'} or None
-        return self._fetch_one(sql)
+        stmt = "SELECT conversation_id FROM conversation_history ORDER BY id DESC LIMIT 1"
+        return self._fetch_one(stmt)
 
     def get_last_conversation(self, conversation_id) -> Optional[Dict]:
-        sql = """SELECT * FROM conversation_history WHERE conversation_id = ? and role = 'user'
-                 ORDER BY id DESC LIMIT 1"""
-        return self._fetch_one(sql, (conversation_id,))
+        stmt = """
+            SELECT * FROM conversation_history 
+            WHERE conversation_id = :conversation_id and role = 'user'
+            ORDER BY id DESC LIMIT 1
+        """
+        return self._fetch_one(stmt, {"conversation_id": conversation_id})
 
     def get_messages_by_npc(self, npc, n_last=20) -> List[Dict]:
-        sql = """SELECT * FROM conversation_history WHERE npc = ?
-                    ORDER BY timestamp DESC LIMIT ?"""
-        params = (npc, n_last)
+        stmt = """
+            SELECT * FROM conversation_history WHERE npc = :npc
+            ORDER BY timestamp DESC LIMIT :n_last
+        """
+        return self._fetch_all(stmt, {"npc": npc, "n_last": n_last})
 
-        return self._fetch_all(sql, params)
     def get_messages_by_team(self, team, n_last=20) -> List[Dict]:
-        sql = """SELECT * FROM conversation_history WHERE team = ?
-                    ORDER BY timestamp DESC LIMIT ?"""
-        params = (team, n_last)
+        stmt = """
+            SELECT * FROM conversation_history WHERE team = :team
+            ORDER BY timestamp DESC LIMIT :n_last
+        """
+        return self._fetch_all(stmt, {"team": team, "n_last": n_last})
 
-        return self._fetch_all(sql, params)
     def get_message_by_id(self, message_id) -> Optional[Dict]:
-        sql = "SELECT * FROM conversation_history WHERE message_id = ?"
-        return self._fetch_one(sql, (message_id,))
+        stmt = "SELECT * FROM conversation_history WHERE message_id = :message_id"
+        return self._fetch_one(stmt, {"message_id": message_id})
 
     def get_most_recent_conversation_id_by_path(self, path) -> Optional[Dict]:
-        sql = """SELECT conversation_id FROM conversation_history WHERE directory_path = ?
-                 ORDER BY timestamp DESC LIMIT 1"""
-        # Returns dict like {'conversation_id': '...'} or None
-        return self._fetch_one(sql, (path,))
+        stmt = """
+            SELECT conversation_id FROM conversation_history WHERE directory_path = :path
+            ORDER BY timestamp DESC LIMIT 1
+        """
+        return self._fetch_one(stmt, {"path": path})
 
     def get_last_conversation_by_path(self, directory_path) -> Optional[List[Dict]]:
         result_dict = self.get_most_recent_conversation_id_by_path(directory_path)
@@ -528,18 +892,24 @@ class CommandHistory:
         return None
 
     def get_conversations_by_id(self, conversation_id: str) -> List[Dict[str, Any]]:
-        sql = """SELECT id, message_id, timestamp, role, content, conversation_id,
-                 directory_path, model, provider, npc, team
-                 FROM conversation_history WHERE conversation_id = ? ORDER BY timestamp ASC"""
-        results = self._fetch_all(sql, (conversation_id,))
+        stmt = """
+            SELECT id, message_id, timestamp, role, content, conversation_id,
+                    directory_path, model, provider, npc, team
+            FROM conversation_history WHERE conversation_id = :conversation_id 
+            ORDER BY timestamp ASC
+        """
+        results = self._fetch_all(stmt, {"conversation_id": conversation_id})
+        
         for message_dict in results:
-             attachments = self.get_message_attachments(message_dict["message_id"])
-             if attachments: message_dict["attachments"] = attachments
+            attachments = self.get_message_attachments(message_dict["message_id"])
+            if attachments:
+                message_dict["attachments"] = attachments
         return results
 
     def get_npc_conversation_stats(self, start_date=None, end_date=None) -> pd.DataFrame:
         date_filter = ""
-        params = {} # Use dict for named parameters with SQLAlchemy/read_sql
+        params = {}
+        
         if start_date and end_date:
             date_filter = "WHERE timestamp BETWEEN :start_date AND :end_date"
             params = {"start_date": start_date, "end_date": end_date}
@@ -550,6 +920,14 @@ class CommandHistory:
             date_filter = "WHERE timestamp <= :end_date"
             params = {"end_date": end_date}
 
+        
+        if 'sqlite' in str(self.engine.url):
+            group_concat_models = "GROUP_CONCAT(DISTINCT model)"
+            group_concat_providers = "GROUP_CONCAT(DISTINCT provider)"
+        else:
+            
+            group_concat_models = "STRING_AGG(DISTINCT model, ',')"
+            group_concat_providers = "STRING_AGG(DISTINCT provider, ',')"
 
         query = f"""
         SELECT
@@ -559,8 +937,8 @@ class CommandHistory:
             COUNT(DISTINCT conversation_id) as total_conversations,
             COUNT(DISTINCT model) as models_used,
             COUNT(DISTINCT provider) as providers_used,
-            GROUP_CONCAT(DISTINCT model) as model_list,
-            GROUP_CONCAT(DISTINCT provider) as provider_list,
+            {group_concat_models} as model_list,
+            {group_concat_providers} as provider_list,
             MIN(timestamp) as first_conversation,
             MAX(timestamp) as last_conversation
         FROM conversation_history
@@ -568,122 +946,127 @@ class CommandHistory:
         GROUP BY npc
         ORDER BY total_messages DESC
         """
+        
         try:
-            # Use pd.read_sql with the appropriate connection object
-            if self._is_sqlalchemy:
-                # read_sql works directly with SQLAlchemy Engine
-                df = pd.read_sql(sql=text(query), con=self.conn, params=params)
-            else:
-                # read_sql works directly with sqlite3 Connection
-                df = pd.read_sql(sql=query, con=self.conn, params=params)
+            df = pd.read_sql(sql=text(query), con=self.engine, params=params)
             return df
         except Exception as e:
-             print(f"Error fetching conversation stats with pandas: {e}")
-             # Fallback or return empty DataFrame
-             return pd.DataFrame(columns=[
-                 'npc', 'total_messages', 'avg_message_length', 'total_conversations',
-                 'models_used', 'providers_used', 'model_list', 'provider_list',
-                 'first_conversation', 'last_conversation'
-             ])
-
+            print(f"Error fetching conversation stats with pandas: {e}")
+            return pd.DataFrame(columns=[
+                'npc', 'total_messages', 'avg_message_length', 'total_conversations',
+                'models_used', 'providers_used', 'model_list', 'provider_list',
+                'first_conversation', 'last_conversation'
+            ])
 
     def get_command_patterns(self, timeframe='day') -> pd.DataFrame:
-        time_group_formats = {
-            'hour': "strftime('%Y-%m-%d %H', timestamp)",
-            'day': "strftime('%Y-%m-%d', timestamp)",
-            'week': "strftime('%Y-%W', timestamp)",
-            'month': "strftime('%Y-%m', timestamp)"
-        }
-        time_group = time_group_formats.get(timeframe, "strftime('%Y-%m-%d', timestamp)") # Default to day
+        
+        if 'sqlite' in str(self.engine.url):
+            time_group_formats = {
+                'hour': "strftime('%Y-%m-%d %H', timestamp)",
+                'day': "strftime('%Y-%m-%d', timestamp)",
+                'week': "strftime('%Y-%W', timestamp)",
+                'month': "strftime('%Y-%m', timestamp)"
+            }
+        else:
+            
+            time_group_formats = {
+                'hour': "TO_CHAR(timestamp::timestamp, 'YYYY-MM-DD HH24')",
+                'day': "TO_CHAR(timestamp::timestamp, 'YYYY-MM-DD')",
+                'week': "TO_CHAR(timestamp::timestamp, 'YYYY-WW')",
+                'month': "TO_CHAR(timestamp::timestamp, 'YYYY-MM')"
+            }
+        
+        time_group = time_group_formats.get(timeframe, time_group_formats['day'])
+
+        
+        if 'sqlite' in str(self.engine.url):
+            substr_func = "SUBSTR"
+            instr_func = "INSTR"
+        else:
+            substr_func = "SUBSTRING"
+            instr_func = "POSITION"
 
         query = f"""
         WITH parsed_commands AS (
             SELECT
                 {time_group} as time_bucket,
                 CASE
-                    WHEN command LIKE '/%%' THEN SUBSTR(command, 2, INSTR(SUBSTR(command, 2), ' ') - 1)
-                    WHEN command LIKE 'npc %%' THEN SUBSTR(command, 5, INSTR(SUBSTR(command, 5), ' ') - 1)
+                    WHEN command LIKE '/%%' THEN {substr_func}(command, 2, {instr_func}({substr_func}(command, 2), ' ') - 1)
+                    WHEN command LIKE 'npc %%' THEN {substr_func}(command, 5, {instr_func}({substr_func}(command, 5), ' ') - 1)
                     ELSE command
                 END as base_command
             FROM command_history
-            WHERE timestamp IS NOT NULL -- Added check for null timestamps
+            WHERE timestamp IS NOT NULL
         )
         SELECT
             time_bucket,
             base_command,
             COUNT(*) as usage_count
         FROM parsed_commands
-        WHERE base_command IS NOT NULL AND base_command != '' -- Filter out potential null/empty commands
+        WHERE base_command IS NOT NULL AND base_command != ''
         GROUP BY time_bucket, base_command
         ORDER BY time_bucket DESC, usage_count DESC
         """
+        
         try:
-             # Use pd.read_sql
-             if self._is_sqlalchemy:
-                  df = pd.read_sql(sql=text(query), con=self.conn)
-             else:
-                  df = pd.read_sql(sql=query, con=self.conn)
-             return df
+            df = pd.read_sql(sql=text(query), con=self.engine)
+            return df
         except Exception as e:
-             print(f"Error fetching command patterns with pandas: {e}")
-             return pd.DataFrame(columns=['time_period', 'command', 'count'])
+            print(f"Error fetching command patterns with pandas: {e}")
+            return pd.DataFrame(columns=['time_period', 'command', 'count'])
 
     def search_commands(self, search_term: str) -> List[Dict]:
         """Searches command history table for a term."""
-        # Use LOWER() for case-insensitive search
-        sql = """
+        stmt = """
             SELECT id, timestamp, command, subcommands, output, location
             FROM command_history
-            WHERE LOWER(command) LIKE LOWER(?) OR LOWER(output) LIKE LOWER(?)
+            WHERE LOWER(command) LIKE LOWER(:search_term) OR LOWER(output) LIKE LOWER(:search_term)
             ORDER BY timestamp DESC
             LIMIT 5
         """
         like_term = f"%{search_term}%"
-        return self._fetch_all(sql, (like_term, like_term))
-    def search_conversations(self, search_term:str) -> List[Dict]:
+        return self._fetch_all(stmt, {"search_term": like_term})
+
+    def search_conversations(self, search_term: str) -> List[Dict]:
         """Searches conversation history table for a term."""
-        # Use LOWER() for case-insensitive search
-        sql = """
+        stmt = """
             SELECT id, message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team
             FROM conversation_history
-            WHERE LOWER(content) LIKE LOWER(?)
+            WHERE LOWER(content) LIKE LOWER(:search_term)
             ORDER BY timestamp DESC
             LIMIT 5
         """
         like_term = f"%{search_term}%"
-        return self._fetch_all(sql, (like_term,))
+        return self._fetch_all(stmt, {"search_term": like_term})
     
     def get_all_commands(self, limit: int = 100) -> List[Dict]:
         """Gets the most recent commands."""
-        sql = """
+        stmt = """
             SELECT id, timestamp, command, subcommands, output, location
             FROM command_history
             ORDER BY id DESC
-            LIMIT ?
+            LIMIT :limit
         """
-        return self._fetch_all(sql, (limit,))
-
+        return self._fetch_all(stmt, {"limit": limit})
 
     def close(self):
-        """Closes the connection if it's a direct sqlite3 connection."""
-        if not self._is_sqlalchemy and self.conn:
+        """Dispose of the SQLAlchemy engine."""
+        if self.engine:
             try:
-                self.conn.close()
-                print("Closed sqlite3 connection.")
+                self.engine.dispose()
+                logging.info("Disposed SQLAlchemy engine.")
             except Exception as e:
-                print(f"Error closing sqlite3 connection: {e}")
-        elif self._is_sqlalchemy:
-             print("SQLAlchemy Engine pool managed automatically.")
-        self.conn = None
-def start_new_conversation(prepend = '') -> str:
+                print(f"Error disposing SQLAlchemy engine: {e}")
+        self.engine = None
+
+def start_new_conversation(prepend: str = None) -> str:
     """
     Starts a new conversation and returns a unique conversation ID.
     """
-    if prepend =='':
+    if prepend is None:
         prepend = 'npcsh'
     return f"{prepend}_{datetime.now().strftime('%Y%m%d%H%M%S')}"
 
-
 def save_conversation_message(
     command_history: CommandHistory,
     conversation_id: str,
@@ -696,94 +1079,62 @@ def save_conversation_message(
     team: str = None,
     attachments: List[Dict] = None,
     message_id: str = None,
-):
+    ):
     """
     Saves a conversation message linked to a conversation ID with optional attachments.
-
-    Args:
-        command_history: The CommandHistory instance
-        conversation_id: The conversation identifier
-        role: The message sender role ('user', 'assistant', etc.)
-        content: The message content
-        wd: Working directory (defaults to current directory)
-        model: The model identifier (optional)
-        provider: The provider identifier (optional)
-        npc: The NPC identifier (optional)
-        attachments: List of attachment dictionaries (optional)
-            Each attachment dict should have:
-            - name: Filename/title
-            - type: MIME type or extension
-            - data: Binary blob data
-            - size: Size in bytes (optional)
-
-    Returns:
-        The message ID
     """
     if wd is None:
         wd = os.getcwd()
-
-    return command_history.add_conversation(
-        role=role,
-        content=content,
-        conversation_id=conversation_id,
-        directory_path=wd,
-        model=model,
-        provider=provider,
-        npc=npc,
-        team=team,
-        attachments=attachments,
-        message_id=message_id,
-    )
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    if message_id is None:
+        message_id = generate_message_id()
 
 
+    return command_history.add_conversation(
+        message_id, 
+        timestamp, 
+        role, 
+        content, 
+        conversation_id, 
+        wd, 
+        model=model, 
+        provider=provider, 
+        npc=npc, 
+        team=team, 
+        attachments=attachments)
 def retrieve_last_conversation(
     command_history: CommandHistory, conversation_id: str
-) -> str:
+    ) -> str:
     """
     Retrieves and formats all messages from the last conversation.
     """
     last_message = command_history.get_last_conversation(conversation_id)
     if last_message:
-        return last_message[3]  # content
+        return last_message['content']  
     return "No previous conversation messages found."
 
-
 def save_attachment_to_message(
     command_history: CommandHistory,
     message_id: str,
     file_path: str,
     attachment_name: str = None,
     attachment_type: str = None,
-):
+    ):
     """
     Helper function to save a file from disk as an attachment.
-
-    Args:
-        command_history: The CommandHistory instance
-        message_id: The message ID to attach to
-        file_path: Path to the file on disk
-        attachment_name: Name to save (defaults to basename)
-        attachment_type: MIME type (defaults to guessing from extension)
-
-    Returns:
-        Boolean indicating success
     """
     try:
-        # Get file name if not specified
         if not attachment_name:
             attachment_name = os.path.basename(file_path)
 
-        # Try to guess MIME type if not specified
         if not attachment_type:
             _, ext = os.path.splitext(file_path)
             if ext:
-                attachment_type = ext.lower()[1:]  # Remove the dot
+                attachment_type = ext.lower()[1:]
 
-        # Read file data
         with open(file_path, "rb") as f:
             data = f.read()
 
-        # Add attachment
         command_history.add_attachment(
             message_id=message_id,
             attachment_name=attachment_name,
@@ -796,46 +1147,29 @@ def save_attachment_to_message(
         print(f"Error saving attachment: {str(e)}")
         return False
 
-def get_available_tables(db_path: str) -> str:
+def get_available_tables(db_path_or_engine: Union[str, Engine]) -> List[Tuple[str]]:
     """
-    Function Description:
-        This function gets the available tables in the database.
-    Args:
-        db_path (str): The database path.
-    Keyword Args:
-        None
-    Returns:
-        str: The available tables in the database.
+    Gets the available tables in the database.
     """
-    if '~' in db_path:
-        db_path = os.path.expanduser(db_path)
+    if isinstance(db_path_or_engine, str):
+        engine = create_engine_from_path(db_path_or_engine)
+    else:
+        engine = db_path_or_engine
+    
     try:
-        with sqlite3.connect(db_path) as conn:
-            cursor = conn.cursor()
-            cursor.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name != 'command_history'"
-            )
-            tables = cursor.fetchall()
-
-            return tables
+        with engine.connect() as conn:
+            if 'sqlite' in str(engine.url):
+                result = conn.execute(text(
+                    "SELECT name FROM sqlite_master WHERE type='table' AND name != 'command_history'"
+                ))
+            else:
+                
+                result = conn.execute(text("""
+                    SELECT table_name FROM information_schema.tables 
+                    WHERE table_schema = 'public' AND table_name != 'command_history'
+                """))
+            
+            return [row[0] for row in result]
     except Exception as e:
         print(f"Error getting available tables: {e}")
-        return ""
-    
-    
-
-'''
-from npcpy.memory.command_history import CommandHistory
-command_history = CommandHistory()
-
-sibiji_messages = command_history.get_messages_by_npc('sibiji', n_last=10)
-
-stats = command_history.get_npc_conversation_stats()
-
-
-
-
-from npcpy.memory.command_history import CommandHistory
-command_history = CommandHistory()
-command_history.create_tool_call_table()
-'''
+        return []