local-deep-research 0.4.4__py3-none-any.whl → 0.5.2__py3-none-any.whl

local_deep_research/web/database/migrate_to_ldr_db.py

@@ -90,7 +90,9 @@ def migrate_to_ldr_db():
     )
 
     # Migrate from deep_research.db
-    migrated_deep_research = migrate_deep_research_db(ldr_conn, LEGACY_DEEP_RESEARCH_DB)
+    migrated_deep_research = migrate_deep_research_db(
+        ldr_conn, LEGACY_DEEP_RESEARCH_DB
+    )
 
     # Re-enable foreign keys and commit
     ldr_cursor.execute("PRAGMA foreign_keys = ON")
@@ -127,7 +129,9 @@ def migrate_research_history_db(ldr_conn, legacy_path):
         logger.info(f"Connected to legacy database: {legacy_path}")
 
         # Get tables from legacy database
-        legacy_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        legacy_cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'"
+        )
         tables = [row[0] for row in legacy_cursor.fetchall()]
 
         for table in tables:
@@ -181,7 +185,9 @@ def migrate_research_history_db(ldr_conn, legacy_path):
                 # Verify data was inserted
                 ldr_cursor.execute(f"SELECT COUNT(*) FROM {table}")
                 count = ldr_cursor.fetchone()[0]
-                logger.info(f"Migrated {count} rows to {table} (expected {len(rows)})")
+                logger.info(
+                    f"Migrated {count} rows to {table} (expected {len(rows)})"
+                )
             else:
                 logger.info(f"No data to migrate from {table}")
 
@@ -218,7 +224,9 @@ def migrate_deep_research_db(ldr_conn, legacy_path):
         logger.info(f"Connected to legacy database: {legacy_path}")
 
         # Get tables from legacy database
-        legacy_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        legacy_cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'"
+        )
         tables = [row[0] for row in legacy_cursor.fetchall()]
 
         # Migrate each table

local_deep_research/web/database/migrations.py

@@ -2,7 +2,14 @@ from loguru import logger
 from sqlalchemy import inspect
 
 from ..services.settings_manager import SettingsManager
-from .models import Base, Journal, Setting
+from .models import (
+    Base,
+    Journal,
+    Setting,
+    ResearchLog,
+    Research,
+    ResearchHistory,
+)
 
 
 def import_default_settings_file(db_session):
@@ -24,14 +31,12 @@ def import_default_settings_file(db_session):
         settings_mgr.load_from_defaults_file(overwrite=False, delete_extra=True)
         logger.info("Successfully imported settings from files")
 
-
         # Update the saved version.
         settings_mgr.update_db_version()
     except Exception as e:
         logger.error("Error importing settings from files: %s", e)
 
 
-
 def run_migrations(engine, db_session=None):
     """
     Run any necessary database migrations
@@ -50,6 +55,18 @@ def run_migrations(engine, db_session=None):
         logger.info("Creating journals table.")
         Base.metadata.create_all(engine, tables=[Journal.__table__])
 
+    if not inspector.has_table(ResearchLog.__tablename__):
+        logger.info("Creating research logs table.")
+        Base.metadata.create_all(engine, tables=[ResearchLog.__table__])
+
+    if not inspector.has_table(Research.__tablename__):
+        logger.info("Creating research table.")
+        Base.metadata.create_all(engine, tables=[Research.__table__])
+
+    if not inspector.has_table(ResearchHistory.__tablename__):
+        logger.info("Creating research table.")
+        Base.metadata.create_all(engine, tables=[ResearchHistory.__table__])
+
     # Import existing settings from files
     if db_session:
         import_default_settings_file(db_session)

local_deep_research/web/database/models.py

@@ -34,6 +34,37 @@ class ResearchStatus(enum.Enum):
     CANCELLED = "cancelled"
 
 
+class ResearchHistory(Base):
+    """Represents the research table."""
+
+    __tablename__ = "research_history"
+
+    # Unique identifier for each record.
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    # The search query.
+    query = Column(Text, nullable=False)
+    # The mode of research (e.g., 'quick_summary', 'detailed_report').
+    mode = Column(Text, nullable=False)
+    # Current status of the research.
+    status = Column(Text, nullable=False)
+    # The timestamp when the research started.
+    created_at = Column(Text, nullable=False)
+    # The timestamp when the research was completed.
+    completed_at = Column(Text)
+    # Duration of the research in seconds.
+    duration_seconds = Column(Integer)
+    # Path to the generated report.
+    report_path = Column(Text)
+    # Additional metadata about the research.
+    research_meta = Column(JSON)
+    # Latest progress log message.
+    progress_log = Column(JSON)
+    # Current progress of the research (as a percentage).
+    progress = Column(Integer)
+    # Title of the research report.
+    title = Column(Text)
+
+
 class Research(Base):
     __tablename__ = "research"
 
@@ -42,7 +73,9 @@ class Research(Base):
     status = Column(
         Enum(ResearchStatus), default=ResearchStatus.PENDING, nullable=False
     )
-    mode = Column(Enum(ResearchMode), default=ResearchMode.QUICK, nullable=False)
+    mode = Column(
+        Enum(ResearchMode), default=ResearchMode.QUICK, nullable=False
+    )
     created_at = Column(DateTime, server_default=func.now(), nullable=False)
     updated_at = Column(
         DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
@@ -52,36 +85,30 @@ class Research(Base):
     end_time = Column(DateTime, nullable=True)
     error_message = Column(Text, nullable=True)
 
-    # Relationships
-    report = relationship(
-        "ResearchReport",
-        back_populates="research",
-        uselist=False,
-        cascade="all, delete-orphan",
-    )
 
+class ResearchLog(Base):
+    __tablename__ = "app_logs"
 
-class ResearchReport(Base):
-    __tablename__ = "research_report"
+    id = Column(
+        Integer, Sequence("reseach_log_id_seq"), primary_key=True, index=True
+    )
 
-    id = Column(Integer, primary_key=True, index=True)
+    timestamp = Column(DateTime, server_default=func.now(), nullable=False)
+    message = Column(Text, nullable=False)
+    # Module that the log message came from.
+    module = Column(Text, nullable=False)
+    # Function that the log message came from.
+    function = Column(Text, nullable=False)
+    # Line number that the log message came from.
+    line_no = Column(Integer, nullable=False)
+    # Log level.
+    level = Column(String(32), nullable=False)
     research_id = Column(
         Integer,
         ForeignKey("research.id", ondelete="CASCADE"),
-        nullable=False,
-        unique=True,
+        nullable=True,
+        index=True,
     )
-    content = Column(Text, nullable=True)
-    created_at = Column(DateTime, server_default=func.now(), nullable=False)
-    updated_at = Column(
-        DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
-    )
-    report_metadata = Column(
-        JSON, nullable=True
-    )  # Additional metadata about the report
-
-    # Relationships
-    research = relationship("Research", back_populates="report")
 
 
 class SettingType(enum.Enum):
@@ -118,6 +145,26 @@ class Setting(Base):
     __table_args__ = (UniqueConstraint("key", name="uix_settings_key"),)
 
 
+class ResearchStrategy(Base):
+    """Database model for tracking research strategies used"""
+
+    __tablename__ = "research_strategies"
+
+    id = Column(Integer, primary_key=True, index=True)
+    research_id = Column(
+        Integer,
+        ForeignKey("research.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+    strategy_name = Column(String(100), nullable=False, index=True)
+    created_at = Column(DateTime, server_default=func.now(), nullable=False)
+
+    # Relationship
+    research = relationship("Research", backref="strategy")
+
+
 class Journal(Base):
     """
     Database model for storing information about academic journals.
@@ -125,7 +172,9 @@ class Journal(Base):
 
     __tablename__ = "journals"
 
-    id = Column(Integer, Sequence("journal_id_seq"), primary_key=True, index=True)
+    id = Column(
+        Integer, Sequence("journal_id_seq"), primary_key=True, index=True
+    )
 
     # Name of the journal
     name = Column(String(255), nullable=False, unique=True, index=True)

local_deep_research/web/database/schema_upgrade.py

@@ -4,10 +4,10 @@ Handles schema upgrades for existing ldr.db databases.
 """
 
 import os
-import sqlite3
 import sys
 
 from loguru import logger
+from sqlalchemy import create_engine, inspect
 
 # Add the parent directory to sys.path to allow relative imports
 sys.path.append(
@@ -19,53 +19,71 @@ try:
 except ImportError:
     # Fallback path if import fails
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    project_root = os.path.abspath(os.path.join(current_dir, "..", "..", "..", ".."))
+    project_root = os.path.abspath(
+        os.path.join(current_dir, "..", "..", "..", "..")
+    )
     DB_PATH = os.path.join(project_root, "src", "data", "ldr.db")
 
+from .models import Base, ResearchStrategy
+
 
-def check_table_exists(conn, table_name):
+def remove_research_log_table(engine):
     """
-    Check if a table exists in the database
+    Remove the redundant research_log table if it exists
 
     Args:
-        conn: SQLite connection
-        table_name: Name of the table
+        engine: SQLAlchemy engine
 
     Returns:
-        bool: True if table exists, False otherwise
+        bool: True if operation was successful, False otherwise
     """
-    cursor = conn.cursor()
-    cursor.execute(
-        "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,)
-    )
-    return cursor.fetchone() is not None
+    try:
+        inspector = inspect(engine)
+
+        # Check if table exists
+        if inspector.has_table("research_log"):
+            # For SQLite, we need to use raw SQL for DROP TABLE
+            with engine.connect() as conn:
+                conn.execute("DROP TABLE research_log")
+                conn.commit()
+            logger.info("Successfully removed redundant 'research_log' table")
+            return True
+        else:
+            logger.info("Table 'research_log' does not exist, no action needed")
+            return True
+    except Exception:
+        logger.exception("Error removing research_log table")
+        return False
 
 
-def remove_research_log_table(conn):
+def create_research_strategy_table(engine):
     """
-    Remove the redundant research_log table if it exists
+    Create the research_strategies table if it doesn't exist
 
     Args:
-        conn: SQLite connection
+        engine: SQLAlchemy engine
 
     Returns:
         bool: True if operation was successful, False otherwise
     """
     try:
-        cursor = conn.cursor()
+        inspector = inspect(engine)
 
         # Check if table exists
-        if check_table_exists(conn, "research_log"):
-            # For SQLite, DROP TABLE is the way to remove a table
-            cursor.execute("DROP TABLE research_log")
-            conn.commit()
-            logger.info("Successfully removed redundant 'research_log' table")
+        if not inspector.has_table("research_strategies"):
+            # Create the table using ORM
+            Base.metadata.create_all(
+                engine, tables=[ResearchStrategy.__table__]
+            )
+            logger.info("Successfully created 'research_strategies' table")
             return True
         else:
-            logger.info("Table 'research_log' does not exist, no action needed")
+            logger.info(
+                "Table 'research_strategies' already exists, no action needed"
+            )
             return True
     except Exception:
-        logger.exception("Error removing research_log table")
+        logger.exception("Error creating research_strategies table")
         return False
 
 
@@ -78,20 +96,22 @@ def run_schema_upgrades():
     """
     # Check if database exists
     if not os.path.exists(DB_PATH):
-        logger.warning(f"Database not found at {DB_PATH}, skipping schema upgrades")
+        logger.warning(
+            f"Database not found at {DB_PATH}, skipping schema upgrades"
+        )
         return False
 
     logger.info(f"Running schema upgrades on {DB_PATH}")
 
     try:
-        # Connect to the database
-        conn = sqlite3.connect(DB_PATH)
+        # Create SQLAlchemy engine
+        engine = create_engine(f"sqlite:///{DB_PATH}")
 
         # 1. Remove the redundant research_log table
-        remove_research_log_table(conn)
+        remove_research_log_table(engine)
 
-        # Close connection
-        conn.close()
+        # 2. Create research_strategies table
+        create_research_strategy_table(engine)
 
         logger.info("Schema upgrades completed successfully")
         return True

local_deep_research/web/models/database.py

@@ -1,10 +1,12 @@
-import json
 import os
 import sqlite3
 from datetime import datetime
 
 from loguru import logger
 
+from ...utilities.db_utils import get_db_session
+from ..database.models import ResearchLog
+
 # Database path
 # Use unified database in data directory
 DATA_DIR = os.path.abspath(
@@ -15,7 +17,9 @@ DB_PATH = os.path.join(DATA_DIR, "ldr.db")
 
 # Legacy database paths (for migration)
 LEGACY_RESEARCH_HISTORY_DB = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "..", "..", "..", "research_history.db")
+    os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "research_history.db"
+    )
 )
 LEGACY_DEEP_RESEARCH_DB = os.path.join(
     os.path.abspath(
@@ -39,26 +43,6 @@ def init_db():
     conn = get_db_connection()
     cursor = conn.cursor()
 
-    # Create the table if it doesn't exist
-    cursor.execute(
-        """
-    CREATE TABLE IF NOT EXISTS research_history (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        query TEXT NOT NULL,
-        mode TEXT NOT NULL,
-        status TEXT NOT NULL,
-        created_at TEXT NOT NULL,
-        completed_at TEXT,
-        duration_seconds INTEGER,
-        report_path TEXT,
-        metadata TEXT,
-        progress_log TEXT,
-        progress INTEGER,
-        title TEXT
-    )
-    """
-    )
-
     # Create a dedicated table for research logs
     cursor.execute(
         """
@@ -97,21 +81,35 @@ def init_db():
     columns = [column[1] for column in cursor.fetchall()]
 
     if "duration_seconds" not in columns:
-        print("Adding missing 'duration_seconds' column to research_history table")
+        logger.info(
+            "Adding missing 'duration_seconds' column to research_history table"
+        )
         cursor.execute(
             "ALTER TABLE research_history ADD COLUMN duration_seconds INTEGER"
         )
 
     # Check if the progress column exists, add it if missing
     if "progress" not in columns:
-        print("Adding missing 'progress' column to research_history table")
-        cursor.execute("ALTER TABLE research_history ADD COLUMN progress INTEGER")
+        logger.info(
+            "Adding missing 'progress' column to research_history table"
+        )
+        cursor.execute(
+            "ALTER TABLE research_history ADD COLUMN progress INTEGER"
+        )
 
     # Check if the title column exists, add it if missing
     if "title" not in columns:
-        print("Adding missing 'title' column to research_history table")
+        logger.info("Adding missing 'title' column to research_history table")
         cursor.execute("ALTER TABLE research_history ADD COLUMN title TEXT")
 
+    # Check if the metadata column exists, and rename it to "research_meta"
+    # if it does.
+    if "metadata" in columns:
+        logger.info("Renaming 'metadata' column to 'research_meta'")
+        cursor.execute(
+            "ALTER TABLE research_history RENAME COLUMN metadata TO research_meta"
+        )
+
     # Enable foreign key support
     cursor.execute("PRAGMA foreign_keys = ON")
 
@@ -180,10 +178,14 @@ def calculate_duration(created_at_str, completed_at_str=None):
         else:  # Older format without T
             # Try different formats
             try:
-                start_time = datetime.strptime(created_at_str, "%Y-%m-%d %H:%M:%S.%f")
+                start_time = datetime.strptime(
+                    created_at_str, "%Y-%m-%d %H:%M:%S.%f"
+                )
             except ValueError:
                 try:
-                    start_time = datetime.strptime(created_at_str, "%Y-%m-%d %H:%M:%S")
+                    start_time = datetime.strptime(
+                        created_at_str, "%Y-%m-%d %H:%M:%S"
+                    )
                 except ValueError:
                     # Last resort fallback
                     start_time = datetime.fromisoformat(
@@ -198,7 +200,7 @@ def calculate_duration(created_at_str, completed_at_str=None):
             start_time = parser.parse(created_at_str)
         except Exception:
             logger.exception(
-                f"Fallback parsing also failed for created_at:" f" {created_at_str}"
+                f"Fallback parsing also failed for created_at: {created_at_str}"
             )
             return None
 
@@ -212,36 +214,6 @@ def calculate_duration(created_at_str, completed_at_str=None):
     return None
 
 
-def add_log_to_db(research_id, message, log_type="info", progress=None, metadata=None):
-    """
-    Store a log entry in the database
-
-    Args:
-        research_id: ID of the research
-        message: Log message text
-        log_type: Type of log (info, error, milestone)
-        progress: Progress percentage (0-100)
-        metadata: Additional metadata as dictionary (will be stored as JSON)
-    """
-    try:
-        timestamp = datetime.utcnow().isoformat()
-        metadata_json = json.dumps(metadata) if metadata else None
-
-        conn = get_db_connection()
-        cursor = conn.cursor()
-        cursor.execute(
-            "INSERT INTO research_logs (research_id, timestamp, message, log_type, progress, metadata) "
-            "VALUES (?, ?, ?, ?, ?, ?)",
-            (research_id, timestamp, message, log_type, progress, metadata_json),
-        )
-        conn.commit()
-        conn.close()
-        return True
-    except Exception:
-        logger.exception("Error adding log to database")
-        return False
-
-
 def get_logs_for_research(research_id):
     """
     Retrieve all logs for a specific research ID
@@ -253,35 +225,23 @@ def get_logs_for_research(research_id):
         List of log entries as dictionaries
     """
     try:
-        conn = get_db_connection()
-        conn.row_factory = sqlite3.Row
-        cursor = conn.cursor()
-        cursor.execute(
-            "SELECT * FROM research_logs WHERE research_id = ? ORDER BY timestamp ASC",
-            (research_id,),
+        session = get_db_session()
+        log_results = (
+            session.query(ResearchLog)
+            .filter(ResearchLog.research_id == research_id)
+            .order_by(ResearchLog.timestamp.asc())
+            .all()
         )
-        results = cursor.fetchall()
-        conn.close()
 
         logs = []
-        for result in results:
-            log_entry = dict(result)
-            # Parse metadata JSON if it exists
-            if log_entry.get("metadata"):
-                try:
-                    log_entry["metadata"] = json.loads(log_entry["metadata"])
-                except Exception:
-                    log_entry["metadata"] = {}
-            else:
-                log_entry["metadata"] = {}
-
+        for result in log_results:
             # Convert entry for frontend consumption
             formatted_entry = {
-                "time": log_entry["timestamp"],
-                "message": log_entry["message"],
-                "progress": log_entry["progress"],
-                "metadata": log_entry["metadata"],
-                "type": log_entry["log_type"],
+                "time": result.timestamp,
+                "message": result.message,
+                "type": result.level,
+                "module": result.module,
+                "line_no": result.line_no,
             }
             logs.append(formatted_entry)
 
@@ -289,3 +249,23 @@ def get_logs_for_research(research_id):
     except Exception:
         logger.exception("Error retrieving logs from database")
         return []
+
+
+@logger.catch
+def get_total_logs_for_research(research_id):
+    """
+    Returns the total number of logs for a given `research_id`.
+
+    Args:
+        research_id (int): The ID of the research.
+
+    Returns:
+        int: Total number of logs for the specified research ID.
+    """
+    session = get_db_session()
+    total_logs = (
+        session.query(ResearchLog)
+        .filter(ResearchLog.research_id == research_id)
+        .count()
+    )
+    return total_logs