local-deep-research 0.4.4__py3-none-any.whl → 0.5.0__py3-none-any.whl

local_deep_research/web/database/migrate_to_ldr_db.py

@@ -90,7 +90,9 @@ def migrate_to_ldr_db():
     )
 
     # Migrate from deep_research.db
-    migrated_deep_research = migrate_deep_research_db(ldr_conn, LEGACY_DEEP_RESEARCH_DB)
+    migrated_deep_research = migrate_deep_research_db(
+        ldr_conn, LEGACY_DEEP_RESEARCH_DB
+    )
 
     # Re-enable foreign keys and commit
     ldr_cursor.execute("PRAGMA foreign_keys = ON")
@@ -127,7 +129,9 @@ def migrate_research_history_db(ldr_conn, legacy_path):
         logger.info(f"Connected to legacy database: {legacy_path}")
 
         # Get tables from legacy database
-        legacy_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        legacy_cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'"
+        )
         tables = [row[0] for row in legacy_cursor.fetchall()]
 
         for table in tables:
@@ -181,7 +185,9 @@ def migrate_research_history_db(ldr_conn, legacy_path):
                 # Verify data was inserted
                 ldr_cursor.execute(f"SELECT COUNT(*) FROM {table}")
                 count = ldr_cursor.fetchone()[0]
-                logger.info(f"Migrated {count} rows to {table} (expected {len(rows)})")
+                logger.info(
+                    f"Migrated {count} rows to {table} (expected {len(rows)})"
+                )
             else:
                 logger.info(f"No data to migrate from {table}")
 
@@ -218,7 +224,9 @@ def migrate_deep_research_db(ldr_conn, legacy_path):
         logger.info(f"Connected to legacy database: {legacy_path}")
 
         # Get tables from legacy database
-        legacy_cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        legacy_cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'"
+        )
         tables = [row[0] for row in legacy_cursor.fetchall()]
 
         # Migrate each table

local_deep_research/web/database/migrations.py

@@ -2,7 +2,7 @@ from loguru import logger
 from sqlalchemy import inspect
 
 from ..services.settings_manager import SettingsManager
-from .models import Base, Journal, Setting
+from .models import Base, Journal, Setting, ResearchLog
 
 
 def import_default_settings_file(db_session):
@@ -24,14 +24,12 @@ def import_default_settings_file(db_session):
         settings_mgr.load_from_defaults_file(overwrite=False, delete_extra=True)
         logger.info("Successfully imported settings from files")
 
-
         # Update the saved version.
         settings_mgr.update_db_version()
     except Exception as e:
         logger.error("Error importing settings from files: %s", e)
 
 
-
 def run_migrations(engine, db_session=None):
     """
     Run any necessary database migrations
@@ -50,6 +48,10 @@ def run_migrations(engine, db_session=None):
         logger.info("Creating journals table.")
         Base.metadata.create_all(engine, tables=[Journal.__table__])
 
+    if not inspector.has_table(ResearchLog.__tablename__):
+        logger.info("Creating research logs table.")
+        Base.metadata.create_all(engine, tables=[ResearchLog.__table__])
+
     # Import existing settings from files
     if db_session:
         import_default_settings_file(db_session)

local_deep_research/web/database/models.py

@@ -42,7 +42,9 @@ class Research(Base):
     status = Column(
         Enum(ResearchStatus), default=ResearchStatus.PENDING, nullable=False
     )
-    mode = Column(Enum(ResearchMode), default=ResearchMode.QUICK, nullable=False)
+    mode = Column(
+        Enum(ResearchMode), default=ResearchMode.QUICK, nullable=False
+    )
     created_at = Column(DateTime, server_default=func.now(), nullable=False)
     updated_at = Column(
         DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
@@ -61,6 +63,31 @@ class Research(Base):
     )
 
 
+class ResearchLog(Base):
+    __tablename__ = "app_logs"
+
+    id = Column(
+        Integer, Sequence("reseach_log_id_seq"), primary_key=True, index=True
+    )
+
+    timestamp = Column(DateTime, server_default=func.now(), nullable=False)
+    message = Column(Text, nullable=False)
+    # Module that the log message came from.
+    module = Column(Text, nullable=False)
+    # Function that the log message came from.
+    function = Column(Text, nullable=False)
+    # Line number that the log message came from.
+    line_no = Column(Integer, nullable=False)
+    # Log level.
+    level = Column(String(32), nullable=False)
+    research_id = Column(
+        Integer,
+        ForeignKey("research.id", ondelete="CASCADE"),
+        nullable=True,
+        index=True,
+    )
+
+
 class ResearchReport(Base):
     __tablename__ = "research_report"
 
@@ -118,6 +145,26 @@ class Setting(Base):
     __table_args__ = (UniqueConstraint("key", name="uix_settings_key"),)
 
 
+class ResearchStrategy(Base):
+    """Database model for tracking research strategies used"""
+
+    __tablename__ = "research_strategies"
+
+    id = Column(Integer, primary_key=True, index=True)
+    research_id = Column(
+        Integer,
+        ForeignKey("research.id", ondelete="CASCADE"),
+        nullable=False,
+        unique=True,
+        index=True,
+    )
+    strategy_name = Column(String(100), nullable=False, index=True)
+    created_at = Column(DateTime, server_default=func.now(), nullable=False)
+
+    # Relationship
+    research = relationship("Research", backref="strategy")
+
+
 class Journal(Base):
     """
     Database model for storing information about academic journals.
@@ -125,7 +172,9 @@ class Journal(Base):
 
     __tablename__ = "journals"
 
-    id = Column(Integer, Sequence("journal_id_seq"), primary_key=True, index=True)
+    id = Column(
+        Integer, Sequence("journal_id_seq"), primary_key=True, index=True
+    )
 
     # Name of the journal
     name = Column(String(255), nullable=False, unique=True, index=True)

local_deep_research/web/database/schema_upgrade.py

@@ -4,10 +4,10 @@ Handles schema upgrades for existing ldr.db databases.
 """
 
 import os
-import sqlite3
 import sys
 
 from loguru import logger
+from sqlalchemy import create_engine, inspect
 
 # Add the parent directory to sys.path to allow relative imports
 sys.path.append(
@@ -19,53 +19,71 @@ try:
 except ImportError:
     # Fallback path if import fails
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    project_root = os.path.abspath(os.path.join(current_dir, "..", "..", "..", ".."))
+    project_root = os.path.abspath(
+        os.path.join(current_dir, "..", "..", "..", "..")
+    )
     DB_PATH = os.path.join(project_root, "src", "data", "ldr.db")
 
+from .models import Base, ResearchStrategy
+
 
-def check_table_exists(conn, table_name):
+def remove_research_log_table(engine):
     """
-    Check if a table exists in the database
+    Remove the redundant research_log table if it exists
 
     Args:
-        conn: SQLite connection
-        table_name: Name of the table
+        engine: SQLAlchemy engine
 
     Returns:
-        bool: True if table exists, False otherwise
+        bool: True if operation was successful, False otherwise
     """
-    cursor = conn.cursor()
-    cursor.execute(
-        "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,)
-    )
-    return cursor.fetchone() is not None
+    try:
+        inspector = inspect(engine)
+
+        # Check if table exists
+        if inspector.has_table("research_log"):
+            # For SQLite, we need to use raw SQL for DROP TABLE
+            with engine.connect() as conn:
+                conn.execute("DROP TABLE research_log")
+                conn.commit()
+            logger.info("Successfully removed redundant 'research_log' table")
+            return True
+        else:
+            logger.info("Table 'research_log' does not exist, no action needed")
+            return True
+    except Exception:
+        logger.exception("Error removing research_log table")
+        return False
 
 
-def remove_research_log_table(conn):
+def create_research_strategy_table(engine):
     """
-    Remove the redundant research_log table if it exists
+    Create the research_strategies table if it doesn't exist
 
     Args:
-        conn: SQLite connection
+        engine: SQLAlchemy engine
 
     Returns:
         bool: True if operation was successful, False otherwise
     """
     try:
-        cursor = conn.cursor()
+        inspector = inspect(engine)
 
         # Check if table exists
-        if check_table_exists(conn, "research_log"):
-            # For SQLite, DROP TABLE is the way to remove a table
-            cursor.execute("DROP TABLE research_log")
-            conn.commit()
-            logger.info("Successfully removed redundant 'research_log' table")
+        if not inspector.has_table("research_strategies"):
+            # Create the table using ORM
+            Base.metadata.create_all(
+                engine, tables=[ResearchStrategy.__table__]
+            )
+            logger.info("Successfully created 'research_strategies' table")
             return True
         else:
-            logger.info("Table 'research_log' does not exist, no action needed")
+            logger.info(
+                "Table 'research_strategies' already exists, no action needed"
+            )
             return True
     except Exception:
-        logger.exception("Error removing research_log table")
+        logger.exception("Error creating research_strategies table")
         return False
 
 
@@ -78,20 +96,22 @@ def run_schema_upgrades():
     """
     # Check if database exists
     if not os.path.exists(DB_PATH):
-        logger.warning(f"Database not found at {DB_PATH}, skipping schema upgrades")
+        logger.warning(
+            f"Database not found at {DB_PATH}, skipping schema upgrades"
+        )
         return False
 
     logger.info(f"Running schema upgrades on {DB_PATH}")
 
     try:
-        # Connect to the database
-        conn = sqlite3.connect(DB_PATH)
+        # Create SQLAlchemy engine
+        engine = create_engine(f"sqlite:///{DB_PATH}")
 
         # 1. Remove the redundant research_log table
-        remove_research_log_table(conn)
+        remove_research_log_table(engine)
 
-        # Close connection
-        conn.close()
+        # 2. Create research_strategies table
+        create_research_strategy_table(engine)
 
         logger.info("Schema upgrades completed successfully")
         return True

local_deep_research/web/models/database.py

@@ -1,10 +1,12 @@
-import json
 import os
 import sqlite3
 from datetime import datetime
 
 from loguru import logger
 
+from ...utilities.db_utils import get_db_session
+from ..database.models import ResearchLog
+
 # Database path
 # Use unified database in data directory
 DATA_DIR = os.path.abspath(
@@ -15,7 +17,9 @@ DB_PATH = os.path.join(DATA_DIR, "ldr.db")
 
 # Legacy database paths (for migration)
 LEGACY_RESEARCH_HISTORY_DB = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "..", "..", "..", "research_history.db")
+    os.path.join(
+        os.path.dirname(__file__), "..", "..", "..", "research_history.db"
+    )
 )
 LEGACY_DEEP_RESEARCH_DB = os.path.join(
     os.path.abspath(
@@ -97,7 +101,9 @@ def init_db():
     columns = [column[1] for column in cursor.fetchall()]
 
     if "duration_seconds" not in columns:
-        print("Adding missing 'duration_seconds' column to research_history table")
+        print(
+            "Adding missing 'duration_seconds' column to research_history table"
+        )
         cursor.execute(
             "ALTER TABLE research_history ADD COLUMN duration_seconds INTEGER"
         )
@@ -105,7 +111,9 @@ def init_db():
     # Check if the progress column exists, add it if missing
     if "progress" not in columns:
         print("Adding missing 'progress' column to research_history table")
-        cursor.execute("ALTER TABLE research_history ADD COLUMN progress INTEGER")
+        cursor.execute(
+            "ALTER TABLE research_history ADD COLUMN progress INTEGER"
+        )
 
     # Check if the title column exists, add it if missing
     if "title" not in columns:
@@ -180,10 +188,14 @@ def calculate_duration(created_at_str, completed_at_str=None):
         else:  # Older format without T
             # Try different formats
             try:
-                start_time = datetime.strptime(created_at_str, "%Y-%m-%d %H:%M:%S.%f")
+                start_time = datetime.strptime(
+                    created_at_str, "%Y-%m-%d %H:%M:%S.%f"
+                )
             except ValueError:
                 try:
-                    start_time = datetime.strptime(created_at_str, "%Y-%m-%d %H:%M:%S")
+                    start_time = datetime.strptime(
+                        created_at_str, "%Y-%m-%d %H:%M:%S"
+                    )
                 except ValueError:
                     # Last resort fallback
                     start_time = datetime.fromisoformat(
@@ -198,7 +210,7 @@ def calculate_duration(created_at_str, completed_at_str=None):
             start_time = parser.parse(created_at_str)
         except Exception:
             logger.exception(
-                f"Fallback parsing also failed for created_at:" f" {created_at_str}"
+                f"Fallback parsing also failed for created_at: {created_at_str}"
             )
             return None
 
@@ -212,36 +224,6 @@ def calculate_duration(created_at_str, completed_at_str=None):
     return None
 
 
-def add_log_to_db(research_id, message, log_type="info", progress=None, metadata=None):
-    """
-    Store a log entry in the database
-
-    Args:
-        research_id: ID of the research
-        message: Log message text
-        log_type: Type of log (info, error, milestone)
-        progress: Progress percentage (0-100)
-        metadata: Additional metadata as dictionary (will be stored as JSON)
-    """
-    try:
-        timestamp = datetime.utcnow().isoformat()
-        metadata_json = json.dumps(metadata) if metadata else None
-
-        conn = get_db_connection()
-        cursor = conn.cursor()
-        cursor.execute(
-            "INSERT INTO research_logs (research_id, timestamp, message, log_type, progress, metadata) "
-            "VALUES (?, ?, ?, ?, ?, ?)",
-            (research_id, timestamp, message, log_type, progress, metadata_json),
-        )
-        conn.commit()
-        conn.close()
-        return True
-    except Exception:
-        logger.exception("Error adding log to database")
-        return False
-
-
 def get_logs_for_research(research_id):
     """
     Retrieve all logs for a specific research ID
@@ -253,35 +235,23 @@ def get_logs_for_research(research_id):
         List of log entries as dictionaries
     """
     try:
-        conn = get_db_connection()
-        conn.row_factory = sqlite3.Row
-        cursor = conn.cursor()
-        cursor.execute(
-            "SELECT * FROM research_logs WHERE research_id = ? ORDER BY timestamp ASC",
-            (research_id,),
+        session = get_db_session()
+        log_results = (
+            session.query(ResearchLog)
+            .filter(ResearchLog.research_id == research_id)
+            .order_by(ResearchLog.timestamp.asc())
+            .all()
         )
-        results = cursor.fetchall()
-        conn.close()
 
         logs = []
-        for result in results:
-            log_entry = dict(result)
-            # Parse metadata JSON if it exists
-            if log_entry.get("metadata"):
-                try:
-                    log_entry["metadata"] = json.loads(log_entry["metadata"])
-                except Exception:
-                    log_entry["metadata"] = {}
-            else:
-                log_entry["metadata"] = {}
-
+        for result in log_results:
             # Convert entry for frontend consumption
             formatted_entry = {
-                "time": log_entry["timestamp"],
-                "message": log_entry["message"],
-                "progress": log_entry["progress"],
-                "metadata": log_entry["metadata"],
-                "type": log_entry["log_type"],
+                "time": result.timestamp,
+                "message": result.message,
+                "type": result.level,
+                "module": result.module,
+                "line_no": result.line_no,
             }
             logs.append(formatted_entry)
 
@@ -289,3 +259,23 @@ def get_logs_for_research(research_id):
     except Exception:
         logger.exception("Error retrieving logs from database")
         return []
+
+
+@logger.catch
+def get_total_logs_for_research(research_id):
+    """
+    Returns the total number of logs for a given `research_id`.
+
+    Args:
+        research_id (int): The ID of the research.
+
+    Returns:
+        int: Total number of logs for the specified research ID.
+    """
+    session = get_db_session()
+    total_logs = (
+        session.query(ResearchLog)
+        .filter(ResearchLog.research_id == research_id)
+        .count()
+    )
+    return total_logs

local_deep_research/web/routes/api_routes.py

@@ -59,9 +59,7 @@ def api_start_research():
                 mode,
                 "in_progress",
                 created_at,
-                json.dumps(
-                    [{"time": created_at, "message": "Research started", "progress": 0}]
-                ),
+                json.dumps([{"time": created_at, "progress": 0}]),
                 json.dumps(research_settings),
             ),
         )
@@ -89,9 +87,11 @@ def api_start_research():
                 "research_id": research_id,
             }
         )
-    except Exception as e:
-        logger.error(f"Error starting research: {str(e)}")
-        return jsonify({"status": "error", "message": str(e)}), 500
+    except Exception:
+        logger.exception("Error starting research")
+        return jsonify(
+            {"status": "error", "message": "Failed to start research"}, 500
+        )
 
 
 @api_bp.route("/status/<int:research_id>", methods=["GET"])
@@ -120,7 +120,9 @@ def api_research_status(research_id):
             try:
                 metadata = json.loads(metadata_str)
             except json.JSONDecodeError:
-                logger.warning(f"Invalid JSON in metadata for research {research_id}")
+                logger.warning(
+                    f"Invalid JSON in metadata for research {research_id}"
+                )
 
         return jsonify(
             {
@@ -144,10 +146,18 @@ def api_terminate_research(research_id):
     try:
         result = cancel_research(research_id)
         return jsonify(
-            {"status": "success", "message": "Research terminated", "result": result}
+            {
+                "status": "success",
+                "message": "Research terminated",
+                "result": result,
+            }
+        )
+    except Exception:
+        logger.exception("Error terminating research")
+        return (
+            jsonify({"status": "error", "message": "Failed to stop research."}),
+            500,
         )
-    except Exception as e:
-        return jsonify({"status": "error", "message": str(e)}), 500
 
 
 @api_bp.route("/resources/<int:research_id>", methods=["GET"])
@@ -158,8 +168,11 @@ def api_get_resources(research_id):
     try:
         resources = get_resources_for_research(research_id)
         return jsonify({"status": "success", "resources": resources})
-    except Exception as e:
-        return jsonify({"status": "error", "message": str(e)}), 500
+    except Exception:
+        logger.exception("Error getting resources for research")
+        return jsonify(
+            {"status": "error", "message": "Failed to get resources"}, 500
+        )
 
 
 @api_bp.route("/resources/<int:research_id>", methods=["POST"])
@@ -182,19 +195,25 @@ def api_add_resource(research_id):
         # Validate required fields
         if not title or not url:
             return (
-                jsonify({"status": "error", "message": "Title and URL are required"}),
+                jsonify(
+                    {"status": "error", "message": "Title and URL are required"}
+                ),
                 400,
             )
 
         # Check if the research exists
         conn = get_db_connection()
         cursor = conn.cursor()
-        cursor.execute("SELECT id FROM research_history WHERE id = ?", (research_id,))
+        cursor.execute(
+            "SELECT id FROM research_history WHERE id = ?", (research_id,)
+        )
         result = cursor.fetchone()
         conn.close()
 
         if not result:
-            return jsonify({"status": "error", "message": "Research not found"}), 404
+            return jsonify(
+                {"status": "error", "message": "Research not found"}
+            ), 404
 
         # Add the resource
         resource_id = add_resource(
@@ -231,10 +250,15 @@ def api_delete_resource(research_id, resource_id):
 
         if success:
             return jsonify(
-                {"status": "success", "message": "Resource deleted successfully"}
+                {
+                    "status": "success",
+                    "message": "Resource deleted successfully",
+                }
             )
         else:
-            return jsonify({"status": "error", "message": "Resource not found"}), 404
+            return jsonify(
+                {"status": "error", "message": "Resource not found"}
+            ), 404
     except Exception as e:
         logger.error(f"Error deleting resource: {str(e)}")
         return jsonify({"status": "error", "message": str(e)}), 500
@@ -252,7 +276,10 @@ def check_ollama_status():
 
         if provider.lower() != "ollama":
             return jsonify(
-                {"running": True, "message": f"Using provider: {provider}, not Ollama"}
+                {
+                    "running": True,
+                    "message": f"Using provider: {provider}, not Ollama",
+                }
             )
 
         # Get Ollama API URL from LLM config
@@ -272,7 +299,9 @@ def check_ollama_status():
             response = requests.get(f"{ollama_base_url}/api/tags", timeout=5)
 
             # Add response details for debugging
-            logger.debug(f"Ollama status check response code: {response.status_code}")
+            logger.debug(
+                f"Ollama status check response code: {response.status_code}"
+            )
 
             if response.status_code == 200:
                 # Try to validate the response content
@@ -420,7 +449,9 @@ def check_ollama_model():
 
                 # Debug log the first bit of the response
                 response_preview = (
-                    str(data)[:500] + "..." if len(str(data)) > 500 else str(data)
+                    str(data)[:500] + "..."
+                    if len(str(data)) > 500
+                    else str(data)
                 )
                 logger.debug(f"Ollama API response data: {response_preview}")
 
@@ -448,7 +479,8 @@ def check_ollama_model():
 
                 # Case-insensitive model name comparison
                 model_exists = any(
-                    m.get("name", "").lower() == model_name.lower() for m in models
+                    m.get("name", "").lower() == model_name.lower()
+                    for m in models
                 )
 
                 if model_exists:
@@ -561,7 +593,9 @@ def api_get_config():
             "search_tool", "auto"
         ),
         "features": {
-            "notifications": current_app.config.get("ENABLE_NOTIFICATIONS", False)
+            "notifications": current_app.config.get(
+                "ENABLE_NOTIFICATIONS", False
+            )
         },
     }
 

local_deep_research/web/routes/benchmark_routes.py

@@ -190,7 +190,10 @@ def get_benchmark_config():
             ],
             "evaluation_models": {
                 "openai_endpoint": [
-                    {"id": "anthropic/claude-3.7-sonnet", "name": "Claude 3.7 Sonnet"}
+                    {
+                        "id": "anthropic/claude-3.7-sonnet",
+                        "name": "Claude 3.7 Sonnet",
+                    }
                 ],
                 "openai": [
                     {"id": "gpt-4o", "name": "GPT-4o"},

local_deep_research/web/routes/globals.py

@@ -0,0 +1,22 @@
+"""
+Stores global state.
+"""
+
+# Active research processes and socket subscriptions
+active_research = {}
+socket_subscriptions = {}
+# Add termination flags dictionary
+termination_flags = {}
+
+
+def get_globals():
+    """
+    Returns:
+        Global state for other modules to access.
+
+    """
+    return {
+        "active_research": active_research,
+        "socket_subscriptions": socket_subscriptions,
+        "termination_flags": termination_flags,
+    }