local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/setup_data_dir.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""
+Data directory setup script for Local Deep Research.
+Creates the data directory for the application database if it doesn't exist.
+"""
+
+import os
+
+
+def setup_data_dir():
+    """Set up the data directory for the application."""
+    # Get the project root directory (3 levels up from this file)
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
+
+    # Define the data directory path
+    data_dir = os.path.join(project_root, "data")
+
+    # Create the data directory if it doesn't exist
+    if not os.path.exists(data_dir):
+        os.makedirs(data_dir)
+        print(f"Created data directory at: {data_dir}")
+    else:
+        print(f"Data directory already exists at: {data_dir}")
+
+    # Return the path to the data directory
+    return data_dir
+
+
+if __name__ == "__main__":
+    data_dir = setup_data_dir()
+    db_path = os.path.join(data_dir, "ldr.db")
+    print(f"Database path: {db_path}")
+    print("Run the following command to migrate your database:")
+    print("python -m src.local_deep_research.migrate_db --backup")

local_deep_research/test_migration.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+"""
+Migration test script for Local Deep Research.
+This script checks the contents of both the legacy and new databases to diagnose migration issues.
+"""
+
+import os
+import sqlite3
+import sys
+import time
+
+
+def check_db_content(db_path, description):
+    """Check what tables and how many rows are in a database."""
+    if not os.path.exists(db_path):
+        print(f"❌ {description} database not found at: {db_path}")
+        return False
+
+    print(f"📊 Examining {description} database at: {db_path}")
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        # Get list of tables
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        tables = [
+            row[0] for row in cursor.fetchall() if not row[0].startswith("sqlite_")
+        ]
+
+        if not tables:
+            print("   ℹ️ No user tables found in database")
+            conn.close()
+            return False
+
+        print(f"   📋 Tables found: {', '.join(tables)}")
+
+        # For each table, count rows
+        for table in tables:
+            cursor.execute(f"SELECT COUNT(*) FROM {table}")
+            count = cursor.fetchone()[0]
+            print(f"   📝 Table '{table}' has {count} rows")
+
+            # If table has rows, show sample
+            if count > 0:
+                cursor.execute(f"SELECT * FROM {table} LIMIT 1")
+                columns = [description[0] for description in cursor.description]
+                print(f"      Columns: {', '.join(columns)}")
+
+                # For specific tables, get key columns
+                if table in [
+                    "research_history",
+                    "research_logs",
+                    "research",
+                    "settings",
+                ]:
+                    key_cols = (
+                        "id, query, status"
+                        if table == "research_history"
+                        else "id, key, value" if table == "settings" else "id, message"
+                    )
+                    cursor.execute(f"SELECT {key_cols} FROM {table} LIMIT 3")
+                    sample = cursor.fetchall()
+                    for row in sample:
+                        print(f"      Sample data: {row}")
+
+        conn.close()
+        return True
+    except Exception as e:
+        print(f"❌ Error examining database: {e}")
+        return False
+
+
+def main():
+    """Main function to test the migration."""
+    # Import necessary constants
+    try:
+        # Set up paths
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        project_root = os.path.abspath(os.path.join(current_dir, "..", ".."))
+
+        # Determine paths
+        data_dir = os.path.join(project_root, "data")
+        new_db_path = os.path.join(data_dir, "ldr.db")
+
+        legacy_research_history_db = os.path.join(
+            project_root, "src", "local_deep_research", "research_history.db"
+        )
+        legacy_deep_research_db = os.path.join(data_dir, "deep_research.db")
+
+        # Print paths for verification
+        print("=" * 60)
+        print("DATABASE PATHS")
+        print("=" * 60)
+        print(f"New database path: {new_db_path}")
+        print(f"Legacy research history DB: {legacy_research_history_db}")
+        print(f"Legacy deep research DB: {legacy_deep_research_db}")
+        print("=" * 60)
+
+        # Check all databases
+        check_db_content(legacy_research_history_db, "Legacy research_history")
+        check_db_content(legacy_deep_research_db, "Legacy deep_research")
+
+        # Now check for the new database or create it if needed
+        if os.path.exists(new_db_path):
+            check_db_content(new_db_path, "New ldr")
+        else:
+            print(f"ℹ️ New database doesn't exist yet at: {new_db_path}")
+            print("Would you like to run a test migration? (y/n)")
+            choice = input("> ").lower()
+            if choice == "y":
+                # Run the migration script directly
+                try:
+                    from src.local_deep_research.setup_data_dir import setup_data_dir
+                except ImportError:
+                    # If that fails, try with the direct import
+                    sys.path.append(
+                        os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+                    )
+                    from local_deep_research.setup_data_dir import setup_data_dir
+
+                setup_data_dir()
+
+                # Import migration function
+                try:
+                    from src.local_deep_research.web.database.migrate_to_ldr_db import (
+                        migrate_to_ldr_db,
+                    )
+                except ImportError:
+                    # If that fails, try with the direct import
+                    from local_deep_research.web.database.migrate_to_ldr_db import (
+                        migrate_to_ldr_db,
+                    )
+
+                print("Running migration...")
+                success = migrate_to_ldr_db()
+
+                # Wait briefly to ensure file system has time to update
+                time.sleep(1)
+
+                if success:
+                    print("\n✅ Migration completed. Checking new database:")
+                    check_db_content(new_db_path, "New ldr")
+                else:
+                    print("❌ Migration failed")
+
+        # Get the paths from the migration script to verify
+        try:
+            try:
+                from src.local_deep_research.web.models.database import (
+                    DB_PATH,
+                    LEGACY_DEEP_RESEARCH_DB,
+                    LEGACY_RESEARCH_HISTORY_DB,
+                )
+            except ImportError:
+                from local_deep_research.web.models.database import (
+                    DB_PATH,
+                    LEGACY_DEEP_RESEARCH_DB,
+                    LEGACY_RESEARCH_HISTORY_DB,
+                )
+
+            print("\n" + "=" * 60)
+            print("PATHS FROM DATABASE MODULE")
+            print("=" * 60)
+            print(f"DB_PATH: {DB_PATH}")
+            print(f"LEGACY_RESEARCH_HISTORY_DB: {LEGACY_RESEARCH_HISTORY_DB}")
+            print(f"LEGACY_DEEP_RESEARCH_DB: {LEGACY_DEEP_RESEARCH_DB}")
+        except ImportError as e:
+            print(f"Could not import paths from database module: {e}")
+
+    except Exception as e:
+        print(f"Error in test script: {e}")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

local_deep_research/utilities/db_utils.py

@@ -0,0 +1,49 @@
+import logging
+import os
+from functools import cache
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..web.services.settings_manager import SettingsManager
+
+logger = logging.getLogger(__name__)
+
+
+# Database path.
+DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "data"))
+DB_PATH = os.path.join(DATA_DIR, "ldr.db")
+
+
+@cache
+def get_db_session() -> Session:
+    """
+    Returns:
+        The singleton DB session.
+    """
+    engine = create_engine(f"sqlite:///{DB_PATH}")
+    session_class = sessionmaker(bind=engine)
+    return session_class()
+
+
+@cache
+def get_settings_manager() -> SettingsManager:
+    """
+    Returns:
+        The singleton settings manager.
+
+    """
+    return SettingsManager(db_session=get_db_session())
+
+
+def get_db_setting(key, default_value=None):
+    """Get a setting from the database with fallback to default value"""
+    try:
+        # Get settings manager which handles database access
+        value = get_settings_manager().get_setting(key)
+
+        if value is not None:
+            return value
+    except Exception as e:
+        logger.error(f"Error getting setting {key} from database: {e}")
+    return default_value

local_deep_research/{utilties → utilities}/enums.py

@@ -1,9 +1,9 @@
 # config/enums.py
-from enum import Enum, auto
+from enum import Enum
+
 
 class KnowledgeAccumulationApproach(Enum):
     QUESTION = "QUESTION"
     ITERATION = "ITERATION"
     NO_KNOWLEDGE = "NO_KNOWLEDGE"
     MAX_NR_OF_CHARACTERS = "MAX_NR_OF_CHARACTERS"
-

local_deep_research/{utilties → utilities}/llm_utils.py

@@ -1,4 +1,4 @@
-# utilties/llm_utils.py
+# utilities/llm_utils.py
 """
 LLM utilities for Local Deep Research.
 
@@ -6,64 +6,75 @@ This module provides utility functions for working with language models
 when the user's llm_config.py is missing or incomplete.
 """
 
-import os
 import logging
-from typing import Dict, Any, Optional
+import os
+from typing import Any, Optional
 
 # Setup logging
 logger = logging.getLogger(__name__)
 
+
 def get_model(
     model_name: Optional[str] = None,
     model_type: Optional[str] = None,
     temperature: Optional[float] = None,
-    **kwargs
+    **kwargs,
 ) -> Any:
     """
     Get a language model instance as fallback when llm_config.get_llm is not available.
-    
+
     Args:
         model_name: Name of the model to use
         model_type: Type of the model provider
         temperature: Model temperature
         **kwargs: Additional parameters
-        
+
     Returns:
         LangChain language model instance
     """
     # Get default values from kwargs or use reasonable defaults
-    model_name = model_name or kwargs.get('DEFAULT_MODEL', 'mistral')
-    model_type = model_type or kwargs.get('DEFAULT_MODEL_TYPE', 'ollama')
-    temperature = temperature or kwargs.get('DEFAULT_TEMPERATURE', 0.7)
-    max_tokens = kwargs.get('max_tokens', kwargs.get('MAX_TOKENS', 30000))
-    
+    model_name = model_name or kwargs.get("DEFAULT_MODEL", "mistral")
+    model_type = model_type or kwargs.get("DEFAULT_MODEL_TYPE", "ollama")
+    temperature = temperature or kwargs.get("DEFAULT_TEMPERATURE", 0.7)
+    max_tokens = kwargs.get("max_tokens", kwargs.get("MAX_TOKENS", 30000))
+
     # Common parameters
     common_params = {
         "temperature": temperature,
         "max_tokens": max_tokens,
     }
-    
+
     # Add additional kwargs
     for key, value in kwargs.items():
-        if key not in ['DEFAULT_MODEL', 'DEFAULT_MODEL_TYPE', 'DEFAULT_TEMPERATURE', 'MAX_TOKENS']:
+        if key not in [
+            "DEFAULT_MODEL",
+            "DEFAULT_MODEL_TYPE",
+            "DEFAULT_TEMPERATURE",
+            "MAX_TOKENS",
+        ]:
             common_params[key] = value
-    
+
     # Try to load the model based on type
     if model_type == "ollama":
         try:
             from langchain_ollama import ChatOllama
+
             return ChatOllama(model=model_name, **common_params)
         except ImportError:
             try:
                 from langchain_community.llms import Ollama
+
                 return Ollama(model=model_name, **common_params)
             except ImportError:
-                logger.error("Neither langchain_ollama nor langchain_community.llms.Ollama available")
+                logger.error(
+                    "Neither langchain_ollama nor langchain_community.llms.Ollama available"
+                )
                 raise
-    
+
     elif model_type == "openai":
         try:
             from langchain_openai import ChatOpenAI
+
             api_key = os.getenv("OPENAI_API_KEY")
             if not api_key:
                 raise ValueError("OPENAI_API_KEY environment variable not set")
@@ -71,46 +82,69 @@ def get_model(
         except ImportError:
             logger.error("langchain_openai not available")
             raise
-    
+
     elif model_type == "anthropic":
         try:
             from langchain_anthropic import ChatAnthropic
+
             api_key = os.getenv("ANTHROPIC_API_KEY")
             if not api_key:
                 raise ValueError("ANTHROPIC_API_KEY environment variable not set")
-            return ChatAnthropic(model=model_name, anthropic_api_key=api_key, **common_params)
+            return ChatAnthropic(
+                model=model_name, anthropic_api_key=api_key, **common_params
+            )
         except ImportError:
             logger.error("langchain_anthropic not available")
             raise
-    
+
     elif model_type == "openai_endpoint":
         try:
             from langchain_openai import ChatOpenAI
+
             api_key = os.getenv("OPENAI_ENDPOINT_API_KEY")
             if not api_key:
                 raise ValueError("OPENAI_ENDPOINT_API_KEY environment variable not set")
-            
-            endpoint_url = kwargs.get("OPENAI_ENDPOINT_URL", "https://openrouter.ai/api/v1")
-            
-            if model_name is None and not kwargs.get("OPENAI_ENDPOINT_REQUIRES_MODEL", True):
-                return ChatOpenAI(api_key=api_key, openai_api_base=endpoint_url, **common_params)
+
+            endpoint_url = kwargs.get(
+                "OPENAI_ENDPOINT_URL", "https://openrouter.ai/api/v1"
+            )
+
+            if model_name is None and not kwargs.get(
+                "OPENAI_ENDPOINT_REQUIRES_MODEL", True
+            ):
+                return ChatOpenAI(
+                    api_key=api_key, openai_api_base=endpoint_url, **common_params
+                )
             else:
-                return ChatOpenAI(model=model_name, api_key=api_key, openai_api_base=endpoint_url, **common_params)
+                return ChatOpenAI(
+                    model=model_name,
+                    api_key=api_key,
+                    openai_api_base=endpoint_url,
+                    **common_params,
+                )
         except ImportError:
             logger.error("langchain_openai not available")
             raise
-    
+
     # Default fallback
     try:
         from langchain_ollama import ChatOllama
+
         logger.warning(f"Unknown model type '{model_type}', defaulting to Ollama")
         return ChatOllama(model=model_name, **common_params)
     except (ImportError, Exception) as e:
         logger.error(f"Failed to load any model: {e}")
-        
+
         # Last resort: create a dummy model
         try:
             from langchain_community.llms.fake import FakeListLLM
-            return FakeListLLM(responses=["No language models are available. Please install Ollama or set up API keys."])
+
+            return FakeListLLM(
+                responses=[
+                    "No language models are available. Please install Ollama or set up API keys."
+                ]
+            )
         except ImportError:
-            raise ValueError("No language models available and could not create dummy model")
+            raise ValueError(
+                "No language models available and could not create dummy model"
+            )