local-deep-research 0.6.1__py3-none-any.whl → 0.6.4__py3-none-any.whl

local_deep_research/web/database/uuid_migration.py

@@ -5,10 +5,9 @@ Migrates all research_id fields from Integer to String (UUID) format.
 This creates a more consistent and scalable ID system across the application.
 """
 
-import sqlite3
-import uuid
 from pathlib import Path
-from loguru import logger
+
+from local_deep_research.web.database.migrations import migrate_to_uuid
 
 
 def get_database_path():
@@ -17,249 +16,5 @@ def get_database_path():
     return data_dir / "ldr.db"
 
 
-def migrate_to_uuid():
-    """
-    Migrate all research_id fields from integers to UUIDs.
-
-    Strategy:
-    1. Add new UUID columns alongside existing integer columns
-    2. Generate UUIDs for existing data (or keep as string versions of integers)
-    3. Update foreign key relationships
-    4. Drop old integer columns and rename UUID columns
-    """
-    db_path = get_database_path()
-
-    if not db_path.exists():
-        logger.info("Database doesn't exist yet, migration not needed")
-        return
-
-    logger.info(f"Starting UUID migration on {db_path}")
-
-    conn = sqlite3.connect(db_path)
-    conn.execute(
-        "PRAGMA foreign_keys = OFF"
-    )  # Disable FK constraints during migration
-
-    try:
-        cursor = conn.cursor()
-
-        # 1. Migrate research_history table (main research IDs)
-        logger.info("Migrating research_history table...")
-
-        # Check if the table exists and has the old structure
-        cursor.execute("PRAGMA table_info(research_history)")
-        columns = cursor.fetchall()
-        has_uuid_id = any(col[1] == "uuid_id" for col in columns)
-
-        if not has_uuid_id:
-            # Add UUID column
-            cursor.execute(
-                "ALTER TABLE research_history ADD COLUMN uuid_id TEXT"
-            )
-
-            # Generate UUIDs for existing records (convert integer ID to UUID format)
-            cursor.execute("SELECT id FROM research_history")
-            existing_ids = cursor.fetchall()
-
-            for (old_id,) in existing_ids:
-                # Generate a deterministic UUID based on the old ID
-                new_uuid = str(
-                    uuid.uuid5(uuid.NAMESPACE_OID, f"research_{old_id}")
-                )
-                cursor.execute(
-                    "UPDATE research_history SET uuid_id = ? WHERE id = ?",
-                    (new_uuid, old_id),
-                )
-
-            logger.info(
-                f"Generated UUIDs for {len(existing_ids)} research records"
-            )
-
-        # 2. Migrate metrics tables
-        logger.info("Migrating metrics tables...")
-
-        # Token usage table
-        cursor.execute("PRAGMA table_info(token_usage)")
-        columns = cursor.fetchall()
-        has_uuid_research_id = any(
-            col[1] == "uuid_research_id" for col in columns
-        )
-
-        if not has_uuid_research_id:
-            cursor.execute(
-                "ALTER TABLE token_usage ADD COLUMN uuid_research_id TEXT"
-            )
-
-            # Convert existing research_ids to UUIDs (deterministic conversion)
-            cursor.execute(
-                "SELECT DISTINCT research_id FROM token_usage WHERE research_id IS NOT NULL"
-            )
-            research_ids = cursor.fetchall()
-
-            for (research_id,) in research_ids:
-                if research_id:
-                    new_uuid = str(
-                        uuid.uuid5(
-                            uuid.NAMESPACE_OID, f"research_{research_id}"
-                        )
-                    )
-                    cursor.execute(
-                        "UPDATE token_usage SET uuid_research_id = ? WHERE research_id = ?",
-                        (new_uuid, research_id),
-                    )
-
-            logger.info(
-                f"Migrated {len(research_ids)} research IDs in token_usage"
-            )
-
-        # Model usage table
-        cursor.execute("PRAGMA table_info(model_usage)")
-        columns = cursor.fetchall()
-        has_uuid_research_id = any(
-            col[1] == "uuid_research_id" for col in columns
-        )
-
-        if not has_uuid_research_id:
-            cursor.execute(
-                "ALTER TABLE model_usage ADD COLUMN uuid_research_id TEXT"
-            )
-
-            cursor.execute(
-                "SELECT DISTINCT research_id FROM model_usage WHERE research_id IS NOT NULL"
-            )
-            research_ids = cursor.fetchall()
-
-            for (research_id,) in research_ids:
-                if research_id:
-                    new_uuid = str(
-                        uuid.uuid5(
-                            uuid.NAMESPACE_OID, f"research_{research_id}"
-                        )
-                    )
-                    cursor.execute(
-                        "UPDATE model_usage SET uuid_research_id = ? WHERE research_id = ?",
-                        (new_uuid, research_id),
-                    )
-
-            logger.info(
-                f"Migrated {len(research_ids)} research IDs in model_usage"
-            )
-
-        # Search calls table
-        cursor.execute("PRAGMA table_info(search_calls)")
-        columns = cursor.fetchall()
-        has_uuid_research_id = any(
-            col[1] == "uuid_research_id" for col in columns
-        )
-
-        if not has_uuid_research_id:
-            cursor.execute(
-                "ALTER TABLE search_calls ADD COLUMN uuid_research_id TEXT"
-            )
-
-            cursor.execute(
-                "SELECT DISTINCT research_id FROM search_calls WHERE research_id IS NOT NULL"
-            )
-            research_ids = cursor.fetchall()
-
-            for (research_id,) in research_ids:
-                if research_id:
-                    new_uuid = str(
-                        uuid.uuid5(
-                            uuid.NAMESPACE_OID, f"research_{research_id}"
-                        )
-                    )
-                    cursor.execute(
-                        "UPDATE search_calls SET uuid_research_id = ? WHERE research_id = ?",
-                        (new_uuid, research_id),
-                    )
-
-            logger.info(
-                f"Migrated {len(research_ids)} research IDs in search_calls"
-            )
-
-        # 3. Migrate benchmark tables
-        logger.info("Migrating benchmark tables...")
-
-        # Check if benchmark_results table exists
-        cursor.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' AND name='benchmark_results'"
-        )
-        if cursor.fetchone():
-            cursor.execute("PRAGMA table_info(benchmark_results)")
-            columns = cursor.fetchall()
-            has_uuid_research_id = any(
-                col[1] == "uuid_research_id" for col in columns
-            )
-
-            if not has_uuid_research_id:
-                cursor.execute(
-                    "ALTER TABLE benchmark_results ADD COLUMN uuid_research_id TEXT"
-                )
-
-                cursor.execute(
-                    "SELECT DISTINCT research_id FROM benchmark_results WHERE research_id IS NOT NULL"
-                )
-                research_ids = cursor.fetchall()
-
-                for (research_id,) in research_ids:
-                    if research_id:
-                        new_uuid = str(
-                            uuid.uuid5(
-                                uuid.NAMESPACE_OID, f"research_{research_id}"
-                            )
-                        )
-                        cursor.execute(
-                            "UPDATE benchmark_results SET uuid_research_id = ? WHERE research_id = ?",
-                            (new_uuid, research_id),
-                        )
-
-                logger.info(
-                    f"Migrated {len(research_ids)} research IDs in benchmark_results"
-                )
-
-        # Commit all changes
-        conn.commit()
-        logger.info("UUID migration completed successfully!")
-
-        # Note: We're keeping both old and new columns for now
-        # The application will use the new UUID columns
-        # Old columns can be dropped in a future migration once everything is stable
-
-    except Exception as e:
-        logger.error(f"Error during UUID migration: {e}")
-        conn.rollback()
-        raise
-    finally:
-        conn.execute("PRAGMA foreign_keys = ON")  # Re-enable FK constraints
-        conn.close()
-
-
-def cleanup_old_columns():
-    """
-    Cleanup migration - drops old integer columns after UUID migration is stable.
-    Run this only after confirming the UUID migration is working correctly.
-    """
-    logger.warning(
-        "This will permanently remove old integer research_id columns!"
-    )
-    logger.warning("Make sure to backup your database before running this!")
-
-    db_path = get_database_path()
-    conn = sqlite3.connect(db_path)
-    conn.execute("PRAGMA foreign_keys = OFF")
-
-    try:
-        # For SQLite, we need to recreate tables to drop columns
-        # This is complex, so we'll leave old columns for now
-        # They can be cleaned up manually if needed
-
-        logger.info("Cleanup deferred - old columns remain for safety")
-
-    finally:
-        conn.execute("PRAGMA foreign_keys = ON")
-        conn.close()
-
-
 if __name__ == "__main__":
     migrate_to_uuid()

local_deep_research/web/models/database.py

@@ -38,85 +38,6 @@ def get_db_connection():
     return conn
 
 
-def init_db():
-    """Initialize the database with necessary tables."""
-    conn = get_db_connection()
-    cursor = conn.cursor()
-
-    # Create a dedicated table for research logs
-    cursor.execute(
-        """
-    CREATE TABLE IF NOT EXISTS research_logs (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        research_id INTEGER NOT NULL,
-        timestamp TEXT NOT NULL,
-        message TEXT NOT NULL,
-        log_type TEXT NOT NULL,
-        progress INTEGER,
-        metadata TEXT,
-        FOREIGN KEY (research_id) REFERENCES research_history (id) ON DELETE CASCADE
-    )
-    """
-    )
-
-    # Create a dedicated table for research resources
-    cursor.execute(
-        """
-    CREATE TABLE IF NOT EXISTS research_resources (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        research_id INTEGER NOT NULL,
-        title TEXT,
-        url TEXT,
-        content_preview TEXT,
-        source_type TEXT,
-        metadata TEXT,
-        created_at TEXT NOT NULL,
-        FOREIGN KEY (research_id) REFERENCES research_history (id) ON DELETE CASCADE
-    )
-    """
-    )
-
-    # Check if the duration_seconds column exists, add it if missing
-    cursor.execute("PRAGMA table_info(research_history)")
-    columns = [column[1] for column in cursor.fetchall()]
-
-    if "duration_seconds" not in columns:
-        logger.info(
-            "Adding missing 'duration_seconds' column to research_history table"
-        )
-        cursor.execute(
-            "ALTER TABLE research_history ADD COLUMN duration_seconds INTEGER"
-        )
-
-    # Check if the progress column exists, add it if missing
-    if "progress" not in columns:
-        logger.info(
-            "Adding missing 'progress' column to research_history table"
-        )
-        cursor.execute(
-            "ALTER TABLE research_history ADD COLUMN progress INTEGER"
-        )
-
-    # Check if the title column exists, add it if missing
-    if "title" not in columns:
-        logger.info("Adding missing 'title' column to research_history table")
-        cursor.execute("ALTER TABLE research_history ADD COLUMN title TEXT")
-
-    # Check if the metadata column exists, and rename it to "research_meta"
-    # if it does.
-    if "metadata" in columns:
-        logger.info("Renaming 'metadata' column to 'research_meta'")
-        cursor.execute(
-            "ALTER TABLE research_history RENAME COLUMN metadata TO research_meta"
-        )
-
-    # Enable foreign key support
-    cursor.execute("PRAGMA foreign_keys = ON")
-
-    conn.commit()
-    conn.close()
-
-
 def calculate_duration(created_at_str, completed_at_str=None):
     """
     Calculate duration in seconds between created_at timestamp and completed_at or now.

local_deep_research/web/routes/settings_routes.py

@@ -16,16 +16,17 @@ from flask import (
 from flask_wtf.csrf import generate_csrf
 from loguru import logger
 
-from ...utilities.db_utils import get_db_setting, get_db_session
+from ...utilities.db_utils import (
+    get_db_setting,
+    get_db_session,
+    get_settings_manager,
+)
 from ...utilities.url_utils import normalize_url
 from ..database.models import Setting, SettingType
 from ..services.settings_service import (
     create_or_update_setting,
-    get_setting,
-    get_settings_manager,
     set_setting,
 )
-from ..services.settings_manager import SettingsManager
 from ..utils.templates import render_template_with_defaults
 
 # Create a Blueprint for settings
@@ -37,20 +38,15 @@ def calculate_warnings():
     warnings = []
 
     try:
-        # Get a fresh database session for safety
-        db_session = get_db_session()
-
         # Get current settings
-        provider = get_setting("llm.provider", "ollama", db_session).lower()
-        local_context = get_setting(
-            "llm.local_context_window_size", 4096, db_session
-        )
+        provider = get_db_setting("llm.provider", "ollama").lower()
+        local_context = get_db_setting("llm.local_context_window_size", 4096)
 
         logger.debug(f"Starting warning calculation - provider={provider}")
 
         # Get dismissal settings
-        dismiss_high_context = get_setting(
-            "app.warnings.dismiss_high_context", False, db_session
+        dismiss_high_context = get_db_setting(
+            "app.warnings.dismiss_high_context", False
         )
 
         # Check warning conditions
@@ -78,15 +74,15 @@ def calculate_warnings():
             )
 
         # Get additional warning settings
-        dismiss_model_mismatch = get_setting(
-            "app.warnings.dismiss_model_mismatch", False, db_session
+        dismiss_model_mismatch = get_db_setting(
+            "app.warnings.dismiss_model_mismatch", False
         )
 
         # Get current strategy and model (these need to be passed from the frontend or retrieved differently)
         # For now, we'll implement basic warnings that don't require form state
 
         # Model mismatch warning (simplified - checking setting instead of form value)
-        current_model = get_setting("llm.model", "", db_session)
+        current_model = get_db_setting("llm.model", "")
         if (
             current_model
             and "70b" in current_model.lower()
@@ -165,7 +161,7 @@ def save_all_settings():
     """Handle saving all settings at once from the unified settings page"""
     db_session = get_db_session()
     # Get the settings manager but we don't need to assign it to a variable right now
-    # get_settings_manager(db_session)
+    # get_db_settings_manager(db_session)
 
     try:
         # Process JSON data
@@ -462,7 +458,7 @@ def reset_to_defaults():
     # Import default settings from files
     try:
         # Create settings manager for the temporary config
-        settings_mgr = get_settings_manager(db_session)
+        settings_mgr = get_settings_manager()
         # Import settings from default files
         settings_mgr.load_from_defaults_file()
 
@@ -499,7 +495,7 @@ def api_get_all_settings():
 
         # Create settings manager
         db_session = get_db_session()
-        settings_manager = get_settings_manager(db_session)
+        settings_manager = get_settings_manager()
 
         # Get settings
         settings = settings_manager.get_all_settings()
@@ -527,15 +523,15 @@ def api_get_all_settings():
 
 
 @settings_bp.route("/api/<path:key>", methods=["GET"])
-def api_get_setting(key):
+def api_get_db_setting(key):
     """Get a specific setting by key"""
     try:
         db_session = get_db_session()
         # No need to assign if not used
-        # get_settings_manager(db_session)
+        # get_db_settings_manager(db_session)
 
         # Get setting
-        value = get_setting(key)
+        value = get_db_setting(key)
         if value is None:
             return jsonify({"error": f"Setting not found: {key}"}), 404
 
@@ -587,7 +583,7 @@ def api_update_setting(key):
         # Get DB session and settings manager
         db_session = get_db_session()
         # Only use settings_manager if needed - we don't need to assign if not used
-        # get_settings_manager(db_session)
+        # get_db_settings_manager(db_session)
 
         # Check if setting exists
         db_setting = (
@@ -687,7 +683,7 @@ def api_delete_setting(key):
     """Delete a setting"""
     try:
         db_session = get_db_session()
-        settings_manager = get_settings_manager(db_session)
+        settings_manager = get_settings_manager()
 
         # Check if setting exists
         db_setting = (
@@ -711,7 +707,7 @@ def api_delete_setting(key):
 def api_import_settings():
     """Import settings from defaults file"""
     try:
-        settings_manager = get_settings_manager(get_db_session())
+        settings_manager = get_settings_manager()
 
         success = settings_manager.load_from_defaults_file()
 
@@ -782,7 +778,7 @@ def api_get_available_models():
     try:
         from datetime import datetime, timedelta
         from ..database.models import ProviderModel
-        from flask import request, current_app
+        from flask import request
 
         # Check if force_refresh is requested
         force_refresh = (
@@ -806,17 +802,16 @@ def api_get_available_models():
         # Check database cache first (unless force_refresh is True)
         if not force_refresh:
             try:
-                db_session = current_app.db_session
-
                 # Define cache expiration (24 hours)
                 cache_expiry = datetime.utcnow() - timedelta(hours=24)
 
                 # Get cached models from database
-                cached_models = (
-                    db_session.query(ProviderModel)
-                    .filter(ProviderModel.last_updated > cache_expiry)
-                    .all()
-                )
+                with get_db_session("settings_routes") as db_session:
+                    cached_models = (
+                        db_session.query(ProviderModel)
+                        .filter(ProviderModel.last_updated > cache_expiry)
+                        .all()
+                    )
 
                 if cached_models:
                     logger.info(
@@ -1269,41 +1264,44 @@ def api_get_available_models():
         # Save fetched models to database cache
         if force_refresh or providers:
             # We fetched fresh data, save it to database
-            try:
-                from datetime import datetime
-
-                db_session = current_app.db_session
-
-                # Clear old cache entries for providers we're updating
-                for provider_key in providers.keys():
-                    provider_name = provider_key.replace("_models", "").upper()
-                    db_session.query(ProviderModel).filter(
-                        ProviderModel.provider == provider_name
-                    ).delete()
-
-                # Insert new models
-                for provider_key, models in providers.items():
-                    provider_name = provider_key.replace("_models", "").upper()
-                    for model in models:
-                        if (
-                            isinstance(model, dict)
-                            and "value" in model
-                            and "label" in model
-                        ):
-                            new_model = ProviderModel(
-                                provider=provider_name,
-                                model_key=model["value"],
-                                model_label=model["label"],
-                                last_updated=datetime.utcnow(),
-                            )
-                            db_session.add(new_model)
-
-                db_session.commit()
-                logger.info("Successfully cached models to database")
-
-            except Exception:
-                logger.exception("Error saving models to database cache")
-                db_session.rollback()
+            with get_db_session("settings_routes") as db_session:
+                try:
+                    from datetime import datetime
+
+                    # Clear old cache entries for providers we're updating
+                    for provider_key in providers.keys():
+                        provider_name = provider_key.replace(
+                            "_models", ""
+                        ).upper()
+                        db_session.query(ProviderModel).filter(
+                            ProviderModel.provider == provider_name
+                        ).delete()
+
+                    # Insert new models
+                    for provider_key, models in providers.items():
+                        provider_name = provider_key.replace(
+                            "_models", ""
+                        ).upper()
+                        for model in models:
+                            if (
+                                isinstance(model, dict)
+                                and "value" in model
+                                and "label" in model
+                            ):
+                                new_model = ProviderModel(
+                                    provider=provider_name,
+                                    model_key=model["value"],
+                                    model_label=model["label"],
+                                    last_updated=datetime.utcnow(),
+                                )
+                                db_session.add(new_model)
+
+                    db_session.commit()
+                    logger.info("Successfully cached models to database")
+
+                except Exception:
+                    logger.exception("Error saving models to database cache")
+                    db_session.rollback()
 
         # Return all options
         return jsonify(
@@ -1858,9 +1856,9 @@ def api_get_rate_limiting_status():
                     "max_wait_seconds": round(max_wait, 2),
                     "last_updated": last_updated,
                     "total_attempts": total_attempts,
-                    "success_rate": round(success_rate * 100, 1)
-                    if success_rate
-                    else 0.0,
+                    "success_rate": (
+                        round(success_rate * 100, 1) if success_rate else 0.0
+                    ),
                 }
             )
 
@@ -1934,12 +1932,11 @@ def get_bulk_settings():
 
         # Fetch all settings at once
         session = get_db_session()
-        settings_manager = SettingsManager(db_session=session)
 
         result = {}
         for key in requested:
             try:
-                value = settings_manager.get_setting(key)
+                value = get_db_setting(key)
                 result[key] = {"value": value, "exists": value is not None}
             except Exception as e:
                 logger.warning(f"Error getting setting {key}: {e}")