local-deep-research 0.2.3__py3-none-any.whl → 0.3.1__py3-none-any.whl

local_deep_research/web/routes/settings_routes.py

@@ -3,11 +3,9 @@ import logging
 import os
 import platform
 import subprocess
-from pathlib import Path
 from typing import Any, Optional, Tuple
 
 import requests
-import toml
 from flask import (
     Blueprint,
     current_app,
@@ -21,8 +19,6 @@ from flask import (
 from flask_wtf.csrf import generate_csrf
 from sqlalchemy.orm import Session
 
-from ...config.config_files import get_config_dir
-from ...web_search_engines.search_engine_factory import get_available_engines
 from ..database.models import Setting, SettingType
 from ..services.settings_service import (
     create_or_update_setting,
@@ -37,23 +33,6 @@ logger = logging.getLogger(__name__)
 # Create a Blueprint for settings
 settings_bp = Blueprint("settings", __name__, url_prefix="/research/settings")
 
-# Legacy config for backwards compatibility
-SEARCH_ENGINES_FILE = None
-CONFIG_DIR = None
-MAIN_CONFIG_FILE = None
-LOCAL_COLLECTIONS_FILE = None
-
-
-def set_config_paths(
-    config_dir, search_engines_file, main_config_file, local_collections_file
-):
-    """Set the config paths for the settings routes (legacy support)"""
-    global CONFIG_DIR, SEARCH_ENGINES_FILE, MAIN_CONFIG_FILE, LOCAL_COLLECTIONS_FILE
-    CONFIG_DIR = config_dir
-    SEARCH_ENGINES_FILE = search_engines_file
-    MAIN_CONFIG_FILE = main_config_file
-    LOCAL_COLLECTIONS_FILE = local_collections_file
-
 
 def get_db_session() -> Session:
     """Get the database session from the app context"""
@@ -105,56 +84,6 @@ def validate_setting(setting: Setting, value: Any) -> Tuple[bool, Optional[str]]
     return True, None
 
 
-def get_all_settings_json():
-    """Get all settings as a JSON-serializable dictionary
-
-    Returns:
-        List of setting dictionaries
-    """
-    db_session = get_db_session()
-    settings_list = []
-
-    # Get all settings
-    settings = (
-        db_session.query(Setting)
-        .order_by(Setting.type, Setting.category, Setting.name)
-        .all()
-    )
-
-    # Convert to dictionaries
-    for setting in settings:
-        # Ensure objects are properly serialized
-        value = setting.value
-
-        # Convert objects to properly formatted JSON strings for display
-        if isinstance(value, (dict, list)) and value:
-            try:
-                # For frontend display, we'll keep objects as they are
-                # The javascript will handle formatting them
-                pass
-            except Exception as e:
-                logger.error(f"Error serializing setting {setting.key}: {e}")
-
-        setting_dict = {
-            "key": setting.key,
-            "value": value,
-            "type": setting.type.value if setting.type else None,
-            "name": setting.name,
-            "description": setting.description,
-            "category": setting.category,
-            "ui_element": setting.ui_element,
-            "options": setting.options,
-            "min_value": setting.min_value,
-            "max_value": setting.max_value,
-            "step": setting.step,
-            "visible": setting.visible,
-            "editable": setting.editable,
-        }
-        settings_list.append(setting_dict)
-
-    return settings_list
-
-
 @settings_bp.route("/", methods=["GET"])
 def settings_page():
     """Main settings dashboard with links to specialized config pages"""
@@ -202,7 +131,6 @@ def save_all_settings():
                 original_values[key] = current_setting.value
 
             # Determine setting type and category
-            setting_type = None
             if key.startswith("llm."):
                 setting_type = SettingType.LLM
                 category = "llm_general"
@@ -231,9 +159,8 @@ def save_all_settings():
                 setting_type = SettingType.APP
                 category = "app_interface"
             else:
-                # Skip keys without a known prefix
-                logger.warning(f"Skipping setting with unknown type: {key}")
-                continue
+                setting_type = None
+                category = None
 
             # Special handling for corrupted or empty values
             if value == "[object Object]" or (
@@ -276,10 +203,6 @@ def save_all_settings():
                 is_valid, error_message = validate_setting(current_setting, value)
 
                 if is_valid:
-                    # Update category if different from our determination
-                    if category and current_setting.category != category:
-                        current_setting.category = category
-
                     # Save the setting
                     success = set_setting(key, value, db_session=db_session)
                     if success:
@@ -425,128 +348,37 @@ def save_all_settings():
         )
 
 
-@settings_bp.route("/reset_to_defaults", methods=["POST"])
+@settings_bp.route("/reset_to_defaults", methods=["GET"])
 def reset_to_defaults():
     """Reset all settings to their default values"""
     db_session = get_db_session()
 
+    # Import default settings from files
     try:
-        # First, delete all existing settings to ensure clean state
-        try:
-            # Get count before deletion
-            settings_count = db_session.query(Setting).count()
-            logger.info(f"Deleting {settings_count} existing settings before reset")
-
-            # Delete all settings
-            db_session.query(Setting).delete()
-            db_session.commit()
-            logger.info("Successfully deleted all existing settings")
-        except Exception as e:
-            logger.error(f"Error deleting existing settings: {e}")
-            db_session.rollback()
-            return (
-                jsonify(
-                    {
-                        "status": "error",
-                        "message": f"Error cleaning existing settings: {str(e)}",
-                    }
-                ),
-                500,
-            )
-
-        # Import default settings from files
-        try:
-            # Import default config files from the defaults directory
-            from importlib.resources import files
-
-            try:
-                defaults_dir = files("local_deep_research.defaults")
-            except ImportError:
-                # Fallback for older Python versions
-                from pkg_resources import resource_filename
+        # Create settings manager for the temporary config
+        settings_mgr = get_settings_manager(db_session)
+        # Import settings from default files
+        settings_mgr.load_from_defaults_file()
 
-                defaults_dir = Path(
-                    resource_filename("local_deep_research", "defaults")
-                )
-
-            logger.info(f"Loading defaults from: {defaults_dir}")
-
-            # Get temporary path to default files
-            import tempfile
-
-            with tempfile.TemporaryDirectory() as temp_dir:
-                # Copy default files to temp directory
-                temp_main = Path(temp_dir) / "settings.toml"
-                temp_search = Path(temp_dir) / "search_engines.toml"
-                temp_collections = Path(temp_dir) / "local_collections.toml"
-
-                # Copy default files (platform independent)
-                import importlib.resources as pkg_resources
-
-                from ... import defaults
-
-                with open(temp_main, "wb") as f:
-                    f.write(pkg_resources.read_binary(defaults, "main.toml"))
-
-                with open(temp_search, "wb") as f:
-                    f.write(pkg_resources.read_binary(defaults, "search_engines.toml"))
-
-                with open(temp_collections, "wb") as f:
-                    f.write(
-                        pkg_resources.read_binary(defaults, "local_collections.toml")
-                    )
-
-                # Create settings manager with temp files
-                # Get configuration directory (not used currently but might be needed in future)
-                # config_dir = get_config_dir() / "config"
-
-                # Create settings manager for the temporary config
-                settings_mgr = get_settings_manager(db_session)
-
-                # Import settings from default files
-                settings_mgr.import_default_settings(
-                    temp_main, temp_search, temp_collections
-                )
-
-                logger.info("Successfully imported settings from default files")
-        except Exception as e:
-            logger.error(f"Error importing default settings: {e}")
-
-            # Fallback to predefined settings if file import fails
-            logger.info("Falling back to predefined settings")
-            # Import here to avoid circular imports
-            from ..database.migrations import (
-                setup_predefined_settings as setup_settings,
-            )
-
-            setup_settings(db_session)
-
-        # Return success
-        return jsonify(
-            {
-                "status": "success",
-                "message": "All settings have been reset to default values",
-            }
-        )
+        logger.info("Successfully imported settings from default files")
 
     except Exception as e:
-        logger.error(f"Error resetting settings to defaults: {e}")
-        return (
-            jsonify(
-                {
-                    "status": "error",
-                    "message": f"Error resetting settings to defaults: {str(e)}",
-                }
-            ),
-            500,
-        )
+        logger.error(f"Error importing default settings: {e}")
 
+        # Fallback to predefined settings if file import fails
+        logger.info("Falling back to predefined settings")
+        # Import here to avoid circular imports
+        from ..database.migrations import setup_predefined_settings as setup_settings
 
-@settings_bp.route("/all_settings", methods=["GET"])
-def get_all_settings_route():
-    """Get all settings for the unified dashboard"""
-    settings_list = get_all_settings_json()
-    return jsonify({"status": "success", "settings": settings_list})
+        setup_settings(db_session)
+
+    # Return success
+    return jsonify(
+        {
+            "status": "success",
+            "message": "All settings have been reset to default values",
+        }
+    )
 
 
 # API Routes
@@ -555,7 +387,6 @@ def api_get_all_settings():
     """Get all settings"""
     try:
         # Get query parameters
-        setting_type = request.args.get("type")
         category = request.args.get("category")
 
         # Create settings manager
@@ -563,14 +394,7 @@ def api_get_all_settings():
         settings_manager = get_settings_manager(db_session)
 
         # Get settings
-        if setting_type:
-            try:
-                setting_type_enum = SettingType[setting_type.upper()]
-                settings = settings_manager.get_all_settings(setting_type_enum)
-            except KeyError:
-                return jsonify({"error": f"Invalid setting type: {setting_type}"}), 400
-        else:
-            settings = settings_manager.get_all_settings()
+        settings = settings_manager.get_all_settings()
 
         # Filter by category if requested
         if category:
@@ -586,7 +410,7 @@ def api_get_all_settings():
 
             settings = filtered_settings
 
-        return jsonify({"settings": settings})
+        return jsonify({"status": "success", "settings": settings})
     except Exception as e:
         logger.error(f"Error getting settings: {e}")
         return jsonify({"error": str(e)}), 500
@@ -741,60 +565,14 @@ def api_delete_setting(key):
         return jsonify({"error": str(e)}), 500
 
 
-@settings_bp.route("/api/export", methods=["POST"])
-def api_export_settings():
-    """Export settings to file"""
-    try:
-        data = request.get_json() or {}
-        setting_type_str = data.get("type")
-
-        db_session = get_db_session()
-        settings_manager = get_settings_manager(db_session)
-
-        # Export settings
-        if setting_type_str:
-            try:
-                setting_type = SettingType[setting_type_str.upper()]
-                success = settings_manager.export_to_file(setting_type)
-            except KeyError:
-                return (
-                    jsonify({"error": f"Invalid setting type: {setting_type_str}"}),
-                    400,
-                )
-        else:
-            success = settings_manager.export_to_file()
-
-        if success:
-            return jsonify({"message": "Settings exported successfully"})
-        else:
-            return jsonify({"error": "Failed to export settings"}), 500
-    except Exception as e:
-        logger.error(f"Error exporting settings: {e}")
-        return jsonify({"error": str(e)}), 500
-
-
 @settings_bp.route("/api/import", methods=["POST"])
 def api_import_settings():
-    """Import settings from file"""
+    """Import settings from defaults file"""
     try:
-        data = request.get_json() or {}
-        setting_type_str = data.get("type")
-
         db_session = get_db_session()
         settings_manager = get_settings_manager(db_session)
 
-        # Import settings
-        if setting_type_str:
-            try:
-                setting_type = SettingType[setting_type_str.upper()]
-                success = settings_manager.import_from_file(setting_type)
-            except KeyError:
-                return (
-                    jsonify({"error": f"Invalid setting type: {setting_type_str}"}),
-                    400,
-                )
-        else:
-            success = settings_manager.import_from_file()
+        success = settings_manager.load_from_defaults_file()
 
         if success:
             return jsonify({"message": "Settings imported successfully"})
@@ -1065,131 +843,67 @@ def api_get_available_models():
 def api_get_available_search_engines():
     """Get available search engines"""
     try:
-        # First try to get engines from search_engines.toml file
-        engines_dict = get_engines_from_file()
-
-        # If we got engines from file, use those
-        if engines_dict:
-            # Make sure searxng is included if it should be
-            if "searxng" not in engines_dict:
-                engines_dict["searxng"] = {
-                    "display_name": "SearXNG (Self-hosted)",
-                    "description": "Self-hosted metasearch engine",
-                    "strengths": ["privacy", "customization", "no API key needed"],
-                }
-
-            # Format as options for dropdown
-            engine_options = [
-                {
-                    "value": key,
-                    "label": engines_dict.get(key, {}).get("display_name", key),
-                }
-                for key in engines_dict.keys()
-            ]
-
-            return jsonify({"engines": engines_dict, "engine_options": engine_options})
+        # Find search engines that are set in the DB.
+        db_session = get_db_session()
+        name_settings = (
+            db_session.query(Setting)
+            .filter(Setting.type == "SEARCH")
+            .filter(Setting.key.startswith("search.engine"))
+            .filter(Setting.key.endswith(".display_name"))
+        ).all()
+
+        # These should all correspond to different search engines.
+        engines_dict = {}
+        for setting in name_settings:
+            key_parts = setting.key.split(".")
+            if key_parts[2] == "auto":
+                # The auto engine is not in the web or local category.
+                engine_name = "auto"
+            else:
+                engine_name = setting.key.split(".")[3]
+            display_name = setting.value
 
-        # Fallback to factory function if file method failed
-        try:
-            # Get available engines
-            search_engines = get_available_engines(include_api_key_services=True)
-
-            # Handle if search_engines is a list (not a dict)
-            if isinstance(search_engines, list):
-                # Convert to dict with engine name as key and display name as value
-                engines_dict = {
-                    engine: engine.replace("_", " ").title()
-                    for engine in search_engines
-                }
+            description = (
+                db_session.query(Setting)
+                .filter(Setting.key == f"search.engine.web.{engine_name}.description")
+                .first()
+            )
+            if description is None:
+                description = ""
             else:
-                engines_dict = search_engines
+                description = description.value
 
-            # Make sure searxng is included
-            if "searxng" not in engines_dict:
-                engines_dict["searxng"] = "SearXNG (Self-hosted)"
+            strengths = (
+                db_session.query(Setting)
+                .filter(Setting.key == f"search.engine.web.{engine_name}.strengths")
+                .first()
+            )
+            if strengths is None:
+                # No strengths in DB.
+                strengths = []
+            else:
+                strengths = strengths.value
 
-            # Format as options for dropdown
-            engine_options = [
-                {
-                    "value": key,
-                    "label": (
-                        value
-                        if isinstance(value, str)
-                        else key.replace("_", " ").title()
-                    ),
-                }
-                for key, value in engines_dict.items()
-            ]
+            engines_dict[engine_name] = dict(
+                display_name=display_name, strengths=strengths, description=description
+            )
 
-            return jsonify({"engines": engines_dict, "engine_options": engine_options})
-        except Exception as e:
-            # If both methods fail, return default engines with searxng
-            logger.error(f"Error getting available search engines from factory: {e}")
-
-            # Use hardcoded defaults from search_engines.toml
-            defaults = {
-                "wikipedia": "Wikipedia",
-                "arxiv": "ArXiv Papers",
-                "pubmed": "PubMed Medical",
-                "github": "GitHub Code",
-                "searxng": "SearXNG (Self-hosted)",
-                "serpapi": "SerpAPI (Google)",
-                "google_pse": "Google PSE",
-                "auto": "Auto-select",
+        # Format as options for dropdown
+        engine_options = [
+            {
+                "value": key,
+                "label": engines_dict.get(key, {}).get("display_name", key),
             }
+            for key in engines_dict.keys()
+        ]
 
-            engine_options = [
-                {"value": key, "label": value} for key, value in defaults.items()
-            ]
+        return jsonify({"engines": engines_dict, "engine_options": engine_options})
 
-            return jsonify({"engines": defaults, "engine_options": engine_options})
     except Exception as e:
         logger.error(f"Error getting available search engines: {e}")
         return jsonify({"error": str(e)}), 500
 
 
-def get_engines_from_file():
-    """Get available search engines directly from the toml file"""
-    try:
-        # Try to load from the actual config directory
-        config_dir = get_config_dir()
-        search_engines_file = config_dir / "config" / "search_engines.toml"
-
-        # If file doesn't exist in user config, try the defaults
-        if not search_engines_file.exists():
-            # Look in the defaults folder instead
-            import inspect
-
-            from ...defaults import search_engines
-
-            # Get the path to the search_engines.toml file
-            module_path = inspect.getfile(search_engines)
-            default_file = Path(module_path)
-
-            if default_file.exists() and default_file.suffix == ".toml":
-                search_engines_file = default_file
-
-        # If we found a file, load it
-        if search_engines_file.exists():
-            data = toml.load(search_engines_file)
-
-            # Filter out the metadata entries (like DEFAULT_SEARCH_ENGINE)
-            engines = {k: v for k, v in data.items() if isinstance(v, dict)}
-
-            # Add display names for each engine
-            for key, engine in engines.items():
-                if "display_name" not in engine:
-                    # Create a display name from the key
-                    engine["display_name"] = key.replace("_", " ").title()
-
-            return engines
-
-        return None
-    except Exception as e:
-        logger.error(f"Error loading search engines from file: {e}")
-        return None
-
-
 # Legacy routes for backward compatibility - these will redirect to the new routes
 @settings_bp.route("/main", methods=["GET"])
 def main_config_page():
@@ -1334,7 +1048,6 @@ def fix_corrupted_settings():
 
         # Search settings
         for key in [
-            "app.research_iterations",
             "app.questions_per_iteration",
             "app.search_engine",
             "app.iterations",
@@ -1472,8 +1185,6 @@ def fix_corrupted_settings():
                     default_value = 10
                 elif setting.key == "search.region":
                     default_value = "us"
-                elif setting.key == "search.research_iterations":
-                    default_value = 2
                 elif setting.key == "search.questions_per_iteration":
                     default_value = 3
                 elif setting.key == "search.searches_per_section":

local_deep_research/web/services/research_service.py

@@ -259,53 +259,53 @@ def run_research_process(
                 f"Overriding system settings with: provider={model_provider}, model={model}, search_engine={search_engine}"
             )
 
-            # Override LLM if model or model_provider specified
-            if model or model_provider:
-                try:
-                    # Get LLM with the overridden settings
-                    # Explicitly create the model with parameters to avoid fallback issues
-                    use_llm = get_llm(
-                        model_name=model,
-                        provider=model_provider,
-                        openai_endpoint_url=custom_endpoint,
-                    )
+        # Override LLM if model or model_provider specified
+        if model or model_provider:
+            try:
+                # Get LLM with the overridden settings
+                # Explicitly create the model with parameters to avoid fallback issues
+                use_llm = get_llm(
+                    model_name=model,
+                    provider=model_provider,
+                    openai_endpoint_url=custom_endpoint,
+                )
 
-                    logger.info(
-                        "Successfully set LLM to: provider=%s, model=%s",
-                        model_provider,
-                        model,
-                    )
-                except Exception as e:
-                    logger.error(
-                        "Error setting LLM provider=%s, model=%s: %s",
-                        model_provider,
-                        model,
-                        str(e),
-                    )
-                    logger.error(traceback.format_exc())
+                logger.info(
+                    "Successfully set LLM to: provider=%s, model=%s",
+                    model_provider,
+                    model,
+                )
+            except Exception as e:
+                logger.error(
+                    "Error setting LLM provider=%s, model=%s: %s",
+                    model_provider,
+                    model,
+                    str(e),
+                )
+                logger.error(traceback.format_exc())
 
-            # Set the progress callback in the system
-            system = AdvancedSearchSystem(llm=use_llm)
-            system.set_progress_callback(progress_callback)
+        # Set the progress callback in the system
+        system = AdvancedSearchSystem(llm=use_llm)
+        system.set_progress_callback(progress_callback)
 
-            # Override search engine if specified
-            if search_engine:
-                try:
-                    if iterations:
-                        system.max_iterations = int(iterations)
-                    if questions_per_iteration:
-                        system.questions_per_iteration = int(questions_per_iteration)
-
-                    # Create a new search object with these settings
-                    system.search = get_search(
-                        search_tool=search_engine, llm_instance=system.model
-                    )
+        # Override search engine if specified
+        if search_engine:
+            try:
+                if iterations:
+                    system.max_iterations = int(iterations)
+                if questions_per_iteration:
+                    system.questions_per_iteration = int(questions_per_iteration)
+
+                # Create a new search object with these settings
+                system.search = get_search(
+                    search_tool=search_engine, llm_instance=system.model
+                )
 
-                    logger.info("Successfully set search engine to: %s", search_engine)
-                except Exception as e:
-                    logger.error(
-                        "Error setting search engine to %s: %s", search_engine, str(e)
-                    )
+                logger.info("Successfully set search engine to: %s", search_engine)
+            except Exception as e:
+                logger.error(
+                    "Error setting search engine to %s: %s", search_engine, str(e)
+                )
 
         # Run the search
         progress_callback("Starting research process", 5, {"phase": "init"})