local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web/routes/settings_routes.py

@@ -0,0 +1,1592 @@
+import json
+import logging
+import os
+import platform
+import subprocess
+from pathlib import Path
+from typing import Any, Optional, Tuple
+
+import requests
+import toml
+from flask import (
+    Blueprint,
+    current_app,
+    flash,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    url_for,
+)
+from flask_wtf.csrf import generate_csrf
+from sqlalchemy.orm import Session
+
+from ...config.config_files import get_config_dir
+from ...web_search_engines.search_engine_factory import get_available_engines
+from ..database.models import Setting, SettingType
+from ..services.settings_service import (
+    create_or_update_setting,
+    get_setting,
+    get_settings_manager,
+    set_setting,
+)
+
+# Initialize logger
+logger = logging.getLogger(__name__)
+
+# Create a Blueprint for settings
+settings_bp = Blueprint("settings", __name__, url_prefix="/research/settings")
+
+# Legacy config for backwards compatibility
+SEARCH_ENGINES_FILE = None
+CONFIG_DIR = None
+MAIN_CONFIG_FILE = None
+LOCAL_COLLECTIONS_FILE = None
+
+
+def set_config_paths(
+    config_dir, search_engines_file, main_config_file, local_collections_file
+):
+    """Set the config paths for the settings routes (legacy support)"""
+    global CONFIG_DIR, SEARCH_ENGINES_FILE, MAIN_CONFIG_FILE, LOCAL_COLLECTIONS_FILE
+    CONFIG_DIR = config_dir
+    SEARCH_ENGINES_FILE = search_engines_file
+    MAIN_CONFIG_FILE = main_config_file
+    LOCAL_COLLECTIONS_FILE = local_collections_file
+
+
+def get_db_session() -> Session:
+    """Get the database session from the app context"""
+    if hasattr(current_app, "db_session"):
+        return current_app.db_session
+    else:
+        return current_app.extensions["sqlalchemy"].session()
+
+
+def validate_setting(setting: Setting, value: Any) -> Tuple[bool, Optional[str]]:
+    """
+    Validate a setting value based on its type and constraints.
+
+    Args:
+        setting: The Setting object to validate against
+        value: The value to validate
+
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    # Convert value based on UI element type
+    if setting.ui_element == "checkbox":
+        # Convert string representations of boolean to actual boolean
+        if isinstance(value, str):
+            value = value.lower() in ("true", "on", "yes", "1")
+    elif setting.ui_element == "number" or setting.ui_element == "slider":
+        try:
+            value = float(value)
+        except (ValueError, TypeError):
+            return False, "Value must be a number"
+
+        # Check min/max constraints if defined
+        if setting.min_value is not None and value < setting.min_value:
+            return False, f"Value must be at least {setting.min_value}"
+        if setting.max_value is not None and value > setting.max_value:
+            return False, f"Value must be at most {setting.max_value}"
+    elif setting.ui_element == "select":
+        # Check if value is in the allowed options
+        if setting.options:
+            # Skip options validation for dynamically populated dropdowns
+            if setting.key not in ["llm.provider", "llm.model"]:
+                allowed_values = [opt.get("value") for opt in setting.options]
+                if value not in allowed_values:
+                    return (
+                        False,
+                        f"Value must be one of: {', '.join(str(v) for v in allowed_values)}",
+                    )
+    # All checks passed
+    return True, None
+
+
+def get_all_settings_json():
+    """Get all settings as a JSON-serializable dictionary
+
+    Returns:
+        List of setting dictionaries
+    """
+    db_session = get_db_session()
+    settings_list = []
+
+    # Get all settings
+    settings = (
+        db_session.query(Setting)
+        .order_by(Setting.type, Setting.category, Setting.name)
+        .all()
+    )
+
+    # Convert to dictionaries
+    for setting in settings:
+        # Ensure objects are properly serialized
+        value = setting.value
+
+        # Convert objects to properly formatted JSON strings for display
+        if isinstance(value, (dict, list)) and value:
+            try:
+                # For frontend display, we'll keep objects as they are
+                # The javascript will handle formatting them
+                pass
+            except Exception as e:
+                logger.error(f"Error serializing setting {setting.key}: {e}")
+
+        setting_dict = {
+            "key": setting.key,
+            "value": value,
+            "type": setting.type.value if setting.type else None,
+            "name": setting.name,
+            "description": setting.description,
+            "category": setting.category,
+            "ui_element": setting.ui_element,
+            "options": setting.options,
+            "min_value": setting.min_value,
+            "max_value": setting.max_value,
+            "step": setting.step,
+            "visible": setting.visible,
+            "editable": setting.editable,
+        }
+        settings_list.append(setting_dict)
+
+    return settings_list
+
+
+@settings_bp.route("/", methods=["GET"])
+def settings_page():
+    """Main settings dashboard with links to specialized config pages"""
+    return render_template("settings_dashboard.html")
+
+
+@settings_bp.route("/save_all_settings", methods=["POST"])
+def save_all_settings():
+    """Handle saving all settings at once from the unified settings page"""
+    db_session = get_db_session()
+    # Get the settings manager but we don't need to assign it to a variable right now
+    # get_settings_manager(db_session)
+
+    try:
+        # Process JSON data
+        form_data = request.get_json()
+        if not form_data:
+            return (
+                jsonify({"status": "error", "message": "No settings data provided"}),
+                400,
+            )
+
+        # Track validation errors
+        validation_errors = []
+        settings_by_type = {}
+
+        # Track changes for logging
+        updated_settings = []
+        created_settings = []
+
+        # Store original values for better messaging
+        original_values = {}
+
+        # Update each setting
+        for key, value in form_data.items():
+            # Skip corrupted keys or empty strings as keys
+            if not key or not isinstance(key, str) or key.strip() == "":
+                continue
+
+            # Get the original value
+            current_setting = (
+                db_session.query(Setting).filter(Setting.key == key).first()
+            )
+            if current_setting:
+                original_values[key] = current_setting.value
+
+            # Determine setting type and category
+            setting_type = None
+            if key.startswith("llm."):
+                setting_type = SettingType.LLM
+                category = "llm_general"
+                if (
+                    "temperature" in key
+                    or "max_tokens" in key
+                    or "batch" in key
+                    or "layers" in key
+                ):
+                    category = "llm_parameters"
+            elif key.startswith("search."):
+                setting_type = SettingType.SEARCH
+                category = "search_general"
+                if (
+                    "iterations" in key
+                    or "results" in key
+                    or "region" in key
+                    or "questions" in key
+                    or "section" in key
+                ):
+                    category = "search_parameters"
+            elif key.startswith("report."):
+                setting_type = SettingType.REPORT
+                category = "report_parameters"
+            elif key.startswith("app."):
+                setting_type = SettingType.APP
+                category = "app_interface"
+            else:
+                # Skip keys without a known prefix
+                logger.warning(f"Skipping setting with unknown type: {key}")
+                continue
+
+            # Special handling for corrupted or empty values
+            if value == "[object Object]" or (
+                isinstance(value, str) and value.strip() in ["{}", "[]", "{", "["]
+            ):
+                if key.startswith("report."):
+                    value = {}
+                else:
+                    # Use default or null for other types
+                    if key == "llm.model":
+                        value = "gpt-3.5-turbo"
+                    elif key == "llm.provider":
+                        value = "openai"
+                    elif key == "search.tool":
+                        value = "auto"
+                    elif key in ["app.theme", "app.default_theme"]:
+                        value = "dark"
+                    else:
+                        value = None
+
+                logger.warning(f"Corrected corrupted value for {key}: {value}")
+
+                # Handle JSON string values (already parsed by JavaScript)
+                if isinstance(value, (dict, list)):
+                    # Keep as is, already parsed
+                    pass
+                # Handle string values that might be JSON
+                elif isinstance(value, str) and (
+                    value.startswith("{") or value.startswith("[")
+                ):
+                    try:
+                        # Try to parse the string as JSON
+                        value = json.loads(value)
+                    except json.JSONDecodeError:
+                        # If it fails to parse, keep as string
+                        pass
+
+            if current_setting:
+                # Validate the setting
+                is_valid, error_message = validate_setting(current_setting, value)
+
+                if is_valid:
+                    # Update category if different from our determination
+                    if category and current_setting.category != category:
+                        current_setting.category = category
+
+                    # Save the setting
+                    success = set_setting(key, value, db_session=db_session)
+                    if success:
+                        updated_settings.append(key)
+
+                    # Track settings by type for exporting
+                    if current_setting.type not in settings_by_type:
+                        settings_by_type[current_setting.type] = []
+                    settings_by_type[current_setting.type].append(current_setting)
+                else:
+                    # Add to validation errors
+                    validation_errors.append(
+                        {
+                            "key": key,
+                            "name": current_setting.name,
+                            "error": error_message,
+                        }
+                    )
+            else:
+                # Create a new setting
+                new_setting = {
+                    "key": key,
+                    "value": value,
+                    "type": setting_type.value.lower(),
+                    "name": key.split(".")[-1].replace("_", " ").title(),
+                    "description": f"Setting for {key}",
+                    "category": category,
+                    "ui_element": "text",  # Default UI element
+                }
+
+                # Determine better UI element based on value type
+                if isinstance(value, bool):
+                    new_setting["ui_element"] = "checkbox"
+                elif isinstance(value, (int, float)) and not isinstance(value, bool):
+                    new_setting["ui_element"] = "number"
+                elif isinstance(value, (dict, list)):
+                    new_setting["ui_element"] = "textarea"
+
+                # Create the setting
+                db_setting = create_or_update_setting(new_setting)
+
+                if db_setting:
+                    created_settings.append(key)
+                    # Track settings by type for exporting
+                    if db_setting.type not in settings_by_type:
+                        settings_by_type[db_setting.type] = []
+                    settings_by_type[db_setting.type].append(db_setting)
+                else:
+                    validation_errors.append(
+                        {
+                            "key": key,
+                            "name": new_setting["name"],
+                            "error": "Failed to create setting",
+                        }
+                    )
+
+        # Report validation errors if any
+        if validation_errors:
+            return (
+                jsonify(
+                    {
+                        "status": "error",
+                        "message": "Validation errors",
+                        "errors": validation_errors,
+                    }
+                ),
+                400,
+            )
+
+        # Export settings to file for each type
+        for setting_type in settings_by_type:
+            get_settings_manager(db_session).export_to_file(setting_type)
+
+        # Get all settings to return to the client for proper state update
+        all_settings = []
+        for setting in db_session.query(Setting).all():
+            # Convert enum to string if present
+            setting_type = setting.type
+            if hasattr(setting_type, "value"):
+                setting_type = setting_type.value
+
+            all_settings.append(
+                {
+                    "key": setting.key,
+                    "value": setting.value,
+                    "name": setting.name,
+                    "description": setting.description,
+                    "type": setting_type,
+                    "category": setting.category,
+                    "ui_element": setting.ui_element,
+                    "editable": setting.editable,
+                    "options": setting.options,
+                }
+            )
+
+        # Customize the success message based on what changed
+        success_message = ""
+        if len(updated_settings) == 1:
+            # For a single update, provide more specific info about what changed
+            key = updated_settings[0]
+            updated_setting = (
+                db_session.query(Setting).filter(Setting.key == key).first()
+            )
+            name = (
+                updated_setting.name
+                if updated_setting
+                else key.split(".")[-1].replace("_", " ").title()
+            )
+
+            # Format the message
+            if key in original_values:
+                # Get original value but comment out if not used
+                # old_value = original_values[key]
+                new_value = updated_setting.value if updated_setting else None
+
+                # If it's a boolean, use "enabled/disabled" language
+                if isinstance(new_value, bool):
+                    state = "enabled" if new_value else "disabled"
+                    success_message = f"{name} {state}"
+                else:
+                    # For non-boolean values
+                    if isinstance(new_value, (dict, list)):
+                        success_message = f"{name} updated"
+                    else:
+                        success_message = f"{name} updated"
+            else:
+                success_message = f"{name} updated"
+        else:
+            # Multiple settings or generic message
+            success_message = f"Settings saved successfully ({len(updated_settings)} updated, {len(created_settings)} created)"
+
+        return jsonify(
+            {
+                "status": "success",
+                "message": success_message,
+                "updated": updated_settings,
+                "created": created_settings,
+                "settings": all_settings,
+            }
+        )
+
+    except Exception as e:
+        logger.error(f"Error saving settings: {e}")
+        return (
+            jsonify({"status": "error", "message": f"Error saving settings: {str(e)}"}),
+            500,
+        )
+
+
+@settings_bp.route("/reset_to_defaults", methods=["POST"])
+def reset_to_defaults():
+    """Reset all settings to their default values"""
+    db_session = get_db_session()
+
+    try:
+        # First, delete all existing settings to ensure clean state
+        try:
+            # Get count before deletion
+            settings_count = db_session.query(Setting).count()
+            logger.info(f"Deleting {settings_count} existing settings before reset")
+
+            # Delete all settings
+            db_session.query(Setting).delete()
+            db_session.commit()
+            logger.info("Successfully deleted all existing settings")
+        except Exception as e:
+            logger.error(f"Error deleting existing settings: {e}")
+            db_session.rollback()
+            return (
+                jsonify(
+                    {
+                        "status": "error",
+                        "message": f"Error cleaning existing settings: {str(e)}",
+                    }
+                ),
+                500,
+            )
+
+        # Import default settings from files
+        try:
+            # Import default config files from the defaults directory
+            from importlib.resources import files
+
+            try:
+                defaults_dir = files("local_deep_research.defaults")
+            except ImportError:
+                # Fallback for older Python versions
+                from pkg_resources import resource_filename
+
+                defaults_dir = Path(
+                    resource_filename("local_deep_research", "defaults")
+                )
+
+            logger.info(f"Loading defaults from: {defaults_dir}")
+
+            # Get temporary path to default files
+            import tempfile
+
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # Copy default files to temp directory
+                temp_main = Path(temp_dir) / "settings.toml"
+                temp_search = Path(temp_dir) / "search_engines.toml"
+                temp_collections = Path(temp_dir) / "local_collections.toml"
+
+                # Copy default files (platform independent)
+                import importlib.resources as pkg_resources
+
+                from ... import defaults
+
+                with open(temp_main, "wb") as f:
+                    f.write(pkg_resources.read_binary(defaults, "main.toml"))
+
+                with open(temp_search, "wb") as f:
+                    f.write(pkg_resources.read_binary(defaults, "search_engines.toml"))
+
+                with open(temp_collections, "wb") as f:
+                    f.write(
+                        pkg_resources.read_binary(defaults, "local_collections.toml")
+                    )
+
+                # Create settings manager with temp files
+                # Get configuration directory (not used currently but might be needed in future)
+                # config_dir = get_config_dir() / "config"
+
+                # Create settings manager for the temporary config
+                settings_mgr = get_settings_manager(db_session)
+
+                # Import settings from default files
+                settings_mgr.import_default_settings(
+                    temp_main, temp_search, temp_collections
+                )
+
+                logger.info("Successfully imported settings from default files")
+        except Exception as e:
+            logger.error(f"Error importing default settings: {e}")
+
+            # Fallback to predefined settings if file import fails
+            logger.info("Falling back to predefined settings")
+            # Import here to avoid circular imports
+            from ..database.migrations import (
+                setup_predefined_settings as setup_settings,
+            )
+
+            setup_settings(db_session)
+
+        # Also export the settings to file for consistency
+        settings_mgr = get_settings_manager(db_session)
+        for setting_type in SettingType:
+            settings_mgr.export_to_file(setting_type)
+
+        # Return success
+        return jsonify(
+            {
+                "status": "success",
+                "message": "All settings have been reset to default values",
+            }
+        )
+
+    except Exception as e:
+        logger.error(f"Error resetting settings to defaults: {e}")
+        return (
+            jsonify(
+                {
+                    "status": "error",
+                    "message": f"Error resetting settings to defaults: {str(e)}",
+                }
+            ),
+            500,
+        )
+
+
+@settings_bp.route("/all_settings", methods=["GET"])
+def get_all_settings_route():
+    """Get all settings for the unified dashboard"""
+    settings_list = get_all_settings_json()
+    return jsonify({"status": "success", "settings": settings_list})
+
+
+# API Routes
+@settings_bp.route("/api", methods=["GET"])
+def api_get_all_settings():
+    """Get all settings"""
+    try:
+        # Get query parameters
+        setting_type = request.args.get("type")
+        category = request.args.get("category")
+
+        # Create settings manager
+        db_session = get_db_session()
+        settings_manager = get_settings_manager(db_session)
+
+        # Get settings
+        if setting_type:
+            try:
+                setting_type_enum = SettingType[setting_type.upper()]
+                settings = settings_manager.get_all_settings(setting_type_enum)
+            except KeyError:
+                return jsonify({"error": f"Invalid setting type: {setting_type}"}), 400
+        else:
+            settings = settings_manager.get_all_settings()
+
+        # Filter by category if requested
+        if category:
+            filtered_settings = {}
+            # Need to get all setting details to check category
+            db_settings = db_session.query(Setting).all()
+            category_keys = [s.key for s in db_settings if s.category == category]
+
+            # Filter settings by keys
+            for key, value in settings.items():
+                if key in category_keys:
+                    filtered_settings[key] = value
+
+            settings = filtered_settings
+
+        return jsonify({"settings": settings})
+    except Exception as e:
+        logger.error(f"Error getting settings: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/<path:key>", methods=["GET"])
+def api_get_setting(key):
+    """Get a specific setting by key"""
+    try:
+        db_session = get_db_session()
+        # No need to assign if not used
+        # get_settings_manager(db_session)
+
+        # Get setting
+        value = get_setting(key, db_session=db_session)
+        if value is None:
+            return jsonify({"error": f"Setting not found: {key}"}), 404
+
+        # Get additional metadata from database.
+        db_setting = db_session.query(Setting).filter(Setting.key == key).first()
+
+        if db_setting:
+            # Return full setting details
+            setting_data = {
+                "key": db_setting.key,
+                "value": db_setting.value,
+                "type": db_setting.type.value,
+                "name": db_setting.name,
+                "description": db_setting.description,
+                "category": db_setting.category,
+                "ui_element": db_setting.ui_element,
+                "options": db_setting.options,
+                "min_value": db_setting.min_value,
+                "max_value": db_setting.max_value,
+                "step": db_setting.step,
+                "visible": db_setting.visible,
+                "editable": db_setting.editable,
+            }
+        else:
+            # Return minimal info
+            setting_data = {"key": key, "value": value}
+
+        return jsonify({"settings": setting_data})
+    except Exception as e:
+        logger.error(f"Error getting setting {key}: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/<path:key>", methods=["PUT"])
+def api_update_setting(key):
+    """Update a setting"""
+    try:
+        # Get request data
+        data = request.get_json()
+        if not data:
+            return jsonify({"error": "No data provided"}), 400
+
+        value = data.get("value")
+        if value is None:
+            return jsonify({"error": "No value provided"}), 400
+
+        # Get DB session and settings manager
+        db_session = get_db_session()
+        # Only use settings_manager if needed - we don't need to assign if not used
+        # get_settings_manager(db_session)
+
+        # Check if setting exists
+        db_setting = db_session.query(Setting).filter(Setting.key == key).first()
+
+        if db_setting:
+            # Check if setting is editable
+            if not db_setting.editable:
+                return jsonify({"error": f"Setting {key} is not editable"}), 403
+
+            # Update setting
+            success = set_setting(key, value)
+            if success:
+                return jsonify({"message": f"Setting {key} updated successfully"})
+            else:
+                return jsonify({"error": f"Failed to update setting {key}"}), 500
+        else:
+            # Create new setting with default metadata
+            setting_dict = {
+                "key": key,
+                "value": value,
+                "name": key.split(".")[-1].replace("_", " ").title(),
+                "description": f"Setting for {key}",
+            }
+
+            # Add additional metadata if provided
+            for field in [
+                "type",
+                "name",
+                "description",
+                "category",
+                "ui_element",
+                "options",
+                "min_value",
+                "max_value",
+                "step",
+                "visible",
+                "editable",
+            ]:
+                if field in data:
+                    setting_dict[field] = data[field]
+
+            # Create setting
+            db_setting = create_or_update_setting(setting_dict)
+
+            if db_setting:
+                return (
+                    jsonify(
+                        {
+                            "message": f"Setting {key} created successfully",
+                            "setting": {
+                                "key": db_setting.key,
+                                "value": db_setting.value,
+                                "type": db_setting.type.value,
+                                "name": db_setting.name,
+                            },
+                        }
+                    ),
+                    201,
+                )
+            else:
+                return jsonify({"error": f"Failed to create setting {key}"}), 500
+    except Exception as e:
+        logger.error(f"Error updating setting {key}: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/<path:key>", methods=["DELETE"])
+def api_delete_setting(key):
+    """Delete a setting"""
+    try:
+        db_session = get_db_session()
+        settings_manager = get_settings_manager(db_session)
+
+        # Check if setting exists
+        db_setting = db_session.query(Setting).filter(Setting.key == key).first()
+        if not db_setting:
+            return jsonify({"error": f"Setting not found: {key}"}), 404
+
+        # Delete setting
+        success = settings_manager.delete_setting(key)
+        if success:
+            return jsonify({"message": f"Setting {key} deleted successfully"})
+        else:
+            return jsonify({"error": f"Failed to delete setting {key}"}), 500
+    except Exception as e:
+        logger.error(f"Error deleting setting {key}: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/export", methods=["POST"])
+def api_export_settings():
+    """Export settings to file"""
+    try:
+        data = request.get_json() or {}
+        setting_type_str = data.get("type")
+
+        db_session = get_db_session()
+        settings_manager = get_settings_manager(db_session)
+
+        # Export settings
+        if setting_type_str:
+            try:
+                setting_type = SettingType[setting_type_str.upper()]
+                success = settings_manager.export_to_file(setting_type)
+            except KeyError:
+                return (
+                    jsonify({"error": f"Invalid setting type: {setting_type_str}"}),
+                    400,
+                )
+        else:
+            success = settings_manager.export_to_file()
+
+        if success:
+            return jsonify({"message": "Settings exported successfully"})
+        else:
+            return jsonify({"error": "Failed to export settings"}), 500
+    except Exception as e:
+        logger.error(f"Error exporting settings: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/import", methods=["POST"])
+def api_import_settings():
+    """Import settings from file"""
+    try:
+        data = request.get_json() or {}
+        setting_type_str = data.get("type")
+
+        db_session = get_db_session()
+        settings_manager = get_settings_manager(db_session)
+
+        # Import settings
+        if setting_type_str:
+            try:
+                setting_type = SettingType[setting_type_str.upper()]
+                success = settings_manager.import_from_file(setting_type)
+            except KeyError:
+                return (
+                    jsonify({"error": f"Invalid setting type: {setting_type_str}"}),
+                    400,
+                )
+        else:
+            success = settings_manager.import_from_file()
+
+        if success:
+            return jsonify({"message": "Settings imported successfully"})
+        else:
+            return jsonify({"error": "Failed to import settings"}), 500
+    except Exception as e:
+        logger.error(f"Error importing settings: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/categories", methods=["GET"])
+def api_get_categories():
+    """Get all setting categories"""
+    try:
+        db_session = get_db_session()
+
+        # Get all distinct categories
+        categories = db_session.query(Setting.category).distinct().all()
+        category_list = [c[0] for c in categories if c[0] is not None]
+
+        return jsonify({"categories": category_list})
+    except Exception as e:
+        logger.error(f"Error getting categories: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/types", methods=["GET"])
+def api_get_types():
+    """Get all setting types"""
+    try:
+        # Get all setting types
+        types = [t.value for t in SettingType]
+        return jsonify({"types": types})
+    except Exception as e:
+        logger.error(f"Error getting types: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/ui_elements", methods=["GET"])
+def api_get_ui_elements():
+    """Get all UI element types"""
+    try:
+        # Define supported UI element types
+        ui_elements = [
+            "text",
+            "select",
+            "checkbox",
+            "slider",
+            "number",
+            "textarea",
+            "color",
+            "date",
+            "file",
+            "password",
+        ]
+
+        return jsonify({"ui_elements": ui_elements})
+    except Exception as e:
+        logger.error(f"Error getting UI elements: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+@settings_bp.route("/api/available-models", methods=["GET"])
+def api_get_available_models():
+    """Get available LLM models from various providers"""
+    try:
+        # Define provider options with generic provider names
+        provider_options = [
+            {"value": "OLLAMA", "label": "Ollama (Local)"},
+            {"value": "OPENAI", "label": "OpenAI (Cloud)"},
+            {"value": "ANTHROPIC", "label": "Anthropic (Cloud)"},
+            {"value": "OPENAI_ENDPOINT", "label": "Custom OpenAI Endpoint"},
+            {"value": "VLLM", "label": "vLLM (Local)"},
+            {"value": "LMSTUDIO", "label": "LM Studio (Local)"},
+            {"value": "LLAMACPP", "label": "Llama.cpp (Local)"},
+        ]
+
+        # Available models by provider
+        providers = {}
+
+        # Try to get Ollama models
+        ollama_models = []
+        try:
+            import json
+            import re
+
+            import requests
+            from flask import current_app
+
+            # Try to query the Ollama API directly
+            try:
+                current_app.logger.info("Attempting to connect to Ollama API")
+
+                base_url = os.getenv(
+                    "OLLAMA_BASE_URL",
+                    "http://localhost:11434",
+                )
+                ollama_response = requests.get(f"{base_url}/api/tags", timeout=5)
+
+                current_app.logger.debug(
+                    f"Ollama API response: Status {ollama_response.status_code}"
+                )
+
+                # Try to parse the response even if status code is not 200 to help with debugging
+                response_text = ollama_response.text
+                current_app.logger.debug(
+                    f"Ollama API raw response: {response_text[:500]}..."
+                )
+
+                if ollama_response.status_code == 200:
+                    try:
+                        ollama_data = ollama_response.json()
+                        current_app.logger.debug(
+                            f"Ollama API JSON data: {json.dumps(ollama_data)[:500]}..."
+                        )
+
+                        if "models" in ollama_data:
+                            # Format for newer Ollama API
+                            current_app.logger.info(
+                                f"Found {len(ollama_data.get('models', []))} models in newer Ollama API format"
+                            )
+                            for model in ollama_data.get("models", []):
+                                # Extract name correctly from the model object
+                                name = model.get("name", "")
+                                if name:
+                                    # Improved display name formatting
+                                    display_name = re.sub(r"[:/]", " ", name).strip()
+                                    display_name = " ".join(
+                                        word.capitalize()
+                                        for word in display_name.split()
+                                    )
+                                    # Create the model entry with value and label
+                                    ollama_models.append(
+                                        {
+                                            "value": name,  # Original model name as value (for API calls)
+                                            "label": f"{display_name} (Ollama)",  # Pretty name as label
+                                            "provider": "OLLAMA",  # Add provider field for consistency
+                                        }
+                                    )
+                                    current_app.logger.debug(
+                                        f"Added Ollama model: {name} -> {display_name}"
+                                    )
+                        else:
+                            # Format for older Ollama API
+                            current_app.logger.info(
+                                f"Found {len(ollama_data)} models in older Ollama API format"
+                            )
+                            for model in ollama_data:
+                                name = model.get("name", "")
+                                if name:
+                                    # Improved display name formatting
+                                    display_name = re.sub(r"[:/]", " ", name).strip()
+                                    display_name = " ".join(
+                                        word.capitalize()
+                                        for word in display_name.split()
+                                    )
+                                    ollama_models.append(
+                                        {
+                                            "value": name,
+                                            "label": f"{display_name} (Ollama)",
+                                            "provider": "OLLAMA",  # Add provider field for consistency
+                                        }
+                                    )
+                                    current_app.logger.debug(
+                                        f"Added Ollama model: {name} -> {display_name}"
+                                    )
+
+                        # Sort models alphabetically
+                        ollama_models.sort(key=lambda x: x["label"])
+
+                    except json.JSONDecodeError as json_err:
+                        current_app.logger.error(
+                            f"Failed to parse Ollama API response as JSON: {json_err}"
+                        )
+                        raise Exception(f"Ollama API returned invalid JSON: {json_err}")
+                else:
+                    current_app.logger.warning(
+                        f"Ollama API returned non-200 status code: {ollama_response.status_code}"
+                    )
+                    raise Exception(
+                        f"Ollama API returned status code {ollama_response.status_code}"
+                    )
+
+            except requests.exceptions.RequestException as e:
+                current_app.logger.warning(f"Could not connect to Ollama API: {str(e)}")
+                # Fallback to default models if Ollama is not running
+                current_app.logger.info(
+                    "Using fallback Ollama models due to connection error"
+                )
+                ollama_models = [
+                    {
+                        "value": "llama3",
+                        "label": "Llama 3 (Ollama)",
+                        "provider": "OLLAMA",
+                    },
+                    {
+                        "value": "mistral",
+                        "label": "Mistral (Ollama)",
+                        "provider": "OLLAMA",
+                    },
+                    {
+                        "value": "gemma:latest",
+                        "label": "Gemma (Ollama)",
+                        "provider": "OLLAMA",
+                    },
+                ]
+
+            # Always set the ollama_models in providers, whether we got real or fallback models
+            providers["ollama_models"] = ollama_models
+            current_app.logger.info(f"Final Ollama models count: {len(ollama_models)}")
+
+            # Log some model names for debugging
+            if ollama_models:
+                model_names = [m["value"] for m in ollama_models[:5]]
+                current_app.logger.info(
+                    f"Sample Ollama models: {', '.join(model_names)}"
+                )
+
+        except Exception as e:
+            current_app.logger.error(f"Error getting Ollama models: {str(e)}")
+            # Use fallback models
+            current_app.logger.info("Using fallback Ollama models due to error")
+            providers["ollama_models"] = [
+                {"value": "llama3", "label": "Llama 3 (Ollama)", "provider": "OLLAMA"},
+                {"value": "mistral", "label": "Mistral (Ollama)", "provider": "OLLAMA"},
+                {
+                    "value": "gemma:latest",
+                    "label": "Gemma (Ollama)",
+                    "provider": "OLLAMA",
+                },
+            ]
+
+        # Add OpenAI models
+        providers["openai_models"] = [
+            {"value": "gpt-4o", "label": "GPT-4o (OpenAI)"},
+            {"value": "gpt-4", "label": "GPT-4 (OpenAI)"},
+            {"value": "gpt-3.5-turbo", "label": "GPT-3.5 Turbo (OpenAI)"},
+        ]
+
+        # Add Anthropic models
+        providers["anthropic_models"] = [
+            {
+                "value": "claude-3-5-sonnet-latest",
+                "label": "Claude 3.5 Sonnet (Anthropic)",
+            },
+            {"value": "claude-3-opus-20240229", "label": "Claude 3 Opus (Anthropic)"},
+            {
+                "value": "claude-3-sonnet-20240229",
+                "label": "Claude 3 Sonnet (Anthropic)",
+            },
+            {"value": "claude-3-haiku-20240307", "label": "Claude 3 Haiku (Anthropic)"},
+        ]
+
+        # Return all options
+        return jsonify({"provider_options": provider_options, "providers": providers})
+
+    except Exception as e:
+        import traceback
+
+        error_trace = traceback.format_exc()
+        current_app.logger.error(
+            f"Error getting available models: {str(e)}\n{error_trace}"
+        )
+        return jsonify({"status": "error", "message": str(e)}), 500
+
+
+@settings_bp.route("/api/available-search-engines", methods=["GET"])
+def api_get_available_search_engines():
+    """Get available search engines"""
+    try:
+        # First try to get engines from search_engines.toml file
+        engines_dict = get_engines_from_file()
+
+        # If we got engines from file, use those
+        if engines_dict:
+            # Make sure searxng is included if it should be
+            if "searxng" not in engines_dict:
+                engines_dict["searxng"] = {
+                    "display_name": "SearXNG (Self-hosted)",
+                    "description": "Self-hosted metasearch engine",
+                    "strengths": ["privacy", "customization", "no API key needed"],
+                }
+
+            # Format as options for dropdown
+            engine_options = [
+                {
+                    "value": key,
+                    "label": engines_dict.get(key, {}).get("display_name", key),
+                }
+                for key in engines_dict.keys()
+            ]
+
+            return jsonify({"engines": engines_dict, "engine_options": engine_options})
+
+        # Fallback to factory function if file method failed
+        try:
+            # Get available engines
+            search_engines = get_available_engines(include_api_key_services=True)
+
+            # Handle if search_engines is a list (not a dict)
+            if isinstance(search_engines, list):
+                # Convert to dict with engine name as key and display name as value
+                engines_dict = {
+                    engine: engine.replace("_", " ").title()
+                    for engine in search_engines
+                }
+            else:
+                engines_dict = search_engines
+
+            # Make sure searxng is included
+            if "searxng" not in engines_dict:
+                engines_dict["searxng"] = "SearXNG (Self-hosted)"
+
+            # Format as options for dropdown
+            engine_options = [
+                {
+                    "value": key,
+                    "label": (
+                        value
+                        if isinstance(value, str)
+                        else key.replace("_", " ").title()
+                    ),
+                }
+                for key, value in engines_dict.items()
+            ]
+
+            return jsonify({"engines": engines_dict, "engine_options": engine_options})
+        except Exception as e:
+            # If both methods fail, return default engines with searxng
+            logger.error(f"Error getting available search engines from factory: {e}")
+
+            # Use hardcoded defaults from search_engines.toml
+            defaults = {
+                "wikipedia": "Wikipedia",
+                "arxiv": "ArXiv Papers",
+                "pubmed": "PubMed Medical",
+                "github": "GitHub Code",
+                "searxng": "SearXNG (Self-hosted)",
+                "serpapi": "SerpAPI (Google)",
+                "google_pse": "Google PSE",
+                "auto": "Auto-select",
+            }
+
+            engine_options = [
+                {"value": key, "label": value} for key, value in defaults.items()
+            ]
+
+            return jsonify({"engines": defaults, "engine_options": engine_options})
+    except Exception as e:
+        logger.error(f"Error getting available search engines: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
+def get_engines_from_file():
+    """Get available search engines directly from the toml file"""
+    try:
+        # Try to load from the actual config directory
+        config_dir = get_config_dir()
+        search_engines_file = config_dir / "config" / "search_engines.toml"
+
+        # If file doesn't exist in user config, try the defaults
+        if not search_engines_file.exists():
+            # Look in the defaults folder instead
+            import inspect
+
+            from ...defaults import search_engines
+
+            # Get the path to the search_engines.toml file
+            module_path = inspect.getfile(search_engines)
+            default_file = Path(module_path)
+
+            if default_file.exists() and default_file.suffix == ".toml":
+                search_engines_file = default_file
+
+        # If we found a file, load it
+        if search_engines_file.exists():
+            data = toml.load(search_engines_file)
+
+            # Filter out the metadata entries (like DEFAULT_SEARCH_ENGINE)
+            engines = {k: v for k, v in data.items() if isinstance(v, dict)}
+
+            # Add display names for each engine
+            for key, engine in engines.items():
+                if "display_name" not in engine:
+                    # Create a display name from the key
+                    engine["display_name"] = key.replace("_", " ").title()
+
+            return engines
+
+        return None
+    except Exception as e:
+        logger.error(f"Error loading search engines from file: {e}")
+        return None
+
+
+# Legacy routes for backward compatibility - these will redirect to the new routes
+@settings_bp.route("/main", methods=["GET"])
+def main_config_page():
+    """Redirect to app settings page"""
+    return redirect(url_for("settings.settings_page"))
+
+
+@settings_bp.route("/collections", methods=["GET"])
+def collections_config_page():
+    """Redirect to app settings page"""
+    return redirect(url_for("settings.settings_page"))
+
+
+@settings_bp.route("/api_keys", methods=["GET"])
+def api_keys_config_page():
+    """Redirect to LLM settings page"""
+    return redirect(url_for("settings.settings_page"))
+
+
+@settings_bp.route("/search_engines", methods=["GET"])
+def search_engines_config_page():
+    """Redirect to search settings page"""
+    return redirect(url_for("settings.settings_page"))
+
+
+@settings_bp.route("/open_file_location", methods=["POST"])
+def open_file_location():
+    """Open the location of a configuration file"""
+    file_path = request.form.get("file_path")
+
+    if not file_path:
+        flash("No file path provided", "error")
+        return redirect(url_for("settings.settings_page"))
+
+    # Get the directory containing the file
+    dir_path = os.path.dirname(os.path.abspath(file_path))
+
+    # Open the directory in the file explorer
+    try:
+        if platform.system() == "Windows":
+            subprocess.Popen(f'explorer "{dir_path}"')
+        elif platform.system() == "Darwin":  # macOS
+            subprocess.Popen(["open", dir_path])
+        else:  # Linux
+            subprocess.Popen(["xdg-open", dir_path])
+
+        flash(f"Opening folder: {dir_path}", "success")
+    except Exception as e:
+        flash(f"Error opening folder: {str(e)}", "error")
+
+    # Redirect back to the settings page
+    return redirect(url_for("settings.settings_page"))
+
+
+@settings_bp.context_processor
+def inject_csrf_token():
+    """Inject CSRF token into the template context for all settings routes."""
+    return dict(csrf_token=generate_csrf)
+
+
+@settings_bp.route("/fix_corrupted_settings", methods=["POST"])
+def fix_corrupted_settings():
+    """Fix corrupted settings in the database"""
+    db_session = get_db_session()
+
+    try:
+        # Track fixed and removed settings
+        fixed_settings = []
+        removed_duplicate_settings = []
+        fixed_scoping_issues = []
+
+        # First, find and remove duplicate settings with the same key
+        # This happens because of errors in settings import/export
+        from sqlalchemy import func as sql_func
+
+        # Find keys with duplicates
+        duplicate_keys = (
+            db_session.query(Setting.key)
+            .group_by(Setting.key)
+            .having(sql_func.count(Setting.key) > 1)
+            .all()
+        )
+        duplicate_keys = [key[0] for key in duplicate_keys]
+
+        # For each duplicate key, keep the latest updated one and remove others
+        for key in duplicate_keys:
+            dupe_settings = (
+                db_session.query(Setting)
+                .filter(Setting.key == key)
+                .order_by(Setting.updated_at.desc())
+                .all()
+            )
+
+            # Keep the first one (most recently updated) and delete the rest
+            for i, setting in enumerate(dupe_settings):
+                if i > 0:  # Skip the first one (keep it)
+                    db_session.delete(setting)
+                    removed_duplicate_settings.append(key)
+
+        # Fix scoping issues - remove app.* settings that should be in other categories
+        # Report settings
+        for key in [
+            "app.enable_fact_checking",
+            "app.knowledge_accumulation",
+            "app.knowledge_accumulation_context_limit",
+            "app.output_dir",
+        ]:
+            setting = db_session.query(Setting).filter(Setting.key == key).first()
+            if setting:
+                # Move to proper category if not already there
+                proper_key = key.replace("app.", "report.")
+                existing_proper = (
+                    db_session.query(Setting).filter(Setting.key == proper_key).first()
+                )
+
+                if not existing_proper:
+                    # Create proper setting
+                    new_setting = Setting(
+                        key=proper_key,
+                        value=setting.value,
+                        type=SettingType.REPORT,
+                        name=setting.name,
+                        description=setting.description,
+                        category=(
+                            setting.category.replace("app", "report")
+                            if setting.category
+                            else "report_parameters"
+                        ),
+                        ui_element=setting.ui_element,
+                        options=setting.options,
+                        min_value=setting.min_value,
+                        max_value=setting.max_value,
+                        step=setting.step,
+                        visible=setting.visible,
+                        editable=setting.editable,
+                    )
+                    db_session.add(new_setting)
+
+                # Delete the app one
+                db_session.delete(setting)
+                fixed_scoping_issues.append(key)
+
+        # Search settings
+        for key in [
+            "app.research_iterations",
+            "app.questions_per_iteration",
+            "app.search_engine",
+            "app.iterations",
+            "app.max_results",
+            "app.region",
+            "app.safe_search",
+            "app.search_language",
+            "app.snippets_only",
+        ]:
+            setting = db_session.query(Setting).filter(Setting.key == key).first()
+            if setting:
+                # Move to proper category if not already there
+                proper_key = key.replace("app.", "search.")
+                existing_proper = (
+                    db_session.query(Setting).filter(Setting.key == proper_key).first()
+                )
+
+                if not existing_proper:
+                    # Create proper setting
+                    new_setting = Setting(
+                        key=proper_key,
+                        value=setting.value,
+                        type=SettingType.SEARCH,
+                        name=setting.name,
+                        description=setting.description,
+                        category=(
+                            setting.category.replace("app", "search")
+                            if setting.category
+                            else "search_parameters"
+                        ),
+                        ui_element=setting.ui_element,
+                        options=setting.options,
+                        min_value=setting.min_value,
+                        max_value=setting.max_value,
+                        step=setting.step,
+                        visible=setting.visible,
+                        editable=setting.editable,
+                    )
+                    db_session.add(new_setting)
+
+                # Delete the app one
+                db_session.delete(setting)
+                fixed_scoping_issues.append(key)
+
+        # LLM settings
+        for key in [
+            "app.model",
+            "app.provider",
+            "app.temperature",
+            "app.max_tokens",
+            "app.openai_endpoint_url",
+            "app.lmstudio_url",
+            "app.llamacpp_model_path",
+        ]:
+            setting = db_session.query(Setting).filter(Setting.key == key).first()
+            if setting:
+                # Move to proper category if not already there
+                proper_key = key.replace("app.", "llm.")
+                existing_proper = (
+                    db_session.query(Setting).filter(Setting.key == proper_key).first()
+                )
+
+                if not existing_proper:
+                    # Create proper setting
+                    new_setting = Setting(
+                        key=proper_key,
+                        value=setting.value,
+                        type=SettingType.LLM,
+                        name=setting.name,
+                        description=setting.description,
+                        category=(
+                            setting.category.replace("app", "llm")
+                            if setting.category
+                            else "llm_parameters"
+                        ),
+                        ui_element=setting.ui_element,
+                        options=setting.options,
+                        min_value=setting.min_value,
+                        max_value=setting.max_value,
+                        step=setting.step,
+                        visible=setting.visible,
+                        editable=setting.editable,
+                    )
+                    db_session.add(new_setting)
+
+                # Delete the app one
+                db_session.delete(setting)
+                fixed_scoping_issues.append(key)
+
+        # Check for settings with corrupted values
+        all_settings = db_session.query(Setting).all()
+        for setting in all_settings:
+            # Check different types of corruption
+            is_corrupted = False
+
+            if setting.value is None:
+                is_corrupted = True
+            elif isinstance(setting.value, str) and setting.value in [
+                "{",
+                "[",
+                "{}",
+                "[]",
+                "[object Object]",
+                "null",
+                "undefined",
+            ]:
+                is_corrupted = True
+            elif isinstance(setting.value, dict) and len(setting.value) == 0:
+                is_corrupted = True
+
+            # Skip if not corrupted
+            if not is_corrupted:
+                continue
+
+            # Get default value from migrations
+            # Import commented out as it's not directly used
+            # from ..database.migrations import setup_predefined_settings
+
+            default_value = None
+
+            # Try to find a matching default setting based on key
+            if setting.key.startswith("llm."):
+                if setting.key == "llm.model":
+                    default_value = "gpt-3.5-turbo"
+                elif setting.key == "llm.provider":
+                    default_value = "openai"
+                elif setting.key == "llm.temperature":
+                    default_value = 0.7
+                elif setting.key == "llm.max_tokens":
+                    default_value = 1024
+            elif setting.key.startswith("search."):
+                if setting.key == "search.tool":
+                    default_value = "auto"
+                elif setting.key == "search.max_results":
+                    default_value = 10
+                elif setting.key == "search.region":
+                    default_value = "us"
+                elif setting.key == "search.research_iterations":
+                    default_value = 2
+                elif setting.key == "search.questions_per_iteration":
+                    default_value = 3
+                elif setting.key == "search.searches_per_section":
+                    default_value = 2
+                elif setting.key == "search.skip_relevance_filter":
+                    default_value = False
+                elif setting.key == "search.safe_search":
+                    default_value = True
+                elif setting.key == "search.search_language":
+                    default_value = "English"
+            elif setting.key.startswith("report."):
+                if setting.key == "report.searches_per_section":
+                    default_value = 2
+                elif setting.key == "report.enable_fact_checking":
+                    default_value = True
+                elif setting.key == "report.detailed_citations":
+                    default_value = True
+            elif setting.key.startswith("app."):
+                if setting.key == "app.theme" or setting.key == "app.default_theme":
+                    default_value = "dark"
+                elif setting.key == "app.enable_notifications":
+                    default_value = True
+                elif (
+                    setting.key == "app.enable_web"
+                    or setting.key == "app.web_interface"
+                ):
+                    default_value = True
+                elif setting.key == "app.host":
+                    default_value = "0.0.0.0"
+                elif setting.key == "app.port":
+                    default_value = 5000
+                elif setting.key == "app.debug":
+                    default_value = True
+
+            # Update the setting with the default value if found
+            if default_value is not None:
+                setting.value = default_value
+                fixed_settings.append(setting.key)
+            else:
+                # If no default found but it's a corrupted JSON, set to empty object
+                if setting.key.startswith("report."):
+                    setting.value = {}
+                    fixed_settings.append(setting.key)
+
+        # Commit changes
+        if fixed_settings or removed_duplicate_settings or fixed_scoping_issues:
+            db_session.commit()
+            logger.info(
+                f"Fixed {len(fixed_settings)} corrupted settings: {', '.join(fixed_settings)}"
+            )
+            if removed_duplicate_settings:
+                logger.info(
+                    f"Removed {len(removed_duplicate_settings)} duplicate settings"
+                )
+            if fixed_scoping_issues:
+                logger.info(f"Fixed {len(fixed_scoping_issues)} scoping issues")
+
+        # Return success
+        return jsonify(
+            {
+                "status": "success",
+                "message": f"Fixed {len(fixed_settings)} corrupted settings, removed {len(removed_duplicate_settings)} duplicates, and fixed {len(fixed_scoping_issues)} scoping issues",
+                "fixed_settings": fixed_settings,
+                "removed_duplicates": removed_duplicate_settings,
+                "fixed_scoping": fixed_scoping_issues,
+            }
+        )
+
+    except Exception as e:
+        logger.error(f"Error fixing corrupted settings: {e}")
+        db_session.rollback()
+        return (
+            jsonify(
+                {
+                    "status": "error",
+                    "message": f"Error fixing corrupted settings: {str(e)}",
+                }
+            ),
+            500,
+        )
+
+
+@settings_bp.route("/api/ollama-status", methods=["GET"])
+def check_ollama_status():
+    """Check if Ollama is running and available"""
+    try:
+        # Set a shorter timeout for the request
+        base_url = os.getenv(
+            "OLLAMA_BASE_URL",
+            "http://localhost:11434",
+        )
+        response = requests.get(f"{base_url}/api/version", timeout=2.0)
+
+        if response.status_code == 200:
+            return jsonify(
+                {"running": True, "version": response.json().get("version", "unknown")}
+            )
+        else:
+            return jsonify(
+                {
+                    "running": False,
+                    "error": f"Ollama returned status code {response.status_code}",
+                }
+            )
+    except requests.exceptions.RequestException as e:
+        logger.info(f"Ollama check failed: {str(e)}")
+        return jsonify({"running": False, "error": str(e)})