local-deep-research 0.5.9__py3-none-any.whl → 0.6.0__py3-none-any.whl

local_deep_research/web/routes/settings_routes.py

@@ -7,7 +7,6 @@ from typing import Any, Optional, Tuple
 import requests
 from flask import (
     Blueprint,
-    current_app,
     flash,
     jsonify,
     redirect,
@@ -16,9 +15,8 @@ from flask import (
 )
 from flask_wtf.csrf import generate_csrf
 from loguru import logger
-from sqlalchemy.orm import Session
 
-from ...utilities.db_utils import get_db_setting
+from ...utilities.db_utils import get_db_setting, get_db_session
 from ...utilities.url_utils import normalize_url
 from ..database.models import Setting, SettingType
 from ..services.settings_service import (
@@ -27,18 +25,89 @@ from ..services.settings_service import (
     get_settings_manager,
     set_setting,
 )
+from ..services.settings_manager import SettingsManager
 from ..utils.templates import render_template_with_defaults
 
 # Create a Blueprint for settings
-settings_bp = Blueprint("settings", __name__, url_prefix="/research/settings")
+settings_bp = Blueprint("settings", __name__, url_prefix="/settings")
 
 
-def get_db_session() -> Session:
-    """Get the database session from the app context"""
-    if hasattr(current_app, "db_session"):
-        return current_app.db_session
-    else:
-        return current_app.extensions["sqlalchemy"].session()
+def calculate_warnings():
+    """Calculate current warning conditions based on settings"""
+    warnings = []
+
+    try:
+        # Get a fresh database session for safety
+        db_session = get_db_session()
+
+        # Get current settings
+        provider = get_setting("llm.provider", "ollama", db_session).lower()
+        local_context = get_setting(
+            "llm.local_context_window_size", 4096, db_session
+        )
+
+        logger.debug(f"Starting warning calculation - provider={provider}")
+
+        # Get dismissal settings
+        dismiss_high_context = get_setting(
+            "app.warnings.dismiss_high_context", False, db_session
+        )
+
+        # Check warning conditions
+        is_local_provider = provider in [
+            "ollama",
+            "llamacpp",
+            "lmstudio",
+            "vllm",
+        ]
+
+        # High context warning for local providers
+        if (
+            is_local_provider
+            and local_context > 8192
+            and not dismiss_high_context
+        ):
+            warnings.append(
+                {
+                    "type": "high_context",
+                    "icon": "⚠️",
+                    "title": "High Context Warning",
+                    "message": f"Context size ({local_context:,} tokens) may cause memory issues with {provider}. Increase VRAM or reduce context size if you experience slowdowns.",
+                    "dismissKey": "app.warnings.dismiss_high_context",
+                }
+            )
+
+        # Get additional warning settings
+        dismiss_model_mismatch = get_setting(
+            "app.warnings.dismiss_model_mismatch", False, db_session
+        )
+
+        # Get current strategy and model (these need to be passed from the frontend or retrieved differently)
+        # For now, we'll implement basic warnings that don't require form state
+
+        # Model mismatch warning (simplified - checking setting instead of form value)
+        current_model = get_setting("llm.model", "", db_session)
+        if (
+            current_model
+            and "70b" in current_model.lower()
+            and is_local_provider
+            and local_context > 8192
+            and not dismiss_model_mismatch
+        ):
+            warnings.append(
+                {
+                    "type": "model_mismatch",
+                    "icon": "🧠",
+                    "title": "Model & Context Warning",
+                    "message": f"Large model ({current_model}) with high context ({local_context:,}) may exceed VRAM. Consider reducing context size or upgrading GPU memory.",
+                    "dismissKey": "app.warnings.dismiss_model_mismatch",
+                }
+            )
+
+    except Exception as e:
+        logger.warning(f"Error calculating warnings: {e}")
+
+    return warnings
 
 
 def validate_setting(
@@ -340,15 +409,37 @@ def save_all_settings():
             # Multiple settings or generic message
             success_message = f"Settings saved successfully ({len(updated_settings)} updated, {len(created_settings)} created)"
 
-        return jsonify(
-            {
-                "status": "success",
-                "message": success_message,
-                "updated": updated_settings,
-                "created": created_settings,
-                "settings": all_settings,
-            }
-        )
+        # Check if any warning-affecting settings were changed and include warnings
+        response_data = {
+            "status": "success",
+            "message": success_message,
+            "updated": updated_settings,
+            "created": created_settings,
+            "settings": all_settings,
+        }
+
+        warning_affecting_keys = [
+            "llm.provider",
+            "search.tool",
+            "search.iterations",
+            "search.questions_per_iteration",
+            "llm.local_context_window_size",
+            "llm.context_window_unrestricted",
+            "llm.context_window_size",
+        ]
+
+        # Check if any warning-affecting settings were changed
+        if any(
+            key in warning_affecting_keys
+            for key in updated_settings + created_settings
+        ):
+            warnings = calculate_warnings()
+            response_data["warnings"] = warnings
+            logger.info(
+                f"Bulk settings update affected warning keys, calculated {len(warnings)} warnings"
+            )
+
+        return jsonify(response_data)
 
     except Exception:
         logger.exception("Error saving settings")
@@ -511,9 +602,29 @@ def api_update_setting(key):
             # Update setting
             success = set_setting(key, value)
             if success:
-                return jsonify(
-                    {"message": f"Setting {key} updated successfully"}
-                )
+                response_data = {
+                    "message": f"Setting {key} updated successfully"
+                }
+
+                # If this is a key that affects warnings, include warning calculations
+                warning_affecting_keys = [
+                    "llm.provider",
+                    "search.tool",
+                    "search.iterations",
+                    "search.questions_per_iteration",
+                    "llm.local_context_window_size",
+                    "llm.context_window_unrestricted",
+                    "llm.context_window_size",
+                ]
+
+                if key in warning_affecting_keys:
+                    warnings = calculate_warnings()
+                    response_data["warnings"] = warnings
+                    logger.debug(
+                        f"Setting {key} changed to {value}, calculated {len(warnings)} warnings"
+                    )
+
+                return jsonify(response_data)
             else:
                 return jsonify(
                     {"error": f"Failed to update setting {key}"}
@@ -669,6 +780,15 @@ def api_get_ui_elements():
 def api_get_available_models():
     """Get available LLM models from various providers"""
     try:
+        from datetime import datetime, timedelta
+        from ..database.models import ProviderModel
+        from flask import request, current_app
+
+        # Check if force_refresh is requested
+        force_refresh = (
+            request.args.get("force_refresh", "false").lower() == "true"
+        )
+
         # Define provider options with generic provider names
         provider_options = [
             {"value": "OLLAMA", "label": "Ollama (Local)"},
@@ -683,6 +803,56 @@ def api_get_available_models():
         # Available models by provider
         providers = {}
 
+        # Check database cache first (unless force_refresh is True)
+        if not force_refresh:
+            try:
+                db_session = current_app.db_session
+
+                # Define cache expiration (24 hours)
+                cache_expiry = datetime.utcnow() - timedelta(hours=24)
+
+                # Get cached models from database
+                cached_models = (
+                    db_session.query(ProviderModel)
+                    .filter(ProviderModel.last_updated > cache_expiry)
+                    .all()
+                )
+
+                if cached_models:
+                    logger.info(
+                        f"Found {len(cached_models)} cached models in database"
+                    )
+
+                    # Group models by provider
+                    for model in cached_models:
+                        provider_key = f"{model.provider.lower()}_models"
+                        if provider_key not in providers:
+                            providers[provider_key] = []
+
+                        providers[provider_key].append(
+                            {
+                                "value": model.model_key,
+                                "label": model.model_label,
+                                "provider": model.provider.upper(),
+                            }
+                        )
+
+                    # If we have cached data for all providers, return it
+                    if providers:
+                        logger.info("Returning cached models from database")
+                        return jsonify(
+                            {
+                                "provider_options": provider_options,
+                                "providers": providers,
+                            }
+                        )
+
+            except Exception as e:
+                logger.warning(
+                    f"Error reading cached models from database: {e}"
+                )
+                # Continue to fetch fresh data
+
         # Try to get Ollama models
         ollama_models = []
         try:
@@ -1096,6 +1266,45 @@ def api_get_available_models():
         providers["anthropic_models"] = anthropic_models
         logger.info(f"Final Anthropic models count: {len(anthropic_models)}")
 
+        # Save fetched models to database cache
+        if force_refresh or providers:
+            # We fetched fresh data, save it to database
+            try:
+                from datetime import datetime
+
+                db_session = current_app.db_session
+
+                # Clear old cache entries for providers we're updating
+                for provider_key in providers.keys():
+                    provider_name = provider_key.replace("_models", "").upper()
+                    db_session.query(ProviderModel).filter(
+                        ProviderModel.provider == provider_name
+                    ).delete()
+
+                # Insert new models
+                for provider_key, models in providers.items():
+                    provider_name = provider_key.replace("_models", "").upper()
+                    for model in models:
+                        if (
+                            isinstance(model, dict)
+                            and "value" in model
+                            and "label" in model
+                        ):
+                            new_model = ProviderModel(
+                                provider=provider_name,
+                                model_key=model["value"],
+                                model_label=model["label"],
+                                last_updated=datetime.utcnow(),
+                            )
+                            db_session.add(new_model)
+
+                db_session.commit()
+                logger.info("Successfully cached models to database")
+
+            except Exception:
+                logger.exception("Error saving models to database cache")
+                db_session.rollback()
+
         # Return all options
         return jsonify(
             {"provider_options": provider_options, "providers": providers}
@@ -1565,6 +1774,17 @@ def fix_corrupted_settings():
         )
 
 
+@settings_bp.route("/api/warnings", methods=["GET"])
+def api_get_warnings():
+    """Get current warnings based on settings"""
+    try:
+        warnings = calculate_warnings()
+        return jsonify({"warnings": warnings})
+    except Exception as e:
+        logger.exception("Error getting warnings")
+        return jsonify({"error": str(e)}), 500
+
+
 @settings_bp.route("/api/ollama-status", methods=["GET"])
 def check_ollama_status():
     """Check if Ollama is running and available"""
@@ -1598,3 +1818,143 @@ def check_ollama_status():
     except requests.exceptions.RequestException as e:
         logger.exception("Ollama check failed")
         return jsonify({"running": False, "error": str(e)})
+
+
+@settings_bp.route("/api/rate-limiting/status", methods=["GET"])
+def api_get_rate_limiting_status():
+    """Get current rate limiting status and statistics"""
+    try:
+        from ...web_search_engines.rate_limiting import get_tracker
+
+        tracker = get_tracker()
+
+        # Get basic status
+        status = {
+            "enabled": tracker.enabled,
+            "exploration_rate": tracker.exploration_rate,
+            "learning_rate": tracker.learning_rate,
+            "memory_window": tracker.memory_window,
+        }
+
+        # Get engine statistics
+        engine_stats = tracker.get_stats()
+        engines = []
+
+        for stat in engine_stats:
+            (
+                engine_type,
+                base_wait,
+                min_wait,
+                max_wait,
+                last_updated,
+                total_attempts,
+                success_rate,
+            ) = stat
+            engines.append(
+                {
+                    "engine_type": engine_type,
+                    "base_wait_seconds": round(base_wait, 2),
+                    "min_wait_seconds": round(min_wait, 2),
+                    "max_wait_seconds": round(max_wait, 2),
+                    "last_updated": last_updated,
+                    "total_attempts": total_attempts,
+                    "success_rate": round(success_rate * 100, 1)
+                    if success_rate
+                    else 0.0,
+                }
+            )
+
+        return jsonify({"status": status, "engines": engines})
+
+    except Exception:
+        logger.exception("Error getting rate limiting status")
+        return jsonify({"error": "An internal error occurred"}), 500
+
+
+@settings_bp.route(
+    "/api/rate-limiting/engines/<engine_type>/reset", methods=["POST"]
+)
+def api_reset_engine_rate_limiting(engine_type):
+    """Reset rate limiting data for a specific engine"""
+    try:
+        from ...web_search_engines.rate_limiting import get_tracker
+
+        tracker = get_tracker()
+        tracker.reset_engine(engine_type)
+
+        return jsonify(
+            {"message": f"Rate limiting data reset for {engine_type}"}
+        )
+
+    except Exception:
+        logger.exception(f"Error resetting rate limiting for {engine_type}")
+        return jsonify({"error": "An internal error occurred"}), 500
+
+
+@settings_bp.route("/api/rate-limiting/cleanup", methods=["POST"])
+def api_cleanup_rate_limiting():
+    """Clean up old rate limiting data"""
+    try:
+        from ...web_search_engines.rate_limiting import get_tracker
+
+        days = request.json.get("days", 30) if request.is_json else 30
+
+        tracker = get_tracker()
+        tracker.cleanup_old_data(days)
+
+        return jsonify(
+            {"message": f"Cleaned up rate limiting data older than {days} days"}
+        )
+
+    except Exception:
+        logger.exception("Error cleaning up rate limiting data")
+        return jsonify({"error": "An internal error occurred"}), 500
+
+
+@settings_bp.route("/api/bulk", methods=["GET"])
+def get_bulk_settings():
+    """Get multiple settings at once for performance."""
+    try:
+        # Get requested settings from query parameters
+        requested = request.args.getlist("keys[]")
+        if not requested:
+            # Default to common settings if none specified
+            requested = [
+                "llm.provider",
+                "llm.model",
+                "search.tool",
+                "search.iterations",
+                "search.questions_per_iteration",
+                "search.search_strategy",
+                "benchmark.evaluation.provider",
+                "benchmark.evaluation.model",
+                "benchmark.evaluation.temperature",
+                "benchmark.evaluation.endpoint_url",
+            ]
+
+        # Fetch all settings at once
+        session = get_db_session()
+        settings_manager = SettingsManager(db_session=session)
+
+        result = {}
+        for key in requested:
+            try:
+                value = settings_manager.get_setting(key)
+                result[key] = {"value": value, "exists": value is not None}
+            except Exception as e:
+                logger.warning(f"Error getting setting {key}: {e}")
+                result[key] = {
+                    "value": None,
+                    "exists": False,
+                    "error": "Failed to retrieve setting",
+                }
+
+        session.close()
+
+        return jsonify({"success": True, "settings": result})
+
+    except Exception:
+        logger.exception("Error getting bulk settings")
+        return jsonify(
+            {"success": False, "error": "An internal error occurred"}
+        ), 500

local_deep_research/web/services/research_service.py

@@ -200,7 +200,7 @@ def run_research_process(
             return
 
         logger.info(
-            "Starting research process for ID %s, query: %s", research_id, query
+            f"Starting research process for ID {research_id}, query: {query}"
         )
 
         # Extract key parameters
@@ -225,18 +225,11 @@ def run_research_process(
 
         # Log all parameters for debugging
         logger.info(
-            "Research parameters: provider=%s, model=%s, search_engine=%s, "
-            "max_results=%s, time_period=%s, iterations=%s, "
-            "questions_per_iteration=%s, custom_endpoint=%s, strategy=%s",
-            model_provider,
-            model,
-            search_engine,
-            max_results,
-            time_period,
-            iterations,
-            questions_per_iteration,
-            custom_endpoint,
-            strategy,
+            f"Research parameters: provider={model_provider}, model={model}, "
+            f"search_engine={search_engine}, max_results={max_results}, "
+            f"time_period={time_period}, iterations={iterations}, "
+            f"questions_per_iteration={questions_per_iteration}, "
+            f"custom_endpoint={custom_endpoint}, strategy={strategy}"
         )
 
         # Set up the AI Context Manager
@@ -269,7 +262,9 @@ def run_research_process(
                 )
                 raise Exception("Research was terminated by user")
 
-            logger.log("milestone", message)
+            # Bind research_id to logger for this specific log
+            bound_logger = logger.bind(research_id=research_id)
+            bound_logger.log("MILESTONE", message)
 
             if "SEARCH_PLAN:" in message:
                 engines = message.split("SEARCH_PLAN:")[1].strip()
@@ -405,15 +400,11 @@ def run_research_process(
                 )
 
                 logger.info(
-                    "Successfully set LLM to: provider=%s, model=%s",
-                    model_provider,
-                    model,
+                    f"Successfully set LLM to: provider={model_provider}, model={model}"
                 )
             except Exception:
                 logger.exception(
-                    "Error setting LLM provider=%s, model=%s",
-                    model_provider,
-                    model,
+                    f"Error setting LLM provider={model_provider}, model={model}"
                 )
 
         # Set the progress callback in the system
@@ -423,12 +414,14 @@ def run_research_process(
         # Override search engine if specified
         if search_engine:
             try:
-                if iterations:
-                    system.max_iterations = int(iterations)
-                if questions_per_iteration:
-                    system.questions_per_iteration = int(
-                        questions_per_iteration
-                    )
+                # For focused-iteration strategy, don't override iterations/questions - use database settings
+                if strategy != "focused-iteration":
+                    if iterations:
+                        system.max_iterations = int(iterations)
+                    if questions_per_iteration:
+                        system.questions_per_iteration = int(
+                            questions_per_iteration
+                        )
 
                 # Create a new search object with these settings
                 system.search = get_search(
@@ -436,11 +429,11 @@ def run_research_process(
                 )
 
                 logger.info(
-                    "Successfully set search engine to: %s", search_engine
+                    f"Successfully set search engine to: {search_engine}"
                 )
             except Exception:
                 logger.exception(
-                    "Error setting search engine to %s", search_engine
+                    f"Error setting search engine to {search_engine}"
                 )
 
         # Run the search
@@ -1032,7 +1025,7 @@ def run_research_process(
                 research.status = status
                 research.completed_at = completed_at
                 research.duration_seconds = duration_seconds
-                research.metadata = metadata
+                research.research_meta = metadata
 
                 # Add error report path if available
                 if "report_path_to_save" in locals() and report_path_to_save:

local_deep_research/web/services/settings_manager.py

@@ -229,6 +229,8 @@ class SettingsManager:
 
                 if commit:
                     self.db_session.commit()
+                    # Emit WebSocket event for settings change
+                    self._emit_settings_changed([key])
 
                 return True
             except SQLAlchemyError as e:
@@ -365,6 +367,8 @@ class SettingsManager:
 
             if commit:
                 self.db_session.commit()
+                # Emit WebSocket event for settings change
+                self._emit_settings_changed([setting_obj.key])
 
             return db_setting
 
@@ -522,6 +526,8 @@ class SettingsManager:
 
         if commit:
             self.db_session.commit()
+            # Emit WebSocket event for all imported settings
+            self._emit_settings_changed(list(settings_data.keys()))
 
     def _create_setting(self, key, value, setting_type):
         """Create a setting with appropriate metadata"""
@@ -578,3 +584,50 @@ class SettingsManager:
 
         # Create the setting in the database
         self.create_or_update_setting(setting_dict, commit=False)
+
+    def _emit_settings_changed(self, changed_keys: list = None):
+        """
+        Emit WebSocket event when settings change
+
+        Args:
+            changed_keys: List of setting keys that changed
+        """
+        try:
+            # Import here to avoid circular imports
+            from .socket_service import SocketIOService
+
+            try:
+                socket_service = SocketIOService()
+            except ValueError:
+                logger.debug(
+                    "Not emitting socket event because server is not initialized."
+                )
+                return
+
+            # Get the changed settings
+            settings_data = {}
+            if changed_keys:
+                for key in changed_keys:
+                    setting_value = self.get_setting(key)
+                    if setting_value is not None:
+                        settings_data[key] = {"value": setting_value}
+
+            # Emit the settings change event
+            from datetime import datetime
+
+            socket_service.emit_socket_event(
+                "settings_changed",
+                {
+                    "changed_keys": changed_keys or [],
+                    "settings": settings_data,
+                    "timestamp": datetime.utcnow().isoformat(),
+                },
+            )
+
+            logger.debug(
+                f"Emitted settings_changed event for keys: {changed_keys}"
+            )
+
+        except Exception as e:
+            logger.exception(f"Failed to emit settings change event: {e}")
+            # Don't let WebSocket emission failures break settings saving

local_deep_research/web/services/settings_service.py

@@ -112,6 +112,8 @@ def bulk_update_settings(
     if commit and success and manager.db_session:
         try:
             manager.db_session.commit()
+            # Emit WebSocket event for all changed settings
+            manager._emit_settings_changed(list(settings_dict.keys()))
         except Exception:
             logger.exception("Error committing bulk settings update")
             manager.db_session.rollback()

local_deep_research/web/static/css/styles.css

@@ -1678,6 +1678,14 @@ textarea:focus, input[type="text"]:focus {
     transition: all 0.3s ease;
 }
 
+/* Allow form controls to be interactive even when panel is collapsed */
+.advanced-options-panel select,
+.advanced-options-panel input,
+.advanced-options-panel button,
+.advanced-options-panel textarea {
+    pointer-events: auto;
+}
+
 .advanced-options-toggle.open + .advanced-options-panel,
 .advanced-options-panel.expanded {
     max-height: 1000px;