local-deep-research 0.2.3__py3-none-any.whl → 0.3.1__py3-none-any.whl

local_deep_research/web/static/js/components/settings.js

@@ -984,7 +984,7 @@
         // Only run this for the main settings dashboard
         if (!settingsContent) return;
 
-        fetch('/research/settings/all_settings')
+        fetch('/research/settings/api')
             .then(response => response.json())
             .then(data => {
                 if (data.status === 'success') {
@@ -1089,7 +1089,6 @@
                 'max_results',
                 'quality_check_urls',
                 'questions_per_iteration',
-                'research_iterations',
                 'region',
                 'search_engine',
                 'searches_per_section',
@@ -1124,7 +1123,7 @@
         const prioritySettings = {
             'app': ['enable_web', 'enable_notifications', 'web_interface', 'theme', 'default_theme', 'dark_mode', 'debug', 'host', 'port'],
             'llm': ['provider', 'model', 'temperature', 'max_tokens', 'api_key', 'openai_endpoint_url', 'lmstudio_url', 'llamacpp_model_path'],
-            'search': ['tool', 'search_engine', 'iterations', 'questions_per_iteration', 'research_iterations', 'max_results', 'region'],
+            'search': ['tool', 'iterations', 'questions_per_iteration', 'max_results', 'region', 'search_engine'],
             'report': ['enable_fact_checking', 'knowledge_accumulation', 'output_dir', 'detailed_citations']
         };
 
@@ -1152,6 +1151,11 @@
                 return false;
             }
 
+            // Filter out settings that are not marked as visible.
+            if (!setting.visible) {
+                return false;
+            }
+
             // If we're on a specific tab, only show settings for that tab
             if (tab !== 'all') {
                 // Only show settings in tab-specific lists for that tab
@@ -3091,7 +3095,15 @@
      * Process settings to handle object values
      */
     function processSettings(settings) {
-        return settings.map(setting => {
+        // Convert to a list.
+        const settingsList = [];
+        for (const key in settings) {
+            const setting = settings[key];
+            setting["key"] = key
+            settingsList.push(setting);
+        }
+
+        return settingsList.map(setting => {
             const processedSetting = {...setting};
 
             // Convert object values to JSON strings for display

local_deep_research/web/static/js/research_form.js

@@ -0,0 +1,106 @@
+/**
+ * Research form handling with settings management
+ */
+
+document.addEventListener('DOMContentLoaded', function() {
+    // Initialize the research form
+    console.log('DOM loaded, initializing research form');
+    initResearchForm();
+});
+
+/**
+ * Initialize the research form with values from settings
+ */
+function initResearchForm() {
+    console.log('Initializing research form...');
+    // Get form elements
+    const iterationsInput = document.getElementById('iterations');
+    const questionsInput = document.getElementById('questions_per_iteration');
+
+    // Fetch all settings at once (more efficient)
+    fetch('/research/settings/api')
+        .then(response => {
+            if (!response.ok) {
+                throw new Error('Failed to fetch settings');
+            }
+            return response.json();
+        })
+        .then(data => {
+            console.log('Loaded settings:', data);
+            if (data && data.status === 'success' && data.settings) {
+                // Find our specific settings
+                const settings = data.settings;
+
+                // Look for the iterations setting
+                for (const key in settings) {
+                    const setting = settings[key];
+                    if (key === 'search.iterations') {
+                        console.log('Found iterations setting:', setting.value);
+                        iterationsInput.value = setting.value;
+                    }
+
+                    if (key === 'search.questions_per_iteration') {
+                        console.log('Found questions setting:', setting.value);
+                        questionsInput.value = setting.value;
+                    }
+                }
+            }
+        })
+        .catch(error => {
+            console.warn('Error loading research settings:', error);
+            // Form will use default values if settings can't be loaded
+        });
+
+    // Add our settings saving to the form submission process
+    patchFormSubmitHandler();
+}
+
+/**
+ * Patch the existing form submit handler to include our settings saving functionality
+ */
+function patchFormSubmitHandler() {
+    // Get the form element
+    const form = document.getElementById('research-form');
+    if (!form) return;
+
+    // Monitor for form submissions using the capture phase to run before other handlers
+    form.addEventListener('submit', function(event) {
+        // Save research settings first, before the main form handler processes the submission
+        saveResearchSettings();
+
+        // Let the event continue normally to the other handlers
+    }, true); // true enables capture phase
+}
+
+/**
+ * Save research settings to the database
+ */
+function saveResearchSettings() {
+    const iterations = document.getElementById('iterations').value;
+    const questions = document.getElementById('questions_per_iteration').value;
+
+    console.log('Saving research settings:', { iterations, questions });
+
+    // Get CSRF token
+    const csrfToken = document.querySelector('meta[name="csrf-token"]').getAttribute('content');
+
+    // Save settings
+    fetch('/research/settings/save_all_settings', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'X-CSRFToken': csrfToken
+        },
+        body: JSON.stringify({
+            'search.iterations': parseInt(iterations),
+            'search.questions_per_iteration': parseInt(questions)
+        })
+    })
+    .then(response => response.json())
+    .then(data => {
+        console.log('Settings saved:', data);
+    })
+    .catch(error => {
+        console.warn('Error saving research settings:', error);
+    });
+}

local_deep_research/web/templates/pages/research.html

@@ -99,9 +99,9 @@
                     </div>
 
                     <div class="form-row">
-                        <!-- Research Iterations -->
+                        <!-- Search Iterations -->
                         <div class="form-group half">
-                            <label for="iterations">Research Iterations</label>
+                            <label for="iterations">Search Iterations</label>
                             <input type="number" id="iterations" name="iterations" class="form-control" min="1" max="5" value="2">
                             <span class="input-help">Number of research cycles to perform</span>
                         </div>
@@ -136,4 +136,5 @@
 {% block component_scripts %}
 <script src="{{ url_for('research.serve_static', path='js/components/custom_dropdown.js') }}"></script>
 <script src="{{ url_for('research.serve_static', path='js/components/research.js') }}"></script>
+<script src="{{ url_for('research.serve_static', path='js/research_form.js') }}"></script>
 {% endblock %}

local_deep_research/web_search_engines/engines/meta_search_engine.py

@@ -1,12 +1,11 @@
 import logging
-import os
 from typing import Any, Dict, List, Optional
 
-from ...config import search_config
+from ...utilities.db_utils import get_db_setting
 from ...web.services.socket_service import emit_socket_event
 from ..search_engine_base import BaseSearchEngine
 from ..search_engine_factory import create_search_engine
-from ..search_engines_config import SEARCH_ENGINES
+from ..search_engines_config import search_config
 from .search_engine_wikipedia import WikipediaSearchEngine
 
 # Setup logging
@@ -66,27 +65,55 @@ class MetaSearchEngine(BaseSearchEngine):
         )
 
     def _get_available_engines(self) -> List[str]:
-        """Get list of available engines, excluding 'meta' and 'auto'"""
+        """Get list of available engines, excluding 'meta' and 'auto', based on user settings"""
         # Filter out 'meta' and 'auto' and check API key availability
         available = []
-        for name, config_ in SEARCH_ENGINES.items():
+
+        for name, config_ in search_config().items():
             if name in ["meta", "auto"]:
                 continue
 
+            # Determine if this is a local engine (starts with "local.")
+            is_local_engine = name.startswith("local.")
+
+            # Determine the appropriate setting path based on engine type
+            if is_local_engine:
+                # Format: search.engine.local.{engine_name}.use_in_auto_search
+                local_name = name.replace("local.", "")
+                auto_search_setting = (
+                    f"search.engine.local.{local_name}.use_in_auto_search"
+                )
+            else:
+                # Format: search.engine.web.{engine_name}.use_in_auto_search
+                auto_search_setting = f"search.engine.web.{name}.use_in_auto_search"
+
+            # Get setting from database, default to False if not found
+            use_in_auto_search = get_db_setting(auto_search_setting, False)
+
+            # Skip engines that aren't enabled for auto search
+            if not use_in_auto_search:
+                logger.info(
+                    f"Skipping {name} engine because it's not enabled for auto search"
+                )
+                continue
+
+            # Skip engines that require API keys if we don't want to use them
             if config_.get("requires_api_key", False) and not self.use_api_key_services:
                 continue
 
+            # Skip engines that require API keys if the key is not available
             if config_.get("requires_api_key", False):
-                api_key_env = config_.get("api_key_env")
-                api_key = os.getenv(api_key_env) if api_key_env else None
+                api_key = config_.get("api_key")
                 if not api_key:
                     continue
 
             available.append(name)
 
-        # Make sure we have at least one engine available
-        if not available and "wikipedia" in SEARCH_ENGINES:
-            available.append("wikipedia")
+        # If no engines are available, raise an error instead of falling back silently
+        if not available:
+            error_msg = "No search engines enabled for auto search. Please enable at least one engine in settings."
+            logger.error(error_msg)
+            raise RuntimeError(error_msg)
 
         return available
 
@@ -109,7 +136,7 @@ class MetaSearchEngine(BaseSearchEngine):
                 # Return engines sorted by reliability
                 return sorted(
                     self.available_engines,
-                    key=lambda x: SEARCH_ENGINES.get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
                     reverse=True,
                 )
 
@@ -117,14 +144,14 @@ class MetaSearchEngine(BaseSearchEngine):
             engines_info = []
             for engine_name in self.available_engines:
                 try:
-                    if engine_name in SEARCH_ENGINES:
-                        strengths = SEARCH_ENGINES[engine_name].get(
+                    if engine_name in search_config():
+                        strengths = search_config()[engine_name].get(
                             "strengths", "General search"
                         )
-                        weaknesses = SEARCH_ENGINES[engine_name].get(
+                        weaknesses = search_config()[engine_name].get(
                             "weaknesses", "None specified"
                         )
-                        description = SEARCH_ENGINES[engine_name].get(
+                        description = search_config()[engine_name].get(
                             "description", engine_name
                         )
                         engines_info.append(
@@ -133,6 +160,17 @@ class MetaSearchEngine(BaseSearchEngine):
                 except KeyError as e:
                     logger.error(f"Missing key for engine {engine_name}: {str(e)}")
 
+            # Only proceed if we have engines available to choose from
+            if not engines_info:
+                logger.warning(
+                    "No engine information available for prompt, using reliability-based sorting instead"
+                )
+                return sorted(
+                    self.available_engines,
+                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    reverse=True,
+                )
+
             prompt = f"""You are a search query analyst. Consider this search query:
 
 QUERY: {query}
@@ -167,7 +205,7 @@ Example output: wikipedia,arxiv,github"""
             if not valid_engines:
                 valid_engines = sorted(
                     self.available_engines,
-                    key=lambda x: SEARCH_ENGINES.get(x, {}).get("reliability", 0),
+                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
                     reverse=True,
                 )
 
@@ -177,7 +215,7 @@ Example output: wikipedia,arxiv,github"""
             # Fall back to reliability-based ordering
             return sorted(
                 self.available_engines,
-                key=lambda x: SEARCH_ENGINES.get(x, {}).get("reliability", 0),
+                key=lambda x: search_config().get(x, {}).get("reliability", 0),
                 reverse=True,
             )
 
@@ -276,10 +314,7 @@ Example output: wikipedia,arxiv,github"""
             List of result dictionaries with full content
         """
         # Check if we should get full content
-        if (
-            hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
-            and search_config.SEARCH_SNIPPETS_ONLY
-        ):
+        if get_db_setting("search.snippets_only", True):
             logger.info("Snippet-only mode, skipping full content retrieval")
             return relevant_items
 

local_deep_research/web_search_engines/engines/search_engine_local.py

@@ -31,6 +31,7 @@ from langchain_core.language_models import BaseLLM
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 from ...config import search_config
+from ...utilities.db_utils import get_db_setting
 from ..search_engine_base import BaseSearchEngine
 
 # Setup logging
@@ -168,8 +169,8 @@ class LocalEmbeddingManager:
             if self.embedding_model_type == "ollama":
                 # Use Ollama for embeddings
                 if not self.ollama_base_url:
-                    self.ollama_base_url = os.getenv(
-                        "OLLAMA_BASE_URL", "http://localhost:11434"
+                    self.ollama_base_url = get_db_setting(
+                        "llm.ollama.url", "http://localhost:11434"
                     )
 
                 logger.info(
@@ -642,6 +643,8 @@ class LocalSearchEngine(BaseSearchEngine):
         chunk_overlap: int = 200,
         cache_dir: str = ".cache/local_search",
         collections: Optional[Dict[str, Dict[str, Any]]] = None,
+        name: str = "",
+        description: str = "",
     ):
         """
         Initialize the local search engine.
@@ -660,10 +663,16 @@ class LocalSearchEngine(BaseSearchEngine):
             chunk_overlap: Overlap between chunks
             cache_dir: Directory to store embedding cache and index
             collections: Dictionary of named collections with paths and descriptions
+            name: Human-readable name of the collection we are searching.
+            description: Human-readable description of the collection we are
+                searching.
         """
         # Initialize the base search engine
         super().__init__(llm=llm, max_filtered_results=max_filtered_results)
 
+        self.name = name
+        self.description = description
+
         # Validate folder paths
         self.folder_paths = paths
         self.valid_folder_paths = []

local_deep_research/web_search_engines/engines/search_engine_local_all.py

@@ -3,14 +3,14 @@ Search engine that searches across all local collections
 """
 
 import logging
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, cast
 
-import toml
 from langchain_core.language_models import BaseLLM
 
-from ...config.config_files import LOCAL_COLLECTIONS_FILE
 from ..search_engine_base import BaseSearchEngine
 from ..search_engine_factory import create_search_engine
+from ..search_engines_config import local_search_engines
+from .search_engine_local import LocalSearchEngine
 
 # Setup logging
 logger = logging.getLogger(__name__)
@@ -46,12 +46,7 @@ class LocalAllSearchEngine(BaseSearchEngine):
         # Find all local collection search engines
         self.local_engines = {}
         try:
-            local_collections = toml.load(LOCAL_COLLECTIONS_FILE)
-
-            for collection_id, collection in local_collections.items():
-                if not collection.get("enabled", True):
-                    continue
-
+            for collection_id in local_search_engines():
                 # Create a search engine for this collection
                 try:
                     engine = create_search_engine(
@@ -59,12 +54,13 @@ class LocalAllSearchEngine(BaseSearchEngine):
                         llm=llm,
                         max_filtered_results=max_filtered_results,
                     )
+                    engine = cast(LocalSearchEngine, engine)
 
                     if engine:
                         self.local_engines[collection_id] = {
                             "engine": engine,
-                            "name": collection.get("name", collection_id),
-                            "description": collection.get("description", ""),
+                            "name": engine.name,
+                            "description": engine.description,
                         }
                 except Exception as e:
                     logger.error(

local_deep_research/web_search_engines/search_engine_factory.py

@@ -2,11 +2,11 @@ import importlib
 import inspect
 import logging
 import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Optional
 
 from ..utilities.db_utils import get_db_setting
 from .search_engine_base import BaseSearchEngine
-from .search_engines_config import DEFAULT_SEARCH_ENGINE, SEARCH_ENGINES
+from .search_engines_config import default_search_engine, search_config
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -28,42 +28,31 @@ def create_search_engine(
         Initialized search engine instance or None if creation failed
     """
     # If engine name not found, use default
-    if engine_name not in SEARCH_ENGINES:
+    if engine_name not in search_config():
         logger.warning(
-            f"Search engine '{engine_name}' not found, using default: {DEFAULT_SEARCH_ENGINE}"
+            f"Search engine '{engine_name}' not found, using default: "
+            f"{default_search_engine()}"
         )
-        engine_name = DEFAULT_SEARCH_ENGINE
+        engine_name = default_search_engine()
 
     # Get engine configuration
-    engine_config = SEARCH_ENGINES[engine_name]
-    from ..config.config_files import settings
+    engine_config = search_config()[engine_name]
 
     # Set default max_results from config if not provided in kwargs
     if "max_results" not in kwargs:
-        max_results = get_db_setting("search.max_results", settings.search.max_results)
+        max_results = get_db_setting("search.max_results", 10)
         if max_results is None:
             max_results = 20
         kwargs["max_results"] = max_results
 
     # Check for API key requirements
     if engine_config.get("requires_api_key", False):
-        api_key_env = engine_config.get("api_key_env")
-
-        # First check environment variable
-        api_key = os.getenv(api_key_env)
+        api_key = os.getenv(f"LDR_{engine_name.upper()}_API_KEY")
         if not api_key:
-            api_key = os.getenv("LDR_" + api_key_env)
-
-        # If not found in environment, check Dynaconf settings
-        if not api_key and api_key_env:
-            # Convert env var name to settings path (e.g., BRAVE_API_KEY -> brave_api_key)
-            settings_key = api_key_env.lower()
-            api_key = settings.get(settings_key)
+            api_key = engine_config.get("api_key")
 
         if not api_key:
-            logger.info(
-                f"Required API key for {engine_name} not found in environment variable: {api_key_env} or settings"
-            )
+            logger.info(f"Required API key for {engine_name} not found in settings.")
             return None
 
     # Check for LLM requirements
@@ -139,7 +128,7 @@ def _create_full_search_wrapper(
 ) -> Optional[BaseSearchEngine]:
     """Create a full search wrapper for the base engine if supported"""
     try:
-        engine_config = SEARCH_ENGINES[engine_name]
+        engine_config = search_config()[engine_name]
 
         # Get full search class details
         module_path = engine_config.get("full_search_module")
@@ -232,47 +221,6 @@ def _create_full_search_wrapper(
         return base_engine
 
 
-def get_available_engines(
-    include_api_key_services: bool = True,
-) -> Union[Dict[str, str], List[str]]:
-    """
-    Get all available search engines from the configuration.
-
-    Args:
-        include_api_key_services: Whether to include engines that require API keys
-
-    Returns:
-        Dictionary of engine names mapped to descriptions, or a list of engine names
-    """
-    try:
-        # Get engines from SEARCH_ENGINES dict
-        available_engines = {}
-
-        for name, config in SEARCH_ENGINES.items():
-            # Skip hidden engines (those that start with _)
-            if name.startswith("_"):
-                continue
-
-            # Skip engines that require API keys if requested
-            if not include_api_key_services and config.get("requires_api_key", False):
-                continue
-
-            # Add to available engines with display name
-            strengths = config.get("strengths", [])
-            description = name.replace("_", " ").title()
-
-            if strengths and len(strengths) > 0:
-                description += f" - {strengths[0]}"
-
-            available_engines[name] = description
-
-        return available_engines
-    except Exception as e:
-        logger.error(f"Error getting available engines: {e}")
-        # Fall back to list of engines directly from keys
-        return list(SEARCH_ENGINES.keys())
-
-
 def get_search(
     search_tool: str,
     llm_instance,