local-deep-research 0.3.12__py3-none-any.whl → 0.4.0__py3-none-any.whl

local_deep_research/web/static/js/components/research.js

@@ -718,18 +718,48 @@
                     usingFallbackModels = true;
                 }
             } else if (providerUpper === 'OPENAI_ENDPOINT') {
-                // For custom endpoints, show a mix of models as examples
                 models = allModels.filter(model => {
                     if (!model || typeof model !== 'object') return false;
 
                     // Skip provider options
                     if (model.value && !model.id && !model.name) return false;
 
-                    // Include OpenAI and Anthropic models as examples
                     const modelProvider = (model.provider || '').toUpperCase();
-                    return modelProvider === 'OPENAI' || modelProvider === 'ANTHROPIC';
+                    return modelProvider === 'OPENAI_ENDPOINT';
                 });
 
+                console.log(`Found ${models.length} models with provider="OPENAI_ENDPOINT"`);
+
+                if (models.length === 0) {
+                    console.log('No OPENAI_ENDPOINT models found, checking for models with "Custom" in label');
+                    models = allModels.filter(model => {
+                        if (!model || typeof model !== 'object') return false;
+                        
+                        // Skip provider options
+                        if (model.value && !model.id && !model.name) return false;
+                        
+                        const modelLabel = (model.label || '').toLowerCase();
+                        return modelLabel.includes('custom');
+                    });
+                    
+                    console.log(`Found ${models.length} models with "Custom" in label`);
+                }
+
+                if (models.length === 0) {
+                    console.log('No OPENAI_ENDPOINT or Custom models found, using OpenAI models as examples');
+                    models = allModels.filter(model => {
+                        if (!model || typeof model !== 'object') return false;
+
+                        // Skip provider options
+                        if (model.value && !model.id && !model.name) return false;
+
+                        const modelProvider = (model.provider || '').toUpperCase();
+                        const modelId = (model.id || model.value || '').toLowerCase();
+                        return modelProvider === 'OPENAI' || 
+                               modelId.includes('gpt');
+                    });
+                }
+
                 // Add fallbacks if necessary
                 if (models.length === 0) {
                     console.log('No models found for custom endpoint, using fallbacks');
@@ -1411,6 +1441,17 @@
             });
         }
 
+        // Process Custom OpenAI Endpoint models
+        if (data.providers && data.providers.openai_endpoint_models) {
+            data.providers.openai_endpoint_models.forEach(model => {
+                formatted.push({
+                    ...model,
+                    id: model.value,
+                    provider: 'OPENAI_ENDPOINT'
+                });
+            });
+        }
+
         return formatted;
     }
 

local_deep_research/web/static/js/components/settings.js

@@ -2761,6 +2761,21 @@
             });
         }
 
+        // Add Custom OpenAI Endpoint models if available
+        if (data.providers && data.providers.openai_endpoint_models && data.providers.openai_endpoint_models.length > 0) {
+            const openai_endpoint_models = data.providers.openai_endpoint_models;
+            console.log('Found OpenAI Endpoint models:', openai_endpoint_models.length);
+
+            // Add provider information to each model
+            openai_endpoint_models.forEach(model => {
+                formattedModels.push({
+                    value: model.value,
+                    label: model.label,
+                    provider: 'OPENAI_ENDPOINT' // Ensure provider field is added
+                });
+            });
+        }
+
         // Update the global modelOptions array
         modelOptions = formattedModels;
         console.log('Final modelOptions:', modelOptions.length, 'models');
@@ -3673,6 +3688,18 @@
 
                 return false;
             }
+            
+            if (providerUpper === 'OPENAI_ENDPOINT') {
+                if (model.provider && model.provider.toUpperCase() === 'OPENAI_ENDPOINT') {
+                    return true;
+                }
+                
+                if (model.label && model.label.toLowerCase().includes('custom')) {
+                    return true;
+                }
+                
+                return false;
+            }
 
             // For other providers, use standard matching
             if (model.provider) {

local_deep_research/web/static/js/services/socket.js

@@ -85,6 +85,29 @@ window.socket = (function() {
             }
         });
 
+        // Add handler for search engine selection events
+        socket.on('search_engine_selected', (data) => {
+            console.log('Received search_engine_selected event:', data);
+            if (data && data.engine) {
+                const engineName = data.engine;
+                const resultCount = data.result_count || 0;
+
+                // Add to log panel
+                if (typeof window.addConsoleLog === 'function') {
+                    // Format engine name - capitalize first letter
+                    const displayEngineName = engineName.charAt(0).toUpperCase() + engineName.slice(1);
+                    const message = `Search engine selected: ${displayEngineName} (found ${resultCount} results)`;
+                    window.addConsoleLog(message, 'info', {
+                        type: 'info',
+                        phase: 'engine_selected',
+                        engine: engineName,
+                        result_count: resultCount,
+                        is_engine_selection: true
+                    });
+                }
+            }
+        });
+
         socket.on('disconnect', (reason) => {
             console.log('Socket disconnected:', reason);
 
@@ -237,6 +260,30 @@ window.socket = (function() {
             });
         }
 
+        // Handle special engine selection events
+        if (data.event === 'search_engine_selected' || (data.engine && data.result_count !== undefined)) {
+            // Extract engine information
+            const engineName = data.engine || 'unknown';
+            const resultCount = data.result_count || 0;
+
+            // Log the event
+            console.log(`Search engine selected: ${engineName} (found ${resultCount} results)`);
+
+            // Add to log panel as an info message with special metadata
+            if (typeof window.addConsoleLog === 'function') {
+                // Format engine name - capitalize first letter
+                const displayEngineName = engineName.charAt(0).toUpperCase() + engineName.slice(1);
+                const message = `Search engine selected: ${displayEngineName} (found ${resultCount} results)`;
+                window.addConsoleLog(message, 'info', {
+                    type: 'info',
+                    phase: 'engine_selected',
+                    engine: engineName,
+                    result_count: resultCount,
+                    is_engine_selection: true
+                });
+            }
+        }
+
         // Initialize message tracking if not exists
         window._processedSocketMessages = window._processedSocketMessages || new Map();
 

local_deep_research/web_search_engines/default_search_engines.py

@@ -0,0 +1,38 @@
+"""
+Default search engine configurations.
+This file can be used to initialize the search engine configurations.
+"""
+
+def get_default_elasticsearch_config():
+    """
+    Returns the default Elasticsearch search engine configuration.
+    
+    Returns:
+        dict: Default configuration for Elasticsearch search engine
+    """
+    return {
+        "module_path": "local_deep_research.web_search_engines.engines.search_engine_elasticsearch",
+        "class_name": "ElasticsearchSearchEngine",
+        "requires_llm": True,
+        "default_params": {
+            "hosts": ["http://172.16.4.131:9200"],
+            "index_name": "sample_documents",
+            "highlight_fields": ["content", "title"],
+            "search_fields": ["content", "title"],
+        },
+        "description": "Search engine for Elasticsearch databases",
+        "strengths": "Efficient for searching document collections and structured data",
+        "weaknesses": "Requires an Elasticsearch instance and properly indexed data",
+        "reliability": "High, depending on your Elasticsearch setup",
+    }
+
+def get_default_search_engine_configs():
+    """
+    Returns a dictionary of default search engine configurations.
+    
+    Returns:
+        dict: Dictionary of default search engine configurations
+    """
+    return {
+        "elasticsearch": get_default_elasticsearch_config(),
+    } 

local_deep_research/web_search_engines/engines/meta_search_engine.py

@@ -1,6 +1,7 @@
-import logging
 from typing import Any, Dict, List, Optional
 
+from loguru import logger
+
 from ...utilities.db_utils import get_db_setting
 from ...web.services.socket_service import emit_socket_event
 from ..search_engine_base import BaseSearchEngine
@@ -8,10 +9,6 @@ from ..search_engine_factory import create_search_engine
 from ..search_engines_config import search_config
 from .search_engine_wikipedia import WikipediaSearchEngine
 
-# Setup logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
 
 class MetaSearchEngine(BaseSearchEngine):
     """
@@ -120,6 +117,8 @@ class MetaSearchEngine(BaseSearchEngine):
     def analyze_query(self, query: str) -> List[str]:
         """
         Analyze the query to determine the best search engines to use.
+        Prioritizes SearXNG for general queries, but selects specialized engines
+        for domain-specific queries (e.g., scientific papers, code).
 
         Args:
             query: The search query
@@ -128,10 +127,57 @@ class MetaSearchEngine(BaseSearchEngine):
             List of search engine names sorted by suitability
         """
         try:
-            # Check if the LLM is available to help select engines
-            if not self.llm:
+            # First check if this is a specialized query that should use specific engines
+            specialized_domains = {
+                "scientific paper": ["arxiv", "pubmed", "wikipedia"],
+                "medical research": ["pubmed", "searxng"],
+                "clinical": ["pubmed", "searxng"],
+                "github": ["github", "searxng"],
+                "repository": ["github", "searxng"],
+                "code": ["github", "searxng"],
+                "programming": ["github", "searxng"],
+            }
+
+            # Quick heuristic check for specialized queries
+            query_lower = query.lower()
+            for term, engines in specialized_domains.items():
+                if term in query_lower:
+                    valid_engines = []
+                    for engine in engines:
+                        if engine in self.available_engines:
+                            valid_engines.append(engine)
+
+                    if valid_engines:
+                        logger.info(
+                            f"Detected specialized query type: {term}, using engines: {valid_engines}"
+                        )
+                        return valid_engines
+
+            # For searches containing "arxiv", prioritize the arxiv engine
+            if "arxiv" in query_lower and "arxiv" in self.available_engines:
+                return ["arxiv"] + [e for e in self.available_engines if e != "arxiv"]
+
+            # For searches containing "pubmed", prioritize the pubmed engine
+            if "pubmed" in query_lower and "pubmed" in self.available_engines:
+                return ["pubmed"] + [e for e in self.available_engines if e != "pubmed"]
+
+            # Check if SearXNG is available and prioritize it for general queries
+            if "searxng" in self.available_engines:
+                # For general queries, return SearXNG first followed by reliability-ordered engines
+                engines_without_searxng = [
+                    e for e in self.available_engines if e != "searxng"
+                ]
+                reliability_sorted = sorted(
+                    engines_without_searxng,
+                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    reverse=True,
+                )
+                return ["searxng"] + reliability_sorted
+
+            # If LLM is not available or SearXNG is not available, fall back to reliability
+            if not self.llm or "searxng" not in self.available_engines:
                 logger.warning(
-                    "No LLM available for query analysis, using default engines"
+                    "No LLM available or SearXNG not available, using reliability-based engines"
                 )
                 # Return engines sorted by reliability
                 return sorted(
@@ -157,8 +203,8 @@ class MetaSearchEngine(BaseSearchEngine):
                         engines_info.append(
                             f"- {engine_name}: {description}\n  Strengths: {strengths}\n  Weaknesses: {weaknesses}"
                         )
-                except KeyError as e:
-                    logger.error(f"Missing key for engine {engine_name}: {str(e)}")
+                except KeyError:
+                    logger.exception(f"Missing key for engine {engine_name}")
 
             # Only proceed if we have engines available to choose from
             if not engines_info:
@@ -171,6 +217,7 @@ class MetaSearchEngine(BaseSearchEngine):
                     reverse=True,
                 )
 
+            # Use a stronger prompt that emphasizes SearXNG preference for general queries
             prompt = f"""You are a search query analyst. Consider this search query:
 
 QUERY: {query}
@@ -179,11 +226,17 @@ I have these search engines available:
 {chr(10).join(engines_info)}
 
 Determine which search engines would be most appropriate for answering this query.
-First analyze the nature of the query (factual, scientific, code-related, etc.)
-Then select the 1-3 most appropriate search engines for this type of query.
+First analyze the nature of the query: Is it factual, scientific, code-related, medical, etc.?
+
+IMPORTANT GUIDELINES:
+- Use SearXNG for most general queries as it combines results from multiple search engines
+- For academic/scientific searches, prefer arXiv
+- For medical research, prefer PubMed
+- For code repositories and programming, prefer GitHub
+- For every other query type, SearXNG is usually the best option
 
-Output ONLY a comma-separated list of the search engine names in order of most appropriate to least appropriate.
-Example output: wikipedia,arxiv,github"""
+Output ONLY a comma-separated list of 1-3 search engine names in order of most appropriate to least appropriate.
+Example output: searxng,wikipedia,brave"""
 
             # Get analysis from LLM
             response = self.llm.invoke(prompt)
@@ -201,7 +254,16 @@ Example output: wikipedia,arxiv,github"""
                 if cleaned_name in self.available_engines:
                     valid_engines.append(cleaned_name)
 
-            # If no valid engines were returned, use default order based on reliability
+            # If SearXNG is available but not selected by the LLM, add it as a fallback
+            if "searxng" in self.available_engines and "searxng" not in valid_engines:
+                # Add it as the last option if the LLM selected others
+                if valid_engines:
+                    valid_engines.append("searxng")
+                # Use it as the first option if no valid engines were selected
+                else:
+                    valid_engines = ["searxng"]
+
+            # If still no valid engines, use reliability-based ordering
             if not valid_engines:
                 valid_engines = sorted(
                     self.available_engines,
@@ -210,14 +272,21 @@ Example output: wikipedia,arxiv,github"""
                 )
 
             return valid_engines
-        except Exception as e:
-            logger.error(f"Error analyzing query with LLM: {str(e)}")
-            # Fall back to reliability-based ordering
-            return sorted(
-                self.available_engines,
-                key=lambda x: search_config().get(x, {}).get("reliability", 0),
-                reverse=True,
-            )
+        except Exception:
+            logger.exception("Error analyzing query with LLM")
+            # Fall back to SearXNG if available, then reliability-based ordering
+            if "searxng" in self.available_engines:
+                return ["searxng"] + sorted(
+                    [e for e in self.available_engines if e != "searxng"],
+                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    reverse=True,
+                )
+            else:
+                return sorted(
+                    self.available_engines,
+                    key=lambda x: search_config().get(x, {}).get("reliability", 0),
+                    reverse=True,
+                )
 
     def _get_previews(self, query: str) -> List[Dict[str, Any]]:
         """
@@ -277,10 +346,8 @@ Example output: wikipedia,arxiv,github"""
                             "search_engine_selected",
                             {"engine": engine_name, "result_count": len(previews)},
                         )
-                    except Exception as socket_error:
-                        logger.error(
-                            f"Socket emit error (non-critical): {str(socket_error)}"
-                        )
+                    except Exception:
+                        logger.exception("Socket emit error (non-critical)")
 
                     return previews
 
@@ -289,7 +356,7 @@ Example output: wikipedia,arxiv,github"""
 
             except Exception as e:
                 error_msg = f"Error getting previews from {engine_name}: {str(e)}"
-                logger.error(error_msg)
+                logger.exception(error_msg)
                 all_errors.append(error_msg)
 
         # If we reach here, all engines failed, use fallback
@@ -325,9 +392,9 @@ Example output: wikipedia,arxiv,github"""
             try:
                 logger.info(f"Using {self._selected_engine_name} to get full content")
                 return self._selected_engine._get_full_content(relevant_items)
-            except Exception as e:
-                logger.error(
-                    f"Error getting full content from {self._selected_engine_name}: {str(e)}"
+            except Exception:
+                logger.exception(
+                    f"Error getting full content from {self._selected_engine_name}"
                 )
                 # Fall back to returning relevant items without full content
                 return relevant_items
@@ -354,8 +421,8 @@ Example output: wikipedia,arxiv,github"""
                 common_params["max_filtered_results"] = self.max_filtered_results
 
             engine = create_search_engine(engine_name, **common_params)
-        except Exception as e:
-            logger.error(f"Error creating engine instance for {engine_name}: {str(e)}")
+        except Exception:
+            logger.exception(f"Error creating engine instance for {engine_name}")
             return None
 
         if engine:

local_deep_research/web_search_engines/engines/search_engine_arxiv.py

@@ -1,14 +1,15 @@
-import logging
 from typing import Any, Dict, List, Optional
 
 import arxiv
 from langchain_core.language_models import BaseLLM
+from loguru import logger
 
+from ...advanced_search_system.filters.journal_reputation_filter import (
+    JournalReputationFilter,
+)
 from ...config import search_config
 from ..search_engine_base import BaseSearchEngine
 
-logger = logging.getLogger(__name__)
-
 
 class ArXivSearchEngine(BaseSearchEngine):
     """arXiv search engine implementation with two-phase approach"""
@@ -37,9 +38,22 @@ class ArXivSearchEngine(BaseSearchEngine):
             llm: Language model for relevance filtering
             max_filtered_results: Maximum number of results to keep after filtering
         """
+        # Initialize the journal reputation filter if needed.
+        content_filters = []
+        journal_filter = JournalReputationFilter.create_default(
+            model=llm, engine_name="arxiv"
+        )
+        if journal_filter is not None:
+            content_filters.append(journal_filter)
+
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
         super().__init__(
-            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+            llm=llm,
+            max_filtered_results=max_filtered_results,
+            max_results=max_results,
+            # We deliberately do this filtering after relevancy checks,
+            # because it is potentially quite slow.
+            content_filters=content_filters,
         )
         self.max_results = max(self.max_results, 25)
         self.sort_by = sort_by
@@ -133,14 +147,15 @@ class ArXivSearchEngine(BaseSearchEngine):
                         if paper.published
                         else None
                     ),
+                    "journal_ref": paper.journal_ref,
                 }
 
                 previews.append(preview)
 
             return previews
 
-        except Exception as e:
-            logger.error(f"Error getting arXiv previews: {e}")
+        except Exception:
+            logger.exception("Error getting arXiv previews")
             return []
 
     def _get_full_content(
@@ -203,7 +218,6 @@ class ArXivSearchEngine(BaseSearchEngine):
                         "categories": paper.categories,
                         "summary": paper.summary,  # Full summary
                         "comment": paper.comment,
-                        "journal_ref": paper.journal_ref,
                         "doi": paper.doi,
                     }
                 )
@@ -263,17 +277,17 @@ class ArXivSearchEngine(BaseSearchEngine):
                                                 "Successfully extracted text from PDF using pdfplumber"
                                             )
                                 except (ImportError, Exception) as e2:
-                                    logger.error(
+                                    logger.exception(
                                         f"PDF text extraction failed: {str(e1)}, then {str(e2)}"
                                     )
                                     logger.error(
                                         "Using paper summary as content instead"
                                     )
-                        except Exception as e:
-                            logger.error(f"Error extracting text from PDF: {e}")
+                        except Exception:
+                            logger.exception("Error extracting text from PDF")
                             logger.error("Using paper summary as content instead")
-                    except Exception as e:
-                        logger.error(f"Error downloading paper {paper.title}: {e}")
+                    except Exception:
+                        logger.exception(f"Error downloading paper {paper.title}")
                         result["pdf_path"] = None
                         pdf_count -= 1  # Decrement counter if download fails
                 elif (
@@ -349,6 +363,7 @@ class ArXivSearchEngine(BaseSearchEngine):
                 "authors": [
                     author.name for author in paper.authors[:3]
                 ],  # First 3 authors
+                "journal_ref": paper.journal_ref,
             }
 
             # Add full content if not in snippet-only mode
@@ -375,7 +390,6 @@ class ArXivSearchEngine(BaseSearchEngine):
                         "categories": paper.categories,
                         "summary": paper.summary,  # Full summary
                         "comment": paper.comment,
-                        "journal_ref": paper.journal_ref,
                         "doi": paper.doi,
                         "content": paper.summary,  # Use summary as content
                         "full_content": paper.summary,  # For consistency
@@ -388,13 +402,13 @@ class ArXivSearchEngine(BaseSearchEngine):
                         # Download the paper
                         paper_path = paper.download_pdf(dirpath=self.download_dir)
                         result["pdf_path"] = str(paper_path)
-                    except Exception as e:
-                        logger.error(f"Error downloading paper: {e}")
+                    except Exception:
+                        logger.exception("Error downloading paper")
 
             return result
 
-        except Exception as e:
-            logger.error(f"Error getting paper details: {e}")
+        except Exception:
+            logger.exception("Error getting paper details")
             return {}
 
     def search_by_author(

local_deep_research/web_search_engines/engines/search_engine_brave.py

@@ -64,11 +64,16 @@ class BraveSearchEngine(BaseSearchEngine):
                 "russian": "ru",
             }
 
-        # Get API key
-        brave_api_key = api_key or os.getenv("BRAVE_API_KEY")
+        # Get API key - check params, env vars, or database
+        from ...utilities.db_utils import get_db_setting
+
+        brave_api_key = api_key
+        if not brave_api_key:
+            brave_api_key = get_db_setting("search.engine.web.brave.api_key")
+
         if not brave_api_key:
             raise ValueError(
-                "BRAVE_API_KEY not found. Please provide api_key or set the BRAVE_API_KEY environment variable."
+                "Brave API key not found. Please provide api_key parameter, set the BRAVE_API_KEY environment variable, or set it in the UI settings."
             )
 
         # Get language code