local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/web_search_engines/search_engine_factory.py

@@ -1,208 +1,292 @@
-import os
 import importlib
 import inspect
 import logging
-from typing import Dict, List, Any, Optional
+import os
+from typing import Any, Dict, List, Optional, Union
 
+from ..utilities.db_utils import get_db_setting
 from .search_engine_base import BaseSearchEngine
-from .search_engines_config import SEARCH_ENGINES, DEFAULT_SEARCH_ENGINE
-from local_deep_research.config import settings
+from .search_engines_config import DEFAULT_SEARCH_ENGINE, SEARCH_ENGINES
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
-def create_search_engine(engine_name: str, llm=None, **kwargs) -> Optional[BaseSearchEngine]:
+def create_search_engine(
+    engine_name: str, llm=None, **kwargs
+) -> Optional[BaseSearchEngine]:
     """
     Create a search engine instance based on the engine name.
-    
+
     Args:
         engine_name: Name of the search engine to create
         llm: Language model instance (required for some engines like meta)
         **kwargs: Additional parameters to override defaults
-        
+
     Returns:
         Initialized search engine instance or None if creation failed
     """
     # If engine name not found, use default
     if engine_name not in SEARCH_ENGINES:
-        logger.warning(f"Search engine '{engine_name}' not found, using default: {DEFAULT_SEARCH_ENGINE}")
+        logger.warning(
+            f"Search engine '{engine_name}' not found, using default: {DEFAULT_SEARCH_ENGINE}"
+        )
         engine_name = DEFAULT_SEARCH_ENGINE
-    
+
     # Get engine configuration
     engine_config = SEARCH_ENGINES[engine_name]
-    from local_deep_research.config import settings
-    
+    from ..config.config_files import settings
+
     # Set default max_results from config if not provided in kwargs
-    if 'max_results' not in kwargs:
-        max_results = settings.search.max_results
-        if max_results == None: max_results=20
-        kwargs['max_results'] = max_results
-    
+    if "max_results" not in kwargs:
+        max_results = get_db_setting("search.max_results", settings.search.max_results)
+        if max_results is None:
+            max_results = 20
+        kwargs["max_results"] = max_results
+
     # Check for API key requirements
     if engine_config.get("requires_api_key", False):
         api_key_env = engine_config.get("api_key_env")
-        
+
         # First check environment variable
         api_key = os.getenv(api_key_env)
         if not api_key:
             api_key = os.getenv("LDR_" + api_key_env)
 
-
         # If not found in environment, check Dynaconf settings
         if not api_key and api_key_env:
             # Convert env var name to settings path (e.g., BRAVE_API_KEY -> brave_api_key)
             settings_key = api_key_env.lower()
             api_key = settings.get(settings_key)
-            
+
         if not api_key:
-            logger.info(f"Required API key for {engine_name} not found in environment variable: {api_key_env} or settings")
+            logger.info(
+                f"Required API key for {engine_name} not found in environment variable: {api_key_env} or settings"
+            )
             return None
-    
+
     # Check for LLM requirements
     if engine_config.get("requires_llm", False) and not llm:
-        logger.info(f"Engine {engine_name} requires an LLM instance but none was provided")
+        logger.info(
+            f"Engine {engine_name} requires an LLM instance but none was provided"
+        )
         return None
-    
+
     try:
         # Load the engine class
         module_path = engine_config["module_path"]
         class_name = engine_config["class_name"]
-        
-        module = importlib.import_module(module_path)
+
+        package = None
+        if module_path.startswith("."):
+            # This is a relative import. Assume it's relative to
+            # `web_search_engines`.
+            package = "local_deep_research.web_search_engines"
+        module = importlib.import_module(module_path, package=package)
         engine_class = getattr(module, class_name)
-        
+
         # Get the engine class's __init__ parameters to filter out unsupported ones
         engine_init_signature = inspect.signature(engine_class.__init__)
         engine_init_params = list(engine_init_signature.parameters.keys())
-        
+
         # Combine default parameters with provided ones
         all_params = {**engine_config.get("default_params", {}), **kwargs}
-        
+
         # Filter out parameters that aren't accepted by the engine class
         # Note: 'self' is always the first parameter of instance methods, so we skip it
-        filtered_params = {k: v for k, v in all_params.items() if k in engine_init_params[1:]}
-        
+        filtered_params = {
+            k: v for k, v in all_params.items() if k in engine_init_params[1:]
+        }
+
         # Add LLM if required
         if engine_config.get("requires_llm", False):
             filtered_params["llm"] = llm
-        
+
         # Add API key if required and not already in filtered_params
-        if engine_config.get("requires_api_key", False) and "api_key" not in filtered_params:
+        if (
+            engine_config.get("requires_api_key", False)
+            and "api_key" not in filtered_params
+        ):
             api_key_env = engine_config.get("api_key_env")
             if api_key_env:
                 api_key = os.getenv(api_key_env)
                 if api_key:
                     filtered_params["api_key"] = api_key
-        
-        logger.info(f"Creating {engine_name} with filtered parameters: {filtered_params.keys()}")
-        
+
+        logger.info(
+            f"Creating {engine_name} with filtered parameters: {filtered_params.keys()}"
+        )
+
         # Create the engine instance with filtered parameters
         engine = engine_class(**filtered_params)
-        
+
         # Check if we need to wrap with full search capabilities
-        if kwargs.get("use_full_search", False) and engine_config.get("supports_full_search", False):
+        if kwargs.get("use_full_search", False) and engine_config.get(
+            "supports_full_search", False
+        ):
             return _create_full_search_wrapper(engine_name, engine, llm, kwargs)
-        
+
         return engine
-        
+
     except Exception as e:
         logger.info(f"Failed to create search engine '{engine_name}': {str(e)}")
         return None
 
 
-def _create_full_search_wrapper(engine_name: str, base_engine: BaseSearchEngine, llm, params: Dict[str, Any]) -> Optional[BaseSearchEngine]:
+def _create_full_search_wrapper(
+    engine_name: str, base_engine: BaseSearchEngine, llm, params: Dict[str, Any]
+) -> Optional[BaseSearchEngine]:
     """Create a full search wrapper for the base engine if supported"""
     try:
         engine_config = SEARCH_ENGINES[engine_name]
-        
+
         # Get full search class details
         module_path = engine_config.get("full_search_module")
         class_name = engine_config.get("full_search_class")
-        
+
         if not module_path or not class_name:
             logger.warning(f"Full search configuration missing for {engine_name}")
             return base_engine
-        
+
         # Import the full search class
         module = importlib.import_module(module_path)
         full_search_class = getattr(module, class_name)
-        
+
         # Get the wrapper's __init__ parameters to filter out unsupported ones
         wrapper_init_signature = inspect.signature(full_search_class.__init__)
-        wrapper_init_params = list(wrapper_init_signature.parameters.keys())[1:]  # Skip 'self'
-        
+        wrapper_init_params = list(wrapper_init_signature.parameters.keys())[
+            1:
+        ]  # Skip 'self'
+
         # Extract relevant parameters for the full search wrapper
         wrapper_params = {k: v for k, v in params.items() if k in wrapper_init_params}
-        
+
         # Special case for SerpAPI which needs the API key directly
         if engine_name == "serpapi" and "serpapi_api_key" in wrapper_init_params:
             serpapi_api_key = os.getenv("SERP_API_KEY")
             if serpapi_api_key:
                 wrapper_params["serpapi_api_key"] = serpapi_api_key
-                
+
             # Map some parameter names to what the wrapper expects
-            if "language" in params and "search_language" not in params and "language" in wrapper_init_params:
+            if (
+                "language" in params
+                and "search_language" not in params
+                and "language" in wrapper_init_params
+            ):
                 wrapper_params["language"] = params["language"]
-                
-            if "safesearch" not in wrapper_params and "safe_search" in params and "safesearch" in wrapper_init_params:
-                wrapper_params["safesearch"] = "active" if params["safe_search"] else "off"
-        
+
+            if (
+                "safesearch" not in wrapper_params
+                and "safe_search" in params
+                and "safesearch" in wrapper_init_params
+            ):
+                wrapper_params["safesearch"] = (
+                    "active" if params["safe_search"] else "off"
+                )
+
         # Special case for Brave which needs the API key directly
         if engine_name == "brave" and "api_key" in wrapper_init_params:
             brave_api_key = os.getenv("BRAVE_API_KEY")
             if brave_api_key:
                 wrapper_params["api_key"] = brave_api_key
-                
+
             # Map some parameter names to what the wrapper expects
-            if "language" in params and "search_language" not in params and "language" in wrapper_init_params:
+            if (
+                "language" in params
+                and "search_language" not in params
+                and "language" in wrapper_init_params
+            ):
                 wrapper_params["language"] = params["language"]
-                
-            if "safesearch" not in wrapper_params and "safe_search" in params and "safesearch" in wrapper_init_params:
-                wrapper_params["safesearch"] = "moderate" if params["safe_search"] else "off"
-        
+
+            if (
+                "safesearch" not in wrapper_params
+                and "safe_search" in params
+                and "safesearch" in wrapper_init_params
+            ):
+                wrapper_params["safesearch"] = (
+                    "moderate" if params["safe_search"] else "off"
+                )
+
         # Always include llm if it's a parameter
         if "llm" in wrapper_init_params:
             wrapper_params["llm"] = llm
-        
+
         # If the wrapper needs the base engine and has a parameter for it
         if "web_search" in wrapper_init_params:
             wrapper_params["web_search"] = base_engine
-        
-        logger.debug(f"Creating full search wrapper for {engine_name} with filtered parameters: {wrapper_params.keys()}")
-        
+
+        logger.debug(
+            f"Creating full search wrapper for {engine_name} with filtered parameters: {wrapper_params.keys()}"
+        )
+
         # Create the full search wrapper with filtered parameters
         full_search = full_search_class(**wrapper_params)
-        
+
         return full_search
-        
+
     except Exception as e:
-        logger.error(f"Failed to create full search wrapper for {engine_name}: {str(e)}")
+        logger.error(
+            f"Failed to create full search wrapper for {engine_name}: {str(e)}"
+        )
         return base_engine
 
 
+def get_available_engines(
+    include_api_key_services: bool = True,
+) -> Union[Dict[str, str], List[str]]:
+    """
+    Get all available search engines from the configuration.
+
+    Args:
+        include_api_key_services: Whether to include engines that require API keys
+
+    Returns:
+        Dictionary of engine names mapped to descriptions, or a list of engine names
+    """
+    try:
+        # Get engines from SEARCH_ENGINES dict
+        available_engines = {}
+
+        for name, config in SEARCH_ENGINES.items():
+            # Skip hidden engines (those that start with _)
+            if name.startswith("_"):
+                continue
+
+            # Skip engines that require API keys if requested
+            if not include_api_key_services and config.get("requires_api_key", False):
+                continue
+
+            # Add to available engines with display name
+            strengths = config.get("strengths", [])
+            description = name.replace("_", " ").title()
 
-def get_available_engines(include_api_key_services: bool = True):
-    """Get a list of available engine names"""
-    if include_api_key_services:
+            if strengths and len(strengths) > 0:
+                description += f" - {strengths[0]}"
+
+            available_engines[name] = description
+
+        return available_engines
+    except Exception as e:
+        logger.error(f"Error getting available engines: {e}")
+        # Fall back to list of engines directly from keys
         return list(SEARCH_ENGINES.keys())
-    else:
-        return [name for name, config in SEARCH_ENGINES.items() 
-                if not config.get("requires_api_key", False)]
-
-
-def get_search(search_tool: str, llm_instance, 
-               max_results: int = 10, 
-               region: str = "us",
-               time_period: str = "y",
-               safe_search: bool = True,
-               search_snippets_only: bool = False,
-               search_language: str = "English",
-               max_filtered_results: Optional[int] = None):
+
+
+def get_search(
+    search_tool: str,
+    llm_instance,
+    max_results: int = 10,
+    region: str = "us",
+    time_period: str = "y",
+    safe_search: bool = True,
+    search_snippets_only: bool = False,
+    search_language: str = "English",
+    max_filtered_results: Optional[int] = None,
+):
     """
     Get search tool instance based on the provided parameters.
-    
+
     Args:
         search_tool: Name of the search engine to use
         llm_instance: Language model instance
@@ -213,7 +297,7 @@ def get_search(search_tool: str, llm_instance,
         search_snippets_only: Whether to return just snippets (vs. full content)
         search_language: Language for search results
         max_filtered_results: Maximum number of results to keep after filtering
-        
+
     Returns:
         Initialized search engine instance
     """
@@ -222,43 +306,49 @@ def get_search(search_tool: str, llm_instance,
         "max_results": max_results,
         "llm": llm_instance,  # Only used by engines that need it
     }
-    
+
     # Add max_filtered_results if provided
     if max_filtered_results is not None:
         params["max_filtered_results"] = max_filtered_results
-    
+
     # Add engine-specific parameters
     if search_tool in ["duckduckgo", "serpapi", "google_pse", "brave"]:
-        params.update({
-            "region": region,
-            "safe_search": safe_search,
-            "use_full_search": not search_snippets_only
-        })
-    
+        params.update(
+            {
+                "region": region,
+                "safe_search": safe_search,
+                "use_full_search": not search_snippets_only,
+            }
+        )
+
     if search_tool in ["serpapi", "brave", "google_pse"]:
         params["search_language"] = search_language
-        
+
     if search_tool == "serpapi":
         params["time_period"] = time_period
-    
+
     # Create and return the search engine
-    logger.info(f"Creating search engine for tool: {search_tool} with params: {params.keys()}")
+    logger.info(
+        f"Creating search engine for tool: {search_tool} with params: {params.keys()}"
+    )
     engine = create_search_engine(search_tool, **params)
-    
+
     # Add debugging to check if engine is None
     if engine is None:
-        logger.error(f"Failed to create search engine for {search_tool} - returned None")
+        logger.error(
+            f"Failed to create search engine for {search_tool} - returned None"
+        )
     else:
         engine_type = type(engine).__name__
         logger.info(f"Successfully created search engine of type: {engine_type}")
         # Check if the engine has run method
-        if hasattr(engine, 'run'):
+        if hasattr(engine, "run"):
             logger.info(f"Engine has 'run' method: {getattr(engine, 'run')}")
         else:
-            logger.error(f"Engine does NOT have 'run' method!")
-            
+            logger.error("Engine does NOT have 'run' method!")
+
         # For SearxNG, check availability flag
-        if hasattr(engine, 'is_available'):
+        if hasattr(engine, "is_available"):
             logger.info(f"Engine availability flag: {engine.is_available}")
-    
+
     return engine

local_deep_research/web_search_engines/search_engines_config.py

@@ -2,11 +2,13 @@
 Configuration file for search engines.
 Loads search engine definitions from the user's configuration.
 """
+
 import logging
 import os
+
 import toml
-from pathlib import Path
-from local_deep_research.config import CONFIG_DIR, LOCAL_COLLECTIONS_FILE
+
+from ..config.config_files import CONFIG_DIR, LOCAL_COLLECTIONS_FILE
 
 logger = logging.getLogger(__name__)
 
@@ -22,23 +24,28 @@ if os.path.exists(SEARCH_ENGINES_FILE):
     try:
         # Load the TOML file directly
         config_data = toml.load(SEARCH_ENGINES_FILE)
-        
+
         # Extract search engine definitions
         for key, value in config_data.items():
             if key == "DEFAULT_SEARCH_ENGINE":
                 DEFAULT_SEARCH_ENGINE = value
             elif isinstance(value, dict):
                 SEARCH_ENGINES[key] = value
-        
-        logger.info(f"Loaded {len(SEARCH_ENGINES)} search engines from configuration file")
+
+        logger.info(
+            f"Loaded {len(SEARCH_ENGINES)} search engines from configuration file"
+        )
+        logger.info(f"\n  {', '.join(sorted(SEARCH_ENGINES.keys()))} \n")
     except Exception as e:
         logger.error(f"Error loading search engines from TOML file: {e}")
 else:
-    logger.warning(f"Search engines configuration file not found: {SEARCH_ENGINES_FILE}")
+    logger.warning(
+        f"Search engines configuration file not found: {SEARCH_ENGINES_FILE}"
+    )
 
 # Add alias for 'auto' if it exists
-if 'auto' in SEARCH_ENGINES and 'meta' not in SEARCH_ENGINES:
-    SEARCH_ENGINES['meta'] = SEARCH_ENGINES['auto']
+if "auto" in SEARCH_ENGINES and "meta" not in SEARCH_ENGINES:
+    SEARCH_ENGINES["meta"] = SEARCH_ENGINES["auto"]
 
 # Register local document collections
 
@@ -52,20 +59,20 @@ if os.path.exists(LOCAL_COLLECTIONS_FILE):
                 "module_path": "local_deep_research.web_search_engines.engines.search_engine_local",
                 "class_name": "LocalSearchEngine",
                 "default_params": config,
-                "requires_llm": True
+                "requires_llm": True,
             }
-            
+
             # Copy these specific fields to the top level if they exist
             for field in ["strengths", "weaknesses", "reliability", "description"]:
                 if field in config:
                     engine_config[field] = config[field]
-            
+
             SEARCH_ENGINES[collection] = engine_config
-            
-        logger.info(f"Registered local document collections as search engines")
+
+        logger.info("Registered local document collections as search engines")
     except Exception as e:
         logger.error(f"Error loading local collections from TOML file: {e}")
 # Ensure the meta search engine is still available at the end if it exists
-if 'auto' in SEARCH_ENGINES:
+if "auto" in SEARCH_ENGINES:
     meta_config = SEARCH_ENGINES["auto"]
-    SEARCH_ENGINES["auto"] = meta_config
+    SEARCH_ENGINES["auto"] = meta_config