local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_brave.py

@@ -1,31 +1,36 @@
-from langchain_community.tools import BraveSearch
-from typing import Dict, List, Any, Optional
+import logging
 import os
+from typing import Any, Dict, List, Optional
+
+from langchain_community.tools import BraveSearch
 from langchain_core.language_models import BaseLLM
 
-from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
-from local_deep_research import config
-import logging
+from ...config import search_config
+from ..search_engine_base import BaseSearchEngine
+
 logger = logging.getLogger(__name__)
 
+
 class BraveSearchEngine(BaseSearchEngine):
     """Brave search engine implementation with two-phase approach"""
 
-    def __init__(self,
-                max_results: int = 10,
-                region: str = "US",
-                time_period: str = "y",
-                safe_search: bool = True,
-                search_language: str = "English",
-                api_key: Optional[str] = None,
-                language_code_mapping: Optional[Dict[str, str]] = None,
-                llm: Optional[BaseLLM] = None,
-                include_full_content: bool = True,
-                max_filtered_results: Optional[int] = None,
-                **kwargs):
+    def __init__(
+        self,
+        max_results: int = 10,
+        region: str = "US",
+        time_period: str = "y",
+        safe_search: bool = True,
+        search_language: str = "English",
+        api_key: Optional[str] = None,
+        language_code_mapping: Optional[Dict[str, str]] = None,
+        llm: Optional[BaseLLM] = None,
+        include_full_content: bool = True,
+        max_filtered_results: Optional[int] = None,
+        **kwargs,
+    ):
         """
         Initialize the Brave search engine.
-        
+
         Args:
             max_results: Maximum number of search results
             region: Region code for search results
@@ -40,9 +45,11 @@ class BraveSearchEngine(BaseSearchEngine):
             **kwargs: Additional parameters (ignored but accepted for compatibility)
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        super().__init__(
+            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+        )
         self.include_full_content = include_full_content
-        
+
         # Set up language code mapping
         if language_code_mapping is None:
             language_code_mapping = {
@@ -56,21 +63,23 @@ class BraveSearchEngine(BaseSearchEngine):
                 "portuguese": "pt",
                 "russian": "ru",
             }
-        
+
         # Get API key
         brave_api_key = api_key or os.getenv("BRAVE_API_KEY")
         if not brave_api_key:
-            raise ValueError("BRAVE_API_KEY not found. Please provide api_key or set the BRAVE_API_KEY environment variable.")
-        
+            raise ValueError(
+                "BRAVE_API_KEY not found. Please provide api_key or set the BRAVE_API_KEY environment variable."
+            )
+
         # Get language code
         language_code = language_code_mapping.get(search_language.lower(), "en")
-        
+
         # Convert time period format to Brave's format
         brave_time_period = f"p{time_period}"
-        
+
         # Convert safe search to Brave's format
         brave_safe_search = "moderate" if safe_search else "off"
-        
+
         # Initialize Brave Search
         self.engine = BraveSearch.from_api_key(
             api_key=brave_api_key,
@@ -80,17 +89,18 @@ class BraveSearchEngine(BaseSearchEngine):
                 "search_lang": language_code,
                 "safesearch": brave_safe_search,
                 "freshness": brave_time_period,
-            }
+            },
         )
-        
+
         # Set user agent for Brave Search
         os.environ["USER_AGENT"] = "Local Deep Research/1.0"
-        
+
         # If full content is requested, initialize FullSearchResults
         if include_full_content:
             # Import FullSearchResults only if needed
             try:
-                from local_deep_research.web_search_engines.engines.full_search import FullSearchResults
+                from .full_search import FullSearchResults
+
                 self.full_search = FullSearchResults(
                     llm=llm,
                     web_search=self.engine,
@@ -98,37 +108,40 @@ class BraveSearchEngine(BaseSearchEngine):
                     max_results=max_results,
                     region=region,
                     time=time_period,
-                    safesearch=brave_safe_search
+                    safesearch=brave_safe_search,
                 )
             except ImportError:
-                logger.warning("Warning: FullSearchResults not available. Full content retrieval disabled.")
+                logger.warning(
+                    "Warning: FullSearchResults not available. Full content retrieval disabled."
+                )
                 self.include_full_content = False
-    
+
     def _get_previews(self, query: str) -> List[Dict[str, Any]]:
         """
         Get preview information from Brave Search.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of preview dictionaries
         """
         logger.info("Getting search results from Brave Search")
-        
+
         try:
             # Get search results from Brave Search
             raw_results = self.engine.run(query[:400])
-            
+
             # Parse results if they're in string format
             if isinstance(raw_results, str):
                 try:
                     import json
+
                     raw_results = json.loads(raw_results)
                 except json.JSONDecodeError:
                     logger.error("Error: Unable to parse BraveSearch response as JSON.")
                     return []
-            
+
             # Format results as previews
             previews = []
             for i, result in enumerate(raw_results):
@@ -138,39 +151,44 @@ class BraveSearchEngine(BaseSearchEngine):
                     "link": result.get("link", ""),
                     "snippet": result.get("snippet", ""),
                     "displayed_link": result.get("link", ""),
-                    "position": i
+                    "position": i,
                 }
-                
+
                 # Store full Brave result for later
                 preview["_full_result"] = result
-                
+
                 previews.append(preview)
-            
+
             # Store the previews for potential full content retrieval
             self._search_results = previews
-            
+
             return previews
-            
+
         except Exception as e:
             logger.error(f"Error getting Brave Search results: {e}")
             return []
-    
-    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    def _get_full_content(
+        self, relevant_items: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """
         Get full content for the relevant search results.
         If include_full_content is True and FullSearchResults is available,
         retrieves full webpage content for the results.
-        
+
         Args:
             relevant_items: List of relevant preview dictionaries
-            
+
         Returns:
             List of result dictionaries with full content if requested
         """
         # Check if we should get full content
-        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+        if (
+            hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
+            and search_config.SEARCH_SNIPPETS_ONLY
+        ):
             logger.info("Snippet-only mode, skipping full content retrieval")
-            
+
             # Return the relevant items with their full Brave information
             results = []
             for item in relevant_items:
@@ -182,28 +200,27 @@ class BraveSearchEngine(BaseSearchEngine):
                         del result["_full_result"]
                 else:
                     result = item
-                    
+
                 results.append(result)
-                
+
             return results
-            
+
         # If full content retrieval is enabled
-        if self.include_full_content and hasattr(self, 'full_search'):
+        if self.include_full_content and hasattr(self, "full_search"):
             logger.info("Retrieving full webpage content")
-            
+
             try:
-                # Extract only the links from relevant items
-                links = [item.get("link") for item in relevant_items if item.get("link")]
-                
                 # Use FullSearchResults to get full content
-                results_with_content = self.full_search._get_full_content(relevant_items)
-                
+                results_with_content = self.full_search._get_full_content(
+                    relevant_items
+                )
+
                 return results_with_content
-                
+
             except Exception as e:
                 logger.error(f"Error retrieving full content: {e}")
                 # Fall back to returning the items without full content
-        
+
         # Return items with their full Brave information
         results = []
         for item in relevant_items:
@@ -217,28 +234,28 @@ class BraveSearchEngine(BaseSearchEngine):
                 result = item.copy()
                 if "_full_result" in result:
                     del result["_full_result"]
-                
+
             results.append(result)
-            
+
         return results
-    
+
     def run(self, query: str) -> List[Dict[str, Any]]:
         """
         Execute a search using Brave Search with the two-phase approach.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of search results
         """
         logger.info("---Execute a search using Brave Search---")
-        
+
         # Use the implementation from the parent class which handles all phases
         results = super().run(query)
-        
+
         # Clean up
-        if hasattr(self, '_search_results'):
+        if hasattr(self, "_search_results"):
             del self._search_results
-            
+
         return results

local_deep_research/web_search_engines/engines/search_engine_ddg.py

@@ -1,26 +1,31 @@
+import logging
+from typing import Any, Dict, List, Optional
+
 from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
-from typing import Dict, List, Any, Optional
 from langchain_core.language_models import BaseLLM
 
-from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
-from local_deep_research.web_search_engines.engines.full_search import FullSearchResults  # Import the FullSearchResults class
-import logging
+from ..search_engine_base import BaseSearchEngine
+from .full_search import FullSearchResults  # Import the FullSearchResults class
+
 logger = logging.getLogger(__name__)
 
+
 class DuckDuckGoSearchEngine(BaseSearchEngine):
     """DuckDuckGo search engine implementation with two-phase retrieval"""
-    
-    def __init__(self,
-                max_results: int = 10,
-                region: str = "us",
-                safe_search: bool = True,
-                llm: Optional[BaseLLM] = None,
-                language: str = "English",
-                include_full_content: bool = False,
-                max_filtered_results=5):
+
+    def __init__(
+        self,
+        max_results: int = 10,
+        region: str = "us",
+        safe_search: bool = True,
+        llm: Optional[BaseLLM] = None,
+        language: str = "English",
+        include_full_content: bool = False,
+        max_filtered_results=5,
+    ):
         """
         Initialize the DuckDuckGo search engine.
-        
+
         Args:
             max_results: Maximum number of search results
             region: Region code for search results
@@ -30,19 +35,21 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
             include_full_content: Whether to include full webpage content in results
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        super().__init__(
+            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+        )
         self.region = region
         self.safe_search = safe_search
         self.language = language
         self.include_full_content = include_full_content
-        
+
         # Initialize the DuckDuckGo wrapper
         self.engine = DuckDuckGoSearchAPIWrapper(
             region=region,
             max_results=max_results,
-            safesearch="moderate" if safe_search else "off"
+            safesearch="moderate" if safe_search else "off",
         )
-        
+
         # Initialize FullSearchResults if full content is requested
         if include_full_content and llm:
             self.full_search = FullSearchResults(
@@ -52,44 +59,44 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
                 max_results=max_results,
                 region=region,
                 time="y",
-                safesearch="Moderate" if safe_search else "Off"
+                safesearch="Moderate" if safe_search else "Off",
             )
-    
+
     def run(self, query: str) -> List[Dict[str, Any]]:
         """
         Execute a search using DuckDuckGo with the two-phase approach.
         Respects config parameters:
         - SEARCH_SNIPPETS_ONLY: If True, only returns snippets without full content
         - SKIP_RELEVANCE_FILTER: If True, returns all results without filtering
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of search results
         """
         logger.info("---Execute a search using DuckDuckGo---")
-        
+
         # Implementation of the two-phase approach (from parent class)
         return super().run(query)
-    
+
     def _get_previews(self, query: str) -> List[Dict[str, Any]]:
         """
         Get preview information (titles and snippets) for initial search results.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of preview dictionaries with 'id', 'title', and 'snippet' keys
         """
         try:
             # Get search results from DuckDuckGo
             results = self.engine.results(query, max_results=self.max_results)
-            
+
             if not isinstance(results, list):
                 return []
-            
+
             # Process results to get previews
             previews = []
             for i, result in enumerate(results):
@@ -97,30 +104,32 @@ class DuckDuckGoSearchEngine(BaseSearchEngine):
                     "id": result.get("link"),  # Use URL as ID for DDG
                     "title": result.get("title", ""),
                     "snippet": result.get("snippet", ""),
-                    "link": result.get("link", "")
+                    "link": result.get("link", ""),
                 }
-                
+
                 previews.append(preview)
-            
+
             return previews
-            
+
         except Exception as e:
             logger.error(f"Error getting DuckDuckGo previews: {e}")
             return []
-    
-    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    def _get_full_content(
+        self, relevant_items: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """
         Get full content for the relevant items by using FullSearchResults.
-        
+
         Args:
             relevant_items: List of relevant preview dictionaries
-            
+
         Returns:
             List of result dictionaries with full content
         """
         # If we have FullSearchResults, use it to get full content
-        if hasattr(self, 'full_search'):
+        if hasattr(self, "full_search"):
             return self.full_search._get_full_content(relevant_items)
-        
+
         # Otherwise, just return the relevant items without full content
-        return relevant_items
+        return relevant_items