local-deep-research 0.1.26__py3-none-any.whl → 0.2.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_serpapi.py

@@ -1,31 +1,36 @@
-from langchain_community.utilities import SerpAPIWrapper
-from typing import Dict, List, Any, Optional
+import logging
 import os
+from typing import Any, Dict, List, Optional
+
+from langchain_community.utilities import SerpAPIWrapper
 from langchain_core.language_models import BaseLLM
 
-from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
-from local_deep_research import config
-import logging
+from ...config import search_config
+from ..search_engine_base import BaseSearchEngine
+
 logger = logging.getLogger(__name__)
 
+
 class SerpAPISearchEngine(BaseSearchEngine):
     """Google search engine implementation using SerpAPI with two-phase approach"""
 
-    def __init__(self,
-                max_results: int = 10,
-                region: str = "us",
-                time_period: str = "y",
-                safe_search: bool = True,
-                search_language: str = "English",
-                api_key: Optional[str] = None,
-                language_code_mapping: Optional[Dict[str, str]] = None,
-                llm: Optional[BaseLLM] = None,
-                include_full_content: bool = False,
-                max_filtered_results: Optional[int] = None,
-                **kwargs):
+    def __init__(
+        self,
+        max_results: int = 10,
+        region: str = "us",
+        time_period: str = "y",
+        safe_search: bool = True,
+        search_language: str = "English",
+        api_key: Optional[str] = None,
+        language_code_mapping: Optional[Dict[str, str]] = None,
+        llm: Optional[BaseLLM] = None,
+        include_full_content: bool = False,
+        max_filtered_results: Optional[int] = None,
+        **kwargs,
+    ):
         """
         Initialize the SerpAPI search engine.
-        
+
         Args:
             max_results: Maximum number of search results
             region: Region code for search results
@@ -40,9 +45,11 @@ class SerpAPISearchEngine(BaseSearchEngine):
             **kwargs: Additional parameters (ignored but accepted for compatibility)
         """
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
+        super().__init__(
+            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+        )
         self.include_full_content = include_full_content
-        
+
         # Set up language code mapping
         if language_code_mapping is None:
             language_code_mapping = {
@@ -56,15 +63,17 @@ class SerpAPISearchEngine(BaseSearchEngine):
                 "portuguese": "pt",
                 "russian": "ru",
             }
-        
+
         # Get API key
         serpapi_api_key = api_key or os.getenv("SERP_API_KEY")
         if not serpapi_api_key:
-            raise ValueError("SERP_API_KEY not found. Please provide api_key or set the SERP_API_KEY environment variable.")
-        
+            raise ValueError(
+                "SERP_API_KEY not found. Please provide api_key or set the SERP_API_KEY environment variable."
+            )
+
         # Get language code
         language_code = language_code_mapping.get(search_language.lower(), "en")
-        
+
         # Initialize SerpAPI wrapper
         self.engine = SerpAPIWrapper(
             serpapi_api_key=serpapi_api_key,
@@ -75,14 +84,15 @@ class SerpAPISearchEngine(BaseSearchEngine):
                 "safe": "active" if safe_search else "off",
                 "tbs": f"qdr:{time_period}",
                 "num": max_results,
-            }
+            },
         )
-        
+
         # If full content is requested, initialize FullSearchResults
         if include_full_content:
             # Import FullSearchResults only if needed
             try:
-                from local_deep_research.web_search_engines.engines.full_search  import FullSearchResults
+                from .full_search import FullSearchResults
+
                 self.full_search = FullSearchResults(
                     llm=llm,
                     web_search=self.engine,
@@ -90,28 +100,30 @@ class SerpAPISearchEngine(BaseSearchEngine):
                     max_results=max_results,
                     region=region,
                     time=time_period,
-                    safesearch="Moderate" if safe_search else "Off"
+                    safesearch="Moderate" if safe_search else "Off",
                 )
             except ImportError:
-                logger.warning("Warning: FullSearchResults not available. Full content retrieval disabled.")
+                logger.warning(
+                    "Warning: FullSearchResults not available. Full content retrieval disabled."
+                )
                 self.include_full_content = False
-    
+
     def _get_previews(self, query: str) -> List[Dict[str, Any]]:
         """
         Get preview information from SerpAPI.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of preview dictionaries
         """
         logger.info("Getting search results from SerpAPI")
-        
+
         try:
             # Get search results from SerpAPI
             organic_results = self.engine.results(query).get("organic_results", [])
-            
+
             # Format results as previews
             previews = []
             for result in organic_results:
@@ -121,39 +133,44 @@ class SerpAPISearchEngine(BaseSearchEngine):
                     "link": result.get("link", ""),
                     "snippet": result.get("snippet", ""),
                     "displayed_link": result.get("displayed_link", ""),
-                    "position": result.get("position")
+                    "position": result.get("position"),
                 }
-                
+
                 # Store full SerpAPI result for later
                 preview["_full_result"] = result
-                
+
                 previews.append(preview)
-            
+
             # Store the previews for potential full content retrieval
             self._search_results = previews
-            
+
             return previews
-            
+
         except Exception as e:
             logger.error(f"Error getting SerpAPI results: {e}")
             return []
-    
-    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    def _get_full_content(
+        self, relevant_items: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """
         Get full content for the relevant search results.
         If include_full_content is True and FullSearchResults is available,
         retrieves full webpage content for the results.
-        
+
         Args:
             relevant_items: List of relevant preview dictionaries
-            
+
         Returns:
             List of result dictionaries with full content if requested
         """
         # Check if we should get full content
-        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+        if (
+            hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
+            and search_config.SEARCH_SNIPPETS_ONLY
+        ):
             logger.info("Snippet-only mode, skipping full content retrieval")
-            
+
             # Return the relevant items with their full SerpAPI information
             results = []
             for item in relevant_items:
@@ -165,30 +182,29 @@ class SerpAPISearchEngine(BaseSearchEngine):
                         del result["_full_result"]
                 else:
                     result = item
-                    
+
                 results.append(result)
-                
+
             return results
-            
+
         # If full content retrieval is enabled
-        if self.include_full_content and hasattr(self, 'full_search'):
+        if self.include_full_content and hasattr(self, "full_search"):
             logger.info("Retrieving full webpage content")
-            
+
             try:
-                # Extract only the links from relevant items
-                links = [item.get("link") for item in relevant_items if item.get("link")]
-                
                 # Use FullSearchResults to get full content
                 # This is a simplified approach - in a real implementation,
                 # you would need to fetch and process the URLs
-                results_with_content = self.full_search._get_full_content(relevant_items)
-                
+                results_with_content = self.full_search._get_full_content(
+                    relevant_items
+                )
+
                 return results_with_content
-                
+
             except Exception as e:
                 logger.info(f"Error retrieving full content: {e}")
                 # Fall back to returning the items without full content
-        
+
         # Return items with their full SerpAPI information
         results = []
         for item in relevant_items:
@@ -202,28 +218,28 @@ class SerpAPISearchEngine(BaseSearchEngine):
                 result = item.copy()
                 if "_full_result" in result:
                     del result["_full_result"]
-                
+
             results.append(result)
-            
+
         return results
-    
+
     def run(self, query: str) -> List[Dict[str, Any]]:
         """
         Execute a search using SerpAPI with the two-phase approach.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of search results
         """
         logger.info("---Execute a search using SerpAPI (Google)---")
-        
+
         # Use the implementation from the parent class which handles all phases
         results = super().run(query)
-        
+
         # Clean up
-        if hasattr(self, '_search_results'):
+        if hasattr(self, "_search_results"):
             del self._search_results
-            
-        return results
+
+        return results