local-deep-research 0.1.26__py3-none-any.whl → 0.2.2__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_searxng.py

@@ -1,42 +1,45 @@
-import requests
 import logging
 import os
-from typing import Dict, List, Any, Optional
-from langchain_core.language_models import BaseLLM
 import time
-import json
+from typing import Any, Dict, List, Optional
+
+import requests
+from langchain_core.language_models import BaseLLM
 
-from web_search_engines.search_engine_base import BaseSearchEngine
-from web_search_engines.engines.full_search import FullSearchResults
-import config
+from ...config import search_config
+from ..search_engine_base import BaseSearchEngine
+from .full_search import FullSearchResults
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+
 class SearXNGSearchEngine(BaseSearchEngine):
     """
     SearXNG search engine implementation that requires an instance URL provided via
     environment variable or configuration. Designed for ethical usage with proper
     rate limiting and single-instance approach.
     """
-    
-    def __init__(self, 
-                max_results: int = 15,
-                instance_url: Optional[str] = None,  # Can be None if using env var
-                categories: Optional[List[str]] = None,
-                engines: Optional[List[str]] = None,
-                language: str = "en",
-                safe_search: int = 1,
-                time_range: Optional[str] = None,
-                delay_between_requests: float = 2.0,
-                llm: Optional[BaseLLM] = None,
-                max_filtered_results: Optional[int] = None,
-                include_full_content: bool = True,
-                api_key: Optional[str] = None):  # API key is actually the instance URL
+
+    def __init__(
+        self,
+        max_results: int = 15,
+        instance_url: Optional[str] = None,  # Can be None if using env var
+        categories: Optional[List[str]] = None,
+        engines: Optional[List[str]] = None,
+        language: str = "en",
+        safe_search: int = 1,
+        time_range: Optional[str] = None,
+        delay_between_requests: float = 0.0,
+        llm: Optional[BaseLLM] = None,
+        max_filtered_results: Optional[int] = None,
+        include_full_content: bool = True,
+        api_key: Optional[str] = None,
+    ):  # API key is actually the instance URL
         """
         Initialize the SearXNG search engine with ethical usage patterns.
-        
+
         Args:
             max_results: Maximum number of search results
             instance_url: URL of your SearXNG instance (preferably self-hosted)
@@ -53,48 +56,62 @@ class SearXNGSearchEngine(BaseSearchEngine):
         """
 
         # Initialize the BaseSearchEngine with LLM, max_filtered_results, and max_results
-        super().__init__(llm=llm, max_filtered_results=max_filtered_results, max_results=max_results)
-        
+        super().__init__(
+            llm=llm, max_filtered_results=max_filtered_results, max_results=max_results
+        )
+
         # Get instance URL from various sources in priority order:
         # 1. api_key parameter (which is actually the instance URL)
         # 2. SEARXNG_INSTANCE environment variable
         # 3. instance_url parameter
         # 4. Default to None, which will disable the engine
-        self.instance_url = api_key or os.getenv("SEARXNG_INSTANCE") or instance_url
-        
+        self.instance_url = api_key or os.getenv("SEARXNG_INSTANCE") or instance_url or "http://localhost:8080"
+
         # Add debug logging for instance URL
-        logger.info(f"SearXNG init - Instance URL sources: api_key={api_key}, env={os.getenv('SEARXNG_INSTANCE')}, param={instance_url}")
-        
+        logger.info(
+            f"SearXNG init - Instance URL sources: api_key={api_key}, env={os.getenv('SEARXNG_INSTANCE')}, param={instance_url}"
+        )
+
         # Validate and normalize the instance URL if provided
         if self.instance_url:
-            self.instance_url = self.instance_url.rstrip('/')
+            self.instance_url = self.instance_url.rstrip("/")
             self.is_available = True
             logger.info(f"SearXNG initialized with instance URL: {self.instance_url}")
         else:
             self.is_available = False
-            logger.error("No SearXNG instance URL provided. The engine is disabled. "
-                       "Set SEARXNG_INSTANCE environment variable or provide instance_url parameter.")
-        
+            logger.error(
+                "No SearXNG instance URL provided. The engine is disabled. "
+                "Set SEARXNG_INSTANCE environment variable or provide instance_url parameter."
+            )
+
         # Add debug logging for all parameters
-        logger.info(f"SearXNG init params: max_results={max_results}, language={language}, "
-                    f"max_filtered_results={max_filtered_results}, is_available={self.is_available}")
-        
+        logger.info(
+            f"SearXNG init params: max_results={max_results}, language={language}, "
+            f"max_filtered_results={max_filtered_results}, is_available={self.is_available}"
+        )
+
         self.max_results = max_results
         self.categories = categories or ["general"]
         self.engines = engines
         self.language = language
         self.safe_search = safe_search
         self.time_range = time_range
-        
-        self.delay_between_requests = float(os.getenv("SEARXNG_DELAY", delay_between_requests))
-        
+
+        self.delay_between_requests = float(
+            os.getenv("SEARXNG_DELAY", delay_between_requests)
+        )
+
         self.include_full_content = include_full_content
-        
+
         if self.is_available:
             self.search_url = f"{self.instance_url}/search"
-            logger.info(f"SearXNG engine initialized with instance: {self.instance_url}")
-            logger.info(f"Rate limiting set to {self.delay_between_requests} seconds between requests")
-        
+            logger.info(
+                f"SearXNG engine initialized with instance: {self.instance_url}"
+            )
+            logger.info(
+                f"Rate limiting set to {self.delay_between_requests} seconds between requests"
+            )
+
             self.full_search = FullSearchResults(
                 llm=llm,
                 web_search=self,
@@ -102,56 +119,63 @@ class SearXNGSearchEngine(BaseSearchEngine):
                 max_results=max_results,
                 region="wt-wt",
                 time="y",
-                safesearch="Moderate" if safe_search == 1 else "Off" if safe_search == 0 else "Strict"
+                safesearch=(
+                    "Moderate"
+                    if safe_search == 1
+                    else "Off" if safe_search == 0 else "Strict"
+                ),
             )
-        
+
         self.last_request_time = 0
-    
+
     def _respect_rate_limit(self):
         """Apply self-imposed rate limiting between requests"""
         current_time = time.time()
         time_since_last_request = current_time - self.last_request_time
-        
 
         if time_since_last_request < self.delay_between_requests:
             wait_time = self.delay_between_requests - time_since_last_request
             logger.info(f"Rate limiting: waiting {wait_time:.2f} seconds")
             time.sleep(wait_time)
-        
+
         self.last_request_time = time.time()
-    
+
     def _get_search_results(self, query: str) -> List[Dict[str, Any]]:
         """
         Get search results from SearXNG with ethical rate limiting.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of search results from SearXNG
         """
         if not self.is_available:
-            logger.error("SearXNG engine is disabled (no instance URL provided) - cannot run search")
+            logger.error(
+                "SearXNG engine is disabled (no instance URL provided) - cannot run search"
+            )
             return []
-            
+
         logger.info(f"SearXNG running search for query: {query}")
-        
+
         try:
             self._respect_rate_limit()
-            
+
             initial_headers = {
                 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-                "Accept-Language": "en-US,en;q=0.9"
+                "Accept-Language": "en-US,en;q=0.9",
             }
-            
+
             try:
-                initial_response = requests.get(self.instance_url, headers=initial_headers, timeout=10)
+                initial_response = requests.get(
+                    self.instance_url, headers=initial_headers, timeout=10
+                )
                 cookies = initial_response.cookies
             except Exception as e:
                 logger.warning(f"Failed to get initial cookies: {e}")
                 cookies = None
-            
+
             params = {
                 "q": query,
                 "categories": ",".join(self.categories),
@@ -159,15 +183,15 @@ class SearXNGSearchEngine(BaseSearchEngine):
                 "format": "html",  # Use HTML format instead of JSON
                 "pageno": 1,
                 "safesearch": self.safe_search,
-                "count": self.max_results
+                "count": self.max_results,
             }
-            
+
             if self.engines:
                 params["engines"] = ",".join(self.engines)
-                
+
             if self.time_range:
                 params["time_range"] = self.time_range
-            
+
             # Browser-like headers
             headers = {
                 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -175,91 +199,105 @@ class SearXNGSearchEngine(BaseSearchEngine):
                 "Accept-Language": "en-US,en;q=0.9",
                 "Referer": self.instance_url + "/",
                 "Connection": "keep-alive",
-                "Upgrade-Insecure-Requests": "1"
+                "Upgrade-Insecure-Requests": "1",
             }
-            
+
             logger.info(f"Sending request to SearXNG instance at {self.instance_url}")
             response = requests.get(
                 self.search_url,
                 params=params,
                 headers=headers,
                 cookies=cookies,
-                timeout=15
+                timeout=15,
             )
-            
+
             if response.status_code == 200:
                 try:
                     from bs4 import BeautifulSoup
-                    
-                    soup = BeautifulSoup(response.text, 'html.parser')
+
+                    soup = BeautifulSoup(response.text, "html.parser")
                     results = []
-                    
-                    result_elements = soup.select('.result-item')
-                    
+
+                    result_elements = soup.select(".result-item")
+
                     if not result_elements:
-                        result_elements = soup.select('.result')
-                    
+                        result_elements = soup.select(".result")
+
                     if not result_elements:
-                        result_elements = soup.select('article') 
-                        
+                        result_elements = soup.select("article")
+
                     if not result_elements:
-                        logger.debug(f"Classes found in HTML: {[c['class'] for c in soup.select('[class]') if 'class' in c.attrs][:10]}")
+                        logger.debug(
+                            f"Classes found in HTML: {[c['class'] for c in soup.select('[class]') if 'class' in c.attrs][:10]}"
+                        )
                         result_elements = soup.select('div[id^="result"]')
-                    
+
                     logger.info(f"Found {len(result_elements)} search result elements")
-                        
+
                     for idx, result_element in enumerate(result_elements):
                         if idx >= self.max_results:
                             break
-                            
+
                         title_element = (
-                            result_element.select_one('.result-title') or 
-                            result_element.select_one('.title') or 
-                            result_element.select_one('h3') or
-                            result_element.select_one('a[href]')
+                            result_element.select_one(".result-title")
+                            or result_element.select_one(".title")
+                            or result_element.select_one("h3")
+                            or result_element.select_one("a[href]")
                         )
-                        
+
                         url_element = (
-                            result_element.select_one('.result-url') or 
-                            result_element.select_one('.url') or
-                            result_element.select_one('a[href]')
+                            result_element.select_one(".result-url")
+                            or result_element.select_one(".url")
+                            or result_element.select_one("a[href]")
                         )
-                        
+
                         content_element = (
-                            result_element.select_one('.result-content') or
-                            result_element.select_one('.content') or
-                            result_element.select_one('.snippet') or 
-                            result_element.select_one('p')
+                            result_element.select_one(".result-content")
+                            or result_element.select_one(".content")
+                            or result_element.select_one(".snippet")
+                            or result_element.select_one("p")
+                        )
+
+                        title = (
+                            title_element.get_text(strip=True) if title_element else ""
                         )
-                        
-                        title = title_element.get_text(strip=True) if title_element else ""
-                        
+
                         url = ""
-                        if url_element and url_element.has_attr('href'):
-                            url = url_element['href']
+                        if url_element and url_element.has_attr("href"):
+                            url = url_element["href"]
                         elif url_element:
                             url = url_element.get_text(strip=True)
-                            
-                        content = content_element.get_text(strip=True) if content_element else ""
-                        
-                        if not url and title_element and title_element.has_attr('href'):
-                            url = title_element['href']
-                            
-                        logger.debug(f"Extracted result {idx}: title={title[:30]}..., url={url[:30]}..., content={content[:30]}...")
-                        
+
+                        content = (
+                            content_element.get_text(strip=True)
+                            if content_element
+                            else ""
+                        )
+
+                        if not url and title_element and title_element.has_attr("href"):
+                            url = title_element["href"]
+
+                        logger.debug(
+                            f"Extracted result {idx}: title={title[:30]}..., url={url[:30]}..., content={content[:30]}..."
+                        )
+
                         # Add to results if we have at least a title or URL
                         if title or url:
-                            results.append({
-                                "title": title,
-                                "url": url,
-                                "content": content,
-                                "engine": "searxng",
-                                "category": "general"
-                            })
-                    
-                    logger.info(f"SearXNG returned {len(results)} results from HTML parsing")
+                            results.append(
+                                {
+                                    "title": title,
+                                    "url": url,
+                                    "content": content,
+                                    "engine": "searxng",
+                                    "category": "general",
+                                }
+                            )
+
+                    logger.info(
+                        f"SearXNG returned {len(results)} results from HTML parsing"
+                    )
                     return results
-                    
+
                 except ImportError:
                     logger.error("BeautifulSoup not available for HTML parsing")
                     return []
@@ -269,123 +307,132 @@ class SearXNGSearchEngine(BaseSearchEngine):
             else:
                 logger.error(f"SearXNG returned status code {response.status_code}")
                 return []
-                
+
         except Exception as e:
             logger.error(f"Error getting SearXNG results: {e}")
             return []
-    
+
     def _get_previews(self, query: str) -> List[Dict[str, Any]]:
         """
         Get preview information for SearXNG search results.
-        
+
         Args:
             query: The search query
-            
+
         Returns:
             List of preview dictionaries
         """
         if not self.is_available:
             logger.warning("SearXNG engine is disabled (no instance URL provided)")
             return []
-            
+
         logger.info(f"Getting SearXNG previews for query: {query}")
-        
+
         results = self._get_search_results(query)
-        
+
         if not results:
             logger.warning(f"No SearXNG results found for query: {query}")
             return []
-        
+
         previews = []
         for i, result in enumerate(results):
             title = result.get("title", "")
             url = result.get("url", "")
             content = result.get("content", "")
-            
+
             preview = {
                 "id": url or f"searxng-result-{i}",
                 "title": title,
                 "link": url,
                 "snippet": content,
                 "engine": result.get("engine", ""),
-                "category": result.get("category", "")
+                "category": result.get("category", ""),
             }
-            
+
             previews.append(preview)
-        
+
         return previews
-    
-    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    def _get_full_content(
+        self, relevant_items: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """
         Get full content for the relevant search results.
-        
+
         Args:
             relevant_items: List of relevant preview dictionaries
-            
+
         Returns:
             List of result dictionaries with full content
         """
         if not self.is_available:
             return relevant_items
-            
-        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+
+        if (
+            hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
+            and search_config.SEARCH_SNIPPETS_ONLY
+        ):
             logger.info("Snippet-only mode, skipping full content retrieval")
             return relevant_items
-        
+
         logger.info("Retrieving full webpage content")
-        
+
         try:
             results_with_content = self.full_search._get_full_content(relevant_items)
             return results_with_content
-            
+
         except Exception as e:
             logger.error(f"Error retrieving full content: {e}")
             return relevant_items
-    
+
     def invoke(self, query: str) -> List[Dict[str, Any]]:
         """Compatibility method for LangChain tools"""
         return self.run(query)
-    
-    def results(self, query: str, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+
+    def results(
+        self, query: str, max_results: Optional[int] = None
+    ) -> List[Dict[str, Any]]:
         """
         Get search results in a format compatible with other search engines.
-        
+
         Args:
             query: The search query
             max_results: Optional override for maximum results
-            
+
         Returns:
             List of search result dictionaries
         """
         if not self.is_available:
             return []
-            
+
         original_max_results = self.max_results
-        
+
         try:
             if max_results is not None:
                 self.max_results = max_results
-                
+
             results = self._get_search_results(query)
-            
+
             formatted_results = []
             for result in results:
-                formatted_results.append({
-                    "title": result.get("title", ""),
-                    "link": result.get("url", ""),
-                    "snippet": result.get("content", "")
-                })
-                
+                formatted_results.append(
+                    {
+                        "title": result.get("title", ""),
+                        "link": result.get("url", ""),
+                        "snippet": result.get("content", ""),
+                    }
+                )
+
             return formatted_results
-            
+
         finally:
             self.max_results = original_max_results
-    
+
     @staticmethod
     def get_self_hosting_instructions() -> str:
         """
         Get instructions for self-hosting a SearXNG instance.
-        
+
         Returns:
             String with installation instructions
         """
@@ -441,15 +488,20 @@ https://searxng.github.io/searxng/admin/installation.html
         Override BaseSearchEngine run method to add SearXNG-specific error handling.
         """
         if not self.is_available:
-            logger.error("SearXNG run method called but engine is not available (missing instance URL)")
+            logger.error(
+                "SearXNG run method called but engine is not available (missing instance URL)"
+            )
             return []
-        
-        logger.info(f"SearXNG run method called with query: {query}")
-        
+
+        logger.info(f"SearXNG search engine running with query: '{query}'")
+        logger.info(f"SearXNG instance URL: {self.instance_url}")
+
         try:
             # Call the parent class's run method
-            return super().run(query)
+            results = super().run(query)
+            logger.info(f"SearXNG search completed with {len(results)} results")
+            return results
         except Exception as e:
             logger.error(f"Error in SearXNG run method: {str(e)}")
             # Return empty results on error
-            return []
+            return []