local-deep-research 0.1.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_wikipedia.py

@@ -0,0 +1,242 @@
+import wikipedia
+from typing import Dict, List, Any, Optional
+from langchain_core.language_models import BaseLLM
+import logging
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+from local_deep_research import config
+
+# Setup logging
+logger = logging.getLogger(__name__)
+
+
+class WikipediaSearchEngine(BaseSearchEngine):
+    """Wikipedia search engine implementation with two-phase approach"""
+    
+    def __init__(self, 
+                max_results: int = 10, 
+                language: str = "en", 
+                include_content: bool = True,
+                sentences: int = 5,
+                llm: Optional[BaseLLM] = None,
+                max_filtered_results: Optional[int] = None,
+                **kwargs):
+        """
+        Initialize the Wikipedia search engine.
+        
+        Args:
+            max_results: Maximum number of search results
+            language: Language code for Wikipedia (e.g., 'en', 'fr', 'es')
+            include_content: Whether to include full page content in results
+            sentences: Number of sentences to include in summary
+            llm: Language model for relevance filtering
+            max_filtered_results: Maximum number of results to keep after filtering
+            **kwargs: Additional parameters (ignored but accepted for compatibility)
+        """
+        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
+        
+        self.max_results = max_results
+        self.include_content = include_content
+        self.sentences = sentences
+        
+        # Set the Wikipedia language
+        wikipedia.set_lang(language)
+    
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information (titles and summaries) for Wikipedia pages.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries
+        """
+        logger.info(f"Getting Wikipedia page previews for query: {query}")
+        
+        try:
+            # Get search results (just titles)
+            search_results = wikipedia.search(query, results=self.max_results)
+            
+            logger.info(f"Found {len(search_results)} Wikipedia results: {search_results}")
+            
+            if not search_results:
+                logger.info(f"No Wikipedia results found for query: {query}")
+                return []
+            
+            # Create a cache for full pages (will be populated on-demand)
+            self._page_cache = {}
+            
+            # Generate previews with summaries
+            previews = []
+            for title in search_results:
+                try:
+                    # Get just the summary, with auto_suggest=False to be more precise
+                    summary = None
+                    try:
+                        summary = wikipedia.summary(title, sentences=self.sentences, auto_suggest=False)
+                    except wikipedia.exceptions.DisambiguationError as e:
+                        # If disambiguation error, try the first option
+                        if e.options and len(e.options) > 0:
+                            logger.info(f"Disambiguation for '{title}', trying first option: {e.options[0]}")
+                            try:
+                                summary = wikipedia.summary(e.options[0], sentences=self.sentences, auto_suggest=False)
+                                title = e.options[0]  # Use the new title
+                            except Exception as inner_e:
+                                logger.error(f"Error with disambiguation option: {inner_e}")
+                                continue
+                        else:
+                            logger.warning(f"Disambiguation with no options for '{title}'")
+                            continue
+                    
+                    if summary:
+                        preview = {
+                            "id": title,  # Use title as ID
+                            "title": title,
+                            "snippet": summary,
+                            "link": f"https://en.wikipedia.org/wiki/{title.replace(' ', '_')}"
+                        }
+                        
+                        previews.append(preview)
+                    
+                except (wikipedia.exceptions.PageError,
+                       wikipedia.exceptions.WikipediaException) as e:
+                    # Skip pages with errors
+                    logger.warning(f"Error getting summary for '{title}': {e}")
+                    continue
+                except Exception as e:
+                    logger.error(f"Unexpected error for '{title}': {e}")
+                    continue
+            
+            logger.info(f"Successfully created {len(previews)} previews from Wikipedia")
+            return previews
+            
+        except Exception as e:
+            logger.error(f"Error getting Wikipedia previews: {e}")
+            return []
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant Wikipedia pages.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        # Check if we should add full content
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            logger.info("Snippet-only mode, skipping full content retrieval")
+            return relevant_items
+            
+        logger.info(f"Getting full content for {len(relevant_items)} relevant Wikipedia pages")
+        
+        results = []
+        for item in relevant_items:
+            title = item.get("id")  # Title stored as ID
+            
+            if not title:
+                results.append(item)
+                continue
+                
+            try:
+                # Get the full page
+                page = wikipedia.page(title, auto_suggest=False)
+                
+                # Create a full result with all information
+                result = {
+                    "title": page.title,
+                    "link": page.url,
+                    "snippet": item.get("snippet", "")  # Keep existing snippet
+                }
+                
+                # Add additional information
+                result["content"] = page.content
+                result["full_content"] = page.content
+                result["categories"] = page.categories
+                result["references"] = page.references
+                result["links"] = page.links
+                result["images"] = page.images
+                result["sections"] = page.sections
+                
+                results.append(result)
+                
+            except (wikipedia.exceptions.DisambiguationError, 
+                   wikipedia.exceptions.PageError,
+                   wikipedia.exceptions.WikipediaException) as e:
+                # If error, use the preview
+                logger.warning(f"Error getting full content for '{title}': {e}")
+                results.append(item)
+            except Exception as e:
+                logger.error(f"Unexpected error getting full content for '{title}': {e}")
+                results.append(item)
+        
+        return results
+    
+    def get_summary(self, title: str, sentences: Optional[int] = None) -> str:
+        """
+        Get a summary of a specific Wikipedia page.
+        
+        Args:
+            title: Title of the Wikipedia page
+            sentences: Number of sentences to include (defaults to self.sentences)
+            
+        Returns:
+            Summary of the page
+        """
+        sentences = sentences or self.sentences
+        try:
+            return wikipedia.summary(title, sentences=sentences, auto_suggest=False)
+        except wikipedia.exceptions.DisambiguationError as e:
+            if e.options and len(e.options) > 0:
+                return wikipedia.summary(e.options[0], sentences=sentences, auto_suggest=False)
+            raise
+    
+    def get_page(self, title: str) -> Dict[str, Any]:
+        """
+        Get detailed information about a specific Wikipedia page.
+        
+        Args:
+            title: Title of the Wikipedia page
+            
+        Returns:
+            Dictionary with page information
+        """
+        # Check if we should include full content
+        include_content = not (hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY)
+        
+        try:
+            page = wikipedia.page(title, auto_suggest=False)
+            
+            result = {
+                "title": page.title,
+                "link": page.url,
+                "snippet": self.get_summary(title, self.sentences)
+            }
+            
+            # Add additional information if requested
+            if include_content:
+                result["content"] = page.content
+                result["full_content"] = page.content
+                result["categories"] = page.categories
+                result["references"] = page.references
+                result["links"] = page.links
+                result["images"] = page.images
+                result["sections"] = page.sections
+            
+            return result
+        except wikipedia.exceptions.DisambiguationError as e:
+            if e.options and len(e.options) > 0:
+                return self.get_page(e.options[0])
+            raise
+    
+    def set_language(self, language: str) -> None:
+        """
+        Change the Wikipedia language.
+        
+        Args:
+            language: Language code (e.g., 'en', 'fr', 'es')
+        """
+        wikipedia.set_lang(language)

local_deep_research/web_search_engines/full_search.py

@@ -0,0 +1,254 @@
+import justext
+from langchain_community.document_loaders import AsyncChromiumLoader
+from langchain_community.document_transformers import BeautifulSoupTransformer
+from langchain_core.language_models import BaseLLM
+from typing import List, Dict, Any, Optional, Union
+import json
+import os
+from .utilties.search_utilities import remove_think_tags
+from datetime import datetime
+from local_deep_research import config
+
+class FullSearchResults:
+    """
+    Enhanced web content retrieval class that works with the BaseSearchEngine architecture.
+    Can be used as a wrapper around web-based search engines like DuckDuckGo and SerpAPI.
+    """
+    
+    def __init__(
+        self,
+        llm: BaseLLM,
+        web_search,        
+        output_format: str = "list",
+        language: str = "English",
+        max_results: int = 10,
+        region: str = "wt-wt",
+        time: str = "y",
+        safesearch: str = "Moderate"
+    ):
+        """
+        Initialize the full search results processor.
+        
+        Args:
+            llm: Language model instance for relevance filtering
+            web_search: Web search engine instance that provides initial results
+            output_format: Format of output ('list' or other formats)
+            language: Language for content processing
+            max_results: Maximum number of search results
+            region: Search region
+            time: Time period for search results
+            safesearch: Safe search setting
+        """
+        self.llm = llm
+        self.output_format = output_format
+        self.language = language
+        self.max_results = max_results
+        self.region = region
+        self.time = time
+        self.safesearch = safesearch
+        self.web_search = web_search
+        os.environ["USER_AGENT"] = "Local Deep Research/1.0"
+
+        self.bs_transformer = BeautifulSoupTransformer()
+        self.tags_to_extract = ["p", "div", "span"]
+    
+    def run(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Legacy method that performs a full search in one step.
+        Respects config parameters:
+        - SEARCH_SNIPPETS_ONLY: If True, only returns snippets without full content
+        - SKIP_RELEVANCE_FILTER: If True, returns all results without filtering
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of search results with full content (unless SEARCH_SNIPPETS_ONLY is True)
+        """
+        # Phase 1: Get search results from the web search engine
+        previews = self._get_previews(query)
+        if not previews:
+            return []
+            
+        # Phase 2: Filter URLs using LLM (unless SKIP_RELEVANCE_FILTER is True)
+        if hasattr(config, 'SKIP_RELEVANCE_FILTER') and config.SKIP_RELEVANCE_FILTER:
+            relevant_items = previews
+            print("Skipping relevance filtering as per config")
+        else:
+            relevant_items = self._filter_relevant_items(previews, query)
+            if not relevant_items:
+                return []
+            
+        # Phase 3: Get full content for relevant items (unless SEARCH_SNIPPETS_ONLY is True)
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            print("Returning snippet-only results as per config")
+            return relevant_items
+        else:
+            results = self._get_full_content(relevant_items)
+            return results
+    
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information from the web search engine.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries
+        """
+        try:
+            # Get search results from the web search engine
+            search_results = self.web_search.invoke(query)
+            
+            if not isinstance(search_results, list):
+                print("Error: Expected search results in list format")
+                return []
+            
+            # Return the results as previews
+            return search_results
+            
+        except Exception as e:
+            print(f"Error getting previews: {e}")
+            return []
+    
+    def _filter_relevant_items(self, previews: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
+        """
+        Filter previews for relevance using LLM.
+        
+        Args:
+            previews: List of preview dictionaries
+            query: The original search query
+            
+        Returns:
+            List of relevant preview dictionaries
+        """
+        # Skip filtering if disabled in config or no previews
+        if not config.QUALITY_CHECK_DDG_URLS or not previews:
+            return previews
+        
+        # Format for LLM evaluation
+        now = datetime.now()
+        current_time = now.strftime("%Y-%m-%d")
+        prompt = f"""ONLY Return a JSON array. The response contains no letters. Evaluate these URLs for:
+            1. Timeliness (today: {current_time})
+            2. Factual accuracy (cross-reference major claims)
+            3. Source reliability (prefer official company websites, established news outlets)
+            4. Direct relevance to query: {query}
+
+            URLs to evaluate:
+            {json.dumps(previews, indent=2)}
+
+            Return a JSON array of indices (0-based) for sources that meet ALL criteria.
+            ONLY Return a JSON array of indices (0-based) and nothing else. No letters. 
+            Example response: \n[0, 2, 4]\n\n"""
+        
+        try:
+            # Get LLM's evaluation
+            response = self.llm.invoke(prompt)
+            
+            # Extract JSON array from response
+            response_text = remove_think_tags(response.content)
+            # Clean up response to handle potential formatting issues
+            response_text = response_text.strip()
+            
+            # Find the first occurrence of '[' and the last occurrence of ']'
+            start_idx = response_text.find('[')
+            end_idx = response_text.rfind(']')
+            
+            if start_idx >= 0 and end_idx > start_idx:
+                array_text = response_text[start_idx:end_idx+1]
+                good_indices = json.loads(array_text)
+                
+                # Return only the results with good indices
+                return [r for i, r in enumerate(previews) if i in good_indices]
+            else:
+                print("Could not find JSON array in response, returning all previews")
+                return previews
+                
+        except Exception as e:
+            print(f"URL filtering error: {e}")
+            # Fall back to returning all previews on error
+            return previews
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant items by retrieving and processing web pages.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        nr_full_text = 0
+        
+        # Extract URLs from relevant items
+        urls = [item.get("link") for item in relevant_items if item.get("link")]
+        
+        if not urls:
+            print("\n === NO VALID LINKS ===\n")
+            return relevant_items
+        
+        try:
+            # Download the full HTML pages for filtered URLs
+            loader = AsyncChromiumLoader(urls)
+            html_docs = loader.load()
+            
+            # Process the HTML using BeautifulSoupTransformer
+            full_docs = self.bs_transformer.transform_documents(
+                html_docs, tags_to_extract=self.tags_to_extract
+            )
+            
+            # Remove boilerplate from each document
+            url_to_content = {}
+            for doc in full_docs:
+                nr_full_text += 1
+                source = doc.metadata.get("source")
+                if source:
+                    cleaned_text = self._remove_boilerplate(doc.page_content)
+                    url_to_content[source] = cleaned_text
+            
+            # Attach the cleaned full content to each result
+            results = []
+            for item in relevant_items:
+                new_item = item.copy()
+                link = item.get("link")
+                new_item["full_content"] = url_to_content.get(link, None)
+                results.append(new_item)
+            
+            print(f"FULL SEARCH WITH FILTERED URLS - Full text retrieved: {nr_full_text}")
+            return results
+            
+        except Exception as e:
+            print(f"Error retrieving full content: {e}")
+            # Return original items if full content retrieval fails
+            return relevant_items
+    
+    def _remove_boilerplate(self, html: str) -> str:
+        """
+        Remove boilerplate content from HTML.
+        
+        Args:
+            html: HTML content
+            
+        Returns:
+            Cleaned text content
+        """
+        if not html or not html.strip():
+            return ""
+        try:
+            paragraphs = justext.justext(html, justext.get_stoplist(self.language))
+            cleaned = "\n".join([p.text for p in paragraphs if not p.is_boilerplate])
+            return cleaned
+        except Exception as e:
+            print(f"Error removing boilerplate: {e}")
+            return html
+    
+    def invoke(self, query: str) -> List[Dict[str, Any]]:
+        """Compatibility method for LangChain tools"""
+        return self.run(query)
+    
+    def __call__(self, query: str) -> List[Dict[str, Any]]:
+        """Make the class callable like a function"""
+        return self.invoke(query)

local_deep_research/web_search_engines/search_engine_base.py

@@ -0,0 +1,197 @@
+from abc import ABC, abstractmethod
+from typing import Dict, List, Any, Optional
+from langchain_core.language_models import BaseLLM
+from datetime import datetime
+import json
+from local_deep_research.utilties.search_utilities import remove_think_tags
+
+import logging
+logger = logging.getLogger(__name__)
+
+class BaseSearchEngine(ABC):
+    """
+    Abstract base class for search engines with two-phase retrieval capability.
+    Handles common parameters and implements the two-phase search approach.
+    """
+    
+    def __init__(self, 
+                 llm: Optional[BaseLLM] = None, 
+                 max_filtered_results: Optional[int] = 5,
+                 **kwargs):
+        """
+        Initialize the search engine with common parameters.
+        
+        Args:
+            llm: Optional language model for relevance filtering
+            max_filtered_results: Maximum number of results to keep after filtering
+            **kwargs: Additional engine-specific parameters
+        """
+        if max_filtered_results == None: max_filtered_results=5
+        self.llm = llm  # LLM for relevance filtering
+        self.max_filtered_results = max_filtered_results  # Limit filtered results
+    
+    def run(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Run the search engine with a given query, retrieving and filtering results.
+        This implements a two-phase retrieval approach: 
+        1. Get preview information for many results
+        2. Filter the previews for relevance
+        3. Get full content for only the relevant results
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of search results with full content (if available)
+        """
+        # Ensure we're measuring time correctly for citation tracking
+        start_time = datetime.now()
+        
+        # Step 1: Get preview information for items
+        previews = self._get_previews(query)
+        if not previews:
+            logger.info(f"Search engine {self.__class__.__name__} returned no preview results for query: {query}")
+            return []
+            
+        # Step 2: Filter previews for relevance with LLM
+        filtered_items = self._filter_for_relevance(previews, query)
+        if not filtered_items:
+            logger.info(f"All preview results were filtered out as irrelevant for query: {query}")
+            # Fall back to preview items if everything was filtered
+            # Access config inside the method to avoid circular import
+            from local_deep_research import config
+            if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+                return previews[:self.max_filtered_results or 5]  # Return unfiltered but limited results
+            else:
+                filtered_items = previews[:self.max_filtered_results or 5]
+        
+        # Step 3: Get full content for filtered items
+        # Import config inside the method to avoid circular import
+        from local_deep_research import config
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            logger.info("Returning snippet-only results as per config")
+            results = filtered_items
+        else:
+            results = self._get_full_content(filtered_items)
+        
+        return results
+    
+    def invoke(self, query: str) -> List[Dict[str, Any]]:
+        """Compatibility method for LangChain tools"""
+        return self.run(query)
+    
+    def _filter_for_relevance(self, previews: List[Dict[str, Any]], query: str) -> List[Dict[str, Any]]:
+        """
+        Filter search results for relevance to the query using an LLM.
+        
+        Checks config.SKIP_RELEVANCE_FILTER to determine whether to perform filtering.
+        
+        Args:
+            previews: List of search result dictionaries with preview information
+            query: The original search query
+            
+        Returns:
+            Filtered list of the most relevant search results
+        """
+        # Import config inside the method to avoid circular import
+        from local_deep_research import config
+        
+        # Skip filtering if configured to do so or if no LLM is available
+        if hasattr(config, 'SKIP_RELEVANCE_FILTER') and config.SKIP_RELEVANCE_FILTER:
+            # Return all previews up to max_filtered_results if no filtering is performed
+            limit = self.max_filtered_results or 5
+            return previews[:limit]
+            
+        # Default implementation uses LLM if available
+        if not self.llm or not previews:
+            # If no LLM available, return all previews as relevant
+            if self.max_filtered_results and len(previews) > self.max_filtered_results:
+                return previews[:self.max_filtered_results]
+            return previews
+        
+        now = datetime.now()
+        current_time = now.strftime("%Y-%m-%d")
+        prompt = f"""Analyze these search results and provide a ranked list of the most relevant ones.
+
+IMPORTANT: Evaluate and rank based on these criteria (in order of importance):
+1. Timeliness - current/recent information as of {current_time}
+2. Direct relevance to query: "{query}"
+3. Source reliability (prefer official sources, established websites)
+4. Factual accuracy (cross-reference major claims)
+
+Search results to evaluate:
+{json.dumps(previews, indent=2)}
+
+Return ONLY a JSON array of indices (0-based) ranked from most to least relevant.
+Include ONLY indices that meet ALL criteria, with the most relevant first.
+Example response: [4, 0, 2]
+
+Respond with ONLY the JSON array, no other text."""
+        
+        try:
+            # Get LLM's evaluation
+            response = self.llm.invoke(prompt)
+            
+            # Extract JSON array from response
+            response_text = remove_think_tags(response.content)
+            # Clean up response to handle potential formatting issues
+            response_text = response_text.strip()
+            
+            # Find the first occurrence of '[' and the last occurrence of ']'
+            start_idx = response_text.find('[')
+            end_idx = response_text.rfind(']')
+            
+            if start_idx >= 0 and end_idx > start_idx:
+                array_text = response_text[start_idx:end_idx+1]
+                ranked_indices = json.loads(array_text)
+                
+                # Return the results in ranked order
+                ranked_results = []
+                for idx in ranked_indices:
+                    if idx < len(previews):
+                        ranked_results.append(previews[idx])
+                
+                # Limit to max_filtered_results if specified
+                if self.max_filtered_results and len(ranked_results) > self.max_filtered_results:
+                    logger.info(f"Limiting filtered results to top {self.max_filtered_results}")
+                    return ranked_results[:self.max_filtered_results]
+                    
+                return ranked_results
+            else:
+                logger.info("Could not find JSON array in response, returning all previews")
+                if self.max_filtered_results and len(previews) > self.max_filtered_results:
+                    return previews[:self.max_filtered_results]
+                return previews
+                
+        except Exception as e:
+            logger.info(f"Relevance filtering error: {e}")
+            # Fall back to returning all previews (or top N) on error
+            if self.max_filtered_results and len(previews) > self.max_filtered_results:
+                return previews[:self.max_filtered_results]
+            return previews
+    
+    @abstractmethod
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information (titles, summaries) for initial search results.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries with at least 'id', 'title', and 'snippet' keys
+        """
+        pass
+    
+    @abstractmethod
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant items.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        pass