local-deep-research 0.1.0__py3-none-any.whl

local_deep_research/web_search_engines/engines/search_engine_google_pse.py

@@ -0,0 +1,283 @@
+from typing import Dict, List, Any, Optional
+import os
+import requests
+import time
+import random
+import logging
+from requests.exceptions import RequestException
+from urllib.parse import quote_plus
+from langchain_core.language_models import BaseLLM
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class GooglePSESearchEngine(BaseSearchEngine):
+    """Google Programmable Search Engine implementation"""
+
+    def __init__(self,
+                max_results: int = 10,
+                region: str = "us",
+                safe_search: bool = True,
+                search_language: str = "English",
+                api_key: Optional[str] = None,
+                search_engine_id: Optional[str] = None,
+                llm: Optional[BaseLLM] = None,
+                include_full_content: bool = False,
+                max_filtered_results: Optional[int] = None,
+                max_retries: int = 3,
+                retry_delay: float = 2.0,
+                **kwargs):
+        """
+        Initialize the Google Programmable Search Engine.
+        
+        Args:
+            max_results: Maximum number of search results
+            region: Region code for search results
+            safe_search: Whether to enable safe search
+            search_language: Language for search results
+            api_key: Google API key (can also be set in GOOGLE_PSE_API_KEY env)
+            search_engine_id: Google CSE ID (can also be set in GOOGLE_PSE_ENGINE_ID env)
+            llm: Language model for relevance filtering
+            include_full_content: Whether to include full webpage content in results
+            max_filtered_results: Maximum number of results to keep after filtering
+            max_retries: Maximum number of retry attempts for API requests
+            retry_delay: Base delay in seconds between retry attempts
+            **kwargs: Additional parameters (ignored but accepted for compatibility)
+        """
+        # Initialize the BaseSearchEngine with the LLM and max_filtered_results
+        super().__init__(llm=llm, max_filtered_results=max_filtered_results)
+        
+        self.max_results = max_results
+        self.include_full_content = include_full_content
+        
+        # Retry configuration
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        
+        # Rate limiting - keep track of last request time
+        self.last_request_time = 0
+        self.min_request_interval = 0.5  # Minimum time between requests in seconds
+        
+        # Language code mapping
+        language_code_mapping = {
+            "english": "en",
+            "spanish": "es",
+            "french": "fr",
+            "german": "de",
+            "italian": "it",
+            "japanese": "ja",
+            "korean": "ko",
+            "portuguese": "pt",
+            "russian": "ru",
+            "chinese": "zh-CN"
+        }
+        
+        # Get language code
+        search_language = search_language.lower()
+        self.language = language_code_mapping.get(search_language, "en")
+        
+        # Safe search setting
+        self.safe = "active" if safe_search else "off"
+        
+        # Region/Country setting
+        self.region = region
+        
+        # API key and Search Engine ID
+        self.api_key = api_key or os.getenv("GOOGLE_PSE_API_KEY")
+        self.search_engine_id = search_engine_id or os.getenv("GOOGLE_PSE_ENGINE_ID")
+        
+        if not self.api_key:
+            raise ValueError("Google API key is required. Set it in the GOOGLE_PSE_API_KEY environment variable.")
+        if not self.search_engine_id:
+            raise ValueError("Google Search Engine ID is required. Set it in the GOOGLE_PSE_ENGINE_ID environment variable.")
+            
+        # Validate connection and credentials
+        self._validate_connection()
+    
+    def _validate_connection(self):
+        """Test the connection to ensure API key and Search Engine ID are valid"""
+        try:
+            # Make a minimal test query
+            response = self._make_request("test")
+            
+            # Check if we got a valid response
+            if response.get("error"):
+                error_msg = response["error"].get("message", "Unknown error")
+                raise ValueError(f"Google PSE API error: {error_msg}")
+                
+            # If we get here, the connection is valid
+            logger.info("Google PSE connection validated successfully")
+            return True
+            
+        except Exception as e:
+            # Log the error and re-raise
+            logger.error(f"Error validating Google PSE connection: {str(e)}")
+            raise
+    
+    def _respect_rate_limit(self):
+        """Ensure we don't exceed rate limits by adding appropriate delay between requests"""
+        current_time = time.time()
+        elapsed = current_time - self.last_request_time
+        
+        # If we've made a request recently, wait until the minimum interval has passed
+        if elapsed < self.min_request_interval:
+            sleep_time = self.min_request_interval - elapsed
+            logger.debug(f"Rate limiting: sleeping for {sleep_time:.2f}s")
+            time.sleep(sleep_time)
+        
+        # Update the last request time
+        self.last_request_time = time.time()
+
+    def _make_request(self, query: str, start_index: int = 1) -> Dict:
+        """
+        Make a request to the Google PSE API with retry logic and rate limiting
+        
+        Args:
+            query: Search query string
+            start_index: Starting index for pagination
+            
+        Returns:
+            JSON response from the API
+            
+        Raises:
+            RequestException: If all retry attempts fail
+        """
+        # Base URL for the API
+        url = "https://www.googleapis.com/customsearch/v1"
+        
+        # Parameters for the request
+        params = {
+            "key": self.api_key,
+            "cx": self.search_engine_id,
+            "q": query,
+            "num": min(10, self.max_results),  # Max 10 per request
+            "start": start_index,
+            "safe": self.safe,
+            "lr": f"lang_{self.language}",
+            "gl": self.region
+        }
+        
+        # Implement retry logic with exponential backoff
+        attempt = 0
+        last_exception = None
+        
+        while attempt < self.max_retries:
+            try:
+                # Respect rate limits
+                self._respect_rate_limit()
+                
+                # Add jitter to retries after the first attempt
+                if attempt > 0:
+                    jitter = random.uniform(0.5, 1.5)
+                    sleep_time = self.retry_delay * (2 ** (attempt - 1)) * jitter
+                    logger.info(f"Retry attempt {attempt+1}/{self.max_retries} for query '{query}'. Waiting {sleep_time:.2f}s")
+                    time.sleep(sleep_time)
+                
+                # Make the request
+                logger.debug(f"Making request to Google PSE API: {query} (start_index={start_index})")
+                response = requests.get(url, params=params, timeout=10)
+                
+                # Check for HTTP errors
+                response.raise_for_status()
+                
+                # Return the JSON response
+                return response.json()
+                
+            except RequestException as e:
+                logger.warning(f"Request error on attempt {attempt+1}/{self.max_retries}: {str(e)}")
+                last_exception = e
+            except Exception as e:
+                logger.warning(f"Error on attempt {attempt+1}/{self.max_retries}: {str(e)}")
+                last_exception = e
+            
+            attempt += 1
+        
+        # If we get here, all retries failed
+        error_msg = f"Failed to get response from Google PSE API after {self.max_retries} attempts"
+        logger.error(error_msg)
+        
+        if last_exception:
+            raise RequestException(f"{error_msg}: {str(last_exception)}")
+        else:
+            raise RequestException(error_msg)
+        
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """Get search result previews/snippets"""
+        results = []
+        
+        # Google PSE API returns a maximum of 10 results per request
+        # We may need to make multiple requests to get the desired number
+        start_index = 1
+        total_results = 0
+        
+        while total_results < self.max_results:
+            try:
+                response = self._make_request(query, start_index)
+                
+                # Break if no items
+                if "items" not in response:
+                    break
+                    
+                items = response.get("items", [])
+                
+                # Process each result
+                for item in items:
+                    title = item.get("title", "")
+                    snippet = item.get("snippet", "")
+                    url = item.get("link", "")
+                    
+                    # Skip results without URL
+                    if not url:
+                        continue
+                        
+                    results.append({
+                        "title": title,
+                        "snippet": snippet,
+                        "url": url,
+                        "source": "Google Programmable Search"
+                    })
+                    
+                    total_results += 1
+                    if total_results >= self.max_results:
+                        break
+                        
+                # Check if there are more results
+                if not items or total_results >= self.max_results:
+                    break
+                    
+                # Update start index for next request
+                start_index += len(items)
+                
+                # Add a small delay between multiple requests to be respectful of the API
+                if total_results < self.max_results:
+                    time.sleep(self.min_request_interval)
+                
+            except Exception as e:
+                logger.error(f"Error getting search results: {str(e)}")
+                break
+                
+        logger.info(f"Retrieved {len(results)} search results for query: '{query}'")
+        return results
+        
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Get full content for search results"""
+        # Use the BaseSearchEngine implementation
+        return super()._get_full_content(relevant_items)
+        
+    def run(self, query: str) -> List[Dict[str, Any]]:
+        """Run the search engine to get results for a query"""
+        # Get search result previews/snippets
+        search_results = self._get_previews(query)
+        
+        # Filter for relevance if we have an LLM and max_filtered_results
+        if self.llm and self.max_filtered_results:
+            search_results = self._filter_for_relevance(query, search_results)
+            
+        # Get full content if needed
+        if self.include_full_content:
+            search_results = self._get_full_content(search_results)
+            
+        return search_results 

local_deep_research/web_search_engines/engines/search_engine_guardian.py

@@ -0,0 +1,337 @@
+import requests
+from typing import Dict, List, Any, Optional
+import os
+from datetime import datetime, timedelta
+from langchain_core.language_models import BaseLLM
+
+from local_deep_research.web_search_engines.search_engine_base import BaseSearchEngine
+from local_deep_research import config
+
+
+class GuardianSearchEngine(BaseSearchEngine):
+    """The Guardian API search engine implementation"""
+    
+    def __init__(self, 
+                max_results: int = 10, 
+                api_key: Optional[str] = None,
+                from_date: Optional[str] = None,
+                to_date: Optional[str] = None,
+                section: Optional[str] = None,
+                order_by: str = "relevance",
+                llm: Optional[BaseLLM] = None):
+        """
+        Initialize The Guardian search engine.
+        
+        Args:
+            max_results: Maximum number of search results
+            api_key: The Guardian API key (can also be set in GUARDIAN_API_KEY env)
+            from_date: Start date for search (YYYY-MM-DD format, default 1 month ago)
+            to_date: End date for search (YYYY-MM-DD format, default today)
+            section: Filter by section (e.g., "politics", "technology", "sport")
+            order_by: Sort order ("relevance", "newest", "oldest")
+            llm: Language model for relevance filtering
+        """
+        # Initialize the BaseSearchEngine with the LLM
+        super().__init__(llm=llm)
+        
+        self.max_results = max_results
+        self.api_key = api_key or os.getenv("GUARDIAN_API_KEY")
+        
+        if not self.api_key:
+            raise ValueError("Guardian API key not found. Please provide api_key or set the GUARDIAN_API_KEY environment variable.")
+        
+        # Set date ranges if not provided
+        if not from_date:
+            # Default to one month ago
+            one_month_ago = datetime.now() - timedelta(days=30)
+            self.from_date = one_month_ago.strftime("%Y-%m-%d")
+        else:
+            self.from_date = from_date
+            
+        if not to_date:
+            # Default to today
+            self.to_date = datetime.now().strftime("%Y-%m-%d")
+        else:
+            self.to_date = to_date
+            
+        self.section = section
+        self.order_by = order_by
+        
+        # API base URL
+        self.api_url = "https://content.guardianapis.com/search"
+    
+    def _get_all_data(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get all article data from The Guardian API in a single call.
+        Always requests all fields for simplicity.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of articles with all data
+        """
+        try:
+            # Always request all fields for simplicity
+            params = {
+                "q": query,
+                "api-key": self.api_key,
+                "from-date": self.from_date,
+                "to-date": self.to_date,
+                "order-by": self.order_by,
+                "page-size": min(self.max_results, 50),  # API maximum is 50
+                "show-fields": "headline,trailText,byline,body,publication",
+                "show-tags": "keyword"
+            }
+            
+            # Add section filter if specified
+            if self.section:
+                params["section"] = self.section
+            
+            # Execute the API request
+            response = requests.get(self.api_url, params=params)
+            response.raise_for_status()
+            
+            data = response.json()
+            
+            # Extract results from the response
+            articles = data.get("response", {}).get("results", [])
+            
+            # Format results to include all data
+            formatted_articles = []
+            for i, article in enumerate(articles):
+                if i >= self.max_results:
+                    break
+                    
+                fields = article.get("fields", {})
+                
+                # Format the article with all fields
+                result = {
+                    "id": article.get("id", ""),
+                    "title": fields.get("headline", article.get("webTitle", "")),
+                    "link": article.get("webUrl", ""),
+                    "snippet": fields.get("trailText", ""),
+                    "publication_date": article.get("webPublicationDate", ""),
+                    "section": article.get("sectionName", ""),
+                    "author": fields.get("byline", ""),
+                    "content": fields.get("body", ""),
+                    "full_content": fields.get("body", "")
+                }
+                
+                # Extract tags/keywords
+                tags = article.get("tags", [])
+                result["keywords"] = [tag.get("webTitle", "") for tag in tags if tag.get("type") == "keyword"]
+                
+                formatted_articles.append(result)
+            
+            return formatted_articles
+            
+        except Exception as e:
+            print(f"Error getting data from The Guardian API: {e}")
+            return []
+    
+    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Get preview information for Guardian articles.
+        Actually gets all data but returns only preview fields.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of preview dictionaries
+        """
+        print("Getting articles from The Guardian API")
+        
+        # Get all article data
+        articles = self._get_all_data(query)
+        
+        # Store full articles for later use (implementation detail)
+        self._full_articles = {a["id"]: a for a in articles}
+        
+        # Return only preview fields for each article
+        previews = []
+        for article in articles:
+            preview = {
+                "id": article["id"],
+                "title": article["title"],
+                "link": article["link"],
+                "snippet": article["snippet"],
+                "publication_date": article["publication_date"],
+                "section": article["section"],
+                "author": article["author"],
+                "keywords": article.get("keywords", [])
+            }
+            previews.append(preview)
+        
+        return previews
+    
+    def _get_full_content(self, relevant_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Get full content for the relevant Guardian articles.
+        Restores full content from the cached data.
+        
+        Args:
+            relevant_items: List of relevant preview dictionaries
+            
+        Returns:
+            List of result dictionaries with full content
+        """
+        print("Adding full content to relevant Guardian articles")
+        
+        # Check if we should add full content
+        if hasattr(config, 'SEARCH_SNIPPETS_ONLY') and config.SEARCH_SNIPPETS_ONLY:
+            return relevant_items
+            
+        # Get full articles for relevant items
+        results = []
+        for item in relevant_items:
+            article_id = item.get("id", "")
+            
+            # Get the full article from our cache
+            if hasattr(self, '_full_articles') and article_id in self._full_articles:
+                results.append(self._full_articles[article_id])
+            else:
+                # If not found (shouldn't happen), just use the preview
+                results.append(item)
+        
+        return results
+    
+    def run(self, query: str) -> List[Dict[str, Any]]:
+        """
+        Execute a search using The Guardian API with the two-phase approach.
+        
+        Args:
+            query: The search query
+            
+        Returns:
+            List of search results
+        """
+        print("---Execute a search using The Guardian---")
+        
+        # Use the implementation from the parent class which handles all phases
+        results = super().run(query)
+        
+        # Clean up the cache after use
+        if hasattr(self, '_full_articles'):
+            del self._full_articles
+            
+        return results
+    
+    def get_article_by_id(self, article_id: str) -> Dict[str, Any]:
+        """
+        Get a specific article by its ID.
+        
+        Args:
+            article_id: The Guardian article ID
+            
+        Returns:
+            Dictionary with article information
+        """
+        try:
+            # Guardian article API URL
+            url = f"https://content.guardianapis.com/{article_id}"
+            
+            # Always request all fields
+            response = requests.get(
+                url, 
+                params={
+                    "api-key": self.api_key,
+                    "show-fields": "headline,trailText,body,byline,publication",
+                    "show-tags": "keyword"
+                }
+            )
+            response.raise_for_status()
+            
+            data = response.json()
+            article = data.get("response", {}).get("content", {})
+            
+            if not article:
+                return {}
+                
+            fields = article.get("fields", {})
+            
+            # Format the article with all fields
+            result = {
+                "id": article_id,
+                "title": fields.get("headline", article.get("webTitle", "")),
+                "link": article.get("webUrl", ""),
+                "snippet": fields.get("trailText", ""),
+                "publication_date": article.get("webPublicationDate", ""),
+                "section": article.get("sectionName", ""),
+                "author": fields.get("byline", "")
+            }
+            
+            # Only include full content if not in snippet-only mode
+            if not hasattr(config, 'SEARCH_SNIPPETS_ONLY') or not config.SEARCH_SNIPPETS_ONLY:
+                result["content"] = fields.get("body", "")
+                result["full_content"] = fields.get("body", "")
+            
+            # Extract tags/keywords
+            tags = article.get("tags", [])
+            result["keywords"] = [tag.get("webTitle", "") for tag in tags if tag.get("type") == "keyword"]
+            
+            return result
+            
+        except Exception as e:
+            print(f"Error getting article details: {e}")
+            return {}
+    
+    def search_by_section(self, section: str, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Search for articles in a specific section.
+        
+        Args:
+            section: The Guardian section name (e.g., "politics", "technology")
+            max_results: Maximum number of search results (defaults to self.max_results)
+            
+        Returns:
+            List of articles in the section
+        """
+        original_section = self.section
+        original_max_results = self.max_results
+        
+        try:
+            # Set section and max_results for this search
+            self.section = section
+            if max_results:
+                self.max_results = max_results
+                
+            # Use empty query to get all articles in the section
+            return self.run("")
+            
+        finally:
+            # Restore original values
+            self.section = original_section
+            self.max_results = original_max_results
+    
+    def get_recent_articles(self, days: int = 7, max_results: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Get recent articles from The Guardian.
+        
+        Args:
+            days: Number of days to look back
+            max_results: Maximum number of results (defaults to self.max_results)
+            
+        Returns:
+            List of recent articles
+        """
+        original_from_date = self.from_date
+        original_order_by = self.order_by
+        original_max_results = self.max_results
+        
+        try:
+            # Set parameters for this search
+            self.from_date = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
+            self.order_by = "newest"
+            if max_results:
+                self.max_results = max_results
+                
+            # Use empty query to get all recent articles
+            return self.run("")
+            
+        finally:
+            # Restore original values
+            self.from_date = original_from_date
+            self.order_by = original_order_by
+            self.max_results = original_max_results