webscout 6.3__py3-none-any.whl → 6.5__py3-none-any.whl

webscout/DWEBS.py

@@ -1,21 +1,99 @@
-from bs4 import BeautifulSoup
 import requests
 from typing import Dict, List, Optional, Union, Any
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from webscout.scout import Scout
 from urllib.parse import quote, urljoin
-from termcolor import colored
+from webscout.litagent import LitAgent
+
 import time
 import random
 import json
 import os
 from datetime import datetime, timedelta
 from functools import lru_cache
-import logging
-from tenacity import retry, stop_after_attempt, wait_exponential
+from webscout.Litlogger import LitLogger, LogFormat, ColorScheme
 
 class GoogleS:
-    """
-    Enhanced Google Search class with support for web search, image search, and advanced filters.
+    """A Python interface for Google search with advanced features
+
+    The GoogleS class provides a powerful interface to perform web searches, image searches,
+    and advanced filtering on Google. Built with love by HAI to keep it
+
+    Basic Usage:
+        >>> from webscout.DWEBS import GoogleS
+        >>> searcher = GoogleS()
+        >>> # Simple web search
+        >>> results = searcher.search("Python programming")
+        >>> for result in results:
+        ...     print(f"Title: {result['title']}")
+        ...     print(f"URL: {result['href']}")
+        ...     print(f"Description: {result['abstract']}")
+
+    Advanced Web Search:
+        >>> # Search with filters
+        >>> results = searcher.search(
+        ...     query="Python tutorials",
+        ...     site="github.com",
+        ...     file_type="pdf",
+        ...     time_period="month",
+        ...     max_results=5
+        ... )
+        >>> # Example response format:
+        >>> {
+        ...     'title': 'Python Tutorial',
+        ...     'href': 'https://example.com/python-tutorial',
+        ...     'abstract': 'Comprehensive Python tutorial covering basics to advanced topics',
+        ...     'index': 0,
+        ...     'type': 'web',
+        ...     'visible_text': ''  # Optional: Contains webpage text if extract_text=True
+        ... }
+
+    Image Search:
+        >>> # Search for images
+        >>> images = searcher.search_images(
+        ...     query="cute puppies",
+        ...     size="large",
+        ...     color="color",
+        ...     type_filter="photo",
+        ...     max_results=5
+        ... )
+        >>> # Example response format:
+        >>> {
+        ...     'title': 'Cute Puppy Image',
+        ...     'thumbnail': 'https://example.com/puppy-thumb.jpg',
+        ...     'full_url': 'https://example.com/puppy-full.jpg',
+        ...     'type': 'image'
+        ... }
+
+    Features:
+        - Web Search: Get detailed web results with title, URL, and description
+        - Image Search: Find images with thumbnails and full-resolution URLs
+        - Advanced Filters: Site-specific search, file types, time periods
+        - Rate Limiting: Smart request handling to avoid blocks
+        - Caching: Save results for faster repeat searches
+        - Retry Logic: Automatic retry on temporary failures
+        - Logging: Optional LitLogger integration for beautiful console output
+        - Proxy Support: Use custom proxies for requests
+        - Concurrent Processing: Multi-threaded requests for better performance
+
+    Response Format:
+        Web Search Results:
+            {
+                'title': str,       # Title of the webpage
+                'href': str,        # URL of the webpage
+                'abstract': str,    # Brief description or snippet
+                'index': int,       # Result position
+                'type': 'web',      # Result type identifier
+                'visible_text': str # Full page text (if extract_text=True)
+            }
+
+        Image Search Results:
+            {
+                'title': str,       # Image title or description
+                'thumbnail': str,   # Thumbnail image URL
+                'full_url': str,    # Full resolution image URL
+                'type': 'image'     # Result type identifier
+            }
     """
 
     SEARCH_TYPES = {
@@ -31,7 +109,8 @@ class GoogleS:
         timeout: Optional[int] = 10,
         max_workers: int = 20,
         cache_dir: Optional[str] = None,
-        rate_limit: float = 0.01
+        rate_limit: float = 0.01,
+        use_litlogger: bool = False
     ):
         """
         Initialize the GoogleS object with enhanced features.
@@ -39,10 +118,11 @@ class GoogleS:
         Args:
             cache_dir: Directory to store search result cache
             rate_limit: Minimum time between requests in seconds
+            use_litlogger: Whether to use LitLogger for logging (default: False)
         """
         self.proxy = proxy
         self.headers = headers if headers else {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"
+            "User-Agent": LitAgent().random()  # Use LitAgent to generate user agent
         }
         self.headers["Referer"] = "https://www.google.com/"
         self.client = requests.Session()
@@ -56,10 +136,16 @@ class GoogleS:
             os.makedirs(cache_dir)
         self.last_request_time = 0
         self.rate_limit = rate_limit
+        self.use_litlogger = use_litlogger
         
-        # Setup logging
-        logging.basicConfig(level=logging.INFO)
-        self.logger = logging.getLogger(__name__)
+        # Setup enhanced logging with LitLogger if enabled
+        if self.use_litlogger:
+            self.logger = LitLogger(
+                name="GoogleS",
+                format=LogFormat.MODERN_EMOJI,
+                color_scheme=ColorScheme.CYBERPUNK,
+                console_output=True
+            )
 
     def _respect_rate_limit(self):
         """Ensure minimum time between requests"""
@@ -69,20 +155,46 @@ class GoogleS:
             time.sleep(self.rate_limit - time_since_last)
         self.last_request_time = time.time()
 
-    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
     def _get_url(self, method: str, url: str, params: Optional[Dict[str, str]] = None,
-                  data: Optional[Union[Dict[str, str], bytes]] = None) -> bytes:
+                  data: Optional[Union[Dict[str, str], bytes]] = None, max_retries: int = 3) -> bytes:
         """
-        Makes an HTTP request with retry logic and rate limiting.
+        Makes an HTTP request with manual retry logic and rate limiting.
+        
+        Args:
+            method (str): HTTP method (GET, POST, etc.)
+            url (str): Target URL
+            params (Optional[Dict[str, str]]): Query parameters
+            data (Optional[Union[Dict[str, str], bytes]]): Request payload
+            max_retries (int): Maximum number of retry attempts
+        
+        Returns:
+            bytes: Response content
         """
         self._respect_rate_limit()
-        try:
-            resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout)
-            resp.raise_for_status()
-            return resp.content
-        except requests.exceptions.RequestException as ex:
-            self.logger.error(f"Request failed: {url} - {str(ex)}")
-            raise
+        
+        for attempt in range(max_retries):
+            try:
+                if self.use_litlogger:
+                    self.logger.debug(f"Making {method} request to {url} (Attempt {attempt + 1})")
+                
+                resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout)
+                resp.raise_for_status()
+                
+                if self.use_litlogger:
+                    self.logger.success(f"Request successful: {resp.status_code}")
+                
+                return resp.content
+            
+            except requests.exceptions.RequestException as ex:
+                if self.use_litlogger:
+                    self.logger.error(f"Request failed: {url} - {str(ex)}")
+                
+                # Exponential backoff
+                if attempt < max_retries - 1:
+                    wait_time = (2 ** attempt) + random.random()
+                    time.sleep(wait_time)
+                else:
+                    raise
 
     @lru_cache(maxsize=100)
     def _cache_key(self, query: str, **kwargs) -> str:
@@ -99,7 +211,11 @@ class GoogleS:
             with open(cache_file, 'r') as f:
                 cached_data = json.load(f)
                 if datetime.fromisoformat(cached_data['timestamp']) + timedelta(hours=24) > datetime.now():
+                    if self.use_litlogger:
+                        self.logger.info(f"Using cached results for: {cache_key}")
                     return cached_data['results']
+        if self.use_litlogger:
+            self.logger.debug(f"No valid cache found for: {cache_key}")
         return None
 
     def _cache_results(self, cache_key: str, results: List[Dict[str, Any]]):
@@ -117,18 +233,49 @@ class GoogleS:
         self,
         query: str,
         max_results: int = 10,
-        size: Optional[str] = None,  # large, medium, icon
-        color: Optional[str] = None,  # color, gray, transparent
-        type_filter: Optional[str] = None,  # face, photo, clipart, lineart
+        size: Optional[str] = None,
+        color: Optional[str] = None,
+        type_filter: Optional[str] = None,
         **kwargs
     ) -> List[Dict[str, str]]:
-        """
-        Perform an image search and return results.
-        
+        """Search for images on Google with style! 
+
         Args:
-            size: Filter by image size
-            color: Filter by color
-            type_filter: Filter by image type
+            query (str): What you're looking for fam
+            max_results (int): How many results you want (default: 10)
+            size (Optional[str]): Image size filter
+                - 'large': Big pics
+                - 'medium': Medium sized
+                - 'icon': Small icons
+            color (Optional[str]): Color filter
+                - 'color': Full color
+                - 'gray': Black and white
+                - 'transparent': Transparent background
+            type_filter (Optional[str]): Type of image
+                - 'face': Just faces
+                - 'photo': Real photos
+                - 'clipart': Vector art
+                - 'lineart': Line drawings
+
+        Returns:
+            List[Dict[str, str]]: List of image results with these keys:
+                - 'thumbnail': Small preview URL
+                - 'full_url': Full resolution image URL
+                - 'title': Image title/description
+                - 'type': Always 'image'
+
+        Example:
+            >>> searcher = GoogleS()
+            >>> # Find some cool nature pics
+            >>> images = searcher.search_images(
+            ...     query="beautiful landscapes",
+            ...     size="large",
+            ...     color="color",
+            ...     max_results=5
+            ... )
+            >>> for img in images:
+            ...     print(f"Found: {img['title']}")
+            ...     print(f"URL: {img['full_url']}")
         """
         params = {
             "q": query,
@@ -144,7 +291,7 @@ class GoogleS:
             params["tbs"] = f"itp:{type_filter}"
 
         content = self._get_url("GET", self.SEARCH_TYPES["image"], params=params)
-        soup = BeautifulSoup(content, 'lxml')
+        soup = Scout(content)  # Use Scout parser
         
         results = []
         for img in soup.find_all("img", class_="rg_i"):
@@ -192,6 +339,9 @@ class GoogleS:
             exclude_terms: List of terms to exclude from search
             exact_phrase: Exact phrase to match
         """
+        if self.use_litlogger:
+            self.logger.info(f"Starting search for: {query}")
+        
         # Build advanced query
         advanced_query = query
         if site:
@@ -202,7 +352,10 @@ class GoogleS:
             advanced_query += " " + " ".join(f"-{term}" for term in exclude_terms)
         if exact_phrase:
             advanced_query = f'"{exact_phrase}"' + advanced_query
-
+            
+        if self.use_litlogger:
+            self.logger.debug(f"Advanced query: {advanced_query}")
+        
         # Check cache first
         cache_key = self._cache_key(advanced_query, region=region, language=language,
                                   safe=safe, time_period=time_period, sort_by=sort_by)
@@ -233,7 +386,8 @@ class GoogleS:
             for future in as_completed(futures):
                 try:
                     resp_content = future.result()
-                    soup = BeautifulSoup(resp_content, 'lxml')  # Use lxml parser
+                    soup = Scout(resp_content)  # Use Scout parser
+                    
                     result_blocks = soup.find_all("div", class_="g")
 
                     if not result_blocks:
@@ -297,9 +451,9 @@ class GoogleS:
 
     def _extract_text_from_webpage(self, html_content: bytes, max_characters: Optional[int] = None) -> str:
         """
-        Extracts visible text from HTML content using lxml parser.
+        Extracts visible text from HTML content using Scout parser.
         """
-        soup = BeautifulSoup(html_content, 'lxml')  # Use lxml parser
+        soup = Scout(html_content)  # Use Scout parser
         for tag in soup(["script", "style", "header", "footer", "nav"]):
             tag.extract()
         visible_text = soup.get_text(strip=True)