webscout 6.3__py3-none-any.whl → 6.4__py3-none-any.whl

webscout/Bing_search.py

@@ -1,154 +1,251 @@
-from bs4 import BeautifulSoup
-import requests
-from typing import Dict, List, Optional, Union
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from urllib.parse import quote, urlparse, parse_qs
-import base64
-import urllib3
-
-# Disable SSL warnings
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class BingS:
-    """Bing search class to get search results from bing.com."""
-
-    _executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=10)
-
-    def __init__(
-        self,
-        headers: Optional[Dict[str, str]] = None,
-        proxy: Optional[str] = None,
-        timeout: Optional[int] = 10,
-    ) -> None:
-        """Initialize the BingS object."""
-        self.proxy: Optional[str] = proxy
-        self.headers = headers if headers else {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-        }
-        self.headers["Referer"] = "https://www.bing.com/"
-        self.client = requests.Session()
-        self.client.headers.update(self.headers)
-        self.client.proxies.update({"http": self.proxy, "https": self.proxy})
-        self.timeout = timeout
-
-    def __enter__(self) -> "BingS":
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.client.close()
-
-    def _get_url(
-        self,
-        method: str,
-        url: str,
-        params: Optional[Dict[str, str]] = None,
-        data: Optional[Union[Dict[str, str], bytes]] = None,
-    ) -> bytes:
-        try:
-            resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout, verify=False)
-        except Exception as ex:
-            raise Exception(f"{url} {type(ex).__name__}: {ex}") from ex
-        if resp.status_code == 200:
-            return resp.content
-        raise Exception(f"{resp.url} returned status code {resp.status_code}. {params=} {data=}")
-
-    def extract_text_from_webpage(self, html_content, max_characters=None):
-        """Extracts visible text from HTML content using BeautifulSoup."""
-        soup = BeautifulSoup(html_content, "html.parser")
-        # Remove unwanted tags
-        for tag in soup(["script", "style", "header", "footer", "nav"]):
-            tag.extract()
-        # Get the remaining visible text
-        visible_text = soup.get_text(separator=' ', strip=True)
-        if max_characters:
-            visible_text = visible_text[:max_characters]
-        return visible_text
-
-    def search(
-        self,
-        keywords: str,
-        max_results: Optional[int] = 10,
-        extract_webpage_text: bool = False,
-        max_extract_characters: Optional[int] = 100,
-    ) -> List[Dict[str, str]]:
-        """Bing text search."""
-        assert keywords, "keywords is mandatory"
-
-        results = []
-        futures = []
-        start = 1
-        while len(results) < max_results:
-            params = {
-                "q": keywords,
-                "first": start
-            }
-            futures.append(self._executor.submit(self._get_url, "GET", "https://www.bing.com/search", params=params))
-            start += 10
-
-            for future in as_completed(futures):
-                try:
-                    resp_content = future.result()
-                    soup = BeautifulSoup(resp_content, "html.parser")
-                    result_block = soup.select('li.b_algo')
-
-                    if not result_block:
-                        break
-
-                    for result in result_block:
-                        try:
-                            link = result.select_one('h2 a')
-                            title = link.text if link else ""
-                            url = link['href'] if link else ""
-                            abstract = result.select_one('.b_caption p')
-                            description = abstract.text if abstract else ""
-
-                            # Remove "WEB" from the beginning of the description if it exists
-                            if description.startswith("WEB"):
-                                description = description[3:].strip()
-
-                            visible_text = ""
-                            if extract_webpage_text:
-                                try:
-                                    actual_url = self._decode_bing_url(url)
-                                    page_content = self._get_url("GET", actual_url)
-                                    visible_text = self.extract_text_from_webpage(
-                                        page_content, max_characters=max_extract_characters
-                                    )
-                                except Exception as e:
-                                    print(f"Error extracting text from {url}: {e}")
-
-                            results.append({
-                                "title": title,
-                                "href": url,
-                                "abstract": description,
-                                "index": len(results),
-                                "type": "web",
-                                "visible_text": visible_text,
-                            })
-
-                            if len(results) >= max_results:
-                                return results
-
-                        except Exception as e:
-                            print(f"Error extracting result: {e}")
-
-                except Exception as e:
-                    print(f"Error fetching URL: {e}")
-
-        return results
-
-    def _decode_bing_url(self, url):
-        if 'bing.com/ck/a' in url:
-            parsed_url = urlparse(url)
-            query_params = parse_qs(parsed_url.query)
-            if 'u' in query_params:
-                encoded_url = query_params['u'][0]
-                return base64.b64decode(encoded_url).decode('utf-8')
-        return url
-
-if __name__ == "__main__":
-    from rich import print
-    searcher = BingS()
-    results = searcher.search("Python development tools", max_results=5, extract_webpage_text=True, max_extract_characters=2000)
-    for result in results:
+from bs4 import BeautifulSoup
+import requests
+from typing import Dict, List, Optional, Union
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from urllib.parse import quote, urlparse, parse_qs
+import base64
+import urllib3
+
+# Disable SSL warnings
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class BingS:
+    """A Python interface for Bing search engine.
+    
+The BingS class provides a simple interface to perform searches on Bing.com
+and extract search results programmatically.
+
+Basic Usage:
+    >>> from webscout.Bing_search import BingS
+    >>> searcher = BingS()
+    >>> results = searcher.search("Python programming")
+    >>> for result in results:
+    ...     print(result['title'], result['href'])
+
+Advanced Usage:
+    >>> # With custom headers and proxy
+    >>> headers = {'User-Agent': 'Custom User Agent'}
+    >>> proxy = 'http://proxy.example.com:8080'
+    >>> searcher = BingS(headers=headers, proxy=proxy)
+    >>> results = searcher.search(
+    ...     "AI developments",
+    ...     max_results=5,
+    ...     extract_webpage_text=True,
+    ...     max_extract_characters=1000
+    ... )
+    >>> # Access result fields
+    >>> for result in results:
+    ...     print(f"Title: {result['title']}")
+    ...     print(f"URL: {result['href']}")
+    ...     print(f"Description: {result['abstract']}")
+    ...     if result['visible_text']:
+    ...         print(f"Page Content: {result['visible_text'][:100]}...")
+
+The class supports context management protocol:
+    >>> with BingS() as searcher:
+    ...     results = searcher.search("Python tutorials")
+
+Return Dictionary Format:
+    {
+        'title': str,       # The title of the search result
+        'href': str,        # The URL of the search result
+        'abstract': str,    # Brief description or snippet
+        'index': int,       # Position in search results
+        'type': str,        # Type of result (always 'web')
+        'visible_text': str # Extracted webpage text (if requested)
+    }
+"""
+
+    _executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=10)
+
+    def __init__(
+        self,
+        headers: Optional[Dict[str, str]] = None,
+        proxy: Optional[str] = None,
+        timeout: Optional[int] = 10,
+    ) -> None:
+        """Initialize a new BingS instance.
+
+        Args:
+            headers (Optional[Dict[str, str]]): Custom HTTP headers for requests.
+                Defaults to a standard User-Agent if not provided.
+            proxy (Optional[str]): Proxy URL to use for requests.
+                Example: 'http://proxy.example.com:8080'
+            timeout (Optional[int]): Request timeout in seconds. Defaults to 10.
+
+        Example:
+            >>> searcher = BingS(
+            ...     headers={'User-Agent': 'Custom UA'},
+            ...     proxy='http://proxy.example.com:8080',
+            ...     timeout=15
+            ... )
+        """
+        self.proxy: Optional[str] = proxy
+        self.headers = headers if headers else {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+        }
+        self.headers["Referer"] = "https://www.bing.com/"
+        self.client = requests.Session()
+        self.client.headers.update(self.headers)
+        self.client.proxies.update({"http": self.proxy, "https": self.proxy})
+        self.timeout = timeout
+
+    def __enter__(self) -> "BingS":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.client.close()
+
+    def _get_url(
+        self,
+        method: str,
+        url: str,
+        params: Optional[Dict[str, str]] = None,
+        data: Optional[Union[Dict[str, str], bytes]] = None,
+    ) -> bytes:
+        try:
+            resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout, verify=False)
+        except Exception as ex:
+            raise Exception(f"{url} {type(ex).__name__}: {ex}") from ex
+        if resp.status_code == 200:
+            return resp.content
+        raise Exception(f"{resp.url} returned status code {resp.status_code}. {params=} {data=}")
+
+    def extract_text_from_webpage(self, html_content, max_characters=None):
+        """Extracts visible text from HTML content using BeautifulSoup."""
+        soup = BeautifulSoup(html_content, "html.parser")
+        # Remove unwanted tags
+        for tag in soup(["script", "style", "header", "footer", "nav"]):
+            tag.extract()
+        # Get the remaining visible text
+        visible_text = soup.get_text(separator=' ', strip=True)
+        if max_characters:
+            visible_text = visible_text[:max_characters]
+        return visible_text
+
+    def search(
+        self,
+        keywords: str,
+        max_results: Optional[int] = 10,
+        extract_webpage_text: bool = False,
+        max_extract_characters: Optional[int] = 100,
+    ) -> List[Dict[str, str]]:
+        """Perform a Bing search and return results.
+
+        Args:
+            keywords (str): Search query string.
+            max_results (Optional[int]): Maximum number of results to return.
+                Defaults to 10.
+            extract_webpage_text (bool): If True, fetches and extracts text from
+                each result webpage. Defaults to False.
+            max_extract_characters (Optional[int]): Maximum number of characters
+                to extract from each webpage. Only used if extract_webpage_text
+                is True. Defaults to 100.
+
+        Returns:
+            List[Dict[str, str]]: List of search results. Each result contains:
+                - title: The title of the search result
+                - href: The URL of the search result
+                - abstract: Brief description or snippet
+                - index: Position in search results
+                - type: Type of result (always 'web')
+                - visible_text: Extracted webpage text (if extract_webpage_text=True)
+
+        Raises:
+            AssertionError: If keywords is empty.
+            Exception: If request fails or returns non-200 status code.
+
+        Example:
+            >>> searcher = BingS()
+            >>> results = searcher.search(
+            ...     "Python tutorials",
+            ...     max_results=5,
+            ...     extract_webpage_text=True
+            ... )
+            >>> for result in results:
+            ...     print(f"Title: {result['title']}")
+            ...     print(f"URL: {result['href']}")
+            ...     print(f"Description: {result['abstract']}")
+            ...     if result['visible_text']:
+            ...         print(f"Content: {result['visible_text'][:100]}...")
+        """
+        assert keywords, "keywords is mandatory"
+
+        results = []
+        futures = []
+        start = 1
+        while len(results) < max_results:
+            params = {
+                "q": keywords,
+                "first": start
+            }
+            futures.append(self._executor.submit(self._get_url, "GET", "https://www.bing.com/search", params=params))
+            start += 10
+
+            for future in as_completed(futures):
+                try:
+                    resp_content = future.result()
+                    soup = BeautifulSoup(resp_content, "html.parser")
+                    result_block = soup.select('li.b_algo')
+
+                    if not result_block:
+                        break
+
+                    for result in result_block:
+                        try:
+                            link = result.select_one('h2 a')
+                            title = link.text if link else ""
+                            url = link['href'] if link else ""
+                            abstract = result.select_one('.b_caption p')
+                            description = abstract.text if abstract else ""
+
+                            # Remove "WEB" from the beginning of the description if it exists
+                            if description.startswith("WEB"):
+                                description = description[3:].strip()
+
+                            visible_text = ""
+                            if extract_webpage_text:
+                                try:
+                                    actual_url = self._decode_bing_url(url)
+                                    page_content = self._get_url("GET", actual_url)
+                                    visible_text = self.extract_text_from_webpage(
+                                        page_content, max_characters=max_extract_characters
+                                    )
+                                except Exception as e:
+                                    print(f"Error extracting text from {url}: {e}")
+
+                            results.append({
+                                "title": title,
+                                "href": url,
+                                "abstract": description,
+                                "index": len(results),
+                                "type": "web",
+                                "visible_text": visible_text,
+                            })
+
+                            if len(results) >= max_results:
+                                return results
+
+                        except Exception as e:
+                            print(f"Error extracting result: {e}")
+
+                except Exception as e:
+                    print(f"Error fetching URL: {e}")
+
+        return results
+
+    def _decode_bing_url(self, url):
+        if 'bing.com/ck/a' in url:
+            parsed_url = urlparse(url)
+            query_params = parse_qs(parsed_url.query)
+            if 'u' in query_params:
+                encoded_url = query_params['u'][0]
+                return base64.b64decode(encoded_url).decode('utf-8')
+        return url
+
+if __name__ == "__main__":
+    from rich import print
+    searcher = BingS()
+    results = searcher.search("Python development tools", max_results=5, extract_webpage_text=True, max_extract_characters=2000)
+    for result in results:
         print(result)

webscout/DWEBS.py

@@ -3,19 +3,97 @@ import requests
 from typing import Dict, List, Optional, Union, Any
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from urllib.parse import quote, urljoin
-from termcolor import colored
+
 import time
 import random
 import json
 import os
 from datetime import datetime, timedelta
 from functools import lru_cache
-import logging
+from .Litlogger import LitLogger, LogFormat, ColorScheme
 from tenacity import retry, stop_after_attempt, wait_exponential
 
 class GoogleS:
-    """
-    Enhanced Google Search class with support for web search, image search, and advanced filters.
+    """A Python interface for Google search with advanced features 🔥
+
+    The GoogleS class provides a powerful interface to perform web searches, image searches,
+    and advanced filtering on Google. Built with love by HAI to keep it 💯
+
+    Basic Usage:
+        >>> from webscout.DWEBS import GoogleS
+        >>> searcher = GoogleS()
+        >>> # Simple web search
+        >>> results = searcher.search("Python programming")
+        >>> for result in results:
+        ...     print(f"Title: {result['title']}")
+        ...     print(f"URL: {result['href']}")
+        ...     print(f"Description: {result['abstract']}")
+
+    Advanced Web Search:
+        >>> # Search with filters
+        >>> results = searcher.search(
+        ...     query="Python tutorials",
+        ...     site="github.com",
+        ...     file_type="pdf",
+        ...     time_period="month",
+        ...     max_results=5
+        ... )
+        >>> # Example response format:
+        >>> {
+        ...     'title': 'Python Tutorial',
+        ...     'href': 'https://example.com/python-tutorial',
+        ...     'abstract': 'Comprehensive Python tutorial covering basics to advanced topics',
+        ...     'index': 0,
+        ...     'type': 'web',
+        ...     'visible_text': ''  # Optional: Contains webpage text if extract_text=True
+        ... }
+
+    Image Search:
+        >>> # Search for images
+        >>> images = searcher.search_images(
+        ...     query="cute puppies",
+        ...     size="large",
+        ...     color="color",
+        ...     type_filter="photo",
+        ...     max_results=5
+        ... )
+        >>> # Example response format:
+        >>> {
+        ...     'title': 'Cute Puppy Image',
+        ...     'thumbnail': 'https://example.com/puppy-thumb.jpg',
+        ...     'full_url': 'https://example.com/puppy-full.jpg',
+        ...     'type': 'image'
+        ... }
+
+    Features:
+        - Web Search: Get detailed web results with title, URL, and description
+        - Image Search: Find images with thumbnails and full-resolution URLs
+        - Advanced Filters: Site-specific search, file types, time periods
+        - Rate Limiting: Smart request handling to avoid blocks
+        - Caching: Save results for faster repeat searches
+        - Retry Logic: Automatic retry on temporary failures
+        - Logging: Optional LitLogger integration for beautiful console output
+        - Proxy Support: Use custom proxies for requests
+        - Concurrent Processing: Multi-threaded requests for better performance
+
+    Response Format:
+        Web Search Results:
+            {
+                'title': str,       # Title of the webpage
+                'href': str,        # URL of the webpage
+                'abstract': str,    # Brief description or snippet
+                'index': int,       # Result position
+                'type': 'web',      # Result type identifier
+                'visible_text': str # Full page text (if extract_text=True)
+            }
+
+        Image Search Results:
+            {
+                'title': str,       # Image title or description
+                'thumbnail': str,   # Thumbnail image URL
+                'full_url': str,    # Full resolution image URL
+                'type': 'image'     # Result type identifier
+            }
     """
 
     SEARCH_TYPES = {
@@ -31,7 +109,8 @@ class GoogleS:
         timeout: Optional[int] = 10,
         max_workers: int = 20,
         cache_dir: Optional[str] = None,
-        rate_limit: float = 0.01
+        rate_limit: float = 0.01,
+        use_litlogger: bool = False
     ):
         """
         Initialize the GoogleS object with enhanced features.
@@ -39,6 +118,7 @@ class GoogleS:
         Args:
             cache_dir: Directory to store search result cache
             rate_limit: Minimum time between requests in seconds
+            use_litlogger: Whether to use LitLogger for logging (default: False)
         """
         self.proxy = proxy
         self.headers = headers if headers else {
@@ -56,10 +136,16 @@ class GoogleS:
             os.makedirs(cache_dir)
         self.last_request_time = 0
         self.rate_limit = rate_limit
+        self.use_litlogger = use_litlogger
         
-        # Setup logging
-        logging.basicConfig(level=logging.INFO)
-        self.logger = logging.getLogger(__name__)
+        # Setup enhanced logging with LitLogger if enabled
+        if self.use_litlogger:
+            self.logger = LitLogger(
+                name="GoogleS",
+                format=LogFormat.MODERN_EMOJI,
+                color_scheme=ColorScheme.CYBERPUNK,
+                console_output=True
+            )
 
     def _respect_rate_limit(self):
         """Ensure minimum time between requests"""
@@ -77,11 +163,16 @@ class GoogleS:
         """
         self._respect_rate_limit()
         try:
+            if self.use_litlogger:
+                self.logger.debug(f"Making {method} request to {url}")
             resp = self.client.request(method, url, params=params, data=data, timeout=self.timeout)
             resp.raise_for_status()
+            if self.use_litlogger:
+                self.logger.success(f"Request successful: {resp.status_code}")
             return resp.content
         except requests.exceptions.RequestException as ex:
-            self.logger.error(f"Request failed: {url} - {str(ex)}")
+            if self.use_litlogger:
+                self.logger.error(f"Request failed: {url} - {str(ex)}")
             raise
 
     @lru_cache(maxsize=100)
@@ -99,7 +190,11 @@ class GoogleS:
             with open(cache_file, 'r') as f:
                 cached_data = json.load(f)
                 if datetime.fromisoformat(cached_data['timestamp']) + timedelta(hours=24) > datetime.now():
+                    if self.use_litlogger:
+                        self.logger.info(f"Using cached results for: {cache_key}")
                     return cached_data['results']
+        if self.use_litlogger:
+            self.logger.debug(f"No valid cache found for: {cache_key}")
         return None
 
     def _cache_results(self, cache_key: str, results: List[Dict[str, Any]]):
@@ -117,18 +212,49 @@ class GoogleS:
         self,
         query: str,
         max_results: int = 10,
-        size: Optional[str] = None,  # large, medium, icon
-        color: Optional[str] = None,  # color, gray, transparent
-        type_filter: Optional[str] = None,  # face, photo, clipart, lineart
+        size: Optional[str] = None,
+        color: Optional[str] = None,
+        type_filter: Optional[str] = None,
         **kwargs
     ) -> List[Dict[str, str]]:
-        """
-        Perform an image search and return results.
-        
+        """Search for images on Google with style! 🖼️
+
         Args:
-            size: Filter by image size
-            color: Filter by color
-            type_filter: Filter by image type
+            query (str): What you're looking for fam
+            max_results (int): How many results you want (default: 10)
+            size (Optional[str]): Image size filter
+                - 'large': Big pics
+                - 'medium': Medium sized
+                - 'icon': Small icons
+            color (Optional[str]): Color filter
+                - 'color': Full color
+                - 'gray': Black and white
+                - 'transparent': Transparent background
+            type_filter (Optional[str]): Type of image
+                - 'face': Just faces
+                - 'photo': Real photos
+                - 'clipart': Vector art
+                - 'lineart': Line drawings
+
+        Returns:
+            List[Dict[str, str]]: List of image results with these keys:
+                - 'thumbnail': Small preview URL
+                - 'full_url': Full resolution image URL
+                - 'title': Image title/description
+                - 'type': Always 'image'
+
+        Example:
+            >>> searcher = GoogleS()
+            >>> # Find some cool nature pics
+            >>> images = searcher.search_images(
+            ...     query="beautiful landscapes",
+            ...     size="large",
+            ...     color="color",
+            ...     max_results=5
+            ... )
+            >>> for img in images:
+            ...     print(f"Found: {img['title']}")
+            ...     print(f"URL: {img['full_url']}")
         """
         params = {
             "q": query,
@@ -192,6 +318,9 @@ class GoogleS:
             exclude_terms: List of terms to exclude from search
             exact_phrase: Exact phrase to match
         """
+        if self.use_litlogger:
+            self.logger.info(f"Starting search for: {query}")
+        
         # Build advanced query
         advanced_query = query
         if site:
@@ -202,7 +331,10 @@ class GoogleS:
             advanced_query += " " + " ".join(f"-{term}" for term in exclude_terms)
         if exact_phrase:
             advanced_query = f'"{exact_phrase}"' + advanced_query
-
+            
+        if self.use_litlogger:
+            self.logger.debug(f"Advanced query: {advanced_query}")
+        
         # Check cache first
         cache_key = self._cache_key(advanced_query, region=region, language=language,
                                   safe=safe, time_period=time_period, sort_by=sort_by)