webscout 2025.10.14.1__py3-none-any.whl → 2025.10.15__py3-none-any.whl

webscout/DWEBS.py

@@ -1,529 +0,0 @@
-"""
-DWEBS - A Google search library with advanced features
-"""
-import random
-from time import sleep
-
-from webscout.scout import Scout
-
-# Import trio before curl_cffi to prevent eventlet socket monkey-patching conflicts
-# See: https://github.com/python-trio/trio/issues/3015
-try:
-    import trio  # noqa: F401
-except ImportError:
-    pass  # trio is optional, ignore if not available
-from concurrent.futures import ThreadPoolExecutor
-from typing import Any, Dict, List, Optional
-from urllib.parse import unquote, urlencode
-
-from curl_cffi.requests import Session
-
-
-class SearchResult:
-    """Class to represent a search result with metadata."""
-
-    def __init__(self, url: str, title: str, description: str):
-        """
-        Initialize a search result.
-        
-        Args:
-            url: The URL of the search result
-            title: The title of the search result
-            description: The description/snippet of the search result
-        """
-        self.url = url
-        self.title = title
-        self.description = description
-        # Additional metadata that can be populated
-        self.metadata: Dict[str, Any] = {}
-
-    def __repr__(self) -> str:
-        """Return string representation of search result."""
-        return f"SearchResult(url={self.url}, title={self.title}, description={self.description})"
-
-
-class GoogleSearch:
-    """Google search implementation with configurable parameters and advanced features."""
-
-    _executor: ThreadPoolExecutor = ThreadPoolExecutor()
-
-    def __init__(
-        self,
-        timeout: int = 10,
-        proxies: Optional[Dict[str, str]] = None,
-        verify: bool = True,
-        lang: str = "en",
-        sleep_interval: float = 0.0,
-        impersonate: str = "chrome110"
-    ):
-        """
-        Initialize GoogleSearch with custom settings.
-
-        Args:
-            timeout: Request timeout in seconds
-            proxies: Proxy configuration for requests
-            verify: Whether to verify SSL certificates
-            lang: Search language
-            sleep_interval: Sleep time between pagination requests
-            impersonate: Browser profile for curl_cffi. Defaults to "chrome110".
-        """
-        self.timeout = timeout # Keep timeout for potential non-session uses or reference
-        self.proxies = proxies if proxies else {}
-        self.verify = verify
-        self.lang = lang
-        self.sleep_interval = sleep_interval
-        self.base_url = "https://www.google.com/search"
-        # Initialize curl_cffi session
-        self.session = Session(
-            proxies=self.proxies,
-            verify=self.verify,
-            timeout=self.timeout,
-            impersonate=impersonate
-        )
-        # Set common headers for the session
-        self.session.headers = {
-            "User-Agent": self._get_useragent(),
-            "Accept-Language": self.lang,
-            "Accept-Encoding": "gzip, deflate, br",
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-        }
-        # Set default cookies for the session
-        self.session.cookies.update({
-            'CONSENT': 'PENDING+987',
-            'SOCS': 'CAESHAgBEhIaAB',
-        })
-
-    def _get_useragent(self) -> str:
-        """
-        Generate a random user agent string.
-        
-        Returns:
-            Random user agent string
-        """
-        lynx_version = f"Lynx/{random.randint(2, 3)}.{random.randint(8, 9)}.{random.randint(0, 2)}"
-        libwww_version = f"libwww-FM/{random.randint(2, 3)}.{random.randint(13, 15)}"
-        ssl_mm_version = f"SSL-MM/{random.randint(1, 2)}.{random.randint(3, 5)}"
-        openssl_version = f"OpenSSL/{random.randint(1, 3)}.{random.randint(0, 4)}.{random.randint(0, 9)}"
-        return f"{lynx_version} {libwww_version} {ssl_mm_version} {openssl_version}"
-
-    def _make_request(self, term: str, results: int, start: int = 0, search_type: str = None) -> str:
-        """
-        Make a request to Google search.
-        
-        Args:
-            term: Search query
-            results: Number of results to request
-            start: Start position for pagination
-            search_type: Type of search ('', 'nws', 'isch')
-            
-        Returns:
-            HTML response content
-        """
-        params = {
-            "q": term,
-            "num": results + 2,  # Request slightly more than needed
-            "hl": self.lang,
-            "start": start,
-        }
-
-        # Add search type if specified
-        if search_type:
-            params["tbm"] = search_type
-
-        try:
-            # Use the curl_cffi session
-            resp = self.session.get(
-                url=self.base_url,
-                params=params,
-                # Headers and cookies are now part of the session
-                # proxies, timeout, verify are handled by the session
-            )
-            resp.raise_for_status()
-            return resp.text
-        except Exception as e:
-            # Provide more specific error context if possible
-            if hasattr(e, 'response') and e.response is not None:
-                 raise RuntimeError(f"Search request failed with status {e.response.status_code}: {str(e)}")
-            else:
-                 raise RuntimeError(f"Search request failed: {str(e)}")
-
-    def _extract_url(self, raw_link: str) -> Optional[str]:
-        """
-        Extract actual URL from Google redirect URL.
-        
-        Args:
-            raw_link: Raw link from Google search
-            
-        Returns:
-            Actual URL or None if invalid
-        """
-        if not raw_link:
-            return None
-
-        if raw_link.startswith("/url?"):
-            try:
-                link = unquote(raw_link.split("&")[0].replace("/url?q=", ""))
-                return link
-            except Exception:
-                return None
-        elif raw_link.startswith("http"):
-            return unquote(raw_link)
-
-        return None
-
-    def _is_valid_result(self, link: str, fetched_links: set, unique: bool) -> bool:
-        """
-        Check if search result is valid.
-        
-        Args:
-            link: URL to check
-            fetched_links: Set of already fetched links
-            unique: Whether to filter duplicate links
-            
-        Returns:
-            Boolean indicating if result is valid
-        """
-        if any(x in link for x in ["google.", "/search?", "webcache."]):
-            return False
-
-        if link in fetched_links and unique:
-            return False
-
-        return True
-
-    def _parse_search_results(
-        self,
-        html: str,
-        num_results: int,
-        fetched_links: set,
-        unique: bool
-    ) -> List[SearchResult]:
-        """
-        Parse search results from HTML.
-        
-        Args:
-            html: HTML content to parse
-            num_results: Maximum number of results to return
-            fetched_links: Set of already fetched links
-            unique: Filter duplicate links
-            
-        Returns:
-            List of SearchResult objects
-        """
-        results = []
-        soup = Scout(html, features="html.parser")
-        result_blocks = soup.find_all("div", class_="ezO2md")
-
-        if not result_blocks:
-            # Try alternative class patterns if the main one doesn't match
-            result_blocks = soup.find_all("div", attrs={"class": lambda c: c and "g" in c.split()})
-
-        for result in result_blocks:
-            # Find the link - looking for various potential Google result classes
-            link_tag = result.find("a", class_=["fuLhoc", "ZWRArf"])
-            if not link_tag:
-                link_tag = result.find("a")
-                if not link_tag:
-                    continue
-
-            raw_link = link_tag.get("href", "")
-            link = self._extract_url(raw_link)
-
-            if not link:
-                continue
-
-            if not self._is_valid_result(link, fetched_links, unique):
-                continue
-
-            # Get title - it's the text content of the link tag for these results
-            title = link_tag.get_text(strip=True)
-            if not title:
-                continue
-
-            # Get description - it's in a span with class FrIlee or potentially other classes
-            description_tag = result.find("span", class_="FrIlee")
-            if not description_tag:
-                description_tag = result.find(["div", "span"], class_=lambda c: c and any(x in c for x in ["snippet", "description", "VwiC3b"]))
-
-            description = description_tag.get_text(strip=True) if description_tag else ""
-
-            # Create result object
-            search_result = SearchResult(link, title, description)
-
-            # Add extra metadata if available
-            citation = result.find("cite")
-            if citation:
-                search_result.metadata["source"] = citation.get_text(strip=True)
-
-            timestamp = result.find("span", class_=lambda c: c and "ZE5qJf" in c)
-            if timestamp:
-                search_result.metadata["date"] = timestamp.get_text(strip=True)
-
-            fetched_links.add(link)
-            results.append(search_result)
-
-            if len(results) >= num_results:
-                break
-
-        return results
-
-    def text(
-        self,
-        keywords: str,
-        region: str = None,
-        safesearch: str = "moderate",
-        max_results: int = 10,
-        start_num: int = 0,
-        unique: bool = True
-    ) -> List[SearchResult]:
-        """
-        Search Google for web results.
-        
-        Args:
-            keywords: Search query
-            region: Region for search results (ISO country code)
-            safesearch: SafeSearch setting ("on", "moderate", "off")
-            max_results: Maximum number of results to return
-            start_num: Starting position for pagination
-            unique: Filter duplicate results
-            
-        Returns:
-            List of SearchResult objects with search results
-        """
-        if not keywords:
-            raise ValueError("Search keywords cannot be empty")
-
-        # Map safesearch values to Google's safe parameter
-        safe_map = {
-            "on": "active",
-            "moderate": "moderate",
-            "off": "off"
-        }
-        safe = safe_map.get(safesearch.lower(), "moderate")
-
-        # Keep track of unique results
-        fetched_results = []
-        fetched_links = set()
-        start = start_num
-
-        while len(fetched_results) < max_results:
-            # Add safe search parameter to the request
-            # Note: This modifies the session params for this specific request type
-            # It might be better to pass params directly to session.get if mixing search types
-            term_with_safe = f"{keywords} safe:{safe}"
-            if region and region.lower() != "all":
-                 term_with_safe += f" location:{region}" # Example of adding region, adjust as needed
-
-            response_html = self._make_request(
-                term=term_with_safe, # Pass term with safe search
-                results=max_results - len(fetched_results),
-                start=start
-            )
-
-            results = self._parse_search_results(
-                html=response_html,
-                num_results=max_results - len(fetched_results),
-                fetched_links=fetched_links,
-                unique=unique
-            )
-
-            if not results:
-                break
-
-            fetched_results.extend(results)
-
-            if len(fetched_results) >= max_results:
-                break
-
-            start += 10 # Google typically uses increments of 10
-            sleep(self.sleep_interval)
-
-        return fetched_results[:max_results]
-
-    def news(
-        self,
-        keywords: str,
-        region: str = None,
-        safesearch: str = "moderate",
-        max_results: int = 10
-    ) -> List[SearchResult]:
-        """
-        Search Google News for news results.
-        
-        Args:
-            keywords: Search query
-            region: Region for search results (ISO country code)
-            safesearch: SafeSearch setting ("on", "moderate", "off")
-            max_results: Maximum number of results to return
-            
-        Returns:
-            List of SearchResult objects with news results
-        """
-        if not keywords:
-            raise ValueError("Search keywords cannot be empty")
-
-        # Map safesearch values to Google's safe parameter
-        safe_map = {
-            "on": "active",
-            "moderate": "moderate",
-            "off": "off"
-        }
-        safe = safe_map.get(safesearch.lower(), "moderate")
-
-        # Keep track of unique results
-        fetched_links = set()
-
-        # Add safe search parameter
-        term_with_safe = f"{keywords} safe:{safe}"
-        if region and region.lower() != "all":
-             term_with_safe += f" location:{region}" # Example
-
-        response_html = self._make_request(
-            term=term_with_safe, # Pass term with safe search
-            results=max_results,
-            search_type="nws"
-        )
-
-        results = self._parse_search_results(
-            html=response_html,
-            num_results=max_results,
-            fetched_links=fetched_links,
-            unique=True # News results are generally unique per request
-        )
-
-        return results[:max_results]
-
-    def suggestions(self, query: str, region: str = None) -> List[str]:
-        """
-        Get search suggestions for a query term.
-        
-        Args:
-            query: Search query
-            region: Region for suggestions (ISO country code)
-        
-        Returns:
-            List of search suggestions
-        """
-        if not query:
-            raise ValueError("Search query cannot be empty")
-
-        try:
-            params = {
-                "client": "firefox",
-                "q": query,
-            }
-
-            # Add region if specified
-            if region and region.lower() != "all":
-                params["gl"] = region
-
-            url = f"https://www.google.com/complete/search?{urlencode(params)}"
-
-            # Use a simpler header set for the suggestions API
-            headers = {
-                "User-Agent": self._get_useragent(),
-                "Accept": "application/json, text/javascript, */*",
-                "Accept-Language": self.lang,
-            }
-
-            # Use session.get but override headers for this specific request
-            response = self.session.get(
-                url=url,
-                headers=headers,
-                params=params # Pass params directly
-                # timeout and verify are handled by session
-            )
-            response.raise_for_status()
-
-            # Response format is typically: ["original query", ["suggestion1", "suggestion2", ...]]
-            data = response.json()
-            if isinstance(data, list) and len(data) > 1 and isinstance(data[1], list):
-                return data[1]
-            return []
-
-        except Exception as e:
-            # Provide more specific error context if possible
-            if hasattr(e, 'response') and e.response is not None:
-                 # Log error or handle differently if needed
-                 print(f"Suggestions request failed with status {e.response.status_code}: {str(e)}")
-            else:
-                 print(f"Suggestions request failed: {str(e)}")
-            # Return empty list on error instead of raising exception
-            return []
-
-
-# Legacy function support for backward compatibility
-def search(term, num_results=10, lang="en", proxy=None, advanced=False, sleep_interval=0, timeout=5, safe="active", ssl_verify=True, region=None, start_num=0, unique=False, impersonate="chrome110"): # Added impersonate
-    """Legacy function for backward compatibility."""
-    google_search = GoogleSearch(
-        timeout=timeout,
-        proxies={"https": proxy, "http": proxy} if proxy else None,
-        verify=ssl_verify,
-        lang=lang,
-        sleep_interval=sleep_interval,
-        impersonate=impersonate # Pass impersonate
-    )
-
-    # Map legacy safe values
-    safe_search_map = {
-        "active": "on",
-        "moderate": "moderate",
-        "off": "off"
-    }
-    safesearch_val = safe_search_map.get(safe, "moderate")
-
-    results = google_search.text(
-        keywords=term,
-        region=region,
-        safesearch=safesearch_val,
-        max_results=num_results,
-        start_num=start_num,
-        unique=unique
-    )
-
-    # Convert to simple URLs if not advanced mode
-    if not advanced:
-        return [result.url for result in results]
-    return results
-
-
-if __name__ == "__main__":
-    from rich import print
-    google = GoogleSearch(
-        timeout=10,  # Optional: Set custom timeout
-        proxies=None,  # Optional: Use proxies
-        verify=True    # Optional: SSL verification
-    )
-
-    # Text Search
-    print("TEXT SEARCH RESULTS:")
-    text_results = google.text(
-        keywords="Python programming",
-        region="us",           # Optional: Region for results
-        safesearch="moderate",  # Optional: "on", "moderate", "off"
-        max_results=3          # Optional: Limit number of results
-    )
-    for result in text_results:
-        print(f"Title: {result.title}")
-        print(f"URL: {result.url}")
-        print(f"Description: {result.description}")
-        print("---")
-
-    # News Search
-    print("\nNEWS SEARCH RESULTS:")
-    news_results = google.news(
-        keywords="artificial intelligence",
-        region="us",
-        safesearch="moderate",
-        max_results=2
-    )
-    for result in news_results:
-        print(f"Title: {result.title}")
-        print(f"URL: {result.url}")
-        print(f"Description: {result.description}")
-        print("---")
-
-    # Search Suggestions
-    print("\nSEARCH SUGGESTIONS:")
-    suggestions = google.suggestions("how to")
-    print(suggestions)

webscout/search/engines/bing.py

@@ -1,84 +0,0 @@
-"""Bing search engine implementation."""
-
-from __future__ import annotations
-
-import base64
-from collections.abc import Mapping
-from time import time
-from typing import Any
-from urllib.parse import parse_qs, urlparse
-
-from ..base import BaseSearchEngine
-from ..results import TextResult
-
-
-def unwrap_bing_url(raw_url: str) -> str | None:
-    """Decode the Bing-wrapped raw_url to extract the original url."""
-    try:
-        parsed = urlparse(raw_url)
-        if parsed.netloc == "www.bing.com" and parsed.path == "/ck/a":
-            query_params = parse_qs(parsed.query)
-            if "u" in query_params:
-                encoded_url = query_params["u"][0]
-                # Decode the base64-like encoding
-                if encoded_url.startswith("a1"):
-                    encoded_url = encoded_url[2:]
-                # Add padding if needed
-                padding = len(encoded_url) % 4
-                if padding:
-                    encoded_url += "=" * (4 - padding)
-                try:
-                    decoded = base64.urlsafe_b64decode(encoded_url).decode()
-                    return decoded
-                except Exception:
-                    pass
-        return raw_url
-    except Exception:
-        return raw_url
-
-
-class Bing(BaseSearchEngine[TextResult]):
-    """Bing search engine."""
-
-    name = "bing"
-    category = "text"
-    provider = "bing"
-
-    search_url = "https://www.bing.com/search"
-    search_method = "GET"
-
-    items_xpath = "//li[contains(@class, 'b_algo')]"
-    elements_xpath: Mapping[str, str] = {
-        "title": ".//h2/a//text()",
-        "href": ".//h2/a/@href",
-        "body": ".//p//text()",
-    }
-
-    def build_payload(
-        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
-    ) -> dict[str, Any]:
-        """Build a payload for the Bing search request."""
-        country, lang = region.lower().split("-")
-        payload = {"q": query, "pq": query, "cc": lang}
-        cookies = {
-            "_EDGE_CD": f"m={lang}-{country}&u={lang}-{country}",
-            "_EDGE_S": f"mkt={lang}-{country}&ui={lang}-{country}",
-        }
-        self.http_client.set_cookies("https://www.bing.com", cookies)
-        if timelimit:
-            d = int(time() // 86400)
-            payload["filters"] = {
-                "d": f"ex1:\"ez1_{d - 1}_{d}\"",
-                "w": f"ex1:\"ez1_{d - 7}_{d}\"",
-                "m": f"ex1:\"ez1_{d - 30}_{d}\"",
-                "y": f"ex1:\"ez1_{d - 365}_{d}\"",
-            }[timelimit]
-        if page > 1:
-            payload["first"] = f"{(page - 1) * 10 + 1}"
-        return payload
-
-    def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
-        """Post-process search results."""
-        for result in results:
-            result.href = unwrap_bing_url(result.href)
-        return results

webscout/search/engines/bing_news.py

@@ -1,52 +0,0 @@
-"""Bing news search engine implementation."""
-
-from __future__ import annotations
-
-from collections.abc import Mapping
-from typing import Any
-
-from ..base import BaseSearchEngine
-from ..results import NewsResult
-
-
-class BingNews(BaseSearchEngine[NewsResult]):
-    """Bing news engine."""
-
-    name = "bing"
-    category = "news"
-    provider = "bing"
-
-    search_url = "https://www.bing.com/news/infinitescrollajax"
-    search_method = "GET"
-
-    items_xpath = "//div[contains(@class, 'newsitem')]"
-    elements_xpath: Mapping[str, str] = {
-        "date": ".//span[@aria-label]//@aria-label",
-        "title": "@data-title",
-        "body": ".//div[@class='snippet']//text()",
-        "url": "@url",
-        "image": ".//a[contains(@class, 'image')]//@src",
-        "source": "@data-author",
-    }
-
-    def build_payload(
-        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
-    ) -> dict[str, Any]:
-        """Build a payload for the Bing search request."""
-        country, lang = region.lower().split("-")
-        payload = {
-            "q": query,
-            "InfiniteScroll": "1",
-            "first": f"{page * 10 + 1}",
-            "SFX": f"{page}",
-            "cc": country,
-            "setlang": lang,
-        }
-        if timelimit:
-            payload["qft"] = {
-                "d": 'interval="4"',  # doesn't exist so it's the same as one hour
-                "w": 'interval="7"',
-                "m": 'interval="9"',
-                "y": 'interval="9"',  # doesn't exist so it's the same as month
-            }[timelimit]
-        return payload