webscout 2025.10.11__py3-none-any.whl → 2025.10.14.1__py3-none-any.whl

webscout/Provider/Andi.py

@@ -6,7 +6,7 @@ from webscout.AIutel import Conversation
 from webscout.AIutel import AwesomePrompts
 from webscout.AIbase import  Provider
 from webscout import exceptions
-from webscout import WEBS
+from ..search import DuckDuckGoSearch
 from webscout.litagent import LitAgent
 
 class AndiSearch(Provider):

webscout/Provider/ChatGPTClone.py

@@ -12,7 +12,8 @@ from webscout.AIutel import Optimizers
 from webscout.AIutel import Conversation
 from webscout.AIutel import AwesomePrompts, sanitize_stream # Import sanitize_stream
 from webscout.AIbase import Provider
-from webscout import WEBS, exceptions
+from ..search import DuckDuckGoSearch
+from .. import exceptions
 # from webscout.litagent import LitAgent
 
 class ChatGPTClone(Provider):

webscout/__init__.py

@@ -1,9 +1,7 @@
 # webscout/__init__.py
 
-from .webscout_search import WEBS
-from .webscout_search_async import AsyncWEBS
+from .search import DuckDuckGoSearch, YepSearch
 from .version import __version__
-from .DWEBS import *
 from .Provider import *
 from .Provider.TTI import *
 from .Provider.TTS import *
@@ -15,7 +13,6 @@ from .swiftcli import *
 from .litagent import LitAgent
 from .scout import *
 from .zeroart import *
-from .yep_search import *
 from .AIutel import *
 
 useragent = LitAgent()

webscout/auth/routes.py

@@ -42,9 +42,8 @@ from .request_processing import (
 )
 from .auth_system import get_auth_components
 from .simple_logger import request_logger
-from webscout.DWEBS import GoogleSearch
-from webscout.yep_search import YepSearch
-from webscout.webscout_search import WEBS
+from ..search import DuckDuckGoSearch, YepSearch
+from ..DWEBS import GoogleSearch
 from webscout.Bing_search import BingSearch
 
 # Setup logger

webscout/cli.py

@@ -1,10 +1,12 @@
 import sys
 from .swiftcli import CLI, option
-from .webscout_search import WEBS  # Import the WEBS class from webscout_search
+from .search import DuckDuckGoSearch, YepSearch  # Import search classes
 from .DWEBS import GoogleSearch  # Import GoogleSearch from DWEBS
-from .yep_search import YepSearch  # Import YepSearch from yep_search
 from .version import __version__
 
+# Alias for backward compatibility
+WEBS = DuckDuckGoSearch
+
 
 def _print_data(data):
     """Prints data in a simple formatted way."""

webscout/search/__init__.py

@@ -0,0 +1,51 @@
+"""Webscout search module - unified search interfaces."""
+
+from .base import BaseSearch, BaseSearchEngine
+from .duckduckgo_main import DuckDuckGoSearch
+from .yep_main import YepSearch
+
+# Import new search engines
+from .engines.bing import Bing
+from .engines.brave import Brave
+from .engines.mojeek import Mojeek
+from .engines.yahoo import Yahoo
+from .engines.yandex import Yandex
+from .engines.wikipedia import Wikipedia
+from .engines.bing_news import BingNews
+from .engines.yahoo_news import YahooNews
+
+# Import result models
+from .results import (
+    TextResult,
+    ImagesResult,
+    VideosResult,
+    NewsResult,
+    BooksResult,
+)
+
+__all__ = [
+    # Base classes
+    "BaseSearch",
+    "BaseSearchEngine",
+    
+    # Main search interfaces
+    "DuckDuckGoSearch",
+    "YepSearch",
+    
+    # Individual engines
+    "Bing",
+    "Brave",
+    "Mojeek",
+    "Yahoo",
+    "Yandex",
+    "Wikipedia",
+    "BingNews",
+    "YahooNews",
+    
+    # Result models
+    "TextResult",
+    "ImagesResult",
+    "VideosResult",
+    "NewsResult",
+    "BooksResult",
+]

webscout/search/base.py

@@ -0,0 +1,195 @@
+"""Base class for search engines."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from functools import cached_property
+from typing import Any, Generic, Literal, TypeVar
+
+try:
+    from lxml import html
+    from lxml.etree import HTMLParser as LHTMLParser
+    LXML_AVAILABLE = True
+except ImportError:
+    LXML_AVAILABLE = False
+    html = None  # type: ignore
+    LHTMLParser = None  # type: ignore
+
+from .http_client import HttpClient
+from .results import BooksResult, ImagesResult, NewsResult, TextResult, VideosResult
+
+logger = logging.getLogger(__name__)
+T = TypeVar("T")
+
+
+class BaseSearchEngine(ABC, Generic[T]):
+    """Abstract base class for all search engine backends."""
+
+    name: str  # unique key, e.g. "google"
+    category: Literal["text", "images", "videos", "news", "books"]
+    provider: str  # source of the search results (e.g. "google", "bing", etc.)
+    disabled: bool = False  # if True, the engine is disabled
+    priority: float = 1
+
+    search_url: str
+    search_method: str  # GET or POST
+    search_headers: Mapping[str, str] = {}
+    items_xpath: str = ""
+    elements_xpath: Mapping[str, str] = {}
+    elements_replace: Mapping[str, str] = {}
+
+    def __init__(self, proxy: str | None = None, timeout: int | None = None, verify: bool = True):
+        """Initialize search engine.
+        
+        Args:
+            proxy: Proxy URL (supports http/https/socks5).
+            timeout: Request timeout in seconds.
+            verify: Whether to verify SSL certificates.
+        """
+        self.http_client = HttpClient(proxy=proxy, timeout=timeout, verify=verify)
+        self.results: list[T] = []
+
+    @property
+    def result_type(self) -> type[T]:
+        """Get result type based on category."""
+        categories = {
+            "text": TextResult,
+            "images": ImagesResult,
+            "videos": VideosResult,
+            "news": NewsResult,
+            "books": BooksResult,
+        }
+        return categories[self.category]  # type: ignore
+
+    @abstractmethod
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        raise NotImplementedError
+
+    def request(self, method: str, url: str, **kwargs: Any) -> str | None:
+        """Make a request to the search engine."""
+        try:
+            response = self.http_client.request(method, url, **kwargs)  # type: ignore
+            return response.text
+        except Exception as ex:
+            logger.error("Error in %s request: %r", self.name, ex)
+            return None
+
+    @cached_property
+    def parser(self) -> Any:
+        """Get HTML parser."""
+        if not LXML_AVAILABLE:
+            logger.warning("lxml not available, HTML parsing disabled")
+            return None
+        return LHTMLParser(remove_blank_text=True, remove_comments=True, remove_pis=True, collect_ids=False)
+
+    def extract_tree(self, html_text: str) -> Any:
+        """Extract html tree from html text."""
+        if not LXML_AVAILABLE or not self.parser:
+            raise ImportError("lxml is required for HTML parsing")
+        return html.fromstring(html_text, parser=self.parser)
+
+    def pre_process_html(self, html_text: str) -> str:
+        """Pre-process html_text before extracting results."""
+        return html_text
+
+    def extract_results(self, html_text: str) -> list[T]:
+        """Extract search results from html text."""
+        if not LXML_AVAILABLE:
+            raise ImportError("lxml is required for result extraction")
+        
+        html_text = self.pre_process_html(html_text)
+        tree = self.extract_tree(html_text)
+        
+        results = []
+        items = tree.xpath(self.items_xpath) if self.items_xpath else []
+        
+        for item in items:
+            result = self.result_type()
+            for key, xpath in self.elements_xpath.items():
+                try:
+                    data = item.xpath(xpath)
+                    if data:
+                        # Join text nodes or get first attribute
+                        value = "".join(data) if isinstance(data, list) else data
+                        setattr(result, key, value.strip() if isinstance(value, str) else value)
+                except Exception as ex:
+                    logger.debug("Error extracting %s: %r", key, ex)
+            results.append(result)
+        
+        return results
+
+    def post_extract_results(self, results: list[T]) -> list[T]:
+        """Post-process search results."""
+        return results
+
+    def search(
+        self,
+        query: str,
+        region: str = "us-en",
+        safesearch: str = "moderate",
+        timelimit: str | None = None,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> list[T] | None:
+        """Search the engine."""
+        payload = self.build_payload(
+            query=query, region=region, safesearch=safesearch, timelimit=timelimit, page=page, **kwargs
+        )
+        if self.search_method == "GET":
+            html_text = self.request(self.search_method, self.search_url, params=payload, headers=self.search_headers)
+        else:
+            html_text = self.request(self.search_method, self.search_url, data=payload, headers=self.search_headers)
+        if not html_text:
+            return None
+        results = self.extract_results(html_text)
+        return self.post_extract_results(results)
+
+
+# Legacy base class for backwards compatibility
+class BaseSearch(ABC):
+    """Base class for synchronous search engines (legacy)."""
+
+    @abstractmethod
+    def text(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Text search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def images(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Images search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def videos(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Videos search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def news(self, *args, **kwargs) -> list[dict[str, str]]:
+        """News search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def answers(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Instant answers."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def suggestions(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Suggestions."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def maps(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Maps search."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def translate(self, *args, **kwargs) -> list[dict[str, str]]:
+        """Translate."""
+        raise NotImplementedError

webscout/search/duckduckgo_main.py

@@ -0,0 +1,54 @@
+"""DuckDuckGo unified search interface."""
+
+from __future__ import annotations
+from typing import Dict, List, Optional
+from .base import BaseSearch
+from .engines.duckduckgo.text import DuckDuckGoTextSearch
+from .engines.duckduckgo.images import DuckDuckGoImages
+from .engines.duckduckgo.videos import DuckDuckGoVideos
+from .engines.duckduckgo.news import DuckDuckGoNews
+from .engines.duckduckgo.answers import DuckDuckGoAnswers
+from .engines.duckduckgo.suggestions import DuckDuckGoSuggestions
+from .engines.duckduckgo.maps import DuckDuckGoMaps
+from .engines.duckduckgo.translate import DuckDuckGoTranslate
+from .engines.duckduckgo.weather import DuckDuckGoWeather
+
+
+class DuckDuckGoSearch(BaseSearch):
+    """Unified DuckDuckGo search interface."""
+
+    def text(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, backend: str = "api", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoTextSearch()
+        return search.run(keywords, region, safesearch, timelimit, backend, max_results)
+
+    def images(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, size: Optional[str] = None, color: Optional[str] = None, type_image: Optional[str] = None, layout: Optional[str] = None, license_image: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoImages()
+        return search.run(keywords, region, safesearch, timelimit, size, color, type_image, layout, license_image, max_results)
+
+    def videos(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, resolution: Optional[str] = None, duration: Optional[str] = None, license_videos: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoVideos()
+        return search.run(keywords, region, safesearch, timelimit, resolution, duration, license_videos, max_results)
+
+    def news(self, keywords: str, region: str = "wt-wt", safesearch: str = "moderate", timelimit: Optional[str] = None, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoNews()
+        return search.run(keywords, region, safesearch, timelimit, max_results)
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        search = DuckDuckGoAnswers()
+        return search.run(keywords)
+
+    def suggestions(self, keywords: str, region: str = "wt-wt") -> List[str]:
+        search = DuckDuckGoSuggestions()
+        return search.run(keywords, region)
+
+    def maps(self, keywords: str, place: Optional[str] = None, street: Optional[str] = None, city: Optional[str] = None, county: Optional[str] = None, state: Optional[str] = None, country: Optional[str] = None, postalcode: Optional[str] = None, latitude: Optional[str] = None, longitude: Optional[str] = None, radius: int = 0, max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = DuckDuckGoMaps()
+        return search.run(keywords, place, street, city, county, state, country, postalcode, latitude, longitude, radius, max_results)
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        search = DuckDuckGoTranslate()
+        return search.run(keywords, from_lang, to_lang)
+
+    def weather(self, keywords: str) -> List[Dict[str, str]]:
+        search = DuckDuckGoWeather()
+        return search.run(keywords)

webscout/search/engines/__init__.py

@@ -0,0 +1,48 @@
+"""__init__.py for engines package - auto-discovers and registers search engines."""
+
+from __future__ import annotations
+
+import importlib
+import inspect
+import logging
+import pkgutil
+from collections import defaultdict
+from typing import Any
+
+from ..base import BaseSearchEngine
+
+logger = logging.getLogger(__name__)
+
+# ENGINES[category][name] = class
+ENGINES: dict[str, dict[str, type[BaseSearchEngine[Any]]]] = defaultdict(dict)
+
+package_name = __name__
+package = importlib.import_module(package_name)
+
+# Auto-discover all search engine classes
+for finder, modname, _ispkg in pkgutil.iter_modules(package.__path__, package_name + "."):
+    try:
+        module = importlib.import_module(modname)
+        for _, cls in inspect.getmembers(module, inspect.isclass):
+            # Must subclass BaseSearchEngine (but not the base itself)
+            if not issubclass(cls, BaseSearchEngine) or cls is BaseSearchEngine:
+                continue
+
+            # Skip any class whose name starts with "Base"
+            if cls.__name__.startswith("Base"):
+                continue
+
+            # Skip disabled engines
+            if getattr(cls, "disabled", False):
+                logger.info("Skipping disabled engine: %s", cls.name)
+                continue
+
+            # Register the engine
+            if hasattr(cls, "name") and hasattr(cls, "category"):
+                ENGINES[cls.category][cls.name] = cls
+                logger.debug("Registered engine: %s (%s)", cls.name, cls.category)
+    except Exception as ex:
+        logger.warning("Failed to import module %s: %r", modname, ex)
+
+
+__all__ = ["ENGINES"]

webscout/search/engines/bing.py

@@ -0,0 +1,84 @@
+"""Bing search engine implementation."""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Mapping
+from time import time
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+def unwrap_bing_url(raw_url: str) -> str | None:
+    """Decode the Bing-wrapped raw_url to extract the original url."""
+    try:
+        parsed = urlparse(raw_url)
+        if parsed.netloc == "www.bing.com" and parsed.path == "/ck/a":
+            query_params = parse_qs(parsed.query)
+            if "u" in query_params:
+                encoded_url = query_params["u"][0]
+                # Decode the base64-like encoding
+                if encoded_url.startswith("a1"):
+                    encoded_url = encoded_url[2:]
+                # Add padding if needed
+                padding = len(encoded_url) % 4
+                if padding:
+                    encoded_url += "=" * (4 - padding)
+                try:
+                    decoded = base64.urlsafe_b64decode(encoded_url).decode()
+                    return decoded
+                except Exception:
+                    pass
+        return raw_url
+    except Exception:
+        return raw_url
+
+
+class Bing(BaseSearchEngine[TextResult]):
+    """Bing search engine."""
+
+    name = "bing"
+    category = "text"
+    provider = "bing"
+
+    search_url = "https://www.bing.com/search"
+    search_method = "GET"
+
+    items_xpath = "//li[contains(@class, 'b_algo')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2/a//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//p//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the Bing search request."""
+        country, lang = region.lower().split("-")
+        payload = {"q": query, "pq": query, "cc": lang}
+        cookies = {
+            "_EDGE_CD": f"m={lang}-{country}&u={lang}-{country}",
+            "_EDGE_S": f"mkt={lang}-{country}&ui={lang}-{country}",
+        }
+        self.http_client.set_cookies("https://www.bing.com", cookies)
+        if timelimit:
+            d = int(time() // 86400)
+            payload["filters"] = {
+                "d": f"ex1:\"ez1_{d - 1}_{d}\"",
+                "w": f"ex1:\"ez1_{d - 7}_{d}\"",
+                "m": f"ex1:\"ez1_{d - 30}_{d}\"",
+                "y": f"ex1:\"ez1_{d - 365}_{d}\"",
+            }[timelimit]
+        if page > 1:
+            payload["first"] = f"{(page - 1) * 10 + 1}"
+        return payload
+
+    def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
+        """Post-process search results."""
+        for result in results:
+            result.href = unwrap_bing_url(result.href)
+        return results

webscout/search/engines/bing_news.py

@@ -0,0 +1,52 @@
+"""Bing news search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import NewsResult
+
+
+class BingNews(BaseSearchEngine[NewsResult]):
+    """Bing news engine."""
+
+    name = "bing"
+    category = "news"
+    provider = "bing"
+
+    search_url = "https://www.bing.com/news/infinitescrollajax"
+    search_method = "GET"
+
+    items_xpath = "//div[contains(@class, 'newsitem')]"
+    elements_xpath: Mapping[str, str] = {
+        "date": ".//span[@aria-label]//@aria-label",
+        "title": "@data-title",
+        "body": ".//div[@class='snippet']//text()",
+        "url": "@url",
+        "image": ".//a[contains(@class, 'image')]//@src",
+        "source": "@data-author",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the Bing search request."""
+        country, lang = region.lower().split("-")
+        payload = {
+            "q": query,
+            "InfiniteScroll": "1",
+            "first": f"{page * 10 + 1}",
+            "SFX": f"{page}",
+            "cc": country,
+            "setlang": lang,
+        }
+        if timelimit:
+            payload["qft"] = {
+                "d": 'interval="4"',  # doesn't exist so it's the same as one hour
+                "w": 'interval="7"',
+                "m": 'interval="9"',
+                "y": 'interval="9"',  # doesn't exist so it's the same as month
+            }[timelimit]
+        return payload

webscout/search/engines/brave.py

@@ -0,0 +1,43 @@
+"""Brave search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+class Brave(BaseSearchEngine[TextResult]):
+    """Brave search engine."""
+
+    name = "brave"
+    category = "text"
+    provider = "brave"
+
+    search_url = "https://search.brave.com/search"
+    search_method = "GET"
+
+    items_xpath = "//div[@data-type='web']"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//div[(contains(@class,'title') or contains(@class,'sitename-container')) and position()=last()]//text()",
+        "href": "./a/@href",
+        "body": ".//div[contains(@class, 'description')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "strict", "moderate": "moderate", "off": "off"}
+        payload = {
+            "q": query,
+            "source": "web",
+            "safesearch": safesearch_base[safesearch.lower()],
+        }
+        if timelimit:
+            payload["tf"] = timelimit
+        if page > 1:
+            payload["offset"] = f"{(page - 1) * 10}"
+        return payload

webscout/search/engines/duckduckgo/__init__.py

@@ -0,0 +1,25 @@
+"""DuckDuckGo search engines package."""
+
+from .answers import DuckDuckGoAnswers
+from .base import DuckDuckGoBase
+from .images import DuckDuckGoImages
+from .maps import DuckDuckGoMaps
+from .news import DuckDuckGoNews
+from .suggestions import DuckDuckGoSuggestions
+from .text import DuckDuckGoTextSearch
+from .translate import DuckDuckGoTranslate
+from .videos import DuckDuckGoVideos
+from .weather import DuckDuckGoWeather
+
+__all__ = [
+    "DuckDuckGoBase",
+    "DuckDuckGoTextSearch",
+    "DuckDuckGoImages",
+    "DuckDuckGoVideos",
+    "DuckDuckGoNews",
+    "DuckDuckGoAnswers",
+    "DuckDuckGoSuggestions",
+    "DuckDuckGoMaps",
+    "DuckDuckGoTranslate",
+    "DuckDuckGoWeather",
+]