webscout 8.3.7__py3-none-any.whl → 2025.10.13__py3-none-any.whl

webscout/search/engines/mojeek.py

@@ -0,0 +1,37 @@
+"""Mojeek search engine implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+class Mojeek(BaseSearchEngine[TextResult]):
+    """Mojeek search engine."""
+
+    name = "mojeek"
+    category = "text"
+    provider = "mojeek"
+
+    search_url = "https://www.mojeek.com/search"
+    search_method = "GET"
+
+    items_xpath = "//ul[contains(@class, 'results')]/li"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//p[@class='s']//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "1", "moderate": "0", "off": "0"}
+        payload = {"q": query, "safe": safesearch_base[safesearch.lower()]}
+        if page > 1:
+            payload["s"] = f"{(page - 1) * 10 + 1}"
+        return payload

webscout/search/engines/wikipedia.py

@@ -0,0 +1,56 @@
+"""Wikipedia text search engine."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+from urllib.parse import quote
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+from ...utils import json_loads
+
+logger = logging.getLogger(__name__)
+
+
+class Wikipedia(BaseSearchEngine[TextResult]):
+    """Wikipedia text search engine."""
+
+    name = "wikipedia"
+    category = "text"
+    provider = "wikipedia"
+    priority = 2
+
+    search_url = "https://{lang}.wikipedia.org/w/api.php?action=opensearch&search={query}"
+    search_method = "GET"
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        _country, lang = region.lower().split("-")
+        encoded_query = quote(query)
+        self.search_url = (
+            f"https://{lang}.wikipedia.org/w/api.php?action=opensearch&profile=fuzzy&limit=1&search={encoded_query}"
+        )
+        payload: dict[str, Any] = {}
+        self.lang = lang  # used in extract_results
+        return payload
+
+    def extract_results(self, html_text: str) -> list[TextResult]:
+        """Extract search results from html text."""
+        json_data = json_loads(html_text)
+        if not json_data or len(json_data) < 4:
+            return []
+        
+        results = []
+        titles, descriptions, urls = json_data[1], json_data[2], json_data[3]
+        
+        for title, description, url in zip(titles, descriptions, urls):
+            result = TextResult()
+            result.title = title
+            result.body = description
+            result.href = url
+            results.append(result)
+        
+        return results

webscout/search/engines/yahoo.py

@@ -0,0 +1,65 @@
+"""Yahoo search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from secrets import token_urlsafe
+from typing import Any
+from urllib.parse import unquote_plus
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+
+def extract_url(u: str) -> str:
+    """Sanitize url."""
+    if "/RU=" in u:
+        start = u.find("/RU=") + 4
+        end = u.find("/RK=", start)
+        if end == -1:
+            end = len(u)
+        return unquote_plus(u[start:end])
+    return u
+
+
+class Yahoo(BaseSearchEngine[TextResult]):
+    """Yahoo search engine."""
+
+    name = "yahoo"
+    category = "text"
+    provider = "bing"
+
+    search_url = "https://search.yahoo.com/search"
+    search_method = "GET"
+
+    items_xpath = "//div[contains(@class, 'relsrch')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//div[contains(@class, 'Title')]//h3//text()",
+        "href": ".//div[contains(@class, 'Title')]//a/@href",
+        "body": ".//div[contains(@class, 'Text')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        self.search_url = (
+            f"https://search.yahoo.com/search;_ylt={token_urlsafe(24 * 3 // 4)};_ylu={token_urlsafe(47 * 3 // 4)}"
+        )
+        payload = {"p": query}
+        if page > 1:
+            payload["b"] = f"{(page - 1) * 7 + 1}"
+        if timelimit:
+            payload["btf"] = timelimit
+        return payload
+
+    def post_extract_results(self, results: list[TextResult]) -> list[TextResult]:
+        """Post-process search results."""
+        post_results = []
+        for result in results:
+            if result.href.startswith("https://www.bing.com/aclick?"):
+                continue
+            if "/RU=" in result.href:
+                result.href = extract_url(result.href)
+            post_results.append(result)
+        return post_results

webscout/search/engines/yahoo_news.py

@@ -0,0 +1,64 @@
+"""Yahoo news search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from secrets import token_urlsafe
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import NewsResult
+
+
+def extract_image(u: str) -> str:
+    """Sanitize image url."""
+    if u and u.startswith("data:image"):
+        return ""
+    return u
+
+
+def extract_source(s: str) -> str:
+    """Remove ' via Yahoo' from string."""
+    return s.replace(" via Yahoo", "") if s else s
+
+
+class YahooNews(BaseSearchEngine[NewsResult]):
+    """Yahoo news search engine."""
+
+    name = "yahoo"
+    category = "news"
+    provider = "bing"
+
+    search_url = "https://news.search.yahoo.com/search"
+    search_method = "GET"
+
+    items_xpath = "//div[contains(@class, 'NewsArticle')]"
+    elements_xpath: Mapping[str, str] = {
+        "date": ".//span[contains(@class, 'fc-2nd')]//text()",
+        "title": ".//h4//a//text()",
+        "url": ".//h4//a/@href",
+        "body": ".//p//text()",
+        "image": ".//img/@src",
+        "source": ".//span[contains(@class, 's-source')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        self.search_url = (
+            f"https://news.search.yahoo.com/search;_ylt={token_urlsafe(24 * 3 // 4)};_ylu={token_urlsafe(47 * 3 // 4)}"
+        )
+        payload = {"p": query}
+        if page > 1:
+            payload["b"] = f"{(page - 1) * 10 + 1}"
+        if timelimit:
+            payload["btf"] = timelimit
+        return payload
+
+    def post_extract_results(self, results: list[NewsResult]) -> list[NewsResult]:
+        """Post-process search results."""
+        for result in results:
+            result.image = extract_image(result.image)
+            result.source = extract_source(result.source)
+        return results

webscout/search/engines/yandex.py

@@ -0,0 +1,43 @@
+"""Yandex search engine."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from random import SystemRandom
+from typing import Any
+
+from ..base import BaseSearchEngine
+from ..results import TextResult
+
+random = SystemRandom()
+
+
+class Yandex(BaseSearchEngine[TextResult]):
+    """Yandex search engine."""
+
+    name = "yandex"
+    category = "text"
+    provider = "yandex"
+
+    search_url = "https://yandex.com/search/"
+    search_method = "GET"
+
+    items_xpath = "//li[contains(@class, 'serp-item')]"
+    elements_xpath: Mapping[str, str] = {
+        "title": ".//h2//text()",
+        "href": ".//h2/a/@href",
+        "body": ".//div[contains(@class, 'text-container')]//text()",
+    }
+
+    def build_payload(
+        self, query: str, region: str, safesearch: str, timelimit: str | None, page: int = 1, **kwargs: Any
+    ) -> dict[str, Any]:
+        """Build a payload for the search request."""
+        safesearch_base = {"on": "1", "moderate": "0", "off": "0"}
+        payload = {
+            "text": query,
+            "family": safesearch_base[safesearch.lower()],
+        }
+        if page > 1:
+            payload["p"] = str(page - 1)
+        return payload

webscout/search/engines/yep/__init__.py

@@ -0,0 +1,13 @@
+"""Yep search engines package."""
+
+from .base import YepBase
+from .images import YepImages
+from .suggestions import YepSuggestions
+from .text import YepSearch as YepTextSearch
+
+__all__ = [
+    "YepBase",
+    "YepTextSearch",
+    "YepImages",
+    "YepSuggestions",
+]

webscout/search/engines/yep/base.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from ....litagent import LitAgent
+from curl_cffi.requests import Session
+
+
+class YepBase:
+    """Base class for Yep search engines."""
+
+    def __init__(
+        self,
+        timeout: int = 20,
+        proxies: dict[str, str] | None = None,
+        verify: bool = True,
+        impersonate: str = "chrome110",
+    ):
+        self.base_url = "https://api.yep.com/fs/2/search"
+        self.timeout = timeout
+        self.session = Session(
+            proxies=proxies,
+            verify=verify,
+            impersonate=impersonate,
+            timeout=timeout,
+        )
+        self.session.headers.update(
+            {
+                **LitAgent().generate_fingerprint(),
+                "Origin": "https://yep.com",
+                "Referer": "https://yep.com/",
+            }
+        )
+

webscout/search/engines/yep/images.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepImages(YepBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        safe_search_map = {
+            "on": "on",
+            "moderate": "moderate",
+            "off": "off"
+        }
+        safe_setting = safe_search_map.get(safesearch.lower(), "moderate")
+
+        params = {
+            "client": "web",
+            "gl": region,
+            "limit": str(max_results) if max_results else "10",
+            "no_correct": "false",
+            "q": keywords,
+            "safeSearch": safe_setting,
+            "type": "images"
+        }
+
+        url = f"{self.base_url}?{urlencode(params)}"
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            raw_results = response.json()
+
+            if not raw_results or len(raw_results) < 2:
+                return []
+
+            formatted_results = []
+            results = raw_results[1].get('results', [])
+
+            for result in results:
+                if result.get("type") != "Image":
+                    continue
+
+                formatted_result = {
+                    "title": self._remove_html_tags(result.get("title", "")),
+                    "image": result.get("image_id", ""),
+                    "thumbnail": result.get("src", ""),
+                    "url": result.get("host_page", ""),
+                    "height": result.get("height", 0),
+                    "width": result.get("width", 0),
+                    "source": result.get("visual_url", "")
+                }
+
+                if "srcset" in result:
+                    formatted_result["thumbnail_hd"] = result["srcset"].split(",")[1].strip().split(" ")[0]
+
+                formatted_results.append(formatted_result)
+
+            if max_results:
+                return formatted_results[:max_results]
+            return formatted_results
+
+        except Exception as e:
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep image search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep image search failed: {str(e)}")
+
+    def _remove_html_tags(self, text: str) -> str:
+        result = ""
+        in_tag = False
+
+        for char in text:
+            if char == '<':
+                in_tag = True
+            elif char == '>':
+                in_tag = False
+            elif not in_tag:
+                result += char
+
+        replacements = {
+            '&nbsp;': ' ',
+            '&amp;': '&',
+            '&lt;': '<',
+            '&gt;': '>',
+            '&quot;': '"',
+            '&apos;': "'",
+        }
+
+        for entity, replacement in replacements.items():
+            result = result.replace(entity, replacement)
+
+        return result.strip()
+

webscout/search/engines/yep/suggestions.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import List
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepSuggestions(YepBase):
+    def run(self, *args, **kwargs) -> List[str]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+
+        params = {
+            "query": keywords,
+            "type": "web",
+            "gl": region
+        }
+
+        url = f"https://api.yep.com/ac/?{urlencode(params)}"
+
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            data = response.json()
+            if isinstance(data, list) and len(data) > 1 and isinstance(data[1], list):
+                return data[1]
+            return []
+
+        except Exception as e:
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep suggestions failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep suggestions failed: {str(e)}")
+

webscout/search/engines/yep/text.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+from urllib.parse import urlencode
+
+from .base import YepBase
+
+
+class YepSearch(YepBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "all")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results")
+
+        safe_search_map = {
+            "on": "on",
+            "moderate": "moderate",
+            "off": "off"
+        }
+        safe_setting = safe_search_map.get(safesearch.lower(), "moderate")
+
+        params = {
+            "client": "web",
+            "gl": region,
+            "limit": str(max_results) if max_results else "10",
+            "no_correct": "false",
+            "q": keywords,
+            "safeSearch": safe_setting,
+            "type": "web"
+        }
+
+        url = f"{self.base_url}?{urlencode(params)}"
+        try:
+            response = self.session.get(url)
+            response.raise_for_status()
+            raw_results = response.json()
+
+            formatted_results = self.format_results(raw_results)
+
+            if max_results:
+                return formatted_results[:max_results]
+            return formatted_results
+        except Exception as e:
+            if hasattr(e, 'response') and e.response is not None:
+                 raise Exception(f"Yep search failed with status {e.response.status_code}: {str(e)}")
+            else:
+                 raise Exception(f"Yep search failed: {str(e)}")
+
+    def format_results(self, raw_results: dict) -> List[Dict]:
+        formatted_results = []
+
+        if not raw_results or len(raw_results) < 2:
+            return formatted_results
+
+        results = raw_results[1].get('results', [])
+
+        for result in results:
+            formatted_result = {
+                "title": self._remove_html_tags(result.get("title", "")),
+                "href": result.get("url", ""),
+                "body": self._remove_html_tags(result.get("snippet", "")),
+                "source": result.get("visual_url", ""),
+                "position": len(formatted_results) + 1,
+                "type": result.get("type", "organic"),
+                "first_seen": result.get("first_seen", None)
+            }
+
+            if "sitelinks" in result:
+                sitelinks = []
+                if "full" in result["sitelinks"]:
+                    sitelinks.extend(result["sitelinks"]["full"])
+                if "short" in result["sitelinks"]:
+                    sitelinks.extend(result["sitelinks"]["short"])
+
+                if sitelinks:
+                    formatted_result["sitelinks"] = [
+                        {
+                            "title": self._remove_html_tags(link.get("title", "")),
+                            "href": link.get("url", "")
+                        }
+                        for link in sitelinks
+                    ]
+
+            formatted_results.append(formatted_result)
+
+        return formatted_results
+
+    def _remove_html_tags(self, text: str) -> str:
+        result = ""
+        in_tag = False
+
+        for char in text:
+            if char == '<':
+                in_tag = True
+            elif char == '>':
+                in_tag = False
+            elif not in_tag:
+                result += char
+
+        replacements = {
+            '&nbsp;': ' ',
+            '&amp;': '&',
+            '&lt;': '<',
+            '&gt;': '>',
+            '&quot;': '"',
+            '&apos;': "'",
+        }
+
+        for entity, replacement in replacements.items():
+            result = result.replace(entity, replacement)
+
+        return result.strip()
+