thordata-sdk 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

thordata/types/common.py

@@ -0,0 +1,141 @@
+"""
+Common types shared across different modules.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class ThordataBaseConfig:
+    """Base class for all config objects with payload conversion."""
+
+    def to_payload(self) -> dict[str, Any]:
+        raise NotImplementedError
+
+
+class Device(str, Enum):
+    DESKTOP = "desktop"
+    MOBILE = "mobile"
+    TABLET = "tablet"
+
+
+class OutputFormat(str, Enum):
+    HTML = "html"
+    PNG = "png"
+
+
+@dataclass
+class CommonSettings:
+    """
+    Common settings for video/audio downloads.
+    Keys strictly aligned with Thordata Video Builder API.
+    """
+
+    resolution: str | None = None
+    video_codec: str | None = None  # vp9, avc
+    audio_format: str | None = None  # opus, mp3
+    bitrate: str | None = None
+    selected_only: str | bool | None = None
+    is_subtitles: str | bool | None = None
+    subtitles_language: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        result = {}
+        for k, v in self.__dict__.items():
+            if v is not None:
+                # API expects explicit string "true"/"false" for booleans
+                if isinstance(v, bool):
+                    result[k] = "true" if v else "false"
+                else:
+                    result[k] = str(v)
+        return result
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict())
+
+
+def normalize_enum_value(value: object, enum_class: type) -> str:
+    """
+    Safely convert an enum or string to its string value.
+    """
+    if isinstance(value, enum_class):
+        return str(getattr(value, "value", value)).lower()
+    if isinstance(value, str):
+        return value.lower()
+    raise TypeError(
+        f"Expected {enum_class.__name__} or str, got {type(value).__name__}"
+    )
+
+
+# --- Geography Enums ---
+
+
+class Continent(str, Enum):
+    AFRICA = "af"
+    ANTARCTICA = "an"
+    ASIA = "as"
+    EUROPE = "eu"
+    NORTH_AMERICA = "na"
+    OCEANIA = "oc"
+    SOUTH_AMERICA = "sa"
+
+
+class Country(str, Enum):
+    US = "us"
+    CA = "ca"
+    MX = "mx"
+    GB = "gb"
+    DE = "de"
+    FR = "fr"
+    ES = "es"
+    IT = "it"
+    NL = "nl"
+    PL = "pl"
+    RU = "ru"
+    UA = "ua"
+    SE = "se"
+    NO = "no"
+    DK = "dk"
+    FI = "fi"
+    CH = "ch"
+    AT = "at"
+    BE = "be"
+    PT = "pt"
+    IE = "ie"
+    CZ = "cz"
+    GR = "gr"
+    CN = "cn"
+    JP = "jp"
+    KR = "kr"
+    IN = "in"
+    AU = "au"
+    NZ = "nz"
+    SG = "sg"
+    HK = "hk"
+    TW = "tw"
+    TH = "th"
+    VN = "vn"
+    ID = "id"
+    MY = "my"
+    PH = "ph"
+    PK = "pk"
+    BD = "bd"
+    BR = "br"
+    AR = "ar"
+    CL = "cl"
+    CO = "co"
+    PE = "pe"
+    VE = "ve"
+    AE = "ae"
+    SA = "sa"
+    IL = "il"
+    TR = "tr"
+    ZA = "za"
+    EG = "eg"
+    NG = "ng"
+    KE = "ke"
+    MA = "ma"

thordata/types/proxy.py

@@ -0,0 +1,340 @@
+"""
+Proxy related types and configurations.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from dataclasses import dataclass
+from enum import Enum, IntEnum
+from typing import Any
+from urllib.parse import quote
+
+# Import geography from common to avoid circular issues
+from .common import Continent
+
+
+class ProxyProduct(str, Enum):
+    RESIDENTIAL = "residential"
+    MOBILE = "mobile"
+    DATACENTER = "datacenter"
+    ISP = "isp"
+
+    @property
+    def default_port(self) -> int:
+        ports = {
+            "residential": 9999,
+            "mobile": 5555,
+            "datacenter": 7777,
+            "isp": 6666,
+        }
+        return ports.get(self.value, 9999)
+
+
+class ProxyType(IntEnum):
+    RESIDENTIAL = 1
+    UNLIMITED = 2
+    DATACENTER = 3
+    ISP = 4
+    MOBILE = 5
+
+
+class SessionType(str, Enum):
+    ROTATING = "rotating"
+    STICKY = "sticky"
+
+
+class ProxyHost(str, Enum):
+    DEFAULT = "pr.thordata.net"
+    NORTH_AMERICA = "t.na.thordata.net"
+    EUROPE = "t.eu.thordata.net"
+
+
+class ProxyPort(IntEnum):
+    RESIDENTIAL = 9999
+    MOBILE = 5555
+    DATACENTER = 7777
+    ISP = 6666
+
+
+@dataclass
+class ProxyConfig:
+    username: str
+    password: str
+    product: ProxyProduct | str = ProxyProduct.RESIDENTIAL
+    host: str | None = None
+    port: int | None = None
+    protocol: str = "https"
+
+    # Geo-targeting
+    continent: str | None = None
+    country: str | None = None
+    state: str | None = None
+    city: str | None = None
+    asn: str | None = None
+
+    # Session control
+    session_id: str | None = None
+    session_duration: int | None = None  # minutes, 1-90
+
+    # Use a set of values for validation logic
+    _VALID_CONTINENTS = {v.value for v in Continent}
+
+    def __post_init__(self) -> None:
+        if isinstance(self.product, str):
+            self.product = ProxyProduct(self.product.lower())
+
+        if self.host is None:
+            host_map = {
+                ProxyProduct.RESIDENTIAL: "pr.thordata.net",
+                ProxyProduct.DATACENTER: "dc.pr.thordata.net",
+                ProxyProduct.MOBILE: "m.pr.thordata.net",
+                ProxyProduct.ISP: "isp.pr.thordata.net",
+            }
+            self.host = host_map.get(self.product, "pr.thordata.net")
+
+        if self.port is None:
+            self.port = self.product.default_port
+
+        self._validate()
+
+    def _validate(self) -> None:
+        if self.protocol not in ("http", "https", "socks5", "socks5h"):
+            raise ValueError(f"Invalid protocol: {self.protocol}")
+
+        if self.session_duration is not None:
+            if not 1 <= self.session_duration <= 90:
+                raise ValueError("session_duration must be between 1 and 90 minutes")
+            if not self.session_id:
+                raise ValueError("session_duration requires session_id")
+
+        if self.asn and not self.country:
+            raise ValueError("ASN targeting requires country")
+
+        if self.continent and self.continent.lower() not in self._VALID_CONTINENTS:
+            raise ValueError(f"Invalid continent code: {self.continent}")
+
+        if self.country and not re.match(r"^[a-zA-Z]{2}$", self.country):
+            raise ValueError("Invalid country code")
+
+    def build_username(self) -> str:
+        base = self.username
+        if not base.startswith("td-customer-"):
+            base = f"td-customer-{base}"
+
+        parts = [base]
+
+        if self.continent:
+            parts.append(f"continent-{self.continent.lower()}")
+        if self.country:
+            parts.append(f"country-{self.country.lower()}")
+        if self.state:
+            parts.append(f"state-{self.state.lower()}")
+        if self.city:
+            parts.append(f"city-{self.city.lower()}")
+        if self.asn:
+            asn_val = (
+                self.asn.upper()
+                if self.asn.upper().startswith("AS")
+                else f"AS{self.asn.upper()}"
+            )
+            parts.append(f"asn-{asn_val}")
+        if self.session_id:
+            parts.append(f"sessid-{self.session_id}")
+        if self.session_duration:
+            parts.append(f"sesstime-{self.session_duration}")
+
+        return "-".join(parts)
+
+    def build_proxy_url(self) -> str:
+        user = self.build_username()
+        proto = "socks5h" if self.protocol == "socks5" else self.protocol
+
+        safe_user = quote(user, safe="")
+        safe_pass = quote(self.password, safe="")
+
+        return f"{proto}://{safe_user}:{safe_pass}@{self.host}:{self.port}"
+
+    def build_proxy_endpoint(self) -> str:
+        proto = "socks5h" if self.protocol == "socks5" else self.protocol
+        return f"{proto}://{self.host}:{self.port}"
+
+    def build_proxy_basic_auth(self) -> str:
+        return f"{self.build_username()}:{self.password}"
+
+    def to_proxies_dict(self) -> dict[str, str]:
+        url = self.build_proxy_url()
+        return {"http": url, "https": url}
+
+    def to_aiohttp_config(self) -> tuple:
+        try:
+            import aiohttp
+
+            return (
+                f"{self.protocol}://{self.host}:{self.port}",
+                aiohttp.BasicAuth(login=self.build_username(), password=self.password),
+            )
+        except ImportError as e:
+            # Fix B904: chain the exception
+            raise ImportError("aiohttp required") from e
+
+
+@dataclass
+class StaticISPProxy:
+    host: str
+    username: str
+    password: str
+    port: int = 6666
+    protocol: str = "https"
+
+    def __post_init__(self) -> None:
+        if self.protocol not in ("http", "https", "socks5", "socks5h"):
+            raise ValueError(f"Invalid protocol: {self.protocol}")
+
+    def build_username(self) -> str:
+        # Static ISP usually doesn't use the 'td-customer-' prefix logic
+        # or special params, it uses raw username.
+        return self.username
+
+    def build_proxy_endpoint(self) -> str:
+        # FIX: Added this method to satisfy client.py interface
+        proto = "socks5h" if self.protocol == "socks5" else self.protocol
+        return f"{proto}://{self.host}:{self.port}"
+
+    def build_proxy_basic_auth(self) -> str:
+        # FIX: Added this method to satisfy client.py interface
+        return f"{self.username}:{self.password}"
+
+    def build_proxy_url(self) -> str:
+        proto = "socks5h" if self.protocol == "socks5" else self.protocol
+        safe_user = quote(self.username, safe="")
+        safe_pass = quote(self.password, safe="")
+        return f"{proto}://{safe_user}:{safe_pass}@{self.host}:{self.port}"
+
+    def to_proxies_dict(self) -> dict[str, str]:
+        url = self.build_proxy_url()
+        return {"http": url, "https": url}
+
+    def to_aiohttp_config(self) -> tuple:
+        try:
+            import aiohttp
+
+            return (
+                f"{self.protocol}://{self.host}:{self.port}",
+                aiohttp.BasicAuth(login=self.username, password=self.password),
+            )
+        except ImportError as e:
+            # Fix B904: chain the exception
+            raise ImportError("aiohttp required") from e
+
+    @classmethod
+    def from_env(cls) -> StaticISPProxy:
+        import os
+
+        host = os.getenv("THORDATA_ISP_HOST")
+        username = os.getenv("THORDATA_ISP_USERNAME")
+        password = os.getenv("THORDATA_ISP_PASSWORD")
+        if not all([host, username, password]):
+            raise ValueError(
+                "THORDATA_ISP_HOST, THORDATA_ISP_USERNAME, and THORDATA_ISP_PASSWORD are required"
+            )
+        return cls(host=host, username=username, password=password)  # type: ignore
+
+
+@dataclass
+class StickySession(ProxyConfig):
+    duration_minutes: int = 10
+    auto_session_id: bool = True
+
+    def __post_init__(self) -> None:
+        if self.auto_session_id and not self.session_id:
+            self.session_id = uuid.uuid4().hex[:12]
+        self.session_duration = self.duration_minutes
+        super().__post_init__()
+
+
+@dataclass
+class ProxyUser:
+    username: str
+    password: str
+    status: bool
+    traffic_limit: int
+    usage_traffic: float
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProxyUser:
+        return cls(
+            username=str(data.get("username", "")),
+            password=str(data.get("password", "")),
+            status=str(data.get("status")).lower() in ("true", "1"),
+            traffic_limit=int(data.get("traffic_limit", 0)),
+            usage_traffic=float(data.get("usage_traffic", 0)),
+        )
+
+    def usage_gb(self) -> float:
+        return self.usage_traffic / (1024 * 1024)
+
+    def limit_gb(self) -> float:
+        if self.traffic_limit == 0:
+            return 0.0
+        return self.traffic_limit / 1024.0
+
+
+@dataclass
+class ProxyUserList:
+    limit: float
+    remaining_limit: float
+    user_count: int
+    users: list[ProxyUser]
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProxyUserList:
+        user_list_raw = data.get("list")
+        if user_list_raw is None:
+            possible_data = data.get("data")
+            user_list_raw = possible_data if isinstance(possible_data, list) else []
+        if not isinstance(user_list_raw, list):
+            user_list_raw = []
+
+        users = [ProxyUser.from_dict(u) for u in user_list_raw]
+
+        return cls(
+            limit=float(data.get("limit", 0)),
+            remaining_limit=float(data.get("remaining_limit", 0)),
+            user_count=int(data.get("user_count", len(users))),
+            users=users,
+        )
+
+
+@dataclass
+class ProxyServer:
+    ip: str
+    port: int
+    username: str
+    password: str
+    expiration_time: int | str | None = None
+    region: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProxyServer:
+        return cls(
+            ip=str(data.get("ip", "")),
+            port=int(data.get("port", 0)),
+            username=str(data.get("username", data.get("user", ""))),
+            password=str(data.get("password", data.get("pwd", ""))),
+            expiration_time=data.get("expiration_time", data.get("expireTime")),
+            region=str(data.get("region")) if data.get("region") else None,
+        )
+
+    def to_proxy_url(self, protocol: str = "https") -> str:
+        return f"{protocol}://{self.username}:{self.password}@{self.ip}:{self.port}"
+
+    def is_expired(self) -> bool:
+        if self.expiration_time is None:
+            return False
+        import time
+
+        if isinstance(self.expiration_time, int):
+            return time.time() > self.expiration_time
+        return False

thordata/types/serp.py

@@ -0,0 +1,224 @@
+"""
+SERP (Search Engine Results Page) related types and configurations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from .common import ThordataBaseConfig
+
+
+class Engine(str, Enum):
+    # Google
+    GOOGLE = "google"
+    GOOGLE_NEWS = "google_news"
+    GOOGLE_SHOPPING = "google_shopping"
+    GOOGLE_VIDEOS = "google_videos"
+    GOOGLE_IMAGES = "google_images"
+    GOOGLE_MAPS = "google_maps"
+    GOOGLE_JOBS = "google_jobs"
+    GOOGLE_PLAY = "google_play"
+    GOOGLE_TRENDS = "google_trends"
+    GOOGLE_SCHOLAR = "google_scholar"
+    GOOGLE_PATENTS = "google_patents"
+    GOOGLE_FINANCE = "google_finance"
+    GOOGLE_FLIGHTS = "google_flights"
+    GOOGLE_LENS = "google_lens"
+    GOOGLE_HOTELS = "google_hotels"
+
+    # Bing
+    BING = "bing"
+    BING_NEWS = "bing_news"
+    BING_SHOPPING = "bing_shopping"
+    BING_IMAGES = "bing_images"
+    BING_VIDEOS = "bing_videos"
+    BING_MAPS = "bing_maps"
+
+    # Others
+    YANDEX = "yandex"
+    DUCKDUCKGO = "duckduckgo"
+    BAIDU = "baidu"
+
+    # Legacy / Compatibility Aliases
+    GOOGLE_SEARCH = "google_search"
+    GOOGLE_WEB = "google_web"
+    GOOGLE_LOCAL = "google_local"
+    GOOGLE_PRODUCT = (
+        "google_product"  # mapped to shopping with product_id internally usually
+    )
+
+
+class GoogleSearchType(str, Enum):
+    SEARCH = "search"
+    NEWS = "news"
+    SHOPPING = "shopping"
+    IMAGES = "images"
+    VIDEOS = "videos"
+    MAPS = "maps"
+    # Add others as needed
+
+
+class BingSearchType(str, Enum):
+    SEARCH = "search"
+    NEWS = "news"
+    SHOPPING = "shopping"
+    IMAGES = "images"
+    VIDEOS = "videos"
+    MAPS = "maps"
+
+
+class GoogleTbm(str, Enum):
+    NEWS = "nws"
+    SHOPPING = "shop"
+    IMAGES = "isch"
+    VIDEOS = "vid"
+
+
+class TimeRange(str, Enum):
+    HOUR = "hour"
+    DAY = "day"
+    WEEK = "week"
+    MONTH = "month"
+    YEAR = "year"
+
+
+@dataclass
+class SerpRequest(ThordataBaseConfig):
+    query: str
+    engine: str = "google"
+    num: int = 10
+    start: int = 0
+
+    # Localization
+    country: str | None = None  # 'gl'
+    language: str | None = None  # 'hl'
+    google_domain: str | None = None
+    countries_filter: str | None = None  # 'cr'
+    languages_filter: str | None = None  # 'lr'
+
+    # Geo-targeting
+    location: str | None = None
+    uule: str | None = None
+
+    # Search type
+    search_type: str | None = None  # 'tbm'
+
+    # Filters
+    safe_search: bool | None = None
+    time_filter: str | None = None  # 'tbs'
+    no_autocorrect: bool = False  # 'nfpr'
+    filter_duplicates: bool | None = None  # 'filter'
+
+    # Device & Rendering
+    device: str | None = None
+    render_js: bool | None = None
+    no_cache: bool | None = None
+
+    # Output
+    output_format: str = "json"
+
+    # Advanced Google
+    ludocid: str | None = None
+    kgmid: str | None = None
+
+    # Pass-through for any other param
+    extra_params: dict[str, Any] = field(default_factory=dict)
+
+    # Mappings
+    SEARCH_TYPE_MAP = {
+        "images": "isch",
+        "shopping": "shop",
+        "news": "nws",
+        "videos": "vid",
+        "isch": "isch",
+        "shop": "shop",
+        "nws": "nws",
+        "vid": "vid",
+    }
+
+    TIME_FILTER_MAP = {
+        "hour": "qdr:h",
+        "day": "qdr:d",
+        "week": "qdr:w",
+        "month": "qdr:m",
+        "year": "qdr:y",
+    }
+
+    def to_payload(self) -> dict[str, Any]:
+        engine = self.engine.lower()
+        payload: dict[str, Any] = {
+            "engine": engine,
+            "num": str(self.num),
+        }
+
+        # JSON output handling
+        fmt = self.output_format.lower()
+        if fmt == "json":
+            payload["json"] = "1"
+        elif fmt == "html":
+            pass  # No json param means HTML
+        elif fmt in ("2", "both", "json+html"):
+            payload["json"] = "2"
+
+        # Query param handling
+        if engine == "yandex":
+            payload["text"] = self.query
+        else:
+            payload["q"] = self.query
+
+        # Basic fields
+        if self.google_domain:
+            payload["google_domain"] = self.google_domain
+        if self.start > 0:
+            payload["start"] = str(self.start)
+        if self.country:
+            payload["gl"] = self.country.lower()
+        if self.language:
+            payload["hl"] = self.language.lower()
+        if self.countries_filter:
+            payload["cr"] = self.countries_filter
+        if self.languages_filter:
+            payload["lr"] = self.languages_filter
+        if self.location:
+            payload["location"] = self.location
+        if self.uule:
+            payload["uule"] = self.uule
+
+        # Search Type (tbm)
+        if self.search_type:
+            val = self.search_type.lower()
+            payload["tbm"] = self.SEARCH_TYPE_MAP.get(val, val)
+
+        # Filters
+        if self.safe_search is not None:
+            payload["safe"] = "active" if self.safe_search else "off"
+
+        if self.time_filter:
+            val = self.time_filter.lower()
+            payload["tbs"] = self.TIME_FILTER_MAP.get(val, val)
+
+        if self.no_autocorrect:
+            payload["nfpr"] = "1"
+        if self.filter_duplicates is not None:
+            payload["filter"] = "1" if self.filter_duplicates else "0"
+
+        # Device & Rendering
+        if self.device:
+            payload["device"] = self.device.lower()
+        if self.render_js is not None:
+            payload["render_js"] = "True" if self.render_js else "False"
+        if self.no_cache is not None:
+            payload["no_cache"] = "True" if self.no_cache else "False"
+
+        # Advanced
+        if self.ludocid:
+            payload["ludocid"] = self.ludocid
+        if self.kgmid:
+            payload["kgmid"] = self.kgmid
+
+        # Merge extras
+        payload.update(self.extra_params)
+        return payload