thordata-sdk 0.2.4__py3-none-any.whl → 1.2.0__py3-none-any.whl

thordata/demo.py

@@ -0,0 +1,138 @@
+"""
+Unified demo entrypoint for the Thordata Python SDK.
+
+This module runs the example scripts from the repository's `examples/` directory
+using `runpy`, so it does not require `examples/` to be an importable package.
+
+Usage:
+    python -m thordata.demo serp
+    python -m thordata.demo universal
+    python -m thordata.demo scraper
+    python -m thordata.demo concurrency
+
+Notes:
+- This entrypoint is primarily intended for repository usage (dev/demo).
+- When installed from PyPI, the `examples/` directory is typically not included.
+"""
+
+from __future__ import annotations
+
+import runpy
+import sys
+from pathlib import Path
+
+
+def _configure_stdio() -> None:
+    # Avoid UnicodeEncodeError on Windows consoles with legacy encodings.
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    if hasattr(sys.stderr, "reconfigure"):
+        sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+
+
+def _load_env() -> None:
+    # Optional .env support for local development
+    try:
+        from dotenv import load_dotenv
+    except ImportError:
+        return
+    load_dotenv()
+
+
+def _repo_root() -> Path:
+    """
+    Resolve repository root based on src layout:
+    <repo>/src/thordata/demo.py -> parents[2] == <repo>
+    """
+    return Path(__file__).resolve().parents[2]
+
+
+def _examples_dir() -> Path:
+    return _repo_root() / "examples"
+
+
+def _demo_map() -> dict[str, Path]:
+    ex = _examples_dir()
+    return {
+        "serp": ex / "demo_serp_api.py",
+        "universal": ex / "demo_universal.py",
+        "scraper": ex / "demo_web_scraper_api.py",
+        "concurrency": ex / "async_high_concurrency.py",
+    }
+
+
+def _usage() -> str:
+    names = ", ".join(sorted(_demo_map().keys()))
+    return f"Usage: python -m thordata.demo [{names}]"
+
+
+def _run_demo(path: Path) -> int:
+    if not path.exists():
+        print(f"Error: demo script not found: {path}")
+        return 2
+
+    # Ensure examples dir is on sys.path (helpful if demo imports local helpers).
+    examples_dir = str(path.parent.resolve())
+    if examples_dir not in sys.path:
+        sys.path.insert(0, examples_dir)
+
+    try:
+        # Load without triggering `if __name__ == "__main__": ...`
+        ns = runpy.run_path(str(path), run_name="__thordata_demo__")
+
+        main_func = ns.get("main")
+        if callable(main_func):
+            return int(main_func())  # type: ignore[arg-type]
+
+        # Fallback: run as __main__ for scripts without main()
+        runpy.run_path(str(path), run_name="__main__")
+        return 0
+
+    except KeyboardInterrupt:
+        raise
+    except SystemExit as e:
+        # In case fallback run as __main__ triggered SystemExit
+        code = e.code
+        if code is None:
+            return 0
+        if isinstance(code, int):
+            return code
+        return 1
+    except Exception as e:
+        import traceback
+
+        print()
+        print("-" * 60)
+        print("[thordata.demo] The demo script raised an exception.")
+        print(f"[thordata.demo] Script: {path.name}")
+        print(f"[thordata.demo] Error:  {type(e).__name__}: {e}")
+        print()
+        print("Note: This is a failure within the demo script itself,")
+        print("      not an issue with the thordata.demo entrypoint.")
+        print("-" * 60)
+        traceback.print_exc()
+        return 1
+
+
+def main() -> int:
+    _configure_stdio()
+    _load_env()
+
+    if len(sys.argv) < 2:
+        print(_usage())
+        return 2
+
+    name = sys.argv[1].strip().lower()
+    mapping = _demo_map()
+
+    path = mapping.get(name)
+    if path is None:
+        print(f"Unknown demo: {name}")
+        print(_usage())
+        return 2
+
+    return _run_demo(path)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

thordata/enums.py

@@ -0,0 +1,384 @@
+"""
+Enumerations for the Thordata Python SDK.
+
+This module provides type-safe enumerations for all Thordata API parameters,
+making it easier to discover available options via IDE autocomplete.
+"""
+
+from enum import Enum, IntEnum
+
+# =============================================================================
+# Continent Enum
+# =============================================================================
+
+
+class Continent(str, Enum):
+    """
+    Continent codes for geo-targeting.
+    """
+
+    AFRICA = "af"
+    ANTARCTICA = "an"
+    ASIA = "as"
+    EUROPE = "eu"
+    NORTH_AMERICA = "na"
+    OCEANIA = "oc"
+    SOUTH_AMERICA = "sa"
+
+
+# =============================================================================
+# Proxy Host Enum
+# =============================================================================
+
+
+class ProxyHost(str, Enum):
+    """
+    Available proxy gateway hosts.
+
+    Note: Dashboard provides user-specific hosts like {shard}.{region}.thordata.net
+    """
+
+    DEFAULT = "pr.thordata.net"
+    NORTH_AMERICA = "t.na.thordata.net"
+    EUROPE = "t.eu.thordata.net"
+
+
+class ProxyPort(IntEnum):
+    """
+    Available proxy gateway ports.
+    """
+
+    RESIDENTIAL = 9999
+    MOBILE = 5555
+    DATACENTER = 7777
+    ISP = 6666
+
+
+# =============================================================================
+# Search Engine Enums
+# =============================================================================
+
+
+class Engine(str, Enum):
+    """
+    Supported search engines for SERP API.
+
+    Engine naming convention:
+    - Base search: {engine} for basic web search (google, bing, yandex, duckduckgo)
+    - Verticals: {engine}_{vertical} (e.g., google_news, bing_images)
+    - Sub-verticals: {engine}_{vertical}_{sub} (e.g., google_scholar_cite)
+    """
+
+    # ===================
+    # Google
+    # ===================
+    GOOGLE = "google"
+    GOOGLE_SEARCH = "google_search"
+    GOOGLE_AI_MODE = "google_ai_mode"
+    GOOGLE_WEB = "google_web"
+    GOOGLE_SHOPPING = "google_shopping"
+    GOOGLE_LOCAL = "google_local"
+    GOOGLE_VIDEOS = "google_videos"
+    GOOGLE_NEWS = "google_news"
+    GOOGLE_FLIGHTS = "google_flights"
+    GOOGLE_IMAGES = "google_images"
+    GOOGLE_LENS = "google_lens"
+    GOOGLE_TRENDS = "google_trends"
+    GOOGLE_HOTELS = "google_hotels"
+    GOOGLE_PLAY = "google_play"
+    GOOGLE_JOBS = "google_jobs"
+    GOOGLE_SCHOLAR = "google_scholar"
+    GOOGLE_SCHOLAR_CITE = "google_scholar_cite"
+    GOOGLE_SCHOLAR_AUTHOR = "google_scholar_author"
+    GOOGLE_MAPS = "google_maps"
+    GOOGLE_FINANCE = "google_finance"
+    GOOGLE_FINANCE_MARKETS = "google_finance_markets"
+    GOOGLE_PATENTS = "google_patents"
+    GOOGLE_PATENTS_DETAILS = "google_patents_details"
+
+    # ===================
+    # Bing
+    # ===================
+    BING = "bing"
+    BING_SEARCH = "bing_search"
+    BING_IMAGES = "bing_images"
+    BING_VIDEOS = "bing_videos"
+    BING_NEWS = "bing_news"
+    BING_MAPS = "bing_maps"
+    BING_SHOPPING = "bing_shopping"
+
+    # ===================
+    # Yandex
+    # ===================
+    YANDEX = "yandex"
+    YANDEX_SEARCH = "yandex_search"
+
+    # ===================
+    # DuckDuckGo
+    # ===================
+    DUCKDUCKGO = "duckduckgo"
+    DUCKDUCKGO_SEARCH = "duckduckgo_search"
+
+
+class GoogleSearchType(str, Enum):
+    """
+    Search types specific to Google.
+
+    These map to the second part of Google engine names.
+    For example, GOOGLE + NEWS = google_news
+    """
+
+    SEARCH = "search"
+    AI_MODE = "ai_mode"
+    WEB = "web"
+    SHOPPING = "shopping"
+    LOCAL = "local"
+    VIDEOS = "videos"
+    NEWS = "news"
+    FLIGHTS = "flights"
+    IMAGES = "images"
+    LENS = "lens"
+    TRENDS = "trends"
+    HOTELS = "hotels"
+    PLAY = "play"
+    JOBS = "jobs"
+    SCHOLAR = "scholar"
+    MAPS = "maps"
+    FINANCE = "finance"
+    PATENTS = "patents"
+
+
+class BingSearchType(str, Enum):
+    """
+    Search types specific to Bing.
+    """
+
+    SEARCH = "search"
+    IMAGES = "images"
+    VIDEOS = "videos"
+    NEWS = "news"
+    MAPS = "maps"
+    SHOPPING = "shopping"
+
+
+class GoogleTbm(str, Enum):
+    """
+    Google tbm (to be matched) parameter values.
+
+    Only available when using specific Google engines that support tbm.
+    """
+
+    NEWS = "nws"
+    SHOPPING = "shop"
+    IMAGES = "isch"
+    VIDEOS = "vid"
+
+
+class Device(str, Enum):
+    """
+    Device types for SERP API.
+    """
+
+    DESKTOP = "desktop"
+    MOBILE = "mobile"
+    TABLET = "tablet"
+
+
+class TimeRange(str, Enum):
+    """
+    Time range filters for search results.
+    """
+
+    HOUR = "hour"
+    DAY = "day"
+    WEEK = "week"
+    MONTH = "month"
+    YEAR = "year"
+
+
+# =============================================================================
+# Proxy Enums
+# =============================================================================
+
+
+class ProxyType(IntEnum):
+    """
+    Types of proxy networks available.
+    """
+
+    RESIDENTIAL = 1
+    UNLIMITED = 2
+    DATACENTER = 3
+    ISP = 4
+    MOBILE = 5
+
+
+class SessionType(str, Enum):
+    """
+    Proxy session types for connection persistence.
+    """
+
+    ROTATING = "rotating"
+    STICKY = "sticky"
+
+
+# =============================================================================
+# Output Format Enums
+# =============================================================================
+
+
+class OutputFormat(str, Enum):
+    """
+    Output formats for Universal Scraping API.
+
+    Currently supported: html, png
+    """
+
+    HTML = "html"
+    PNG = "png"
+
+
+class DataFormat(str, Enum):
+    """
+    Data formats for task result download.
+    """
+
+    JSON = "json"
+    CSV = "csv"
+    XLSX = "xlsx"
+
+
+# =============================================================================
+# Task Status Enums
+# =============================================================================
+
+
+class TaskStatus(str, Enum):
+    """
+    Possible statuses for async scraping tasks.
+    """
+
+    PENDING = "pending"
+    RUNNING = "running"
+    READY = "ready"
+    SUCCESS = "success"
+    FINISHED = "finished"
+    FAILED = "failed"
+    ERROR = "error"
+    CANCELLED = "cancelled"
+    UNKNOWN = "unknown"
+
+    @classmethod
+    def is_terminal(cls, status: "TaskStatus") -> bool:
+        """Check if a status is terminal (no more updates expected)."""
+        return status in {
+            cls.READY,
+            cls.SUCCESS,
+            cls.FINISHED,
+            cls.FAILED,
+            cls.ERROR,
+            cls.CANCELLED,
+        }
+
+    @classmethod
+    def is_success(cls, status: "TaskStatus") -> bool:
+        """Check if a status indicates success."""
+        return status in {cls.READY, cls.SUCCESS, cls.FINISHED}
+
+    @classmethod
+    def is_failure(cls, status: "TaskStatus") -> bool:
+        """Check if a status indicates failure."""
+        return status in {cls.FAILED, cls.ERROR}
+
+
+# =============================================================================
+# Country Enum (Common Countries)
+# =============================================================================
+
+
+class Country(str, Enum):
+    """
+    Common country codes for geo-targeting.
+    """
+
+    # North America
+    US = "us"
+    CA = "ca"
+    MX = "mx"
+
+    # Europe
+    GB = "gb"
+    DE = "de"
+    FR = "fr"
+    ES = "es"
+    IT = "it"
+    NL = "nl"
+    PL = "pl"
+    RU = "ru"
+    UA = "ua"
+    SE = "se"
+    NO = "no"
+    DK = "dk"
+    FI = "fi"
+    CH = "ch"
+    AT = "at"
+    BE = "be"
+    PT = "pt"
+    IE = "ie"
+    CZ = "cz"
+    GR = "gr"
+
+    # Asia Pacific
+    CN = "cn"
+    JP = "jp"
+    KR = "kr"
+    IN = "in"
+    AU = "au"
+    NZ = "nz"
+    SG = "sg"
+    HK = "hk"
+    TW = "tw"
+    TH = "th"
+    VN = "vn"
+    ID = "id"
+    MY = "my"
+    PH = "ph"
+    PK = "pk"
+    BD = "bd"
+
+    # South America
+    BR = "br"
+    AR = "ar"
+    CL = "cl"
+    CO = "co"
+    PE = "pe"
+    VE = "ve"
+
+    # Middle East & Africa
+    AE = "ae"
+    SA = "sa"
+    IL = "il"
+    TR = "tr"
+    ZA = "za"
+    EG = "eg"
+    NG = "ng"
+    KE = "ke"
+    MA = "ma"
+
+
+# =============================================================================
+# Helper Functions
+# =============================================================================
+
+
+def normalize_enum_value(value: object, enum_class: type) -> str:
+    """
+    Safely convert an enum or string to its string value.
+    """
+    if isinstance(value, enum_class):
+        return str(getattr(value, "value", value)).lower()
+    if isinstance(value, str):
+        return value.lower()
+    raise TypeError(
+        f"Expected {enum_class.__name__} or str, got {type(value).__name__}"
+    )