thordata-sdk 1.3.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

thordata/core/__init__.py

@@ -0,0 +1,23 @@
+"""
+Core networking modules for Thordata SDK.
+"""
+
+from .async_http_client import AsyncThordataHttpSession
+from .http_client import ThordataHttpSession
+from .tunnel import (
+    HAS_PYSOCKS,
+    UpstreamProxySocketFactory,
+    create_tls_in_tls,
+    parse_upstream_proxy,
+    socks5_handshake,
+)
+
+__all__ = [
+    "ThordataHttpSession",
+    "AsyncThordataHttpSession",
+    "parse_upstream_proxy",
+    "UpstreamProxySocketFactory",
+    "create_tls_in_tls",
+    "socks5_handshake",
+    "HAS_PYSOCKS",
+]

thordata/core/async_http_client.py

@@ -0,0 +1,91 @@
+"""
+Core Async HTTP Session management for Thordata SDK.
+Wraps aiohttp.ClientSession with retry logic and standard headers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import aiohttp
+
+from .. import __version__ as _sdk_version
+from .._utils import build_user_agent
+from ..exceptions import ThordataNetworkError, ThordataTimeoutError
+from ..retry import RetryConfig, with_retry
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncThordataHttpSession:
+    """
+    Async wrapper for HTTP requests with built-in retry logic.
+    """
+
+    def __init__(self, timeout: int = 30, retry_config: RetryConfig | None = None):
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+        self._retry_config = retry_config or RetryConfig()
+        self._session: aiohttp.ClientSession | None = None
+        self._headers = {
+            "User-Agent": build_user_agent(_sdk_version, "aiohttp"),
+            "Accept-Encoding": "gzip, deflate",
+        }
+
+    async def _ensure_session(self) -> aiohttp.ClientSession:
+        if self._session is None or self._session.closed:
+            self._session = aiohttp.ClientSession(
+                timeout=self._timeout, headers=self._headers, trust_env=True
+            )
+        return self._session
+
+    async def close(self) -> None:
+        if self._session and not self._session.closed:
+            await self._session.close()
+            self._session = None
+
+    async def request(
+        self,
+        method: str,
+        url: str,
+        params: dict[str, Any] | None = None,
+        data: Any = None,
+        headers: dict[str, str] | None = None,
+        timeout: int | None = None,
+        proxy: str | None = None,
+        proxy_auth: aiohttp.BasicAuth | None = None,
+    ) -> aiohttp.ClientResponse:
+        """
+        Execute async HTTP request with automatic retry logic.
+        """
+        session = await self._ensure_session()
+
+        # Determine timeout
+        req_timeout = aiohttp.ClientTimeout(total=timeout) if timeout else self._timeout
+
+        @with_retry(self._retry_config)
+        async def _do_request() -> aiohttp.ClientResponse:
+            try:
+                return await session.request(
+                    method=method,
+                    url=url,
+                    params=params,
+                    data=data,
+                    headers=headers,
+                    timeout=req_timeout,
+                    proxy=proxy,
+                    proxy_auth=proxy_auth,
+                )
+            except asyncio.TimeoutError as e:
+                # Map asyncio timeout to SDK timeout for retry handler
+                raise ThordataTimeoutError(
+                    f"Async request timed out: {e}", original_error=e
+                ) from e
+            except aiohttp.ClientError as e:
+                # Map aiohttp errors to SDK network error
+                raise ThordataNetworkError(
+                    f"Async request failed: {e}", original_error=e
+                ) from e
+
+        return await _do_request()

thordata/core/http_client.py

@@ -0,0 +1,79 @@
+"""
+Core HTTP Session management for Thordata SDK.
+Handles authentication injection, retries, and session lifecycle.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import requests
+
+from .. import __version__ as _sdk_version
+from .._utils import build_user_agent
+from ..exceptions import ThordataNetworkError, ThordataTimeoutError
+from ..retry import RetryConfig, with_retry
+
+
+class ThordataHttpSession:
+    """
+    Wrapper around requests.Session with built-in retry and Thordata headers.
+    """
+
+    def __init__(
+        self,
+        timeout: int = 30,
+        retry_config: RetryConfig | None = None,
+        trust_env: bool = True,
+    ):
+        self._session = requests.Session()
+        self._session.trust_env = trust_env
+        self._timeout = timeout
+        self._retry_config = retry_config or RetryConfig()
+
+        # Default Headers
+        self._session.headers.update(
+            {
+                "User-Agent": build_user_agent(_sdk_version, "requests"),
+                "Accept-Encoding": "gzip, deflate",
+            }
+        )
+
+    def close(self) -> None:
+        self._session.close()
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        params: dict[str, Any] | None = None,
+        data: Any = None,
+        headers: dict[str, str] | None = None,
+        timeout: int | None = None,
+        stream: bool = False,
+    ) -> requests.Response:
+        """
+        Execute HTTP request with automatic retry logic.
+        """
+        effective_timeout = timeout if timeout is not None else self._timeout
+
+        @with_retry(self._retry_config)
+        def _do_request() -> requests.Response:
+            return self._session.request(
+                method=method,
+                url=url,
+                params=params,
+                data=data,
+                headers=headers,
+                timeout=effective_timeout,
+                stream=stream,
+            )
+
+        try:
+            return _do_request()
+        except requests.Timeout as e:
+            raise ThordataTimeoutError(
+                f"Request timed out: {e}", original_error=e
+            ) from e
+        except requests.RequestException as e:
+            raise ThordataNetworkError(f"Request failed: {e}", original_error=e) from e

thordata/core/tunnel.py

@@ -0,0 +1,287 @@
+"""
+Low-level networking tunnel utilities.
+Handles Upstream Proxies, TLS-in-TLS wrapping, and SOCKS handshakes.
+"""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import logging
+import socket
+import ssl
+import time
+from typing import Any
+from urllib.parse import urlparse
+
+try:
+    import socks
+
+    HAS_PYSOCKS = True
+except ImportError:
+    HAS_PYSOCKS = False
+
+logger = logging.getLogger(__name__)
+
+
+def parse_upstream_proxy() -> dict[str, Any] | None:
+    import os
+
+    upstream_url = os.environ.get("THORDATA_UPSTREAM_PROXY", "").strip()
+    if not upstream_url:
+        return None
+
+    parsed = urlparse(upstream_url)
+    scheme = (parsed.scheme or "").lower()
+
+    # Normalize scheme
+    if scheme in ("socks5", "socks5h"):
+        scheme = "socks5"
+    elif scheme in ("http", "https"):
+        scheme = "http"
+    else:
+        return None
+
+    return {
+        "scheme": scheme,
+        "host": parsed.hostname or "127.0.0.1",
+        "port": parsed.port or 7890,
+        "username": parsed.username,
+        "password": parsed.password,
+    }
+
+
+class UpstreamProxySocketFactory:
+    """Creates sockets routed through an upstream proxy."""
+
+    def __init__(self, upstream_config: dict[str, Any]):
+        self.config = upstream_config
+
+    def create_connection(
+        self,
+        address: tuple[str, int],
+        timeout: float | None = None,
+    ) -> socket.socket:
+        scheme = self.config["scheme"]
+        if timeout is None or timeout <= 0:
+            timeout = 30.0
+
+        if scheme == "socks5":
+            return self._create_socks_connection(address, timeout)
+        else:
+            return self._create_http_tunnel(address, timeout)
+
+    def _create_socks_connection(
+        self, address: tuple[str, int], timeout: float
+    ) -> socket.socket:
+        if not HAS_PYSOCKS:
+            raise ImportError("PySocks is required for upstream SOCKS proxy.")
+
+        sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.set_proxy(
+            socks.SOCKS5,
+            self.config["host"],
+            self.config["port"],
+            rdns=True,
+            username=self.config.get("username"),
+            password=self.config.get("password"),
+        )
+        sock.settimeout(timeout)
+        try:
+            sock.connect(address)
+        except Exception:
+            sock.close()
+            raise
+        return sock
+
+    def _create_http_tunnel(
+        self, address: tuple[str, int], timeout: float
+    ) -> socket.socket:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(timeout)
+
+        try:
+            # 1. Connect to Upstream
+            sock.connect((self.config["host"], self.config["port"]))
+
+            # 2. Send CONNECT
+            target_host, target_port = address
+            connect_req = f"CONNECT {target_host}:{target_port} HTTP/1.1\r\n"
+            connect_req += f"Host: {target_host}:{target_port}\r\n"
+
+            if self.config.get("username"):
+                creds = f"{self.config['username']}:{self.config.get('password','')}"
+                b64_creds = base64.b64encode(creds.encode()).decode()
+                connect_req += f"Proxy-Authorization: Basic {b64_creds}\r\n"
+
+            connect_req += "\r\n"
+            sock.sendall(connect_req.encode())
+
+            # 3. Read Response (Byte by byte to avoid over-reading)
+            resp = b""
+            while b"\r\n\r\n" not in resp:
+                chunk = sock.recv(1)
+                if not chunk:
+                    raise ConnectionError(
+                        "Upstream proxy closed connection during CONNECT"
+                    )
+                resp += chunk
+
+            # Fix: Decode bytes safely for string formatting
+            status_line = resp.split(b"\r\n")[0]
+            if b"200" not in status_line:
+                status_str = status_line.decode("utf-8", errors="replace")
+                raise ConnectionError(f"Upstream proxy CONNECT failed: {status_str}")
+
+        except Exception:
+            sock.close()
+            raise
+
+        return sock
+
+
+class TLSInTLSSocket:
+    """
+    A wrapper around SSLObject to make it behave like a socket for requests.
+    """
+
+    def __init__(
+        self,
+        outer: ssl.SSLSocket,
+        ssl_obj: ssl.SSLObject,
+        incoming: ssl.MemoryBIO,
+        outgoing: ssl.MemoryBIO,
+    ):
+        self._outer = outer
+        self._ssl = ssl_obj
+        self._incoming = incoming
+        self._outgoing = outgoing
+        self._timeout: float | None = None
+
+    def settimeout(self, t: float | None) -> None:
+        self._timeout = t
+        self._outer.settimeout(t)
+
+    def sendall(self, data: bytes) -> None:
+        try:
+            self._ssl.write(data)
+            enc = self._outgoing.read()
+            if enc:
+                self._outer.sendall(enc)
+        except ssl.SSLWantReadError:
+            pass
+
+    def recv(self, bufsize: int) -> bytes:
+        while True:
+            try:
+                return self._ssl.read(bufsize)
+            except ssl.SSLWantReadError:
+                try:
+                    data = self._outer.recv(8192)
+                    if not data:
+                        return b""
+                    self._incoming.write(data)
+                except socket.timeout:
+                    raise
+            except ssl.SSLWantWriteError:
+                enc = self._outgoing.read()
+                if enc:
+                    self._outer.sendall(enc)
+
+    def close(self) -> None:
+        with contextlib.suppress(Exception):
+            self._outer.close()
+
+
+def create_tls_in_tls(
+    outer_sock: ssl.SSLSocket, hostname: str, timeout: float
+) -> TLSInTLSSocket:
+    ctx = ssl.create_default_context()
+    incoming = ssl.MemoryBIO()
+    outgoing = ssl.MemoryBIO()
+    ssl_obj = ctx.wrap_bio(incoming, outgoing, server_hostname=hostname)
+
+    outer_sock.settimeout(timeout)
+    start_time = time.time()
+
+    while True:
+        if time.time() - start_time > timeout:
+            raise TimeoutError("TLS handshake timed out")
+        try:
+            ssl_obj.do_handshake()
+            break
+        except ssl.SSLWantReadError:
+            data = outgoing.read()
+            if data:
+                outer_sock.sendall(data)
+            try:
+                chunk = outer_sock.recv(4096)
+                if not chunk:
+                    raise ConnectionError("Connection closed")
+                incoming.write(chunk)
+            except socket.timeout:
+                pass
+        except ssl.SSLWantWriteError:
+            data = outgoing.read()
+            if data:
+                outer_sock.sendall(data)
+
+    return TLSInTLSSocket(outer_sock, ssl_obj, incoming, outgoing)
+
+
+def socks5_handshake(
+    sock: socket.socket,
+    target_host: str,
+    target_port: int,
+    user: str | None,
+    pwd: str | None,
+) -> socket.socket:
+    # 1. Auth Method
+    if user and pwd:
+        sock.sendall(b"\x05\x02\x00\x02")
+    else:
+        sock.sendall(b"\x05\x01\x00")
+
+    resp = sock.recv(2)
+    if not resp or resp[0] != 0x05:
+        # Fix: Decode bytes safely
+        resp_str = resp.decode("utf-8", errors="replace") if resp else "Empty"
+        raise ConnectionError(f"Invalid SOCKS5 init response: {resp_str}")
+
+    if resp[1] == 0x02:  # User/Pass
+        u_bytes = (user or "").encode()
+        p_bytes = (pwd or "").encode()
+        auth_payload = (
+            b"\x01" + bytes([len(u_bytes)]) + u_bytes + bytes([len(p_bytes)]) + p_bytes
+        )
+        sock.sendall(auth_payload)
+        auth_resp = sock.recv(2)
+        if not auth_resp or auth_resp[1] != 0x00:
+            raise ConnectionError("SOCKS5 Authentication failed")
+    elif resp[1] == 0xFF:
+        raise ConnectionError("No acceptable authentication methods")
+
+    # 2. Connect
+    req = (
+        b"\x05\x01\x00\x03"
+        + bytes([len(target_host)])
+        + target_host.encode()
+        + target_port.to_bytes(2, "big")
+    )
+    sock.sendall(req)
+
+    # 3. Response
+    resp = sock.recv(4)
+    if not resp or resp[1] != 0x00:
+        err = str(resp[1]) if (resp and len(resp) > 1) else "Empty/Unknown"
+        raise ConnectionError(f"SOCKS5 Connect failed, error code: {err}")
+
+    atype = resp[3]
+    if atype == 1:
+        sock.recv(4 + 2)
+    elif atype == 3:
+        sock.recv(sock.recv(1)[0] + 2)
+    elif atype == 4:
+        sock.recv(16 + 2)
+
+    return sock

thordata/demo.py

@@ -25,9 +25,9 @@ from pathlib import Path
 def _configure_stdio() -> None:
     # Avoid UnicodeEncodeError on Windows consoles with legacy encodings.
     if hasattr(sys.stdout, "reconfigure"):
-        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+        sys.stdout.reconfigure(encoding="utf-8", errors="replace")  # type: ignore[attr-defined]
     if hasattr(sys.stderr, "reconfigure"):
-        sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+        sys.stderr.reconfigure(encoding="utf-8", errors="replace")  # type: ignore[attr-defined]
 
 
 def _load_env() -> None: