curlpro 0.2.0__py3-none-win_amd64.whl

curlpro/__init__.py

@@ -0,0 +1,61 @@
+"""curlpro — an HTTP client with a browser's network fingerprint.
+
+    import curlpro
+
+    curlpro.load_profiles("profiles")
+    r = curlpro.get("https://example.com", impersonate="chrome-151-windows")
+    print(r.status, r.text[:200])
+
+A profile can also be added at runtime, without waiting for a library
+release::
+
+    curlpro.register_profile(json.load(open("chrome-152-windows.json")))
+
+The library reproduces the network layer: the TLS ClientHello, HTTP/2 frames,
+header order and case. The JavaScript fingerprint (canvas, WebGL, navigator)
+belongs to the browser and is not covered here.
+"""
+
+from __future__ import annotations
+
+from ._ffi import HTTPError, Timeout, CurlProError, WebSocketClosed
+from .expect import Expect, ExpectationFailed
+from .aio import AsyncSession, AsyncStreamResponse, AsyncWebSocket
+from .cookies import Cookie, Cookies
+from .session import Redirect, Response, Session, delete, get, head, options, patch, post, put
+from .profiles import Profile, ensure_loaded, list_profiles, load_profiles, register_profile
+from .stream import StreamResponse
+from .websocket import WebSocket
+
+__all__ = [
+    "AsyncSession",
+    "AsyncStreamResponse",
+    "AsyncWebSocket",
+    "Cookie",
+    "Cookies",
+    "CurlProError",
+    "Expect",
+    "ExpectationFailed",
+    "HTTPError",
+    "Redirect",
+    "Timeout",
+    "Profile",
+    "Response",
+    "Session",
+    "StreamResponse",
+    "WebSocket",
+    "WebSocketClosed",
+    "delete",
+    "ensure_loaded",
+    "get",
+    "head",
+    "list_profiles",
+    "load_profiles",
+    "options",
+    "patch",
+    "post",
+    "put",
+    "register_profile",
+]
+
+__version__ = "0.1.0"

curlpro/_completions.py

@@ -0,0 +1,142 @@
+"""Collector of finished native calls.
+
+An async request goes into a goroutine, and Python learns that it finished
+from here. There is exactly one thread per process, no matter how many calls
+are in flight: it waits inside the native part, where ctypes releases the
+GIL, so the event loop keeps running.
+
+The result itself is picked up by the event loop — that call is quick, just a
+memory copy. Reaching into a loop from another thread is only allowed through
+call_soon_threadsafe, and that is how the future is woken.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from typing import Any
+
+from ._ffi import _call, _lib, call_framed_out
+
+# How long a single wait inside the native part lasts. Waking up four times a
+# second while idle costs nothing, and the thread exits almost immediately
+# once nobody uses it any more.
+_WAIT_MS = 250
+
+
+class Completions:
+    """The single per-process collector of results."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._waiters: dict[int, tuple[asyncio.AbstractEventLoop, asyncio.Future]] = {}
+        # Results that arrived before their waiter: the work can finish
+        # between the start call and register — routine for a stream read,
+        # which takes microseconds.
+        self._ready: dict[int, tuple[Any, bytes]] = {}
+        self._thread: threading.Thread | None = None
+        self._stop = False
+
+    def register(self, request_id: int, future: asyncio.Future) -> None:
+        loop = asyncio.get_running_loop()
+        with self._lock:
+            done = self._ready.pop(request_id, None)
+            if done is not None:
+                # Registration happens on the event loop, so the future can
+                # be completed right here, without call_soon_threadsafe.
+                future.set_result(done)
+                return
+            self._waiters[request_id] = (loop, future)
+            if self._thread is None or not self._thread.is_alive():
+                self._stop = False
+                self._thread = threading.Thread(
+                    target=self._reap, name="curlpro-completions", daemon=True
+                )
+                self._thread.start()
+
+    def forget(self, request_id: int) -> None:
+        """Drops the wait: the call was cancelled and nobody needs its result."""
+        with self._lock:
+            self._waiters.pop(request_id, None)
+            self._ready.pop(request_id, None)
+
+    def _reap(self) -> None:
+        while True:
+            with self._lock:
+                if self._stop or not (self._waiters or self._ready):
+                    self._thread = None
+                    return
+            rid = int(_lib.curlpro_result_wait(_WAIT_MS))
+            if rid == 0:
+                continue
+            with self._lock:
+                entry = self._waiters.pop(rid, None)
+                if entry is None:
+                    # No waiter yet. Work is started and registered in two
+                    # steps, and a fast one — reading a chunk of the body —
+                    # can finish in between. The result is set aside for
+                    # register to pick up.
+                    #
+                    # It used to be thrown away here, and such a call hung
+                    # forever: 24 concurrent stream reads lost one of them.
+                    #
+                    # It is taken under the same lock: otherwise register
+                    # slips between the check and the hand-off, and the
+                    # waiter and its ready result never meet again.
+                    #
+                    # A cancelled result never lands here: cancelling removes
+                    # the call from the native registry, leaving nothing to take.
+                    done = _take(rid)
+                    if done is not None:
+                        self._ready[rid] = done
+                    continue
+            loop, future = entry
+            try:
+                loop.call_soon_threadsafe(_settle, future, rid)
+            except RuntimeError:
+                # The event loop is already closed — nobody to hand it to.
+                _take(rid)
+
+
+def _take(request_id: int) -> tuple[Any, bytes] | None:
+    try:
+        return call_framed_out("curlpro_result_take", request_id)
+    except Exception:  # noqa: BLE001 — already taken, or the session is closed
+        return None
+
+
+def _settle(future: asyncio.Future, request_id: int) -> None:
+    """Runs on the event loop: takes the result and wakes the waiter."""
+    if future.done():  # the task was cancelled while the result was on its way
+        _take(request_id)
+        return
+    try:
+        payload, content = call_framed_out("curlpro_result_take", request_id)
+    except BaseException as exc:  # noqa: BLE001 — hand the error to the waiter
+        future.set_exception(exc)
+        return
+    future.set_result((payload, content))
+
+
+#: One instance per process: the native ready queue is single too.
+completions = Completions()
+
+
+async def settle(started: dict) -> tuple[Any, bytes]:
+    """Awaits the result of work already started: a request, a read, a receive.
+
+    Cancelling the task drops the wait and cancels the work natively. For a
+    request that frees the connection at once; for a stream read or a message
+    receive there is nothing to cancel — whatever was taken off the wire is
+    lost, which is why the stream or socket is closed after a cancellation.
+    """
+    request_id = int(started["request"])
+    loop = asyncio.get_running_loop()
+    future: asyncio.Future = loop.create_future()
+    completions.register(request_id, future)
+    try:
+        return await future
+    except asyncio.CancelledError:
+        completions.forget(request_id)
+        _call("curlpro_request_cancel", request_id)
+        raise

curlpro/_ffi.py

@@ -0,0 +1,302 @@
+"""Loading the native library and calling it through ctypes.
+
+Every export returns a char* holding a JSON envelope
+``{"ok":…, "error":…, "code":…, "data":…}``. That string is allocated in C and
+must be released with ``curlpro_free`` or it leaks. :func:`_call` does the
+releasing, which is why no pointer ever escapes this module.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import json
+import os
+import platform
+from pathlib import Path
+from typing import Any
+
+
+class CurlProError(RuntimeError):
+    """An error raised by the native part.
+
+    ``code`` is the machine-readable code when the native side knows one:
+    ``timeout``, ``session_closed``, ``too_large``, ``ws_closed``,
+    ``ws_too_big``, ``ws_protocol``. Never branch on the message text — it is
+    for humans.
+    """
+
+    def __init__(self, message: str, code: str | None = None):
+        super().__init__(message)
+        self.code = code
+
+
+class Timeout(CurlProError):
+    """The request ran out of time.
+
+    A class of its own because a timeout is the one outcome a scraper treats
+    differently from other network errors: it retries it.
+    """
+
+
+class HTTPError(CurlProError):
+    """A response with an error status; raised by :meth:`Response.raise_for_status`.
+
+    That method used to raise a bare RuntimeError, indistinguishable from an
+    internal failure. ``response`` stays attached: an error response usually
+    carries a body, and that body is the reason to look at it.
+    """
+
+    def __init__(self, message: str, response=None, code: str | None = None):  # noqa: ANN001
+        super().__init__(message, code)
+        self.response = response
+        self.status = getattr(response, "status", None)
+
+
+class WebSocketClosed(CurlProError):
+    """The WebSocket is closed: by the server's Close frame or by the caller.
+
+    A class of its own so that ``for message in ws`` stops on a close only,
+    while read timeouts and protocol errors reach the caller.
+    """
+
+
+def _raise(envelope: dict, name: str) -> None:
+    code = envelope.get("code")
+    message = envelope.get("error") or f"{name}: unknown error"
+    if code == "ws_closed":
+        raise WebSocketClosed(message, code)
+    if code == "timeout":
+        raise Timeout(message, code)
+    raise CurlProError(message, code)
+
+
+def _library_name() -> str:
+    system = platform.system()
+    if system == "Windows":
+        return "curlpro.dll"
+    if system == "Darwin":
+        return "libcurlpro.dylib"
+    return "libcurlpro.so"
+
+
+def _candidates() -> list[Path]:
+    """Where the library is looked for, from an explicit path to a source build."""
+    name = _library_name()
+    out: list[Path] = []
+    if env := os.environ.get("CURLPRO_LIBRARY"):
+        out.append(Path(env))
+    here = Path(__file__).resolve().parent
+    out.append(here / "lib" / name)          # packed into the wheel
+    out.append(here.parent.parent / "dist" / name)  # a build from the repository
+    return out
+
+
+def _load() -> ctypes.CDLL:
+    tried = []
+    for path in _candidates():
+        if path.is_file():
+            return ctypes.CDLL(str(path))
+        tried.append(str(path))
+    raise CurlProError(
+        "native library not found. Looked in:\n  "
+        + "\n  ".join(tried)
+        + "\nBuild it: CGO_ENABLED=1 go build -buildmode=c-shared -o dist/"
+        + _library_name()
+        + " ./lib\n"
+        + "(cgo needs a C compiler; without CGO_ENABLED=1 the build fails with\n"
+        + "\"build constraints exclude all Go files\", which names neither)\n"
+        + "or point CURLPRO_LIBRARY at an existing build"
+    )
+
+
+_lib = _load()
+
+_lib.curlpro_free.argtypes = [ctypes.c_void_p]
+_lib.curlpro_free.restype = None
+
+for _name, _args in (
+    ("curlpro_version", []),
+    ("curlpro_profiles_list", []),
+    ("curlpro_profiles_load_dir", [ctypes.c_char_p]),
+    ("curlpro_profile_register", [ctypes.c_char_p]),
+    ("curlpro_session_new", [ctypes.c_char_p]),
+    ("curlpro_session_close", [ctypes.c_longlong]),
+    (
+        "curlpro_request",
+        [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)],
+    ),
+    (
+        "curlpro_stream_open",
+        [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)],
+    ),
+    ("curlpro_stream_close", [ctypes.c_longlong]),
+    ("curlpro_ws_connect", [ctypes.c_longlong, ctypes.c_char_p]),
+    (
+        "curlpro_ws_send",
+        [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)],
+    ),
+    ("curlpro_ws_recv", [ctypes.c_longlong, ctypes.POINTER(ctypes.c_int)]),
+    ("curlpro_ws_close", [ctypes.c_longlong, ctypes.c_int, ctypes.c_char_p]),
+    ("curlpro_session_set_header", [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_char_p]),
+    ("curlpro_session_remove_header", [ctypes.c_longlong, ctypes.c_char_p]),
+    ("curlpro_session_reset_headers", [ctypes.c_longlong]),
+    ("curlpro_session_headers", [ctypes.c_longlong]),
+    ("curlpro_session_cookies", [ctypes.c_longlong]),
+    ("curlpro_session_set_cookies", [ctypes.c_longlong, ctypes.c_char_p]),
+    ("curlpro_session_clear_cookies", [ctypes.c_longlong]),
+    ("curlpro_request_start", [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int]),
+    ("curlpro_stream_open_start", [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int]),
+    ("curlpro_stream_read_start", [ctypes.c_longlong, ctypes.c_int]),
+    ("curlpro_ws_connect_start", [ctypes.c_longlong, ctypes.c_char_p]),
+    ("curlpro_ws_send_start", [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int]),
+    ("curlpro_ws_recv_start", [ctypes.c_longlong]),
+    ("curlpro_result_take", [ctypes.c_longlong, ctypes.POINTER(ctypes.c_int)]),
+    ("curlpro_request_cancel", [ctypes.c_longlong]),
+    ("curlpro_debug_counts", []),
+):
+    try:
+        _fn = getattr(_lib, _name)
+    except AttributeError:
+        raise CurlProError(
+            f"the library exports no {_name}: it was built from an older revision.\n"
+            f"Rebuild it: .\\build.ps1 (writes to dist/)"
+        ) from None
+    _fn.argtypes = _args
+    # c_void_p and not c_char_p: ctypes converts c_char_p into bytes and
+    # loses the original pointer, which curlpro_free needs.
+    _fn.restype = ctypes.c_void_p
+
+# Waiting for completions and counting in-flight calls return numbers, not
+# pointers. ctypes performs the blocking wait with the GIL released, and the
+# async path stands on that: the collector thread waits without blocking the
+# event loop.
+_lib.curlpro_result_wait.argtypes = [ctypes.c_int]
+_lib.curlpro_result_wait.restype = ctypes.c_longlong
+_lib.curlpro_async_pending.argtypes = []
+_lib.curlpro_async_pending.restype = ctypes.c_longlong
+
+# read returns a byte count, not a pointer: the caller owns the buffer.
+_lib.curlpro_stream_read.argtypes = [ctypes.c_longlong, ctypes.c_char_p, ctypes.c_int]
+_lib.curlpro_stream_read.restype = ctypes.c_int
+
+
+def stream_read(stream_id: int, size: int) -> bytes:
+    """Reads up to size bytes. An empty result means the body ended."""
+    buf = ctypes.create_string_buffer(size)
+    n = _lib.curlpro_stream_read(stream_id, buf, size)
+    if n < 0:
+        raise CurlProError("stream read failed")
+    return buf.raw[:n]
+
+
+def _call(name: str, *args: Any) -> Any:
+    """Calls an export, unwraps the envelope and frees the C string."""
+    ptr = getattr(_lib, name)(*args)
+    if not ptr:
+        raise CurlProError(f"{name}: the native side returned NULL")
+    try:
+        raw = ctypes.string_at(ptr).decode("utf-8")
+    finally:
+        _lib.curlpro_free(ptr)
+
+    envelope = json.loads(raw)
+    if not envelope.get("ok"):
+        _raise(envelope, name)
+    return envelope.get("data")
+
+
+# Minimum version of the native part: major and minor. Raise it together
+# with lib/curlpro.go whenever Python starts depending on a new export or field.
+REQUIRED_VERSION = (0, 12)
+
+
+def _check_version() -> None:
+    """Checks the library version against what this Python expects.
+
+    Without the check a mismatch is silent: both sides tolerate unknown JSON
+    fields, so an older library quietly ignores new options — the request
+    goes out without them, and that looks like a logic bug rather than a
+    stale build.
+    """
+    raw = _call("curlpro_version").get("version", "")
+    try:
+        got = tuple(int(p) for p in raw.split(".")[:2])
+    except ValueError:
+        raise CurlProError(f"the library reported an unreadable version {raw!r}") from None
+
+    if got < REQUIRED_VERSION:
+        want = ".".join(str(p) for p in REQUIRED_VERSION)
+        raise CurlProError(
+            f"library version {raw} is older than the required {want}.x, "
+            f"so some options would be silently ignored.\n"
+            f"Rebuild it: .\\build.ps1 (writes to dist/)"
+        )
+
+
+_check_version()
+
+
+# Request and response bodies travel as binary, separate from the JSON.
+#
+# As a string inside JSON they were not merely copied one extra time:
+# arbitrary bytes are not valid UTF-8, so the response was corrupted — 10,000
+# random bytes came back as 18,502 after the round trip.
+#
+# Frame layout: [uint32 LE JSON length][JSON][raw body].
+_HEADER = 4
+
+
+def _frame(meta: Any, body: bytes = b"") -> bytes:
+    js = encode(meta)
+    return len(js).to_bytes(_HEADER, "little") + js + body
+
+
+def _unframe(name: str, raw: bytes) -> tuple[Any, bytes]:
+    if len(raw) < _HEADER:
+        raise CurlProError(f"{name}: frame is shorter than its header ({len(raw)} bytes)")
+    meta_len = int.from_bytes(raw[:_HEADER], "little")
+    envelope = json.loads(raw[_HEADER : _HEADER + meta_len])
+    if not envelope.get("ok"):
+        _raise(envelope, name)
+    return envelope.get("data"), raw[_HEADER + meta_len :]
+
+
+def call_framed(name: str, *args: Any, body: bytes = b"", meta: Any) -> tuple[Any, bytes]:
+    """Calls an export with frames and returns (data, body)."""
+    payload = _frame(meta, body)
+    out_len = ctypes.c_int(0)
+    ptr = getattr(_lib, name)(*args, payload, len(payload), ctypes.byref(out_len))
+    if not ptr:
+        raise CurlProError(f"{name}: the native side returned NULL")
+    try:
+        raw = ctypes.string_at(ptr, out_len.value)
+    finally:
+        _lib.curlpro_free(ptr)
+    return _unframe(name, raw)
+
+
+def call_with_frame(name: str, *args: Any, body: bytes = b"", meta: Any) -> Any:
+    """A frame going in, a plain JSON envelope coming back.
+
+    This is how an async request starts: the body goes out as a frame, and
+    all that returns is the request number — there is no response yet.
+    """
+    payload = _frame(meta, body)
+    return _call(name, *args, payload, len(payload))
+
+
+def call_framed_out(name: str, *args: Any) -> tuple[Any, bytes]:
+    """Like call_framed but without an input frame — for recv-style calls."""
+    out_len = ctypes.c_int(0)
+    ptr = getattr(_lib, name)(*args, ctypes.byref(out_len))
+    if not ptr:
+        raise CurlProError(f"{name}: the native side returned NULL")
+    try:
+        raw = ctypes.string_at(ptr, out_len.value)
+    finally:
+        _lib.curlpro_free(ptr)
+    return _unframe(name, raw)
+
+
+def encode(obj: Any) -> bytes:
+    return json.dumps(obj, ensure_ascii=False).encode("utf-8")