wpr-runtime-core 0.2.0__py3-none-any.whl

wpr_runtime_core/iocp_pipe.py

@@ -0,0 +1,204 @@
+"""Windows Proactor JSON-line pipe transport with no per-RPC executor hop."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from typing import Any
+
+
+class _PipeProtocol(asyncio.Protocol):
+    def __init__(self, multiplex=False):
+        self.multiplex = multiplex
+        self.pending: dict[int, asyncio.Future] = {}
+        self.transport = None
+        self.buffer = bytearray()
+        self.response: asyncio.Future | None = None
+        self.closed = asyncio.get_running_loop().create_future()
+        self.flow_waiter: asyncio.Future | None = None
+
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def pause_writing(self):
+        self.flow_waiter = asyncio.get_running_loop().create_future()
+
+    def resume_writing(self):
+        if self.flow_waiter is not None and not self.flow_waiter.done():
+            self.flow_waiter.set_result(None)
+        self.flow_waiter = None
+
+    def data_received(self, data):
+        callback_ns = time.perf_counter_ns()
+        self.buffer.extend(data)
+        while (newline := self.buffer.find(b"\n")) >= 0:
+            if newline >= 16 * 1024 * 1024:
+                self.fail(RuntimeError("WPR RPC response exceeded the framing limit"))
+                return
+            line = bytes(self.buffer[: newline + 1])
+            del self.buffer[: newline + 1]
+            try:
+                started = time.perf_counter_ns()
+                value = json.loads(line)
+                if not isinstance(value, dict):
+                    raise TypeError("invalid WPR RPC envelope")
+                decoded_ms = (time.perf_counter_ns() - started) / 1_000_000
+                ipc = value.get("_wpr_ipc")
+                if isinstance(ipc, dict):
+                    ipc["client_callback_ns"] = callback_ns
+                    ipc["client_decoded_ns"] = time.perf_counter_ns()
+                future = (
+                    self.pending.pop(value.get("request_id"), None)
+                    if self.multiplex
+                    else self.response
+                )
+                if future is None or future.done():
+                    raise RuntimeError("unexpected uncorrelated WPR pipe response")
+                future.set_result((value, decoded_ms, len(line)))
+            except (ValueError, TypeError, RuntimeError) as error:
+                self.fail(error)
+                return
+        if len(self.buffer) > 16 * 1024 * 1024:
+            self.fail(RuntimeError("WPR RPC response exceeded the framing limit"))
+
+    def fail(self, error):
+        for future in self.pending.values():
+            if not future.done():
+                future.set_exception(error)
+        if self.response is not None and not self.response.done():
+            self.response.set_exception(error)
+        if self.transport is not None:
+            self.transport.abort()
+
+    def connection_lost(self, error):
+        error = error or RuntimeError("gecko worker closed the RPC pipe")
+        for future in self.pending.values():
+            if not future.done():
+                future.set_exception(error)
+        if self.response is not None and not self.response.done():
+            self.response.set_exception(error)
+        if self.flow_waiter is not None and not self.flow_waiter.done():
+            self.flow_waiter.set_exception(error)
+        if not self.closed.done():
+            self.closed.set_result(None)
+
+
+class IocpJsonLinePipe:
+    """Serial or request-ID-multiplexed RPC over a Proactor duplex pipe."""
+
+    def __init__(self, pipe_name: str, *, multiplex=False, on_cancel=None):
+        self.multiplex = multiplex
+        self.on_cancel = on_cancel
+        self._next_id = 0
+        self.pipe_name = pipe_name
+        self.protocol: _PipeProtocol | None = None
+        self.transport = None
+        self._closed = False
+
+    async def connect(self):
+        if self._closed:
+            raise RuntimeError("IOCP pipe is closed")
+        if self.transport is not None:
+            return
+        loop = asyncio.get_running_loop()
+        connect = getattr(loop, "create_pipe_connection", None)
+        if connect is None:
+            raise RuntimeError("IOCP backend requires a Windows Proactor event loop")
+        deadline = loop.time() + 60
+        delay = 0.002
+        while True:
+            try:
+                self.transport, self.protocol = await asyncio.wait_for(
+                    connect(lambda: _PipeProtocol(self.multiplex), self.pipe_name),
+                    max(0.001, deadline - loop.time()),
+                )
+                return
+            except OSError as error:
+                if (
+                    getattr(error, "winerror", None) not in {2, 231}
+                    or loop.time() >= deadline
+                ):
+                    raise
+                await asyncio.sleep(delay)
+                delay = min(0.05, delay * 2)
+
+    async def request(self, payload: dict[str, Any], timing_out=None):
+        started = time.perf_counter_ns()
+        phase = started
+        was_connected = self.transport is not None
+        await self.connect()
+        timings = {
+            "pipe_connect_ms": 0.0
+            if was_connected
+            else (time.perf_counter_ns() - phase) / 1_000_000
+        }
+        assert self.protocol is not None and self.transport is not None
+        future = asyncio.get_running_loop().create_future()
+        self._next_id += 1
+        request_id = self._next_id
+        if self.multiplex:
+            payload = {**payload, "request_id": request_id}
+            self.protocol.pending[request_id] = future
+        else:
+            self.protocol.response = future
+        try:
+            phase = time.perf_counter_ns()
+            encoded = (json.dumps(payload, separators=(",", ":")) + "\n").encode()
+            timings["json_encode_ms"] = (time.perf_counter_ns() - phase) / 1_000_000
+            phase = time.perf_counter_ns()
+            self.transport.write(encoded)
+            if self.protocol.flow_waiter is not None:
+                await self.protocol.flow_waiter
+            timings["pipe_write_ms"] = (time.perf_counter_ns() - phase) / 1_000_000
+            phase = time.perf_counter_ns()
+            value, decode_ms, size = await asyncio.shield(future)
+            resumed_ns = time.perf_counter_ns()
+            ipc = value.get("_wpr_ipc")
+            if isinstance(ipc, dict):
+                timings["request_id"] = request_id
+                timings["ipc_timeline"] = {**ipc, "client_resumed_ns": resumed_ns}
+            timings["pipe_read_ms"] = (time.perf_counter_ns() - phase) / 1_000_000
+            timings["json_decode_ms"] = decode_ms
+            timings["response_metadata_bytes"] = size
+            return value
+        except asyncio.CancelledError:
+            if (
+                self.multiplex
+                and self.on_cancel is not None
+                and payload.get("method") == "network.fetch"
+            ):
+                try:
+                    await self.on_cancel(request_id, future, payload)
+                except BaseException:
+                    future.cancel()
+                    self.transport.abort()
+                    self._closed = True
+                    raise
+            else:
+                future.cancel()
+                self.transport.abort()
+                self._closed = True
+            raise
+        except BaseException:
+            future.cancel()
+            self.transport.abort()
+            self._closed = True
+            raise
+        finally:
+            if self.multiplex:
+                self.protocol.pending.pop(request_id, None)
+            else:
+                self.protocol.response = None
+            if timing_out is not None:
+                timing_out.update(timings)
+                timing_out["pipe_total_ms"] = (
+                    time.perf_counter_ns() - started
+                ) / 1_000_000
+
+    async def aclose(self):
+        self._closed = True
+        if self.transport is not None:
+            self.transport.close()
+            assert self.protocol is not None
+            await asyncio.shield(self.protocol.closed)

wpr_runtime_core/pipe.py

@@ -0,0 +1,169 @@
+"""Windows named-pipe transport for the public WPR JSON protocol."""
+
+from __future__ import annotations
+
+import ctypes
+import json
+import os
+import time
+from ctypes import wintypes
+from typing import Any
+
+if os.name == "nt":
+    import msvcrt
+
+    KERNEL32 = ctypes.WinDLL("kernel32", use_last_error=True)
+    KERNEL32.CreateFileW.argtypes = [
+        wintypes.LPCWSTR,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        ctypes.c_void_p,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.HANDLE,
+    ]
+    KERNEL32.CreateFileW.restype = wintypes.HANDLE
+    KERNEL32.WaitNamedPipeW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD]
+    KERNEL32.WaitNamedPipeW.restype = wintypes.BOOL
+else:
+    KERNEL32 = None
+GENERIC_READ = 0x80000000
+GENERIC_WRITE = 0x40000000
+OPEN_EXISTING = 3
+INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
+
+
+def connect_pipe(pipe_name: str, timeout_seconds: float = 60.0):
+    if KERNEL32 is None:
+        raise OSError("native WPR named-pipe transport requires Windows")
+    deadline = time.monotonic() + timeout_seconds
+    while time.monotonic() < deadline:
+        handle = KERNEL32.CreateFileW(
+            pipe_name,
+            GENERIC_READ | GENERIC_WRITE,
+            0,
+            None,
+            OPEN_EXISTING,
+            0,
+            None,
+        )
+        invalid = {-1, 0xFFFFFFFF, 0xFFFFFFFFFFFFFFFF, INVALID_HANDLE_VALUE}
+        if int(handle) not in invalid and int(handle) != 0:
+            return os.fdopen(msvcrt.open_osfhandle(handle, 0), "r+b", buffering=0)
+        KERNEL32.WaitNamedPipeW(pipe_name, 1000)
+    raise TimeoutError(f"named pipe did not become available: {pipe_name}")
+
+
+def _elapsed_ms(started_ns: int, ended_ns: int) -> float:
+    return (ended_ns - started_ns) / 1_000_000
+
+
+class JsonLinePipe:
+    """Synchronous JSON-lines RPC channel, optionally retained across calls."""
+
+    def __init__(self, pipe_name: str, *, persistent: bool = False):
+        self.pipe_name = pipe_name
+        self.persistent = persistent
+        self._pipe = None
+        self._pending = bytearray()
+        self.read_mode = os.environ.get("WPR_PIPE_READ_MODE", "chunk")
+        if self.read_mode not in {"byte", "chunk"}:
+            raise ValueError("WPR_PIPE_READ_MODE must be 'byte' or 'chunk'")
+
+    def close(self) -> None:
+        pipe, self._pipe = self._pipe, None
+        if pipe is not None:
+            pipe.close()
+        self._pending.clear()
+
+    def _readline(self, pipe) -> tuple[bytes, int]:
+        if self.read_mode == "byte":
+            line = pipe.readline()
+            return line, len(line)
+        reads = 0
+        while True:
+            newline = self._pending.find(b"\n")
+            if newline >= 0:
+                if newline >= 16 * 1024 * 1024:
+                    raise RuntimeError("WPR RPC response exceeded the framing limit")
+                line = bytes(self._pending[: newline + 1])
+                del self._pending[: newline + 1]
+                return line, reads
+            chunk = pipe.read(64 * 1024)
+            reads += 1
+            if not chunk:
+                return b"", reads
+            self._pending.extend(chunk)
+            if len(self._pending) > 16 * 1024 * 1024:
+                raise RuntimeError("WPR RPC response exceeded the framing limit")
+
+    def request(
+        self,
+        payload: dict[str, Any],
+        timing_out: dict[str, float] | None = None,
+    ) -> dict[str, Any]:
+        if timing_out is not None:
+            timing_out.clear()
+        timings: dict[str, float] = {"pipe_connect_ms": 0.0}
+        started_ns = time.perf_counter_ns()
+        pipe = self._pipe if self.persistent else None
+        try:
+            if pipe is None:
+                phase_ns = time.perf_counter_ns()
+                pipe = connect_pipe(self.pipe_name)
+                timings["pipe_connect_ms"] = _elapsed_ms(
+                    phase_ns, time.perf_counter_ns()
+                )
+                if self.persistent:
+                    self._pipe = pipe
+
+            phase_ns = time.perf_counter_ns()
+            encoded = (json.dumps(payload, separators=(",", ":")) + "\n").encode()
+            timings["json_encode_ms"] = _elapsed_ms(phase_ns, time.perf_counter_ns())
+
+            phase_ns = time.perf_counter_ns()
+            offset = 0
+            while offset < len(encoded):
+                written = pipe.write(memoryview(encoded)[offset:])
+                if not written:
+                    raise RuntimeError("gecko worker closed the RPC pipe while writing")
+                offset += written
+            timings["pipe_write_ms"] = _elapsed_ms(phase_ns, time.perf_counter_ns())
+
+            phase_ns = time.perf_counter_ns()
+            line, read_calls = self._readline(pipe)
+            timings["pipe_read_ms"] = _elapsed_ms(phase_ns, time.perf_counter_ns())
+            timings["pipe_read_calls"] = float(read_calls)
+            timings["response_metadata_bytes"] = float(len(line))
+            if not line:
+                raise RuntimeError("gecko worker closed the RPC pipe")
+
+            phase_ns = time.perf_counter_ns()
+            value = json.loads(line)
+            timings["json_decode_ms"] = _elapsed_ms(phase_ns, time.perf_counter_ns())
+            if not isinstance(value, dict):
+                raise RuntimeError("invalid WPR RPC envelope")  # noqa: TRY004 -- protocol error
+            return value
+        except BaseException:
+            if self.persistent:
+                self.close()
+            raise
+        finally:
+            if not self.persistent and pipe is not None:
+                phase_ns = time.perf_counter_ns()
+                pipe.close()
+                timings["pipe_close_ms"] = _elapsed_ms(phase_ns, time.perf_counter_ns())
+            if timing_out is not None:
+                timing_out.update(timings)
+                timing_out["pipe_total_ms"] = _elapsed_ms(
+                    started_ns, time.perf_counter_ns()
+                )
+
+
+def request(
+    payload: dict[str, Any],
+    pipe_name: str,
+    timing_out: dict[str, float] | None = None,
+) -> dict[str, Any]:
+    """Send one RPC using the legacy connect/send/close pipe lifecycle."""
+    return JsonLinePipe(pipe_name).request(payload, timing_out)

wpr_runtime_core/protocols.py

@@ -0,0 +1,23 @@
+"""Small structural interfaces shared by network and browser data consumers."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class ReleaseSession(Protocol):
+    @property
+    def is_open(self) -> bool: ...
+
+    async def data_release(self, blob_id: int) -> Any: ...
+
+
+class BodyLease(Protocol):
+    descriptor: Any
+    mapping_open_ms: float
+    copy_ms: float
+    release_timings: dict[str, Any]
+
+    def open(self) -> Any: ...
+    def read_bytes(self) -> bytes: ...
+    async def release(self) -> dict[str, Any]: ...

wpr_runtime_core/registry.py

@@ -0,0 +1,157 @@
+"""Resolve actual network engines from versioned manifests, never UA labels."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from .installer import (
+    RUNTIME_RELEASES,
+    config_path,
+    configured_runtime_home,
+    select_release,
+)
+from .session import RuntimeConfig
+
+CLIENT_API_VERSION = 1
+HTTP_PROTOCOL = "wpr-http-v1"
+
+
+class RuntimeCompatibilityError(ValueError):
+    """An installed runtime does not satisfy the selected native engine ABI."""
+
+
+def engine_major(value: str | int) -> str:
+    if isinstance(value, bool) or not re.fullmatch(r"[1-9]\d*(?:\.\d+)*", str(value)):
+        raise RuntimeCompatibilityError(f"invalid Firefox engine version: {value!r}")
+    return str(value).split(".", 1)[0]
+
+
+@dataclass(frozen=True)
+class RuntimeManifest:
+    home: Path
+    firefox_major: str
+    runtime_version: str
+    network_worker: Path
+    library_dir: Path
+    protocol: str
+    manifest_schema: str
+
+    @classmethod
+    def load(cls, home: str | Path, requested: str | int) -> RuntimeManifest:
+        home = Path(home).expanduser().resolve()
+        path = home / "manifest.json"
+        if not path.is_file():
+            raise FileNotFoundError(
+                f"runtime manifest missing: {path}; install the selected runtime first"
+            )
+        data = json.loads(path.read_text(encoding="utf-8-sig"))
+        if data.get("schema") != "wpr.runtime-manifest.v1":
+            raise RuntimeCompatibilityError("unsupported runtime manifest schema")
+        version = str(data.get("version", ""))
+        component = data.get("components", {}).get("network", {})
+        major = str(component.get("firefox_major") or engine_major(version))
+        if major != engine_major(version):
+            raise RuntimeCompatibilityError("Firefox major conflicts with runtime build version")
+        if major != engine_major(requested):
+            raise RuntimeCompatibilityError(
+                f"requested Firefox {requested}, manifest contains Firefox {major}"
+            )
+        if "." in str(requested) and str(requested) != version:
+            raise RuntimeCompatibilityError(
+                f"requested runtime {requested}, installed runtime is {version}"
+            )
+        protocols = data.get("protocols", [])
+        protocol = component.get(
+            "protocol", HTTP_PROTOCOL if HTTP_PROTOCOL in protocols else ""
+        )
+        if protocol != HTTP_PROTOCOL or protocol not in protocols:
+            raise RuntimeCompatibilityError(
+                f"expected protocol {HTTP_PROTOCOL}, found {protocol!r}"
+            )
+        abi = component.get("client_api", {"min": 1, "max": 1})
+        if not int(abi["min"]) <= CLIENT_API_VERSION <= int(abi["max"]):
+            raise RuntimeCompatibilityError(
+                f"runtime client API range {abi} excludes {CLIENT_API_VERSION}"
+            )
+        if data.get("platform") != "windows-x86_64":
+            raise RuntimeCompatibilityError(
+                f"unsupported runtime platform: {data.get('platform')!r}"
+            )
+        relative = component.get("worker") or data.get("network_worker")
+        if not isinstance(relative, str):
+            raise RuntimeCompatibilityError("network worker path missing from manifest")
+        worker = (home / relative).resolve()
+        if home not in worker.parents:
+            raise RuntimeCompatibilityError("network worker escapes runtime directory")
+        if not worker.is_file():
+            raise FileNotFoundError(worker)
+        library_dir = (home / component.get("library_dir", "runtime")).resolve()
+        if home not in library_dir.parents or not library_dir.is_dir():
+            raise RuntimeCompatibilityError("runtime library directory is invalid")
+        entry = data.get("files", {}).get(relative)
+        if not isinstance(entry, dict) or not entry.get("sha256"):
+            raise RuntimeCompatibilityError("network worker hash missing from manifest")
+        with worker.open("rb") as stream:
+            digest = hashlib.sha256()
+            for block in iter(lambda: stream.read(1024 * 1024), b""):
+                digest.update(block)
+        if digest.hexdigest() != entry["sha256"]:
+            raise RuntimeCompatibilityError(
+                "network worker SHA256 differs from manifest"
+            )
+        return cls(home, major, version, worker, library_dir, protocol, data["schema"])
+
+    def config(self) -> RuntimeConfig:
+        defaults = RuntimeConfig.bundled()
+        return replace(
+            defaults,
+            package_root=self.home,
+            runtime_dir=self.library_dir,
+            worker_exe=self.home / "runtime" / "gecko-web-worker.exe",
+            network_worker_exe=self.network_worker,
+        )
+
+
+def resolve_runtime(
+    version: str | int = "155", *, runtime_home: str | Path | None = None
+) -> RuntimeManifest:
+    major = engine_major(version)
+    if runtime_home is None:
+        try:
+            settings = json.loads(config_path().read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            settings = {}
+        runtime_home = os.environ.get(f"WPR_RUNTIME_HOME_{major}") or os.environ.get(
+            "WPR_RUNTIME_HOME"
+        )
+        runtime_home = runtime_home or settings.get("network_runtimes", {}).get(str(version))
+        if runtime_home is None and "." not in str(version):
+            runtime_home = settings.get("network_runtimes", {}).get(major)
+        runtime_home = runtime_home or settings.get("runtimes", {}).get(str(version))
+        runtime_home = runtime_home or settings.get("runtimes", {}).get(major)
+        legacy = configured_runtime_home()
+        if runtime_home is None and legacy is not None:
+            try:
+                data = json.loads(
+                    (legacy / "manifest.json").read_text(encoding="utf-8")
+                )
+                if engine_major(data["version"]) == major:
+                    runtime_home = legacy
+            except (OSError, ValueError, KeyError):
+                pass
+        if runtime_home is None:
+            if major not in RUNTIME_RELEASES:
+                raise RuntimeCompatibilityError(
+                    f"Firefox {major} has no published runtime in this catalog; available: {', '.join(RUNTIME_RELEASES)}"
+                )
+            release = select_release(version, product="network")
+            base = Path(
+                os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local"
+            )
+            runtime_home = base / "WPR" / "network-runtimes" / f"firefox{major}" / release["version"]
+    return RuntimeManifest.load(runtime_home, version)

wpr_runtime_core/runtime-catalog.json

@@ -0,0 +1,37 @@
+{
+  "schema": "wpr.runtime-catalog.v2",
+  "client_api": 1,
+  "engines": {
+    "firefox155": {
+      "default_build": "155.0.6",
+      "builds": {
+        "155.0.6": {
+          "version": "155.0.6",
+          "firefox_major": 155,
+          "protocol": "wpr-http-v1",
+          "client_api_min": 1,
+          "client_api_max": 1,
+          "platform": "windows-x86_64",
+          "products": {
+            "browser": {
+              "product": "wpr-runtime",
+              "repository": "childclm/gecko-web-runtime",
+              "tag": "v155.0.6",
+              "asset": "wpr-runtime-155-win64.zip",
+              "url": "https://github.com/childclm/gecko-web-runtime/releases/download/v155.0.6/wpr-runtime-155-win64.zip",
+              "sha256": "4830a4e7b56292892261e67ed8079c43631bb542c847b17e09c0989c9ef6d5dc"
+            },
+            "network": {
+              "product": "wpr-network-runtime",
+              "repository": "childclm/gecko-web-runtime",
+              "tag": "network-155.0.6",
+              "asset": "wpr-network-155.0.6-win64.zip",
+              "url": "https://github.com/childclm/gecko-web-runtime/releases/download/network-155.0.6/wpr-network-155.0.6-win64.zip",
+              "sha256": "7df04dff79ee942fb2d3180bb9b7d06b34708cb395c2d6a416c33fba1bd5bc80"
+            }
+          }
+        }
+      }
+    }
+  }
+}