wpr-runtime-core 0.2.0__py3-none-any.whl

wpr_runtime_core/__init__.py

@@ -0,0 +1,3 @@
+"""Shared runtime infrastructure; browser APIs live in gecko_web_runtime."""
+
+__version__ = "0.1.0"

wpr_runtime_core/_leases.py

@@ -0,0 +1,120 @@
+"""Acknowledged, session-scoped deferred release of network response blobs."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+
+_logger = logging.getLogger("wpr_runtime_core.leases")
+
+
+class DeferredBlobReleases:
+    max_batch = 32
+    flush_bytes = 8 * 1024 * 1024
+    idle_seconds = 0.05
+
+    def __init__(self, flush: Callable[[], Awaitable[None]]) -> None:
+        self.pending: dict[int, int] = {}
+        self.reserved: set[int] = set()
+        self._changed = asyncio.Event()
+        self._flush = flush
+        self._timer: asyncio.TimerHandle | None = None
+        self._background: asyncio.Task | None = None
+        self._drain_lock = asyncio.Lock()
+        self._stopped = False
+
+    async def enqueue(self, blob_id: int, size: int) -> dict:
+        if self._stopped:
+            raise RuntimeError("deferred release queue is closed")
+        self.pending.setdefault(blob_id, size)
+        if self._timer is None:
+            self._timer = asyncio.get_running_loop().call_later(
+                self.idle_seconds, self._idle_flush
+            )
+        if (
+            len(self.pending) >= self.max_batch
+            or sum(self.pending.values()) >= self.flush_bytes
+        ):
+            await self.drain()
+        return {
+            "released": blob_id not in self.pending,
+            "release_deferred": blob_id in self.pending,
+            "pending_release_count": len(self.pending),
+        }
+
+    def snapshot(self) -> list[int]:
+        return [item for item in self.pending if item not in self.reserved][
+            : self.max_batch
+        ]
+
+    def reserve_batch(self) -> list[int]:
+        batch = self.snapshot()
+        self.reserved.update(batch)
+        return batch
+
+    def unreserve(self, batch: list[int]) -> None:
+        self.reserved.difference_update(batch)
+        self._changed.set()
+
+    def acknowledge(self, batch: list[int], receipt: object) -> None:
+        if not isinstance(receipt, list) or any(
+            type(item) is not int for item in receipt
+        ):
+            raise RuntimeError("Worker omitted valid blob release acknowledgements")
+        if set(receipt) != set(batch) or len(receipt) != len(batch):
+            raise RuntimeError(
+                "Worker blob release acknowledgements did not match the batch"
+            )
+        for blob_id in batch:
+            self.pending.pop(blob_id, None)
+        self.unreserve(batch)
+        if not self.pending and self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+
+    def _idle_flush(self) -> None:
+        self._timer = None
+        if self.pending and not self._stopped and self._background is None:
+            self._background = asyncio.create_task(self.drain())
+            self._background.add_done_callback(self._background_done)
+
+    def _background_done(self, task: asyncio.Task) -> None:
+        self._background = None
+        if not task.cancelled() and (error := task.exception()) is not None:
+            _logger.warning("http.blob_release_flush_failed error=%s", error)
+
+    async def drain(self) -> None:
+        async with self._drain_lock:
+            while self.pending and not self._stopped:
+                self._changed.clear()
+                before = self.snapshot()
+                if not before:
+                    await self._changed.wait()
+                    continue
+                await self._flush()
+                if before == self.snapshot():
+                    raise RuntimeError("Worker made no progress flushing blob releases")
+
+    async def close(self) -> None:
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+        await self.drain()
+        self._stopped = True
+        if self._background is not None:
+            await self._background
+
+    def discard(self) -> None:
+        self._stopped = True
+        self.pending.clear()
+        self.reserved.clear()
+        self._changed.set()
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+        if (
+            self._background is not None
+            and self._background is not asyncio.current_task()
+        ):
+            self._background.cancel()

wpr_runtime_core/data_plane.py

@@ -0,0 +1,148 @@
+"""Large-result data-plane leases for the WPR control RPC.
+
+The Worker returns a small JSON descriptor for a named read-only mapping.  The
+consumer maps it only when needed and explicitly releases the Worker-side
+handle after closing its local view.
+"""
+
+from __future__ import annotations
+
+import mmap
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from .protocols import ReleaseSession
+
+
+@dataclass(frozen=True)
+class SharedBlobDescriptor:
+    transport: str
+    blob_id: int
+    mapping_name: str
+    size: int
+    encoding: str = "utf-8"
+    readonly: bool = True
+    mapping_size: int | None = None
+    offset: int = 0
+    generation: int | None = None
+
+    @classmethod
+    def from_value(cls, value: Any) -> SharedBlobDescriptor:
+        if not isinstance(value, dict) or value.get("transport") not in {
+            "shared_memory",
+            "shared_memory_ring",
+        }:
+            raise ValueError("invalid shared blob descriptor")
+        size = int(value["size"])
+        mapping_size = int(value.get("mapping_size") or size)
+        offset = int(value.get("offset", 0))
+        if size < 0 or offset < 0 or mapping_size < size + offset:
+            raise ValueError("invalid shared blob bounds")
+        return cls(
+            transport=str(value["transport"]),
+            blob_id=int(value["blob_id"]),
+            mapping_name=str(value["mapping_name"]),
+            size=size,
+            encoding=str(value.get("encoding") or "utf-8"),
+            readonly=bool(value.get("readonly", True)),
+            mapping_size=mapping_size,
+            offset=offset,
+            generation=(int(value["generation"]) if "generation" in value else None),
+        )
+
+
+class SharedBlob:
+    """A mapped blob with an explicit asynchronous release handshake."""
+
+    def __init__(
+        self,
+        descriptor: SharedBlobDescriptor,
+        session: ReleaseSession,
+    ) -> None:
+        self.descriptor = descriptor
+        self._session = session
+        self._mapping: mmap.mmap | None = None
+        self._released = False
+        self.mapping_open_ms = 0.0
+        self.copy_ms = 0.0
+        self.release_timings: dict[str, float] = {}
+
+    def open(self) -> SharedBlob:
+        if self._mapping is None:
+            if self.descriptor.size < 0:
+                raise ValueError("shared blob size must be non-negative")
+            # Windows mmap opens an existing named CreateFileMapping object via
+            # the tagname argument.  Keep a one-byte view for an empty result.
+            started_ns = time.perf_counter_ns()
+            self._mapping = mmap.mmap(
+                -1,
+                max(1, int(self.descriptor.mapping_size or self.descriptor.size)),
+                tagname=self.descriptor.mapping_name,
+                access=mmap.ACCESS_READ,
+            )
+            self.mapping_open_ms += (time.perf_counter_ns() - started_ns) / 1_000_000
+        return self
+
+    def view(self) -> memoryview:
+        if self._mapping is None:
+            self.open()
+        assert self._mapping is not None
+        start = self.descriptor.offset
+        return memoryview(self._mapping)[start : start + self.descriptor.size]
+
+    def read_bytes(self) -> bytes:
+        started_ns = time.perf_counter_ns()
+        data = self.view().tobytes()
+        self.copy_ms += (time.perf_counter_ns() - started_ns) / 1_000_000
+        return data
+
+    def read_text(self) -> str:
+        return self.read_bytes().decode(self.descriptor.encoding)
+
+    def close(self) -> None:
+        if self._mapping is not None:
+            self._mapping.close()
+            self._mapping = None
+
+    async def release(self) -> dict[str, Any]:
+        if self._released:
+            return {"released": True, "already_released": True}
+        self.close()
+        if not self._session.is_open:
+            # A Worker may be aborted after Python mapped the response. The
+            # OS mapping remains readable while this local view is open, and
+            # the dead process has already discarded its own blob registry.
+            self._released = True
+            return {"released": False, "worker_closed": True}
+        try:
+            if getattr(self._session, "defer_body_releases", False):
+                response = await self._session.queue_body_release(
+                    self.descriptor.blob_id, self.descriptor.size
+                )
+                self.release_timings = {
+                    "release_deferred": float(response["release_deferred"])
+                }
+            elif hasattr(self._session, "data_release_with_timing"):
+                (
+                    response,
+                    self.release_timings,
+                ) = await self._session.data_release_with_timing(
+                    self.descriptor.blob_id
+                )
+            else:
+                response = await self._session.data_release(self.descriptor.blob_id)
+        except Exception:
+            if self._session.is_open:
+                raise
+            self._released = True
+            return {"released": False, "worker_closed": True}
+        self._released = True
+        return response
+
+    async def __aenter__(self) -> SharedBlob:  # noqa: PYI034
+        return self.open()
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        await self.release()

wpr_runtime_core/installer.py

@@ -0,0 +1,326 @@
+"""Download and install a pinned WPR runtime release."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+import uuid
+import warnings
+import zipfile
+from importlib.resources import files
+from pathlib import Path
+
+RUNTIME_CATALOG = json.loads(
+    files(__package__).joinpath("runtime-catalog.json").read_text(encoding="utf-8")
+)
+RUNTIME_RELEASES = {
+    engine.removeprefix("firefox"): {
+        **definition["builds"][definition["default_build"]],
+        **definition["builds"][definition["default_build"]]["products"]["browser"],
+    }
+    for engine, definition in RUNTIME_CATALOG["engines"].items()
+}
+NETWORK_RELEASES = {
+    engine.removeprefix("firefox"): {
+        **definition["builds"][definition["default_build"]],
+        **definition["builds"][definition["default_build"]]["products"]["network"],
+    }
+    for engine, definition in RUNTIME_CATALOG["engines"].items()
+}
+
+
+def select_release(version: str | int, *, product: str = "browser") -> dict[str, str | int]:
+    """Resolve one exact native product without crossing engine versions."""
+    from .registry import RuntimeCompatibilityError, engine_major
+
+    major = engine_major(version)
+    if product not in {"browser", "network"}:
+        raise ValueError(f"unknown runtime product: {product}")
+    engine = RUNTIME_CATALOG["engines"].get(f"firefox{major}")
+    if engine is None:
+        raise RuntimeCompatibilityError(
+            f"Firefox {major} has no published runtime; available: {', '.join(RUNTIME_RELEASES)}"
+        )
+    build = str(version) if "." in str(version) else engine["default_build"]
+    build_info = engine["builds"].get(build)
+    if build_info is None:
+        raise RuntimeCompatibilityError(
+            f"Firefox {version} has no published runtime; available builds: {', '.join(engine['builds'])}"
+        )
+    # Keep the browser default mapping patchable for the historical installer
+    # compatibility tests and existing third-party imports.
+    if product == "browser" and build == engine["default_build"]:
+        release = RUNTIME_RELEASES[major]
+    else:
+        release = build_info["products"][product]
+    if (build_info["firefox_major"] != int(major)
+            or build_info["version"] != build
+            or release["product"] != ("wpr-runtime" if product == "browser" else "wpr-network-runtime")):
+        raise RuntimeCompatibilityError("runtime catalog entry is inconsistent")
+    return {**build_info, **release}
+    return release
+
+
+def _github_token() -> str | None:
+    for name in ("WPR_GITHUB_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"):
+        value = os.environ.get(name)
+        if value:
+            return value.strip()
+    # Git Credential Manager may already hold the user's HTTPS credential.
+    try:
+        query = "protocol=https\nhost=github.com\n\n"
+        result = subprocess.run(
+            ["git", "credential", "fill"],
+            input=query,
+            text=True,
+            capture_output=True,
+            timeout=10,
+            check=True,
+        )
+        for line in result.stdout.splitlines():
+            if line.startswith("password=") and line[9:]:
+                return line[9:].strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return None
+
+
+def _github_asset_request(release: dict[str, str]) -> urllib.request.Request:
+    token = _github_token()
+    if not token:
+        return urllib.request.Request(
+            release["url"],
+            headers={"User-Agent": "gecko-web-runtime-client/1.3.1"},
+        )
+
+    api = (
+        f"https://api.github.com/repos/{release['repository']}"
+        f"/releases/tags/{release['tag']}"
+    )
+    metadata_request = urllib.request.Request(
+        api,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "User-Agent": "gecko-web-runtime-client/1.3.1",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    with _open_github(metadata_request, timeout=30) as response:
+        metadata = json.load(response)
+    asset = next(
+        (
+            item
+            for item in metadata.get("assets", [])
+            if item.get("name") == release["asset"]
+        ),
+        None,
+    )
+    if not asset:
+        raise RuntimeError(f"GitHub release asset not found: {release['asset']}")
+    return urllib.request.Request(
+        f"https://api.github.com/repos/{release['repository']}"
+        f"/releases/assets/{asset['id']}",
+        headers={
+            "Accept": "application/octet-stream",
+            "Authorization": f"Bearer {token}",
+            "User-Agent": "gecko-web-runtime-client/1.3.1",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+
+
+def _open_github(request: urllib.request.Request, *, timeout: int):
+    """Retry GitHub through a direct route if a system proxy breaks TLS."""
+    try:
+        return urllib.request.urlopen(request, timeout=timeout)
+    except urllib.error.URLError:
+        host = urllib.parse.urlsplit(request.full_url).hostname or ""
+        if host not in {"api.github.com", "github.com"}:
+            raise
+        return urllib.request.build_opener(urllib.request.ProxyHandler({})).open(
+            request, timeout=timeout
+        )
+
+
+def config_path() -> Path:
+    base = os.environ.get("LOCALAPPDATA")
+    if not base:
+        base = str(Path.home() / "AppData" / "Local")
+    return Path(base) / "WPR" / "config.json"
+
+
+def configured_runtime_home() -> Path | None:
+    path = config_path()
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        value = data.get("runtime_home")
+        return Path(value).resolve() if value else None
+    except (FileNotFoundError, OSError, json.JSONDecodeError):
+        return None
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _extract_safely(archive: Path, destination: Path) -> None:
+    destination.mkdir(parents=True, exist_ok=True)
+    root = destination.resolve()
+    with zipfile.ZipFile(archive) as source:
+        for member in source.infolist():
+            target = (destination / member.filename).resolve()
+            if target != root and root not in target.parents:
+                raise RuntimeError(f"unsafe path in runtime archive: {member.filename}")
+        source.extractall(destination)
+
+
+def _runtime_is_current(destination: Path, version: str, product: str = "browser") -> bool:
+    try:
+        manifest = json.loads(
+            (destination / "manifest.json").read_text(encoding="utf-8")
+        )
+        if manifest.get("version") != version:
+            return False
+        if product == "network":
+            if manifest.get("product") != "wpr-network-runtime":
+                return False
+            from .registry import RuntimeManifest
+            RuntimeManifest.load(destination, version)
+        elif not (destination / "runtime" / "gecko-web-worker.exe").is_file():
+            return False
+    except (FileNotFoundError, OSError, ValueError, KeyError, TypeError):
+        return False
+    return True
+
+
+def install_runtime(
+    version: str,
+    runtime_home: str | os.PathLike[str] | None = None,
+    *,
+    force: bool = False,
+    product: str = "browser",
+) -> Path:
+    """Install a pinned runtime and persist its location for the client."""
+    if os.name != "nt":
+        raise OSError("wpr-runtime-155 requires Windows")
+    release = select_release(version, product=product)
+    key = str(release["firefox_major"])
+
+    if runtime_home is None:
+        local = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
+        if product == "browser":
+            destination = Path(local) / "WPR" / "runtime" / key
+        else:
+            destination = Path(local) / "WPR" / "network-runtimes" / f"firefox{key}" / release["version"]
+    else:
+        destination = Path(runtime_home)
+    destination = destination.expanduser().resolve()
+
+    if (
+        destination.exists()
+        and not force
+        and _runtime_is_current(destination, release["version"], product)
+    ):
+        _write_config(destination, product=product)
+        return destination
+
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    stage = destination.parent / f".{destination.name}.staging-{uuid.uuid4().hex}"
+    backup = None
+    moved_new = False
+    preserve_stage = False
+    try:
+        with tempfile.TemporaryDirectory(prefix="wpr-runtime-download-") as temp:
+            archive = Path(temp) / "runtime.zip"
+            request = _github_asset_request(release)
+            with (
+                _open_github(request, timeout=120) as response,
+                archive.open("wb") as output,
+            ):
+                shutil.copyfileobj(response, output)
+            actual = _sha256(archive)
+            if actual != release["sha256"]:
+                raise RuntimeError(
+                    f"runtime SHA256 mismatch: expected {release['sha256']}, got {actual}"
+                )
+            _extract_safely(archive, stage)
+
+        if not _runtime_is_current(stage, release["version"], product):
+            raise RuntimeError(
+                f"{product} runtime archive is missing its Worker or manifest"
+            )
+        if destination.exists():
+            backup = (
+                destination.parent / f"{destination.name}.backup-{uuid.uuid4().hex}"
+            )
+            destination.rename(backup)
+        try:
+            stage.replace(destination)
+            moved_new = True
+            if not _runtime_is_current(destination, release["version"], product):
+                raise RuntimeError("installed runtime failed post-swap validation")
+            _write_config(destination, product=product)
+        except BaseException:
+            try:
+                if moved_new:
+                    destination.rename(stage)
+                if backup is not None:
+                    backup.rename(destination)
+            except OSError as rollback_error:
+                preserve_stage = True
+                raise RuntimeError(
+                    f"runtime upgrade rollback incomplete; inspect {destination}, {backup}, and {stage}"
+                ) from rollback_error
+            raise
+        if backup is not None:
+            try:
+                shutil.rmtree(backup)
+            except OSError:
+                warnings.warn(
+                    f"Upgrade succeeded, but the temporary backup remains at {backup}",
+                    stacklevel=2,
+                )
+        return destination
+    finally:
+        if not preserve_stage and stage.exists():
+            shutil.rmtree(stage, ignore_errors=True)
+
+
+def _write_config(destination: Path, *, product: str = "browser") -> None:
+    path = config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    staged = path.with_name(f".{path.name}.staging-{uuid.uuid4().hex}")
+    try:
+        settings = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        settings = {}
+    manifest = json.loads((destination / "manifest.json").read_text(encoding="utf-8"))
+    major = str(manifest["version"]).split(".", 1)[0]
+    if product == "network":
+        settings.setdefault("network_runtimes", {})[str(manifest["version"])] = str(destination)
+        if NETWORK_RELEASES.get(major, {}).get("version", manifest["version"]) == manifest["version"]:
+            settings["network_runtimes"][major] = str(destination)
+    else:
+        settings.setdefault("runtimes", {})[major] = str(destination)
+        settings["runtimes"][str(manifest["version"])] = str(destination)
+        settings["runtime_home"] = str(destination)
+    try:
+        staged.write_text(
+            json.dumps(settings, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        staged.replace(path)
+    finally:
+        staged.unlink(missing_ok=True)