canarywire 0.1.0a1__py3-none-any.whl

canarywire/__init__.py

@@ -0,0 +1,3 @@
+"""canarywire: a regression test for your PII gateway's trust boundary."""
+
+__version__ = "0.1.0a1"

canarywire/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from canarywire.cli import main
+
+sys.exit(main())

canarywire/canaries.py

@@ -0,0 +1,15 @@
+"""Canaries: synthetic values in real formats, assigned per template instance for a run."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Canary:
+    """One tested value: a template's canary instance (`name`) with its type and template."""
+
+    name: str
+    value: str
+    type: str = ""
+    template: str = ""

canarywire/capture/app.py

@@ -0,0 +1,344 @@
+"""The capture server: records every upstream request and relays it to the connected runner."""
+
+from __future__ import annotations
+
+import contextlib
+import itertools
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+
+import anyio
+from starlette.applications import Starlette
+from starlette.background import BackgroundTask
+from starlette.responses import PlainTextResponse, Response, StreamingResponse
+from starlette.routing import Route, WebSocketRoute
+from starlette.websockets import WebSocketDisconnect
+
+from canarywire.capture.relay import (
+    NoRunnerError,
+    Relay,
+    RunnerLostError,
+    Session,
+    forward,
+    notify_done,
+)
+from canarywire.rpc import ChannelClosedError, Peer, RpcError
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
+    from starlette.requests import Request
+    from starlette.types import Receive, Scope, Send
+    from starlette.websockets import WebSocket
+
+    from canarywire.capture.recorder import Recorder
+
+METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]
+SESSION_CONFLICT = -32000
+POLICY_VIOLATION = 1008
+MIN_STATUS = 100
+MAX_STATUS = 599
+SUCCESS_MIN = 200
+NO_CONTENT = 204
+NOT_MODIFIED = 304
+HEADER_PAIR = 2
+PID_HEADER = "x-canarywire-pid"
+DROPPED_HEADERS = frozenset({"content-length", "transfer-encoding", "connection"})
+# Credentials the gateway sends its upstream: relayed to the runner, never written to disk.
+SECRET_HEADERS = frozenset(
+    {"authorization", "proxy-authorization", "x-api-key", "api-key", "x-goog-api-key", "cookie"}
+)
+REDACTED = "[redacted]"
+
+
+class StarletteChannel:
+    """RPC channel over the capture's side of the runner WebSocket."""
+
+    def __init__(self, websocket: WebSocket) -> None:
+        """Wrap an accepted WebSocket."""
+        self._websocket = websocket
+
+    async def send(self, text: str) -> None:
+        """Send one text frame."""
+        try:
+            await self._websocket.send_text(text)
+        except (WebSocketDisconnect, RuntimeError) as exc:
+            raise ChannelClosedError from exc
+
+    async def receive(self) -> str:
+        """Receive one text frame."""
+        try:
+            return await self._websocket.receive_text()
+        except (WebSocketDisconnect, RuntimeError, KeyError) as exc:
+            raise ChannelClosedError from exc
+
+    async def close(self) -> None:
+        """Close the socket with a policy-violation code."""
+        with contextlib.suppress(RuntimeError):
+            await self._websocket.close(code=POLICY_VIOLATION)
+
+
+@dataclass(frozen=True)
+class ResponsePlan:
+    """A validated `upstream.request` result: a plain body or a list of stream chunks."""
+
+    status: int
+    headers: list[tuple[str, str]]
+    body: str | None
+    chunks: list[tuple[str, int]] | None
+
+    @classmethod
+    def parse(cls, result: object) -> ResponsePlan:
+        """Validate a runner answer; raise ValueError if malformed."""
+        try:
+            return cls._parse(result)
+        except (AttributeError, TypeError) as exc:
+            raise ValueError(f"malformed response spec: {exc!r}") from exc
+
+    @classmethod
+    def _parse(cls, result: object) -> ResponsePlan:
+        if not isinstance(result, dict):
+            raise ValueError("response spec must be an object")
+        status = result.get("status")
+        if isinstance(status, bool) or not isinstance(status, int):
+            raise ValueError("status must be an integer")
+        if not MIN_STATUS <= status <= MAX_STATUS:
+            raise ValueError("status out of range")
+        headers = _parse_headers(result.get("headers", []))
+        body = result.get("body")
+        stream = result.get("stream")
+        if (body is None) == (stream is None):
+            raise ValueError("exactly one of body and stream is required")
+        if body is not None:
+            if not isinstance(body, str):
+                raise ValueError("body must be a string")
+            if body and not _may_have_body(status):
+                raise ValueError(f"status {status} cannot carry a body")
+            return cls(status, headers, body, None)
+        if not isinstance(stream, list):
+            raise ValueError("stream must be a list")
+        chunks: list[tuple[str, int]] = []
+        for chunk in stream:
+            data = chunk.get("data")
+            delay = chunk.get("delay_ms", 0)
+            if not isinstance(data, str) or isinstance(delay, bool) or not isinstance(delay, int):
+                raise ValueError("stream chunk needs data (string) and delay_ms (integer)")
+            chunks.append((data, max(delay, 0)))
+        if chunks and not _may_have_body(status):
+            raise ValueError(f"status {status} cannot carry a body")
+        return cls(status, headers, None, chunks)
+
+
+def _may_have_body(status: int) -> bool:
+    """1xx, 204 and 304 responses must not carry a body (RFC 9110)."""
+    return status >= SUCCESS_MIN and status not in (NO_CONTENT, NOT_MODIFIED)
+
+
+def _parse_headers(raw: object) -> list[tuple[str, str]]:
+    """Headers must be a list of [name, value] string pairs; anything else is a runner error."""
+    if not isinstance(raw, list):
+        raise ValueError("headers must be a list of [name, value] pairs")
+    headers: list[tuple[str, str]] = []
+    for pair in raw:
+        if not (
+            isinstance(pair, list)
+            and len(pair) == HEADER_PAIR
+            and all(isinstance(part, str) for part in pair)
+        ):
+            raise ValueError("each header must be a [name, value] pair of strings")
+        headers.append((pair[0], pair[1]))
+    return headers
+
+
+def create_app(recorder: Recorder) -> Starlette:
+    """Build the capture ASGI app; `recorder` receives one entry per upstream request."""
+    relay = Relay()
+    ids = itertools.count(1)
+
+    async def health(_: Request) -> Response:
+        # The pid lets `serve --detach` tell its own child from another capture on the same port.
+        return PlainTextResponse("ok", headers={PID_HEADER: str(os.getpid())})
+
+    async def reserved(_: Request) -> Response:
+        return PlainTextResponse("not found", status_code=404)
+
+    async def runner_socket(websocket: WebSocket) -> None:
+        await websocket.accept()
+        started: list[Session] = []
+
+        async def start(params: object) -> object:
+            if started:
+                raise RpcError(SESSION_CONFLICT, "session already started")
+            if relay.session is not None:
+                raise RpcError(SESSION_CONFLICT, "another runner session is active", close=True)
+            session = Session.from_params(peer, params)
+            started.append(session)
+            relay.start(session)
+            return {}
+
+        peer = Peer(StarletteChannel(websocket), {"session.start": start})
+        try:
+            await peer.run()
+        finally:
+            for session in started:
+                relay.end(session)
+
+    async def upstream(request: Request) -> Response:
+        body = (await request.body()).decode("utf-8", errors="replace")
+        params: dict[str, Any] = {
+            "id": f"u{next(ids)}",
+            "method": request.method,
+            "path": request.url.path,
+            "query": request.url.query,
+            "headers": [
+                [name.decode("latin-1").lower(), value.decode("latin-1")]
+                for name, value in request.headers.raw
+            ],
+            "body": body,
+            "json": _parse_json(body),
+        }
+        session = relay.session
+        entry: dict[str, Any] = {
+            "ts": _now(),
+            "run_id": session.run_id if session else None,
+            "seed": session.seed if session else None,
+            "id": params["id"],
+            "request": _redacted(params),
+        }
+        try:
+            plan = ResponsePlan.parse(await forward(session, params))
+        except NoRunnerError:
+            return _refused(recorder, entry, 503, "no_runner", "no canarywire runner connected")
+        except RunnerLostError:
+            return _refused(recorder, entry, 503, "runner_lost", "canarywire runner disconnected")
+        except TimeoutError:
+            return _refused(recorder, entry, 504, "timeout", "canarywire runner did not answer")
+        except (RpcError, ValueError):
+            return _refused(recorder, entry, 502, "runner_error", "canarywire runner failed")
+        if plan.body is not None:
+            return _body_response(plan, plan.body, entry, recorder, session)
+        return _stream_response(plan, plan.chunks or [], entry, recorder, session)
+
+    return Starlette(
+        routes=[
+            Route("/_canarywire/health", health),
+            WebSocketRoute("/_canarywire/ws", runner_socket),
+            Route("/_canarywire/{rest:path}", reserved, methods=METHODS),
+            Route("/{path:path}", upstream, methods=METHODS),
+        ]
+    )
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _parse_json(body: str) -> Any:
+    if not body:
+        return None
+    try:
+        return json.loads(body)
+    except (ValueError, RecursionError):
+        return None
+
+
+def _redacted(params: dict[str, Any]) -> dict[str, Any]:
+    """The request as recorded: `params` with secret header values replaced."""
+    headers = [
+        [name, REDACTED if name in SECRET_HEADERS else value] for name, value in params["headers"]
+    ]
+    return {**params, "headers": headers}
+
+
+def _refused(
+    recorder: Recorder, entry: dict[str, Any], status: int, outcome: str, message: str
+) -> Response:
+    entry["response"] = {"status": status, "body": message}
+    entry["outcome"] = outcome
+    recorder.write(entry)
+    return PlainTextResponse(message, status_code=status)
+
+
+def _with_headers(response: Response, headers: list[tuple[str, str]]) -> Response:
+    for name, value in headers:
+        if name.lower() not in DROPPED_HEADERS:
+            response.headers.append(name, value)
+    return response
+
+
+def _body_response(
+    plan: ResponsePlan,
+    body: str,
+    entry: dict[str, Any],
+    recorder: Recorder,
+    session: Session | None,
+) -> Response:
+    async def finish() -> None:
+        entry["response"] = {"status": plan.status, "headers": plan.headers, "body": body}
+        entry["outcome"] = "completed"
+        recorder.write(entry)
+        await notify_done(
+            session, {"id": entry["id"], "outcome": "completed", "delivered_chunks": 1}
+        )
+
+    response = Response(body.encode(), status_code=plan.status, background=BackgroundTask(finish))
+    return _with_headers(response, plan.headers)
+
+
+class ClosingStreamingResponse(StreamingResponse):
+    """A StreamingResponse that always closes its generator when the response task ends.
+
+    On client disconnect Starlette cancels the send loop but never closes the body iterator, so
+    the generator's `finally` (which records the outcome and sends `upstream.done`) would
+    otherwise wait for garbage collection.
+    """
+
+    def __init__(self, content: AsyncGenerator[bytes, None], status_code: int) -> None:
+        """Stream `content` with `status_code`."""
+        super().__init__(content, status_code=status_code)
+        self._generator = content
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        """Send the response, then close the generator however sending ended."""
+        try:
+            await super().__call__(scope, receive, send)
+        finally:
+            with anyio.CancelScope(shield=True):
+                await self._generator.aclose()
+
+
+def _stream_response(
+    plan: ResponsePlan,
+    chunks: list[tuple[str, int]],
+    entry: dict[str, Any],
+    recorder: Recorder,
+    session: Session | None,
+) -> Response:
+    async def content() -> AsyncGenerator[bytes, None]:
+        delivered: list[str] = []
+        outcome = "aborted"
+        try:
+            for data, delay_ms in chunks:
+                if delay_ms:
+                    await anyio.sleep(delay_ms / 1000)
+                yield data.encode()
+                delivered.append(data)
+            outcome = "completed"
+        finally:
+            with anyio.CancelScope(shield=True):
+                entry["response"] = {
+                    "status": plan.status,
+                    "headers": plan.headers,
+                    "chunks": delivered,
+                }
+                entry["outcome"] = outcome
+                recorder.write(entry)
+                await notify_done(
+                    session,
+                    {"id": entry["id"], "outcome": outcome, "delivered_chunks": len(delivered)},
+                )
+
+    return _with_headers(ClosingStreamingResponse(content(), status_code=plan.status), plan.headers)

canarywire/capture/daemon.py

@@ -0,0 +1,197 @@
+"""Capture process lifecycle: foreground serve, --detach, stop, and the pid file."""
+
+from __future__ import annotations
+
+import contextlib
+import ipaddress
+import os
+import re
+import signal
+import subprocess
+import sys
+from pathlib import Path
+
+import anyio
+import httpx
+import uvicorn
+
+from canarywire.capture.app import PID_HEADER, create_app
+from canarywire.capture.recorder import Recorder
+from canarywire.config import parse_listen
+
+STATE_DIR = Path(".canarywire")
+PID_FILE = "capture.pid"
+LOG_FILE = "capture.log"
+RECORD_FILE = "capture.jsonl"
+PS = "/bin/ps"
+POLL = 0.05
+KILL_WAIT = 5.0
+LOG_TAIL = 20
+# `canarywire serve` as whole words: `python -m canarywire serve …` or `…/bin/canarywire serve …`,
+# never `notcanarywire serve` or `canarywire serverless`.
+CAPTURE_COMMAND = re.compile(r"(?:^|[\s/])canarywire serve(?:\s|$)")
+
+
+def read_pid(path: Path) -> int | None:
+    """The pid in a pid file, or None if it is missing or unparsable."""
+    try:
+        return int(path.read_text().strip())
+    except (OSError, ValueError):
+        return None
+
+
+def is_capture(pid: int) -> bool:
+    """True if `pid` is a running `canarywire serve` (not a recycled pid, not a zombie)."""
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        pass  # exists but belongs to someone else; ps decides
+    # -ww: unlimited width. procps otherwise cuts the command at $COLUMNS, and a long venv path
+    # would push `canarywire serve` out of view.
+    result = subprocess.run(  # noqa: S603 - fixed argv, pid is an int
+        [PS, "-ww", "-o", "command=", "-p", str(pid)], capture_output=True, text=True, check=False
+    )
+    return CAPTURE_COMMAND.search(result.stdout) is not None
+
+
+def live_pid(state_dir: Path) -> int | None:
+    """The pid of the live capture named by the pid file, if any."""
+    pid = read_pid(state_dir / PID_FILE)
+    return pid if pid is not None and is_capture(pid) else None
+
+
+def serve_foreground(host: str, port: int, state_dir: Path) -> int:
+    """Run the capture in this process until SIGTERM or SIGINT."""
+    if (pid := live_pid(state_dir)) is not None:
+        print(f"canarywire: capture already running (pid {pid})", file=sys.stderr)
+        return 1
+    state_dir.mkdir(parents=True, exist_ok=True)
+    pid_file = state_dir / PID_FILE
+    pid_file.write_text(f"{os.getpid()}\n")
+    # uvicorn installs its own SIGTERM handler, shuts down gracefully, then restores the
+    # previous handler and re-raises the signal. Without a no-op handler in place first, that
+    # re-raise kills the process before this function's `finally` (and the pid file cleanup) runs.
+    previous_sigterm = signal.signal(signal.SIGTERM, lambda *_: None)
+    try:
+        uvicorn.run(
+            create_app(Recorder(state_dir / RECORD_FILE)),
+            host=host,
+            port=port,
+            log_level="warning",
+            ws="websockets-sansio",
+            lifespan="off",
+        )
+    finally:
+        signal.signal(signal.SIGTERM, previous_sigterm)
+        if read_pid(pid_file) == os.getpid():
+            pid_file.unlink(missing_ok=True)
+    return 0
+
+
+async def serve_detached(listen: str, state_dir: Path, ready_timeout: float) -> int:
+    """Start `canarywire serve` in a new session; return once it answers its health check."""
+    if (pid := live_pid(state_dir)) is not None:
+        print(f"canarywire: capture already running (pid {pid})", file=sys.stderr)
+        return 1
+    host, port = parse_listen(listen)
+    state_dir.mkdir(parents=True, exist_ok=True)
+    log_path = state_dir / LOG_FILE
+    with log_path.open("ab") as log:
+        process = subprocess.Popen(  # noqa: S603 - argv built from our own interpreter
+            [sys.executable, "-m", "canarywire", "serve", "--listen", listen],
+            stdin=subprocess.DEVNULL,
+            stdout=log,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+    url = f"http://{_probe_host(host)}:{port}/_canarywire/health"
+    if await _wait_healthy(url, process, ready_timeout):
+        print(f"canarywire: capture listening on {listen} (pid {process.pid})")
+        return 0
+    process.kill()
+    process.wait()
+    print("canarywire: capture failed to start; last log lines:", file=sys.stderr)
+    print(_tail(log_path), file=sys.stderr)
+    return 1
+
+
+async def stop(state_dir: Path, grace: float) -> int:
+    """Stop the capture named by the pid file; succeed when none is running."""
+    pid_file = state_dir / PID_FILE
+    pid = read_pid(pid_file)
+    if pid is None or not is_capture(pid):
+        pid_file.unlink(missing_ok=True)
+        print("canarywire: no capture running")
+        return 0
+    if not _signal(pid, signal.SIGTERM):
+        return 1
+    gone = await _wait_gone(pid, grace)
+    if not gone:
+        if not _signal(pid, signal.SIGKILL):
+            return 1
+        gone = await _wait_gone(pid, KILL_WAIT)
+    if not gone:
+        print(f"canarywire: capture did not stop (pid {pid})", file=sys.stderr)
+        return 1
+    pid_file.unlink(missing_ok=True)
+    print(f"canarywire: capture stopped (pid {pid})")
+    return 0
+
+
+def _probe_host(host: str) -> str:
+    with contextlib.suppress(ValueError):
+        address = ipaddress.ip_address(host)
+        if address.is_unspecified:
+            return "127.0.0.1"
+        if address.version == 6:  # noqa: PLR2004 - IP version number
+            return f"[{host}]"
+    return host
+
+
+async def _wait_healthy(url: str, process: subprocess.Popen[bytes], timeout: float) -> bool:
+    """True once *our child* answers the health check.
+
+    Another capture already bound to the port would answer too; its pid header tells it apart.
+    The child writes the pid file before it serves, so a healthy child also owns the pid file.
+    """
+    # Our own child on the address it bound: never through a proxy from the environment.
+    async with httpx.AsyncClient(timeout=1.0, trust_env=False) as client:
+        with anyio.move_on_after(timeout):
+            while process.poll() is None:
+                with contextlib.suppress(httpx.TransportError):
+                    response = await client.get(url)
+                    if response.status_code == httpx.codes.OK and response.headers.get(
+                        PID_HEADER
+                    ) == str(process.pid):
+                        return True
+                await anyio.sleep(POLL)
+    return False
+
+
+async def _wait_gone(pid: int, timeout: float) -> bool:
+    with anyio.move_on_after(timeout):
+        while is_capture(pid):
+            await anyio.sleep(POLL)
+        return True
+    return False
+
+
+def _signal(pid: int, sig: signal.Signals) -> bool:
+    """Send `sig` to `pid`; False (with a stderr message) only if permission was denied."""
+    try:
+        os.kill(pid, sig)
+    except ProcessLookupError:
+        return True
+    except PermissionError:
+        print(f"canarywire: cannot signal pid {pid}: permission denied", file=sys.stderr)
+        return False
+    return True
+
+
+def _tail(path: Path) -> str:
+    try:
+        return "\n".join(path.read_text(errors="replace").splitlines()[-LOG_TAIL:])
+    except OSError:
+        return "<no log>"

canarywire/capture/recorder.py

@@ -0,0 +1,24 @@
+"""Append-only record of every upstream exchange."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from pathlib import Path
+
+
+class Recorder:
+    """Appends one JSON line per upstream exchange; every line is flushed immediately."""
+
+    def __init__(self, path: Path) -> None:
+        """Record to `path`, creating its directory on first write."""
+        self.path = path
+
+    def write(self, entry: Mapping[str, Any]) -> None:
+        """Append one entry."""
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open("a", encoding="utf-8") as file:
+            file.write(json.dumps(entry) + "\n")

canarywire/capture/relay.py

@@ -0,0 +1,99 @@
+"""The single runner session and forwarding of upstream requests to it."""
+
+from __future__ import annotations
+
+import contextlib
+import math
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from canarywire.rpc import INVALID_PARAMS, ChannelClosedError, RpcError
+
+if TYPE_CHECKING:
+    from canarywire.rpc import Peer
+
+
+class NoRunnerError(Exception):
+    """No runner session is active."""
+
+
+class RunnerLostError(Exception):
+    """The runner's socket closed before it answered."""
+
+
+@dataclass(frozen=True)
+class Session:
+    """The connected runner: its RPC peer and what it sent in `session.start`."""
+
+    peer: Peer
+    run_id: str
+    seed: int
+    upstream_response_timeout: float
+
+    @classmethod
+    def from_params(cls, peer: Peer, params: object) -> Session:
+        """Validate `session.start` params; raise RpcError(INVALID_PARAMS) if malformed."""
+        if not isinstance(params, dict):
+            raise RpcError(INVALID_PARAMS, "params must be an object")
+        run_id = params.get("run_id")
+        seed = params.get("seed")
+        timeout = params.get("upstream_response_timeout")
+        if not isinstance(run_id, str) or not run_id:
+            raise RpcError(INVALID_PARAMS, "run_id: expected a non-empty string")
+        if isinstance(seed, bool) or not isinstance(seed, int):
+            raise RpcError(INVALID_PARAMS, "seed: expected an integer")
+        if (
+            isinstance(timeout, bool)
+            or not isinstance(timeout, int | float)
+            or not math.isfinite(timeout)  # json.loads accepts Infinity and NaN
+            or timeout <= 0
+        ):
+            raise RpcError(
+                INVALID_PARAMS, "upstream_response_timeout: expected a finite positive number"
+            )
+        return cls(peer, run_id, seed, float(timeout))
+
+
+class Relay:
+    """Holds the single active runner session."""
+
+    def __init__(self) -> None:
+        """Start with no session."""
+        self._session: Session | None = None
+
+    @property
+    def session(self) -> Session | None:
+        """The active session, if any."""
+        return self._session
+
+    def start(self, session: Session) -> None:
+        """Make `session` the active one."""
+        self._session = session
+
+    def end(self, session: Session) -> None:
+        """Forget `session` if it is still the active one."""
+        if self._session is session:
+            self._session = None
+
+
+async def forward(session: Session | None, params: dict[str, Any]) -> Any:
+    """Ask the runner how to answer an upstream request.
+
+    Raises NoRunnerError, RunnerLostError, TimeoutError or RpcError.
+    """
+    if session is None:
+        raise NoRunnerError
+    try:
+        return await session.peer.call(
+            "upstream.request", params, timeout=session.upstream_response_timeout
+        )
+    except ChannelClosedError as exc:
+        raise RunnerLostError from exc
+
+
+async def notify_done(session: Session | None, params: dict[str, Any]) -> None:
+    """Tell the runner how delivery of its answer ended; ignored if it is gone."""
+    if session is None:
+        return
+    with contextlib.suppress(ChannelClosedError):
+        await session.peer.notify("upstream.done", params)