pytest-zygote 0.1.0__tar.gz

pytest_zygote-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,76 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        include:
+          # The plugin forks, so macOS is a separate platform, not just another interpreter.
+          - os: macos-latest
+            python: "3.12"
+          # Oldest supported pytest, against the newest interpreter that accepts it.
+          - os: ubuntu-latest
+            python: "3.11"
+            pytest: "==7.0.0"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - run: pip install -e ".[test]" "pytest${{ matrix.pytest }}"
+      - run: pytest -q
+        env:
+          OBJC_DISABLE_INITIALIZE_FORK_SAFETY: YES
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install ruff
+      - run: ruff check .
+      - run: ruff format --check .
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [test, lint]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build twine
+      - run: python -m build
+      - run: twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    # Tags only. Uses PyPI trusted publishing, so there is no token to store.
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    needs: [build]
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

pytest_zygote-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.venv/
+dist/
+build/
+*.egg-info/

pytest_zygote-0.1.0/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+## 0.1.0
+
+First release.
+
+- `--workers N` (or `auto`) runs tests in workers forked from the collecting process.
+- `gc.freeze()` before the fork, so the garbage collector stops dirtying shared pages.
+- Workers that die mid-test are reported against the test they were running and replaced;
+  `--workers-max-restarts` caps that.
+- `--workers-no-freeze` for measuring what the freeze is worth.

pytest_zygote-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pedro Fuentes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pytest_zygote-0.1.0/PKG-INFO

@@ -0,0 +1,170 @@
+Metadata-Version: 2.5
+Name: pytest-zygote
+Version: 0.1.0
+Summary: Run pytest in parallel workers forked from one warm process, so the imports are paid once
+Project-URL: Homepage, https://github.com/pedrofuentes79/pytest-zygote
+Project-URL: Issues, https://github.com/pedrofuentes79/pytest-zygote/issues
+Project-URL: Changelog, https://github.com/pedrofuentes79/pytest-zygote/blob/main/CHANGELOG.md
+Author-email: Pedro Fuentes <pedrofuentes7799@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: fork,memory,parallel,plugin,pytest,xdist
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.9
+Requires-Dist: pytest>=7.0
+Provides-Extra: test
+Requires-Dist: pytest-xdist>=3.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pytest-zygote
+
+Run pytest in parallel without paying for the imports N times.
+
+```bash
+pip install pytest-zygote
+pytest --workers 4
+```
+
+Tested on Python 3.9-3.14 and pytest 7.0-9.1.
+
+## The problem
+
+A pytest-xdist worker is a fresh interpreter. execnet's popen gateway starts each one as
+`python -u -B -c ...`, so every worker imports your whole dependency tree from scratch and
+keeps its own copy in memory. If importing your app costs 400 MB — a few ML or LLM libraries
+will do it — then `-n 4` costs 2 GB before a single assertion runs, and almost none of it is
+shareable: it is private anonymous heap, not file-backed pages.
+
+That is what puts a ceiling on parallelism in a container. You want more workers because your
+tests wait on the network; you can't have them because each one costs another 400 MB.
+
+## What this does
+
+Collect in the parent, `gc.freeze()`, then `fork()` a worker per slot. Workers inherit the
+already-imported heap as copy-on-write pages, so the imports are paid once for the whole run.
+
+Measured on a suite of 1089 tests whose imports cost ~430 MB (litellm, pydantic-ai, pandas,
+matplotlib), peak memory of the whole container:
+
+| | wall | peak memory |
+|---|---|---|
+| one process | 28.3 s | 510 MB |
+| `pytest -n 4` (xdist) | 21.3 s | 1945 MB |
+| `pytest --workers 4` | 16.1 s | 810 MB |
+| `pytest --workers 8` | 14.7 s | 1004 MB |
+| `pytest --workers 12` | 13.8 s | 1104 MB |
+
+**A worker costs ~46 MB here; an xdist worker costs ~490 MB.** It is also faster than xdist,
+because a forked worker starts warm and skips the import that each xdist worker repeats.
+
+The saving is real only because of `gc.freeze()`. CPython's generational collector writes to
+every object header when it runs, which dirties copy-on-write pages and un-shares them one
+collection at a time. `gc.freeze()` moves everything imported so far into a permanent
+generation the collector never walks. Without it a worker's private dirty memory here is
+~195 MB instead of ~22 MB. The technique is Instagram's, upstreamed as `gc.freeze()` in
+Python 3.7.
+
+## Compared with pytest-xdist
+
+The difference is entirely in how a worker is born.
+
+| | pytest-xdist | pytest-zygote |
+|---|---|---|
+| how a worker starts | a new interpreter (`python -u -B -c ...` via execnet) | `fork()` of the process that just collected |
+| who imports your code | every worker, separately | the controller, once |
+| who collects | every worker repeats collection | the controller only |
+| memory per worker* | ~490 MB | ~46 MB |
+| time before the first test | each worker pays the full import | workers start warm |
+| distribution | `load`, `loadscope`, `loadfile`, `loadgroup`, `worksteal`, `each` | dynamic, per test, with a one-test lookahead |
+| workers on other machines | yes, over ssh or a socket | no, this box only |
+| Windows | yes | no — `fork()` only, so Linux and macOS |
+| `--looponfail` | yes | no |
+| a worker that dies | `--max-worker-restart` | replaced, and the test it killed is reported as failed |
+| maturity | since 2010, used everywhere | new |
+
+\* On the suite in the table above, whose imports cost ~430 MB. If your imports are cheap,
+both numbers collapse and the distinction stops mattering.
+
+**Use xdist** when you need workers on other machines, a `--dist` mode that pins related tests
+to one worker, Windows, or simply when it already works for you. It is the mature, general
+answer and this plugin does not try to replace it.
+
+**Use zygote** when your imports are expensive and memory is what caps your parallelism —
+a container with a hard limit, CI with a small runner, a laptop that swaps at `-n 8`. The
+worse your import cost, the bigger the gap. The suite above needs 1945 MB to run four xdist
+workers; it runs twelve forked workers in 1104 MB.
+
+They are mutually exclusive in one run: passing `--workers` and `-n` together is an error,
+not a silent choice of one.
+
+## Options
+
+| | |
+|---|---|
+| `--workers N` | run tests in N forked workers, or `auto` for one per core. Default `0`, which leaves pytest alone |
+| `--workers-max-restarts N` | cap how many dead workers get replaced. Default: one per worker; `0` disables |
+| `--workers-no-freeze` | skip `gc.freeze()`. Diagnostic — it costs most of the saving |
+
+The worker count is capped at the number of collected tests. Each worker gets
+`PYTEST_ZYGOTE_WORKER=gw<n>` in its environment, like xdist's `PYTEST_XDIST_WORKER`.
+
+## When a worker dies
+
+A worker killed mid-test — in a container, usually the memory cgroup — would otherwise take
+its test down silently, and with `-q` you would not even learn which one. The controller sees
+the socket close, reports that test as failed with the signal name, puts the worker's queued
+work back, and forks a replacement:
+
+```
+zygote worker gw0 killed by signal 9 (SIGKILL) while running this test.
+In a container SIGKILL here is almost always the memory cgroup.
+```
+
+## How reports get back
+
+Workers run `runtestprotocol(item, log=False)` and stream their reports to the controller as
+length-prefixed JSON over a `socketpair()`. Only the controller calls
+`pytest_runtest_logreport`, so the terminal reporter, `--junit-xml`, coverage and any plugin
+of your own that listens to reports all see one ordered stream from one process. Nothing needs
+locking and nothing is written twice.
+
+Each worker is also told the test *after* the one it is running. pytest finalizes a fixture
+when the next test no longer needs it, so without that lookahead every module- and
+class-scoped fixture would tear down after each test.
+
+## Limits
+
+- **Needs `os.fork()`** — Linux and macOS. Not Windows.
+- **Mutually exclusive with xdist's `-n`.** Pass one or the other.
+- **`--workers` is also pytest-parallel's flag.** That plugin was archived in May 2024; if you
+  have both installed, argparse fails at startup and you have to drop one.
+- **Scheduling is per test**, so a module-scoped fixture is set up once per worker that draws a
+  test from that module — the same trade xdist makes without `--dist loadfile`.
+- **`-x` can overshoot** by a test or two, since several are in flight when the first failure
+  lands.
+- **Your conftest must not open things at import time.** A module-level DB connection, event
+  loop or thread pool created during collection is inherited by every fork. Anything created
+  inside a fixture is fine, because that runs in the worker.
+
+## macOS
+
+CI runs the suite on macOS, but the numbers above and the day-to-day use are Linux. macOS
+treats forking a process that has already initialized certain Objective-C frameworks as
+unsafe; if you hit `__NSPlaceholderDictionary initialize` crashes, set
+`OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES`, the same workaround xdist users apply.
+
+## License
+
+MIT.

pytest_zygote-0.1.0/README.md

@@ -0,0 +1,140 @@
+# pytest-zygote
+
+Run pytest in parallel without paying for the imports N times.
+
+```bash
+pip install pytest-zygote
+pytest --workers 4
+```
+
+Tested on Python 3.9-3.14 and pytest 7.0-9.1.
+
+## The problem
+
+A pytest-xdist worker is a fresh interpreter. execnet's popen gateway starts each one as
+`python -u -B -c ...`, so every worker imports your whole dependency tree from scratch and
+keeps its own copy in memory. If importing your app costs 400 MB — a few ML or LLM libraries
+will do it — then `-n 4` costs 2 GB before a single assertion runs, and almost none of it is
+shareable: it is private anonymous heap, not file-backed pages.
+
+That is what puts a ceiling on parallelism in a container. You want more workers because your
+tests wait on the network; you can't have them because each one costs another 400 MB.
+
+## What this does
+
+Collect in the parent, `gc.freeze()`, then `fork()` a worker per slot. Workers inherit the
+already-imported heap as copy-on-write pages, so the imports are paid once for the whole run.
+
+Measured on a suite of 1089 tests whose imports cost ~430 MB (litellm, pydantic-ai, pandas,
+matplotlib), peak memory of the whole container:
+
+| | wall | peak memory |
+|---|---|---|
+| one process | 28.3 s | 510 MB |
+| `pytest -n 4` (xdist) | 21.3 s | 1945 MB |
+| `pytest --workers 4` | 16.1 s | 810 MB |
+| `pytest --workers 8` | 14.7 s | 1004 MB |
+| `pytest --workers 12` | 13.8 s | 1104 MB |
+
+**A worker costs ~46 MB here; an xdist worker costs ~490 MB.** It is also faster than xdist,
+because a forked worker starts warm and skips the import that each xdist worker repeats.
+
+The saving is real only because of `gc.freeze()`. CPython's generational collector writes to
+every object header when it runs, which dirties copy-on-write pages and un-shares them one
+collection at a time. `gc.freeze()` moves everything imported so far into a permanent
+generation the collector never walks. Without it a worker's private dirty memory here is
+~195 MB instead of ~22 MB. The technique is Instagram's, upstreamed as `gc.freeze()` in
+Python 3.7.
+
+## Compared with pytest-xdist
+
+The difference is entirely in how a worker is born.
+
+| | pytest-xdist | pytest-zygote |
+|---|---|---|
+| how a worker starts | a new interpreter (`python -u -B -c ...` via execnet) | `fork()` of the process that just collected |
+| who imports your code | every worker, separately | the controller, once |
+| who collects | every worker repeats collection | the controller only |
+| memory per worker* | ~490 MB | ~46 MB |
+| time before the first test | each worker pays the full import | workers start warm |
+| distribution | `load`, `loadscope`, `loadfile`, `loadgroup`, `worksteal`, `each` | dynamic, per test, with a one-test lookahead |
+| workers on other machines | yes, over ssh or a socket | no, this box only |
+| Windows | yes | no — `fork()` only, so Linux and macOS |
+| `--looponfail` | yes | no |
+| a worker that dies | `--max-worker-restart` | replaced, and the test it killed is reported as failed |
+| maturity | since 2010, used everywhere | new |
+
+\* On the suite in the table above, whose imports cost ~430 MB. If your imports are cheap,
+both numbers collapse and the distinction stops mattering.
+
+**Use xdist** when you need workers on other machines, a `--dist` mode that pins related tests
+to one worker, Windows, or simply when it already works for you. It is the mature, general
+answer and this plugin does not try to replace it.
+
+**Use zygote** when your imports are expensive and memory is what caps your parallelism —
+a container with a hard limit, CI with a small runner, a laptop that swaps at `-n 8`. The
+worse your import cost, the bigger the gap. The suite above needs 1945 MB to run four xdist
+workers; it runs twelve forked workers in 1104 MB.
+
+They are mutually exclusive in one run: passing `--workers` and `-n` together is an error,
+not a silent choice of one.
+
+## Options
+
+| | |
+|---|---|
+| `--workers N` | run tests in N forked workers, or `auto` for one per core. Default `0`, which leaves pytest alone |
+| `--workers-max-restarts N` | cap how many dead workers get replaced. Default: one per worker; `0` disables |
+| `--workers-no-freeze` | skip `gc.freeze()`. Diagnostic — it costs most of the saving |
+
+The worker count is capped at the number of collected tests. Each worker gets
+`PYTEST_ZYGOTE_WORKER=gw<n>` in its environment, like xdist's `PYTEST_XDIST_WORKER`.
+
+## When a worker dies
+
+A worker killed mid-test — in a container, usually the memory cgroup — would otherwise take
+its test down silently, and with `-q` you would not even learn which one. The controller sees
+the socket close, reports that test as failed with the signal name, puts the worker's queued
+work back, and forks a replacement:
+
+```
+zygote worker gw0 killed by signal 9 (SIGKILL) while running this test.
+In a container SIGKILL here is almost always the memory cgroup.
+```
+
+## How reports get back
+
+Workers run `runtestprotocol(item, log=False)` and stream their reports to the controller as
+length-prefixed JSON over a `socketpair()`. Only the controller calls
+`pytest_runtest_logreport`, so the terminal reporter, `--junit-xml`, coverage and any plugin
+of your own that listens to reports all see one ordered stream from one process. Nothing needs
+locking and nothing is written twice.
+
+Each worker is also told the test *after* the one it is running. pytest finalizes a fixture
+when the next test no longer needs it, so without that lookahead every module- and
+class-scoped fixture would tear down after each test.
+
+## Limits
+
+- **Needs `os.fork()`** — Linux and macOS. Not Windows.
+- **Mutually exclusive with xdist's `-n`.** Pass one or the other.
+- **`--workers` is also pytest-parallel's flag.** That plugin was archived in May 2024; if you
+  have both installed, argparse fails at startup and you have to drop one.
+- **Scheduling is per test**, so a module-scoped fixture is set up once per worker that draws a
+  test from that module — the same trade xdist makes without `--dist loadfile`.
+- **`-x` can overshoot** by a test or two, since several are in flight when the first failure
+  lands.
+- **Your conftest must not open things at import time.** A module-level DB connection, event
+  loop or thread pool created during collection is inherited by every fork. Anything created
+  inside a fixture is fine, because that runs in the worker.
+
+## macOS
+
+CI runs the suite on macOS, but the numbers above and the day-to-day use are Linux. macOS
+treats forking a process that has already initialized certain Objective-C frameworks as
+unsafe; if you hit `__NSPlaceholderDictionary initialize` crashes, set
+`OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES`, the same workaround xdist users apply.
+
+## License
+
+MIT.

pytest_zygote-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "pytest-zygote"
+version = "0.1.0"
+description = "Run pytest in parallel workers forked from one warm process, so the imports are paid once"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"
+authors = [{ name = "Pedro Fuentes", email = "pedrofuentes7799@gmail.com" }]
+keywords = ["pytest", "parallel", "fork", "memory", "xdist", "plugin"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Framework :: Pytest",
+    "Intended Audience :: Developers",
+    "Operating System :: MacOS",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = ["pytest>=7.0"]
+
+[project.optional-dependencies]
+test = ["pytest-xdist>=3.0"]
+
+[project.urls]
+Homepage = "https://github.com/pedrofuentes79/pytest-zygote"
+Issues = "https://github.com/pedrofuentes79/pytest-zygote/issues"
+Changelog = "https://github.com/pedrofuentes79/pytest-zygote/blob/main/CHANGELOG.md"
+
+[project.entry-points.pytest11]
+zygote = "pytest_zygote.plugin"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pytest_zygote"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 103
+# The plugin runs on 3.9; 3.14's PEP 758 would let the formatter strip the parentheses
+# from `except (A, B)`, which every older Python rejects.
+target-version = "py39"
+
+[tool.ruff.lint]
+# Pinned rather than left to the default, so CI lints the same as any contributor's machine.
+select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "RUF"]

pytest_zygote-0.1.0/src/pytest_zygote/plugin.py

@@ -0,0 +1,402 @@
+"""Run tests in workers forked from one warmed-up process instead of fresh interpreters.
+
+pytest-xdist starts each worker as a new interpreter (execnet's popen gateway), so every
+worker re-imports the whole dependency tree and pays its full resident cost. Here collection
+and every import happen once, in the controller; workers are ``fork()`` children that share
+that heap copy-on-write, so N workers cost roughly one worker plus what the tests allocate.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import gc
+import json
+import os
+import selectors
+import signal
+import socket
+import struct
+import sys
+import traceback
+from typing import Any
+
+import pytest
+from _pytest.reports import TestReport
+from _pytest.runner import runtestprotocol
+
+_HEADER = struct.Struct("!I")
+
+
+class _PeerGone(OSError):
+    """The other end of a worker socket closed; an OSError so one except covers socket errors too."""
+
+
+def _count(value: str) -> int:
+    """Worker count, or 'auto' for one per core — the spelling xdist and pytest-parallel both take."""
+    if value == "auto":
+        return os.cpu_count() or 1
+    try:
+        count = int(value)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"expected a number or 'auto', got {value!r}") from None
+    if count < 0:
+        raise argparse.ArgumentTypeError("cannot be negative")
+    return count
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("zygote", "forked workers that share the controller's memory")
+    group.addoption(
+        "--workers",
+        metavar="N",
+        type=_count,
+        default=0,
+        help="Run tests in N workers forked from this process, or 'auto' for one per core.",
+    )
+    group.addoption(
+        "--workers-no-freeze",
+        action="store_true",
+        help="Skip gc.freeze() before forking; diagnostic, costs most of the memory saving.",
+    )
+    group.addoption(
+        "--workers-max-restarts",
+        metavar="N",
+        type=int,
+        default=-1,
+        help="Replace at most N workers that die mid-run (default: one per worker; 0 disables).",
+    )
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtestloop(session: pytest.Session) -> object | None:
+    config = session.config
+    workers: int = config.getoption("workers") or 0
+    if workers < 1 or config.option.collectonly:
+        return None
+    if not hasattr(os, "fork"):
+        raise pytest.UsageError("--workers needs os.fork(), which this platform lacks")
+    if config.getoption("numprocesses", None):
+        raise pytest.UsageError("--workers and -n are alternatives; pass only one")
+    if session.testsfailed and not config.option.continue_on_collection_errors:
+        raise session.Failed(f"{session.testsfailed} error(s) during collection")
+    if session.items:
+        _Controller(session, workers).run()
+    return True
+
+
+# --------------------------------------------------------------------------- wire
+
+
+def _send(sock: socket.socket, msg: dict[str, Any]) -> None:
+    payload = json.dumps(msg).encode()
+    sock.sendall(_HEADER.pack(len(payload)) + payload)
+
+
+def _recv_one(sock: socket.socket) -> dict[str, Any]:
+    """Block until one whole message has arrived; raise _PeerGone if the peer closed first."""
+
+    def exact(n: int) -> bytes:
+        buf = b""
+        while len(buf) < n:
+            chunk = sock.recv(n - len(buf))
+            if not chunk:
+                raise _PeerGone
+            buf += chunk
+        return buf
+
+    (size,) = _HEADER.unpack(exact(_HEADER.size))
+    return json.loads(exact(size))
+
+
+class _Reader:
+    """Reassembles length-prefixed messages from a non-blocking socket."""
+
+    def __init__(self, sock: socket.socket) -> None:
+        self._sock = sock
+        self._buf = bytearray()
+
+    def read(self) -> list[dict[str, Any]]:
+        chunk = self._sock.recv(1 << 16)
+        if not chunk:
+            raise _PeerGone
+        self._buf += chunk
+        out = []
+        while len(self._buf) >= _HEADER.size:
+            (size,) = _HEADER.unpack_from(self._buf)
+            end = _HEADER.size + size
+            if len(self._buf) < end:
+                break
+            out.append(json.loads(bytes(self._buf[_HEADER.size : end])))
+            del self._buf[:end]
+        return out
+
+
+# --------------------------------------------------------------------------- controller
+
+
+class _Worker:
+    def __init__(self, wid: int, pid: int, sock: socket.socket) -> None:
+        self.id = wid
+        self.pid = pid
+        self.sock = sock
+        self.reader = _Reader(sock)
+        self.running: int | None = None  # item index in flight, for crash attribution
+        self.committed: int | None = None  # item promised as this worker's next
+        self.alive = True
+
+
+class _Controller:
+    def __init__(self, session: pytest.Session, workers: int) -> None:
+        self.session = session
+        self.config = session.config
+        self.items = session.items
+        self.count = max(1, min(workers, len(session.items)))
+        self.queue = list(range(len(session.items)))
+        self.workers: list[_Worker] = []
+        restarts = self.config.getoption("workers_max_restarts")
+        restarts = self.count if restarts is None or restarts < 0 else int(restarts)
+        self.restarts_left: int = restarts
+        self._capman: Any = None
+        self._next_id = 0
+
+    def run(self) -> None:
+        capman: Any = self.config.pluginmanager.getplugin("capturemanager")
+        self._capman = capman
+        if capman:
+            # Fork with the real stdout/stderr fds restored: a child that inherited the
+            # controller's capture would write into its temp files, interleaving with
+            # every other child.
+            capman.suspend_global_capture(in_=True)
+        if not self.config.getoption("workers_no_freeze"):
+            gc.collect()
+            gc.freeze()
+        self._announce()
+        self._spawn(capman)
+        try:
+            self._dispatch()
+        except BaseException:
+            self._signal_all(signal.SIGKILL)
+            raise
+        finally:
+            self._reap()
+            if capman:
+                capman.resume_global_capture()
+
+    def _announce(self) -> None:
+        reporter: Any = self.config.pluginmanager.getplugin("terminalreporter")
+        if reporter is None:
+            return
+        frozen = (
+            ""
+            if self.config.getoption("workers_no_freeze")
+            else f", {gc.get_freeze_count()} objects frozen"
+        )
+        plural = "worker" if self.count == 1 else "workers"
+        reporter.write_line(f"zygote: {self.count} {plural} forked from pid {os.getpid()}{frozen}")
+
+    def _spawn(self, capman: Any) -> None:
+        for _ in range(self.count):
+            self.workers.append(self._fork_one(capman))
+
+    def _fork_one(self, capman: Any) -> _Worker:
+        wid = self._next_id
+        self._next_id += 1
+        ours, theirs = socket.socketpair()
+        pid = os.fork()
+        if pid == 0:
+            ours.close()
+            os._exit(_worker_main(self.session, theirs, wid, capman))
+        theirs.close()
+        ours.setblocking(False)
+        return _Worker(wid, pid, ours)
+
+    def _dispatch(self) -> None:
+        selector = selectors.DefaultSelector()
+        for worker in self.workers:
+            selector.register(worker.sock, selectors.EVENT_READ, worker)
+        live = len(self.workers)
+        while live:
+            for key, _ in selector.select():
+                worker = key.data
+                try:
+                    messages = worker.reader.read()
+                except OSError:
+                    selector.unregister(worker.sock)
+                    live -= 1
+                    self._bury(worker)
+                    replacement = self._replace(worker)
+                    if replacement is not None:
+                        selector.register(replacement.sock, selectors.EVENT_READ, replacement)
+                        live += 1
+                    continue
+                for message in messages:
+                    self._handle(worker, message)
+                if not worker.alive:
+                    selector.unregister(worker.sock)
+                    live -= 1
+        selector.close()
+
+    def _replace(self, dead: _Worker) -> _Worker | None:
+        """Fork a fresh worker so a killed one does not strand the tests still queued."""
+        if not self.queue or self._should_stop() or self.restarts_left <= 0:
+            return None
+        self.restarts_left -= 1
+        worker = self._fork_one(self._capman)
+        self.workers.append(worker)
+        reporter: Any = self.config.pluginmanager.getplugin("terminalreporter")
+        if reporter is not None:
+            reporter.write_line(f"zygote: replaced dead gw{dead.id} with gw{worker.id}")
+        return worker
+
+    def _handle(self, worker: _Worker, message: dict[str, Any]) -> None:
+        hook = self.config.hook
+        kind = message["t"]
+        if kind == "idle":
+            self._assign(worker)
+        elif kind == "start":
+            hook.pytest_runtest_logstart(nodeid=message["nodeid"], location=tuple(message["location"]))
+        elif kind == "report":
+            report = hook.pytest_report_from_serializable(config=self.config, data=message["data"])
+            hook.pytest_runtest_logreport(report=_retuple(report))
+        elif kind == "finish":
+            worker.running = None
+            hook.pytest_runtest_logfinish(
+                nodeid=message["nodeid"], location=tuple(message["location"])
+            )
+        elif kind == "bye":
+            worker.alive = False
+
+    def _assign(self, worker: _Worker) -> None:
+        """Hand out the next test plus the one after it, so teardown knows its nextitem.
+
+        pytest finalizes a fixture when the following test no longer needs it, which means
+        the worker must know that test before it tears down. Committing one item ahead buys
+        that without giving up dynamic scheduling.
+        """
+        if self._should_stop():
+            _send(worker.sock, {"t": "stop"})
+            return
+        index = worker.committed if worker.committed is not None else self._take()
+        if index is None:
+            _send(worker.sock, {"t": "stop"})
+            return
+        worker.committed = None if self._should_stop() else self._take()
+        worker.running = index
+        _send(worker.sock, {"t": "run", "i": index, "next": worker.committed})
+
+    def _take(self) -> int | None:
+        return self.queue.pop(0) if self.queue else None
+
+    def _should_stop(self) -> bool:
+        return bool(self.session.shouldstop or self.session.shouldfail)
+
+    def _bury(self, worker: _Worker) -> None:
+        """Report the in-flight test as failed, so an OOM-killed worker never loses one silently."""
+        worker.alive = False
+        status = 0
+        with contextlib.suppress(ChildProcessError):
+            _, status = os.waitpid(worker.pid, 0)
+        if worker.committed is not None:
+            self.queue.insert(0, worker.committed)
+            worker.committed = None
+        if worker.running is None:
+            return
+        item = self.items[worker.running]
+        worker.running = None
+        self.config.hook.pytest_runtest_logreport(report=_crash_report(item, worker.id, status))
+
+    def _signal_all(self, sig: int) -> None:
+        for worker in self.workers:
+            with contextlib.suppress(ProcessLookupError):
+                os.kill(worker.pid, sig)
+
+    def _reap(self) -> None:
+        for worker in self.workers:
+            with contextlib.suppress(ChildProcessError):
+                os.waitpid(worker.pid, 0)
+            worker.sock.close()
+
+
+def _retuple(report: Any) -> Any:
+    """Undo JSON's list-for-tuple substitution; the terminal reporter asserts on a skip's tuple."""
+    if isinstance(report.longrepr, list):
+        report.longrepr = tuple(report.longrepr)
+    if isinstance(report.location, list):
+        report.location = tuple(report.location)
+    return report
+
+
+def _crash_report(item: pytest.Item, wid: int, status: int) -> TestReport:
+    signum = status & 0x7F
+    how = (
+        f"killed by signal {signum} ({signal.Signals(signum).name})"
+        if signum
+        else f"exited with code {status >> 8}"
+    )
+    hint = (
+        "\nIn a container SIGKILL here is almost always the memory cgroup."
+        if signum == signal.SIGKILL
+        else ""
+    )
+    return TestReport(
+        nodeid=item.nodeid,
+        location=item.location,
+        keywords={},
+        outcome="failed",
+        longrepr=f"zygote worker gw{wid} {how} while running this test.{hint}",
+        when="call",
+    )
+
+
+# --------------------------------------------------------------------------- worker
+
+
+def _worker_main(session: pytest.Session, sock: socket.socket, wid: int, capman: Any) -> int:
+    config = session.config
+    try:
+        reporter = config.pluginmanager.getplugin("terminalreporter")
+        if reporter is not None:
+            config.pluginmanager.unregister(reporter)
+        if capman:
+            capman.stop_global_capturing()
+            capman.start_global_capturing()
+            capman.suspend_global_capture(in_=True)
+        os.environ["PYTEST_ZYGOTE_WORKER"] = f"gw{wid}"
+        _worker_loop(session, sock)
+        return 0
+    # A worker must report why it stopped, whatever stopped it.
+    except BaseException:
+        traceback.print_exc(file=sys.stderr)
+        return 1
+    finally:
+        # os._exit() skips stdio flushing, and under -s the worker's own prints are still buffered.
+        for stream in (sys.stdout, sys.stderr):
+            with contextlib.suppress(OSError, ValueError):
+                stream.flush()
+        with contextlib.suppress(OSError):
+            sock.close()
+
+
+def _worker_loop(session: pytest.Session, sock: socket.socket) -> None:
+    config = session.config
+    items = session.items
+    while True:
+        _send(sock, {"t": "idle"})
+        try:
+            message = _recv_one(sock)
+        except _PeerGone:
+            return
+        if message["t"] == "stop":
+            _send(sock, {"t": "bye"})
+            return
+        item = items[message["i"]]
+        nextitem = items[message["next"]] if message["next"] is not None else None
+        location = list(item.location)
+        _send(sock, {"t": "start", "nodeid": item.nodeid, "location": location})
+        # log=False keeps pytest_runtest_logreport out of the worker: the controller is the
+        # only process that fires it, so terminal output and result files stay single-writer.
+        for report in runtestprotocol(item, log=False, nextitem=nextitem):
+            data = config.hook.pytest_report_to_serializable(config=config, report=report)
+            _send(sock, {"t": "report", "data": data})
+        _send(sock, {"t": "finish", "nodeid": item.nodeid, "location": location})

pytest_zygote-0.1.0/tests/conftest.py

@@ -0,0 +1 @@
+pytest_plugins = ["pytester"]

pytest_zygote-0.1.0/tests/test_plugin.py

@@ -0,0 +1,193 @@
+"""Zygote's own suite: every case runs a real pytest in a subprocess and checks what it reported.
+
+The plugin forks, so these cannot run in-process — a nested fork inside the suite that is
+already under test tells you nothing about either.
+"""
+
+import importlib.util
+import os
+
+import pytest
+
+
+@pytest.fixture
+def run(pytester):
+    def go(*args, workers=2):
+        return pytester.runpytest_subprocess(f"--workers={workers}", *args)
+
+    return go
+
+
+def test_matches_a_serial_run(pytester, run):
+    pytester.makepyfile(
+        test_mixed="""
+        import pytest
+
+        @pytest.mark.parametrize("n", range(12))
+        def test_passes(n): assert n >= 0
+
+        def test_fails(): assert False
+
+        @pytest.mark.skip(reason="deliberate")
+        def test_skipped(): pass
+
+        @pytest.mark.xfail(reason="deliberate")
+        def test_xfails(): assert False
+        """
+    )
+    serial = pytester.runpytest_subprocess()
+    forked = run("-rsxX")
+    assert forked.parseoutcomes() == serial.parseoutcomes()
+    forked.assert_outcomes(passed=12, failed=1, skipped=1, xfailed=1)
+
+
+def test_failure_keeps_traceback_and_captured_output(pytester, run):
+    pytester.makepyfile(
+        test_loud="""
+        def test_loud():
+            print("MARKER-STDOUT")
+            raise RuntimeError("MARKER-BOOM")
+        """
+    )
+    result = run(workers=1)
+    result.stdout.fnmatch_lines(
+        ["*RuntimeError: MARKER-BOOM*", "*Captured stdout call*", "MARKER-STDOUT"]
+    )
+
+
+def test_skip_reason_survives_serialization(pytester, run):
+    """JSON has no tuples, and the terminal reporter asserts a skip's longrepr is one."""
+    pytester.makepyfile(
+        test_skips="""
+        import pytest
+
+        @pytest.mark.skip(reason="MARKER-REASON")
+        def test_skipped(): pass
+        """
+    )
+    result = run("-rs", workers=1)
+    result.stdout.fnmatch_lines(["*MARKER-REASON*"])
+    assert "INTERNALERROR" not in result.stdout.str()
+
+
+def test_dead_worker_is_reported_not_lost(pytester, run):
+    pytester.makepyfile(
+        test_crash="""
+        import os, signal
+
+        def test_before(): assert True
+
+        def test_kills_worker():
+            os.kill(os.getpid(), signal.SIGKILL)
+
+        def test_after(): assert True
+        """
+    )
+    result = run(workers=1)
+    result.assert_outcomes(passed=2, failed=1)
+    result.stdout.fnmatch_lines(["*killed by signal 9*"])
+
+
+def test_dead_worker_is_replaced_so_the_queue_drains(pytester, run):
+    followers = "\n".join(f"def test_after_{i}(): assert True" for i in range(8))
+    pytester.makepyfile(
+        test_crash="import os, signal\n"
+        "def test_kills_worker():\n"
+        "    os.kill(os.getpid(), signal.SIGKILL)\n" + followers
+    )
+    result = run(workers=1)
+    result.assert_outcomes(passed=8, failed=1)
+    result.stdout.fnmatch_lines(["*replaced dead gw0*"])
+
+
+def test_restarts_can_be_disabled(pytester, run):
+    pytester.makepyfile(
+        test_crash="""
+        import os, signal
+
+        def test_kills_worker():
+            os.kill(os.getpid(), signal.SIGKILL)
+
+        def test_never_reached(): assert True
+        """
+    )
+    result = run("--workers-max-restarts=0", workers=1)
+    result.assert_outcomes(failed=1)
+
+
+def test_a_scoped_fixture_is_torn_down_once_per_worker(pytester, run):
+    """One worker means one setup for the module fixture, and a teardown only after the last test."""
+    pytester.makepyfile(
+        test_scoped="""
+        import pytest
+
+        @pytest.fixture(scope="module")
+        def resource():
+            print("SETUP-MARKER")
+            yield
+            print("TEARDOWN-MARKER")
+
+        def test_one(resource): assert True
+        def test_two(resource): assert True
+        def test_three(resource): assert True
+        """
+    )
+    result = run("-s", workers=1)
+    assert result.stdout.str().count("SETUP-MARKER") == 1
+    assert result.stdout.str().count("TEARDOWN-MARKER") == 1
+
+
+def test_maxfail_stops_the_run(pytester, run):
+    pytester.makepyfile(
+        test_many="""
+        import pytest
+
+        @pytest.mark.parametrize("n", range(40))
+        def test_fails(n): assert False
+        """
+    )
+    result = run("--maxfail=2", workers=2)
+    assert result.parseoutcomes()["failed"] < 40
+
+
+def test_zero_workers_leaves_pytest_alone(pytester):
+    pytester.makepyfile(test_plain="def test_ok(): assert True")
+    result = pytester.runpytest_subprocess()
+    result.assert_outcomes(passed=1)
+    assert "zygote:" not in result.stdout.str()
+
+
+def test_auto_means_one_worker_per_core(pytester):
+    """The count is also capped at the number of tests, so give it more tests than cores."""
+    cores = os.cpu_count() or 1
+    pytester.makepyfile(
+        test_plenty="\n".join(f"def test_{i}(): assert True" for i in range(cores * 2))
+    )
+    result = pytester.runpytest_subprocess("--workers", "auto")
+    result.assert_outcomes(passed=cores * 2)
+    result.stdout.fnmatch_lines([f"zygote: {cores} worker* forked*"])
+
+
+def test_a_bad_worker_count_is_rejected(pytester):
+    pytester.makepyfile(test_plain="def test_ok(): assert True")
+    result = pytester.runpytest_subprocess("--workers", "many")
+    result.stderr.fnmatch_lines(["*expected a number or 'auto'*"])
+
+
+def test_worker_identity_is_exported(pytester, run):
+    pytester.makepyfile(
+        test_env="""
+        import os
+
+        def test_worker_env():
+            assert os.environ["PYTEST_ZYGOTE_WORKER"].startswith("gw")
+        """
+    )
+    run(workers=1).assert_outcomes(passed=1)
+
+
+@pytest.mark.skipif(importlib.util.find_spec("xdist") is None, reason="needs pytest-xdist to own -n")
+def test_rejects_xdist_at_the_same_time(pytester, run):
+    pytester.makepyfile(test_plain="def test_ok(): assert True")
+    result = run("-n", "2")
+    result.stderr.fnmatch_lines(["*alternatives*"])