testguard-cli 0.6.0 → 0.7.0

package/src/probe/runners/python/_testguard_report.py

@@ -0,0 +1,141 @@
+"""
+Shared report writing for TestGuard's two Python reporters.
+
+TestGuard never installs anything into the project under test. Both reporters
+are put on PYTHONPATH from the tool's own package and loaded from there, so a
+project with zero test dependencies stays a project with zero test
+dependencies.
+
+The report is TestGuard's own shape, not jest's. `unittest` and `pytest` both
+distinguish an assertion failure from a collection/import error natively, and
+that distinction is the whole point of the adapter: flattening it into a
+foreign shape would throw away the best signal Python offers and move the
+verdict semantics out of the tested JavaScript and into here.
+
+Shape (schemaVersion 1):
+
+    {
+      "testguard": 1,
+      "runner": "pytest" | "unittest",
+      "rootdir": "<absolute dir the run started from>",
+      "collectionErrors": [{"file": str, "message": str}],
+      "tests": [{"id": "<file>::<name>", "file": str, "name": str,
+                 "outcome": "passed"|"failed"|"skipped"|"xfail"|"xpass",
+                 "phase": "setup"|"call"|"teardown",
+                 "exceptionType": str|null, "message": str|null}],
+      "provenance": {"<relative target path>": "<absolute path actually
+                      imported>" | null}
+    }
+
+`provenance` is the Python-specific precondition. A scratch git worktree
+isolates Node for free, because Node loads files by path. Python does not: an
+editable install (PEP 660 puts an import hook ahead of every path entry) or a
+site-packages copy means the interpreter loads the ORIGINAL file while
+TestGuard has faulted the copy. The baseline stays green, every fault
+SURVIVES, and the result is a confident lie. So every reporter reports which
+file the interpreter actually loaded, and the JavaScript side refuses to
+proceed when that is not the file it faulted.
+"""
+
+import json
+import os
+import sys
+
+ENV_OUT = "TESTGUARD_REPORT"
+ENV_TARGETS = "TESTGUARD_TARGETS"
+
+MAX_MESSAGE = 4000
+
+
+def clip(text):
+    """One bounded string. A pytest longrepr can be megabytes; the report is read whole."""
+    if text is None:
+        return None
+    s = str(text)
+    return s if len(s) <= MAX_MESSAGE else s[:MAX_MESSAGE] + "\n… (truncated)"
+
+
+def targets():
+    """Fault target files (project-relative) whose import provenance must be proved."""
+    raw = os.environ.get(ENV_TARGETS)
+    if not raw:
+        return []
+    try:
+        value = json.loads(raw)
+    except ValueError:
+        return []
+    return [t for t in value if isinstance(t, str)] if isinstance(value, list) else []
+
+
+def module_tail(rootdir, rel):
+    """
+    The trailing path that identifies `rel` as a module wherever it is imported
+    from: its basename preceded by every parent directory that is a package
+    (`__init__.py`). `src/websec/core.py` with `src/websec/__init__.py` present
+    and no `src/__init__.py` gives `websec/core.py` — which is what a
+    site-packages copy of the same module would also end with.
+    """
+    parts = [os.path.basename(rel)]
+    directory = os.path.dirname(os.path.join(rootdir, rel))
+    # Bounded: a package nesting deeper than this is pathological, and an
+    # unbounded walk on a malformed tree would climb to the filesystem root.
+    for _ in range(32):
+        if not os.path.isfile(os.path.join(directory, "__init__.py")):
+            break
+        parts.insert(0, os.path.basename(directory))
+        parent = os.path.dirname(directory)
+        if parent == directory:
+            break
+        directory = parent
+    return os.path.join(*parts)
+
+
+def provenance(rootdir):
+    """
+    For each target: the absolute file the interpreter actually loaded for it,
+    or None when no module with that tail was loaded at all.
+
+    Read from `sys.modules` after the tests have run, so it reflects the real
+    `sys.path` the suite ran under — including anything a `conftest.py`, a
+    `pyproject.toml` `pythonpath`, or an editable-install hook did to it. A
+    separate `find_spec` subprocess could not see any of that.
+    """
+    out = {}
+    loaded = []
+    for module in list(sys.modules.values()):
+        path = getattr(module, "__file__", None)
+        if not path:
+            continue
+        try:
+            loaded.append(os.path.realpath(path))
+        except OSError:
+            continue
+    for rel in targets():
+        tail = module_tail(rootdir, rel)
+        match = None
+        for path in loaded:
+            if path == tail or path.endswith(os.sep + tail):
+                match = path
+                break
+        out[rel] = match
+    return out
+
+
+def write(runner, rootdir, tests, collection_errors):
+    """Write the report where TestGuard asked for it. Never to stdout: a test may print anything."""
+    out_path = os.environ.get(ENV_OUT)
+    if not out_path:
+        return
+    document = {
+        "testguard": 1,
+        "runner": runner,
+        "rootdir": rootdir,
+        "collectionErrors": collection_errors,
+        "tests": tests,
+        "provenance": provenance(rootdir),
+    }
+    directory = os.path.dirname(out_path)
+    if directory and not os.path.isdir(directory):
+        os.makedirs(directory, exist_ok=True)
+    with open(out_path, "w", encoding="utf-8") as handle:
+        json.dump(document, handle)

package/src/probe/runners/python/_testguard_unittest_main.py

@@ -0,0 +1,168 @@
+"""
+TestGuard's stdlib `unittest` runner.
+
+Run as `python -m _testguard_unittest_main <test files…>` from a PYTHONPATH
+entry inside the tool's own package. It needs nothing installed anywhere: a
+project whose test dependencies are the standard library stays that way, which
+is the whole reason this file exists rather than a pytest plugin requirement.
+
+Two deliberate departures from `python -m unittest`:
+
+  * Test modules are imported here, not by the loader. `unittest` turns an
+    import failure into a synthetic `_FailedTest` that *errors at run time* —
+    indistinguishable, from the outside, from a test that ran and failed. That
+    distinction is the difference between "your tests caught the fault" and
+    "the fault stopped the suite from loading", so it is made explicitly.
+
+  * Nothing is written to stdout or stderr. `unittest`'s own runner prints its
+    summary to stderr, where it interleaves with whatever the tests print —
+    the exact trap that makes hand-rolled harnesses report a suite as green
+    when it is not. The result is a JSON file or nothing at all.
+"""
+
+import importlib
+import os
+import sys
+import traceback
+import unittest
+
+import _testguard_report as tg
+
+
+def _module_for(path):
+    """
+    `(top_level_dir, dotted_name)` for a test file, the way `unittest discover`
+    would resolve it: walk up while the directory is a package, and the first
+    directory that is not becomes the import root.
+    """
+    absolute = os.path.abspath(path)
+    parts = [os.path.splitext(os.path.basename(absolute))[0]]
+    directory = os.path.dirname(absolute)
+    for _ in range(32):
+        if not os.path.isfile(os.path.join(directory, "__init__.py")):
+            break
+        parts.insert(0, os.path.basename(directory))
+        parent = os.path.dirname(directory)
+        if parent == directory:
+            break
+        directory = parent
+    return directory, ".".join(parts)
+
+
+def _rel(path):
+    if not path:
+        return ""
+    try:
+        return os.path.relpath(path, os.getcwd()).replace(os.sep, "/")
+    except ValueError:
+        return str(path).replace(os.sep, "/")
+
+
+def _file_of(test):
+    module = sys.modules.get(getattr(test, "__module__", "") or getattr(type(test), "__module__", ""))
+    return _rel(getattr(module, "__file__", None))
+
+
+class Result(unittest.TestResult):
+    """
+    Every outcome `unittest` can produce, kept apart.
+
+    `addFailure` (an assertion rejected the behaviour) and `addError` (the test
+    body raised something else) are both recorded as `failed`: the JavaScript
+    runners already treat a thrown TypeError in a test body as the suite
+    noticing, and a Python adapter that disagreed would make the same fault
+    kill under jest and survive under unittest. The exception class is reported
+    alongside so the difference is never lost, only kept out of the verdict.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.records = {}
+
+    def _record(self, test):
+        key = test.id()
+        if key not in self.records:
+            file_rel = _file_of(test)
+            name = key
+            module_name = getattr(test, "__module__", "")
+            if module_name and name.startswith(module_name + "."):
+                name = name[len(module_name) + 1:]
+            self.records[key] = {
+                "id": "%s::%s" % (file_rel or "", name),
+                "file": file_rel,
+                "name": name,
+                "outcome": "passed",
+                "phase": "call",
+                "exceptionType": None,
+                "message": None,
+            }
+        return self.records[key]
+
+    def _fail(self, test, err, outcome="failed"):
+        record = self._record(test)
+        record["outcome"] = outcome
+        record["exceptionType"] = err[0].__name__ if err and err[0] else None
+        record["message"] = tg.clip("".join(traceback.format_exception(*err))) if err else None
+
+    def startTest(self, test):
+        super().startTest(test)
+        self._record(test)
+
+    def addFailure(self, test, err):
+        super().addFailure(test, err)
+        self._fail(test, err)
+
+    def addError(self, test, err):
+        super().addError(test, err)
+        self._fail(test, err)
+
+    def addSubTest(self, test, subtest, err):
+        super().addSubTest(test, subtest, err)
+        if err is not None:
+            self._fail(test, err)
+
+    def addSkip(self, test, reason):
+        super().addSkip(test, reason)
+        record = self._record(test)
+        record["outcome"] = "skipped"
+        record["message"] = tg.clip(reason)
+
+    def addExpectedFailure(self, test, err):
+        super().addExpectedFailure(test, err)
+        self._fail(test, err, outcome="xfail")
+
+    def addUnexpectedSuccess(self, test):
+        super().addUnexpectedSuccess(test)
+        self._record(test)["outcome"] = "xpass"
+
+
+def main(argv):
+    collection_errors = []
+    loader = unittest.defaultTestLoader
+    suite = unittest.TestSuite()
+
+    if os.getcwd() not in sys.path:
+        sys.path.insert(0, os.getcwd())
+
+    for path in argv:
+        top_level, dotted = _module_for(path)
+        if top_level not in sys.path:
+            sys.path.insert(0, top_level)
+        try:
+            module = importlib.import_module(dotted)
+        except BaseException:  # SyntaxError and SystemExit included: both are load failures, not verdicts
+            collection_errors.append({"file": _rel(path), "message": tg.clip(traceback.format_exc())})
+            continue
+        try:
+            suite.addTests(loader.loadTestsFromModule(module))
+        except BaseException:
+            collection_errors.append({"file": _rel(path), "message": tg.clip(traceback.format_exc())})
+
+    result = Result()
+    suite.run(result)
+    tg.write("unittest", os.getcwd(), list(result.records.values()), collection_errors)
+    return 0 if result.wasSuccessful() and not collection_errors else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/src/probe/runners/python.mjs

@@ -0,0 +1,316 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { spawn } from 'node:child_process';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { runProcess, listTestFiles, firstInformativeLine } from './shared.mjs';
+
+/**
+ * The Python runner. Selected **per file**: any `.py` defender runs here
+ * whatever the project runner is, which is what makes a repository holding
+ * both a JavaScript and a Python package probeable in one run. Selected for
+ * the whole project by `--runner python|pytest|unittest`, or by `auto` when
+ * neither vitest nor jest resolves.
+ *
+ * Two engines, one adapter. `pytest` when the project's interpreter can import
+ * it, stdlib `unittest` otherwise — and `unittest` is not a fallback for the
+ * poor: it is the only engine that runs on a project whose test dependencies
+ * are the standard library, which is a deliberate choice in exactly the
+ * codebases most worth probing. The engine that ran is what the evidence
+ * records as `runner.name`; TestGuard never says pytest ran when it did not.
+ *
+ * Neither engine is installed into the project. The reporters live in
+ * `runners/python/` inside this package and reach the interpreter through
+ * PYTHONPATH — `-p _testguard_pytest_plugin` for pytest, `-m
+ * _testguard_unittest_main` for unittest. The project's dependency set is
+ * never touched.
+ *
+ * Report protocol: TestGuard's own JSON shape (see `python/_testguard_report.py`),
+ * written to a file named by `TESTGUARD_REPORT`, never parsed from stdout —
+ * `unittest` writes its summary to stderr where it interleaves with whatever
+ * the tests print, and reading that stream is how hand-rolled harnesses come
+ * to report a red suite as green.
+ *
+ * Python outcomes map onto the spec's testRun as follows:
+ *
+ *   passed              → passed
+ *   failed              → failed, counts toward assertionFailures (can kill).
+ *                         An `AssertionError` and a `TypeError` raised by the
+ *                         test body are alike here, as they are under jest:
+ *                         both are the suite rejecting the behaviour. The
+ *                         exception class is reported, never scored.
+ *   failed, timed out   → failed, counts toward timeouts (never a kill)
+ *   xfail               → neither passed nor failed
+ *   xpass               → the suite's own expectation is wrong, so the run is
+ *                         NOT green — counted as failed with no assertion
+ *                         failure, so it reaches FLAKY-DEFENDER through the
+ *                         ordinary N-run rules rather than killing anything
+ *   skipped             → neither passed nor failed
+ *   collection error    → the suite could not load: `error`, never a kill.
+ *                         This is what a fault whose replacement does not
+ *                         parse looks like from Python, and it is how
+ *                         `fault-invalid` is reached.
+ */
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+export const PYTHON_DIR = join(HERE, 'python');
+
+export const name = 'python';
+
+/**
+ * pytest's default `python_files`, plus unittest's default discovery pattern
+ * (`test*.py`). A helper named `testutils.py` matches and collects nothing,
+ * which costs a little; a test file that did not match would be invisible to
+ * discovery and report `nocover`, which costs the truth.
+ */
+export const testGlobs = ['**/test_*.py', '**/test*.py', '**/*_test.py'];
+
+const PY_FILE = /\.py$/;
+const TIMEOUT = /\bTimeout >|\btimed out\b|\bTimeoutError\b/i;
+
+/**
+ * Interpreters to try, best first: the active virtualenv, the project's own,
+ * then PATH.
+ *
+ * An explicit `--python` is NOT one of these. It is the only candidate, and it
+ * is returned alone: an interpreter the user named and TestGuard quietly
+ * replaced with another would change which dependencies the tests ran against
+ * without changing anything the evidence says.
+ */
+function candidates(projectDir, python) {
+  const out = [];
+  const win = process.platform === 'win32';
+  const binOf = (root) => (win ? join(root, 'Scripts', 'python.exe') : join(root, 'bin', 'python'));
+  const explicit = python ?? process.env.TESTGUARD_PYTHON;
+  if (explicit) return [{ path: explicit, source: 'project', explicit: true }];
+  if (process.env.VIRTUAL_ENV) out.push({ path: binOf(process.env.VIRTUAL_ENV), source: 'project' });
+  for (const dir of ['.venv', 'venv', '.env']) out.push({ path: binOf(join(projectDir, dir)), source: 'project' });
+  out.push({ path: win ? 'python.exe' : 'python3', source: 'path' });
+  out.push({ path: win ? 'py.exe' : 'python', source: 'path' });
+  return out;
+}
+
+const PROBE = [
+  'import json,sys',
+  'v=".".join(map(str,sys.version_info[:3]))',
+  'try:',
+  ' import pytest; p=pytest.__version__',
+  'except Exception: p=None',
+  'print(json.dumps({"python":v,"pytest":p}))',
+].join('\n');
+
+const interpreters = new Map(); // projectDir -> resolution promise
+
+/** Forget resolved interpreters (tests create and delete projects at the same paths). */
+export const resetInterpreterCache = () => interpreters.clear();
+
+/**
+ * Ask an interpreter what it is and whether it can import pytest.
+ *
+ * Deliberately NOT run with `-I`. Isolated mode ignores `PYTHONPATH` and the
+ * user site directory, so it would answer "no pytest" for an interpreter that
+ * will import pytest perfectly well during the actual run — and TestGuard
+ * would silently choose the stdlib engine for a suite written in pytest, which
+ * then collects nothing. The check has to see what the run will see.
+ */
+function ask(path, budgetMs) {
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(path, ['-c', PROBE], { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1' } });
+    } catch {
+      return resolve(null);
+    }
+    let out = '';
+    child.on('error', () => resolve(null));
+    child.stdout.on('data', (d) => (out += d));
+    child.stderr.on('data', () => {});
+    const timer = setTimeout(() => child.kill('SIGKILL'), budgetMs);
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code !== 0) return resolve(null);
+      try {
+        resolve(JSON.parse(out.trim().split('\n').pop()));
+      } catch {
+        resolve(null);
+      }
+    });
+  });
+}
+
+/**
+ * The interpreter this project's tests run under, and whether it can import
+ * pytest. Resolved against the **original** project directory, never the
+ * scratch worktree: a virtualenv is gitignored, so it is not in the worktree
+ * checkout, and it is referenced by absolute path the way virtualenvs are
+ * meant to be — never copied and never symlinked, which would break the
+ * `pyvenv.cfg` prefix detection that makes one work at all.
+ */
+export function resolveInterpreter({ projectDir, python, budgetMs = 30_000 }) {
+  const key = `${projectDir}\0${python ?? ''}`;
+  if (interpreters.has(key)) return interpreters.get(key);
+  const task = (async () => {
+    const tried = [];
+    for (const candidate of candidates(projectDir, python)) {
+      if (!candidate.explicit && candidate.source === 'project' && !existsSync(candidate.path)) continue;
+      const answer = await ask(candidate.path, budgetMs);
+      if (answer) return { ...candidate, ...answer };
+      tried.push(candidate.path);
+    }
+    const explicit = python ?? process.env.TESTGUARD_PYTHON;
+    return { error: explicit
+      ? `${explicit} is not a usable Python interpreter`
+      : `no usable Python interpreter (tried ${tried.join(', ') || 'nothing'}); install Python 3.8+, or point --python / TESTGUARD_PYTHON at one` };
+  })();
+  interpreters.set(key, task);
+  return task;
+}
+
+/**
+ * Resolvable = an interpreter answers, and — when an engine is pinned — that
+ * engine is available. `--runner pytest` on a machine that has no pytest is a
+ * precondition failure, never a silent fall back to unittest: which engine ran
+ * changes what the evidence means.
+ */
+/**
+ * Which engine runs, given what the interpreter answered.
+ *
+ * Pure, and separate from the spawn, because the interesting branch is the one
+ * that is hard to stage: an interpreter that resolves but cannot import
+ * pytest. Testing that through `check()` would mean building a bare virtualenv
+ * inside a unit test, so the decision is lifted out and the spawn is left with
+ * nothing to decide.
+ */
+export function chooseEngine(pinned, found) {
+  const engine = pinned ?? (found.pytest ? 'pytest' : 'unittest');
+  if (engine === 'pytest' && !found.pytest) {
+    return { ok: false, message: `pytest is not importable from ${found.path} (pip install pytest, or use --runner unittest for the stdlib runner)` };
+  }
+  return { ok: true, engine, version: engine === 'pytest' ? found.pytest : `CPython ${found.python}` };
+}
+
+export function makeCheck(pinned) {
+  return async ({ projectDir, sourceDir, python, budgetMs = 30_000 }) => {
+    // Always the ORIGINAL project directory when there is one: a virtualenv is
+    // gitignored, so the scratch worktree does not contain it.
+    const found = await resolveInterpreter({ projectDir: sourceDir ?? projectDir, python, budgetMs });
+    if (found.error) return { ok: false, message: found.error };
+    const chosen = chooseEngine(pinned, found);
+    if (!chosen.ok) return chosen;
+    return { ...chosen, interpreter: found.path, source: found.source };
+  };
+}
+
+/** Any `.py` file. A Python test can never run under vitest, so ownership needs no configuration to read. */
+export const owns = (projectDir, file) => PY_FILE.test(file);
+
+export const tests = (projectDir) => listTestFiles(projectDir, testGlobs).filter((f) => owns(projectDir, f));
+
+/**
+ * TestGuard's Python report → the spec's testRun. Pure; exported for the
+ * sample-driven tests. `provenance` rides along for the import check that
+ * `probe` performs once the fault is live; it is not part of the testRun.
+ */
+export function parseReport(report, durationMs) {
+  const all = report.tests ?? [];
+  const collectionErrors = report.collectionErrors ?? [];
+  const failed = all.filter((t) => t.outcome === 'failed');
+  const timedOut = failed.filter((t) => TIMEOUT.test(t.message ?? ''));
+  const xpass = all.filter((t) => t.outcome === 'xpass');
+  const passed = all.filter((t) => t.outcome === 'passed').length;
+  const total = all.length;
+  const summary = { total, passed, failed: failed.length + xpass.length };
+
+  let outcome;
+  if (collectionErrors.length > 0 || total === 0) outcome = 'error';
+  else if (failed.length > 0 || xpass.length > 0) outcome = 'fail';
+  else outcome = 'pass';
+
+  const loadMessage = outcome === 'error'
+    ? (collectionErrors.length
+      ? firstInformativeLine(collectionErrors.map((e) => e.message ?? '').join('\n'))
+      : 'python collected no tests')
+    : undefined;
+
+  return {
+    run: { outcome, tests: summary, assertionFailures: failed.length - timedOut.length, durationMs },
+    timeouts: timedOut.length,
+    loadMessage,
+    failedTests: [...failed, ...xpass].map((t) => t.id),
+    provenance: report.provenance ?? {},
+  };
+}
+
+/**
+ * Environment for one run. Three things matter beyond finding the reporter:
+ *
+ *   PYTHONDONTWRITEBYTECODE + PYTHONPYCACHEPREFIX — `.pyc` files are validated
+ *     by the source's mtime **in whole seconds** and its size. A fault that
+ *     preserves the byte length and is applied within the same second as the
+ *     file it replaces can therefore be served from a stale cache and never
+ *     execute: a green run that proves nothing. The prefix moves every cache
+ *     lookup out of the tree, so an `--in-place` run cannot read the user's
+ *     existing `__pycache__` either, and DONTWRITEBYTECODE means the prefix
+ *     directory is never actually created.
+ *
+ *   TESTGUARD_TARGETS — the fault's file, so the reporter can say which file
+ *     the interpreter really loaded for it.
+ */
+export function environment(targets) {
+  const existing = process.env.PYTHONPATH;
+  return {
+    PYTHONPATH: existing ? `${PYTHON_DIR}${process.platform === 'win32' ? ';' : ':'}${existing}` : PYTHON_DIR,
+    PYTHONDONTWRITEBYTECODE: '1',
+    PYTHONPYCACHEPREFIX: '{out}.pycache',
+    PYTHONUNBUFFERED: '1',
+    TESTGUARD_REPORT: '{out}',
+    TESTGUARD_TARGETS: JSON.stringify(targets ?? []),
+  };
+}
+
+/**
+ * `--maxfail=0` is appended on every pytest run, and is not optional: a
+ * project whose `addopts` carries `-x` would otherwise stop at the first
+ * failure. The verdict would survive that (one assertion failure is still a
+ * kill) but the list of failing tests would not, and that list is what names
+ * an undeclared killer during escalation. `0` means no limit and overrides
+ * the configured value; the rest of the project's `addopts` is left alone,
+ * because TestGuard runs a project's tests the way the project runs them.
+ */
+export function argvFor({ engine, interpreter, files, serial }) {
+  if (engine === 'pytest') {
+    return [interpreter, '-m', 'pytest', '-p', '_testguard_pytest_plugin', '-p', 'no:cacheprovider',
+      '--maxfail=0', '-q', ...(serial ? ['-p', 'no:xdist'] : []), ...files];
+  }
+  return [interpreter, '-m', '_testguard_unittest_main', ...files];
+}
+
+export function makeRun(pinned) {
+  const check = makeCheck(pinned);
+  return async (opts) => {
+    const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python });
+    if (!resolved.ok) {
+      return { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs: 0 }, timeouts: 0, loadMessage: resolved.message, failedTests: [], provenance: {} };
+    }
+    return runProcess({
+      ...opts,
+      env: environment(opts.targets),
+      parse: parseReport,
+      argv: (files) => argvFor({ engine: resolved.engine, interpreter: resolved.interpreter, files, serial: opts.serial }),
+    });
+  };
+}
+
+export const check = makeCheck(null);
+export const run = makeRun(null);
+
+/** `--runner pytest` / `--runner unittest`: the same adapter with the engine pinned. */
+export function pinned(engine) {
+  return { name: engine, testGlobs, owns, tests, parseReport, fatalEdit, check: makeCheck(engine), run: makeRun(engine) };
+}
+
+/**
+ * The negative control's edit, for Python: an unterminated group, which is a
+ * `SyntaxError` at import time whatever the module contained. See the
+ * JavaScript runners for why this exists.
+ */
+export const fatalEdit = () => '# testguard negative control: this file must not parse\n(\n';

package/src/probe/runners/shared.mjs

@@ -136,6 +136,13 @@ export function firstInformativeLine(message) {
 }
 
 export function parseReport(report, durationMs) {
+  // A report that parsed as JSON but carries no `testResults` is not a suite
+  // that ran zero tests — it is a report of a shape this parser does not
+  // understand, which is what `--runner-cmd` pointed at a non-jest reporter
+  // produces. The verdict is already safe (a non-green baseline makes the
+  // claim unverifiable rather than passing), but without this the user is told
+  // `defenders-failed-to-load` with no reason at all.
+  const unknownShape = report == null || typeof report !== 'object' || !Array.isArray(report.testResults);
   const files = report.testResults ?? [];
   const loadFailed = files.some((f) => f.status === 'failed' && (f.assertionResults?.length ?? 0) === 0);
   const results = files.flatMap((f) => (f.assertionResults ?? []).map((t) => ({ ...t, id: `${f.name}::${t.fullName ?? t.title ?? ''}` }))).filter((t) => t.status === 'failed');
@@ -153,7 +160,9 @@ export function parseReport(report, durationMs) {
   // jest opens a load failure with "● Test suite failed to run" and puts the
   // cause lines below; vitest puts it on the first line. Take the first line
   // that names an error, else the first non-empty one.
-  const loadMessage = loadFailed ? firstInformativeLine(files.find((f) => f.message)?.message ?? '') : undefined;
+  const loadMessage = unknownShape
+    ? 'the runner produced a report with no `testResults` array: this parser reads the jest-compatible JSON report, so a --runner-cmd must emit that shape (vitest --reporter=json, jest --json)'
+    : loadFailed ? firstInformativeLine(files.find((f) => f.message)?.message ?? '') : undefined;
   return { run, timeouts, loadMessage, failedTests };
 }
 

package/src/probe/runners/vitest.mjs

@@ -5,4 +5,16 @@ export const testGlobs = TEST_GLOBS;
 export const check = (opts) => checkRunner({ ...opts, pkg: 'vitest', bin: 'vitest' });
 export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
 // --serial: one file at a time in one process, so a contended machine cannot turn a slow suite into a TIMEOUT verdict.
-export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [...runnerArgv(opts.projectDir, 'vitest', 'vitest'), 'run', ...files, '--reporter=json', `--outputFile=${outFile}`, ...(opts.serial ? ['--no-file-parallelism'] : [])] });
+/** The command line, as data. Exported so it is falsifiable without spawning anything. */
+export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
+  [...runnerArgv(projectDir, 'vitest', 'vitest'), 'run', ...files, '--reporter=json', `--outputFile=${outFile}`, ...(serial ? ['--no-file-parallelism'] : [])];
+export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
+
+/**
+ * The negative control's edit: content this runner's loader cannot possibly
+ * parse. Used to prove the defenders actually execute a file before a
+ * `survived` verdict about it is allowed to stand. An unterminated group is
+ * a syntax error at every JavaScript/TypeScript parser, whatever the file
+ * contained before.
+ */
+export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';