testguard-cli 0.5.0 → 0.7.0

package/src/probe/runners/playwright.mjs

@@ -0,0 +1,142 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join, relative, resolve, sep } from 'node:path';
+import { npx, runProcess, listTestFiles, firstInformativeLine } from './shared.mjs';
+import { globToRegExp } from '../../util/glob.mjs';
+
+/**
+ * Playwright runner. Selected **per file**, not per project: a defender that
+ * lives under Playwright's `testDir` runs here, everything else runs under
+ * the project runner (vitest/jest). One claim may list both kinds. Selected
+ * for the whole project only with `--runner playwright`.
+ *
+ * Report protocol: `--reporter=json` with the JSON written to a file we name
+ * (PLAYWRIGHT_JSON_OUTPUT_FILE; the older PLAYWRIGHT_JSON_OUTPUT_NAME is set
+ * too), never parsed from stdout. Playwright's test statuses map onto the
+ * spec's testRun as follows, and the mapping is the point of the adapter:
+ *
+ *   expected            → passed
+ *   unexpected/failed   → failed, counts toward assertionFailures (can kill)
+ *   unexpected/timedOut → failed, counts toward timeouts (never a kill)
+ *   flaky               → failed on an attempt and passed on retry: the run
+ *                         is NOT green. Counted as failed with neither an
+ *                         assertion failure nor a timeout, so a green-looking
+ *                         exit code becomes FLAKY-DEFENDER through the
+ *                         ordinary N-run rules, exactly as GATE-SEMANTICS says.
+ *   skipped             → neither passed nor failed
+ */
+
+export const name = 'playwright';
+
+const CONFIG_FILES = ['playwright.config.ts', 'playwright.config.mts', 'playwright.config.js', 'playwright.config.mjs', 'playwright.config.cjs'];
+const SPEC_GLOBS = ['**/*.spec.js', '**/*.spec.mjs', '**/*.spec.cjs', '**/*.spec.ts', '**/*.spec.mts', '**/*.spec.tsx', '**/*.spec.jsx',
+  '**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**/*.test.ts', '**/*.test.mts', '**/*.test.tsx', '**/*.test.jsx'];
+
+const configCache = new Map();
+
+/**
+ * The project's Playwright config, read as text (it may be TypeScript, so it
+ * is never imported): `{ path, testDir }` or null when there is none.
+ * `testDir` defaults to the config's own directory, as Playwright does.
+ */
+export function loadConfig(projectDir) {
+  if (configCache.has(projectDir)) return configCache.get(projectDir);
+  let out = null;
+  for (const f of CONFIG_FILES) {
+    const path = join(projectDir, f);
+    if (!existsSync(path)) continue;
+    const text = readFileSync(path, 'utf8');
+    const m = /\btestDir\s*:\s*(['"`])([^'"`]+)\1/.exec(text);
+    const testDir = relative(projectDir, resolve(projectDir, m ? m[2] : '.')).split(sep).join('/') || '.';
+    out = { path: f, testDir };
+    break;
+  }
+  configCache.set(projectDir, out);
+  return out;
+}
+
+/** Clear the config cache (tests create and delete projects at the same paths). */
+export const resetConfigCache = () => configCache.clear();
+
+export const testGlobs = SPEC_GLOBS;
+const SPEC_RES = SPEC_GLOBS.map(globToRegExp);
+
+/** Does this test file belong to Playwright: is there a config, and does the file live under its testDir? */
+export function owns(projectDir, file) {
+  const cfg = loadConfig(projectDir);
+  if (!cfg) return false;
+  if (!SPEC_RES.some((re) => re.test(file))) return false;
+  return cfg.testDir === '.' ? true : file.startsWith(cfg.testDir + '/');
+}
+
+/**
+ * Resolvable = a config exists AND `@playwright/test` resolves from the
+ * project. Deliberately not `npx --no-install playwright --version`: the npx
+ * cache and a Python `playwright` on PATH both answer that from a project
+ * that has no Playwright at all, and `playwright test` would then run
+ * against nothing. The package `playwright test` needs is the precondition.
+ */
+export async function check({ projectDir }) {
+  const cfg = loadConfig(projectDir);
+  if (!cfg) return { ok: false, message: 'no playwright.config.* in the project' };
+  try {
+    const pkg = JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve('@playwright/test/package.json'), 'utf8'));
+    return { ok: true, version: pkg.version };
+  } catch {
+    return { ok: false, message: '@playwright/test is not installed in the project (npm i -D @playwright/test)' };
+  }
+}
+
+export const tests = (projectDir) => listTestFiles(projectDir, SPEC_GLOBS).filter((f) => owns(projectDir, f));
+
+/**
+ * Playwright's JSON report → the spec's testRun. Pure; exported for the
+ * sample-driven tests. `durationMs` is the wall clock measured by the runner.
+ */
+export function parseReport(report, durationMs) {
+  const tests = [];
+  const walk = (suite) => {
+    for (const spec of suite.specs ?? []) {
+      for (const t of spec.tests ?? []) {
+        const results = t.results ?? [];
+        const last = results[results.length - 1];
+        tests.push({ id: `${suite.file ?? spec.file ?? ''}::${spec.title}`, status: t.status, finalResult: last?.status, error: results.find((r) => r.error)?.error?.message });
+      }
+    }
+    for (const s of suite.suites ?? []) walk(s);
+  };
+  for (const s of report.suites ?? []) walk(s);
+
+  const expected = tests.filter((t) => t.status === 'expected').length;
+  const unexpected = tests.filter((t) => t.status === 'unexpected');
+  const flaky = tests.filter((t) => t.status === 'flaky').length;
+  const timeouts = unexpected.filter((t) => t.finalResult === 'timedOut' || t.finalResult === 'interrupted').length;
+  const assertionFailures = unexpected.length - timeouts;
+  const failedTests = [...unexpected, ...tests.filter((t) => t.status === 'flaky')].map((t) => t.id);
+  const total = tests.length;
+  const summary = { total, passed: expected, failed: unexpected.length + flaky };
+
+  const globalErrors = report.errors ?? [];
+  let outcome;
+  if (total === 0) outcome = 'error';
+  else if (unexpected.length > 0 || flaky > 0) outcome = 'fail';
+  else outcome = 'pass';
+  const loadMessage = outcome === 'error'
+    ? (globalErrors.length ? firstInformativeLine(globalErrors.map((e) => e.message ?? '').join('\n')) : 'playwright found no tests')
+    : undefined;
+  return { run: { outcome, tests: summary, assertionFailures, durationMs }, timeouts, loadMessage, failedTests, flaky };
+}
+
+export const run = (opts) => {
+  const env = { PLAYWRIGHT_JSON_OUTPUT_FILE: '{out}', PLAYWRIGHT_JSON_OUTPUT_NAME: '{out}' };
+  return runProcess({ ...opts, env, parse: parseReport, argv: (files) => [npx, 'playwright', 'test', '--reporter=json', ...(opts.serial ? ['--workers=1'] : []), ...files] });
+};
+
+/**
+ * The negative control's edit: content this runner's loader cannot possibly
+ * parse. Used to prove the defenders actually execute a file before a
+ * `survived` verdict about it is allowed to stand. An unterminated group is
+ * a syntax error at every JavaScript/TypeScript parser, whatever the file
+ * contained before.
+ */
+export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';

package/src/probe/runners/python/_testguard_pytest_plugin.py

@@ -0,0 +1,132 @@
+"""
+TestGuard's pytest reporter.
+
+Loaded with `-p _testguard_pytest_plugin` from a PYTHONPATH entry inside the
+tool's own package. Nothing is installed into the project under test, so
+`pytest` needs no report plugin of its own and the project's dependency set is
+untouched — which matters, because the suites most worth probing are often the
+ones that deliberately have none.
+
+Only long-stable hooks are used, so one file serves pytest 7, 8 and 9.
+"""
+
+import os
+
+import _testguard_report as tg
+
+_tests = {}            # nodeid -> record being assembled
+_files = {}            # nodeid -> absolute test file
+_collection_errors = []
+
+# `unittest.expectedFailure` on a test that passes reaches pytest as a plain
+# failure whose whole report is this string; pytest's own
+# `@pytest.mark.xfail` reports the same situation as a pass carrying
+# `wasxfail`. Both are the suite's expectation being wrong rather than an
+# assertion rejecting the fault, so both must land on `xpass` — otherwise the
+# same test would be kill-eligible under pytest and not under unittest.
+_UNEXPECTED_SUCCESS = "Unexpected success"
+
+# Worst-first: a later phase never downgrades a test that already failed.
+_RANK = {"failed": 4, "xpass": 3, "xfail": 2, "skipped": 1, "passed": 0}
+
+
+def _rel(path):
+    """Project-relative, forward-slashed. pytest's rootdir may be an ancestor of the probed project."""
+    if not path:
+        return ""
+    try:
+        return os.path.relpath(path, os.getcwd()).replace(os.sep, "/")
+    except ValueError:  # different drive on Windows
+        return str(path).replace(os.sep, "/")
+
+
+def _record(nodeid):
+    if nodeid not in _tests:
+        file_rel = _rel(_files.get(nodeid)) or nodeid.split("::")[0]
+        _tests[nodeid] = {
+            "id": nodeid if nodeid.startswith(file_rel) else "%s::%s" % (file_rel, nodeid),
+            "file": file_rel,
+            "name": nodeid.split("::", 1)[1] if "::" in nodeid else nodeid,
+            "outcome": "passed",
+            "phase": "call",
+            "exceptionType": None,
+            "message": None,
+        }
+    return _tests[nodeid]
+
+
+def _apply(record, outcome, phase, message, exception_type):
+    if _RANK[outcome] < _RANK[record["outcome"]]:
+        return
+    record["outcome"] = outcome
+    record["phase"] = phase
+    if message is not None:
+        record["message"] = tg.clip(message)
+    if exception_type is not None:
+        record["exceptionType"] = exception_type
+
+
+def pytest_collection_modifyitems(items):
+    """Absolute path per test, taken while the items still carry it."""
+    for item in items:
+        path = getattr(item, "path", None) or getattr(item, "fspath", None)
+        if path is not None:
+            _files[item.nodeid] = str(path)
+
+
+def pytest_exception_interact(node, call, report):
+    """
+    The exception's class name, which no report object carries. Recorded but
+    never allowed to change a verdict: an assertion failure and a TypeError
+    raised by the test body are both the suite rejecting the behaviour, and the
+    JavaScript runners already treat them alike. It is reported because it is
+    the cheapest thing to look at when a kill turns out to be the wrong kind.
+
+    pytest calls this hook *after* `pytest_runtest_logreport` for the same
+    phase, so the record already exists and is amended in place rather than
+    read from a map that would still be empty.
+    """
+    name = getattr(getattr(call, "excinfo", None), "typename", None)
+    nodeid = getattr(report, "nodeid", "") or getattr(node, "nodeid", "")
+    if name and nodeid in _tests:
+        _tests[nodeid]["exceptionType"] = name
+
+
+def pytest_collectreport(report):
+    """
+    A module that will not import. This is the load failure: it is what a fault
+    whose replacement does not parse looks like from here, and it must never be
+    mistaken for the tests rejecting the fault.
+    """
+    if getattr(report, "failed", False):
+        _collection_errors.append({
+            "file": _rel(str(getattr(report, "fspath", "") or "")) or report.nodeid,
+            "message": tg.clip(report.longrepr),
+        })
+
+
+def pytest_runtest_logreport(report):
+    phase = getattr(report, "when", "call")
+    record = _record(report.nodeid)
+    exception_type = None
+    message = tg.clip(report.longrepr) if report.longrepr is not None else None
+
+    if report.failed:
+        unexpected_success = isinstance(message, str) and message.startswith(_UNEXPECTED_SUCCESS)
+        _apply(record, "xpass" if unexpected_success else "failed", phase, message, exception_type)
+    elif report.skipped:
+        # pytest reports an xfail as a skip carrying `wasxfail`.
+        _apply(record, "xfail" if hasattr(report, "wasxfail") else "skipped", phase, message, exception_type)
+    elif report.passed and hasattr(report, "wasxfail"):
+        # A test expected to fail that passed: the suite's own expectation is
+        # wrong, so the run is not green — but nothing asserted anything about
+        # the fault, so it can never be a kill.
+        _apply(record, "xpass", phase, message, exception_type)
+
+
+def pytest_internalerror(excrepr):
+    _collection_errors.append({"file": "", "message": tg.clip(excrepr)})
+
+
+def pytest_sessionfinish(session, exitstatus):
+    tg.write("pytest", os.getcwd(), list(_tests.values()), _collection_errors)

package/src/probe/runners/python/_testguard_report.py

@@ -0,0 +1,141 @@
+"""
+Shared report writing for TestGuard's two Python reporters.
+
+TestGuard never installs anything into the project under test. Both reporters
+are put on PYTHONPATH from the tool's own package and loaded from there, so a
+project with zero test dependencies stays a project with zero test
+dependencies.
+
+The report is TestGuard's own shape, not jest's. `unittest` and `pytest` both
+distinguish an assertion failure from a collection/import error natively, and
+that distinction is the whole point of the adapter: flattening it into a
+foreign shape would throw away the best signal Python offers and move the
+verdict semantics out of the tested JavaScript and into here.
+
+Shape (schemaVersion 1):
+
+    {
+      "testguard": 1,
+      "runner": "pytest" | "unittest",
+      "rootdir": "<absolute dir the run started from>",
+      "collectionErrors": [{"file": str, "message": str}],
+      "tests": [{"id": "<file>::<name>", "file": str, "name": str,
+                 "outcome": "passed"|"failed"|"skipped"|"xfail"|"xpass",
+                 "phase": "setup"|"call"|"teardown",
+                 "exceptionType": str|null, "message": str|null}],
+      "provenance": {"<relative target path>": "<absolute path actually
+                      imported>" | null}
+    }
+
+`provenance` is the Python-specific precondition. A scratch git worktree
+isolates Node for free, because Node loads files by path. Python does not: an
+editable install (PEP 660 puts an import hook ahead of every path entry) or a
+site-packages copy means the interpreter loads the ORIGINAL file while
+TestGuard has faulted the copy. The baseline stays green, every fault
+SURVIVES, and the result is a confident lie. So every reporter reports which
+file the interpreter actually loaded, and the JavaScript side refuses to
+proceed when that is not the file it faulted.
+"""
+
+import json
+import os
+import sys
+
+ENV_OUT = "TESTGUARD_REPORT"
+ENV_TARGETS = "TESTGUARD_TARGETS"
+
+MAX_MESSAGE = 4000
+
+
+def clip(text):
+    """One bounded string. A pytest longrepr can be megabytes; the report is read whole."""
+    if text is None:
+        return None
+    s = str(text)
+    return s if len(s) <= MAX_MESSAGE else s[:MAX_MESSAGE] + "\n… (truncated)"
+
+
+def targets():
+    """Fault target files (project-relative) whose import provenance must be proved."""
+    raw = os.environ.get(ENV_TARGETS)
+    if not raw:
+        return []
+    try:
+        value = json.loads(raw)
+    except ValueError:
+        return []
+    return [t for t in value if isinstance(t, str)] if isinstance(value, list) else []
+
+
+def module_tail(rootdir, rel):
+    """
+    The trailing path that identifies `rel` as a module wherever it is imported
+    from: its basename preceded by every parent directory that is a package
+    (`__init__.py`). `src/websec/core.py` with `src/websec/__init__.py` present
+    and no `src/__init__.py` gives `websec/core.py` — which is what a
+    site-packages copy of the same module would also end with.
+    """
+    parts = [os.path.basename(rel)]
+    directory = os.path.dirname(os.path.join(rootdir, rel))
+    # Bounded: a package nesting deeper than this is pathological, and an
+    # unbounded walk on a malformed tree would climb to the filesystem root.
+    for _ in range(32):
+        if not os.path.isfile(os.path.join(directory, "__init__.py")):
+            break
+        parts.insert(0, os.path.basename(directory))
+        parent = os.path.dirname(directory)
+        if parent == directory:
+            break
+        directory = parent
+    return os.path.join(*parts)
+
+
+def provenance(rootdir):
+    """
+    For each target: the absolute file the interpreter actually loaded for it,
+    or None when no module with that tail was loaded at all.
+
+    Read from `sys.modules` after the tests have run, so it reflects the real
+    `sys.path` the suite ran under — including anything a `conftest.py`, a
+    `pyproject.toml` `pythonpath`, or an editable-install hook did to it. A
+    separate `find_spec` subprocess could not see any of that.
+    """
+    out = {}
+    loaded = []
+    for module in list(sys.modules.values()):
+        path = getattr(module, "__file__", None)
+        if not path:
+            continue
+        try:
+            loaded.append(os.path.realpath(path))
+        except OSError:
+            continue
+    for rel in targets():
+        tail = module_tail(rootdir, rel)
+        match = None
+        for path in loaded:
+            if path == tail or path.endswith(os.sep + tail):
+                match = path
+                break
+        out[rel] = match
+    return out
+
+
+def write(runner, rootdir, tests, collection_errors):
+    """Write the report where TestGuard asked for it. Never to stdout: a test may print anything."""
+    out_path = os.environ.get(ENV_OUT)
+    if not out_path:
+        return
+    document = {
+        "testguard": 1,
+        "runner": runner,
+        "rootdir": rootdir,
+        "collectionErrors": collection_errors,
+        "tests": tests,
+        "provenance": provenance(rootdir),
+    }
+    directory = os.path.dirname(out_path)
+    if directory and not os.path.isdir(directory):
+        os.makedirs(directory, exist_ok=True)
+    with open(out_path, "w", encoding="utf-8") as handle:
+        json.dump(document, handle)

package/src/probe/runners/python/_testguard_unittest_main.py

@@ -0,0 +1,168 @@
+"""
+TestGuard's stdlib `unittest` runner.
+
+Run as `python -m _testguard_unittest_main <test files…>` from a PYTHONPATH
+entry inside the tool's own package. It needs nothing installed anywhere: a
+project whose test dependencies are the standard library stays that way, which
+is the whole reason this file exists rather than a pytest plugin requirement.
+
+Two deliberate departures from `python -m unittest`:
+
+  * Test modules are imported here, not by the loader. `unittest` turns an
+    import failure into a synthetic `_FailedTest` that *errors at run time* —
+    indistinguishable, from the outside, from a test that ran and failed. That
+    distinction is the difference between "your tests caught the fault" and
+    "the fault stopped the suite from loading", so it is made explicitly.
+
+  * Nothing is written to stdout or stderr. `unittest`'s own runner prints its
+    summary to stderr, where it interleaves with whatever the tests print —
+    the exact trap that makes hand-rolled harnesses report a suite as green
+    when it is not. The result is a JSON file or nothing at all.
+"""
+
+import importlib
+import os
+import sys
+import traceback
+import unittest
+
+import _testguard_report as tg
+
+
+def _module_for(path):
+    """
+    `(top_level_dir, dotted_name)` for a test file, the way `unittest discover`
+    would resolve it: walk up while the directory is a package, and the first
+    directory that is not becomes the import root.
+    """
+    absolute = os.path.abspath(path)
+    parts = [os.path.splitext(os.path.basename(absolute))[0]]
+    directory = os.path.dirname(absolute)
+    for _ in range(32):
+        if not os.path.isfile(os.path.join(directory, "__init__.py")):
+            break
+        parts.insert(0, os.path.basename(directory))
+        parent = os.path.dirname(directory)
+        if parent == directory:
+            break
+        directory = parent
+    return directory, ".".join(parts)
+
+
+def _rel(path):
+    if not path:
+        return ""
+    try:
+        return os.path.relpath(path, os.getcwd()).replace(os.sep, "/")
+    except ValueError:
+        return str(path).replace(os.sep, "/")
+
+
+def _file_of(test):
+    module = sys.modules.get(getattr(test, "__module__", "") or getattr(type(test), "__module__", ""))
+    return _rel(getattr(module, "__file__", None))
+
+
+class Result(unittest.TestResult):
+    """
+    Every outcome `unittest` can produce, kept apart.
+
+    `addFailure` (an assertion rejected the behaviour) and `addError` (the test
+    body raised something else) are both recorded as `failed`: the JavaScript
+    runners already treat a thrown TypeError in a test body as the suite
+    noticing, and a Python adapter that disagreed would make the same fault
+    kill under jest and survive under unittest. The exception class is reported
+    alongside so the difference is never lost, only kept out of the verdict.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.records = {}
+
+    def _record(self, test):
+        key = test.id()
+        if key not in self.records:
+            file_rel = _file_of(test)
+            name = key
+            module_name = getattr(test, "__module__", "")
+            if module_name and name.startswith(module_name + "."):
+                name = name[len(module_name) + 1:]
+            self.records[key] = {
+                "id": "%s::%s" % (file_rel or "", name),
+                "file": file_rel,
+                "name": name,
+                "outcome": "passed",
+                "phase": "call",
+                "exceptionType": None,
+                "message": None,
+            }
+        return self.records[key]
+
+    def _fail(self, test, err, outcome="failed"):
+        record = self._record(test)
+        record["outcome"] = outcome
+        record["exceptionType"] = err[0].__name__ if err and err[0] else None
+        record["message"] = tg.clip("".join(traceback.format_exception(*err))) if err else None
+
+    def startTest(self, test):
+        super().startTest(test)
+        self._record(test)
+
+    def addFailure(self, test, err):
+        super().addFailure(test, err)
+        self._fail(test, err)
+
+    def addError(self, test, err):
+        super().addError(test, err)
+        self._fail(test, err)
+
+    def addSubTest(self, test, subtest, err):
+        super().addSubTest(test, subtest, err)
+        if err is not None:
+            self._fail(test, err)
+
+    def addSkip(self, test, reason):
+        super().addSkip(test, reason)
+        record = self._record(test)
+        record["outcome"] = "skipped"
+        record["message"] = tg.clip(reason)
+
+    def addExpectedFailure(self, test, err):
+        super().addExpectedFailure(test, err)
+        self._fail(test, err, outcome="xfail")
+
+    def addUnexpectedSuccess(self, test):
+        super().addUnexpectedSuccess(test)
+        self._record(test)["outcome"] = "xpass"
+
+
+def main(argv):
+    collection_errors = []
+    loader = unittest.defaultTestLoader
+    suite = unittest.TestSuite()
+
+    if os.getcwd() not in sys.path:
+        sys.path.insert(0, os.getcwd())
+
+    for path in argv:
+        top_level, dotted = _module_for(path)
+        if top_level not in sys.path:
+            sys.path.insert(0, top_level)
+        try:
+            module = importlib.import_module(dotted)
+        except BaseException:  # SyntaxError and SystemExit included: both are load failures, not verdicts
+            collection_errors.append({"file": _rel(path), "message": tg.clip(traceback.format_exc())})
+            continue
+        try:
+            suite.addTests(loader.loadTestsFromModule(module))
+        except BaseException:
+            collection_errors.append({"file": _rel(path), "message": tg.clip(traceback.format_exc())})
+
+    result = Result()
+    suite.run(result)
+    tg.write("unittest", os.getcwd(), list(result.records.values()), collection_errors)
+    return 0 if result.wasSuccessful() and not collection_errors else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))