counterproof 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- counterproof/__init__.py +12 -0
- counterproof/__main__.py +3 -0
- counterproof/adapters/__init__.py +1 -0
- counterproof/adapters/base.py +163 -0
- counterproof/adapters/detect.py +71 -0
- counterproof/adapters/jest_adapter.py +314 -0
- counterproof/adapters/junit.py +93 -0
- counterproof/adapters/node.py +185 -0
- counterproof/adapters/pytest_adapter.py +412 -0
- counterproof/adapters/vitest_adapter.py +346 -0
- counterproof/change/__init__.py +1 -0
- counterproof/change/classify.py +107 -0
- counterproof/change/diff.py +128 -0
- counterproof/change/intent.py +58 -0
- counterproof/change/resolver.py +99 -0
- counterproof/change/self_config.py +113 -0
- counterproof/change/task.py +104 -0
- counterproof/cli.py +615 -0
- counterproof/config.py +262 -0
- counterproof/data/counterproof.toml +49 -0
- counterproof/doctor.py +132 -0
- counterproof/errors.py +27 -0
- counterproof/eval/__init__.py +1 -0
- counterproof/eval/corpus.py +261 -0
- counterproof/eval/harness.py +455 -0
- counterproof/eval/leakage.py +156 -0
- counterproof/eval/table.py +158 -0
- counterproof/evidence/__init__.py +1 -0
- counterproof/evidence/hashing.py +35 -0
- counterproof/evidence/seed.py +60 -0
- counterproof/evidence/store.py +275 -0
- counterproof/git.py +149 -0
- counterproof/impact/__init__.py +1 -0
- counterproof/impact/baseline.py +70 -0
- counterproof/impact/selector.py +62 -0
- counterproof/integrations/__init__.py +1 -0
- counterproof/integrations/claude_hook.py +205 -0
- counterproof/integrations/git_hook.py +140 -0
- counterproof/integrations/mcp_server.py +166 -0
- counterproof/llm/__init__.py +1 -0
- counterproof/llm/anthropic_provider.py +68 -0
- counterproof/llm/budget.py +77 -0
- counterproof/llm/cache.py +107 -0
- counterproof/llm/credentials.py +68 -0
- counterproof/llm/factory.py +73 -0
- counterproof/llm/openai_compat.py +84 -0
- counterproof/llm/prices.py +60 -0
- counterproof/llm/prompts/equivalence_judge.v1.md +32 -0
- counterproof/llm/prompts/equivalence_judge.v1.typescript.md +33 -0
- counterproof/llm/prompts/mutant_forge.v1.md +31 -0
- counterproof/llm/prompts/mutant_forge.v2.md +35 -0
- counterproof/llm/prompts/oracle_writer.v1.md +41 -0
- counterproof/llm/prompts/oracle_writer.v1.typescript.md +45 -0
- counterproof/llm/prompts/triage_judge.v1.md +40 -0
- counterproof/llm/prompts/triage_judge.v2.md +43 -0
- counterproof/llm/prompts/triage_judge.v2.typescript.md +43 -0
- counterproof/llm/provider.py +144 -0
- counterproof/llm/redact.py +92 -0
- counterproof/llm/roles.py +210 -0
- counterproof/llm/schemas.py +52 -0
- counterproof/log.py +55 -0
- counterproof/models.py +272 -0
- counterproof/orchestrator.py +767 -0
- counterproof/probes/__init__.py +1 -0
- counterproof/probes/blind_oracle/__init__.py +1 -0
- counterproof/probes/blind_oracle/export.py +77 -0
- counterproof/probes/blind_oracle/interface.py +382 -0
- counterproof/probes/blind_oracle/interface_ts.py +535 -0
- counterproof/probes/blind_oracle/probe.py +192 -0
- counterproof/probes/blind_oracle/runner.py +161 -0
- counterproof/probes/blind_oracle/triage.py +103 -0
- counterproof/probes/blind_oracle/writer.py +308 -0
- counterproof/probes/blind_oracle/writer_ts.py +366 -0
- counterproof/probes/claim_audit/__init__.py +1 -0
- counterproof/probes/claim_audit/extract.py +151 -0
- counterproof/probes/claim_audit/match.py +392 -0
- counterproof/probes/claim_audit/parsers/__init__.py +35 -0
- counterproof/probes/claim_audit/parsers/base.py +45 -0
- counterproof/probes/claim_audit/parsers/claude_code.py +179 -0
- counterproof/probes/claim_audit/probe.py +144 -0
- counterproof/probes/integrity/__init__.py +1 -0
- counterproof/probes/integrity/overfit.py +246 -0
- counterproof/probes/integrity/overfit_ts.py +173 -0
- counterproof/probes/integrity/probe.py +158 -0
- counterproof/probes/integrity/python_rules.py +303 -0
- counterproof/probes/integrity/waivers.py +62 -0
- counterproof/probes/lang.py +107 -0
- counterproof/probes/mutants/__init__.py +1 -0
- counterproof/probes/mutants/apply.py +73 -0
- counterproof/probes/mutants/equivalence.py +172 -0
- counterproof/probes/mutants/fuzz.py +152 -0
- counterproof/probes/mutants/fuzz_ts.py +200 -0
- counterproof/probes/mutants/llm_forge.py +164 -0
- counterproof/probes/mutants/probe.py +314 -0
- counterproof/probes/mutants/rule_forge_py.py +363 -0
- counterproof/probes/mutants/rule_forge_ts.py +329 -0
- counterproof/probes/mutants/runner.py +128 -0
- counterproof/probes/mutants/scope.py +84 -0
- counterproof/probes/mutants/select.py +57 -0
- counterproof/probes/pyast.py +442 -0
- counterproof/probes/red_green.py +216 -0
- counterproof/probes/tsast.py +587 -0
- counterproof/py.typed +0 -0
- counterproof/report/__init__.py +15 -0
- counterproof/report/bundle.py +59 -0
- counterproof/report/coach.py +150 -0
- counterproof/report/html.py +116 -0
- counterproof/report/markdown.py +259 -0
- counterproof/report/sarif.py +137 -0
- counterproof/report/templates/report.html.j2 +98 -0
- counterproof/report/terminal.py +31 -0
- counterproof/verdict/__init__.py +1 -0
- counterproof/verdict/engine.py +25 -0
- counterproof/verdict/rules.py +314 -0
- counterproof/workspace/__init__.py +1 -0
- counterproof/workspace/env.py +106 -0
- counterproof/workspace/links.py +54 -0
- counterproof/workspace/manager.py +136 -0
- counterproof/workspace/sandbox_docker.py +335 -0
- counterproof/workspace/sandbox_none.py +113 -0
- counterproof-1.0.0.dist-info/METADATA +332 -0
- counterproof-1.0.0.dist-info/RECORD +125 -0
- counterproof-1.0.0.dist-info/WHEEL +4 -0
- counterproof-1.0.0.dist-info/entry_points.txt +2 -0
- counterproof-1.0.0.dist-info/licenses/LICENSE +202 -0
counterproof/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Counterproof: is an AI-written change actually proven by its tests?"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version("counterproof")
|
|
9
|
+
except PackageNotFoundError: # pragma: no cover - running from a source tree
|
|
10
|
+
__version__ = "0.0.0+unknown"
|
|
11
|
+
|
|
12
|
+
__all__ = ["__version__"]
|
counterproof/__main__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Test-runner adapters behind one interface (FR-AD)."""
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Common adapter interface (PRD section 7.3) and shared failure classification."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Final, Protocol
|
|
9
|
+
|
|
10
|
+
from counterproof.adapters.junit import JUnitCase
|
|
11
|
+
from counterproof.models import FailureKind, RunResult, TestId
|
|
12
|
+
from counterproof.workspace.manager import Workspace
|
|
13
|
+
|
|
14
|
+
NOT_REPORTED: Final = "no result reported"
|
|
15
|
+
"""Message for requested tests missing from the report (for example after a collection abort)."""
|
|
16
|
+
|
|
17
|
+
FileLine = tuple[str, int]
|
|
18
|
+
"""Workspace-relative POSIX path and 1-based line number."""
|
|
19
|
+
CoverageMap = dict[FileLine, set[str]]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class PrepResult:
|
|
24
|
+
ok: bool
|
|
25
|
+
message: str = ""
|
|
26
|
+
interpreter: str | None = None
|
|
27
|
+
env: dict[str, str] = field(default_factory=dict)
|
|
28
|
+
capabilities: frozenset[str] = frozenset()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class CollectionError(Exception):
|
|
32
|
+
"""Test discovery failed (syntax error, import error at collection time)."""
|
|
33
|
+
|
|
34
|
+
def __init__(self, message: str, output: str = "") -> None:
|
|
35
|
+
super().__init__(message)
|
|
36
|
+
self.output = output
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Adapter(Protocol):
|
|
40
|
+
name: str
|
|
41
|
+
|
|
42
|
+
def detect(self, repo: Path) -> float: ...
|
|
43
|
+
|
|
44
|
+
def is_test_file(self, path: str) -> bool: ...
|
|
45
|
+
|
|
46
|
+
def prepare(self, ws: Workspace) -> PrepResult: ...
|
|
47
|
+
|
|
48
|
+
def list_tests(self, ws: Workspace, files: list[str] | None = None) -> list[TestId]: ...
|
|
49
|
+
|
|
50
|
+
def run(
|
|
51
|
+
self,
|
|
52
|
+
ws: Workspace,
|
|
53
|
+
tests: list[str],
|
|
54
|
+
*,
|
|
55
|
+
timeout_s: float,
|
|
56
|
+
fail_fast: bool,
|
|
57
|
+
junit_path: Path,
|
|
58
|
+
coverage_file: Path | None = None,
|
|
59
|
+
) -> RunResult: ...
|
|
60
|
+
|
|
61
|
+
def coverage_map(
|
|
62
|
+
self,
|
|
63
|
+
ws: Workspace,
|
|
64
|
+
tests: list[str],
|
|
65
|
+
*,
|
|
66
|
+
timeout_s: float,
|
|
67
|
+
junit_path: Path,
|
|
68
|
+
coverage_file: Path,
|
|
69
|
+
) -> CoverageMap | None: ...
|
|
70
|
+
|
|
71
|
+
def related_tests(self, src_files: list[str], all_tests: list[TestId]) -> list[TestId]: ...
|
|
72
|
+
|
|
73
|
+
def failure_kind(self, case: JUnitCase) -> FailureKind: ...
|
|
74
|
+
|
|
75
|
+
def run_script(
|
|
76
|
+
self, ws: Workspace, script: Path, args: list[str], *, timeout_s: float = 120.0
|
|
77
|
+
) -> str | None: ...
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_COLLECTION_HINTS: Final = re.compile(
|
|
81
|
+
r"(collection failure|ERROR collecting|SyntaxError|IndentationError|"
|
|
82
|
+
r"ImportError while importing test module)"
|
|
83
|
+
)
|
|
84
|
+
_ENV_HINTS: Final = re.compile(
|
|
85
|
+
r"(ModuleNotFoundError: No module named|ConnectionRefusedError|ConnectionError|"
|
|
86
|
+
r"socket\.gaierror|Temporary failure in name resolution)"
|
|
87
|
+
)
|
|
88
|
+
_ASSERTION_HINTS: Final = re.compile(
|
|
89
|
+
r"(AssertionError|^assert |\nE\s+assert |DID NOT RAISE|Failed: DID NOT|"
|
|
90
|
+
r"AssertionError:|expect\(|toBe|toEqual)"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def classify_failure(case: JUnitCase, *, timed_out: bool = False) -> FailureKind:
|
|
95
|
+
"""FR-AD-04, change-agnostic part.
|
|
96
|
+
|
|
97
|
+
``API_ABSENT`` needs to know which symbols the change added, so the red-green probe (M2)
|
|
98
|
+
refines ``EXCEPTION_BEHAVIOURAL`` and ``ENV`` results into ``API_ABSENT`` where it applies.
|
|
99
|
+
"""
|
|
100
|
+
if timed_out:
|
|
101
|
+
return "TIMEOUT"
|
|
102
|
+
blob = f"{case.message}\n{case.text}"
|
|
103
|
+
if case.is_collection_error or _COLLECTION_HINTS.search(blob):
|
|
104
|
+
return "COLLECTION"
|
|
105
|
+
if _ASSERTION_HINTS.search(blob):
|
|
106
|
+
return "ASSERTION"
|
|
107
|
+
if _ENV_HINTS.search(blob):
|
|
108
|
+
return "ENV"
|
|
109
|
+
return "EXCEPTION_BEHAVIOURAL"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_API_HINTS: Final = re.compile(
|
|
113
|
+
r"(ImportError|cannot import name|AttributeError|has no attribute|TypeError|"
|
|
114
|
+
r"unexpected keyword argument|positional argument|is not a function|is not callable|"
|
|
115
|
+
r"ModuleNotFoundError|is not a constructor|does not provide an export named|"
|
|
116
|
+
r"Failed to load url|Failed to resolve import|Cannot find module)"
|
|
117
|
+
)
|
|
118
|
+
# JavaScript: `(0 , __vi_import_0__.splitCents) is not a function`, missing ESM exports and
|
|
119
|
+
# modules that do not exist on the old code.
|
|
120
|
+
_JS_NOT_CALLABLE: Final = re.compile(r"([A-Za-z_$][\w$]*)\)? is not a (?:function|constructor)")
|
|
121
|
+
_JS_EXPORT: Final = re.compile(r"does not provide an export named '([\w$]+)'")
|
|
122
|
+
_JS_MODULE: Final = re.compile(
|
|
123
|
+
r"(?:Failed to load url|Failed to resolve import|Cannot find module)\s+['\"]?([^'\"\s)]+)"
|
|
124
|
+
)
|
|
125
|
+
_QUOTED: Final = re.compile(r"['\"]([A-Za-z_][\w.]*)['\"]")
|
|
126
|
+
_UNEXPECTED_KW: Final = re.compile(r"(\w+)\(\) got an unexpected keyword argument '(\w+)'")
|
|
127
|
+
_POSITIONAL: Final = re.compile(r"(\w+)\(\) takes .* positional arguments? but .* given")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def refine_api_absent(
|
|
131
|
+
kind: FailureKind,
|
|
132
|
+
text: str,
|
|
133
|
+
added_symbols: set[str],
|
|
134
|
+
new_modules: set[str],
|
|
135
|
+
) -> FailureKind:
|
|
136
|
+
"""FR-AD-04 ``API_ABSENT``: the failure names a symbol, argument or module the change added.
|
|
137
|
+
|
|
138
|
+
``added_symbols`` holds names and ``func(param)`` pairs from :func:`pyast.added_symbols`.
|
|
139
|
+
A test that cannot even call the old code is weak evidence (FR-RG-05), not a behavioural
|
|
140
|
+
failure, so it must not count as red-green proof.
|
|
141
|
+
"""
|
|
142
|
+
if kind not in {"EXCEPTION_BEHAVIOURAL", "ENV", "COLLECTION"} or not _API_HINTS.search(text):
|
|
143
|
+
return kind
|
|
144
|
+
names = {s for s in added_symbols if "(" not in s}
|
|
145
|
+
params = {tuple(s[:-1].split("(", 1)) for s in added_symbols if "(" in s}
|
|
146
|
+
for quoted in _QUOTED.findall(text):
|
|
147
|
+
if quoted in names or quoted.rsplit(".", 1)[-1] in names or quoted in new_modules:
|
|
148
|
+
return "API_ABSENT"
|
|
149
|
+
for func, arg in _UNEXPECTED_KW.findall(text):
|
|
150
|
+
if (func, arg) in params:
|
|
151
|
+
return "API_ABSENT"
|
|
152
|
+
for func in _POSITIONAL.findall(text):
|
|
153
|
+
if any(f == func for f, _ in params):
|
|
154
|
+
return "API_ABSENT"
|
|
155
|
+
for name in [*_JS_NOT_CALLABLE.findall(text), *_JS_EXPORT.findall(text)]:
|
|
156
|
+
if name in names:
|
|
157
|
+
return "API_ABSENT"
|
|
158
|
+
module_stems = {m.rsplit(".", 1)[-1] for m in new_modules}
|
|
159
|
+
for spec in _JS_MODULE.findall(text):
|
|
160
|
+
stem = re.split(r"[/\\]", spec)[-1].split(".", 1)[0]
|
|
161
|
+
if stem and stem in module_stems:
|
|
162
|
+
return "API_ABSENT"
|
|
163
|
+
return kind
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""StackDetector: choose adapters from project manifests (PRD section 6.1, FR-AD-06).
|
|
2
|
+
|
|
3
|
+
Every adapter scores the repository; those at or above :data:`MIN_CONFIDENCE` are active.
|
|
4
|
+
A run uses the adapter that owns the changed code: Python files belong to pytest, TS/JS
|
|
5
|
+
files to Vitest or Jest. Projects in subdirectories of a monorepo (a nested
|
|
6
|
+
``package.json`` or ``pyproject.toml``) are not run separately yet.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Mapping, Sequence
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Final
|
|
14
|
+
|
|
15
|
+
from counterproof.adapters.base import Adapter
|
|
16
|
+
from counterproof.adapters.jest_adapter import JestAdapter
|
|
17
|
+
from counterproof.adapters.pytest_adapter import PytestAdapter
|
|
18
|
+
from counterproof.adapters.vitest_adapter import VitestAdapter
|
|
19
|
+
from counterproof.models import ChangeSet
|
|
20
|
+
from counterproof.probes import lang
|
|
21
|
+
|
|
22
|
+
MIN_CONFIDENCE: Final = 0.5
|
|
23
|
+
_LANGUAGE: Final = {"pytest": "python", "vitest": "typescript", "jest": "typescript"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def detect_adapters(
|
|
27
|
+
repo: Path,
|
|
28
|
+
*,
|
|
29
|
+
env_pass: Sequence[str] = (),
|
|
30
|
+
source_env: Mapping[str, str] | None = None,
|
|
31
|
+
deps_root: Path | None = None,
|
|
32
|
+
) -> list[Adapter]:
|
|
33
|
+
"""Adapters whose confidence for ``repo`` is at least :data:`MIN_CONFIDENCE`.
|
|
34
|
+
|
|
35
|
+
``repo`` is the tree to inspect (the head worktree); ``deps_root`` is where untracked
|
|
36
|
+
dependencies live (the user's repository: its ``.venv`` and ``node_modules``).
|
|
37
|
+
"""
|
|
38
|
+
deps = deps_root or repo
|
|
39
|
+
candidates: list[Adapter] = [
|
|
40
|
+
PytestAdapter(deps, env_pass=env_pass, source_env=source_env),
|
|
41
|
+
VitestAdapter(deps, env_pass=env_pass, source_env=source_env),
|
|
42
|
+
JestAdapter(deps, env_pass=env_pass, source_env=source_env),
|
|
43
|
+
]
|
|
44
|
+
scored = [(a.detect(repo), i, a) for i, a in enumerate(candidates)]
|
|
45
|
+
return [
|
|
46
|
+
a for score, _, a in sorted(scored, key=lambda s: (-s[0], s[1])) if score >= MIN_CONFIDENCE
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def owner_language(path: str) -> str | None:
|
|
51
|
+
if lang.is_python(path):
|
|
52
|
+
return "python"
|
|
53
|
+
if lang.is_typescript(path):
|
|
54
|
+
return "typescript"
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def choose_adapter(adapters: Sequence[Adapter], change: ChangeSet) -> Adapter | None:
|
|
59
|
+
"""FR-AD-06: the active adapter whose language owns most of the changed code files."""
|
|
60
|
+
if not adapters:
|
|
61
|
+
return None
|
|
62
|
+
counts: dict[str, int] = {}
|
|
63
|
+
for f in change.files:
|
|
64
|
+
language = owner_language(f.path)
|
|
65
|
+
if language and f.role in {"source", "test"}:
|
|
66
|
+
counts[language] = counts.get(language, 0) + 1
|
|
67
|
+
if not counts:
|
|
68
|
+
return adapters[0]
|
|
69
|
+
ranked = sorted(adapters, key=lambda a: -counts.get(_LANGUAGE.get(a.name, ""), 0))
|
|
70
|
+
best = ranked[0]
|
|
71
|
+
return best if counts.get(_LANGUAGE.get(best.name, ""), 0) else adapters[0]
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""JestAdapter (FR-AD-03), checked against Jest 30.5 and jest-junit 17 (November 2026).
|
|
2
|
+
|
|
3
|
+
* Run: ``jest --ci --reporters=jest-junit --runTestsByPath <files> -t '^(?:names)$'``.
|
|
4
|
+
jest-junit is configured through its environment variables: ``classname`` is the file
|
|
5
|
+
path, ``name`` is ``<describe> > ... > <title>`` (ancestors joined by `` > ``), so node
|
|
6
|
+
ids match Vitest's: ``<file>::<full name>``. Jest's ``-t`` matches the full name joined
|
|
7
|
+
with spaces. Suite load errors are reported as a case named after the file.
|
|
8
|
+
* List: ``jest --listTests --json`` gives files; test names come from a static scan of each
|
|
9
|
+
file (tree-sitter). Related tests: ``jest --listTests --findRelatedTests`` (runs nothing).
|
|
10
|
+
* No per-test coverage in v1. Jest's cache goes to the run directory.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
from collections.abc import Mapping, Sequence
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Final
|
|
20
|
+
|
|
21
|
+
from counterproof.adapters.base import (
|
|
22
|
+
NOT_REPORTED,
|
|
23
|
+
CollectionError,
|
|
24
|
+
CoverageMap,
|
|
25
|
+
PrepResult,
|
|
26
|
+
classify_failure,
|
|
27
|
+
)
|
|
28
|
+
from counterproof.adapters.junit import JUnitCase, parse_junit
|
|
29
|
+
from counterproof.adapters.node import (
|
|
30
|
+
declared_dependencies,
|
|
31
|
+
ensure_node_modules,
|
|
32
|
+
js_regex_escape,
|
|
33
|
+
node_env,
|
|
34
|
+
node_version,
|
|
35
|
+
package_bin,
|
|
36
|
+
resolve_node,
|
|
37
|
+
)
|
|
38
|
+
from counterproof.change.classify import is_test_path
|
|
39
|
+
from counterproof.evidence.hashing import sha256_bytes, sha256_file
|
|
40
|
+
from counterproof.log import get_logger
|
|
41
|
+
from counterproof.models import FailureInfo, FailureKind, RunResult, TestId, TestOutcome
|
|
42
|
+
from counterproof.probes import tsast
|
|
43
|
+
from counterproof.workspace.manager import Workspace
|
|
44
|
+
from counterproof.workspace.sandbox_docker import SandboxHost
|
|
45
|
+
|
|
46
|
+
log = get_logger("adapters.jest")
|
|
47
|
+
|
|
48
|
+
COLLECT_TIMEOUT_S: Final = 180.0
|
|
49
|
+
MAX_CMDLINE_CHARS: Final = 7000
|
|
50
|
+
SUITE_ERROR: Final = "Test suite failed to run"
|
|
51
|
+
CONFIG_FILES: Final = tuple(
|
|
52
|
+
f"jest.config.{ext}" for ext in ("js", "cjs", "mjs", "ts", "cts", "mts", "json")
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class JestAdapter(SandboxHost):
|
|
57
|
+
name = "jest"
|
|
58
|
+
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
repo_root: Path,
|
|
62
|
+
*,
|
|
63
|
+
env_pass: Sequence[str] = (),
|
|
64
|
+
source_env: Mapping[str, str] | None = None,
|
|
65
|
+
) -> None:
|
|
66
|
+
self.repo_root = repo_root
|
|
67
|
+
self.env_pass = tuple(env_pass)
|
|
68
|
+
self.source_env: Mapping[str, str] = os.environ if source_env is None else source_env
|
|
69
|
+
self.node = resolve_node(self.source_env)
|
|
70
|
+
self._listed: Workspace | None = None
|
|
71
|
+
|
|
72
|
+
def detect(self, repo: Path) -> float:
|
|
73
|
+
deps = declared_dependencies(repo)
|
|
74
|
+
if any((repo / name).is_file() for name in CONFIG_FILES):
|
|
75
|
+
return 1.0
|
|
76
|
+
if "jest" in deps and "vitest" not in deps:
|
|
77
|
+
return 0.9
|
|
78
|
+
return 0.4 if "jest" in deps else 0.0
|
|
79
|
+
|
|
80
|
+
def is_test_file(self, path: str) -> bool:
|
|
81
|
+
return tsast.is_ts_path(path) and is_test_path(path)
|
|
82
|
+
|
|
83
|
+
def child_env(self, ws: Workspace, junit_path: Path | None = None) -> dict[str, str]:
|
|
84
|
+
env = node_env(self.source_env, self.env_pass, self.node)
|
|
85
|
+
env.update(
|
|
86
|
+
JEST_JUNIT_CLASSNAME="{filepath}",
|
|
87
|
+
JEST_JUNIT_TITLE="{classname} > {title}",
|
|
88
|
+
JEST_JUNIT_ANCESTOR_SEPARATOR=" > ",
|
|
89
|
+
JEST_JUNIT_REPORT_TEST_SUITE_ERRORS="true",
|
|
90
|
+
)
|
|
91
|
+
if junit_path is not None:
|
|
92
|
+
env["JEST_JUNIT_OUTPUT_FILE"] = str(junit_path)
|
|
93
|
+
return env
|
|
94
|
+
|
|
95
|
+
def _bin(self, ws: Workspace) -> Path | None:
|
|
96
|
+
return package_bin(ws.root / "node_modules", "jest", "jest")
|
|
97
|
+
|
|
98
|
+
def _node(self) -> str:
|
|
99
|
+
if self.node is None:
|
|
100
|
+
raise CollectionError("node not found on PATH (set COUNTERPROOF_NODE)")
|
|
101
|
+
return self.node
|
|
102
|
+
|
|
103
|
+
def _base_cmd(self, ws: Workspace) -> list[str]:
|
|
104
|
+
return [
|
|
105
|
+
self._node(),
|
|
106
|
+
str(self._bin(ws)),
|
|
107
|
+
"--ci",
|
|
108
|
+
"--rootDir",
|
|
109
|
+
str(ws.root),
|
|
110
|
+
"--cacheDirectory",
|
|
111
|
+
str(ws.root.parent / f".jest-cache-{ws.name}"),
|
|
112
|
+
"--watchman=false",
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
def prepare(self, ws: Workspace) -> PrepResult:
|
|
116
|
+
if self.node is None:
|
|
117
|
+
return PrepResult(
|
|
118
|
+
ok=False, message="node not found on PATH (set COUNTERPROOF_NODE to its path)"
|
|
119
|
+
)
|
|
120
|
+
problem = ensure_node_modules(self.repo_root, ws)
|
|
121
|
+
if problem:
|
|
122
|
+
return PrepResult(ok=False, message=problem)
|
|
123
|
+
missing = [p for p in ("jest", "jest-junit") if not (ws.root / "node_modules" / p).is_dir()]
|
|
124
|
+
if self._bin(ws) is None or missing:
|
|
125
|
+
return PrepResult(
|
|
126
|
+
ok=False,
|
|
127
|
+
message=f"{' and '.join(missing or ['jest'])} not installed in node_modules; "
|
|
128
|
+
"install the project's dependencies first (FR-AD-05)",
|
|
129
|
+
)
|
|
130
|
+
env = self.child_env(ws)
|
|
131
|
+
version = node_version(self.node, env, ws.root)
|
|
132
|
+
if version is None:
|
|
133
|
+
return PrepResult(ok=False, message=f"cannot run {self.node}")
|
|
134
|
+
data = json.loads(
|
|
135
|
+
(ws.root / "node_modules" / "jest" / "package.json").read_text(encoding="utf-8")
|
|
136
|
+
)
|
|
137
|
+
log.info("jest %s on node %s (%s)", data.get("version"), version, self.node)
|
|
138
|
+
return PrepResult(
|
|
139
|
+
ok=True,
|
|
140
|
+
interpreter=self.node,
|
|
141
|
+
env=env,
|
|
142
|
+
capabilities=frozenset({"jest"}),
|
|
143
|
+
message=f"jest {data.get('version')}, node {version}",
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def _list_files(self, ws: Workspace, extra: list[str]) -> list[str]:
|
|
147
|
+
cmd = [*self._base_cmd(ws), "--listTests", "--json", *extra]
|
|
148
|
+
proc = self.run_process(
|
|
149
|
+
cmd, cwd=ws.root, env=self.child_env(ws), timeout_s=COLLECT_TIMEOUT_S
|
|
150
|
+
)
|
|
151
|
+
start = proc.stdout.find("[")
|
|
152
|
+
try:
|
|
153
|
+
files = json.loads(proc.stdout[start:]) if start >= 0 else None
|
|
154
|
+
except json.JSONDecodeError:
|
|
155
|
+
files = None
|
|
156
|
+
if proc.timed_out or proc.exit_code != 0 or not isinstance(files, list):
|
|
157
|
+
raise CollectionError(
|
|
158
|
+
f"jest --listTests failed (exit {proc.exit_code})", proc.stdout + proc.stderr
|
|
159
|
+
)
|
|
160
|
+
root = ws.root.resolve()
|
|
161
|
+
out: list[str] = []
|
|
162
|
+
for f in files:
|
|
163
|
+
try:
|
|
164
|
+
out.append(Path(str(f)).resolve().relative_to(root).as_posix())
|
|
165
|
+
except ValueError:
|
|
166
|
+
continue
|
|
167
|
+
return sorted(out)
|
|
168
|
+
|
|
169
|
+
def list_tests(self, ws: Workspace, files: list[str] | None = None) -> list[TestId]:
|
|
170
|
+
problem = ensure_node_modules(self.repo_root, ws)
|
|
171
|
+
if problem:
|
|
172
|
+
raise CollectionError(problem)
|
|
173
|
+
listed = self._list_files(ws, [])
|
|
174
|
+
if files:
|
|
175
|
+
listed = [f for f in listed if f in set(files)]
|
|
176
|
+
self._listed = ws
|
|
177
|
+
tests: list[TestId] = []
|
|
178
|
+
for rel in listed:
|
|
179
|
+
text = (ws.root / rel).read_text(encoding="utf-8", errors="replace")
|
|
180
|
+
analysed = tsast.analyse_test_file(text, rel)
|
|
181
|
+
if analysed is None:
|
|
182
|
+
raise CollectionError(f"cannot parse test file {rel}")
|
|
183
|
+
tests += [TestId(adapter=self.name, file=rel, name=n) for n in analysed.tests]
|
|
184
|
+
return tests
|
|
185
|
+
|
|
186
|
+
def related_tests(self, src_files: list[str], all_tests: list[TestId]) -> list[TestId]:
|
|
187
|
+
if self._listed is None or not src_files:
|
|
188
|
+
return []
|
|
189
|
+
try:
|
|
190
|
+
files = set(self._list_files(self._listed, ["--findRelatedTests", *src_files]))
|
|
191
|
+
except CollectionError as exc:
|
|
192
|
+
log.warning("jest --findRelatedTests failed: %s", exc)
|
|
193
|
+
return []
|
|
194
|
+
return [t for t in all_tests if t.file in files]
|
|
195
|
+
|
|
196
|
+
def _targets(self, tests: list[str]) -> list[str]:
|
|
197
|
+
files = sorted({t.partition("::")[0] for t in tests})
|
|
198
|
+
names = sorted({t.partition("::")[2].replace(" > ", " ") for t in tests})
|
|
199
|
+
pattern = "^(?:" + "|".join(js_regex_escape(n) for n in names) + ")$"
|
|
200
|
+
args = ["--runTestsByPath", *files, "-t", pattern]
|
|
201
|
+
if sum(len(a) + 1 for a in args) > MAX_CMDLINE_CHARS:
|
|
202
|
+
return ["--runTestsByPath", *files]
|
|
203
|
+
return args
|
|
204
|
+
|
|
205
|
+
def run(
|
|
206
|
+
self,
|
|
207
|
+
ws: Workspace,
|
|
208
|
+
tests: list[str],
|
|
209
|
+
*,
|
|
210
|
+
timeout_s: float,
|
|
211
|
+
fail_fast: bool,
|
|
212
|
+
junit_path: Path,
|
|
213
|
+
coverage_file: Path | None = None,
|
|
214
|
+
) -> RunResult:
|
|
215
|
+
junit_path.parent.mkdir(parents=True, exist_ok=True)
|
|
216
|
+
junit_path.unlink(missing_ok=True)
|
|
217
|
+
problem = ensure_node_modules(self.repo_root, ws)
|
|
218
|
+
cmd = [*self._base_cmd(ws), "--reporters=jest-junit"]
|
|
219
|
+
if fail_fast:
|
|
220
|
+
cmd.append("--bail=1")
|
|
221
|
+
cmd += self._targets(tests)
|
|
222
|
+
if problem:
|
|
223
|
+
out, exit_code, duration, timed_out = problem, 1, 0.0, False
|
|
224
|
+
else:
|
|
225
|
+
proc = self.run_process(
|
|
226
|
+
cmd, cwd=ws.root, env=self.child_env(ws, junit_path), timeout_s=timeout_s
|
|
227
|
+
)
|
|
228
|
+
out = f"{proc.stdout}\n{proc.stderr}"
|
|
229
|
+
exit_code, duration, timed_out = proc.exit_code, proc.duration_s, proc.timed_out
|
|
230
|
+
junit_path.with_suffix(".log").write_text(
|
|
231
|
+
f"$ {' '.join(cmd)}\n[exit {exit_code}{' TIMEOUT' if timed_out else ''}]\n{out}",
|
|
232
|
+
encoding="utf-8",
|
|
233
|
+
)
|
|
234
|
+
outcomes, failures, durations = self._map_results(
|
|
235
|
+
parse_junit(junit_path), tests, timed_out=timed_out, fail_fast=fail_fast
|
|
236
|
+
)
|
|
237
|
+
return RunResult(
|
|
238
|
+
tests=outcomes,
|
|
239
|
+
failures=failures,
|
|
240
|
+
duration_s=round(duration, 6),
|
|
241
|
+
command=cmd,
|
|
242
|
+
exit_code=exit_code,
|
|
243
|
+
junit_sha256=sha256_file(junit_path) if junit_path.is_file() else sha256_bytes(b""),
|
|
244
|
+
test_durations=dict(sorted(durations.items())),
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
def _map_results(
|
|
248
|
+
self, cases: list[JUnitCase], tests: list[str], *, timed_out: bool, fail_fast: bool
|
|
249
|
+
) -> tuple[dict[str, TestOutcome], dict[str, FailureInfo], dict[str, float]]:
|
|
250
|
+
wanted = set(tests)
|
|
251
|
+
outcomes: dict[str, TestOutcome] = {}
|
|
252
|
+
failures: dict[str, FailureInfo] = {}
|
|
253
|
+
durations: dict[str, float] = {}
|
|
254
|
+
for case in cases:
|
|
255
|
+
if case.classname == SUITE_ERROR:
|
|
256
|
+
file = case.name.replace("\\", "/")
|
|
257
|
+
for t in tests:
|
|
258
|
+
if t.partition("::")[0] == file and t not in outcomes:
|
|
259
|
+
outcomes[t] = "error"
|
|
260
|
+
failures[t] = FailureInfo(
|
|
261
|
+
kind="COLLECTION", message=SUITE_ERROR, traceback_excerpt=case.text
|
|
262
|
+
)
|
|
263
|
+
continue
|
|
264
|
+
file = case.classname.replace("\\", "/")
|
|
265
|
+
name = case.name.removeprefix(" > ")
|
|
266
|
+
nodeid = f"{file}::{name}"
|
|
267
|
+
if nodeid not in wanted:
|
|
268
|
+
continue
|
|
269
|
+
outcomes[nodeid] = case.outcome
|
|
270
|
+
durations[nodeid] = case.time_s
|
|
271
|
+
if case.outcome in {"fail", "error"}:
|
|
272
|
+
failures[nodeid] = FailureInfo(
|
|
273
|
+
kind=self.failure_kind(case), message=case.message, traceback_excerpt=case.text
|
|
274
|
+
)
|
|
275
|
+
stopped_early = fail_fast and any(o in {"fail", "error"} for o in outcomes.values())
|
|
276
|
+
for t in tests:
|
|
277
|
+
if t in outcomes:
|
|
278
|
+
continue
|
|
279
|
+
if timed_out:
|
|
280
|
+
outcomes[t] = "timeout"
|
|
281
|
+
failures[t] = FailureInfo(kind="TIMEOUT", message="run timed out")
|
|
282
|
+
elif not stopped_early:
|
|
283
|
+
outcomes[t] = "error"
|
|
284
|
+
failures[t] = FailureInfo(kind="COLLECTION", message=NOT_REPORTED)
|
|
285
|
+
return outcomes, failures, durations
|
|
286
|
+
|
|
287
|
+
def failure_kind(self, case: JUnitCase) -> FailureKind:
|
|
288
|
+
return classify_failure(case)
|
|
289
|
+
|
|
290
|
+
def run_script(
|
|
291
|
+
self, ws: Workspace, script: Path, args: list[str], *, timeout_s: float = 120.0
|
|
292
|
+
) -> str | None:
|
|
293
|
+
"""Run a generated Jest test file inside the worktree (fast-check fuzzer)."""
|
|
294
|
+
if ensure_node_modules(self.repo_root, ws) or self.node is None:
|
|
295
|
+
return None
|
|
296
|
+
rel = script.relative_to(ws.root).as_posix()
|
|
297
|
+
proc = self.run_process(
|
|
298
|
+
[*self._base_cmd(ws), "--runTestsByPath", rel],
|
|
299
|
+
cwd=ws.root,
|
|
300
|
+
env=self.child_env(ws),
|
|
301
|
+
timeout_s=timeout_s,
|
|
302
|
+
)
|
|
303
|
+
return None if proc.timed_out or proc.exit_code != 0 else proc.stdout + proc.stderr
|
|
304
|
+
|
|
305
|
+
def coverage_map(
|
|
306
|
+
self,
|
|
307
|
+
ws: Workspace,
|
|
308
|
+
tests: list[str],
|
|
309
|
+
*,
|
|
310
|
+
timeout_s: float,
|
|
311
|
+
junit_path: Path,
|
|
312
|
+
coverage_file: Path,
|
|
313
|
+
) -> CoverageMap | None:
|
|
314
|
+
return None
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""JUnit XML parsing (pytest ``junit_family=xunit2``; Vitest and Jest later).
|
|
2
|
+
|
|
3
|
+
pytest identifies a case by ``classname`` (dotted module path, plus classes) and ``name``.
|
|
4
|
+
We map those back to node ids by computing the expected key for every requested node id, so
|
|
5
|
+
parametrised ids and nested classes round-trip exactly. Collection errors appear as a case
|
|
6
|
+
with an empty classname whose ``name`` is the dotted module path.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import xml.etree.ElementTree as ET
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Final, Literal
|
|
15
|
+
|
|
16
|
+
CaseOutcome = Literal["pass", "fail", "error", "skip"]
|
|
17
|
+
COLLECTION_FAILURE: Final = "collection failure"
|
|
18
|
+
MAX_TEXT: Final = 4000
|
|
19
|
+
_TAG_OUTCOME: Final[dict[str, CaseOutcome]] = {
|
|
20
|
+
"failure": "fail",
|
|
21
|
+
"error": "error",
|
|
22
|
+
"skipped": "skip",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class JUnitCase:
|
|
28
|
+
classname: str
|
|
29
|
+
name: str
|
|
30
|
+
outcome: CaseOutcome
|
|
31
|
+
message: str
|
|
32
|
+
text: str
|
|
33
|
+
time_s: float
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def is_collection_error(self) -> bool:
|
|
37
|
+
return self.outcome == "error" and (
|
|
38
|
+
self.message == COLLECTION_FAILURE or (not self.classname and "." in self.name)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_junit(path: Path) -> list[JUnitCase]:
|
|
43
|
+
"""Parse a junit file; a missing or empty file yields no cases."""
|
|
44
|
+
if not path.is_file() or path.stat().st_size == 0:
|
|
45
|
+
return []
|
|
46
|
+
try:
|
|
47
|
+
root = ET.parse(path).getroot() # noqa: S314 - runner-written file in our run dir
|
|
48
|
+
except ET.ParseError:
|
|
49
|
+
return []
|
|
50
|
+
cases: list[JUnitCase] = []
|
|
51
|
+
for case in root.iter("testcase"):
|
|
52
|
+
outcome: CaseOutcome = "pass"
|
|
53
|
+
message = text = ""
|
|
54
|
+
for child in case:
|
|
55
|
+
mapped = _TAG_OUTCOME.get(child.tag)
|
|
56
|
+
if mapped is None:
|
|
57
|
+
continue
|
|
58
|
+
outcome = mapped
|
|
59
|
+
message = child.get("message", "") or ""
|
|
60
|
+
text = (child.text or "")[:MAX_TEXT]
|
|
61
|
+
if mapped != "skip":
|
|
62
|
+
break
|
|
63
|
+
try:
|
|
64
|
+
time_s = float(case.get("time", "0") or 0)
|
|
65
|
+
except ValueError:
|
|
66
|
+
time_s = 0.0
|
|
67
|
+
cases.append(
|
|
68
|
+
JUnitCase(
|
|
69
|
+
classname=case.get("classname", "") or "",
|
|
70
|
+
name=case.get("name", "") or "",
|
|
71
|
+
outcome=outcome,
|
|
72
|
+
message=message,
|
|
73
|
+
text=text,
|
|
74
|
+
time_s=time_s,
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
return cases
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def module_of(file: str) -> str:
|
|
81
|
+
"""``tests/test_a.py`` -> ``tests.test_a``."""
|
|
82
|
+
pure = PurePosixPath(file)
|
|
83
|
+
return ".".join([*pure.parent.parts, pure.stem]) if pure.parent.parts else pure.stem
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def junit_key(nodeid: str) -> tuple[str, str]:
|
|
87
|
+
"""``tests/test_a.py::TestK::test_p[1]`` -> ``("tests.test_a.TestK", "test_p[1]")``."""
|
|
88
|
+
file, _, rest = nodeid.partition("::")
|
|
89
|
+
parts = rest.split("::") if rest else []
|
|
90
|
+
module = module_of(file)
|
|
91
|
+
if not parts:
|
|
92
|
+
return "", module
|
|
93
|
+
return ".".join([module, *parts[:-1]]), parts[-1]
|