jitcatch 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jitcatch/__init__.py +2 -0
- jitcatch/__main__.py +5 -0
- jitcatch/adapters/__init__.py +36 -0
- jitcatch/adapters/base.py +137 -0
- jitcatch/adapters/golang.py +88 -0
- jitcatch/adapters/java.py +136 -0
- jitcatch/adapters/javascript.py +165 -0
- jitcatch/adapters/python.py +187 -0
- jitcatch/assessors/__init__.py +94 -0
- jitcatch/assessors/failure_parse.py +126 -0
- jitcatch/assessors/llm_judge.py +75 -0
- jitcatch/assessors/rules.py +343 -0
- jitcatch/cli.py +338 -0
- jitcatch/config.py +259 -0
- jitcatch/engine.py +521 -0
- jitcatch/github.py +90 -0
- jitcatch/gitutil.py +231 -0
- jitcatch/hook.py +98 -0
- jitcatch/jsonutil.py +87 -0
- jitcatch/llm/__init__.py +21 -0
- jitcatch/llm/base.py +100 -0
- jitcatch/llm/providers.py +127 -0
- jitcatch/models.py +150 -0
- jitcatch/prompts.py +191 -0
- jitcatch/report.py +163 -0
- jitcatch/targeting.py +44 -0
- jitcatch/templates.py +76 -0
- jitcatch/userconfig.py +188 -0
- jitcatch-0.2.0.dist-info/METADATA +164 -0
- jitcatch-0.2.0.dist-info/RECORD +34 -0
- jitcatch-0.2.0.dist-info/WHEEL +5 -0
- jitcatch-0.2.0.dist-info/entry_points.txt +2 -0
- jitcatch-0.2.0.dist-info/licenses/LICENSE +21 -0
- jitcatch-0.2.0.dist-info/top_level.txt +1 -0
jitcatch/engine.py
ADDED
|
@@ -0,0 +1,521 @@
|
|
|
1
|
+
"""The Just-in-Time catching pipeline.
|
|
2
|
+
|
|
3
|
+
change ──► targeting ──► intent & risks ──► mutants of the parent ──► tests (pass parent, kill mutant)
|
|
4
|
+
│ │
|
|
5
|
+
└──── dodgy diff: tests that pass parent ────────┤
|
|
6
|
+
▼
|
|
7
|
+
run on change: fail = weak catch, pass = hardening test
|
|
8
|
+
▼
|
|
9
|
+
assessors (rules + LLM ensemble) ──► ranked strong-catch candidates
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import difflib
|
|
14
|
+
import fnmatch
|
|
15
|
+
import logging
|
|
16
|
+
import time
|
|
17
|
+
import uuid
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from pathlib import Path, PurePosixPath
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from . import prompts
|
|
23
|
+
from .adapters import Adapter, TestPlacement, adapter_for
|
|
24
|
+
from .assessors import assess_weak_catch
|
|
25
|
+
from .config import Config
|
|
26
|
+
from .gitutil import show_file, worktree
|
|
27
|
+
from .jsonutil import LLMFormatError, extract_code, extract_json
|
|
28
|
+
from .llm import LLMClient, LLMError
|
|
29
|
+
from .models import (Assessment, ChangeContext, GeneratedTest, Intent, Mutant, Outcome, Risk, TestRun)
|
|
30
|
+
from .targeting import RiskScore, score_change
|
|
31
|
+
|
|
32
|
+
log = logging.getLogger("jitcatch")
|
|
33
|
+
|
|
34
|
+
CHANGE_KINDS = {"refactor", "dead_code_removal", "additive", "bugfix", "feature", "other"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class Target:
|
|
39
|
+
path: str
|
|
40
|
+
adapter: Adapter
|
|
41
|
+
base_text: str
|
|
42
|
+
head_text: str
|
|
43
|
+
patch: str
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class RunResult:
|
|
48
|
+
ctx: ChangeContext
|
|
49
|
+
risk: RiskScore | None = None
|
|
50
|
+
intent: Intent | None = None
|
|
51
|
+
targets: list[str] = field(default_factory=list)
|
|
52
|
+
skipped: list[dict[str, str]] = field(default_factory=list)
|
|
53
|
+
mutants: list[Mutant] = field(default_factory=list)
|
|
54
|
+
tests: list[GeneratedTest] = field(default_factory=list)
|
|
55
|
+
assessments: dict[str, Assessment] = field(default_factory=dict)
|
|
56
|
+
errors: list[str] = field(default_factory=list)
|
|
57
|
+
llm_calls: int = 0
|
|
58
|
+
started_at: float = field(default_factory=time.time)
|
|
59
|
+
finished_at: float = 0.0
|
|
60
|
+
threshold: float = 0.2
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def weak_catches(self) -> list[GeneratedTest]:
|
|
64
|
+
return [t for t in self.tests if t.is_weak_catch]
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def hardening(self) -> list[GeneratedTest]:
|
|
68
|
+
return [t for t in self.tests if t.is_hardening and not t.is_flaky]
|
|
69
|
+
|
|
70
|
+
def candidates(self) -> list[tuple[GeneratedTest, Assessment]]:
|
|
71
|
+
"""Weak catches above the reporting threshold, best first."""
|
|
72
|
+
by_id = {t.id: t for t in self.tests}
|
|
73
|
+
ranked = sorted(
|
|
74
|
+
((by_id[a.test_id], a) for a in self.assessments.values() if not a.discarded),
|
|
75
|
+
key=lambda ta: ta[1].combined, reverse=True,
|
|
76
|
+
)
|
|
77
|
+
return [(t, a) for t, a in ranked if a.combined >= self.threshold]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class Engine:
|
|
81
|
+
def __init__(self, cfg: Config, repo: Path, ctx: ChangeContext, generator: LLMClient,
|
|
82
|
+
judges: list[LLMClient], logprob_judge: LLMClient | None, adapters: list[Adapter],
|
|
83
|
+
force: bool = False):
|
|
84
|
+
self.cfg = cfg
|
|
85
|
+
self.repo = repo
|
|
86
|
+
self.ctx = ctx
|
|
87
|
+
self.gen = generator
|
|
88
|
+
self.judges = judges
|
|
89
|
+
self.logprob_judge = logprob_judge
|
|
90
|
+
self.adapters = adapters
|
|
91
|
+
self.force = force
|
|
92
|
+
self.run_cfg = cfg.run
|
|
93
|
+
self.result = RunResult(ctx=ctx, threshold=float(cfg.run["report_threshold"]))
|
|
94
|
+
self._placements: dict[str, TestPlacement] = {}
|
|
95
|
+
self._adapter_ok: dict[str, tuple[bool, str]] = {}
|
|
96
|
+
self._existing_baseline: dict[str, bool | None] = {}
|
|
97
|
+
|
|
98
|
+
# ------------------------------------------------------------------ orchestration
|
|
99
|
+
def run(self) -> RunResult:
|
|
100
|
+
r = self.result
|
|
101
|
+
targets = self._select_targets()
|
|
102
|
+
r.targets = [t.path for t in targets]
|
|
103
|
+
r.risk = score_change(self.ctx, self.cfg.targeting["sensitive_paths"],
|
|
104
|
+
lambda p: any(a.is_test_file(p) for a in self.adapters))
|
|
105
|
+
if not targets:
|
|
106
|
+
log.info("no testable modified source files in this change")
|
|
107
|
+
return self._finish()
|
|
108
|
+
if not self.force and r.risk.score < float(self.run_cfg["min_risk_score"]):
|
|
109
|
+
r.skipped.append({"path": "*", "reason": f"risk score {r.risk.score} below min_risk_score"})
|
|
110
|
+
log.info("change risk %.3f below threshold; skipping (use --force to override)", r.risk.score)
|
|
111
|
+
return self._finish()
|
|
112
|
+
|
|
113
|
+
log.info("%d target file(s): %s", len(targets), ", ".join(t.path for t in targets))
|
|
114
|
+
log.info("creating worktrees for parent %s and change %s…", self.ctx.base_sha[:8], self.ctx.head_sha[:8])
|
|
115
|
+
link = list(self.run_cfg.get("link_paths") or [])
|
|
116
|
+
with worktree(self.repo, self.ctx.base_sha, "base", link) as base_wt, \
|
|
117
|
+
worktree(self.repo, self.ctx.head_sha, "head", link) as head_wt:
|
|
118
|
+
log.info("inferring the change's intent and risks…")
|
|
119
|
+
r.intent = self._infer_intent(targets)
|
|
120
|
+
for target in targets:
|
|
121
|
+
if "intent_aware" in self.run_cfg["workflows"] and r.intent:
|
|
122
|
+
self._intent_aware(target, base_wt, head_wt)
|
|
123
|
+
if "dodgy_diff" in self.run_cfg["workflows"]:
|
|
124
|
+
self._dodgy_diff(target, base_wt, head_wt)
|
|
125
|
+
self._assess()
|
|
126
|
+
return self._finish()
|
|
127
|
+
|
|
128
|
+
def _finish(self) -> RunResult:
|
|
129
|
+
self.result.finished_at = time.time()
|
|
130
|
+
return self.result
|
|
131
|
+
|
|
132
|
+
# ------------------------------------------------------------------ targeting
|
|
133
|
+
def _select_targets(self) -> list[Target]:
|
|
134
|
+
inc = self.run_cfg.get("include") or []
|
|
135
|
+
exc = self.run_cfg.get("exclude") or []
|
|
136
|
+
chosen: list[Target] = []
|
|
137
|
+
for f in self.ctx.files:
|
|
138
|
+
if inc and not any(fnmatch.fnmatch(f.path, g) for g in inc):
|
|
139
|
+
continue
|
|
140
|
+
if any(fnmatch.fnmatch(f.path, g) for g in exc):
|
|
141
|
+
self._skip(f.path, "excluded by config")
|
|
142
|
+
continue
|
|
143
|
+
ad = adapter_for(self.adapters, f.path)
|
|
144
|
+
if ad is None:
|
|
145
|
+
continue
|
|
146
|
+
if f.status not in ("M", "T"):
|
|
147
|
+
self._skip(f.path, {"A": "new file (no parent behaviour to regress)", "D": "deleted file",
|
|
148
|
+
"R": "renamed file (not supported)", "C": "copied file (not supported)"}
|
|
149
|
+
.get(f.status, f"unsupported change status {f.status}"))
|
|
150
|
+
continue
|
|
151
|
+
ok, why = self._adapter_available(ad)
|
|
152
|
+
if not ok:
|
|
153
|
+
self._skip(f.path, f"{ad.name} toolchain unavailable: {why}")
|
|
154
|
+
continue
|
|
155
|
+
ok, why = ad.can_test(f.path)
|
|
156
|
+
if not ok:
|
|
157
|
+
self._skip(f.path, why)
|
|
158
|
+
continue
|
|
159
|
+
base_text = show_file(self.repo, self.ctx.base_sha, f.old_path)
|
|
160
|
+
head_text = show_file(self.repo, self.ctx.head_sha, f.path)
|
|
161
|
+
if base_text is None or head_text is None:
|
|
162
|
+
self._skip(f.path, "could not read file at base/head")
|
|
163
|
+
continue
|
|
164
|
+
if len(base_text) > int(self.run_cfg["max_file_chars"]):
|
|
165
|
+
self._skip(f.path, f"file larger than max_file_chars ({len(base_text)} chars)")
|
|
166
|
+
continue
|
|
167
|
+
chosen.append(Target(f.path, ad, base_text, head_text, f.patch))
|
|
168
|
+
chosen.sort(key=lambda t: -len(t.patch))
|
|
169
|
+
limit = int(self.run_cfg["max_target_files"])
|
|
170
|
+
for t in chosen[limit:]:
|
|
171
|
+
self._skip(t.path, f"over max_target_files={limit}")
|
|
172
|
+
return chosen[:limit]
|
|
173
|
+
|
|
174
|
+
def _adapter_available(self, ad: Adapter) -> tuple[bool, str]:
|
|
175
|
+
if ad.name not in self._adapter_ok:
|
|
176
|
+
self._adapter_ok[ad.name] = ad.available()
|
|
177
|
+
return self._adapter_ok[ad.name]
|
|
178
|
+
|
|
179
|
+
def _skip(self, path: str, reason: str) -> None:
|
|
180
|
+
self.result.skipped.append({"path": path, "reason": reason})
|
|
181
|
+
log.info("skip %s: %s", path, reason)
|
|
182
|
+
|
|
183
|
+
# ------------------------------------------------------------------ LLM helpers
|
|
184
|
+
def _ask(self, user: str, max_tokens: int | None = None) -> str:
|
|
185
|
+
self.result.llm_calls += 1
|
|
186
|
+
t0 = time.monotonic()
|
|
187
|
+
text = self.gen.complete(prompts.GEN_SYSTEM, user, max_tokens=max_tokens).text
|
|
188
|
+
log.info(" model replied in %.0fs", time.monotonic() - t0)
|
|
189
|
+
return text
|
|
190
|
+
|
|
191
|
+
def _clip(self, s: str, key: str = "max_patch_chars") -> str:
|
|
192
|
+
n = int(self.run_cfg[key])
|
|
193
|
+
return s if len(s) <= n else s[: n // 2] + "\n... [truncated] ...\n" + s[-n // 2 :]
|
|
194
|
+
|
|
195
|
+
# ------------------------------------------------------------------ intent & risks
|
|
196
|
+
def _infer_intent(self, targets: list[Target]) -> Intent | None:
|
|
197
|
+
files_block = "\n".join(
|
|
198
|
+
f"### `{t.path}`\n```{t.adapter.fence_langs[0]}\n{self._clip(t.head_text, 'max_file_chars')}\n```"
|
|
199
|
+
for t in targets
|
|
200
|
+
)
|
|
201
|
+
user = prompts.INTENT_USER.format(
|
|
202
|
+
title=self.ctx.title or "(none)", description=self.ctx.description or "(none)",
|
|
203
|
+
patch=self._clip(self.ctx.full_patch), files=files_block,
|
|
204
|
+
max_risks=int(self.run_cfg["max_risks_per_file"]),
|
|
205
|
+
)
|
|
206
|
+
try:
|
|
207
|
+
data = extract_json(self._ask(user, 4000))
|
|
208
|
+
except (LLMError, LLMFormatError) as e:
|
|
209
|
+
self.result.errors.append(f"intent inference failed: {e}")
|
|
210
|
+
log.warning("intent inference failed: %s", e)
|
|
211
|
+
return None
|
|
212
|
+
if not isinstance(data, dict):
|
|
213
|
+
self.result.errors.append("intent inference returned non-object JSON")
|
|
214
|
+
return None
|
|
215
|
+
kind = str(data.get("change_kind", "other")).strip().lower()
|
|
216
|
+
paths = {t.path for t in targets}
|
|
217
|
+
per_file: dict[str, int] = {}
|
|
218
|
+
risks: list[Risk] = []
|
|
219
|
+
for item in data.get("risks") or []:
|
|
220
|
+
if not isinstance(item, dict) or not item.get("description"):
|
|
221
|
+
continue
|
|
222
|
+
path = str(item.get("file", "")).strip().lstrip("./")
|
|
223
|
+
if path not in paths:
|
|
224
|
+
if len(paths) == 1:
|
|
225
|
+
path = next(iter(paths))
|
|
226
|
+
else:
|
|
227
|
+
match = [p for p in paths if p.endswith(path) or path.endswith(PurePosixPath(p).name)]
|
|
228
|
+
if len(match) != 1:
|
|
229
|
+
continue
|
|
230
|
+
path = match[0]
|
|
231
|
+
if per_file.get(path, 0) >= int(self.run_cfg["max_risks_per_file"]):
|
|
232
|
+
continue
|
|
233
|
+
per_file[path] = per_file.get(path, 0) + 1
|
|
234
|
+
risks.append(Risk(id=f"r{len(risks) + 1}", description=str(item["description"]).strip(), file=path))
|
|
235
|
+
intent = Intent(summary=str(data.get("intent", "")).strip(),
|
|
236
|
+
change_kind=kind if kind in CHANGE_KINDS else "other", risks=risks)
|
|
237
|
+
log.info("intent: %s [%s]; %d risks", intent.summary, intent.change_kind, len(risks))
|
|
238
|
+
return intent
|
|
239
|
+
|
|
240
|
+
# ------------------------------------------------------------------ intent-aware workflow
|
|
241
|
+
def _intent_aware(self, t: Target, base_wt: Path, head_wt: Path) -> None:
|
|
242
|
+
assert self.result.intent is not None
|
|
243
|
+
risks = [r for r in self.result.intent.risks if r.file == t.path]
|
|
244
|
+
for i, risk in enumerate(risks, 1):
|
|
245
|
+
log.info("[%s] risk %d/%d: generating mutant…", t.path, i, len(risks))
|
|
246
|
+
mutant = self._make_mutant(t, risk, base_wt)
|
|
247
|
+
if mutant is None:
|
|
248
|
+
continue
|
|
249
|
+
log.info("[%s] risk %d/%d: writing a test that should catch mutant %s…", t.path, i, len(risks), mutant.id)
|
|
250
|
+
test = self._generate_test(t, "intent_aware", mutant=mutant)
|
|
251
|
+
if test is not None:
|
|
252
|
+
self._evaluate(test, t, base_wt, head_wt, mutant)
|
|
253
|
+
|
|
254
|
+
def _make_mutant(self, t: Target, risk: Risk, base_wt: Path) -> Mutant | None:
|
|
255
|
+
fence = t.adapter.fence_langs[0]
|
|
256
|
+
user = prompts.MUTANT_USER.format(path=t.path, intent=self.result.intent.summary if self.result.intent else "",
|
|
257
|
+
risk=risk.description, lang=fence, source=t.base_text)
|
|
258
|
+
mid = f"m{len(self.result.mutants) + 1}"
|
|
259
|
+
try:
|
|
260
|
+
data = extract_json(self._ask(user, 4000))
|
|
261
|
+
edits = data.get("edits") if isinstance(data, dict) else None
|
|
262
|
+
if not edits:
|
|
263
|
+
raise LLMFormatError("no edits")
|
|
264
|
+
mutated = apply_edits(t.base_text, edits)
|
|
265
|
+
except (LLMError, LLMFormatError, ValueError) as e:
|
|
266
|
+
log.info("mutant for %s/%s rejected: %s", t.path, risk.id, e)
|
|
267
|
+
self.result.mutants.append(Mutant(mid, risk, t.path, t.base_text, t.base_text,
|
|
268
|
+
survived_existing_tests=None, note=f"invalid: {e}"))
|
|
269
|
+
return None
|
|
270
|
+
mutant = Mutant(mid, risk, t.path, t.base_text, mutated,
|
|
271
|
+
note=str(data.get("explanation", ""))[:500] if isinstance(data, dict) else "")
|
|
272
|
+
self.result.mutants.append(mutant)
|
|
273
|
+
if mutated == t.base_text:
|
|
274
|
+
mutant.note = "invalid: identical to parent"
|
|
275
|
+
return None
|
|
276
|
+
syntax = _syntax_error(t.adapter, mutated, t.path)
|
|
277
|
+
if syntax:
|
|
278
|
+
mutant.note = f"invalid: {syntax}"
|
|
279
|
+
return None
|
|
280
|
+
if self.run_cfg.get("check_mutants_with_existing_tests"):
|
|
281
|
+
if t.path not in self._existing_baseline:
|
|
282
|
+
log.info("[%s] running existing tests on the parent (once per file)…", t.path)
|
|
283
|
+
if self._existing_tests_pass_on_parent(t, base_wt) is False:
|
|
284
|
+
mutant.note += " | existing tests already fail on parent; check skipped"
|
|
285
|
+
return mutant
|
|
286
|
+
log.info("[%s] running existing tests against mutant %s…", t.path, mid)
|
|
287
|
+
with _swapped(base_wt / t.path, mutated):
|
|
288
|
+
run = t.adapter.run_existing_tests(base_wt, t.path, float(self.run_cfg["existing_tests_timeout_s"]))
|
|
289
|
+
if run is not None and run.outcome in (Outcome.FAIL, Outcome.ERROR):
|
|
290
|
+
mutant.survived_existing_tests = False
|
|
291
|
+
mutant.note += " | killed or broken by existing tests (discarded)"
|
|
292
|
+
log.info("mutant %s discarded: existing tests %s", mid, run.outcome.value)
|
|
293
|
+
return None
|
|
294
|
+
mutant.survived_existing_tests = True if run is not None and run.passed else None
|
|
295
|
+
return mutant
|
|
296
|
+
|
|
297
|
+
def _existing_tests_pass_on_parent(self, t: Target, base_wt: Path) -> bool | None:
|
|
298
|
+
"""Run the existing tests once on the unmodified parent. A mutant can only be judged
|
|
299
|
+
"killed by existing tests" if those tests pass without it. None = no runner for this target."""
|
|
300
|
+
if t.path not in self._existing_baseline:
|
|
301
|
+
run = t.adapter.run_existing_tests(base_wt, t.path, float(self.run_cfg["existing_tests_timeout_s"]))
|
|
302
|
+
ok = None if run is None else run.passed
|
|
303
|
+
self._existing_baseline[t.path] = ok
|
|
304
|
+
if ok is False:
|
|
305
|
+
msg = (f"existing tests do not pass on the parent for {t.path} ({run.outcome.value}); "
|
|
306
|
+
"mutants are kept without the existing-tests check")
|
|
307
|
+
self.result.errors.append(msg)
|
|
308
|
+
log.warning(msg)
|
|
309
|
+
return self._existing_baseline[t.path]
|
|
310
|
+
|
|
311
|
+
# ------------------------------------------------------------------ dodgy-diff workflow
|
|
312
|
+
def _dodgy_diff(self, t: Target, base_wt: Path, head_wt: Path) -> None:
|
|
313
|
+
n = int(self.run_cfg["dodgy_tests_per_file"])
|
|
314
|
+
for i in range(n):
|
|
315
|
+
log.info("[%s] dodgy-diff test %d/%d: writing a test for the parent's behaviour…", t.path, i + 1, n)
|
|
316
|
+
test = self._generate_test(t, "dodgy_diff", variation=i)
|
|
317
|
+
if test is not None:
|
|
318
|
+
self._evaluate(test, t, base_wt, head_wt, None)
|
|
319
|
+
|
|
320
|
+
# ------------------------------------------------------------------ test generation
|
|
321
|
+
def _example_block(self, t: Target) -> str:
|
|
322
|
+
rel = t.adapter.related_test(t.path) or t.adapter.any_test_example()
|
|
323
|
+
if not rel:
|
|
324
|
+
return ""
|
|
325
|
+
code = show_file(self.repo, self.ctx.base_sha, rel)
|
|
326
|
+
if code is None:
|
|
327
|
+
p = self.repo / rel
|
|
328
|
+
code = p.read_text(encoding="utf-8", errors="replace") if p.exists() else None
|
|
329
|
+
if not code:
|
|
330
|
+
return ""
|
|
331
|
+
code = code if len(code) < 6000 else code[:6000] + "\n# ... (truncated)\n"
|
|
332
|
+
return prompts.EXAMPLE_BLOCK.format(path=rel, lang=t.adapter.fence_langs[0], code=code)
|
|
333
|
+
|
|
334
|
+
def _generate_test(self, t: Target, workflow: str, mutant: Mutant | None = None,
|
|
335
|
+
variation: int = 0) -> GeneratedTest | None:
|
|
336
|
+
tid = uuid.uuid4().hex[:6]
|
|
337
|
+
pl = t.adapter.placement(t.path, tid, t.base_text)
|
|
338
|
+
fence = t.adapter.fence_langs[0]
|
|
339
|
+
if mutant is not None:
|
|
340
|
+
user = prompts.TEST_FOR_MUTANT_USER.format(
|
|
341
|
+
path=t.path, risk=mutant.risk.description, instructions=pl.instructions, test_path=pl.rel_path,
|
|
342
|
+
lang=fence, source=t.base_text, mutant_diff=unified(t.base_text, mutant.mutated, t.path),
|
|
343
|
+
example=self._example_block(t))
|
|
344
|
+
else:
|
|
345
|
+
variation_txt = ("" if variation == 0 else
|
|
346
|
+
f"\n(Variation {variation + 1}: cover different inputs/branches of the changed code "
|
|
347
|
+
"than an obvious first test would.)\n")
|
|
348
|
+
user = prompts.TEST_DODGY_USER.format(
|
|
349
|
+
path=t.path, instructions=pl.instructions, test_path=pl.rel_path, lang=fence,
|
|
350
|
+
source=t.base_text, patch=self._clip(t.patch), example=self._example_block(t),
|
|
351
|
+
variation=variation_txt)
|
|
352
|
+
try:
|
|
353
|
+
code = extract_code(self._ask(user, 6000), t.adapter.fence_langs)
|
|
354
|
+
except LLMError as e:
|
|
355
|
+
self.result.errors.append(f"test generation failed for {t.path}: {e}")
|
|
356
|
+
return None
|
|
357
|
+
if not code.strip():
|
|
358
|
+
return None
|
|
359
|
+
test = GeneratedTest(id=tid, workflow=workflow, target_file=t.path, test_path=pl.rel_path, code=code,
|
|
360
|
+
risk=mutant.risk if mutant else None, mutant_id=mutant.id if mutant else None)
|
|
361
|
+
self._placements[tid] = pl
|
|
362
|
+
self.result.tests.append(test)
|
|
363
|
+
return test
|
|
364
|
+
|
|
365
|
+
# ------------------------------------------------------------------ execution
|
|
366
|
+
def _run(self, test: GeneratedTest, t: Target, wt: Path) -> TestRun:
|
|
367
|
+
pl = self._placements[test.id]
|
|
368
|
+
with _temp_file(wt / test.test_path, test.code):
|
|
369
|
+
return t.adapter.run_test(wt, test.test_path, pl, float(self.run_cfg["test_timeout_s"]))
|
|
370
|
+
|
|
371
|
+
def _evaluate(self, test: GeneratedTest, t: Target, base_wt: Path, head_wt: Path, mutant: Mutant | None) -> None:
|
|
372
|
+
max_fix = int(self.run_cfg["max_fix_attempts"])
|
|
373
|
+
pl = self._placements[test.id]
|
|
374
|
+
fence = t.adapter.fence_langs[0]
|
|
375
|
+
# 1. must build and pass on the parent (with an LLM repair loop)
|
|
376
|
+
while True:
|
|
377
|
+
test.attempts += 1
|
|
378
|
+
log.info(" test %s: running on parent (attempt %d)…", test.id, test.attempts)
|
|
379
|
+
run = self._run(test, t, base_wt)
|
|
380
|
+
test.base_run = run
|
|
381
|
+
if run.passed or run.outcome == Outcome.INFRA or test.attempts > max_fix:
|
|
382
|
+
break
|
|
383
|
+
log.info(" test %s: %s on parent; asking the model to repair it (%d/%d)…",
|
|
384
|
+
test.id, run.outcome.value, test.attempts, max_fix)
|
|
385
|
+
user = prompts.TEST_FIX_USER.format(
|
|
386
|
+
path=t.path, instructions=pl.instructions, lang=fence, source=t.base_text,
|
|
387
|
+
test_path=test.test_path, test_code=test.code, outcome=run.outcome.value,
|
|
388
|
+
output=self._clip(run.output, "max_patch_chars"))
|
|
389
|
+
try:
|
|
390
|
+
test.code = extract_code(self._ask(user, 6000), t.adapter.fence_langs)
|
|
391
|
+
except LLMError as e:
|
|
392
|
+
self.result.errors.append(f"test repair failed: {e}")
|
|
393
|
+
break
|
|
394
|
+
if not test.base_run.passed:
|
|
395
|
+
test.discarded_reason = f"does not pass on parent ({test.base_run.outcome.value})"
|
|
396
|
+
log.info("test %s discarded: %s", test.id, test.discarded_reason)
|
|
397
|
+
return
|
|
398
|
+
reruns = int(self.run_cfg["flaky_reruns"])
|
|
399
|
+
if reruns > 0:
|
|
400
|
+
test.base_rerun = self._run(test, t, base_wt)
|
|
401
|
+
if not test.base_rerun.passed:
|
|
402
|
+
test.discarded_reason = "flaky on parent"
|
|
403
|
+
return
|
|
404
|
+
# 2. mutant kill (informational; intent-aware workflow)
|
|
405
|
+
if mutant is not None:
|
|
406
|
+
log.info(" test %s: running on mutant %s…", test.id, mutant.id)
|
|
407
|
+
with _swapped(base_wt / t.path, mutant.mutated):
|
|
408
|
+
test.mutant_run = self._run(test, t, base_wt)
|
|
409
|
+
# 3. run on the change
|
|
410
|
+
log.info(" test %s: running on the change…", test.id)
|
|
411
|
+
test.head_run = self._run(test, t, head_wt)
|
|
412
|
+
if not test.head_run.passed and reruns > 0:
|
|
413
|
+
test.head_rerun = self._run(test, t, head_wt)
|
|
414
|
+
state = ("WEAK CATCH" if test.is_weak_catch else "hardening" if test.is_hardening else test.head_run.outcome.value)
|
|
415
|
+
log.info("test %s [%s] on %s: %s", test.id, test.workflow, t.path, state)
|
|
416
|
+
|
|
417
|
+
# ------------------------------------------------------------------ assessment
|
|
418
|
+
def _assess(self) -> None:
|
|
419
|
+
by_path = {}
|
|
420
|
+
for f in self.ctx.files:
|
|
421
|
+
by_path[f.path] = adapter_for(self.adapters, f.path)
|
|
422
|
+
catches = self.result.weak_catches
|
|
423
|
+
for i, test in enumerate(catches, 1):
|
|
424
|
+
log.info("judging weak catch %d/%d (test %s)…", i, len(catches), test.id)
|
|
425
|
+
ad = by_path.get(test.target_file)
|
|
426
|
+
fence = ad.fence_langs[0] if ad else ""
|
|
427
|
+
a = assess_weak_catch(test, self.ctx, self.result.intent, ad.name if ad else "", fence,
|
|
428
|
+
self.judges, self.logprob_judge, self.cfg.assessors,
|
|
429
|
+
int(self.run_cfg["max_patch_chars"]))
|
|
430
|
+
self.result.llm_calls += 0 if a.discarded else len(self.judges) + (1 if self.logprob_judge else 0)
|
|
431
|
+
self.result.assessments[test.id] = a
|
|
432
|
+
log.info("assessed %s: combined=%.3f%s", test.id, a.combined, " (discarded)" if a.discarded else "")
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
# ---------------------------------------------------------------------- utilities
|
|
436
|
+
|
|
437
|
+
def apply_edits(text: str, edits: list[dict[str, Any]]) -> str:
|
|
438
|
+
out = text
|
|
439
|
+
for e in edits:
|
|
440
|
+
if not isinstance(e, dict):
|
|
441
|
+
raise ValueError("edit is not an object")
|
|
442
|
+
search, replace = e.get("search"), e.get("replace")
|
|
443
|
+
if not isinstance(search, str) or not isinstance(replace, str) or not search:
|
|
444
|
+
raise ValueError("edit needs non-empty 'search' and string 'replace'")
|
|
445
|
+
count = out.count(search)
|
|
446
|
+
if count == 0:
|
|
447
|
+
# tolerate trailing-whitespace differences
|
|
448
|
+
stripped = "\n".join(l.rstrip() for l in search.splitlines())
|
|
449
|
+
norm = "\n".join(l.rstrip() for l in out.splitlines())
|
|
450
|
+
if norm.count(stripped) == 1:
|
|
451
|
+
out = norm.replace(stripped, replace, 1) + ("\n" if text.endswith("\n") else "")
|
|
452
|
+
continue
|
|
453
|
+
raise ValueError(f"search text not found: {search[:80]!r}")
|
|
454
|
+
if count > 1:
|
|
455
|
+
raise ValueError(f"search text is not unique ({count} matches): {search[:80]!r}")
|
|
456
|
+
out = out.replace(search, replace, 1)
|
|
457
|
+
return out
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def unified(a: str, b: str, path: str) -> str:
|
|
461
|
+
return "".join(difflib.unified_diff(a.splitlines(True), b.splitlines(True),
|
|
462
|
+
fromfile=f"a/{path}", tofile=f"b/{path}"))
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _syntax_error(adapter: Adapter, text: str, path: str) -> str | None:
|
|
466
|
+
if adapter.name == "python":
|
|
467
|
+
try:
|
|
468
|
+
compile(text, path, "exec")
|
|
469
|
+
except SyntaxError as e:
|
|
470
|
+
return f"syntax error: {e}"
|
|
471
|
+
return None
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
class _swapped:
|
|
475
|
+
"""Temporarily replace a file's contents."""
|
|
476
|
+
|
|
477
|
+
def __init__(self, path: Path, content: str):
|
|
478
|
+
self.path, self.content = path, content
|
|
479
|
+
self.original: bytes | None = None
|
|
480
|
+
|
|
481
|
+
def __enter__(self):
|
|
482
|
+
self.original = self.path.read_bytes()
|
|
483
|
+
self.path.write_text(self.content, encoding="utf-8")
|
|
484
|
+
return self.path
|
|
485
|
+
|
|
486
|
+
def __exit__(self, *exc):
|
|
487
|
+
assert self.original is not None
|
|
488
|
+
self.path.write_bytes(self.original)
|
|
489
|
+
return False
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
class _temp_file:
|
|
493
|
+
"""Write a file for the duration of a block, creating (and later removing) missing parent dirs."""
|
|
494
|
+
|
|
495
|
+
def __init__(self, path: Path, content: str):
|
|
496
|
+
self.path, self.content = path, content
|
|
497
|
+
self.created: list[Path] = []
|
|
498
|
+
|
|
499
|
+
def __enter__(self):
|
|
500
|
+
d = self.path.parent
|
|
501
|
+
missing = []
|
|
502
|
+
while not d.exists():
|
|
503
|
+
missing.append(d)
|
|
504
|
+
d = d.parent
|
|
505
|
+
for m in reversed(missing):
|
|
506
|
+
m.mkdir()
|
|
507
|
+
self.created.append(m)
|
|
508
|
+
self.path.write_text(self.content, encoding="utf-8")
|
|
509
|
+
return self.path
|
|
510
|
+
|
|
511
|
+
def __exit__(self, *exc):
|
|
512
|
+
try:
|
|
513
|
+
self.path.unlink()
|
|
514
|
+
except FileNotFoundError:
|
|
515
|
+
pass
|
|
516
|
+
for d in reversed(self.created):
|
|
517
|
+
try:
|
|
518
|
+
d.rmdir()
|
|
519
|
+
except OSError:
|
|
520
|
+
pass
|
|
521
|
+
return False
|
jitcatch/github.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""GitHub Actions integration: read the PR event, post/update a sticky PR comment."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import urllib.error
|
|
7
|
+
import urllib.request
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
from .report import COMMENT_MARKER
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class PullRequest:
|
|
15
|
+
number: int
|
|
16
|
+
title: str
|
|
17
|
+
body: str
|
|
18
|
+
base_sha: str
|
|
19
|
+
head_sha: str
|
|
20
|
+
repo: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def read_event() -> PullRequest | None:
|
|
24
|
+
path = os.environ.get("GITHUB_EVENT_PATH")
|
|
25
|
+
if not path or not os.path.exists(path):
|
|
26
|
+
return None
|
|
27
|
+
with open(path, encoding="utf-8") as f:
|
|
28
|
+
ev = json.load(f)
|
|
29
|
+
pr = ev.get("pull_request")
|
|
30
|
+
if not pr:
|
|
31
|
+
return None
|
|
32
|
+
return PullRequest(
|
|
33
|
+
number=int(pr["number"]), title=pr.get("title") or "", body=pr.get("body") or "",
|
|
34
|
+
base_sha=pr["base"]["sha"], head_sha=pr["head"]["sha"],
|
|
35
|
+
repo=os.environ.get("GITHUB_REPOSITORY") or ev.get("repository", {}).get("full_name", ""),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class GitHubAPI:
|
|
40
|
+
def __init__(self, token: str, api_url: str | None = None):
|
|
41
|
+
self.token = token
|
|
42
|
+
self.api = (api_url or os.environ.get("GITHUB_API_URL") or "https://api.github.com").rstrip("/")
|
|
43
|
+
|
|
44
|
+
def _req(self, method: str, path: str, body: dict | None = None):
|
|
45
|
+
data = json.dumps(body).encode() if body is not None else None
|
|
46
|
+
req = urllib.request.Request(self.api + path, data=data, method=method, headers={
|
|
47
|
+
"Authorization": f"Bearer {self.token}", "Accept": "application/vnd.github+json",
|
|
48
|
+
"X-GitHub-Api-Version": "2022-11-28", "Content-Type": "application/json",
|
|
49
|
+
"User-Agent": "jitcatch",
|
|
50
|
+
})
|
|
51
|
+
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
52
|
+
raw = resp.read()
|
|
53
|
+
return json.loads(raw) if raw else None
|
|
54
|
+
|
|
55
|
+
def find_comment(self, repo: str, number: int) -> int | None:
|
|
56
|
+
page = 1
|
|
57
|
+
while True:
|
|
58
|
+
items = self._req("GET", f"/repos/{repo}/issues/{number}/comments?per_page=100&page={page}") or []
|
|
59
|
+
for c in items:
|
|
60
|
+
if COMMENT_MARKER in (c.get("body") or ""):
|
|
61
|
+
return int(c["id"])
|
|
62
|
+
if len(items) < 100:
|
|
63
|
+
return None
|
|
64
|
+
page += 1
|
|
65
|
+
|
|
66
|
+
def upsert_comment(self, repo: str, number: int, body: str, create_if_missing: bool = True) -> str:
|
|
67
|
+
if len(body) > 65000: # GitHub comment limit is 65536 chars
|
|
68
|
+
body = body[:64000] + "\n\n… report truncated; see the workflow artifact for the full report.\n"
|
|
69
|
+
existing = self.find_comment(repo, number)
|
|
70
|
+
if existing is not None:
|
|
71
|
+
self._req("PATCH", f"/repos/{repo}/issues/comments/{existing}", {"body": body})
|
|
72
|
+
return "updated"
|
|
73
|
+
if not create_if_missing:
|
|
74
|
+
return "skipped"
|
|
75
|
+
self._req("POST", f"/repos/{repo}/issues/{number}/comments", {"body": body})
|
|
76
|
+
return "created"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def set_output(name: str, value: str) -> None:
|
|
80
|
+
path = os.environ.get("GITHUB_OUTPUT")
|
|
81
|
+
if path:
|
|
82
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
83
|
+
f.write(f"{name}={value}\n")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def write_step_summary(md: str) -> None:
|
|
87
|
+
path = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
88
|
+
if path:
|
|
89
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
90
|
+
f.write(md)
|