tycho-cli 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tycho/__init__.py +9 -0
- tycho/__main__.py +12 -0
- tycho/astdiff.py +102 -0
- tycho/checks.py +681 -0
- tycho/cli.py +516 -0
- tycho/config.py +177 -0
- tycho/doctor.py +304 -0
- tycho/events.py +393 -0
- tycho/fsstate.py +30 -0
- tycho/gitstate.py +46 -0
- tycho/harness.py +303 -0
- tycho/hook.py +212 -0
- tycho/init.py +1018 -0
- tycho/model.py +147 -0
- tycho/opencode.py +154 -0
- tycho/report.py +29 -0
- tycho/runlog.py +54 -0
- tycho/state.py +469 -0
- tycho/status.py +171 -0
- tycho/verify.py +202 -0
- tycho/version.py +99 -0
- tycho_cli-0.0.1.dist-info/METADATA +379 -0
- tycho_cli-0.0.1.dist-info/RECORD +26 -0
- tycho_cli-0.0.1.dist-info/WHEEL +4 -0
- tycho_cli-0.0.1.dist-info/entry_points.txt +2 -0
- tycho_cli-0.0.1.dist-info/licenses/LICENSE +201 -0
tycho/checks.py
ADDED
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
"""The deterministic checks. Each is a pure function of the Session snapshot.
|
|
2
|
+
|
|
3
|
+
A check returns exactly one CheckResult. When a check doesn't apply (no relevant
|
|
4
|
+
input) it returns UNSUPPORTED with a reason — never a false FAIL. `run_checks`
|
|
5
|
+
drives the registry and honours `config.disabled_checks`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import shlex
|
|
12
|
+
from fnmatch import fnmatchcase
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
|
|
15
|
+
from . import runlog
|
|
16
|
+
from .astdiff import assertion_delta, skip_or_mock_added
|
|
17
|
+
from .model import CheckResult, CheckStatus, Session
|
|
18
|
+
|
|
19
|
+
_TEST_RUNNERS = (
|
|
20
|
+
"python -m pytest",
|
|
21
|
+
"python3 -m pytest",
|
|
22
|
+
"python -m unittest",
|
|
23
|
+
"poetry run pytest",
|
|
24
|
+
"uv run pytest",
|
|
25
|
+
"pytest",
|
|
26
|
+
"go test",
|
|
27
|
+
"npm test",
|
|
28
|
+
"npm run test",
|
|
29
|
+
"cargo test",
|
|
30
|
+
"make test",
|
|
31
|
+
"make check",
|
|
32
|
+
"jest",
|
|
33
|
+
"vitest",
|
|
34
|
+
"mocha",
|
|
35
|
+
"npx jest",
|
|
36
|
+
"npx vitest",
|
|
37
|
+
"npx mocha",
|
|
38
|
+
"yarn test",
|
|
39
|
+
"pnpm test",
|
|
40
|
+
"mvn test",
|
|
41
|
+
"mvnw test",
|
|
42
|
+
"gradle test",
|
|
43
|
+
"gradlew test",
|
|
44
|
+
"dotnet test",
|
|
45
|
+
"ctest",
|
|
46
|
+
"meson test",
|
|
47
|
+
"bazel test",
|
|
48
|
+
"phpunit",
|
|
49
|
+
"pest",
|
|
50
|
+
"codecept",
|
|
51
|
+
"codeception",
|
|
52
|
+
"composer test",
|
|
53
|
+
"rspec",
|
|
54
|
+
"bundle exec rspec",
|
|
55
|
+
"rake test",
|
|
56
|
+
"swift test",
|
|
57
|
+
"xcodebuild test",
|
|
58
|
+
"dart test",
|
|
59
|
+
"flutter test",
|
|
60
|
+
"tox",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# The shell tools an agent runs commands through. Runner detection isn't Bash-specific:
|
|
64
|
+
# a PowerShell/pwsh/Shell tool runs `pytest` just the same, and a Bash-only filter made
|
|
65
|
+
# those runs — and every check that depends on them — invisible (TYCHO-27). Cursor already
|
|
66
|
+
# maps Shell→Bash in its reader; this covers tools/harnesses that don't normalize.
|
|
67
|
+
_SHELL_TOOLS = frozenset({"Bash", "Shell", "sh", "PowerShell", "powershell", "pwsh"})
|
|
68
|
+
|
|
69
|
+
# Ephemeral-env wrappers put their own flags between the wrapper and the real runner:
|
|
70
|
+
# `uv run --python 3.12 --with pytest python -m pytest -q`. The plain runner match misses
|
|
71
|
+
# that, so for a wrapper we also look for a *multi-word* runner phrase later in the segment
|
|
72
|
+
# (TYCHO-27). Multi-word phrases like `python -m pytest` are never a `--with` value, which
|
|
73
|
+
# keeps this from firing on `uv run --with pytest ruff check`.
|
|
74
|
+
_RUN_WRAPPERS = ("uv run", "uvx", "poetry run", "pdm run", "hatch run", "rye run", "npx", "pnpm dlx", "bunx")
|
|
75
|
+
_PHRASE_RUNNERS = tuple(r for r in _TEST_RUNNERS if " " in r)
|
|
76
|
+
|
|
77
|
+
# `<python> -m <module>` — the module IS the runner, so the leading executable can be
|
|
78
|
+
# anything, including a shell variable (`"$PY" -m pytest`) or path we can't resolve
|
|
79
|
+
# statically. Recognising the module directly is what makes a variable-indirected
|
|
80
|
+
# interpreter visible instead of counted as "no test ran" (TYCHO-88).
|
|
81
|
+
_MODULE_RUNNERS = ("pytest", "unittest", "nose2")
|
|
82
|
+
|
|
83
|
+
# Wrappers that carry the real command *inside* an argument, so the runner is invisible
|
|
84
|
+
# until we descend into it (TYCHO-89). `_unwrap` peels these; the shapes it knows are
|
|
85
|
+
# `<w> ... -- <cmd>` (wsl/env), `<shell> -c '<cmd>'`, `ssh <host> <cmd>`, and our own
|
|
86
|
+
# `tycho run` escape hatch (TYCHO-90). `wsl.exe`/`bash.exe` reduce to the bare name first.
|
|
87
|
+
_DASHDASH_WRAPPERS = ("wsl", "env")
|
|
88
|
+
_C_SHELLS = ("bash", "sh", "zsh", "dash", "ash")
|
|
89
|
+
_EXE_SUFFIX = re.compile(r"\.(?:exe|bat|cmd|ps1)$", re.IGNORECASE)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _looks_like_interpreter(tok: str) -> bool:
|
|
93
|
+
"""True for a token that could be a Python interpreter — a real name (`python3.12`,
|
|
94
|
+
the `py` launcher) or an unresolved shell/Windows variable (`$PY`, `%PY%`, `${PY}`)
|
|
95
|
+
whose value we can't see. Keeps the `-m pytest` rule off `echo -m pytest` (TYCHO-88)."""
|
|
96
|
+
if tok[:1] in "$%{":
|
|
97
|
+
return True
|
|
98
|
+
name = tok.lower()
|
|
99
|
+
return name == "py" or bool(re.fullmatch(r"python[0-9.]*|pypy[0-9.]*", name))
|
|
100
|
+
|
|
101
|
+
# Split a shell command into segments so a runner name buried in a quoted
|
|
102
|
+
# echo/grep argument doesn't count as "the tests ran".
|
|
103
|
+
_SEGMENT_SEP = re.compile(r"&&|\|\||[;|\n()]")
|
|
104
|
+
# The same separators, kept as capture groups: *which* separator follows the runner is the
|
|
105
|
+
# whole question in `_status_is_masked`, so unlike `_SEGMENT_SEP` we can't discard them.
|
|
106
|
+
_SEGMENT_TOKENS = re.compile(r"(&&|\|\||[;|\n()])")
|
|
107
|
+
_ENV_PREFIX = re.compile(r"^(?:\s*\w+=\S+\s+)+")
|
|
108
|
+
|
|
109
|
+
# How much of a runner's output to read back for its summary. pytest's verdict is the last
|
|
110
|
+
# line; the slack absorbs the trailing warnings block and blank lines that follow it under
|
|
111
|
+
# `-q`. Deliberately small: the further up the blob we look, the more we're reading the run
|
|
112
|
+
# rather than its conclusion.
|
|
113
|
+
_SUMMARY_TAIL_LINES = 12
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _normalize_segment(segment: str) -> str:
|
|
117
|
+
"""Strip env prefixes and any leading path from the executable so
|
|
118
|
+
`.venv/bin/python -m pytest` reads as the same runner as bare `pytest`."""
|
|
119
|
+
segment = _ENV_PREFIX.sub("", segment.strip())
|
|
120
|
+
try:
|
|
121
|
+
parts = shlex.split(segment)
|
|
122
|
+
except ValueError:
|
|
123
|
+
parts = []
|
|
124
|
+
if parts:
|
|
125
|
+
exe = parts[0].rsplit("/", 1)[-1]
|
|
126
|
+
# Windows interpreters carry a suffix (`python.exe`); strip it so a venv
|
|
127
|
+
# runner reads the same as its POSIX `python` bare name (TYCHO-52).
|
|
128
|
+
exe = re.sub(r"\.(?:exe|bat|cmd|ps1)$", "", exe, flags=re.IGNORECASE)
|
|
129
|
+
segment = " ".join([exe, *parts[1:]])
|
|
130
|
+
return segment
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _is_runner(segment: str) -> bool:
|
|
134
|
+
"""True if this already-normalized segment invokes a test/build runner."""
|
|
135
|
+
java_runner = segment.startswith("java ") and ("junit" in segment.lower() or "testng" in segment.lower())
|
|
136
|
+
if java_runner or any(segment == r or segment.startswith(f"{r} ") for r in _TEST_RUNNERS):
|
|
137
|
+
return True
|
|
138
|
+
# `<interpreter> -m pytest|unittest` regardless of the interpreter token, so a variable
|
|
139
|
+
# or unresolved-path exe still counts (TYCHO-88). Guarded so `echo -m pytest` doesn't.
|
|
140
|
+
parts = segment.split()
|
|
141
|
+
if (
|
|
142
|
+
len(parts) >= 3
|
|
143
|
+
and parts[1] == "-m"
|
|
144
|
+
and parts[2] in _MODULE_RUNNERS
|
|
145
|
+
and _looks_like_interpreter(parts[0])
|
|
146
|
+
):
|
|
147
|
+
return True
|
|
148
|
+
# An ephemeral-env wrapper (`uv run …`) with a multi-word runner phrase further along
|
|
149
|
+
# counts as a runner even though the flags in between defeat the plain prefix match.
|
|
150
|
+
if any(segment == w or segment.startswith(f"{w} ") for w in _RUN_WRAPPERS):
|
|
151
|
+
padded = f" {segment} "
|
|
152
|
+
return any(f" {r} " in padded for r in _PHRASE_RUNNERS)
|
|
153
|
+
return False
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _unwrap(segment: str) -> str | None:
|
|
157
|
+
"""If `segment` wraps the real command inside an argument, return that inner command
|
|
158
|
+
(quotes intact) to recurse into; else None. Runs on the RAW segment, not a normalized
|
|
159
|
+
one, so a quoted `-c '<cmd>'` stays a single token instead of splitting apart (TYCHO-89).
|
|
160
|
+
|
|
161
|
+
Peels one layer; `_runner_segment` recurses, so `wsl -- bash -c 'pytest'` unwinds
|
|
162
|
+
`wsl -- …` → `bash -c 'pytest'` → `pytest`.
|
|
163
|
+
"""
|
|
164
|
+
try:
|
|
165
|
+
parts = shlex.split(segment)
|
|
166
|
+
except ValueError:
|
|
167
|
+
return None
|
|
168
|
+
if not parts:
|
|
169
|
+
return None
|
|
170
|
+
head = _EXE_SUFFIX.sub("", parts[0].rsplit("/", 1)[-1]).lower()
|
|
171
|
+
|
|
172
|
+
# Our opt-in escape hatch: `tycho run [--] <cmd>` execs <cmd> and forwards its real exit
|
|
173
|
+
# code, so the runner inside is both visible here and trustworthy at runtime (TYCHO-90).
|
|
174
|
+
if head == "tycho" and len(parts) >= 2 and parts[1] == "run":
|
|
175
|
+
rest = parts[2:]
|
|
176
|
+
if rest and rest[0] == "--":
|
|
177
|
+
rest = rest[1:]
|
|
178
|
+
return shlex.join(rest) if rest else None
|
|
179
|
+
|
|
180
|
+
# `<wrapper> ... -- <cmd...>` : the command is everything past the first `--`
|
|
181
|
+
# (`wsl.exe -d Ubuntu -- <cmd>`, `env FOO=bar -- <cmd>`).
|
|
182
|
+
if head in _DASHDASH_WRAPPERS and "--" in parts:
|
|
183
|
+
rest = parts[parts.index("--") + 1 :]
|
|
184
|
+
return shlex.join(rest) if rest else None
|
|
185
|
+
|
|
186
|
+
# `<shell> ... -c '<cmd>'` : the command is the single arg after a -c-family flag.
|
|
187
|
+
if head in _C_SHELLS:
|
|
188
|
+
for i in range(1, len(parts) - 1):
|
|
189
|
+
flag = parts[i]
|
|
190
|
+
if flag.startswith("-") and "c" in flag.lstrip("-").lower():
|
|
191
|
+
return parts[i + 1]
|
|
192
|
+
|
|
193
|
+
# `ssh <host> <cmd...>` : the remote command trails the host. Conservative — only the
|
|
194
|
+
# no-option form, so an option value is never mistaken for the host.
|
|
195
|
+
if head == "ssh" and len(parts) >= 3 and not parts[1].startswith("-"):
|
|
196
|
+
return shlex.join(parts[2:])
|
|
197
|
+
|
|
198
|
+
# `docker run <img> <cmd>` is not unwrapped — unlike the shells above it needs the
|
|
199
|
+
# image and its flags skipped first; add when a docker test workflow needs it (TYCHO-89).
|
|
200
|
+
return None
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _runner_segment(cmd: str) -> str | None:
|
|
204
|
+
"""The command segment that is a test/build runner, or None. Splitting on shell
|
|
205
|
+
separators keeps a runner name quoted inside an echo/grep from counting; unwrapping
|
|
206
|
+
descends into wrappers (wsl/ssh/`bash -c`/`tycho run`) that hide it (TYCHO-89/90)."""
|
|
207
|
+
for segment in _SEGMENT_SEP.split(cmd):
|
|
208
|
+
norm = _normalize_segment(segment)
|
|
209
|
+
if _is_runner(norm):
|
|
210
|
+
return norm
|
|
211
|
+
inner = _unwrap(segment)
|
|
212
|
+
if inner is not None:
|
|
213
|
+
found = _runner_segment(inner)
|
|
214
|
+
if found is not None:
|
|
215
|
+
return found
|
|
216
|
+
return None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _status_is_masked(cmd: str) -> bool:
|
|
220
|
+
"""True when the exit status the harness recorded is *not* the runner's own.
|
|
221
|
+
|
|
222
|
+
The shell reports one status for the whole command: the last thing it ran. So a
|
|
223
|
+
runner's failure only reaches us if nothing overwrote it, and there are three ways it
|
|
224
|
+
gets overwritten (TYCHO-26/31/61) — all of them real commands agents write:
|
|
225
|
+
|
|
226
|
+
pytest | tail -1 the pipeline's status is tail's (no pipefail here)
|
|
227
|
+
pytest; echo done the status is echo's; `;` discards what came before
|
|
228
|
+
pytest || true `||` swallows the failure by construction
|
|
229
|
+
|
|
230
|
+
`&&` is the one shape that is *safe* and must not be flagged: `pytest && echo ok`
|
|
231
|
+
fails the whole command when pytest fails, so a recorded success really does mean the
|
|
232
|
+
runner succeeded. Flagging it would cost us the most common honest invocation there is.
|
|
233
|
+
|
|
234
|
+
A runner reached through a wrapper (`wsl -- bash -c 'pytest | tail'`) is masked when its
|
|
235
|
+
*inner* command is — the wrapper can only forward as clean a status as it was given — so
|
|
236
|
+
the wrapper's inner command is checked too (TYCHO-89).
|
|
237
|
+
|
|
238
|
+
Trusting a masked status is how a red suite gets reported VERIFIED, which is the
|
|
239
|
+
single worst thing this program can do — so when in doubt, say masked and let
|
|
240
|
+
`_outcome` fall back to what the runner said in its own output.
|
|
241
|
+
"""
|
|
242
|
+
parts = _SEGMENT_TOKENS.split(cmd) # [segment, sep, segment, sep, ..., segment]
|
|
243
|
+
for i in range(0, len(parts), 2):
|
|
244
|
+
seg = parts[i]
|
|
245
|
+
if not _is_runner(_normalize_segment(seg)):
|
|
246
|
+
inner = _unwrap(seg)
|
|
247
|
+
if inner is None or _runner_segment(inner) is None:
|
|
248
|
+
continue # not the runner segment, wrapped or otherwise — keep looking
|
|
249
|
+
if _status_is_masked(inner):
|
|
250
|
+
return True # a masking operator inside the wrapper hides the runner's status
|
|
251
|
+
# Walk what follows this runner (or its wrapper): the first separator that redirects,
|
|
252
|
+
# swallows, or supersedes the status is the one that masks it.
|
|
253
|
+
for j in range(i + 1, len(parts), 2):
|
|
254
|
+
sep = parts[j]
|
|
255
|
+
rest = parts[j + 1] if j + 1 < len(parts) else ""
|
|
256
|
+
if sep in ("|", "||"):
|
|
257
|
+
return True # status replaced downstream, or the failure swallowed
|
|
258
|
+
if sep in (";", "\n") and rest.strip():
|
|
259
|
+
return True # something ran after; its status is what got recorded
|
|
260
|
+
return False # nothing after it can overwrite the status — trust the exit code
|
|
261
|
+
return False # no runner in this command at all
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _captured_output(event) -> str:
|
|
265
|
+
"""The runner's own words, tail-first — or "" when the harness kept none.
|
|
266
|
+
|
|
267
|
+
Only the tail, and that is the point. Claude Code caps `toolUseResult.stdout` at 30k
|
|
268
|
+
characters and keeps the *head* (verified against 2356 real payloads — see
|
|
269
|
+
docs/harness-support.md), while pytest prints its summary *last*. So on a long run the
|
|
270
|
+
verdict is exactly what got cut, and reading the tail means a truncated capture finds
|
|
271
|
+
no summary and honestly reports nothing — where scanning the whole blob could match a
|
|
272
|
+
stray "5 passed" from the middle of a run that ended red.
|
|
273
|
+
"""
|
|
274
|
+
result = event.result or {}
|
|
275
|
+
text = "\n".join(str(result.get(key) or "") for key in ("stdout", "stderr")).strip()
|
|
276
|
+
return "\n".join(text.splitlines()[-_SUMMARY_TAIL_LINES:]) if text else ""
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _outcome(event) -> bool | None:
|
|
280
|
+
"""Did this runner invocation fail? True = failed, False = passed, None = can't tell.
|
|
281
|
+
|
|
282
|
+
One predicate for every caller, so `command_execution` and `_last_green_run_ts` can
|
|
283
|
+
never disagree about what "green" means — a split brain there is how a run gets
|
|
284
|
+
reported green by one check and stale-against-nothing by the other.
|
|
285
|
+
|
|
286
|
+
The exit code wins whenever it is genuinely the runner's. Otherwise fall back to the
|
|
287
|
+
runner's own summary line, which is evidence rather than inference — but weaker
|
|
288
|
+
evidence, so callers say so in their output.
|
|
289
|
+
"""
|
|
290
|
+
if not _status_is_masked(event.input.get("command") or "") and event.is_error is not None:
|
|
291
|
+
return event.is_error
|
|
292
|
+
return runlog.outcome(_captured_output(event))
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def command_execution(session: Session) -> CheckResult:
|
|
296
|
+
runners = _runner_events(session.turn_events)
|
|
297
|
+
if not runners:
|
|
298
|
+
return _r(
|
|
299
|
+
"command_execution",
|
|
300
|
+
CheckStatus.UNSUPPORTED,
|
|
301
|
+
f"no test/build command ran this {_scope(session)}",
|
|
302
|
+
)
|
|
303
|
+
last = max(runners, key=lambda e: e.ts)
|
|
304
|
+
raw = last.input.get("command", "")
|
|
305
|
+
cmd = _short(_runner_segment(raw) or raw)
|
|
306
|
+
masked = _status_is_masked(raw)
|
|
307
|
+
outcome = _outcome(last)
|
|
308
|
+
if outcome is None:
|
|
309
|
+
why = (
|
|
310
|
+
"its exit status was masked by the shell" if masked
|
|
311
|
+
else "no exit status was recorded"
|
|
312
|
+
)
|
|
313
|
+
return _r(
|
|
314
|
+
"command_execution",
|
|
315
|
+
CheckStatus.UNSUPPORTED,
|
|
316
|
+
f"`{cmd}` ran but {why}, and its output carries no summary — Tycho can't confirm it passed",
|
|
317
|
+
)
|
|
318
|
+
# Say where the verdict came from. A status recovered from output is real evidence but
|
|
319
|
+
# weaker than an exit code, and a reader who can't tell which they got can't judge it.
|
|
320
|
+
via = " (read from its output — exit status masked by the shell)" if masked else ""
|
|
321
|
+
if outcome:
|
|
322
|
+
return _r("command_execution", CheckStatus.FAIL, f"`{cmd}` ran but reported an error{via}")
|
|
323
|
+
return _r("command_execution", CheckStatus.PASS, f"`{cmd}` ran without error{via}")
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def test_freshness(session: Session) -> CheckResult:
|
|
327
|
+
green_ts = _last_green_run_ts(session)
|
|
328
|
+
if green_ts is None:
|
|
329
|
+
return _r("test_freshness", CheckStatus.UNSUPPORTED, "no passing test run to check against")
|
|
330
|
+
source_edits = [fe for fe in session.edits if _is_source_path(fe.path)]
|
|
331
|
+
if not source_edits:
|
|
332
|
+
return _r("test_freshness", CheckStatus.UNSUPPORTED, "no source edits to check against the run")
|
|
333
|
+
stale = []
|
|
334
|
+
for fe in source_edits:
|
|
335
|
+
fs = session.files.get(fe.path)
|
|
336
|
+
if fs and fs.mtime is not None and fs.mtime > green_ts:
|
|
337
|
+
stale.append((fe.path, fs.mtime))
|
|
338
|
+
if stale:
|
|
339
|
+
path, mt = max(stale, key=lambda x: x[1])
|
|
340
|
+
# Session-scoped by design (TYCHO-23): staleness is a fact about the tree *now*, so an
|
|
341
|
+
# uncovered source from an earlier turn still counts. But don't *imply* a this-turn edit:
|
|
342
|
+
# reserve the "edited Ns after" phrasing for turn edits, and word an earlier-turn
|
|
343
|
+
# staleness as the live condition it is.
|
|
344
|
+
if path in {fe.path for fe in session.turn_edits}:
|
|
345
|
+
evidence = f"{path} edited {int(mt - green_ts)}s after the last passing test run"
|
|
346
|
+
else:
|
|
347
|
+
evidence = f"{path} still uncovered since the last passing run (last edited in an earlier turn)"
|
|
348
|
+
return _r("test_freshness", CheckStatus.STALE, evidence)
|
|
349
|
+
return _r("test_freshness", CheckStatus.PASS, "sources unchanged since the last passing run")
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def test_provenance(session: Session) -> CheckResult:
|
|
353
|
+
test_edits = [fe for fe in session.edits if _is_test_path(fe.path)]
|
|
354
|
+
if not test_edits:
|
|
355
|
+
return _r("test_provenance", CheckStatus.UNSUPPORTED, "no test files touched this session")
|
|
356
|
+
green_ts = _last_green_run_ts(session)
|
|
357
|
+
if green_ts is None:
|
|
358
|
+
return _r("test_provenance", CheckStatus.UNSUPPORTED, "no passing test run to check against")
|
|
359
|
+
after = [(fe.path, fe.ts) for fe in test_edits if fe.ts > green_ts]
|
|
360
|
+
if after:
|
|
361
|
+
path, ts = max(after, key=lambda x: x[1])
|
|
362
|
+
return _r(
|
|
363
|
+
"test_provenance",
|
|
364
|
+
CheckStatus.FAIL,
|
|
365
|
+
f"{path} modified {int(ts - green_ts)}s after the last passing run — that run didn't cover it",
|
|
366
|
+
)
|
|
367
|
+
return _r("test_provenance", CheckStatus.PASS, "tests unchanged since the last passing run")
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def assertion_weakening(session: Session) -> CheckResult:
|
|
371
|
+
return _ast_check(session, "assertion_weakening", assertion_delta,
|
|
372
|
+
"no assertions removed or weakened in edited tests")
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def skip_mock_injection(session: Session) -> CheckResult:
|
|
376
|
+
return _ast_check(session, "skip_mock_injection", skip_or_mock_added,
|
|
377
|
+
"no skips or mocks injected into edited tests")
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def file_state(session: Session) -> CheckResult:
|
|
381
|
+
paths = {fe.path for fe in session.turn_edits}
|
|
382
|
+
if not paths:
|
|
383
|
+
return _r(
|
|
384
|
+
"file_state", CheckStatus.UNSUPPORTED, f"no files created or edited this {_scope(session)}"
|
|
385
|
+
)
|
|
386
|
+
broken = []
|
|
387
|
+
for path in sorted(paths):
|
|
388
|
+
fs = session.files.get(path)
|
|
389
|
+
if fs is None or not fs.exists:
|
|
390
|
+
broken.append(f"{path} — missing")
|
|
391
|
+
elif not (fs.current_text or "").strip():
|
|
392
|
+
broken.append(f"{path} — empty")
|
|
393
|
+
if broken:
|
|
394
|
+
return _r("file_state", CheckStatus.FAIL, "; ".join(broken))
|
|
395
|
+
return _r(
|
|
396
|
+
"file_state",
|
|
397
|
+
CheckStatus.PASS,
|
|
398
|
+
f"all {len(paths)} file(s) edited this {_scope(session)} present on disk",
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def git_state(session: Session) -> CheckResult:
|
|
403
|
+
if not session.git.is_repo:
|
|
404
|
+
return _r("git_state", CheckStatus.UNSUPPORTED, "not a git repository")
|
|
405
|
+
edited = {fe.path for fe in session.turn_edits}
|
|
406
|
+
if not edited:
|
|
407
|
+
return _r(
|
|
408
|
+
"git_state",
|
|
409
|
+
CheckStatus.UNSUPPORTED,
|
|
410
|
+
f"no edits this {_scope(session)} to reconcile against git",
|
|
411
|
+
)
|
|
412
|
+
# Only in-repo paths are git's to speak to. `_relpath` keeps an out-of-repo edit
|
|
413
|
+
# absolute; judging one against this repo's diff would pass on file_state's evidence
|
|
414
|
+
# (the file exists on disk), not git's, and report it "reconciled" (TYCHO-45).
|
|
415
|
+
in_repo = {p for p in edited if _is_in_repo(p)}
|
|
416
|
+
outside = edited - in_repo
|
|
417
|
+
if not in_repo:
|
|
418
|
+
return _r(
|
|
419
|
+
"git_state",
|
|
420
|
+
CheckStatus.UNSUPPORTED,
|
|
421
|
+
f"{len(outside)} path(s) edited outside the repo — nothing here for git to reconcile",
|
|
422
|
+
)
|
|
423
|
+
changed = set(session.git.changed_paths)
|
|
424
|
+
# light git check — a phantom is an edit that's neither in the working diff
|
|
425
|
+
# nor on disk. Richer commit-claim verification arrives with manual --claim/--since.
|
|
426
|
+
phantom = [
|
|
427
|
+
p for p in sorted(in_repo)
|
|
428
|
+
if p not in changed and not (session.files.get(p) and session.files[p].exists)
|
|
429
|
+
]
|
|
430
|
+
if phantom:
|
|
431
|
+
return _r("git_state", CheckStatus.FAIL, f"claimed edits absent from repo: {', '.join(phantom)}")
|
|
432
|
+
evidence = (
|
|
433
|
+
f"{len(in_repo)} path(s) edited this {_scope(session)} reconciled with git "
|
|
434
|
+
f"({len(in_repo & changed)} uncommitted)"
|
|
435
|
+
)
|
|
436
|
+
if outside:
|
|
437
|
+
evidence += f"; {len(outside)} outside the repo — not tracked here"
|
|
438
|
+
return _r("git_state", CheckStatus.PASS, evidence)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def scope_drift(session: Session) -> CheckResult:
|
|
442
|
+
globs = session.config.scope_include
|
|
443
|
+
excludes = session.config.scope_exclude
|
|
444
|
+
if not globs:
|
|
445
|
+
# Point the reader at the fix — a bare "zero-config" is a dead end in the Stop output
|
|
446
|
+
# (TYCHO-74). `tycho scope add` is the concrete action; `tycho scope`/`tycho help`
|
|
447
|
+
# cover the rest, so no dedicated scope-help command is needed.
|
|
448
|
+
return _r("scope_drift", CheckStatus.UNSUPPORTED,
|
|
449
|
+
"no [scope] set — run `tycho scope add '<glob>'` to bound edits (zero-config)")
|
|
450
|
+
edited = {fe.path for fe in session.turn_edits}
|
|
451
|
+
if not edited:
|
|
452
|
+
return _r(
|
|
453
|
+
"scope_drift",
|
|
454
|
+
CheckStatus.UNSUPPORTED,
|
|
455
|
+
f"no edits this {_scope(session)} to check scope against",
|
|
456
|
+
)
|
|
457
|
+
# In scope iff it matches an include glob AND no exclude glob — exclude wins (TYCHO-78).
|
|
458
|
+
def _in_scope(p: str) -> bool:
|
|
459
|
+
return any(fnmatchcase(p, g) for g in globs) and not any(fnmatchcase(p, g) for g in excludes)
|
|
460
|
+
|
|
461
|
+
outside = [p for p in sorted(edited) if not _in_scope(p)]
|
|
462
|
+
if outside:
|
|
463
|
+
return _r("scope_drift", CheckStatus.FAIL, f"edits outside stated scope: {', '.join(outside)}")
|
|
464
|
+
within = f"all edits within scope {list(globs)}"
|
|
465
|
+
return _r("scope_drift", CheckStatus.PASS, f"{within} (excluding {list(excludes)})" if excludes else within)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
# tool_call_provenance families (TYCHO-91). Each is (label, claim pattern, tool-name
|
|
469
|
+
# substrings). BROAD by design: a claim of a family requires *some* tool call of that family
|
|
470
|
+
# in the turn — not a content match, which no generalizable tool schema supports and which
|
|
471
|
+
# would risk a false FAIL (deferred to tycho-web/TYCHO-85 if ever needed). The claim patterns
|
|
472
|
+
# are first-person, past-tense assertions of a *completed* action (never future/hypothetical),
|
|
473
|
+
# anchored on unambiguous cues (a web verb, or a ticket KEY next to a ticket verb) so an honest
|
|
474
|
+
# turn is never failed; a claim we can't classify is simply not counted. This table is the
|
|
475
|
+
# calibration knob — widen coverage by adding rows.
|
|
476
|
+
_PROV_WEB = re.compile(
|
|
477
|
+
r"\b(?:searched (?:the web|online)|web[- ]?searched|googled|"
|
|
478
|
+
r"browsed to|fetched the (?:page|url|web ?site|site))\b",
|
|
479
|
+
re.IGNORECASE,
|
|
480
|
+
)
|
|
481
|
+
# An issue-tracker action is claimed when a ticket anchor (a KEY like TYCHO-95, or "Jira
|
|
482
|
+
# ticket/issue/card") sits within a short window of an action cue — in *either* order, since
|
|
483
|
+
# real prose says both "moved TYCHO-29 to In Progress" and "TYCHO-29 moved to Done" (TYCHO-95).
|
|
484
|
+
# Two cue kinds, both conservative enough to never false-FAIL an honest/future turn:
|
|
485
|
+
# - a *past-tense* action verb (a future "I'll move TYCHO-40" uses base "move", never "moved");
|
|
486
|
+
# - a *two-status arrow* ("Hold → In Review"), which only ever reports a transition that
|
|
487
|
+
# happened — a plan is written "I'll move it to In Review", never "Hold → In Review".
|
|
488
|
+
_ISSUE_ANCHOR = r"(?:\b[A-Z][A-Z0-9]+-\d+\b|\b[Jj]ira (?:ticket|issue|card)\b)"
|
|
489
|
+
_ISSUE_STATUS = r"(?:To ?Do|In Progress|In Review|Backlog|Blocked|Hold|Done)"
|
|
490
|
+
_ISSUE_CUE = (
|
|
491
|
+
r"(?i:\b(?:created|filed|moved|transitioned|assigned|re-?opened|closed|linked|"
|
|
492
|
+
rf"commented on)\b|{_ISSUE_STATUS}\s*(?:→|-+>|↔)\s*{_ISSUE_STATUS})"
|
|
493
|
+
)
|
|
494
|
+
_PROV_ISSUE = re.compile(
|
|
495
|
+
rf"{_ISSUE_CUE}[^.\n]{{0,40}}?{_ISSUE_ANCHOR}|{_ISSUE_ANCHOR}[^.\n]{{0,40}}?{_ISSUE_CUE}",
|
|
496
|
+
)
|
|
497
|
+
_PROVENANCE_FAMILIES = (
|
|
498
|
+
("a web search/fetch", _PROV_WEB, ("search", "fetch", "browse", "web")),
|
|
499
|
+
("an issue-tracker action", _PROV_ISSUE, ("jira", "atlassian", "issue", "linear")),
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _claimed_families(session: Session) -> list[tuple[str, tuple[str, ...]]]:
|
|
504
|
+
"""(label, tool-substrings) for each provenance family whose claim appears in the turn's
|
|
505
|
+
assistant prose. Shared by the check and `has_verifiable_activity` so they can't disagree."""
|
|
506
|
+
prose = "\n".join(m.text for m in session.turn_messages)
|
|
507
|
+
return [(label, tools) for label, pat, tools in _PROVENANCE_FAMILIES if pat.search(prose)]
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def tool_call_provenance(session: Session) -> CheckResult:
|
|
511
|
+
"""Did the agent's prose claim a tool action that never happened? (TYCHO-91)
|
|
512
|
+
|
|
513
|
+
Deterministic and broad, never an LLM judge: a claim of a family (web search, issue-tracker
|
|
514
|
+
action) must be backed by *some* tool call of that family in the turn. An unbacked claim is
|
|
515
|
+
a fabricated action — the thing a user can't see in the chat — so it FAILs. No prose to read
|
|
516
|
+
(a harness we don't mine yet) or no recognized claim is UNSUPPORTED, never a false FAIL.
|
|
517
|
+
"""
|
|
518
|
+
if not session.turn_messages:
|
|
519
|
+
return _r("tool_call_provenance", CheckStatus.UNSUPPORTED, "no assistant prose captured to check")
|
|
520
|
+
claimed = _claimed_families(session)
|
|
521
|
+
if not claimed:
|
|
522
|
+
return _r("tool_call_provenance", CheckStatus.UNSUPPORTED, "no tool-action claims recognized in the turn")
|
|
523
|
+
tools = [e.tool.lower() for e in session.turn_events]
|
|
524
|
+
unbacked = [label for label, subs in claimed if not any(s in t for t in tools for s in subs)]
|
|
525
|
+
if unbacked:
|
|
526
|
+
return _r(
|
|
527
|
+
"tool_call_provenance",
|
|
528
|
+
CheckStatus.FAIL,
|
|
529
|
+
f"claimed {', '.join(unbacked)} with no matching tool call this turn",
|
|
530
|
+
)
|
|
531
|
+
backed = ", ".join(label for label, _ in claimed)
|
|
532
|
+
return _r("tool_call_provenance", CheckStatus.PASS, f"claimed actions are backed by tool calls ({backed})")
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
CHECKS = (
|
|
536
|
+
command_execution,
|
|
537
|
+
tool_call_provenance,
|
|
538
|
+
test_freshness,
|
|
539
|
+
test_provenance,
|
|
540
|
+
assertion_weakening,
|
|
541
|
+
skip_mock_injection,
|
|
542
|
+
file_state,
|
|
543
|
+
git_state,
|
|
544
|
+
scope_drift,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
_TEST_CHECKS = frozenset({
|
|
548
|
+
"command_execution", "test_freshness", "test_provenance",
|
|
549
|
+
"assertion_weakening", "skip_mock_injection",
|
|
550
|
+
})
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def run_checks(session: Session) -> list[CheckResult]:
|
|
554
|
+
disabled = set(session.config.disabled_checks)
|
|
555
|
+
if not session.has_tests:
|
|
556
|
+
disabled.update(_TEST_CHECKS)
|
|
557
|
+
return [check(session) for check in CHECKS if check.__name__ not in disabled]
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def has_verifiable_activity(session: Session) -> bool:
|
|
561
|
+
"""Whether an automatic Stop verdict would have meaningful work to report.
|
|
562
|
+
|
|
563
|
+
Turn-scoped, and that is the whole of TYCHO-17: session-scoped, a read-only turn
|
|
564
|
+
in a session that edited anything earlier still looked like "activity", so every
|
|
565
|
+
later Stop re-reported long-committed work as though it were this turn's. A turn
|
|
566
|
+
that did nothing verifiable earns silence, not a stale green tick.
|
|
567
|
+
|
|
568
|
+
A tool-action *claim* in the prose counts too (TYCHO-91): an MCP-only turn ("I created
|
|
569
|
+
the ticket") has no edits or runners, but its claim is exactly what tool_call_provenance
|
|
570
|
+
exists to check — so the hook must speak rather than stay silent on it.
|
|
571
|
+
"""
|
|
572
|
+
return bool(
|
|
573
|
+
session.turn_edits or _runner_events(session.turn_events) or _claimed_families(session)
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
# --- helpers ----------------------------------------------------------------
|
|
578
|
+
|
|
579
|
+
def _r(name: str, status: CheckStatus, evidence: str) -> CheckResult:
|
|
580
|
+
return CheckResult(name, status, evidence)
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _scope(session: Session) -> str:
|
|
584
|
+
"""What the turn-scoped checks should *call* their scope in the evidence line.
|
|
585
|
+
|
|
586
|
+
A 0.0 boundary means nothing narrowed the view — `tycho verify` auditing a whole
|
|
587
|
+
session, or a harness that can't mark turns — so the honest word is "session".
|
|
588
|
+
Saying "turn" there would re-tell the exact lie TYCHO-17 is about.
|
|
589
|
+
"""
|
|
590
|
+
return "turn" if session.turn_start else "session"
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _runner_events(events) -> list:
|
|
594
|
+
"""The test/build runner invocations among ``events`` — pass the scope you mean."""
|
|
595
|
+
return [
|
|
596
|
+
e
|
|
597
|
+
for e in events
|
|
598
|
+
if e.tool in _SHELL_TOOLS and _runner_segment(e.input.get("command") or "") is not None
|
|
599
|
+
]
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _last_green_run_ts(session: Session) -> float | None:
|
|
603
|
+
# Session-scoped on purpose: a run three turns back still covers a source that
|
|
604
|
+
# hasn't changed since, and test_freshness/test_provenance are exactly the checks
|
|
605
|
+
# whose job is to reason across turns.
|
|
606
|
+
greens = [e.ts for e in _runner_events(session.events) if _outcome(e) is False]
|
|
607
|
+
return max(greens) if greens else None
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
# Prose and pictures. A green test run can't be invalidated by editing them, so
|
|
611
|
+
# test_freshness must not count them as sources — a doc edit after a passing run is
|
|
612
|
+
# not staleness, and STALE sinks the whole verdict. Kept deliberately narrow: config
|
|
613
|
+
# and lockfiles stay sources, because changing a dependency really can break tests.
|
|
614
|
+
_PROSE_SUFFIXES = frozenset({
|
|
615
|
+
".md", ".rst", ".txt", ".adoc",
|
|
616
|
+
".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", ".pdf",
|
|
617
|
+
})
|
|
618
|
+
_PROSE_NAMES = frozenset({"LICENSE", "NOTICE", "AUTHORS", "CODEOWNERS"})
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _is_prose_path(path: str) -> bool:
|
|
622
|
+
"""True for a file whose edits can't change what a test run proves."""
|
|
623
|
+
base = path.replace("\\", "/").rsplit("/", 1)[-1]
|
|
624
|
+
if base in _PROSE_NAMES:
|
|
625
|
+
return True
|
|
626
|
+
dot = base.rfind(".")
|
|
627
|
+
return dot > 0 and base[dot:].lower() in _PROSE_SUFFIXES
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _is_source_path(path: str) -> bool:
|
|
631
|
+
"""True for a file a test run actually covers — not a test, not prose."""
|
|
632
|
+
return not _is_test_path(path) and not _is_prose_path(path)
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _is_test_path(path: str) -> bool:
|
|
636
|
+
p = path.replace("\\", "/")
|
|
637
|
+
base = p.rsplit("/", 1)[-1]
|
|
638
|
+
return (
|
|
639
|
+
"/tests/" in f"/{p}"
|
|
640
|
+
or p.startswith("tests/")
|
|
641
|
+
or base.startswith("test_")
|
|
642
|
+
or base.endswith("_test.py")
|
|
643
|
+
or base == "conftest.py"
|
|
644
|
+
)
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def _is_in_repo(path: str) -> bool:
|
|
648
|
+
"""True for a path `_relpath` left repo-relative. An out-of-repo edit is kept
|
|
649
|
+
absolute (native *or* POSIX flavor — the host can't judge the other), so those are
|
|
650
|
+
not ours to reconcile against this repo's git (TYCHO-45)."""
|
|
651
|
+
return not Path(path).is_absolute() and not PurePosixPath(path).is_absolute()
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _ast_check(session: Session, name: str, differ, clean_msg: str) -> CheckResult:
|
|
655
|
+
test_edits = [fe for fe in session.edits if _is_test_path(fe.path)]
|
|
656
|
+
if not test_edits:
|
|
657
|
+
return _r(name, CheckStatus.UNSUPPORTED, "no edited test files to diff")
|
|
658
|
+
# earliest original per test path = the file's state before the session's first edit
|
|
659
|
+
firsts: dict[str, str] = {}
|
|
660
|
+
for fe in sorted(test_edits, key=lambda e: e.ts):
|
|
661
|
+
if fe.original is not None:
|
|
662
|
+
firsts.setdefault(fe.path, fe.original)
|
|
663
|
+
if not firsts:
|
|
664
|
+
# Test files WERE edited but no pre-session baseline is available — the harness
|
|
665
|
+
# omitted `originalFile` and git couldn't supply it (untracked). A capability gap,
|
|
666
|
+
# not an all-clear: say so distinctly so `•` doesn't read as "tests untouched" (TYCHO-32).
|
|
667
|
+
missing = ", ".join(sorted({fe.path for fe in test_edits}))
|
|
668
|
+
return _r(name, CheckStatus.UNSUPPORTED, f"edited test file(s) with no pre-session baseline to diff: {missing}")
|
|
669
|
+
findings = []
|
|
670
|
+
for path, before in firsts.items():
|
|
671
|
+
fs = session.files.get(path)
|
|
672
|
+
after = fs.current_text if fs else None
|
|
673
|
+
findings.extend(f"{path}: {f}" for f in differ(before, after))
|
|
674
|
+
if findings:
|
|
675
|
+
return _r(name, CheckStatus.FAIL, "; ".join(findings))
|
|
676
|
+
return _r(name, CheckStatus.PASS, clean_msg)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _short(cmd: str, limit: int = 50) -> str:
|
|
680
|
+
cmd = cmd.strip().splitlines()[0] if cmd.strip() else cmd
|
|
681
|
+
return cmd if len(cmd) <= limit else cmd[: limit - 1] + "…"
|