agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,563 @@
1
+ """Check frozen search tasks using captured read-only evidence, without executing commands."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ import shlex
8
+ import stat
9
+ from collections.abc import Mapping
10
+ from dataclasses import dataclass
11
+ from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath
12
+ from typing import Any, Literal
13
+
14
+
15
+ _INDEPENDENT_TOOLS = {"Read", "Grep", "Glob", "StructuredOutput"}
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class SearchHit:
20
+ path: str
21
+ line: int
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class SearchTask:
26
+ scope: str
27
+ symbol: str
28
+ files: Mapping[str, str]
29
+ status: Literal["found", "not_found", "unavailable"]
30
+ hits: tuple[SearchHit, ...]
31
+
32
+
33
+ @dataclass(frozen=True)
34
+ class SearchCheck:
35
+ correct: bool | None
36
+ evidence_complete: bool
37
+ reasons: tuple[str, ...]
38
+
39
+ @property
40
+ def accepted(self) -> bool:
41
+ return self.correct is True and self.evidence_complete and not self.reasons
42
+
43
+ @property
44
+ def classification(self) -> str:
45
+ if self.accepted:
46
+ return "accepted"
47
+ if self.correct is False and self.evidence_complete and not self.reasons:
48
+ return "wrong_task"
49
+ return "unknown"
50
+
51
+
52
+ @dataclass
53
+ class _Proof:
54
+ output: list[set[str]]
55
+ covered: set[str]
56
+ missing: bool = False
57
+ code: int = 0
58
+ scope_checked: bool = False
59
+ cwd: Path | None = None
60
+ status_marker: str | None = None
61
+ changes_cwd: bool = False
62
+
63
+
64
+ def _path(value: str, cwd: Path, root: Path) -> Path:
65
+ path = (cwd / value).resolve()
66
+ if not path.is_relative_to(root):
67
+ raise ValueError("outside fixture")
68
+ return path
69
+
70
+
71
+ def _relative_name(path: PurePath, root: PurePath) -> str:
72
+ return path.relative_to(root).as_posix()
73
+
74
+
75
+ def _fixture(task: SearchTask, root: Path) -> set[str]:
76
+ if not root.is_dir() or root.is_symlink() or not re.fullmatch(r"[A-Za-z_]\w*", task.symbol):
77
+ raise ValueError("invalid fixture")
78
+ for name in (task.scope, *task.files):
79
+ logical = PurePosixPath(name)
80
+ if not name or "\\" in name or PureWindowsPath(name).drive or logical.is_absolute() \
81
+ or logical.as_posix() != name or ".." in logical.parts:
82
+ raise ValueError("invalid fixture paths")
83
+ scope = _path(task.scope, root, root)
84
+ paths = list(root.rglob("*"))
85
+ modes = [path.lstat().st_mode for path in paths]
86
+ if any(not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)) for mode in modes):
87
+ raise ValueError("nonregular fixture entry")
88
+ actual = {_relative_name(path, root): path.read_bytes().decode("utf-8")
89
+ for path, mode in zip(paths, modes) if stat.S_ISREG(mode)}
90
+ if actual != dict(task.files):
91
+ raise ValueError("fixture changed")
92
+ scoped = {name for name in task.files if name.endswith(".py") and
93
+ (root / name).is_relative_to(scope)}
94
+ hits = tuple(sorted((name, index) for name in scoped
95
+ for index, line in enumerate(task.files[name].splitlines(), 1)
96
+ if task.symbol in line))
97
+ status = "found" if hits else "not_found" if scope.is_dir() else "unavailable"
98
+ if task.status != status or sorted((hit.path, hit.line) for hit in task.hits) != list(hits):
99
+ raise ValueError("invalid task ground truth")
100
+ return scoped
101
+
102
+
103
+ def _stream(text: str) -> tuple[list[tuple[str, dict[str, Any], dict[str, Any]]], object]:
104
+ calls: dict[str, tuple[str, dict[str, Any]]] = {}
105
+ results: dict[str, dict[str, Any]] = {}
106
+ overlap: dict[str, set[str]] = {}
107
+ final: dict[str, Any] | None = None
108
+ initialized = False
109
+ for line in text.splitlines():
110
+ event = json.loads(line)
111
+ if not isinstance(event, dict) or final is not None:
112
+ raise ValueError("invalid stream")
113
+ if event.get("type") == "system" and event.get("subtype") == "init":
114
+ if initialized:
115
+ raise ValueError("duplicate init")
116
+ initialized = True
117
+ if event.get("type") == "result":
118
+ final = event
119
+ if event.get("type") not in {"assistant", "user"}:
120
+ continue
121
+ message = event.get("message")
122
+ if not isinstance(message, dict):
123
+ raise ValueError("invalid message")
124
+ blocks = message.get("content", [])
125
+ if not isinstance(blocks, list):
126
+ raise ValueError("invalid content")
127
+ for block in blocks:
128
+ if not isinstance(block, dict):
129
+ raise ValueError("invalid block")
130
+ kind = block.get("type")
131
+ if kind == "tool_use":
132
+ key, name, data = block.get("id"), block.get("name"), block.get("input")
133
+ if not isinstance(key, str) or not key or not isinstance(name, str) \
134
+ or not isinstance(data, dict):
135
+ raise ValueError("invalid call")
136
+ call = (name, data)
137
+ if key in calls and calls[key] != call:
138
+ raise ValueError("conflicting call")
139
+ pending = {other for other, (tool, _) in calls.items()
140
+ if tool not in _INDEPENDENT_TOOLS and other not in results}
141
+ if name not in _INDEPENDENT_TOOLS and key not in calls and pending:
142
+ group = {key} | pending
143
+ for other in pending:
144
+ group |= overlap.get(other, set())
145
+ for other in group:
146
+ overlap[other] = group
147
+ calls[key] = call
148
+ elif kind == "tool_result":
149
+ key = block.get("tool_use_id")
150
+ if not isinstance(key, str) or key not in calls:
151
+ raise ValueError("orphan result")
152
+ value = block.get("content", "")
153
+ if isinstance(value, list) and all(isinstance(b, dict) and
154
+ b.get("type") == "text" and isinstance(b.get("text"), str) for b in value):
155
+ value = "\n".join(b["text"] for b in value)
156
+ error = block.get("is_error", False)
157
+ if not isinstance(value, str) or type(error) is not bool:
158
+ raise ValueError("invalid result")
159
+ result = {"text": value, "error": error}
160
+ if key in results and results[key] != result:
161
+ raise ValueError("conflicting result")
162
+ results[key] = result
163
+ if not initialized or final is None or final.get("subtype") != "success" \
164
+ or final.get("is_error", False) is not False or set(calls) != set(results):
165
+ raise ValueError("incomplete stream")
166
+ return [(name, data, {**results[key], "call_id": key, "overlap": frozenset(overlap.get(key, ()))})
167
+ for key, (name, data) in calls.items()], final.get("structured_output")
168
+
169
+
170
+ def _answer(value: object, task: SearchTask) -> bool | None:
171
+ if not isinstance(value, dict) or set(value) != {"status", "hits"} \
172
+ or not isinstance(value["status"], str) \
173
+ or value["status"] not in {"found", "not_found", "unavailable"}:
174
+ return None
175
+ hits = value["hits"]
176
+ if not isinstance(hits, list) or any(
177
+ not isinstance(hit, dict) or set(hit) != {"path", "line"}
178
+ or not isinstance(hit["path"], str) or type(hit["line"]) is not int or hit["line"] < 1
179
+ for hit in hits
180
+ ):
181
+ return None
182
+ return value["status"] == task.status and sorted((h["path"], h["line"]) for h in hits) == sorted(
183
+ (hit.path, hit.line) for hit in task.hits)
184
+
185
+
186
+ def _tokens(command: str) -> list[str]:
187
+ # Quoted globs are arguments; unquoted expansion would need a real shell and stays unknown.
188
+ quote, escaped = "", False
189
+ for char in command:
190
+ if escaped:
191
+ escaped = False
192
+ elif char == "\\" and quote != "'":
193
+ escaped = True
194
+ elif char in "\"'" and (not quote or char == quote):
195
+ quote = "" if quote else char
196
+ elif char in "$`" and quote != "'" or char in "*?[{" and not quote:
197
+ raise ValueError("shell expansion")
198
+ if "\n" in command or "\0" in command:
199
+ raise ValueError("unsupported shell syntax")
200
+ lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|<>()")
201
+ lexer.whitespace_split = True
202
+ lexer.commenters = ""
203
+ return list(lexer)
204
+
205
+
206
+ def _search(args: list[str], name: str, cwd: Path, root: Path,
207
+ task: SearchTask) -> _Proof:
208
+ positional: list[str] = []
209
+ pattern: str | None = None
210
+ python_only, recursive, number, filename = False, name == "rg", False, False
211
+ hidden, no_ignore = name == "grep", name == "grep"
212
+ index = 0
213
+ while index < len(args):
214
+ arg = args[index]
215
+ if arg == "--":
216
+ positional.extend(args[index + 1:])
217
+ break
218
+ if arg in {"-g", "--glob", "--include", "-t", "--type", "-e", "--regexp"}:
219
+ index += 1
220
+ if index >= len(args):
221
+ raise ValueError("missing option")
222
+ value = args[index]
223
+ if arg in {"-g", "--glob", "--include"} and value in {"*.py", "**/*.py"}:
224
+ python_only = True
225
+ elif arg in {"-t", "--type"} and value in {"py", "python"} and name == "rg":
226
+ python_only = True
227
+ elif arg in {"-e", "--regexp"} and pattern is None:
228
+ pattern = value
229
+ else:
230
+ raise ValueError("unsupported option")
231
+ elif arg in {"--glob=*.py", "--include=*.py", "-g*.py", "-tpy", "-tpython",
232
+ "--type=py", "--type=python"}:
233
+ python_only = True
234
+ elif arg in {"--line-number", "--with-filename", "--no-heading", "--fixed-strings",
235
+ "--color=never", "--no-ignore", "--no-ignore-vcs", "--hidden"}:
236
+ number |= arg == "--line-number"
237
+ filename |= arg == "--with-filename"
238
+ hidden |= arg == "--hidden"
239
+ no_ignore |= arg == "--no-ignore"
240
+ elif arg.startswith("-"):
241
+ allowed = set("nHFrRI") if name == "grep" else set("nHF")
242
+ if len(arg) < 2 or not set(arg[1:]) <= allowed:
243
+ raise ValueError("unsupported option")
244
+ recursive |= bool(set(arg[1:]) & set("rR"))
245
+ number |= "n" in arg
246
+ filename |= "H" in arg
247
+ else:
248
+ positional.append(arg)
249
+ index += 1
250
+ if pattern is None and positional:
251
+ pattern = positional.pop(0)
252
+ if pattern != task.symbol or not number:
253
+ raise ValueError("unsupported search")
254
+ targets = [_path(value, cwd, root) for value in positional or ["."]]
255
+ if not no_ignore and any(PurePosixPath(file).name in {".gitignore", ".ignore", ".rgignore"}
256
+ for file in task.files):
257
+ raise ValueError("unsupported ignore rules")
258
+ if any(not path.exists() for path in targets):
259
+ if len(targets) != 1 or targets[0] != _path(task.scope, root, root):
260
+ raise ValueError("unrelated missing scope")
261
+ displays = {positional[0], str(targets[0]), targets[0].as_posix()}
262
+ errors = {f"{name}: {path}: No such file or directory{suffix}" for path in displays
263
+ for suffix in ("", " (os error 2)")}
264
+ return _Proof([errors], set(), True, 2)
265
+ selected = {file for file in task.files if any(
266
+ root / file == path or recursive and path.is_dir() and (root / file).is_relative_to(path)
267
+ for path in targets) and (not python_only or file.endswith(".py")) and
268
+ (hidden or not any(part.startswith(".") for part in PurePosixPath(file).parts))}
269
+ output: list[set[str]] = []
270
+ show_path = filename or len(targets) > 1 or any(path.is_dir() for path in targets)
271
+ for file in sorted(selected):
272
+ path = root / file
273
+ for line, text in enumerate(task.files[file].splitlines(), 1):
274
+ if task.symbol not in text:
275
+ continue
276
+ if show_path:
277
+ names = {str(path), path.as_posix(), file}
278
+ if path.is_relative_to(cwd):
279
+ relative = _relative_name(path, cwd)
280
+ names |= {relative, "./" + relative}
281
+ output.append({f"{display}:{line}:{text}" for display in names})
282
+ else:
283
+ output.append({f"{line}:{text}"})
284
+ scope = _path(task.scope, root, root)
285
+ return _Proof(output, {file for file in selected if file.endswith(".py")},
286
+ code=0 if output else 1,
287
+ scope_checked=recursive and any(scope.is_relative_to(path) for path in targets))
288
+
289
+
290
+ def _status_suffix(command: str) -> tuple[str, str | None]:
291
+ """Recognize only a final expanded status echo after an unquoted separator."""
292
+ quote, escaped = "", False
293
+ separators = []
294
+ for index, char in enumerate(command):
295
+ if escaped:
296
+ escaped = False
297
+ elif char == "\\" and quote != "'":
298
+ escaped = True
299
+ elif char in "\"'" and (not quote or char == quote):
300
+ quote = "" if quote else char
301
+ elif char == ";" and not quote:
302
+ separators.append(index)
303
+ if quote or "\n" in command or "\0" in command:
304
+ raise ValueError("unsupported shell syntax")
305
+ end = len(command.rstrip(" \t"))
306
+ if separators and separators[-1] == end - 1:
307
+ end = separators.pop()
308
+ if separators:
309
+ start = separators[-1]
310
+ suffix = command[start + 1:end].strip(" \t")
311
+ match = re.fullmatch(r'echo[ \t]+(?:"([^"$`\\]*)\$\?"|([A-Za-z0-9_:=./-]*)\$\?)', suffix)
312
+ if match:
313
+ prefix = match[1] if match[1] is not None else match[2]
314
+ if all(32 <= ord(char) <= 126 for char in prefix):
315
+ return command[:start], prefix
316
+ return command, None
317
+
318
+
319
+ def _bash(command: str, root: Path, task: SearchTask, cwd: Path) -> _Proof:
320
+ command, marker = _status_suffix(command)
321
+ tokens = _tokens(command)
322
+ if tokens[-2:] == ["||", "true"] or tokens[-2:] == ["||", ":"]:
323
+ tokens = tokens[:-2]
324
+ masked = True
325
+ else:
326
+ masked = False
327
+ if not tokens or any(token in {"|", "||", "&", "(", ")", ">", ">>", "<", "<<"}
328
+ for token in tokens):
329
+ raise ValueError("unsupported shell plan")
330
+ parts: list[tuple[str, list[str]]] = []
331
+ current: list[str] = []
332
+ link = ";"
333
+ for token in tokens:
334
+ if token in {";", "&&"}:
335
+ if not current:
336
+ raise ValueError("empty command")
337
+ parts.append((link, current))
338
+ current, link = [], token
339
+ else:
340
+ current.append(token)
341
+ if current:
342
+ parts.append((link, current))
343
+ code = 0
344
+ proof = _Proof([], set())
345
+ for link, argv in parts:
346
+ if link == "&&" and code != 0:
347
+ continue
348
+ name, args = argv[0], argv[1:]
349
+ if name == "cd" and len(args) == 1:
350
+ target = _path(args[0], cwd, root)
351
+ if not target.is_dir():
352
+ raise ValueError("failed prerequisite")
353
+ cwd, code = target, 0
354
+ proof.changes_cwd = True
355
+ elif name == "test" and len(args) == 2 and args[0] in {"-d", "-f", "-e"}:
356
+ path = _path(args[1], cwd, root)
357
+ exists = path.is_dir() if args[0] == "-d" else path.is_file() if args[0] == "-f" \
358
+ else path.exists()
359
+ code = 0 if exists else 1
360
+ elif name == "pwd" and not args:
361
+ proof.output.append({str(cwd), cwd.as_posix()})
362
+ code = 0
363
+ elif name == "ls" and (not args or args[0] == "-1" or
364
+ len(args) == 1 and not args[0].startswith("-")):
365
+ paths = args[1:] if args[:1] == ["-1"] else args
366
+ if len(paths) > 1:
367
+ raise ValueError("unsupported directory listing")
368
+ display = paths[0] if paths else "."
369
+ path = _path(display, cwd, root)
370
+ if path.is_dir():
371
+ children = {(root / file).relative_to(path).parts[0] for file in task.files
372
+ if (root / file).is_relative_to(path)}
373
+ proof.output.extend({child} for child in sorted(children) if not child.startswith("."))
374
+ code = 0
375
+ elif not path.exists() and path == _path(task.scope, root, root):
376
+ proof.output.append({f"ls: {value}: No such file or directory"
377
+ for value in {display, str(path), path.as_posix()}})
378
+ proof.missing, code = True, 1
379
+ else:
380
+ raise ValueError("unsupported directory listing")
381
+ elif name in {"ls", "stat"} and (len(args) == 1 or name == "ls" and
382
+ len(args) == 2 and args[0] in {"-d", "-ld", "-dl"}):
383
+ path = _path(args[-1], cwd, root)
384
+ if path != _path(task.scope, root, root) or path.exists():
385
+ raise ValueError("unsupported scope probe")
386
+ displays = {args[-1], str(path), path.as_posix()}
387
+ proof.output.append({f"{name}: {display}: {prefix}No such file or directory"
388
+ for display in displays for prefix in ("", "stat: ")})
389
+ proof.missing, code = True, 1
390
+ elif name in {"rg", "grep"}:
391
+ step = _search(args, name, cwd, root, task)
392
+ proof.output.extend(step.output)
393
+ proof.covered |= step.covered
394
+ proof.missing |= step.missing
395
+ proof.scope_checked |= step.scope_checked
396
+ code = step.code
397
+ else:
398
+ raise ValueError("unsupported command")
399
+ proof.code = 0 if masked else code
400
+ if marker is not None:
401
+ proof.status_marker = marker + str(proof.code)
402
+ proof.output.append({proof.status_marker})
403
+ proof.code = 0
404
+ proof.cwd = cwd
405
+ return proof
406
+
407
+
408
+ def _output_matches(output: list[set[str]], text: str) -> bool:
409
+ remaining = list(output)
410
+ for line in text.splitlines():
411
+ index = next((i for i, expected in enumerate(remaining) if line in expected), None)
412
+ if index is None:
413
+ return False
414
+ remaining.pop(index)
415
+ return not remaining
416
+
417
+
418
+ def _native(name: str, data: dict[str, Any], result: dict[str, Any], root: Path,
419
+ task: SearchTask) -> _Proof:
420
+ if name in {"Read", "Grep"} and result["error"]:
421
+ value = data.get("file_path" if name == "Read" else "path")
422
+ if not isinstance(value, str) or _path(value, root, root) != _path(task.scope, root, root) \
423
+ or (root / task.scope).exists():
424
+ raise ValueError("unrelated native failure")
425
+ expected_errors = {"File does not exist."} if name == "Read" else {
426
+ f"Path does not exist: {value}", f"Error: Path does not exist: {value}"}
427
+ if result["text"] not in expected_errors:
428
+ raise ValueError("unsupported native failure")
429
+ return _Proof([], set(), True)
430
+ if name == "Read":
431
+ if set(data) != {"file_path"} or not isinstance(data["file_path"], str) or result["error"]:
432
+ raise ValueError("partial or failed read")
433
+ path = _path(data["file_path"], root, root)
434
+ relative = _relative_name(path, root)
435
+ if relative not in task.files:
436
+ raise ValueError("unfrozen read")
437
+ lines = result["text"].splitlines()
438
+ observed = []
439
+ for line in lines:
440
+ match = re.fullmatch(r"\s*(\d+)(?:→|\t)(.*)", line)
441
+ if match is None:
442
+ raise ValueError("unsupported read output")
443
+ observed.append((int(match[1]), match[2]))
444
+ expected = list(enumerate(task.files[relative].splitlines(), 1))
445
+ if observed != expected:
446
+ raise ValueError("incomplete read")
447
+ return _Proof([], {relative})
448
+ if name != "Grep" or set(data) - {"pattern", "path", "glob", "type", "output_mode", "-n"} \
449
+ or data.get("output_mode", "content") != "content" or data.get("-n", True) is not True \
450
+ or result["error"]:
451
+ raise ValueError("unsupported native search")
452
+ args = ["-n", "-e", data["pattern"]]
453
+ if "glob" in data:
454
+ args.extend(["-g", data["glob"]])
455
+ if "type" in data:
456
+ args.extend(["-t", data["type"]])
457
+ args.append(data.get("path", "."))
458
+ proof = _search(args, "rg", root, root, task)
459
+ text = result["text"]
460
+ if not proof.output and text == "No matches found" and not proof.missing:
461
+ return proof
462
+ if proof.missing or not _output_matches(proof.output, text):
463
+ raise ValueError("native output mismatch")
464
+ return proof
465
+
466
+
467
+ def _shell_metadata(command: object) -> bool:
468
+ return isinstance(command, str) and re.fullmatch(
469
+ r'''printf (["'])AUT_SHELL=zsh:%s\\{1,2}n\1 "\$ZSH_VERSION"''', command) is not None
470
+
471
+
472
+ def _state_free_overlap(group: frozenset[str],
473
+ pairs: list[tuple[str, dict[str, Any], dict[str, Any]]],
474
+ root: Path, task: SearchTask, cwd: Path) -> bool:
475
+ for name, data, result in pairs:
476
+ if result["call_id"] not in group:
477
+ continue
478
+ command = data.get("command")
479
+ if name == "Bash" and _shell_metadata(command):
480
+ continue
481
+ try:
482
+ if name != "Bash" or not isinstance(command, str) or _bash(
483
+ command, root, task, cwd
484
+ ).changes_cwd:
485
+ return False
486
+ except (ValueError, TypeError, KeyError, AttributeError, OSError):
487
+ return False
488
+ return True
489
+
490
+
491
+ def check_search_stream(text: str, task: SearchTask, workspace: Path) -> SearchCheck:
492
+ """Compare facts and complete evidence independently against a trusted full fixture."""
493
+ try:
494
+ root = workspace.resolve()
495
+ if workspace.is_symlink():
496
+ raise ValueError("symlink workspace")
497
+ scoped = _fixture(task, root)
498
+ except (OSError, ValueError, TypeError, AttributeError):
499
+ return SearchCheck(None, False, ("fixture_or_ground_truth_invalid",))
500
+ try:
501
+ pairs, answer = _stream(text)
502
+ except (ValueError, TypeError, KeyError):
503
+ return SearchCheck(None, False, ("stream_integrity_invalid",))
504
+ correct = _answer(answer, task)
505
+ covered: set[str] = set()
506
+ missing, scope_checked = False, False
507
+ bash_tainted, bash_cwd = False, root
508
+ checked_overlap: set[frozenset[str]] = set()
509
+ for name, data, result in pairs:
510
+ try:
511
+ if name == "Bash":
512
+ command = data["command"]
513
+ group = result["overlap"]
514
+ if group and group not in checked_overlap:
515
+ checked_overlap.add(group)
516
+ bash_tainted |= not _state_free_overlap(group, pairs, root, task, bash_cwd)
517
+ if _shell_metadata(command):
518
+ continue
519
+ if bash_tainted:
520
+ continue
521
+ try:
522
+ proof = _bash(command, root, task, bash_cwd)
523
+ except (ValueError, TypeError, KeyError, AttributeError, OSError):
524
+ bash_tainted = True
525
+ continue
526
+ output = result["text"]
527
+ if result["error"]:
528
+ prefix = f"Exit code {proof.code}"
529
+ if proof.code == 0 or not (output == prefix or output.startswith(prefix + "\n")):
530
+ bash_tainted |= proof.cwd != bash_cwd
531
+ continue
532
+ output = output[len(prefix):].lstrip("\n")
533
+ elif proof.code != 0:
534
+ bash_tainted |= proof.cwd != bash_cwd
535
+ continue
536
+ if proof.status_marker is not None and (
537
+ not output.splitlines() or output.splitlines()[-1] != proof.status_marker
538
+ ):
539
+ bash_tainted |= proof.cwd != bash_cwd
540
+ continue
541
+ if not _output_matches(proof.output, output):
542
+ bash_tainted |= proof.cwd != bash_cwd
543
+ continue
544
+ bash_cwd = proof.cwd or bash_cwd
545
+ else:
546
+ bash_tainted |= name not in _INDEPENDENT_TOOLS
547
+ proof = _native(name, data, result, root, task)
548
+ covered |= proof.covered
549
+ missing |= proof.missing
550
+ scope_checked |= proof.scope_checked
551
+ except (ValueError, TypeError, KeyError, AttributeError, OSError):
552
+ continue
553
+ complete = missing if task.status == "unavailable" else scoped <= covered and (
554
+ bool(scoped) or scope_checked)
555
+ try:
556
+ _fixture(task, root)
557
+ except (OSError, ValueError, TypeError, AttributeError):
558
+ return SearchCheck(correct, False, ("fixture_or_ground_truth_invalid",))
559
+ reasons: tuple[str, ...] = ("invalid_answer_schema",) if correct is None else ()
560
+ if not complete:
561
+ reasons += ("uncertain_shell_state" if bash_tainted else
562
+ "unsupported_or_incomplete_scope_evidence",)
563
+ return SearchCheck(correct, complete, reasons)