agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,563 @@
|
|
|
1
|
+
"""Check frozen search tasks using captured read-only evidence, without executing commands."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import shlex
|
|
8
|
+
import stat
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
_INDEPENDENT_TOOLS = {"Read", "Grep", "Glob", "StructuredOutput"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class SearchHit:
|
|
20
|
+
path: str
|
|
21
|
+
line: int
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class SearchTask:
|
|
26
|
+
scope: str
|
|
27
|
+
symbol: str
|
|
28
|
+
files: Mapping[str, str]
|
|
29
|
+
status: Literal["found", "not_found", "unavailable"]
|
|
30
|
+
hits: tuple[SearchHit, ...]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class SearchCheck:
|
|
35
|
+
correct: bool | None
|
|
36
|
+
evidence_complete: bool
|
|
37
|
+
reasons: tuple[str, ...]
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def accepted(self) -> bool:
|
|
41
|
+
return self.correct is True and self.evidence_complete and not self.reasons
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def classification(self) -> str:
|
|
45
|
+
if self.accepted:
|
|
46
|
+
return "accepted"
|
|
47
|
+
if self.correct is False and self.evidence_complete and not self.reasons:
|
|
48
|
+
return "wrong_task"
|
|
49
|
+
return "unknown"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class _Proof:
|
|
54
|
+
output: list[set[str]]
|
|
55
|
+
covered: set[str]
|
|
56
|
+
missing: bool = False
|
|
57
|
+
code: int = 0
|
|
58
|
+
scope_checked: bool = False
|
|
59
|
+
cwd: Path | None = None
|
|
60
|
+
status_marker: str | None = None
|
|
61
|
+
changes_cwd: bool = False
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _path(value: str, cwd: Path, root: Path) -> Path:
|
|
65
|
+
path = (cwd / value).resolve()
|
|
66
|
+
if not path.is_relative_to(root):
|
|
67
|
+
raise ValueError("outside fixture")
|
|
68
|
+
return path
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _relative_name(path: PurePath, root: PurePath) -> str:
|
|
72
|
+
return path.relative_to(root).as_posix()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _fixture(task: SearchTask, root: Path) -> set[str]:
|
|
76
|
+
if not root.is_dir() or root.is_symlink() or not re.fullmatch(r"[A-Za-z_]\w*", task.symbol):
|
|
77
|
+
raise ValueError("invalid fixture")
|
|
78
|
+
for name in (task.scope, *task.files):
|
|
79
|
+
logical = PurePosixPath(name)
|
|
80
|
+
if not name or "\\" in name or PureWindowsPath(name).drive or logical.is_absolute() \
|
|
81
|
+
or logical.as_posix() != name or ".." in logical.parts:
|
|
82
|
+
raise ValueError("invalid fixture paths")
|
|
83
|
+
scope = _path(task.scope, root, root)
|
|
84
|
+
paths = list(root.rglob("*"))
|
|
85
|
+
modes = [path.lstat().st_mode for path in paths]
|
|
86
|
+
if any(not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)) for mode in modes):
|
|
87
|
+
raise ValueError("nonregular fixture entry")
|
|
88
|
+
actual = {_relative_name(path, root): path.read_bytes().decode("utf-8")
|
|
89
|
+
for path, mode in zip(paths, modes) if stat.S_ISREG(mode)}
|
|
90
|
+
if actual != dict(task.files):
|
|
91
|
+
raise ValueError("fixture changed")
|
|
92
|
+
scoped = {name for name in task.files if name.endswith(".py") and
|
|
93
|
+
(root / name).is_relative_to(scope)}
|
|
94
|
+
hits = tuple(sorted((name, index) for name in scoped
|
|
95
|
+
for index, line in enumerate(task.files[name].splitlines(), 1)
|
|
96
|
+
if task.symbol in line))
|
|
97
|
+
status = "found" if hits else "not_found" if scope.is_dir() else "unavailable"
|
|
98
|
+
if task.status != status or sorted((hit.path, hit.line) for hit in task.hits) != list(hits):
|
|
99
|
+
raise ValueError("invalid task ground truth")
|
|
100
|
+
return scoped
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _stream(text: str) -> tuple[list[tuple[str, dict[str, Any], dict[str, Any]]], object]:
|
|
104
|
+
calls: dict[str, tuple[str, dict[str, Any]]] = {}
|
|
105
|
+
results: dict[str, dict[str, Any]] = {}
|
|
106
|
+
overlap: dict[str, set[str]] = {}
|
|
107
|
+
final: dict[str, Any] | None = None
|
|
108
|
+
initialized = False
|
|
109
|
+
for line in text.splitlines():
|
|
110
|
+
event = json.loads(line)
|
|
111
|
+
if not isinstance(event, dict) or final is not None:
|
|
112
|
+
raise ValueError("invalid stream")
|
|
113
|
+
if event.get("type") == "system" and event.get("subtype") == "init":
|
|
114
|
+
if initialized:
|
|
115
|
+
raise ValueError("duplicate init")
|
|
116
|
+
initialized = True
|
|
117
|
+
if event.get("type") == "result":
|
|
118
|
+
final = event
|
|
119
|
+
if event.get("type") not in {"assistant", "user"}:
|
|
120
|
+
continue
|
|
121
|
+
message = event.get("message")
|
|
122
|
+
if not isinstance(message, dict):
|
|
123
|
+
raise ValueError("invalid message")
|
|
124
|
+
blocks = message.get("content", [])
|
|
125
|
+
if not isinstance(blocks, list):
|
|
126
|
+
raise ValueError("invalid content")
|
|
127
|
+
for block in blocks:
|
|
128
|
+
if not isinstance(block, dict):
|
|
129
|
+
raise ValueError("invalid block")
|
|
130
|
+
kind = block.get("type")
|
|
131
|
+
if kind == "tool_use":
|
|
132
|
+
key, name, data = block.get("id"), block.get("name"), block.get("input")
|
|
133
|
+
if not isinstance(key, str) or not key or not isinstance(name, str) \
|
|
134
|
+
or not isinstance(data, dict):
|
|
135
|
+
raise ValueError("invalid call")
|
|
136
|
+
call = (name, data)
|
|
137
|
+
if key in calls and calls[key] != call:
|
|
138
|
+
raise ValueError("conflicting call")
|
|
139
|
+
pending = {other for other, (tool, _) in calls.items()
|
|
140
|
+
if tool not in _INDEPENDENT_TOOLS and other not in results}
|
|
141
|
+
if name not in _INDEPENDENT_TOOLS and key not in calls and pending:
|
|
142
|
+
group = {key} | pending
|
|
143
|
+
for other in pending:
|
|
144
|
+
group |= overlap.get(other, set())
|
|
145
|
+
for other in group:
|
|
146
|
+
overlap[other] = group
|
|
147
|
+
calls[key] = call
|
|
148
|
+
elif kind == "tool_result":
|
|
149
|
+
key = block.get("tool_use_id")
|
|
150
|
+
if not isinstance(key, str) or key not in calls:
|
|
151
|
+
raise ValueError("orphan result")
|
|
152
|
+
value = block.get("content", "")
|
|
153
|
+
if isinstance(value, list) and all(isinstance(b, dict) and
|
|
154
|
+
b.get("type") == "text" and isinstance(b.get("text"), str) for b in value):
|
|
155
|
+
value = "\n".join(b["text"] for b in value)
|
|
156
|
+
error = block.get("is_error", False)
|
|
157
|
+
if not isinstance(value, str) or type(error) is not bool:
|
|
158
|
+
raise ValueError("invalid result")
|
|
159
|
+
result = {"text": value, "error": error}
|
|
160
|
+
if key in results and results[key] != result:
|
|
161
|
+
raise ValueError("conflicting result")
|
|
162
|
+
results[key] = result
|
|
163
|
+
if not initialized or final is None or final.get("subtype") != "success" \
|
|
164
|
+
or final.get("is_error", False) is not False or set(calls) != set(results):
|
|
165
|
+
raise ValueError("incomplete stream")
|
|
166
|
+
return [(name, data, {**results[key], "call_id": key, "overlap": frozenset(overlap.get(key, ()))})
|
|
167
|
+
for key, (name, data) in calls.items()], final.get("structured_output")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _answer(value: object, task: SearchTask) -> bool | None:
|
|
171
|
+
if not isinstance(value, dict) or set(value) != {"status", "hits"} \
|
|
172
|
+
or not isinstance(value["status"], str) \
|
|
173
|
+
or value["status"] not in {"found", "not_found", "unavailable"}:
|
|
174
|
+
return None
|
|
175
|
+
hits = value["hits"]
|
|
176
|
+
if not isinstance(hits, list) or any(
|
|
177
|
+
not isinstance(hit, dict) or set(hit) != {"path", "line"}
|
|
178
|
+
or not isinstance(hit["path"], str) or type(hit["line"]) is not int or hit["line"] < 1
|
|
179
|
+
for hit in hits
|
|
180
|
+
):
|
|
181
|
+
return None
|
|
182
|
+
return value["status"] == task.status and sorted((h["path"], h["line"]) for h in hits) == sorted(
|
|
183
|
+
(hit.path, hit.line) for hit in task.hits)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _tokens(command: str) -> list[str]:
|
|
187
|
+
# Quoted globs are arguments; unquoted expansion would need a real shell and stays unknown.
|
|
188
|
+
quote, escaped = "", False
|
|
189
|
+
for char in command:
|
|
190
|
+
if escaped:
|
|
191
|
+
escaped = False
|
|
192
|
+
elif char == "\\" and quote != "'":
|
|
193
|
+
escaped = True
|
|
194
|
+
elif char in "\"'" and (not quote or char == quote):
|
|
195
|
+
quote = "" if quote else char
|
|
196
|
+
elif char in "$`" and quote != "'" or char in "*?[{" and not quote:
|
|
197
|
+
raise ValueError("shell expansion")
|
|
198
|
+
if "\n" in command or "\0" in command:
|
|
199
|
+
raise ValueError("unsupported shell syntax")
|
|
200
|
+
lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|<>()")
|
|
201
|
+
lexer.whitespace_split = True
|
|
202
|
+
lexer.commenters = ""
|
|
203
|
+
return list(lexer)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _search(args: list[str], name: str, cwd: Path, root: Path,
|
|
207
|
+
task: SearchTask) -> _Proof:
|
|
208
|
+
positional: list[str] = []
|
|
209
|
+
pattern: str | None = None
|
|
210
|
+
python_only, recursive, number, filename = False, name == "rg", False, False
|
|
211
|
+
hidden, no_ignore = name == "grep", name == "grep"
|
|
212
|
+
index = 0
|
|
213
|
+
while index < len(args):
|
|
214
|
+
arg = args[index]
|
|
215
|
+
if arg == "--":
|
|
216
|
+
positional.extend(args[index + 1:])
|
|
217
|
+
break
|
|
218
|
+
if arg in {"-g", "--glob", "--include", "-t", "--type", "-e", "--regexp"}:
|
|
219
|
+
index += 1
|
|
220
|
+
if index >= len(args):
|
|
221
|
+
raise ValueError("missing option")
|
|
222
|
+
value = args[index]
|
|
223
|
+
if arg in {"-g", "--glob", "--include"} and value in {"*.py", "**/*.py"}:
|
|
224
|
+
python_only = True
|
|
225
|
+
elif arg in {"-t", "--type"} and value in {"py", "python"} and name == "rg":
|
|
226
|
+
python_only = True
|
|
227
|
+
elif arg in {"-e", "--regexp"} and pattern is None:
|
|
228
|
+
pattern = value
|
|
229
|
+
else:
|
|
230
|
+
raise ValueError("unsupported option")
|
|
231
|
+
elif arg in {"--glob=*.py", "--include=*.py", "-g*.py", "-tpy", "-tpython",
|
|
232
|
+
"--type=py", "--type=python"}:
|
|
233
|
+
python_only = True
|
|
234
|
+
elif arg in {"--line-number", "--with-filename", "--no-heading", "--fixed-strings",
|
|
235
|
+
"--color=never", "--no-ignore", "--no-ignore-vcs", "--hidden"}:
|
|
236
|
+
number |= arg == "--line-number"
|
|
237
|
+
filename |= arg == "--with-filename"
|
|
238
|
+
hidden |= arg == "--hidden"
|
|
239
|
+
no_ignore |= arg == "--no-ignore"
|
|
240
|
+
elif arg.startswith("-"):
|
|
241
|
+
allowed = set("nHFrRI") if name == "grep" else set("nHF")
|
|
242
|
+
if len(arg) < 2 or not set(arg[1:]) <= allowed:
|
|
243
|
+
raise ValueError("unsupported option")
|
|
244
|
+
recursive |= bool(set(arg[1:]) & set("rR"))
|
|
245
|
+
number |= "n" in arg
|
|
246
|
+
filename |= "H" in arg
|
|
247
|
+
else:
|
|
248
|
+
positional.append(arg)
|
|
249
|
+
index += 1
|
|
250
|
+
if pattern is None and positional:
|
|
251
|
+
pattern = positional.pop(0)
|
|
252
|
+
if pattern != task.symbol or not number:
|
|
253
|
+
raise ValueError("unsupported search")
|
|
254
|
+
targets = [_path(value, cwd, root) for value in positional or ["."]]
|
|
255
|
+
if not no_ignore and any(PurePosixPath(file).name in {".gitignore", ".ignore", ".rgignore"}
|
|
256
|
+
for file in task.files):
|
|
257
|
+
raise ValueError("unsupported ignore rules")
|
|
258
|
+
if any(not path.exists() for path in targets):
|
|
259
|
+
if len(targets) != 1 or targets[0] != _path(task.scope, root, root):
|
|
260
|
+
raise ValueError("unrelated missing scope")
|
|
261
|
+
displays = {positional[0], str(targets[0]), targets[0].as_posix()}
|
|
262
|
+
errors = {f"{name}: {path}: No such file or directory{suffix}" for path in displays
|
|
263
|
+
for suffix in ("", " (os error 2)")}
|
|
264
|
+
return _Proof([errors], set(), True, 2)
|
|
265
|
+
selected = {file for file in task.files if any(
|
|
266
|
+
root / file == path or recursive and path.is_dir() and (root / file).is_relative_to(path)
|
|
267
|
+
for path in targets) and (not python_only or file.endswith(".py")) and
|
|
268
|
+
(hidden or not any(part.startswith(".") for part in PurePosixPath(file).parts))}
|
|
269
|
+
output: list[set[str]] = []
|
|
270
|
+
show_path = filename or len(targets) > 1 or any(path.is_dir() for path in targets)
|
|
271
|
+
for file in sorted(selected):
|
|
272
|
+
path = root / file
|
|
273
|
+
for line, text in enumerate(task.files[file].splitlines(), 1):
|
|
274
|
+
if task.symbol not in text:
|
|
275
|
+
continue
|
|
276
|
+
if show_path:
|
|
277
|
+
names = {str(path), path.as_posix(), file}
|
|
278
|
+
if path.is_relative_to(cwd):
|
|
279
|
+
relative = _relative_name(path, cwd)
|
|
280
|
+
names |= {relative, "./" + relative}
|
|
281
|
+
output.append({f"{display}:{line}:{text}" for display in names})
|
|
282
|
+
else:
|
|
283
|
+
output.append({f"{line}:{text}"})
|
|
284
|
+
scope = _path(task.scope, root, root)
|
|
285
|
+
return _Proof(output, {file for file in selected if file.endswith(".py")},
|
|
286
|
+
code=0 if output else 1,
|
|
287
|
+
scope_checked=recursive and any(scope.is_relative_to(path) for path in targets))
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _status_suffix(command: str) -> tuple[str, str | None]:
|
|
291
|
+
"""Recognize only a final expanded status echo after an unquoted separator."""
|
|
292
|
+
quote, escaped = "", False
|
|
293
|
+
separators = []
|
|
294
|
+
for index, char in enumerate(command):
|
|
295
|
+
if escaped:
|
|
296
|
+
escaped = False
|
|
297
|
+
elif char == "\\" and quote != "'":
|
|
298
|
+
escaped = True
|
|
299
|
+
elif char in "\"'" and (not quote or char == quote):
|
|
300
|
+
quote = "" if quote else char
|
|
301
|
+
elif char == ";" and not quote:
|
|
302
|
+
separators.append(index)
|
|
303
|
+
if quote or "\n" in command or "\0" in command:
|
|
304
|
+
raise ValueError("unsupported shell syntax")
|
|
305
|
+
end = len(command.rstrip(" \t"))
|
|
306
|
+
if separators and separators[-1] == end - 1:
|
|
307
|
+
end = separators.pop()
|
|
308
|
+
if separators:
|
|
309
|
+
start = separators[-1]
|
|
310
|
+
suffix = command[start + 1:end].strip(" \t")
|
|
311
|
+
match = re.fullmatch(r'echo[ \t]+(?:"([^"$`\\]*)\$\?"|([A-Za-z0-9_:=./-]*)\$\?)', suffix)
|
|
312
|
+
if match:
|
|
313
|
+
prefix = match[1] if match[1] is not None else match[2]
|
|
314
|
+
if all(32 <= ord(char) <= 126 for char in prefix):
|
|
315
|
+
return command[:start], prefix
|
|
316
|
+
return command, None
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _bash(command: str, root: Path, task: SearchTask, cwd: Path) -> _Proof:
|
|
320
|
+
command, marker = _status_suffix(command)
|
|
321
|
+
tokens = _tokens(command)
|
|
322
|
+
if tokens[-2:] == ["||", "true"] or tokens[-2:] == ["||", ":"]:
|
|
323
|
+
tokens = tokens[:-2]
|
|
324
|
+
masked = True
|
|
325
|
+
else:
|
|
326
|
+
masked = False
|
|
327
|
+
if not tokens or any(token in {"|", "||", "&", "(", ")", ">", ">>", "<", "<<"}
|
|
328
|
+
for token in tokens):
|
|
329
|
+
raise ValueError("unsupported shell plan")
|
|
330
|
+
parts: list[tuple[str, list[str]]] = []
|
|
331
|
+
current: list[str] = []
|
|
332
|
+
link = ";"
|
|
333
|
+
for token in tokens:
|
|
334
|
+
if token in {";", "&&"}:
|
|
335
|
+
if not current:
|
|
336
|
+
raise ValueError("empty command")
|
|
337
|
+
parts.append((link, current))
|
|
338
|
+
current, link = [], token
|
|
339
|
+
else:
|
|
340
|
+
current.append(token)
|
|
341
|
+
if current:
|
|
342
|
+
parts.append((link, current))
|
|
343
|
+
code = 0
|
|
344
|
+
proof = _Proof([], set())
|
|
345
|
+
for link, argv in parts:
|
|
346
|
+
if link == "&&" and code != 0:
|
|
347
|
+
continue
|
|
348
|
+
name, args = argv[0], argv[1:]
|
|
349
|
+
if name == "cd" and len(args) == 1:
|
|
350
|
+
target = _path(args[0], cwd, root)
|
|
351
|
+
if not target.is_dir():
|
|
352
|
+
raise ValueError("failed prerequisite")
|
|
353
|
+
cwd, code = target, 0
|
|
354
|
+
proof.changes_cwd = True
|
|
355
|
+
elif name == "test" and len(args) == 2 and args[0] in {"-d", "-f", "-e"}:
|
|
356
|
+
path = _path(args[1], cwd, root)
|
|
357
|
+
exists = path.is_dir() if args[0] == "-d" else path.is_file() if args[0] == "-f" \
|
|
358
|
+
else path.exists()
|
|
359
|
+
code = 0 if exists else 1
|
|
360
|
+
elif name == "pwd" and not args:
|
|
361
|
+
proof.output.append({str(cwd), cwd.as_posix()})
|
|
362
|
+
code = 0
|
|
363
|
+
elif name == "ls" and (not args or args[0] == "-1" or
|
|
364
|
+
len(args) == 1 and not args[0].startswith("-")):
|
|
365
|
+
paths = args[1:] if args[:1] == ["-1"] else args
|
|
366
|
+
if len(paths) > 1:
|
|
367
|
+
raise ValueError("unsupported directory listing")
|
|
368
|
+
display = paths[0] if paths else "."
|
|
369
|
+
path = _path(display, cwd, root)
|
|
370
|
+
if path.is_dir():
|
|
371
|
+
children = {(root / file).relative_to(path).parts[0] for file in task.files
|
|
372
|
+
if (root / file).is_relative_to(path)}
|
|
373
|
+
proof.output.extend({child} for child in sorted(children) if not child.startswith("."))
|
|
374
|
+
code = 0
|
|
375
|
+
elif not path.exists() and path == _path(task.scope, root, root):
|
|
376
|
+
proof.output.append({f"ls: {value}: No such file or directory"
|
|
377
|
+
for value in {display, str(path), path.as_posix()}})
|
|
378
|
+
proof.missing, code = True, 1
|
|
379
|
+
else:
|
|
380
|
+
raise ValueError("unsupported directory listing")
|
|
381
|
+
elif name in {"ls", "stat"} and (len(args) == 1 or name == "ls" and
|
|
382
|
+
len(args) == 2 and args[0] in {"-d", "-ld", "-dl"}):
|
|
383
|
+
path = _path(args[-1], cwd, root)
|
|
384
|
+
if path != _path(task.scope, root, root) or path.exists():
|
|
385
|
+
raise ValueError("unsupported scope probe")
|
|
386
|
+
displays = {args[-1], str(path), path.as_posix()}
|
|
387
|
+
proof.output.append({f"{name}: {display}: {prefix}No such file or directory"
|
|
388
|
+
for display in displays for prefix in ("", "stat: ")})
|
|
389
|
+
proof.missing, code = True, 1
|
|
390
|
+
elif name in {"rg", "grep"}:
|
|
391
|
+
step = _search(args, name, cwd, root, task)
|
|
392
|
+
proof.output.extend(step.output)
|
|
393
|
+
proof.covered |= step.covered
|
|
394
|
+
proof.missing |= step.missing
|
|
395
|
+
proof.scope_checked |= step.scope_checked
|
|
396
|
+
code = step.code
|
|
397
|
+
else:
|
|
398
|
+
raise ValueError("unsupported command")
|
|
399
|
+
proof.code = 0 if masked else code
|
|
400
|
+
if marker is not None:
|
|
401
|
+
proof.status_marker = marker + str(proof.code)
|
|
402
|
+
proof.output.append({proof.status_marker})
|
|
403
|
+
proof.code = 0
|
|
404
|
+
proof.cwd = cwd
|
|
405
|
+
return proof
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _output_matches(output: list[set[str]], text: str) -> bool:
|
|
409
|
+
remaining = list(output)
|
|
410
|
+
for line in text.splitlines():
|
|
411
|
+
index = next((i for i, expected in enumerate(remaining) if line in expected), None)
|
|
412
|
+
if index is None:
|
|
413
|
+
return False
|
|
414
|
+
remaining.pop(index)
|
|
415
|
+
return not remaining
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _native(name: str, data: dict[str, Any], result: dict[str, Any], root: Path,
|
|
419
|
+
task: SearchTask) -> _Proof:
|
|
420
|
+
if name in {"Read", "Grep"} and result["error"]:
|
|
421
|
+
value = data.get("file_path" if name == "Read" else "path")
|
|
422
|
+
if not isinstance(value, str) or _path(value, root, root) != _path(task.scope, root, root) \
|
|
423
|
+
or (root / task.scope).exists():
|
|
424
|
+
raise ValueError("unrelated native failure")
|
|
425
|
+
expected_errors = {"File does not exist."} if name == "Read" else {
|
|
426
|
+
f"Path does not exist: {value}", f"Error: Path does not exist: {value}"}
|
|
427
|
+
if result["text"] not in expected_errors:
|
|
428
|
+
raise ValueError("unsupported native failure")
|
|
429
|
+
return _Proof([], set(), True)
|
|
430
|
+
if name == "Read":
|
|
431
|
+
if set(data) != {"file_path"} or not isinstance(data["file_path"], str) or result["error"]:
|
|
432
|
+
raise ValueError("partial or failed read")
|
|
433
|
+
path = _path(data["file_path"], root, root)
|
|
434
|
+
relative = _relative_name(path, root)
|
|
435
|
+
if relative not in task.files:
|
|
436
|
+
raise ValueError("unfrozen read")
|
|
437
|
+
lines = result["text"].splitlines()
|
|
438
|
+
observed = []
|
|
439
|
+
for line in lines:
|
|
440
|
+
match = re.fullmatch(r"\s*(\d+)(?:→|\t)(.*)", line)
|
|
441
|
+
if match is None:
|
|
442
|
+
raise ValueError("unsupported read output")
|
|
443
|
+
observed.append((int(match[1]), match[2]))
|
|
444
|
+
expected = list(enumerate(task.files[relative].splitlines(), 1))
|
|
445
|
+
if observed != expected:
|
|
446
|
+
raise ValueError("incomplete read")
|
|
447
|
+
return _Proof([], {relative})
|
|
448
|
+
if name != "Grep" or set(data) - {"pattern", "path", "glob", "type", "output_mode", "-n"} \
|
|
449
|
+
or data.get("output_mode", "content") != "content" or data.get("-n", True) is not True \
|
|
450
|
+
or result["error"]:
|
|
451
|
+
raise ValueError("unsupported native search")
|
|
452
|
+
args = ["-n", "-e", data["pattern"]]
|
|
453
|
+
if "glob" in data:
|
|
454
|
+
args.extend(["-g", data["glob"]])
|
|
455
|
+
if "type" in data:
|
|
456
|
+
args.extend(["-t", data["type"]])
|
|
457
|
+
args.append(data.get("path", "."))
|
|
458
|
+
proof = _search(args, "rg", root, root, task)
|
|
459
|
+
text = result["text"]
|
|
460
|
+
if not proof.output and text == "No matches found" and not proof.missing:
|
|
461
|
+
return proof
|
|
462
|
+
if proof.missing or not _output_matches(proof.output, text):
|
|
463
|
+
raise ValueError("native output mismatch")
|
|
464
|
+
return proof
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _shell_metadata(command: object) -> bool:
|
|
468
|
+
return isinstance(command, str) and re.fullmatch(
|
|
469
|
+
r'''printf (["'])AUT_SHELL=zsh:%s\\{1,2}n\1 "\$ZSH_VERSION"''', command) is not None
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _state_free_overlap(group: frozenset[str],
|
|
473
|
+
pairs: list[tuple[str, dict[str, Any], dict[str, Any]]],
|
|
474
|
+
root: Path, task: SearchTask, cwd: Path) -> bool:
|
|
475
|
+
for name, data, result in pairs:
|
|
476
|
+
if result["call_id"] not in group:
|
|
477
|
+
continue
|
|
478
|
+
command = data.get("command")
|
|
479
|
+
if name == "Bash" and _shell_metadata(command):
|
|
480
|
+
continue
|
|
481
|
+
try:
|
|
482
|
+
if name != "Bash" or not isinstance(command, str) or _bash(
|
|
483
|
+
command, root, task, cwd
|
|
484
|
+
).changes_cwd:
|
|
485
|
+
return False
|
|
486
|
+
except (ValueError, TypeError, KeyError, AttributeError, OSError):
|
|
487
|
+
return False
|
|
488
|
+
return True
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def check_search_stream(text: str, task: SearchTask, workspace: Path) -> SearchCheck:
|
|
492
|
+
"""Compare facts and complete evidence independently against a trusted full fixture."""
|
|
493
|
+
try:
|
|
494
|
+
root = workspace.resolve()
|
|
495
|
+
if workspace.is_symlink():
|
|
496
|
+
raise ValueError("symlink workspace")
|
|
497
|
+
scoped = _fixture(task, root)
|
|
498
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
499
|
+
return SearchCheck(None, False, ("fixture_or_ground_truth_invalid",))
|
|
500
|
+
try:
|
|
501
|
+
pairs, answer = _stream(text)
|
|
502
|
+
except (ValueError, TypeError, KeyError):
|
|
503
|
+
return SearchCheck(None, False, ("stream_integrity_invalid",))
|
|
504
|
+
correct = _answer(answer, task)
|
|
505
|
+
covered: set[str] = set()
|
|
506
|
+
missing, scope_checked = False, False
|
|
507
|
+
bash_tainted, bash_cwd = False, root
|
|
508
|
+
checked_overlap: set[frozenset[str]] = set()
|
|
509
|
+
for name, data, result in pairs:
|
|
510
|
+
try:
|
|
511
|
+
if name == "Bash":
|
|
512
|
+
command = data["command"]
|
|
513
|
+
group = result["overlap"]
|
|
514
|
+
if group and group not in checked_overlap:
|
|
515
|
+
checked_overlap.add(group)
|
|
516
|
+
bash_tainted |= not _state_free_overlap(group, pairs, root, task, bash_cwd)
|
|
517
|
+
if _shell_metadata(command):
|
|
518
|
+
continue
|
|
519
|
+
if bash_tainted:
|
|
520
|
+
continue
|
|
521
|
+
try:
|
|
522
|
+
proof = _bash(command, root, task, bash_cwd)
|
|
523
|
+
except (ValueError, TypeError, KeyError, AttributeError, OSError):
|
|
524
|
+
bash_tainted = True
|
|
525
|
+
continue
|
|
526
|
+
output = result["text"]
|
|
527
|
+
if result["error"]:
|
|
528
|
+
prefix = f"Exit code {proof.code}"
|
|
529
|
+
if proof.code == 0 or not (output == prefix or output.startswith(prefix + "\n")):
|
|
530
|
+
bash_tainted |= proof.cwd != bash_cwd
|
|
531
|
+
continue
|
|
532
|
+
output = output[len(prefix):].lstrip("\n")
|
|
533
|
+
elif proof.code != 0:
|
|
534
|
+
bash_tainted |= proof.cwd != bash_cwd
|
|
535
|
+
continue
|
|
536
|
+
if proof.status_marker is not None and (
|
|
537
|
+
not output.splitlines() or output.splitlines()[-1] != proof.status_marker
|
|
538
|
+
):
|
|
539
|
+
bash_tainted |= proof.cwd != bash_cwd
|
|
540
|
+
continue
|
|
541
|
+
if not _output_matches(proof.output, output):
|
|
542
|
+
bash_tainted |= proof.cwd != bash_cwd
|
|
543
|
+
continue
|
|
544
|
+
bash_cwd = proof.cwd or bash_cwd
|
|
545
|
+
else:
|
|
546
|
+
bash_tainted |= name not in _INDEPENDENT_TOOLS
|
|
547
|
+
proof = _native(name, data, result, root, task)
|
|
548
|
+
covered |= proof.covered
|
|
549
|
+
missing |= proof.missing
|
|
550
|
+
scope_checked |= proof.scope_checked
|
|
551
|
+
except (ValueError, TypeError, KeyError, AttributeError, OSError):
|
|
552
|
+
continue
|
|
553
|
+
complete = missing if task.status == "unavailable" else scoped <= covered and (
|
|
554
|
+
bool(scoped) or scope_checked)
|
|
555
|
+
try:
|
|
556
|
+
_fixture(task, root)
|
|
557
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
558
|
+
return SearchCheck(correct, False, ("fixture_or_ground_truth_invalid",))
|
|
559
|
+
reasons: tuple[str, ...] = ("invalid_answer_schema",) if correct is None else ()
|
|
560
|
+
if not complete:
|
|
561
|
+
reasons += ("uncertain_shell_state" if bash_tainted else
|
|
562
|
+
"unsupported_or_incomplete_scope_evidence",)
|
|
563
|
+
return SearchCheck(correct, complete, reasons)
|