agent-coach-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_coach_cli-0.1.0.dist-info/METADATA +242 -0
- agent_coach_cli-0.1.0.dist-info/RECORD +19 -0
- agent_coach_cli-0.1.0.dist-info/WHEEL +5 -0
- agent_coach_cli-0.1.0.dist-info/entry_points.txt +2 -0
- agent_coach_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
- agent_coach_cli-0.1.0.dist-info/top_level.txt +1 -0
- agentcoach/__init__.py +1 -0
- agentcoach/__main__.py +3 -0
- agentcoach/cli.py +254 -0
- agentcoach/collect.py +336 -0
- agentcoach/heuristics.py +134 -0
- agentcoach/i18n.py +88 -0
- agentcoach/judge.py +328 -0
- agentcoach/paths.py +24 -0
- agentcoach/redact.py +52 -0
- agentcoach/report.py +559 -0
- agentcoach/rubric/default.json +561 -0
- agentcoach/rubric.py +46 -0
- agentcoach/store.py +102 -0
agentcoach/collect.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
"""Read local Claude Code and Codex CLI transcripts into a common shape.
|
|
2
|
+
|
|
3
|
+
Only the human side matters for coaching, so a Session keeps the user's
|
|
4
|
+
prompts verbatim and squeezes everything the agent did between them into a
|
|
5
|
+
one-line summary (tool counts + a short excerpt of its last reply).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import glob
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass, field, asdict
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Iterable, Iterator
|
|
17
|
+
|
|
18
|
+
from .paths import coach_home
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Turn:
|
|
23
|
+
index: int
|
|
24
|
+
text: str
|
|
25
|
+
ts: str = ""
|
|
26
|
+
images: int = 0
|
|
27
|
+
slash: str = "" # e.g. "/clear" when the turn was a slash command
|
|
28
|
+
agent_after: str = "" # summary of what the agent did before the next human turn
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Session:
|
|
33
|
+
source: str # "claude" | "codex"
|
|
34
|
+
id: str
|
|
35
|
+
project: str
|
|
36
|
+
path: str
|
|
37
|
+
started: str
|
|
38
|
+
ended: str
|
|
39
|
+
turns: list[Turn] = field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
def to_dict(self) -> dict:
|
|
42
|
+
return asdict(self)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# ---------------------------------------------------------------- helpers
|
|
46
|
+
|
|
47
|
+
_NOISE_PREFIXES = (
|
|
48
|
+
"<local-command-", "<command-message>", "<task-notification", "<system-reminder",
|
|
49
|
+
"<side-sessions-event", "<ci-monitor-event", "Caveat:", "[Request interrupted",
|
|
50
|
+
"<environment_context>", "<user_instructions>", "# AGENTS.md", "<permissions",
|
|
51
|
+
"<turn_aborted>", "<user_shell_command>", "<subagent_notification", "<image",
|
|
52
|
+
"This session is being continued from a previous conversation", # auto compaction summary
|
|
53
|
+
"<artifact-view-context", "<ide_", "<bash-input>", "<bash-stdout>",
|
|
54
|
+
)
|
|
55
|
+
_SLASH_RE = re.compile(r"<command-name>\s*(/[\w:-]+)\s*</command-name>")
|
|
56
|
+
_ARGS_RE = re.compile(r"<command-args>(.*?)</command-args>", re.S)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _parse_ts(s: str) -> datetime | None:
|
|
60
|
+
if not s:
|
|
61
|
+
return None
|
|
62
|
+
try:
|
|
63
|
+
return datetime.fromisoformat(s.replace("Z", "+00:00"))
|
|
64
|
+
except ValueError:
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _summarize_agent(tools: dict[str, int], last_text: str) -> str:
|
|
69
|
+
parts = []
|
|
70
|
+
if tools:
|
|
71
|
+
top = sorted(tools.items(), key=lambda kv: -kv[1])[:5]
|
|
72
|
+
parts.append("tools: " + ", ".join(f"{k}x{v}" for k, v in top))
|
|
73
|
+
if last_text:
|
|
74
|
+
t = " ".join(last_text.split())
|
|
75
|
+
parts.append("reply: " + (t[:280] + "…" if len(t) > 280 else t))
|
|
76
|
+
return " | ".join(parts)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _is_temp_project(dirname: str) -> bool:
|
|
80
|
+
low = dirname.lower()
|
|
81
|
+
return "-temp-" in low or "appdata-local-temp" in low or "agent-coach-judge" in low
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------- Claude Code
|
|
85
|
+
|
|
86
|
+
def claude_root() -> Path:
|
|
87
|
+
return Path(os.environ.get("CLAUDE_CONFIG_DIR", Path.home() / ".claude")) / "projects"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _claude_human_text(d: dict) -> tuple[str, int] | None:
|
|
91
|
+
"""Return (text, image_count) when this record is a prompt a human typed."""
|
|
92
|
+
if d.get("type") != "user" or d.get("isSidechain") or d.get("isMeta"):
|
|
93
|
+
return None
|
|
94
|
+
origin = d.get("origin")
|
|
95
|
+
if isinstance(origin, dict) and origin.get("kind") not in (None, "human"):
|
|
96
|
+
return None
|
|
97
|
+
content = (d.get("message") or {}).get("content")
|
|
98
|
+
images = 0
|
|
99
|
+
if isinstance(content, str):
|
|
100
|
+
text = content
|
|
101
|
+
elif isinstance(content, list):
|
|
102
|
+
if any(isinstance(c, dict) and c.get("type") == "tool_result" for c in content):
|
|
103
|
+
return None
|
|
104
|
+
texts = []
|
|
105
|
+
for c in content:
|
|
106
|
+
if not isinstance(c, dict):
|
|
107
|
+
continue
|
|
108
|
+
if c.get("type") == "text":
|
|
109
|
+
texts.append(c.get("text", ""))
|
|
110
|
+
elif c.get("type") == "image":
|
|
111
|
+
images += 1
|
|
112
|
+
text = "\n".join(texts)
|
|
113
|
+
else:
|
|
114
|
+
return None
|
|
115
|
+
return text, images
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def parse_claude_file(path: str, include_sdk: bool = False) -> Session | None:
|
|
119
|
+
project = Path(path).parent.name
|
|
120
|
+
turns: list[Turn] = []
|
|
121
|
+
tools: dict[str, int] = {}
|
|
122
|
+
last_text = ""
|
|
123
|
+
first_ts = last_ts = ""
|
|
124
|
+
sid = Path(path).stem
|
|
125
|
+
saw_human = saw_script = False
|
|
126
|
+
|
|
127
|
+
def flush():
|
|
128
|
+
nonlocal tools, last_text
|
|
129
|
+
if turns:
|
|
130
|
+
turns[-1].agent_after = _summarize_agent(tools, last_text)
|
|
131
|
+
tools, last_text = {}, ""
|
|
132
|
+
|
|
133
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
134
|
+
for line in fh:
|
|
135
|
+
try:
|
|
136
|
+
d = json.loads(line)
|
|
137
|
+
except json.JSONDecodeError:
|
|
138
|
+
continue
|
|
139
|
+
ts = d.get("timestamp") or ""
|
|
140
|
+
if d.get("type") == "user" and not d.get("isSidechain"):
|
|
141
|
+
origin_human = isinstance(d.get("origin"), dict) and d["origin"].get("kind") == "human"
|
|
142
|
+
if origin_human or d.get("turnOrigin") == "human":
|
|
143
|
+
saw_human = True
|
|
144
|
+
elif d.get("entrypoint") == "sdk-cli" or d.get("turnOrigin") == "sdk":
|
|
145
|
+
saw_script = True
|
|
146
|
+
# peer messages, task notifications, scripted prompts: not typed by a person
|
|
147
|
+
if d.get("turnOrigin") not in (None, "human") and not origin_human and not include_sdk:
|
|
148
|
+
continue
|
|
149
|
+
if d.get("type") == "assistant" and not d.get("isSidechain"):
|
|
150
|
+
for c in (d.get("message") or {}).get("content") or []:
|
|
151
|
+
if not isinstance(c, dict):
|
|
152
|
+
continue
|
|
153
|
+
if c.get("type") == "tool_use":
|
|
154
|
+
n = c.get("name", "?")
|
|
155
|
+
tools[n] = tools.get(n, 0) + 1
|
|
156
|
+
elif c.get("type") == "text" and c.get("text", "").strip():
|
|
157
|
+
last_text = c["text"]
|
|
158
|
+
continue
|
|
159
|
+
got = _claude_human_text(d)
|
|
160
|
+
if not got:
|
|
161
|
+
continue
|
|
162
|
+
text, images = got
|
|
163
|
+
slash = ""
|
|
164
|
+
m = _SLASH_RE.search(text)
|
|
165
|
+
if m:
|
|
166
|
+
slash = m.group(1)
|
|
167
|
+
a = _ARGS_RE.search(text)
|
|
168
|
+
text = (slash + " " + (a.group(1).strip() if a else "")).strip()
|
|
169
|
+
elif text.lstrip().startswith(_NOISE_PREFIXES):
|
|
170
|
+
continue
|
|
171
|
+
text = text.strip()
|
|
172
|
+
if not text and not images:
|
|
173
|
+
continue
|
|
174
|
+
if turns and turns[-1].text == text: # resumed sessions repeat the prompt
|
|
175
|
+
continue
|
|
176
|
+
flush()
|
|
177
|
+
turns.append(Turn(index=len(turns) + 1, text=text, ts=ts, images=images, slash=slash))
|
|
178
|
+
first_ts = first_ts or ts
|
|
179
|
+
last_ts = ts or last_ts
|
|
180
|
+
flush()
|
|
181
|
+
scripted = saw_script and not saw_human # `claude -p` or an app driving the SDK
|
|
182
|
+
if (scripted and not include_sdk) or not turns:
|
|
183
|
+
return None
|
|
184
|
+
return Session("claude", sid, project, path, first_ts, last_ts or first_ts, turns)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def iter_claude(include_sdk: bool = False) -> Iterator[Session]:
|
|
188
|
+
root = claude_root()
|
|
189
|
+
for path in glob.glob(str(root / "*" / "*.jsonl")):
|
|
190
|
+
if _is_temp_project(Path(path).parent.name):
|
|
191
|
+
continue
|
|
192
|
+
s = parse_claude_file(path, include_sdk)
|
|
193
|
+
if s:
|
|
194
|
+
yield s
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ---------------------------------------------------------------- Codex CLI
|
|
198
|
+
|
|
199
|
+
def codex_root() -> Path:
|
|
200
|
+
return Path(os.environ.get("CODEX_HOME", Path.home() / ".codex"))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def parse_codex_file(path: str, include_sdk: bool = False) -> Session | None:
|
|
204
|
+
turns: list[Turn] = []
|
|
205
|
+
tools: dict[str, int] = {}
|
|
206
|
+
last_text = ""
|
|
207
|
+
sid, project, first_ts, last_ts = Path(path).stem, "", "", ""
|
|
208
|
+
from_items: list[tuple[str, str, int]] = []
|
|
209
|
+
from_events: list[tuple[str, str, int]] = []
|
|
210
|
+
order: list[tuple] = [] # interleaving of user turns and agent activity
|
|
211
|
+
|
|
212
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
213
|
+
for line in fh:
|
|
214
|
+
try:
|
|
215
|
+
d = json.loads(line)
|
|
216
|
+
except json.JSONDecodeError:
|
|
217
|
+
continue
|
|
218
|
+
ts = d.get("timestamp") or ""
|
|
219
|
+
p = d.get("payload") or {}
|
|
220
|
+
t, pt = d.get("type"), p.get("type")
|
|
221
|
+
if t == "session_meta":
|
|
222
|
+
sid = p.get("id") or sid
|
|
223
|
+
cwd = p.get("cwd") or ""
|
|
224
|
+
project = re.split(r"[\\/]", cwd.rstrip("\\/"))[-1] if cwd else "" # cwd may be a Windows path
|
|
225
|
+
# `codex exec`, or another program driving Codex through the app-server
|
|
226
|
+
# (default originator "codex_cli_rs"); the TUI/IDE set their own originator
|
|
227
|
+
scripted = p.get("source") == "exec" or p.get("originator") == "codex_cli_rs" \
|
|
228
|
+
or p.get("thread_source") not in (None, "user")
|
|
229
|
+
if scripted and not include_sdk:
|
|
230
|
+
return None
|
|
231
|
+
if _is_temp_project(cwd.replace("\\", "-").replace("/", "-")):
|
|
232
|
+
return None
|
|
233
|
+
elif t == "response_item" and pt == "message" and p.get("role") == "user":
|
|
234
|
+
texts, images = [], 0
|
|
235
|
+
for c in p.get("content") or []:
|
|
236
|
+
if c.get("type") in ("input_text", "text"):
|
|
237
|
+
texts.append(c.get("text", ""))
|
|
238
|
+
elif c.get("type") in ("input_image", "image"):
|
|
239
|
+
images += 1
|
|
240
|
+
text = "\n".join(texts).strip()
|
|
241
|
+
if text.startswith(_NOISE_PREFIXES):
|
|
242
|
+
continue
|
|
243
|
+
order.append(("user", text, ts, images))
|
|
244
|
+
elif t == "event_msg" and pt == "user_message":
|
|
245
|
+
from_events.append((p.get("message", "").strip(), ts, len(p.get("images") or [])))
|
|
246
|
+
elif t == "response_item" and pt in ("function_call", "custom_tool_call", "local_shell_call"):
|
|
247
|
+
order.append(("tool", p.get("name") or pt))
|
|
248
|
+
elif t == "response_item" and pt == "message" and p.get("role") == "assistant":
|
|
249
|
+
txt = " ".join(c.get("text", "") for c in p.get("content") or [] if isinstance(c, dict))
|
|
250
|
+
if txt.strip():
|
|
251
|
+
order.append(("say", txt))
|
|
252
|
+
|
|
253
|
+
has_items = any(o[0] == "user" for o in order)
|
|
254
|
+
if not has_items: # older rollouts only log user prompts as events
|
|
255
|
+
order = [("user", txt, ts, im) for txt, ts, im in from_events] + [o for o in order if o[0] != "user"]
|
|
256
|
+
|
|
257
|
+
def flush():
|
|
258
|
+
nonlocal tools, last_text
|
|
259
|
+
if turns:
|
|
260
|
+
turns[-1].agent_after = _summarize_agent(tools, last_text)
|
|
261
|
+
tools, last_text = {}, ""
|
|
262
|
+
|
|
263
|
+
for o in order:
|
|
264
|
+
if o[0] == "user":
|
|
265
|
+
_, text, ts, images = o
|
|
266
|
+
if not text and not images:
|
|
267
|
+
continue
|
|
268
|
+
flush()
|
|
269
|
+
slash = text.split()[0] if text.startswith("/") and text.split() else ""
|
|
270
|
+
turns.append(Turn(index=len(turns) + 1, text=text, ts=ts, images=images, slash=slash))
|
|
271
|
+
first_ts = first_ts or ts
|
|
272
|
+
last_ts = ts or last_ts
|
|
273
|
+
elif o[0] == "tool":
|
|
274
|
+
tools[o[1]] = tools.get(o[1], 0) + 1
|
|
275
|
+
elif o[0] == "say":
|
|
276
|
+
last_text = o[1]
|
|
277
|
+
flush()
|
|
278
|
+
if not turns:
|
|
279
|
+
return None
|
|
280
|
+
return Session("codex", sid, project, path, first_ts, last_ts or first_ts, turns)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def iter_codex(include_sdk: bool = False) -> Iterator[Session]:
|
|
284
|
+
root = codex_root()
|
|
285
|
+
paths = glob.glob(str(root / "sessions" / "**" / "*.jsonl"), recursive=True)
|
|
286
|
+
paths += glob.glob(str(root / "archived_sessions" / "*.jsonl"))
|
|
287
|
+
seen = set()
|
|
288
|
+
for path in paths:
|
|
289
|
+
s = parse_codex_file(path, include_sdk)
|
|
290
|
+
if s and s.id not in seen:
|
|
291
|
+
seen.add(s.id)
|
|
292
|
+
yield s
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ---------------------------------------------------------------- selection
|
|
296
|
+
|
|
297
|
+
def collect(sources: Iterable[str], since: datetime | None = None,
|
|
298
|
+
include_sdk: bool = False, min_chars: int = 15) -> list[Session]:
|
|
299
|
+
out: list[Session] = []
|
|
300
|
+
for src in sources:
|
|
301
|
+
it = iter_claude(include_sdk) if src == "claude" else iter_codex(include_sdk)
|
|
302
|
+
for s in it:
|
|
303
|
+
end = _parse_ts(s.ended)
|
|
304
|
+
if since and (end is None or end < since):
|
|
305
|
+
continue
|
|
306
|
+
typed = sum(len(t.text) for t in s.turns if not t.slash)
|
|
307
|
+
if typed < min_chars:
|
|
308
|
+
continue
|
|
309
|
+
out.append(s)
|
|
310
|
+
if not include_sdk:
|
|
311
|
+
# the same opening prompt in 3+ sessions is a template a script sent, not a person typing
|
|
312
|
+
firsts: dict[str, int] = {}
|
|
313
|
+
for s in out:
|
|
314
|
+
k = s.turns[0].text[:200]
|
|
315
|
+
firsts[k] = firsts.get(k, 0) + 1
|
|
316
|
+
out = [s for s in out if firsts[s.turns[0].text[:200]] < 3]
|
|
317
|
+
out.sort(key=lambda s: s.ended, reverse=True)
|
|
318
|
+
return out
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def pick_sample(sessions: list[Session], limit: int) -> list[Session]:
|
|
322
|
+
"""Spread the sample over the whole window, preferring sessions with more turns."""
|
|
323
|
+
if len(sessions) <= limit:
|
|
324
|
+
return sessions
|
|
325
|
+
# interleaved buckets each span the window; take the richest session from each
|
|
326
|
+
buckets = [sessions[i::limit] for i in range(limit)]
|
|
327
|
+
picked = []
|
|
328
|
+
for b in buckets:
|
|
329
|
+
b = sorted(b, key=lambda s: -min(len(s.turns), 30))
|
|
330
|
+
picked.append(b[0])
|
|
331
|
+
picked.sort(key=lambda s: s.ended, reverse=True)
|
|
332
|
+
return picked
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def now_utc() -> datetime:
|
|
336
|
+
return datetime.now(timezone.utc)
|
agentcoach/heuristics.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Offline scorer: keyword/shape signals only, no model calls.
|
|
2
|
+
|
|
3
|
+
It is deliberately rough. It exists so a first report works with zero setup
|
|
4
|
+
(and in a classroom without spending anyone's quota); the LLM judge is the
|
|
5
|
+
real coach. Each rule returns 0/1/2 or None (not applicable).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from .collect import Session
|
|
12
|
+
|
|
13
|
+
R = lambda p: re.compile(p, re.I) # noqa: E731
|
|
14
|
+
|
|
15
|
+
PATH = R(r"(@[\w./\\-]+|[\w\-]+[/\\][\w./\\-]+|\b[\w\-]+\.(py|ts|tsx|js|jsx|go|rs|java|kt|cs|cpp|c|h|rb|php|md|json|ya?ml|toml|html|css|sql|sh|ps1)\b|:\d+\b)")
|
|
16
|
+
SYMPTOM = R(r"(에러|오류|실패|안 ?돼|안 ?됨|깨져|버그|증상|재현|error|fail|exception|traceback|crash|bug|doesn'?t work|broken)")
|
|
17
|
+
VERIFY = R(r"(테스트|test|pytest|jest|통과|pass|빌드|build|lint|린트|타입체크|typecheck|검증|확인해|재현|돌려 ?봐|실행해 ?봐|스크린샷|screenshot|until .* pass|될 때까지)")
|
|
18
|
+
CONSTRAINT = R(r"(하지 ?마|말고|없이|금지|건드리지|유지|그대로 두|만 ?(수정|고쳐)|패턴(을|대로)? 따라|don'?t|do not|without|no new|keep|only|avoid|must not|instead of)")
|
|
19
|
+
SCOPE = R(r"(최소(한)?|간단히|간단하게|빠르게 대충|넉넉|꼼꼼|빠짐없이|엣지 ?케이스|edge case|빈 ?상태|에러 ?상태|be ambitious|minimal|just the|fully|end[- ]to[- ]end|전부|모두 다)")
|
|
20
|
+
PLAN = R(r"(계획|플랜|plan|설계|먼저 .*(말해|알려|정리)|질문(해|하고)|인터뷰|interview|spec|스펙|단계별|마일스톤|milestone)")
|
|
21
|
+
EVIDENCE = R(r"(보여줘|보여 ?달|출력(을|도)? ?(보여|붙여)|로그(를|도)? ?(보여|붙여)|증거|스크린샷|캡처|evidence|show me|paste the output|결과(를)? (보여|붙여))")
|
|
22
|
+
CORRECTION = R(r"^(아니|아냐|그게 아니|다시|또|여전히|still|no[,. ]|wrong|not what|그대로잖|안 ?바뀌|똑같)")
|
|
23
|
+
ERRDUMP = R(r"(Traceback|Error:|Exception|at .+:\d+|\bline \d+|```)")
|
|
24
|
+
STOPS = R(r"(멈춰|멈추고|물어보고|물어봐|확인받|승인(하면|받|한 ?뒤)|제출 ?전|하기 ?전에|직전에|묻지 ?말고|알아서 (진행|끝까지)|ask me|stop and|before (you )?(submit|delete|push|send)|keep going|don'?t ask)")
|
|
25
|
+
DURABLE = R(r"(앞으로|항상|매번|무조건|절대|from now on|always|every time|never)")
|
|
26
|
+
MISS_SIG = [
|
|
27
|
+
("repeat", R(r"(라니까|랬잖|했잖아|말했잖|말했는데|아까 (말|얘기)|분명히 (말|얘기))")),
|
|
28
|
+
("redo", R(r"(되돌려|원래대로|원상복구|롤백|revert|undo|다시 해)")),
|
|
29
|
+
("mishap", R(r"(왜 (이렇게|그렇게|이걸|그걸|.{0,12}했)|하지 말(라|랬)|손해|날렸|망가|잘못 (눌|했|만))")),
|
|
30
|
+
("correction", R(r"^\s*(아니|아냐|그게 아니|그거 말고|그런 뜻|틀렸|안 ?바뀌|여전히|똑같|still|no[,. ]|wrong|not what)")),
|
|
31
|
+
]
|
|
32
|
+
RECORD = R(r"(CLAUDE\.md|AGENTS\.md|메모리|기억해|규칙(으로|에)|스킬|skill|훅|hook|memory)")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _task_prompts(s: Session) -> list:
|
|
36
|
+
"""Turns that start or substantially reframe work (long enough to carry a spec)."""
|
|
37
|
+
out = [t for t in s.turns if not t.slash and (t.index == 1 or len(t.text) >= 60)]
|
|
38
|
+
return out or [t for t in s.turns if not t.slash][:1]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def score_session(s: Session, rubric: dict) -> dict:
|
|
42
|
+
tasks = _task_prompts(s)
|
|
43
|
+
typed = [t for t in s.turns if not t.slash]
|
|
44
|
+
all_text = "\n".join(t.text for t in typed)
|
|
45
|
+
task_text = "\n".join(t.text for t in tasks)
|
|
46
|
+
big = len(typed) >= 6 or len(task_text) >= 400
|
|
47
|
+
first = tasks[0] if tasks else None
|
|
48
|
+
|
|
49
|
+
def first_turn(rx):
|
|
50
|
+
for t in typed:
|
|
51
|
+
if rx.search(t.text):
|
|
52
|
+
return t.index
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
def graded(rx, text_first, text_any):
|
|
56
|
+
if rx.search(text_first):
|
|
57
|
+
return 2
|
|
58
|
+
if rx.search(text_any):
|
|
59
|
+
return 1
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
dims = {}
|
|
63
|
+
ft = first.text if first else ""
|
|
64
|
+
actionable = bool(re.search(r"(해줘|해 ?주세요|만들어|고쳐|수정|추가|바꿔|구현|fix|add|make|build|implement|change|refactor|write)", all_text, re.I))
|
|
65
|
+
|
|
66
|
+
ctx = 2 if (PATH.search(ft) and (SYMPTOM.search(ft) or len(ft) > 120)) else (1 if PATH.search(task_text) or SYMPTOM.search(ft) or len(ft) > 200 else 0)
|
|
67
|
+
dims["specific_context"] = ctx if actionable else None
|
|
68
|
+
dims["done_criteria"] = graded(VERIFY, ft, all_text) if actionable else None
|
|
69
|
+
dims["constraints"] = graded(CONSTRAINT, ft, task_text) if actionable else None
|
|
70
|
+
has_material = any(t.images for t in typed) or "@" in all_text or ERRDUMP.search(all_text)
|
|
71
|
+
mentions_err = SYMPTOM.search(all_text)
|
|
72
|
+
dims["hand_over"] = (2 if has_material else 0) if (mentions_err or any(t.images for t in typed)) else None
|
|
73
|
+
dims["scope_signal"] = (graded(SCOPE, ft, task_text)) if actionable and big else None
|
|
74
|
+
planned = any(t.slash in ("/plan", "/goal") for t in s.turns) or PLAN.search(ft)
|
|
75
|
+
dims["plan_first"] = (2 if planned else (1 if PLAN.search(all_text) else 0)) if big else None
|
|
76
|
+
claimed_done = any(re.search(r"(완료|끝났|done|fixed|passed|통과)", t.agent_after, re.I) for t in s.turns)
|
|
77
|
+
dims["evidence"] = (2 if EVIDENCE.search(all_text) else 0) if claimed_done and actionable else None
|
|
78
|
+
|
|
79
|
+
# longest run of consecutive correction-style prompts without a reset
|
|
80
|
+
run = best = 0
|
|
81
|
+
for t in s.turns:
|
|
82
|
+
if t.slash in ("/clear", "/rewind", "/compact", "/new"):
|
|
83
|
+
run = 0
|
|
84
|
+
elif CORRECTION.search(t.text.strip()):
|
|
85
|
+
run += 1
|
|
86
|
+
best = max(best, run)
|
|
87
|
+
else:
|
|
88
|
+
run = 0
|
|
89
|
+
corrections = sum(1 for t in typed if CORRECTION.search(t.text.strip()))
|
|
90
|
+
dims["steering"] = None if corrections == 0 else (2 if best <= 1 else 1 if best == 2 else 0)
|
|
91
|
+
|
|
92
|
+
risky = re.search(r"(제출|지원|결제|구매|삭제|지워|전송|보내|게시|업로드|푸시|push|submit|delete|send|publish|pay)", all_text, re.I)
|
|
93
|
+
dims["stop_conditions"] = graded(STOPS, ft, task_text) if actionable and (big or risky) else None
|
|
94
|
+
|
|
95
|
+
durable_turns = [t for t in typed if DURABLE.search(t.text)]
|
|
96
|
+
if not durable_turns:
|
|
97
|
+
dims["reusable_guidance"] = None
|
|
98
|
+
elif any(RECORD.search(t.text) for t in typed):
|
|
99
|
+
dims["reusable_guidance"] = 2
|
|
100
|
+
else:
|
|
101
|
+
dims["reusable_guidance"] = 0 if len(durable_turns) >= 3 else 1
|
|
102
|
+
|
|
103
|
+
resets = sum(1 for t in s.turns if t.slash in ("/clear", "/compact", "/new", "/fork"))
|
|
104
|
+
if len(typed) < 15:
|
|
105
|
+
dims["focused_session"] = None
|
|
106
|
+
elif len(typed) >= 60 and resets == 0:
|
|
107
|
+
dims["focused_session"] = 0
|
|
108
|
+
else:
|
|
109
|
+
dims["focused_session"] = 1 if (len(typed) >= 30 and resets == 0) else 2
|
|
110
|
+
|
|
111
|
+
rx_for = {"specific_context": PATH, "done_criteria": VERIFY, "constraints": CONSTRAINT,
|
|
112
|
+
"scope_signal": SCOPE, "plan_first": PLAN, "evidence": EVIDENCE, "steering": CORRECTION,
|
|
113
|
+
"hand_over": ERRDUMP, "stop_conditions": STOPS, "reusable_guidance": DURABLE}
|
|
114
|
+
out = {}
|
|
115
|
+
for d in rubric["dimensions"]:
|
|
116
|
+
did = d["id"]
|
|
117
|
+
sc = dims.get(did)
|
|
118
|
+
out[did] = {"score": sc, "evidence_turn": first_turn(rx_for[did]) if did in rx_for else None,
|
|
119
|
+
"note": ""}
|
|
120
|
+
# misses: what happened next tells us the turn missed; offline we can't tell whose fault it was
|
|
121
|
+
misses = []
|
|
122
|
+
for i, t in enumerate(s.turns[:-1]):
|
|
123
|
+
nxt = s.turns[i + 1]
|
|
124
|
+
if t.slash or nxt.slash:
|
|
125
|
+
continue
|
|
126
|
+
for kind, rx in MISS_SIG:
|
|
127
|
+
if rx.search(nxt.text.strip()[:160]):
|
|
128
|
+
misses.append({"turn": t.index, "signal_turn": nxt.index, "signal": kind, "cause": "unclassified",
|
|
129
|
+
"context_was_enough": None, "original": t.text[:160], "what_went_wrong": nxt.text[:160],
|
|
130
|
+
"better": "", "why": ""})
|
|
131
|
+
break
|
|
132
|
+
return {"task_summary": (ft[:80] + "…") if len(ft) > 80 else ft, "dimensions": out, "coaching": [],
|
|
133
|
+
"misses": misses[:8],
|
|
134
|
+
"judge": "heuristic"}
|
agentcoach/i18n.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Language selection and CLI messages (Korean and English)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import locale
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
SUPPORTED = ("en", "ko")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def detect_lang() -> str:
|
|
11
|
+
"""AGENT_COACH_LANG wins; otherwise follow the OS locale; default English."""
|
|
12
|
+
env = os.environ.get("AGENT_COACH_LANG", "").lower()
|
|
13
|
+
if env in SUPPORTED:
|
|
14
|
+
return env
|
|
15
|
+
candidates = [os.environ.get(k, "") for k in ("LC_ALL", "LC_MESSAGES", "LANG")]
|
|
16
|
+
try:
|
|
17
|
+
candidates.append(locale.getlocale()[0] or "")
|
|
18
|
+
except ValueError:
|
|
19
|
+
pass
|
|
20
|
+
for c in candidates:
|
|
21
|
+
c = c.lower()
|
|
22
|
+
if c.startswith("ko") or "korean" in c:
|
|
23
|
+
return "ko"
|
|
24
|
+
return "en"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
M = {
|
|
28
|
+
"bad_source": {"en": "Unknown source: {bad} (choose from claude, codex)",
|
|
29
|
+
"ko": "알 수 없는 source: {bad} (claude, codex 중에서)"},
|
|
30
|
+
"all_time": {"en": "all time", "ko": "전체"},
|
|
31
|
+
"scan_head": {"en": "Window: since {since} · {n} conversations",
|
|
32
|
+
"ko": "기간: {since} 이후 · 대화 {n}개"},
|
|
33
|
+
"scan_tail": {"en": "{turns} prompts you typed. Run `agent-coach run` to grade them (up to {max} conversations by default).",
|
|
34
|
+
"ko": "사람이 쓴 지시 {turns}개. `agent-coach run` 으로 채점합니다 (기본 최대 {max}개 대화)."},
|
|
35
|
+
"nothing": {"en": "No conversations to grade. Widen the window with --days 30 or --all.",
|
|
36
|
+
"ko": "채점할 대화가 없습니다. --days 30 이나 --all 로 기간을 넓혀 보세요."},
|
|
37
|
+
"grading": {"en": "Grading {k} of {n} conversations · judge: {judge}",
|
|
38
|
+
"ko": "대화 {n}개 중 {k}개를 채점합니다 · 채점기: {judge}"},
|
|
39
|
+
"done": {"en": "done", "ko": "완료"},
|
|
40
|
+
"fallback": {"en": "failed, using keyword scoring", "ko": "실패, 키워드 채점으로 대체"},
|
|
41
|
+
"all_failed": {"en": "Every LLM grading call failed; saving keyword scores only. Use --verbose to see why.",
|
|
42
|
+
"ko": "LLM 채점이 모두 실패해 키워드 채점 결과만 저장합니다. --verbose 로 원인을 확인하세요."},
|
|
43
|
+
"grouping": {"en": "Grouping prompt gaps that keep coming back…", "ko": "반복되는 지시 빈틈을 묶는 중…"},
|
|
44
|
+
"result": {"en": "Overall {overall} · {gaps} of {total} missed responses were gaps in your prompt\nRun saved {path}\nReport {out}",
|
|
45
|
+
"ko": "종합 {overall}점 · 놓친 응답 {total}곳 중 내 지시 빈틈 {gaps}곳\n기록 {path}\n리포트 {out}"},
|
|
46
|
+
"no_runs": {"en": "No runs yet. Run `agent-coach run` first.",
|
|
47
|
+
"ko": "아직 채점 기록이 없습니다. 먼저 `agent-coach run` 을 실행하세요."},
|
|
48
|
+
"no_history": {"en": "No runs yet", "ko": "기록 없음"},
|
|
49
|
+
"hist_head": {"en": ("Date", "Overall", "Convos", "Judge"), "ko": ("날짜", "종합", "대화", "채점기")},
|
|
50
|
+
"desc": {"en": "Coach how you direct Claude Code and Codex, graded against the official guides using your own local transcripts.",
|
|
51
|
+
"ko": "Claude Code·Codex 대화 기록으로 공식 가이드 기준 지시 습관을 코칭합니다."},
|
|
52
|
+
"h_rubric": {"en": "Rubric JSON path (default: built-in, grounded in the official guides)",
|
|
53
|
+
"ko": "채점표 JSON 경로 (기본: 공식 가이드 기반)"},
|
|
54
|
+
"h_source": {"en": "Which tools to read: claude,codex (default both)", "ko": "claude,codex 중 선택 (기본 둘 다)"},
|
|
55
|
+
"h_days": {"en": "Last N days (default: since the previous run; 14 days on the first run)",
|
|
56
|
+
"ko": "최근 N일 (기본: 지난 채점 이후, 첫 회는 14일)"},
|
|
57
|
+
"h_all": {"en": "No time window", "ko": "기간 제한 없이 전체"},
|
|
58
|
+
"h_sdk": {"en": "Also include scripted `claude -p` / `codex exec` conversations",
|
|
59
|
+
"ko": "스크립트로 돌린 claude -p / codex exec 대화도 포함"},
|
|
60
|
+
"h_max": {"en": "Max conversations to grade (default 20)", "ko": "채점할 최대 대화 수 (기본 20)"},
|
|
61
|
+
"h_lang": {"en": "Language for the report and coaching (default: from your OS locale)",
|
|
62
|
+
"ko": "리포트와 코칭 언어 (기본: OS 언어 설정)"},
|
|
63
|
+
"h_scan": {"en": "Preview which conversations would be graded (no model calls)",
|
|
64
|
+
"ko": "채점 대상 대화를 미리 봅니다 (모델 호출 없음)"},
|
|
65
|
+
"h_run": {"en": "Grade, save the run and open the report", "ko": "채점하고 기록을 남긴 뒤 리포트를 엽니다"},
|
|
66
|
+
"h_judge": {"en": "Judge (default auto: claude → codex → keywords)", "ko": "채점기 (기본 auto: claude → codex → 키워드)"},
|
|
67
|
+
"h_model": {"en": "Model for the judge (e.g. sonnet, haiku, gpt-5.5)", "ko": "채점에 쓸 모델 (예: sonnet, haiku, gpt-5.5)"},
|
|
68
|
+
"h_workers": {"en": "Parallel grading calls", "ko": "동시 채점 수"},
|
|
69
|
+
"h_noopen": {"en": "Don't open the report in a browser", "ko": "리포트를 브라우저로 열지 않음"},
|
|
70
|
+
"h_report": {"en": "Rebuild the report from saved runs", "ko": "저장된 기록으로 리포트만 다시 만듭니다"},
|
|
71
|
+
"h_history": {"en": "Score table per run", "ko": "회차별 점수 표"},
|
|
72
|
+
"h_summary": {"en": "Print the latest run as text: weakest habits, rewrites, missed responses, rules to add",
|
|
73
|
+
"ko": "최근 회차를 글로 출력: 약한 습관, 고쳐 쓰기, 놓친 응답, 추가할 규칙"},
|
|
74
|
+
"sum_head": {"en": "agent-coach · {date} · overall {overall}/100 · {n} conversations",
|
|
75
|
+
"ko": "agent-coach · {date} · 종합 {overall}/100 · 대화 {n}개"},
|
|
76
|
+
"sum_weak": {"en": "Weakest habits", "ko": "약한 습관"},
|
|
77
|
+
"sum_rewrites": {"en": "Rewrites of your prompts", "ko": "내 지시 고쳐 쓰기"},
|
|
78
|
+
"sum_gaps": {"en": "Missed responses caused by a gap in your prompt", "ko": "내 지시의 빈틈 때문에 놓친 응답"},
|
|
79
|
+
"sum_rules": {"en": "Rules worth adding", "ko": "추가할 만한 규칙"},
|
|
80
|
+
"sum_wrote": {"en": "wrote", "ko": "쓴 지시"},
|
|
81
|
+
"sum_better": {"en": "better", "ko": "이렇게"},
|
|
82
|
+
"sum_why": {"en": "why", "ko": "이유"},
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def t(key: str, lang: str, **kw):
|
|
87
|
+
msg = M[key].get(lang) or M[key]["en"]
|
|
88
|
+
return msg.format(**kw) if isinstance(msg, str) and kw else msg
|