codeer-cli 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/PKG-INFO +1 -1
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/pyproject.toml +1 -1
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/cli.py +4 -1
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/eval_cmd.py +146 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/eval_.py +4 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/uv.lock +1 -1
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/.gitignore +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/API_REFERENCE.md +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/README.md +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/__init__.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/_validate.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/agents.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/chats.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/client.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/__init__.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/_util.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/agent.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/check.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/history.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/kb.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/commands/profile.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/constants.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/histories.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/kb.py +0 -0
- {codeer_cli-0.1.3 → codeer_cli-0.1.4}/src/codeer_cli/parse.py +0 -0
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
codeer check
|
|
4
4
|
codeer agent list|get|apply|diff|versions
|
|
5
5
|
codeer kb list|files|upload|faq-list|faq-get|faq-create|faq-update|faq-delete
|
|
6
|
-
codeer eval list|evaluators|evaluator-create|evaluator-update|run|export|reconcile|cases-apply|rubrics|rubrics-apply
|
|
6
|
+
codeer eval list|case-update|case-delete|evaluators|evaluator-create|evaluator-update|run|export|reconcile|cases-apply|rubrics|rubrics-apply
|
|
7
7
|
codeer history list|get|conversations|negative-feedback
|
|
8
8
|
"""
|
|
9
9
|
|
|
@@ -29,12 +29,15 @@ Safe workflow for coding agents:
|
|
|
29
29
|
codeer agent get <agent-id> --full
|
|
30
30
|
codeer kb list
|
|
31
31
|
codeer eval list --agent <agent-id>
|
|
32
|
+
codeer eval case-update --case <case-id> --input "..." --dry-run
|
|
32
33
|
codeer eval evaluators
|
|
33
34
|
codeer agent diff --agent <agent-id> --from-version <n> --to-version <n>
|
|
34
35
|
codeer eval reconcile --agent <agent-id> --manifest .codeer/eval_cases.json
|
|
35
36
|
|
|
36
37
|
Preview mutations before applying:
|
|
37
38
|
codeer agent apply --payload agent.json --dry-run
|
|
39
|
+
codeer eval case-update --case <case-id> --input "..." --dry-run
|
|
40
|
+
codeer eval case-delete --case <case-id> --dry-run
|
|
38
41
|
codeer eval cases-apply --agent <agent-id> --cases eval_cases.json --dry-run
|
|
39
42
|
codeer eval rubrics-apply --rubrics rubrics.json --dry-run
|
|
40
43
|
codeer kb upload --dir kb --name "Product KB" --dry-run
|
|
@@ -46,6 +46,39 @@ def register(subparsers):
|
|
|
46
46
|
help="Write stripped full case payloads to this file; stdout stays compact unless --full.")
|
|
47
47
|
p.set_defaults(func=run_list)
|
|
48
48
|
|
|
49
|
+
# codeer eval case-update
|
|
50
|
+
p = sub.add_parser("case-update", help="Update one eval case by UUID; run --dry-run first")
|
|
51
|
+
p.add_argument("--case", required=True, dest="case_id", help="Eval case UUID")
|
|
52
|
+
g = p.add_mutually_exclusive_group()
|
|
53
|
+
g.add_argument("--input", help="New eval case input text")
|
|
54
|
+
g.add_argument("--input-file", help="Path to new eval case input text")
|
|
55
|
+
g = p.add_mutually_exclusive_group()
|
|
56
|
+
g.add_argument("--expected-output", help="New expected_output text")
|
|
57
|
+
g.add_argument("--expected-output-file", help="Path to new expected_output text")
|
|
58
|
+
g = p.add_mutually_exclusive_group()
|
|
59
|
+
g.add_argument("--rubric", help="New case-level rubric text")
|
|
60
|
+
g.add_argument("--rubric-file", help="Path to new case-level rubric text")
|
|
61
|
+
g = p.add_mutually_exclusive_group()
|
|
62
|
+
g.add_argument("--note", help="New case note text")
|
|
63
|
+
g.add_argument("--note-file", help="Path to new case note text")
|
|
64
|
+
g = p.add_mutually_exclusive_group()
|
|
65
|
+
g.add_argument("--meta-json", help="New case meta JSON object")
|
|
66
|
+
g.add_argument("--meta-file", help="Path to new case meta JSON object")
|
|
67
|
+
p.add_argument("--attachment-ids", default=None,
|
|
68
|
+
help="Comma-separated file UUIDs to set as the case attachments")
|
|
69
|
+
p.add_argument("--dry-run", action="store_true",
|
|
70
|
+
help="Validate inputs and print intended mutation without writing server state.")
|
|
71
|
+
p.add_argument("--out", default=None)
|
|
72
|
+
p.set_defaults(func=run_case_update)
|
|
73
|
+
|
|
74
|
+
# codeer eval case-delete
|
|
75
|
+
p = sub.add_parser("case-delete", help="Delete one eval case by UUID; run --dry-run first")
|
|
76
|
+
p.add_argument("--case", required=True, dest="case_id", help="Eval case UUID")
|
|
77
|
+
p.add_argument("--dry-run", action="store_true",
|
|
78
|
+
help="Print intended deletion without writing server state.")
|
|
79
|
+
p.add_argument("--out", default=None)
|
|
80
|
+
p.set_defaults(func=run_case_delete)
|
|
81
|
+
|
|
49
82
|
# codeer eval evaluators
|
|
50
83
|
p = sub.add_parser(
|
|
51
84
|
"evaluators",
|
|
@@ -206,6 +239,119 @@ def run_list(args, client) -> int:
|
|
|
206
239
|
return 0
|
|
207
240
|
|
|
208
241
|
|
|
242
|
+
# ---------------------------------------------------------------------------
|
|
243
|
+
# eval case-update / case-delete
|
|
244
|
+
# ---------------------------------------------------------------------------
|
|
245
|
+
|
|
246
|
+
def _read_text_arg(value: str | None, file_path: str | None) -> str | None:
|
|
247
|
+
if file_path is not None:
|
|
248
|
+
return Path(file_path).read_text()
|
|
249
|
+
return value
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _read_meta_arg(value: str | None, file_path: str | None) -> dict | None:
|
|
253
|
+
if file_path is not None:
|
|
254
|
+
raw = Path(file_path).read_text()
|
|
255
|
+
elif value is not None:
|
|
256
|
+
raw = value
|
|
257
|
+
else:
|
|
258
|
+
return None
|
|
259
|
+
|
|
260
|
+
meta = json.loads(raw)
|
|
261
|
+
if not isinstance(meta, dict):
|
|
262
|
+
raise ValueError("case meta must be a JSON object")
|
|
263
|
+
return meta
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def run_case_update(args, client) -> int:
|
|
267
|
+
try:
|
|
268
|
+
input_text = _read_text_arg(args.input, args.input_file)
|
|
269
|
+
expected_output = _read_text_arg(args.expected_output, args.expected_output_file)
|
|
270
|
+
rubric = _read_text_arg(args.rubric, args.rubric_file)
|
|
271
|
+
note = _read_text_arg(args.note, args.note_file)
|
|
272
|
+
meta = _read_meta_arg(args.meta_json, args.meta_file)
|
|
273
|
+
except (OSError, json.JSONDecodeError, ValueError) as e:
|
|
274
|
+
log(f"error: {e}")
|
|
275
|
+
return 2
|
|
276
|
+
attachment_ids = _ids(args.attachment_ids)
|
|
277
|
+
|
|
278
|
+
has_update = any(
|
|
279
|
+
value is not None
|
|
280
|
+
for value in (input_text, expected_output, rubric, note, meta, attachment_ids)
|
|
281
|
+
)
|
|
282
|
+
if not has_update:
|
|
283
|
+
log(
|
|
284
|
+
"error: provide at least one of --input, --input-file, --expected-output, "
|
|
285
|
+
"--expected-output-file, --rubric, --rubric-file, --note, --note-file, "
|
|
286
|
+
"--meta-json, --meta-file, --attachment-ids"
|
|
287
|
+
)
|
|
288
|
+
return 2
|
|
289
|
+
|
|
290
|
+
if args.dry_run:
|
|
291
|
+
current = strip_noisy_fields(eval_mod.get_case(client, args.case_id))
|
|
292
|
+
out = {
|
|
293
|
+
"dry_run": True,
|
|
294
|
+
"operation": "case_update",
|
|
295
|
+
"method": "PUT",
|
|
296
|
+
"path": f"/external/eval/cases/{args.case_id}",
|
|
297
|
+
"case_id": args.case_id,
|
|
298
|
+
"current": _case_summary(current, full=True),
|
|
299
|
+
"updates": {
|
|
300
|
+
"input_chars": len(input_text) if input_text is not None else None,
|
|
301
|
+
"expected_output_chars": (
|
|
302
|
+
len(expected_output) if expected_output is not None else None
|
|
303
|
+
),
|
|
304
|
+
"rubric_chars": len(rubric) if rubric is not None else None,
|
|
305
|
+
"note_chars": len(note) if note is not None else None,
|
|
306
|
+
"meta": meta,
|
|
307
|
+
"attachment_ids": attachment_ids,
|
|
308
|
+
},
|
|
309
|
+
"would_write_server_state": True,
|
|
310
|
+
"next_step": "Review this summary, then rerun without --dry-run after approval.",
|
|
311
|
+
}
|
|
312
|
+
print_json(out)
|
|
313
|
+
write_json(args.out, out)
|
|
314
|
+
return 0
|
|
315
|
+
|
|
316
|
+
updated = eval_mod.update_case(
|
|
317
|
+
client,
|
|
318
|
+
args.case_id,
|
|
319
|
+
input=input_text,
|
|
320
|
+
expected_output=expected_output,
|
|
321
|
+
rubric=rubric,
|
|
322
|
+
attachment_ids=attachment_ids,
|
|
323
|
+
meta=meta,
|
|
324
|
+
note=note,
|
|
325
|
+
)
|
|
326
|
+
out = strip_noisy_fields(updated)
|
|
327
|
+
print_json(out)
|
|
328
|
+
write_json(args.out, out)
|
|
329
|
+
return 0
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def run_case_delete(args, client) -> int:
|
|
333
|
+
if args.dry_run:
|
|
334
|
+
current = strip_noisy_fields(eval_mod.get_case(client, args.case_id))
|
|
335
|
+
out = {
|
|
336
|
+
"dry_run": True,
|
|
337
|
+
"operation": "case_delete",
|
|
338
|
+
"method": "DELETE",
|
|
339
|
+
"path": f"/external/eval/cases/{args.case_id}",
|
|
340
|
+
"case_id": args.case_id,
|
|
341
|
+
"current": _case_summary(current, full=True),
|
|
342
|
+
"would_write_server_state": True,
|
|
343
|
+
"next_step": "Review this summary, then rerun without --dry-run after approval.",
|
|
344
|
+
}
|
|
345
|
+
print_json(out)
|
|
346
|
+
write_json(args.out, out)
|
|
347
|
+
return 0
|
|
348
|
+
|
|
349
|
+
deleted = strip_noisy_fields(eval_mod.delete_case(client, args.case_id))
|
|
350
|
+
print_json(deleted)
|
|
351
|
+
write_json(args.out, deleted)
|
|
352
|
+
return 0
|
|
353
|
+
|
|
354
|
+
|
|
209
355
|
# ---------------------------------------------------------------------------
|
|
210
356
|
# eval evaluators
|
|
211
357
|
# ---------------------------------------------------------------------------
|
|
@@ -83,6 +83,10 @@ def update_case(
|
|
|
83
83
|
return client.put(f"/external/eval/cases/{case_id}", json=body)
|
|
84
84
|
|
|
85
85
|
|
|
86
|
+
def delete_case(client: CodeerClient, case_id: str) -> dict:
|
|
87
|
+
return client.delete(f"/external/eval/cases/{case_id}")
|
|
88
|
+
|
|
89
|
+
|
|
86
90
|
|
|
87
91
|
# --- evaluators -----------------------------------------------------------
|
|
88
92
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|