codeer-cli 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl

codeer_cli/commands/_util.py

@@ -62,5 +62,7 @@ def print_json(value: Any) -> None:
 def write_json(path: str | None, value: Any) -> None:
     if not path:
         return
-    Path(path).write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n")
+    out = Path(path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n")
     log(f"wrote full detail to {path}")

codeer_cli/commands/eval_cmd.py

@@ -653,6 +653,64 @@ def _group_case_ids_by_evaluator(pairs: list[dict[str, str]]) -> dict[str, list[
     return dict(grouped)
 
 
+def _pairs_from_rubric_batches(
+    client,
+    *,
+    case_ids: list[str],
+    evaluator_ids: list[str],
+) -> list[dict[str, str]]:
+    pairs: list[dict[str, str]] = []
+    for evaluator_id in evaluator_ids:
+        for row in eval_mod.get_rubrics_batch(client, case_ids=case_ids, evaluator_id=evaluator_id):
+            if row.get("rubric"):
+                case_id = row.get("case_id") or row.get("evaluation_case_id")
+                if case_id:
+                    pairs.append({"case_id": str(case_id), "evaluator_id": evaluator_id})
+    return pairs
+
+
+def _pair_key(pair: dict[str, str]) -> tuple[str, str]:
+    return pair["case_id"], pair["evaluator_id"]
+
+
+def _skipped_pairs_from_trigger_response(response: Any) -> list[dict[str, str]]:
+    if not isinstance(response, dict):
+        return []
+    payload = response.get("data") if isinstance(response.get("data"), dict) else response
+    skipped = payload.get("skipped_pairs") if isinstance(payload, dict) else None
+    if not isinstance(skipped, list):
+        return []
+
+    out: list[dict[str, str]] = []
+    for row in skipped:
+        if not isinstance(row, dict):
+            continue
+        case_id = row.get("case_id")
+        evaluator_id = row.get("evaluator_id")
+        if not case_id or not evaluator_id:
+            continue
+        out.append({
+            "case_id": str(case_id),
+            "evaluator_id": str(evaluator_id),
+            "reason": str(row.get("reason") or "skipped"),
+        })
+    return out
+
+
+def _remove_non_runnable_skipped_pairs(
+    pairs: list[dict[str, str]],
+    skipped_pairs: list[dict[str, str]],
+) -> list[dict[str, str]]:
+    non_runnable = {
+        _pair_key(pair)
+        for pair in skipped_pairs
+        if pair.get("reason") == "not_assigned"
+    }
+    if not non_runnable:
+        return pairs
+    return [pair for pair in pairs if _pair_key(pair) not in non_runnable]
+
+
 def run_run(args, client) -> int:
     workspace_id, _ = client.resolve_scope()
     if args.latest or not args.history:
@@ -682,18 +740,22 @@ def run_run(args, client) -> int:
     evaluator_ids = [args.evaluator] if args.evaluator else (_ids(args.evaluators) or [])
     requested_evaluator_ids = evaluator_ids or None
 
-    assignment_rows = eval_mod.get_case_evaluator_infos(client, case_ids=case_ids)
-    assigned_by_case = _assigned_evaluators_by_case(assignment_rows)
-    pairs, skipped_unassigned = _planned_eval_pairs(
-        case_ids=case_ids,
-        assigned_by_case=assigned_by_case,
-        requested_evaluator_ids=requested_evaluator_ids,
-    )
+    skipped_unassigned: list[dict[str, str]] = []
+    if requested_evaluator_ids:
+        pairs = [
+            {"case_id": case_id, "evaluator_id": evaluator_id}
+            for evaluator_id in requested_evaluator_ids
+            for case_id in case_ids
+        ]
+    else:
+        evaluator_ids = [e["id"] for e in eval_mod.list_evaluators(client, workspace_id)]
+        pairs = _pairs_from_rubric_batches(
+            client,
+            case_ids=case_ids,
+            evaluator_ids=evaluator_ids,
+        )
     if not pairs:
-        if skipped_unassigned:
-            log("error: none of the requested case/evaluator pairs are assigned")
-        else:
-            log("error: no assigned case/evaluator pairs to run")
+        log("error: no case/evaluator pairs to run")
         print_json({
             "agent_id": args.agent,
             "history_id": args.history,
@@ -710,14 +772,47 @@ def run_run(args, client) -> int:
     case_label_by_id = {c["id"]: truncate(c.get("input") or "", 60) for c in case_objs}
     evaluator_name_by_id = {e["id"]: e.get("name", e["id"]) for e in evaluators}
 
+    requested_pairs = list(pairs)
+    requested_pair_count = len(requested_pairs)
+    log(f"triggering: {requested_pair_count} case/evaluator pairs on history {args.history}")
+    trigger_response: list[dict[str, Any]] = []
+    skipped_pairs: list[dict[str, str]] = []
+    for ev_id, ev_case_ids in _group_case_ids_by_evaluator(pairs).items():
+        response = eval_mod.trigger(
+            client,
+            case_ids=ev_case_ids,
+            evaluator_ids=[ev_id],
+            agent_history_id=args.history,
+        )
+        response_skipped = _skipped_pairs_from_trigger_response(response)
+        skipped_pairs.extend(response_skipped)
+        trigger_response.append({
+            "evaluator_id": ev_id,
+            "case_ids": ev_case_ids,
+            "response": response,
+            "skipped_pairs": response_skipped,
+        })
+
+    pairs = _remove_non_runnable_skipped_pairs(pairs, skipped_pairs)
+    skipped_unassigned = [pair for pair in skipped_pairs if pair.get("reason") == "not_assigned"]
     if skipped_unassigned:
-        log(f"skipping {len(skipped_unassigned)} unassigned requested pairs")
-    log(f"triggering: {len(pairs)} assigned case/evaluator pairs on history {args.history}")
-    trigger_response = eval_mod.trigger_pairs(
-        client,
-        case_evaluator_pairs=pairs,
-        agent_history_id=args.history,
-    )
+        log(f"skipping {len(skipped_unassigned)} not-assigned pairs from polling")
+    if not pairs:
+        log("error: no runnable case/evaluator pairs after trigger response")
+        print_json({
+            "agent_id": args.agent,
+            "history_id": args.history,
+            "requested_case_count": len(case_ids),
+            "requested_evaluator_count": len(requested_evaluator_ids or evaluator_ids),
+            "requested_pair_count": requested_pair_count,
+            "triggered_pair_count": 0,
+            "skipped_pair_count": len(skipped_pairs),
+            "skipped_unassigned_count": len(skipped_unassigned),
+            "trigger_response": trigger_response,
+            "skipped_pairs": skipped_pairs,
+            "skipped_unassigned": skipped_unassigned,
+        })
+        return 2
 
     deadline = time.time() + args.poll_timeout
     results_by_eval: dict[str, list[dict]] = {}
@@ -825,22 +920,27 @@ def run_run(args, client) -> int:
         "history_id": args.history,
         "requested_case_count": len(case_ids),
         "requested_evaluator_count": len(requested_evaluator_ids or evaluator_ids),
+        "requested_pair_count": requested_pair_count,
         "triggered_pair_count": len(pairs),
         "scored_pair_count": len(scored_pair_keys),
+        "skipped_pair_count": len(skipped_pairs),
         "skipped_unassigned_count": len(skipped_unassigned),
         "all_perfect": all_perfect,
         "result_count": len(result_summaries),
         "non_perfect_count": len(non_perfect),
         "wrote_full_detail": bool(args.out),
         "trigger_response": trigger_response,
+        "skipped_pairs": skipped_pairs,
         "skipped_unassigned": skipped_unassigned,
         "results": result_summaries,
     }
     full_out = {
         "agent_id": args.agent,
         "history_id": args.history,
+        "requested_pairs": requested_pairs,
         "triggered_pairs": pairs,
         "trigger_response": trigger_response,
+        "skipped_pairs": skipped_pairs,
         "skipped_unassigned": skipped_unassigned,
         "all_perfect": all_perfect,
         "results": flat,
@@ -1466,35 +1566,44 @@ def run_rubrics(args, client) -> int:
         log("error: no cases for this agent")
         return 2
 
-    assignment_rows = eval_mod.get_case_evaluator_infos(client, case_ids=case_ids)
-    assigned_by_case = _assigned_evaluators_by_case(assignment_rows)
-
     if args.evaluators:
         evaluator_ids = _ids(args.evaluators) or []
         evaluators = [eval_mod.get_evaluator(client, eid) for eid in evaluator_ids]
-    elif args.all_pairs:
+    else:
         evaluators = eval_mod.list_evaluators(client, workspace_id)
         evaluator_ids = [e["id"] for e in evaluators]
+
+    rubrics = eval_mod.get_case_rubrics(
+        client, agent_id=args.agent, workspace_id=workspace_id,
+        evaluator_ids=evaluator_ids, case_ids=case_ids,
+    )
+    assigned_by_case = {
+        cid: {
+            ev_id: {"evaluator_id": ev_id, "rubric": rubric_text}
+            for ev_id, rubric_text in (rubrics.get(cid) or {}).items()
+            if rubric_text
+        }
+        for cid in case_ids
+    }
+
+    if args.evaluators or args.all_pairs:
+        pass
     else:
         evaluator_ids = _dedupe_preserve_order([
             evaluator_id
             for case_id in case_ids
             for evaluator_id in assigned_by_case.get(case_id, {})
         ])
-        evaluators = [eval_mod.get_evaluator(client, eid) for eid in evaluator_ids]
+        evaluator_id_set = set(evaluator_ids)
+        evaluators = [e for e in evaluators if e["id"] in evaluator_id_set]
     evaluator_name = {e["id"]: e.get("name", e["id"]) for e in evaluators}
     if not evaluator_ids:
-        log("error: no assigned evaluators for these cases")
+        log("error: no evaluators with configured rubrics for these cases")
         return 2
 
-    mode = "all requested pairs" if args.evaluators or args.all_pairs else "assigned pairs"
+    mode = "all requested pairs" if args.evaluators or args.all_pairs else "pairs with configured rubrics"
     log(f"reading {mode}: {len(case_ids)} cases, {len(evaluator_ids)} evaluators...")
 
-    rubrics = eval_mod.get_case_rubrics(
-        client, agent_id=args.agent, workspace_id=workspace_id,
-        evaluator_ids=evaluator_ids, case_ids=case_ids,
-    )
-
     if args.full:
         for cid in case_ids:
             log("=" * 80)

codeer_cli/eval_.py

@@ -54,8 +54,26 @@ def create_case(
     return client.post("/external/eval/cases", json=body)
 
 
+def _unwrap_list_response(value: Any, *keys: str) -> list[dict]:
+    """Normalize list endpoints that may return either a bare list or envelope."""
+    if isinstance(value, list):
+        return value
+    if isinstance(value, dict):
+        for key in keys:
+            rows = value.get(key)
+            if isinstance(rows, list):
+                return rows
+    return []
+
+
 def list_cases(client: CodeerClient, agent_id: str) -> list[dict]:
-    return client.get(f"/external/eval/agents/{agent_id}/cases")
+    return _unwrap_list_response(
+        client.get(f"/external/eval/agents/{agent_id}/cases"),
+        "cases",
+        "evaluation_cases",
+        "data",
+        "items",
+    )
 
 
 def get_case(client: CodeerClient, case_id: str) -> dict:

{codeer_cli-0.1.7.dist-info → codeer_cli-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeer-cli
-Version: 0.1.7
+Version: 0.1.8
 Summary: Command line tools for managing Codeer agents over the Codeer API.
 Project-URL: Homepage, https://www.codeer.ai
 Author: Codeer.AI

{codeer_cli-0.1.7.dist-info → codeer_cli-0.1.8.dist-info}/RECORD

@@ -5,19 +5,19 @@ codeer_cli/chats.py,sha256=YVrZJhoa-d67o6tzX6riGXsbA-ehyhOxrZ8zRCcJNro,2675
 codeer_cli/cli.py,sha256=g-WR2D5MkaUdc13ZrpRCavXD1940CHE9eBELC034tic,4443
 codeer_cli/client.py,sha256=LpHVqf1IYNg1wFfIHnO9q4xg2h3IiGOitzCnvwB-Bcw,9809
 codeer_cli/constants.py,sha256=D1pV3wCoqYybrKGKeoupYjjFWLfaFviKp1yL7oh6Qso,2323
-codeer_cli/eval_.py,sha256=XwmPxNOtxSyZq2EOae9FZe5NFp0YyVbIswTn22S8noc,17050
+codeer_cli/eval_.py,sha256=4borDzuU7DOqWi1XidfaELmp7eYjV2FH90uKXgGIDuM,17541
 codeer_cli/histories.py,sha256=tk28git_peX4x703CIDU8u72JtlGaytyrtlHfxlK-7A,5979
 codeer_cli/kb.py,sha256=Ad4h65NByq5Rq5BTeMghLTKlWRhmOC2jxL0BaTGX3EM,10631
 codeer_cli/parse.py,sha256=qrjZn0MUTjGfucp4cwxy8Pt7WS-0x15kK5F7kWTY8Ps,21818
 codeer_cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-codeer_cli/commands/_util.py,sha256=VOB_HMWYzHFNY1ElLOED1HB6fpFpsniqH6Yx3VUlMrY,1644
+codeer_cli/commands/_util.py,sha256=X9-9cYgnBYz94hVG4Pe9SsI97DHNv3HE5_7GlPMn93I,1708
 codeer_cli/commands/agent.py,sha256=amvfVVrbPOkbYKCGvA6EJB30C-aY6WSdfR7u7FklXbs,14793
 codeer_cli/commands/check.py,sha256=lTxolx1mIJ8jldPhJ5FXqie9nbCLVOO-sDPOHTSy1-w,3817
-codeer_cli/commands/eval_cmd.py,sha256=fQu8ZRzGO7GWL_Og9NeZ0xYwHoN_iocmCnWw-kPNRkc,68599
+codeer_cli/commands/eval_cmd.py,sha256=V_6OptwWaQ8QLtzJeBqEtlHLUpYXWce1UneeMGPog14,72456
 codeer_cli/commands/history.py,sha256=Jv7t0GhSZcbZ8OuIXZT34CixXt7ECEVP3nZ-WW_Ya9E,12026
 codeer_cli/commands/kb.py,sha256=kVEinBVM6NN8_0djOqIQErh46dLmArwFngXMNzvGeAI,28345
 codeer_cli/commands/profile.py,sha256=IdlXC_6cqobsfN3JRrAnt-1OgBUsIFneS9QtR4Un6Kc,6521
-codeer_cli-0.1.7.dist-info/METADATA,sha256=z8yvV20PQ1ysK27LpsMuurXdAPWJ4Kz-o9-LuxrHczg,5999
-codeer_cli-0.1.7.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-codeer_cli-0.1.7.dist-info/entry_points.txt,sha256=-nXIrlm5SR5r7gg3y8AS0tN66MwmvNHsrlwLNQNGD50,47
-codeer_cli-0.1.7.dist-info/RECORD,,
+codeer_cli-0.1.8.dist-info/METADATA,sha256=dj78KQR8zotgqs8U_VaO5hmIF3ib8XfdGYLDyG5kLOo,5999
+codeer_cli-0.1.8.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+codeer_cli-0.1.8.dist-info/entry_points.txt,sha256=-nXIrlm5SR5r7gg3y8AS0tN66MwmvNHsrlwLNQNGD50,47
+codeer_cli-0.1.8.dist-info/RECORD,,