codeer-cli 0.1.13__py3-none-any.whl → 0.1.14__py3-none-any.whl

codeer_cli/commands/eval_cmd.py

@@ -128,6 +128,12 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    p.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Judge LLM model ID (default: system default)",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_create)
@@ -140,6 +146,18 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    g = p.add_mutually_exclusive_group()
+    g.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Set the judge LLM model ID",
+    )
+    g.add_argument(
+        "--clear-judge-model",
+        action="store_true",
+        help="Clear the evaluator override and use the system default judge model",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_update)
@@ -249,6 +267,7 @@ def _evaluator_summary(evaluator: dict, *, full: bool = False) -> dict:
         "id": evaluator.get("id"),
         "name": evaluator.get("name"),
         "description": evaluator.get("description"),
+        "judge_llm_model_id": evaluator.get("judge_llm_model_id"),
         "system_prompt_template_chars": len(template),
         "has_tool_steps_placeholder": "{tool_steps}" in template,
         "has_output_placeholder": "{output}" in template,
@@ -534,6 +553,10 @@ def run_evaluator_create(args, client) -> int:
             "workspace_id": workspace_id,
             "name": args.name,
             "description": args.description,
+            "judge_model": {
+                "action": "set" if args.judge_model is not None else "use_system_default",
+                "model_id": args.judge_model,
+            },
             "system_prompt_template_chars": len(system_prompt_template or ""),
             "would_write_server_state": True,
             "next_step": "Review this summary, then rerun without --dry-run after approval.",
@@ -545,6 +568,7 @@ def run_evaluator_create(args, client) -> int:
         name=args.name,
         system_prompt_template=system_prompt_template,
         description=args.description,
+        judge_llm_model_id=args.judge_model,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0
@@ -560,13 +584,27 @@ def run_evaluator_update(args, client) -> int:
     else:
         system_prompt_template = args.system_prompt_template
 
-    if args.name is None and args.description is None and system_prompt_template is None:
+    if (
+        args.name is None
+        and args.description is None
+        and system_prompt_template is None
+        and args.judge_model is None
+        and not args.clear_judge_model
+    ):
         log(
             "error: provide at least one of --name, --description, "
-            "--system-prompt-template, --system-prompt-template-file"
+            "--system-prompt-template, --system-prompt-template-file, "
+            "--judge-model, --clear-judge-model"
         )
         return 2
 
+    if args.clear_judge_model:
+        judge_model = {"action": "clear_to_system_default", "model_id": None}
+    elif args.judge_model is not None:
+        judge_model = {"action": "set", "model_id": args.judge_model}
+    else:
+        judge_model = {"action": "unchanged"}
+
     if args.dry_run:
         print_json({
             "dry_run": True,
@@ -574,6 +612,7 @@ def run_evaluator_update(args, client) -> int:
             "evaluator_id": args.evaluator,
             "name": args.name,
             "description": args.description,
+            "judge_model": judge_model,
             "system_prompt_template_chars": (
                 len(system_prompt_template) if system_prompt_template is not None else None
             ),
@@ -582,12 +621,20 @@ def run_evaluator_update(args, client) -> int:
         })
         return 0
 
+    evaluator_kwargs: dict[str, Any] = {
+        "name": args.name,
+        "system_prompt_template": system_prompt_template,
+        "description": args.description,
+    }
+    if args.clear_judge_model:
+        evaluator_kwargs["judge_llm_model_id"] = None
+    elif args.judge_model is not None:
+        evaluator_kwargs["judge_llm_model_id"] = args.judge_model
+
     evaluator = eval_mod.update_evaluator(
         client,
         evaluator_id=args.evaluator,
-        name=args.name,
-        system_prompt_template=system_prompt_template,
-        description=args.description,
+        **evaluator_kwargs,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0

codeer_cli/eval_.py

@@ -12,6 +12,13 @@ from typing import Any, List, Optional
 from .client import CodeerClient
 
 
+class _UnsetType:
+    pass
+
+
+_UNSET = _UnsetType()
+
+
 # --- cases ----------------------------------------------------------------
 
 def create_case(
@@ -186,6 +193,7 @@ def create_evaluator(
     name: str,
     system_prompt_template: str,
     description: Optional[str] = None,
+    judge_llm_model_id: Optional[str] = None,
 ) -> dict:
     body: dict[str, Any] = {
         "name": name,
@@ -193,6 +201,8 @@ def create_evaluator(
     }
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not None:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.post("/external/eval/evaluators", json=body)
 
 
@@ -211,6 +221,7 @@ def update_evaluator(
     name: Optional[str] = None,
     system_prompt_template: Optional[str] = None,
     description: Optional[str] = None,
+    judge_llm_model_id: str | None | _UnsetType = _UNSET,
 ) -> dict:
     body: dict[str, Any] = {}
     if name is not None:
@@ -219,6 +230,8 @@ def update_evaluator(
         body["system_prompt_template"] = system_prompt_template
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not _UNSET:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.put(f"/external/eval/evaluators/{evaluator_id}", json=body)
 
 

{codeer_cli-0.1.13.dist-info → codeer_cli-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: codeer-cli
-Version: 0.1.13
+Version: 0.1.14
 Summary: Command line tools for managing Codeer agents over the Codeer API.
 Project-URL: Homepage, https://www.codeer.ai
 Author: Codeer.AI
@@ -123,6 +123,34 @@ List the active cloud models without opening the Codeer web app:
 codeer model list --type text
 ```
 
+## Custom evaluator judge models
+
+Custom evaluator create/update commands can select a judge LLM model by ID:
+
+```bash
+codeer eval evaluator-create \
+  --name "Correctness" \
+  --system-prompt-template-file evaluator-prompt.txt \
+  --judge-model <model-id> \
+  --dry-run
+
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --judge-model <model-id> \
+  --dry-run
+```
+
+Omit the judge-model flags on update to leave the current setting unchanged.
+Use `--clear-judge-model` to explicitly clear the override and return to the
+system default:
+
+```bash
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --clear-judge-model \
+  --dry-run
+```
+
 ## Agent human handoff
 
 `codeer agent apply` accepts the same `human_handoff` object as the Agent API.

{codeer_cli-0.1.13.dist-info → codeer_cli-0.1.14.dist-info}/RECORD

@@ -5,7 +5,7 @@ codeer_cli/chats.py,sha256=Vg1DIu8fT6mPyPpfqLpExKvXhdZEWCyJGPvalQy3BXg,7989
 codeer_cli/cli.py,sha256=ghZMn7LUxDDvb3f82mDYW5hIFLk7dtlrCrGGP40Gygw,4637
 codeer_cli/client.py,sha256=apUycjK-OvHEquAj58FqOy8FHmHjA6K9dScaYQ4M-VU,13175
 codeer_cli/constants.py,sha256=D1pV3wCoqYybrKGKeoupYjjFWLfaFviKp1yL7oh6Qso,2323
-codeer_cli/eval_.py,sha256=2Tz4qokctocZ0fqLYL8Sk9UV8yJ7p-9zYiUFljsRRCs,17478
+codeer_cli/eval_.py,sha256=4lDaT-gf_nCatnRoHA0L9MDBxCDgT0ORBWC5ku04pts,17827
 codeer_cli/histories.py,sha256=WvpqvuprPc1nJJCTElyM_eoUabyooACfmRCwYlcvPnI,6637
 codeer_cli/kb.py,sha256=Ad4h65NByq5Rq5BTeMghLTKlWRhmOC2jxL0BaTGX3EM,10631
 codeer_cli/models.py,sha256=Ama9fx8hyRMdysiUNA3Ty0sYIYwzFDkOcAZg-id3-vQ,362
@@ -14,12 +14,12 @@ codeer_cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 codeer_cli/commands/_util.py,sha256=X9-9cYgnBYz94hVG4Pe9SsI97DHNv3HE5_7GlPMn93I,1708
 codeer_cli/commands/agent.py,sha256=BjctOF0tflt8yVGzdc6320sWXVUAYrF39tNIiTxtQx4,16319
 codeer_cli/commands/check.py,sha256=lTxolx1mIJ8jldPhJ5FXqie9nbCLVOO-sDPOHTSy1-w,3817
-codeer_cli/commands/eval_cmd.py,sha256=Zlt_YIA3D5-Y8WL1HZ6PxZgL-TcwFTDoXvO2gv7yfOM,71989
+codeer_cli/commands/eval_cmd.py,sha256=S-ydzWZppPLMCu2i_HOK0OW5ZaoebxcLO--WDYOIuAk,73548
 codeer_cli/commands/history.py,sha256=2p9VuPwyexr7Q2FFIBSuv9hNzn7VdIMa-AtvtP-8LvI,17253
 codeer_cli/commands/kb.py,sha256=kVEinBVM6NN8_0djOqIQErh46dLmArwFngXMNzvGeAI,28345
 codeer_cli/commands/model.py,sha256=91LDo_YeHi3POZV7KTCkoCcgtK5p87LwrHX1lZgfCps,1698
 codeer_cli/commands/profile.py,sha256=IdlXC_6cqobsfN3JRrAnt-1OgBUsIFneS9QtR4Un6Kc,6521
-codeer_cli-0.1.13.dist-info/METADATA,sha256=zslhdIB39lnbLxfGOM9sv8zrZdzDNzi6xjZWCDLrPQU,7885
-codeer_cli-0.1.13.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-codeer_cli-0.1.13.dist-info/entry_points.txt,sha256=-nXIrlm5SR5r7gg3y8AS0tN66MwmvNHsrlwLNQNGD50,47
-codeer_cli-0.1.13.dist-info/RECORD,,
+codeer_cli-0.1.14.dist-info/METADATA,sha256=Dyw4s9TmhHF1Jsn9X0NSN9H-qouENxBsxIMRC4d_gb4,8547
+codeer_cli-0.1.14.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+codeer_cli-0.1.14.dist-info/entry_points.txt,sha256=-nXIrlm5SR5r7gg3y8AS0tN66MwmvNHsrlwLNQNGD50,47
+codeer_cli-0.1.14.dist-info/RECORD,,