codeer-cli 0.1.12__tar.gz → 0.1.14__tar.gz

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: codeer-cli
-Version: 0.1.12
+Version: 0.1.14
 Summary: Command line tools for managing Codeer agents over the Codeer API.
 Project-URL: Homepage, https://www.codeer.ai
 Author: Codeer.AI
@@ -123,6 +123,34 @@ List the active cloud models without opening the Codeer web app:
 codeer model list --type text
 ```
 
+## Custom evaluator judge models
+
+Custom evaluator create/update commands can select a judge LLM model by ID:
+
+```bash
+codeer eval evaluator-create \
+  --name "Correctness" \
+  --system-prompt-template-file evaluator-prompt.txt \
+  --judge-model <model-id> \
+  --dry-run
+
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --judge-model <model-id> \
+  --dry-run
+```
+
+Omit the judge-model flags on update to leave the current setting unchanged.
+Use `--clear-judge-model` to explicitly clear the override and return to the
+system default:
+
+```bash
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --clear-judge-model \
+  --dry-run
+```
+
 ## Agent human handoff
 
 `codeer agent apply` accepts the same `human_handoff` object as the Agent API.

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/README.md

@@ -105,6 +105,34 @@ List the active cloud models without opening the Codeer web app:
 codeer model list --type text
 ```
 
+## Custom evaluator judge models
+
+Custom evaluator create/update commands can select a judge LLM model by ID:
+
+```bash
+codeer eval evaluator-create \
+  --name "Correctness" \
+  --system-prompt-template-file evaluator-prompt.txt \
+  --judge-model <model-id> \
+  --dry-run
+
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --judge-model <model-id> \
+  --dry-run
+```
+
+Omit the judge-model flags on update to leave the current setting unchanged.
+Use `--clear-judge-model` to explicitly clear the override and return to the
+system default:
+
+```bash
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --clear-judge-model \
+  --dry-run
+```
+
 ## Agent human handoff
 
 `codeer agent apply` accepts the same `human_handoff` object as the Agent API.

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "codeer-cli"
-version = "0.1.12"
+version = "0.1.14"
 description = "Command line tools for managing Codeer agents over the Codeer API."
 readme = "README.md"
 requires-python = ">=3.11"

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/src/codeer_cli/agents.py

@@ -20,6 +20,7 @@ def create(
     unified_tools: Optional[List[dict]] = None,
     use_search: bool = False,
     llm_model: Optional[str] = None,
+    llm_model_settings: Optional[dict[str, dict[str, Any]]] = None,
     description: Optional[str] = None,
     suggested_questions: Optional[List[str]] = None,
     primary_object_ids: Optional[List[int]] = None,
@@ -41,6 +42,8 @@ def create(
         body["description"] = description
     if llm_model is not None:
         body["llm_model"] = llm_model
+    if llm_model_settings is not None:
+        body["llm_model_settings"] = llm_model_settings
     if validated_handoff is not None:
         body["human_handoff"] = validated_handoff
     return client.post("/external/agents", json=body)
@@ -57,6 +60,7 @@ def update(
     version_note: str = "",
     description: Optional[str] = None,
     llm_model: Optional[str] = None,
+    llm_model_settings: Optional[dict[str, dict[str, Any]]] = None,
     suggested_questions: Optional[List[str]] = None,
     primary_object_ids: Optional[List[int]] = None,
     attachment_ids: Optional[List[str]] = None,
@@ -79,6 +83,8 @@ def update(
         body["description"] = description
     if llm_model is not None:
         body["llm_model"] = llm_model
+    if llm_model_settings is not None:
+        body["llm_model_settings"] = llm_model_settings
     if validated_handoff is not None:
         body["human_handoff"] = validated_handoff
     return client.patch(f"/external/agents/{agent_id}", json=body)

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/src/codeer_cli/commands/agent.py

@@ -89,7 +89,9 @@ def register(subparsers):
 def _tool_summary(tools: list[dict] | None) -> list[dict]:
     out = []
     for t in tools or []:
-        form = t.get("custom_form_schema") if isinstance(t.get("custom_form_schema"), dict) else {}
+        form = t.get("custom_form_schema")
+        if not isinstance(form, dict):
+            form = {}
         out.append({
             "id": t.get("id"),
             "type": t.get("type"),
@@ -169,6 +171,8 @@ def run_apply(args, client) -> int:
         log(f"error: invalid agent payload: {exc}")
         return 2
 
+    llm_model_settings_provided = "llm_model_settings" in body
+
     if args.dry_run:
         operation = "update" if args.agent_id else "create"
         result = {
@@ -181,6 +185,8 @@ def run_apply(args, client) -> int:
             "tool_count": len(validated_tools),
             "use_search": body.get("use_search", False),
             "llm_model": body.get("llm_model"),
+            "llm_model_settings_provided": llm_model_settings_provided,
+            "llm_model_settings": body.get("llm_model_settings"),
             "human_handoff": {
                 "enabled": bool((validated_handoff or {}).get("enabled")),
                 "idle_timeout_minutes": (validated_handoff or {}).get("idle_timeout_minutes"),
@@ -193,6 +199,12 @@ def run_apply(args, client) -> int:
         print_json(result)
         return 0
 
+    model_settings_kwargs = (
+        {"llm_model_settings": body["llm_model_settings"]}
+        if llm_model_settings_provided
+        else {}
+    )
+
     if args.agent_id:
         body.pop("workspace_id", None)
         agents_mod.update(
@@ -208,6 +220,7 @@ def run_apply(args, client) -> int:
             primary_object_ids=body.get("primary_object_ids") or [],
             attachment_ids=body.get("attachment_ids") or [],
             human_handoff=validated_handoff,
+            **model_settings_kwargs,
         )
         agent_id = args.agent_id
         log(f"PUT /agents/{agent_id} ok")
@@ -227,6 +240,7 @@ def run_apply(args, client) -> int:
             primary_object_ids=body.get("primary_object_ids") or [],
             attachment_ids=body.get("attachment_ids") or [],
             human_handoff=validated_handoff,
+            **model_settings_kwargs,
         )
         agent_id = agent["id"]
         log(f"POST /agents ok, id={agent_id}")

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/src/codeer_cli/commands/eval_cmd.py

@@ -128,6 +128,12 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    p.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Judge LLM model ID (default: system default)",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_create)
@@ -140,6 +146,18 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    g = p.add_mutually_exclusive_group()
+    g.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Set the judge LLM model ID",
+    )
+    g.add_argument(
+        "--clear-judge-model",
+        action="store_true",
+        help="Clear the evaluator override and use the system default judge model",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_update)
@@ -249,6 +267,7 @@ def _evaluator_summary(evaluator: dict, *, full: bool = False) -> dict:
         "id": evaluator.get("id"),
         "name": evaluator.get("name"),
         "description": evaluator.get("description"),
+        "judge_llm_model_id": evaluator.get("judge_llm_model_id"),
         "system_prompt_template_chars": len(template),
         "has_tool_steps_placeholder": "{tool_steps}" in template,
         "has_output_placeholder": "{output}" in template,
@@ -534,6 +553,10 @@ def run_evaluator_create(args, client) -> int:
             "workspace_id": workspace_id,
             "name": args.name,
             "description": args.description,
+            "judge_model": {
+                "action": "set" if args.judge_model is not None else "use_system_default",
+                "model_id": args.judge_model,
+            },
             "system_prompt_template_chars": len(system_prompt_template or ""),
             "would_write_server_state": True,
             "next_step": "Review this summary, then rerun without --dry-run after approval.",
@@ -545,6 +568,7 @@ def run_evaluator_create(args, client) -> int:
         name=args.name,
         system_prompt_template=system_prompt_template,
         description=args.description,
+        judge_llm_model_id=args.judge_model,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0
@@ -560,13 +584,27 @@ def run_evaluator_update(args, client) -> int:
     else:
         system_prompt_template = args.system_prompt_template
 
-    if args.name is None and args.description is None and system_prompt_template is None:
+    if (
+        args.name is None
+        and args.description is None
+        and system_prompt_template is None
+        and args.judge_model is None
+        and not args.clear_judge_model
+    ):
         log(
             "error: provide at least one of --name, --description, "
-            "--system-prompt-template, --system-prompt-template-file"
+            "--system-prompt-template, --system-prompt-template-file, "
+            "--judge-model, --clear-judge-model"
         )
         return 2
 
+    if args.clear_judge_model:
+        judge_model = {"action": "clear_to_system_default", "model_id": None}
+    elif args.judge_model is not None:
+        judge_model = {"action": "set", "model_id": args.judge_model}
+    else:
+        judge_model = {"action": "unchanged"}
+
     if args.dry_run:
         print_json({
             "dry_run": True,
@@ -574,6 +612,7 @@ def run_evaluator_update(args, client) -> int:
             "evaluator_id": args.evaluator,
             "name": args.name,
             "description": args.description,
+            "judge_model": judge_model,
             "system_prompt_template_chars": (
                 len(system_prompt_template) if system_prompt_template is not None else None
             ),
@@ -582,12 +621,20 @@ def run_evaluator_update(args, client) -> int:
         })
         return 0
 
+    evaluator_kwargs: dict[str, Any] = {
+        "name": args.name,
+        "system_prompt_template": system_prompt_template,
+        "description": args.description,
+    }
+    if args.clear_judge_model:
+        evaluator_kwargs["judge_llm_model_id"] = None
+    elif args.judge_model is not None:
+        evaluator_kwargs["judge_llm_model_id"] = args.judge_model
+
     evaluator = eval_mod.update_evaluator(
         client,
         evaluator_id=args.evaluator,
-        name=args.name,
-        system_prompt_template=system_prompt_template,
-        description=args.description,
+        **evaluator_kwargs,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/src/codeer_cli/eval_.py

@@ -12,6 +12,13 @@ from typing import Any, List, Optional
 from .client import CodeerClient
 
 
+class _UnsetType:
+    pass
+
+
+_UNSET = _UnsetType()
+
+
 # --- cases ----------------------------------------------------------------
 
 def create_case(
@@ -186,6 +193,7 @@ def create_evaluator(
     name: str,
     system_prompt_template: str,
     description: Optional[str] = None,
+    judge_llm_model_id: Optional[str] = None,
 ) -> dict:
     body: dict[str, Any] = {
         "name": name,
@@ -193,6 +201,8 @@ def create_evaluator(
     }
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not None:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.post("/external/eval/evaluators", json=body)
 
 
@@ -211,6 +221,7 @@ def update_evaluator(
     name: Optional[str] = None,
     system_prompt_template: Optional[str] = None,
     description: Optional[str] = None,
+    judge_llm_model_id: str | None | _UnsetType = _UNSET,
 ) -> dict:
     body: dict[str, Any] = {}
     if name is not None:
@@ -219,6 +230,8 @@ def update_evaluator(
         body["system_prompt_template"] = system_prompt_template
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not _UNSET:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.put(f"/external/eval/evaluators/{evaluator_id}", json=body)
 
 

codeer_cli-0.1.14/tests/test_agent_model_settings.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from codeer_cli import agents
+from codeer_cli.commands import agent as agent_cmd
+
+
+MODEL_ID = "openai/gpt-5.6-luna"
+MODEL_SETTINGS = {MODEL_ID: {"thinking": "xhigh"}}
+
+
+class FakeClient:
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str, dict]] = []
+
+    def post(self, path: str, **kwargs):
+        self.calls.append(("POST", path, kwargs))
+        return {"id": "agent-1"}
+
+    def patch(self, path: str, **kwargs):
+        self.calls.append(("PATCH", path, kwargs))
+        return {"id": "agent-1"}
+
+
+class AgentModelSettingsPayloadTests(unittest.TestCase):
+    def test_create_forwards_model_settings(self) -> None:
+        client = FakeClient()
+
+        agents.create(
+            client,  # type: ignore[arg-type]
+            workspace_id="workspace-1",
+            name="Agent",
+            system_prompt="Help safely.",
+            llm_model=MODEL_ID,
+            llm_model_settings=MODEL_SETTINGS,
+        )
+
+        self.assertEqual(client.calls[0][0:2], ("POST", "/external/agents"))
+        self.assertEqual(
+            client.calls[0][2]["json"]["llm_model_settings"],
+            MODEL_SETTINGS,
+        )
+
+    def test_update_forwards_model_settings(self) -> None:
+        client = FakeClient()
+
+        agents.update(
+            client,  # type: ignore[arg-type]
+            "agent-1",
+            name="Agent",
+            system_prompt="Help safely.",
+            unified_tools=[],
+            use_search=False,
+            llm_model=MODEL_ID,
+            llm_model_settings=MODEL_SETTINGS,
+        )
+
+        self.assertEqual(
+            client.calls[0][0:2],
+            ("PATCH", "/external/agents/agent-1"),
+        )
+        self.assertEqual(
+            client.calls[0][2]["json"]["llm_model_settings"],
+            MODEL_SETTINGS,
+        )
+
+    def test_update_forwards_empty_map_to_clear_settings(self) -> None:
+        client = FakeClient()
+
+        agents.update(
+            client,  # type: ignore[arg-type]
+            "agent-1",
+            name="Agent",
+            system_prompt="Help safely.",
+            unified_tools=[],
+            use_search=False,
+            llm_model_settings={},
+        )
+
+        self.assertEqual(client.calls[0][2]["json"]["llm_model_settings"], {})
+
+    def test_update_omits_model_settings_when_unspecified(self) -> None:
+        client = FakeClient()
+
+        agents.update(
+            client,  # type: ignore[arg-type]
+            "agent-1",
+            name="Agent",
+            system_prompt="Help safely.",
+            unified_tools=[],
+            use_search=False,
+        )
+
+        self.assertNotIn("llm_model_settings", client.calls[0][2]["json"])
+
+
+class AgentModelSettingsCommandTests(unittest.TestCase):
+    def _run_apply(
+        self,
+        payload: dict,
+        *,
+        agent_id: str | None = None,
+        dry_run: bool = False,
+    ) -> tuple[int, dict]:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            payload_path = Path(tmpdir) / "agent.json"
+            payload_path.write_text(json.dumps(payload))
+            args = SimpleNamespace(
+                payload=str(payload_path),
+                agent_id=agent_id,
+                dry_run=dry_run,
+                note="settings regression",
+                out=None,
+            )
+            stdout = io.StringIO()
+            with contextlib.redirect_stdout(stdout):
+                status = agent_cmd.run_apply(args, object())
+        return status, json.loads(stdout.getvalue()) if stdout.getvalue() else {}
+
+    def test_apply_create_forwards_model_settings(self) -> None:
+        payload = {
+            "workspace_id": "workspace-1",
+            "name": "Agent",
+            "system_prompt": "Help safely.",
+            "llm_model": MODEL_ID,
+            "llm_model_settings": MODEL_SETTINGS,
+        }
+        history = [{"id": "history-1", "version_number": 1, "status": "draft"}]
+
+        with (
+            patch.object(
+                agent_cmd.agents_mod,
+                "create",
+                return_value={"id": "agent-1"},
+            ) as create,
+            patch.object(agent_cmd.agents_mod, "list_versions", return_value=history),
+        ):
+            status, _ = self._run_apply(payload)
+
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            create.call_args.kwargs["llm_model_settings"],
+            MODEL_SETTINGS,
+        )
+
+    def test_apply_update_preserves_present_and_empty_settings(self) -> None:
+        history = [{"id": "history-2", "version_number": 2, "status": "draft"}]
+
+        for model_settings in (MODEL_SETTINGS, {}):
+            with self.subTest(model_settings=model_settings):
+                payload = {
+                    "name": "Agent",
+                    "system_prompt": "Help safely.",
+                    "llm_model_settings": model_settings,
+                }
+                with (
+                    patch.object(agent_cmd.agents_mod, "update") as update,
+                    patch.object(
+                        agent_cmd.agents_mod,
+                        "list_versions",
+                        return_value=history,
+                    ),
+                ):
+                    status, _ = self._run_apply(payload, agent_id="agent-1")
+
+                self.assertEqual(status, 0)
+                self.assertEqual(
+                    update.call_args.kwargs["llm_model_settings"],
+                    model_settings,
+                )
+
+    def test_apply_update_does_not_supply_omitted_settings(self) -> None:
+        payload = {
+            "name": "Agent",
+            "system_prompt": "Help safely.",
+        }
+        history = [{"id": "history-2", "version_number": 2, "status": "draft"}]
+
+        with (
+            patch.object(agent_cmd.agents_mod, "update") as update,
+            patch.object(agent_cmd.agents_mod, "list_versions", return_value=history),
+        ):
+            status, _ = self._run_apply(payload, agent_id="agent-1")
+
+        self.assertEqual(status, 0)
+        self.assertNotIn("llm_model_settings", update.call_args.kwargs)
+
+    def test_apply_dry_run_distinguishes_settings_states(self) -> None:
+        cases = (
+            (MODEL_SETTINGS, True, MODEL_SETTINGS),
+            ({}, True, {}),
+            (None, False, None),
+        )
+
+        for model_settings, expected_provided, expected_settings in cases:
+            with self.subTest(model_settings=model_settings):
+                payload: dict[str, object] = {
+                    "name": "Agent",
+                    "system_prompt": "Help safely.",
+                }
+                if expected_provided:
+                    payload["llm_model_settings"] = model_settings
+
+                status, result = self._run_apply(
+                    payload,
+                    agent_id="agent-1",
+                    dry_run=True,
+                )
+
+                self.assertEqual(status, 0)
+                self.assertEqual(
+                    result["llm_model_settings_provided"],
+                    expected_provided,
+                )
+                self.assertEqual(
+                    result["llm_model_settings"],
+                    expected_settings,
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

codeer_cli-0.1.14/tests/test_eval_evaluators.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from types import SimpleNamespace
+
+from codeer_cli import eval_ as eval_mod
+from codeer_cli.commands.eval_cmd import (
+    register,
+    run_evaluator_create,
+    run_evaluator_update,
+)
+
+
+class FakeClient:
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str, dict]] = []
+
+    def post(self, path: str, **kwargs):
+        self.calls.append(("POST", path, kwargs))
+        return {"id": "created", **kwargs["json"]}
+
+    def put(self, path: str, **kwargs):
+        self.calls.append(("PUT", path, kwargs))
+        return {"id": "updated", **kwargs["json"]}
+
+    def resolve_scope(self):
+        return "ws-1", "org-1"
+
+
+class EvaluatorRequestTests(unittest.TestCase):
+    def test_create_evaluator_sends_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.create_evaluator(
+            client,  # type: ignore[arg-type]
+            workspace_id="ws-1",
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+            judge_llm_model_id="model-1",
+        )
+
+        self.assertEqual(client.calls[0][0:2], ("POST", "/external/eval/evaluators"))
+        self.assertEqual(client.calls[0][2]["json"]["judge_llm_model_id"], "model-1")
+
+    def test_create_evaluator_omits_unspecified_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.create_evaluator(
+            client,  # type: ignore[arg-type]
+            workspace_id="ws-1",
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+        )
+
+        self.assertNotIn("judge_llm_model_id", client.calls[0][2]["json"])
+
+    def test_update_evaluator_sends_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.update_evaluator(
+            client,  # type: ignore[arg-type]
+            "eval-1",
+            judge_llm_model_id="model-2",
+        )
+
+        self.assertEqual(client.calls[0][2]["json"], {"judge_llm_model_id": "model-2"})
+
+    def test_update_evaluator_omits_unspecified_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.update_evaluator(
+            client,  # type: ignore[arg-type]
+            "eval-1",
+            name="Updated",
+        )
+
+        self.assertEqual(client.calls[0][2]["json"], {"name": "Updated"})
+
+    def test_update_evaluator_sends_null_to_clear_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.update_evaluator(
+            client,  # type: ignore[arg-type]
+            "eval-1",
+            judge_llm_model_id=None,
+        )
+
+        self.assertEqual(client.calls[0][2]["json"], {"judge_llm_model_id": None})
+
+
+class EvaluatorCliTests(unittest.TestCase):
+    def test_create_dry_run_reports_set_judge_model(self) -> None:
+        client = FakeClient()
+        args = SimpleNamespace(
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+            system_prompt_template_file=None,
+            description=None,
+            judge_model="model-1",
+            dry_run=True,
+        )
+
+        report, result = self._capture(run_evaluator_create, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"], {"action": "set", "model_id": "model-1"}
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_create_dry_run_reports_system_default_when_model_is_omitted(self) -> None:
+        client = FakeClient()
+        args = SimpleNamespace(
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+            system_prompt_template_file=None,
+            description=None,
+            judge_model=None,
+            dry_run=True,
+        )
+
+        report, result = self._capture(run_evaluator_create, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"],
+            {"action": "use_system_default", "model_id": None},
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_update_dry_run_reports_set_judge_model(self) -> None:
+        client = FakeClient()
+        args = self._update_args(judge_model="model-2")
+
+        report, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"], {"action": "set", "model_id": "model-2"}
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_update_dry_run_reports_unspecified_judge_model_as_unchanged(self) -> None:
+        client = FakeClient()
+        args = self._update_args(name="Updated")
+
+        report, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(report["judge_model"], {"action": "unchanged"})
+        self.assertEqual(client.calls, [])
+
+    def test_update_dry_run_reports_explicit_clear(self) -> None:
+        client = FakeClient()
+        args = self._update_args(clear_judge_model=True)
+
+        report, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"],
+            {"action": "clear_to_system_default", "model_id": None},
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_update_clear_flag_sends_explicit_null(self) -> None:
+        client = FakeClient()
+        args = self._update_args(clear_judge_model=True, dry_run=False)
+
+        _, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(client.calls[0][2]["json"], {"judge_llm_model_id": None})
+
+    def test_update_other_field_does_not_send_judge_model(self) -> None:
+        client = FakeClient()
+        args = self._update_args(name="Updated", dry_run=False)
+
+        _, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(client.calls[0][2]["json"], {"name": "Updated"})
+
+    def test_parser_rejects_set_and_clear_together(self) -> None:
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers(dest="group")
+        register(subparsers)
+
+        with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit):
+            parser.parse_args(
+                [
+                    "eval",
+                    "evaluator-update",
+                    "--evaluator",
+                    "eval-1",
+                    "--judge-model",
+                    "model-1",
+                    "--clear-judge-model",
+                ]
+            )
+
+    @staticmethod
+    def _capture(func, args, client) -> tuple[dict, int]:
+        stdout = io.StringIO()
+        with redirect_stdout(stdout):
+            result = func(args, client)
+        return json.loads(stdout.getvalue()), result
+
+    @staticmethod
+    def _update_args(**overrides) -> SimpleNamespace:
+        values = {
+            "evaluator": "eval-1",
+            "name": None,
+            "system_prompt_template": None,
+            "system_prompt_template_file": None,
+            "description": None,
+            "judge_model": None,
+            "clear_judge_model": False,
+            "dry_run": True,
+        }
+        values.update(overrides)
+        return SimpleNamespace(**values)
+
+
+if __name__ == "__main__":
+    unittest.main()

{codeer_cli-0.1.12 → codeer_cli-0.1.14}/uv.lock

@@ -26,7 +26,7 @@ wheels = [
 
 [[package]]
 name = "codeer-cli"
-version = "0.1.12"
+version = "0.1.14"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },