codeer-cli 0.1.13__tar.gz → 0.1.14__tar.gz

{codeer_cli-0.1.13 → codeer_cli-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: codeer-cli
-Version: 0.1.13
+Version: 0.1.14
 Summary: Command line tools for managing Codeer agents over the Codeer API.
 Project-URL: Homepage, https://www.codeer.ai
 Author: Codeer.AI
@@ -123,6 +123,34 @@ List the active cloud models without opening the Codeer web app:
 codeer model list --type text
 ```
 
+## Custom evaluator judge models
+
+Custom evaluator create/update commands can select a judge LLM model by ID:
+
+```bash
+codeer eval evaluator-create \
+  --name "Correctness" \
+  --system-prompt-template-file evaluator-prompt.txt \
+  --judge-model <model-id> \
+  --dry-run
+
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --judge-model <model-id> \
+  --dry-run
+```
+
+Omit the judge-model flags on update to leave the current setting unchanged.
+Use `--clear-judge-model` to explicitly clear the override and return to the
+system default:
+
+```bash
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --clear-judge-model \
+  --dry-run
+```
+
 ## Agent human handoff
 
 `codeer agent apply` accepts the same `human_handoff` object as the Agent API.

{codeer_cli-0.1.13 → codeer_cli-0.1.14}/README.md

@@ -105,6 +105,34 @@ List the active cloud models without opening the Codeer web app:
 codeer model list --type text
 ```
 
+## Custom evaluator judge models
+
+Custom evaluator create/update commands can select a judge LLM model by ID:
+
+```bash
+codeer eval evaluator-create \
+  --name "Correctness" \
+  --system-prompt-template-file evaluator-prompt.txt \
+  --judge-model <model-id> \
+  --dry-run
+
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --judge-model <model-id> \
+  --dry-run
+```
+
+Omit the judge-model flags on update to leave the current setting unchanged.
+Use `--clear-judge-model` to explicitly clear the override and return to the
+system default:
+
+```bash
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --clear-judge-model \
+  --dry-run
+```
+
 ## Agent human handoff
 
 `codeer agent apply` accepts the same `human_handoff` object as the Agent API.

{codeer_cli-0.1.13 → codeer_cli-0.1.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "codeer-cli"
-version = "0.1.13"
+version = "0.1.14"
 description = "Command line tools for managing Codeer agents over the Codeer API."
 readme = "README.md"
 requires-python = ">=3.11"

{codeer_cli-0.1.13 → codeer_cli-0.1.14}/src/codeer_cli/commands/eval_cmd.py

@@ -128,6 +128,12 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    p.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Judge LLM model ID (default: system default)",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_create)
@@ -140,6 +146,18 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    g = p.add_mutually_exclusive_group()
+    g.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Set the judge LLM model ID",
+    )
+    g.add_argument(
+        "--clear-judge-model",
+        action="store_true",
+        help="Clear the evaluator override and use the system default judge model",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_update)
@@ -249,6 +267,7 @@ def _evaluator_summary(evaluator: dict, *, full: bool = False) -> dict:
         "id": evaluator.get("id"),
         "name": evaluator.get("name"),
         "description": evaluator.get("description"),
+        "judge_llm_model_id": evaluator.get("judge_llm_model_id"),
         "system_prompt_template_chars": len(template),
         "has_tool_steps_placeholder": "{tool_steps}" in template,
         "has_output_placeholder": "{output}" in template,
@@ -534,6 +553,10 @@ def run_evaluator_create(args, client) -> int:
             "workspace_id": workspace_id,
             "name": args.name,
             "description": args.description,
+            "judge_model": {
+                "action": "set" if args.judge_model is not None else "use_system_default",
+                "model_id": args.judge_model,
+            },
             "system_prompt_template_chars": len(system_prompt_template or ""),
             "would_write_server_state": True,
             "next_step": "Review this summary, then rerun without --dry-run after approval.",
@@ -545,6 +568,7 @@ def run_evaluator_create(args, client) -> int:
         name=args.name,
         system_prompt_template=system_prompt_template,
         description=args.description,
+        judge_llm_model_id=args.judge_model,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0
@@ -560,13 +584,27 @@ def run_evaluator_update(args, client) -> int:
     else:
         system_prompt_template = args.system_prompt_template
 
-    if args.name is None and args.description is None and system_prompt_template is None:
+    if (
+        args.name is None
+        and args.description is None
+        and system_prompt_template is None
+        and args.judge_model is None
+        and not args.clear_judge_model
+    ):
         log(
             "error: provide at least one of --name, --description, "
-            "--system-prompt-template, --system-prompt-template-file"
+            "--system-prompt-template, --system-prompt-template-file, "
+            "--judge-model, --clear-judge-model"
         )
         return 2
 
+    if args.clear_judge_model:
+        judge_model = {"action": "clear_to_system_default", "model_id": None}
+    elif args.judge_model is not None:
+        judge_model = {"action": "set", "model_id": args.judge_model}
+    else:
+        judge_model = {"action": "unchanged"}
+
     if args.dry_run:
         print_json({
             "dry_run": True,
@@ -574,6 +612,7 @@ def run_evaluator_update(args, client) -> int:
             "evaluator_id": args.evaluator,
             "name": args.name,
             "description": args.description,
+            "judge_model": judge_model,
             "system_prompt_template_chars": (
                 len(system_prompt_template) if system_prompt_template is not None else None
             ),
@@ -582,12 +621,20 @@ def run_evaluator_update(args, client) -> int:
         })
         return 0
 
+    evaluator_kwargs: dict[str, Any] = {
+        "name": args.name,
+        "system_prompt_template": system_prompt_template,
+        "description": args.description,
+    }
+    if args.clear_judge_model:
+        evaluator_kwargs["judge_llm_model_id"] = None
+    elif args.judge_model is not None:
+        evaluator_kwargs["judge_llm_model_id"] = args.judge_model
+
     evaluator = eval_mod.update_evaluator(
         client,
         evaluator_id=args.evaluator,
-        name=args.name,
-        system_prompt_template=system_prompt_template,
-        description=args.description,
+        **evaluator_kwargs,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0

{codeer_cli-0.1.13 → codeer_cli-0.1.14}/src/codeer_cli/eval_.py

@@ -12,6 +12,13 @@ from typing import Any, List, Optional
 from .client import CodeerClient
 
 
+class _UnsetType:
+    pass
+
+
+_UNSET = _UnsetType()
+
+
 # --- cases ----------------------------------------------------------------
 
 def create_case(
@@ -186,6 +193,7 @@ def create_evaluator(
     name: str,
     system_prompt_template: str,
     description: Optional[str] = None,
+    judge_llm_model_id: Optional[str] = None,
 ) -> dict:
     body: dict[str, Any] = {
         "name": name,
@@ -193,6 +201,8 @@ def create_evaluator(
     }
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not None:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.post("/external/eval/evaluators", json=body)
 
 
@@ -211,6 +221,7 @@ def update_evaluator(
     name: Optional[str] = None,
     system_prompt_template: Optional[str] = None,
     description: Optional[str] = None,
+    judge_llm_model_id: str | None | _UnsetType = _UNSET,
 ) -> dict:
     body: dict[str, Any] = {}
     if name is not None:
@@ -219,6 +230,8 @@ def update_evaluator(
         body["system_prompt_template"] = system_prompt_template
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not _UNSET:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.put(f"/external/eval/evaluators/{evaluator_id}", json=body)
 
 

codeer_cli-0.1.14/tests/test_eval_evaluators.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from types import SimpleNamespace
+
+from codeer_cli import eval_ as eval_mod
+from codeer_cli.commands.eval_cmd import (
+    register,
+    run_evaluator_create,
+    run_evaluator_update,
+)
+
+
+class FakeClient:
+    def __init__(self) -> None:
+        self.calls: list[tuple[str, str, dict]] = []
+
+    def post(self, path: str, **kwargs):
+        self.calls.append(("POST", path, kwargs))
+        return {"id": "created", **kwargs["json"]}
+
+    def put(self, path: str, **kwargs):
+        self.calls.append(("PUT", path, kwargs))
+        return {"id": "updated", **kwargs["json"]}
+
+    def resolve_scope(self):
+        return "ws-1", "org-1"
+
+
+class EvaluatorRequestTests(unittest.TestCase):
+    def test_create_evaluator_sends_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.create_evaluator(
+            client,  # type: ignore[arg-type]
+            workspace_id="ws-1",
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+            judge_llm_model_id="model-1",
+        )
+
+        self.assertEqual(client.calls[0][0:2], ("POST", "/external/eval/evaluators"))
+        self.assertEqual(client.calls[0][2]["json"]["judge_llm_model_id"], "model-1")
+
+    def test_create_evaluator_omits_unspecified_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.create_evaluator(
+            client,  # type: ignore[arg-type]
+            workspace_id="ws-1",
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+        )
+
+        self.assertNotIn("judge_llm_model_id", client.calls[0][2]["json"])
+
+    def test_update_evaluator_sends_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.update_evaluator(
+            client,  # type: ignore[arg-type]
+            "eval-1",
+            judge_llm_model_id="model-2",
+        )
+
+        self.assertEqual(client.calls[0][2]["json"], {"judge_llm_model_id": "model-2"})
+
+    def test_update_evaluator_omits_unspecified_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.update_evaluator(
+            client,  # type: ignore[arg-type]
+            "eval-1",
+            name="Updated",
+        )
+
+        self.assertEqual(client.calls[0][2]["json"], {"name": "Updated"})
+
+    def test_update_evaluator_sends_null_to_clear_judge_model(self) -> None:
+        client = FakeClient()
+
+        eval_mod.update_evaluator(
+            client,  # type: ignore[arg-type]
+            "eval-1",
+            judge_llm_model_id=None,
+        )
+
+        self.assertEqual(client.calls[0][2]["json"], {"judge_llm_model_id": None})
+
+
+class EvaluatorCliTests(unittest.TestCase):
+    def test_create_dry_run_reports_set_judge_model(self) -> None:
+        client = FakeClient()
+        args = SimpleNamespace(
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+            system_prompt_template_file=None,
+            description=None,
+            judge_model="model-1",
+            dry_run=True,
+        )
+
+        report, result = self._capture(run_evaluator_create, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"], {"action": "set", "model_id": "model-1"}
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_create_dry_run_reports_system_default_when_model_is_omitted(self) -> None:
+        client = FakeClient()
+        args = SimpleNamespace(
+            name="Correctness",
+            system_prompt_template="Judge {output}",
+            system_prompt_template_file=None,
+            description=None,
+            judge_model=None,
+            dry_run=True,
+        )
+
+        report, result = self._capture(run_evaluator_create, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"],
+            {"action": "use_system_default", "model_id": None},
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_update_dry_run_reports_set_judge_model(self) -> None:
+        client = FakeClient()
+        args = self._update_args(judge_model="model-2")
+
+        report, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"], {"action": "set", "model_id": "model-2"}
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_update_dry_run_reports_unspecified_judge_model_as_unchanged(self) -> None:
+        client = FakeClient()
+        args = self._update_args(name="Updated")
+
+        report, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(report["judge_model"], {"action": "unchanged"})
+        self.assertEqual(client.calls, [])
+
+    def test_update_dry_run_reports_explicit_clear(self) -> None:
+        client = FakeClient()
+        args = self._update_args(clear_judge_model=True)
+
+        report, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(
+            report["judge_model"],
+            {"action": "clear_to_system_default", "model_id": None},
+        )
+        self.assertEqual(client.calls, [])
+
+    def test_update_clear_flag_sends_explicit_null(self) -> None:
+        client = FakeClient()
+        args = self._update_args(clear_judge_model=True, dry_run=False)
+
+        _, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(client.calls[0][2]["json"], {"judge_llm_model_id": None})
+
+    def test_update_other_field_does_not_send_judge_model(self) -> None:
+        client = FakeClient()
+        args = self._update_args(name="Updated", dry_run=False)
+
+        _, result = self._capture(run_evaluator_update, args, client)
+
+        self.assertEqual(result, 0)
+        self.assertEqual(client.calls[0][2]["json"], {"name": "Updated"})
+
+    def test_parser_rejects_set_and_clear_together(self) -> None:
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers(dest="group")
+        register(subparsers)
+
+        with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit):
+            parser.parse_args(
+                [
+                    "eval",
+                    "evaluator-update",
+                    "--evaluator",
+                    "eval-1",
+                    "--judge-model",
+                    "model-1",
+                    "--clear-judge-model",
+                ]
+            )
+
+    @staticmethod
+    def _capture(func, args, client) -> tuple[dict, int]:
+        stdout = io.StringIO()
+        with redirect_stdout(stdout):
+            result = func(args, client)
+        return json.loads(stdout.getvalue()), result
+
+    @staticmethod
+    def _update_args(**overrides) -> SimpleNamespace:
+        values = {
+            "evaluator": "eval-1",
+            "name": None,
+            "system_prompt_template": None,
+            "system_prompt_template_file": None,
+            "description": None,
+            "judge_model": None,
+            "clear_judge_model": False,
+            "dry_run": True,
+        }
+        values.update(overrides)
+        return SimpleNamespace(**values)
+
+
+if __name__ == "__main__":
+    unittest.main()

{codeer_cli-0.1.13 → codeer_cli-0.1.14}/uv.lock

@@ -26,7 +26,7 @@ wheels = [
 
 [[package]]
 name = "codeer-cli"
-version = "0.1.13"
+version = "0.1.14"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },