codeer-cli 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

codeer_cli/agents.py

@@ -20,6 +20,7 @@ def create(
     unified_tools: Optional[List[dict]] = None,
     use_search: bool = False,
     llm_model: Optional[str] = None,
+    llm_model_settings: Optional[dict[str, dict[str, Any]]] = None,
     description: Optional[str] = None,
     suggested_questions: Optional[List[str]] = None,
     primary_object_ids: Optional[List[int]] = None,
@@ -41,6 +42,8 @@ def create(
         body["description"] = description
     if llm_model is not None:
         body["llm_model"] = llm_model
+    if llm_model_settings is not None:
+        body["llm_model_settings"] = llm_model_settings
     if validated_handoff is not None:
         body["human_handoff"] = validated_handoff
     return client.post("/external/agents", json=body)
@@ -57,6 +60,7 @@ def update(
     version_note: str = "",
     description: Optional[str] = None,
     llm_model: Optional[str] = None,
+    llm_model_settings: Optional[dict[str, dict[str, Any]]] = None,
     suggested_questions: Optional[List[str]] = None,
     primary_object_ids: Optional[List[int]] = None,
     attachment_ids: Optional[List[str]] = None,
@@ -79,6 +83,8 @@ def update(
         body["description"] = description
     if llm_model is not None:
         body["llm_model"] = llm_model
+    if llm_model_settings is not None:
+        body["llm_model_settings"] = llm_model_settings
     if validated_handoff is not None:
         body["human_handoff"] = validated_handoff
     return client.patch(f"/external/agents/{agent_id}", json=body)

codeer_cli/commands/agent.py

@@ -89,7 +89,9 @@ def register(subparsers):
 def _tool_summary(tools: list[dict] | None) -> list[dict]:
     out = []
     for t in tools or []:
-        form = t.get("custom_form_schema") if isinstance(t.get("custom_form_schema"), dict) else {}
+        form = t.get("custom_form_schema")
+        if not isinstance(form, dict):
+            form = {}
         out.append({
             "id": t.get("id"),
             "type": t.get("type"),
@@ -169,6 +171,8 @@ def run_apply(args, client) -> int:
         log(f"error: invalid agent payload: {exc}")
         return 2
 
+    llm_model_settings_provided = "llm_model_settings" in body
+
     if args.dry_run:
         operation = "update" if args.agent_id else "create"
         result = {
@@ -181,6 +185,8 @@ def run_apply(args, client) -> int:
             "tool_count": len(validated_tools),
             "use_search": body.get("use_search", False),
             "llm_model": body.get("llm_model"),
+            "llm_model_settings_provided": llm_model_settings_provided,
+            "llm_model_settings": body.get("llm_model_settings"),
             "human_handoff": {
                 "enabled": bool((validated_handoff or {}).get("enabled")),
                 "idle_timeout_minutes": (validated_handoff or {}).get("idle_timeout_minutes"),
@@ -193,6 +199,12 @@ def run_apply(args, client) -> int:
         print_json(result)
         return 0
 
+    model_settings_kwargs = (
+        {"llm_model_settings": body["llm_model_settings"]}
+        if llm_model_settings_provided
+        else {}
+    )
+
     if args.agent_id:
         body.pop("workspace_id", None)
         agents_mod.update(
@@ -208,6 +220,7 @@ def run_apply(args, client) -> int:
             primary_object_ids=body.get("primary_object_ids") or [],
             attachment_ids=body.get("attachment_ids") or [],
             human_handoff=validated_handoff,
+            **model_settings_kwargs,
         )
         agent_id = args.agent_id
         log(f"PUT /agents/{agent_id} ok")
@@ -227,6 +240,7 @@ def run_apply(args, client) -> int:
             primary_object_ids=body.get("primary_object_ids") or [],
             attachment_ids=body.get("attachment_ids") or [],
             human_handoff=validated_handoff,
+            **model_settings_kwargs,
         )
         agent_id = agent["id"]
         log(f"POST /agents ok, id={agent_id}")

codeer_cli/commands/eval_cmd.py

@@ -128,6 +128,12 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    p.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Judge LLM model ID (default: system default)",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_create)
@@ -140,6 +146,18 @@ def register(subparsers):
     g.add_argument("--system-prompt-template", help="Evaluator system prompt template text")
     g.add_argument("--system-prompt-template-file", help="Path to evaluator system prompt template")
     p.add_argument("--description", default=None)
+    g = p.add_mutually_exclusive_group()
+    g.add_argument(
+        "--judge-model",
+        default=None,
+        metavar="MODEL_ID",
+        help="Set the judge LLM model ID",
+    )
+    g.add_argument(
+        "--clear-judge-model",
+        action="store_true",
+        help="Clear the evaluator override and use the system default judge model",
+    )
     p.add_argument("--dry-run", action="store_true",
                    help="Validate inputs and print intended mutation without writing server state.")
     p.set_defaults(func=run_evaluator_update)
@@ -249,6 +267,7 @@ def _evaluator_summary(evaluator: dict, *, full: bool = False) -> dict:
         "id": evaluator.get("id"),
         "name": evaluator.get("name"),
         "description": evaluator.get("description"),
+        "judge_llm_model_id": evaluator.get("judge_llm_model_id"),
         "system_prompt_template_chars": len(template),
         "has_tool_steps_placeholder": "{tool_steps}" in template,
         "has_output_placeholder": "{output}" in template,
@@ -534,6 +553,10 @@ def run_evaluator_create(args, client) -> int:
             "workspace_id": workspace_id,
             "name": args.name,
             "description": args.description,
+            "judge_model": {
+                "action": "set" if args.judge_model is not None else "use_system_default",
+                "model_id": args.judge_model,
+            },
             "system_prompt_template_chars": len(system_prompt_template or ""),
             "would_write_server_state": True,
             "next_step": "Review this summary, then rerun without --dry-run after approval.",
@@ -545,6 +568,7 @@ def run_evaluator_create(args, client) -> int:
         name=args.name,
         system_prompt_template=system_prompt_template,
         description=args.description,
+        judge_llm_model_id=args.judge_model,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0
@@ -560,13 +584,27 @@ def run_evaluator_update(args, client) -> int:
     else:
         system_prompt_template = args.system_prompt_template
 
-    if args.name is None and args.description is None and system_prompt_template is None:
+    if (
+        args.name is None
+        and args.description is None
+        and system_prompt_template is None
+        and args.judge_model is None
+        and not args.clear_judge_model
+    ):
         log(
             "error: provide at least one of --name, --description, "
-            "--system-prompt-template, --system-prompt-template-file"
+            "--system-prompt-template, --system-prompt-template-file, "
+            "--judge-model, --clear-judge-model"
         )
         return 2
 
+    if args.clear_judge_model:
+        judge_model = {"action": "clear_to_system_default", "model_id": None}
+    elif args.judge_model is not None:
+        judge_model = {"action": "set", "model_id": args.judge_model}
+    else:
+        judge_model = {"action": "unchanged"}
+
     if args.dry_run:
         print_json({
             "dry_run": True,
@@ -574,6 +612,7 @@ def run_evaluator_update(args, client) -> int:
             "evaluator_id": args.evaluator,
             "name": args.name,
             "description": args.description,
+            "judge_model": judge_model,
             "system_prompt_template_chars": (
                 len(system_prompt_template) if system_prompt_template is not None else None
             ),
@@ -582,12 +621,20 @@ def run_evaluator_update(args, client) -> int:
         })
         return 0
 
+    evaluator_kwargs: dict[str, Any] = {
+        "name": args.name,
+        "system_prompt_template": system_prompt_template,
+        "description": args.description,
+    }
+    if args.clear_judge_model:
+        evaluator_kwargs["judge_llm_model_id"] = None
+    elif args.judge_model is not None:
+        evaluator_kwargs["judge_llm_model_id"] = args.judge_model
+
     evaluator = eval_mod.update_evaluator(
         client,
         evaluator_id=args.evaluator,
-        name=args.name,
-        system_prompt_template=system_prompt_template,
-        description=args.description,
+        **evaluator_kwargs,
     )
     print_json(_evaluator_summary(evaluator, full=True))
     return 0

codeer_cli/eval_.py

@@ -12,6 +12,13 @@ from typing import Any, List, Optional
 from .client import CodeerClient
 
 
+class _UnsetType:
+    pass
+
+
+_UNSET = _UnsetType()
+
+
 # --- cases ----------------------------------------------------------------
 
 def create_case(
@@ -186,6 +193,7 @@ def create_evaluator(
     name: str,
     system_prompt_template: str,
     description: Optional[str] = None,
+    judge_llm_model_id: Optional[str] = None,
 ) -> dict:
     body: dict[str, Any] = {
         "name": name,
@@ -193,6 +201,8 @@ def create_evaluator(
     }
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not None:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.post("/external/eval/evaluators", json=body)
 
 
@@ -211,6 +221,7 @@ def update_evaluator(
     name: Optional[str] = None,
     system_prompt_template: Optional[str] = None,
     description: Optional[str] = None,
+    judge_llm_model_id: str | None | _UnsetType = _UNSET,
 ) -> dict:
     body: dict[str, Any] = {}
     if name is not None:
@@ -219,6 +230,8 @@ def update_evaluator(
         body["system_prompt_template"] = system_prompt_template
     if description is not None:
         body["description"] = description
+    if judge_llm_model_id is not _UNSET:
+        body["judge_llm_model_id"] = judge_llm_model_id
     return client.put(f"/external/eval/evaluators/{evaluator_id}", json=body)
 
 

{codeer_cli-0.1.12.dist-info → codeer_cli-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: codeer-cli
-Version: 0.1.12
+Version: 0.1.14
 Summary: Command line tools for managing Codeer agents over the Codeer API.
 Project-URL: Homepage, https://www.codeer.ai
 Author: Codeer.AI
@@ -123,6 +123,34 @@ List the active cloud models without opening the Codeer web app:
 codeer model list --type text
 ```
 
+## Custom evaluator judge models
+
+Custom evaluator create/update commands can select a judge LLM model by ID:
+
+```bash
+codeer eval evaluator-create \
+  --name "Correctness" \
+  --system-prompt-template-file evaluator-prompt.txt \
+  --judge-model <model-id> \
+  --dry-run
+
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --judge-model <model-id> \
+  --dry-run
+```
+
+Omit the judge-model flags on update to leave the current setting unchanged.
+Use `--clear-judge-model` to explicitly clear the override and return to the
+system default:
+
+```bash
+codeer eval evaluator-update \
+  --evaluator <evaluator-id> \
+  --clear-judge-model \
+  --dry-run
+```
+
 ## Agent human handoff
 
 `codeer agent apply` accepts the same `human_handoff` object as the Agent API.

{codeer_cli-0.1.12.dist-info → codeer_cli-0.1.14.dist-info}/RECORD

@@ -1,25 +1,25 @@
 codeer_cli/__init__.py,sha256=z3rkUdVCt11c8QE3-YFkG5cFObM6QhPe2yKugA8k3aA,1915
 codeer_cli/_validate.py,sha256=lqMwkhsBy1g0o5CINBp1HdBCjMawgUaEsZygjJaAPL0,6943
-codeer_cli/agents.py,sha256=HWfz6QgU4wn07df5s3twxVJfAhb0vLJHIfxTjvDMUzU,6045
+codeer_cli/agents.py,sha256=Yt_Rw4c9omsTnXsTe5n_9mD4TpDcx_1WuUkhtKmFY0g,6371
 codeer_cli/chats.py,sha256=Vg1DIu8fT6mPyPpfqLpExKvXhdZEWCyJGPvalQy3BXg,7989
 codeer_cli/cli.py,sha256=ghZMn7LUxDDvb3f82mDYW5hIFLk7dtlrCrGGP40Gygw,4637
 codeer_cli/client.py,sha256=apUycjK-OvHEquAj58FqOy8FHmHjA6K9dScaYQ4M-VU,13175
 codeer_cli/constants.py,sha256=D1pV3wCoqYybrKGKeoupYjjFWLfaFviKp1yL7oh6Qso,2323
-codeer_cli/eval_.py,sha256=2Tz4qokctocZ0fqLYL8Sk9UV8yJ7p-9zYiUFljsRRCs,17478
+codeer_cli/eval_.py,sha256=4lDaT-gf_nCatnRoHA0L9MDBxCDgT0ORBWC5ku04pts,17827
 codeer_cli/histories.py,sha256=WvpqvuprPc1nJJCTElyM_eoUabyooACfmRCwYlcvPnI,6637
 codeer_cli/kb.py,sha256=Ad4h65NByq5Rq5BTeMghLTKlWRhmOC2jxL0BaTGX3EM,10631
 codeer_cli/models.py,sha256=Ama9fx8hyRMdysiUNA3Ty0sYIYwzFDkOcAZg-id3-vQ,362
 codeer_cli/parse.py,sha256=-txMtIxPfeg_i4McEskVKxo8p4438gKS-sFTI38RoBs,21772
 codeer_cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codeer_cli/commands/_util.py,sha256=X9-9cYgnBYz94hVG4Pe9SsI97DHNv3HE5_7GlPMn93I,1708
-codeer_cli/commands/agent.py,sha256=6qtYcQYgdJE26V6Nw6parB_3RYAYXcOpPE9iJLuLGHo,15888
+codeer_cli/commands/agent.py,sha256=BjctOF0tflt8yVGzdc6320sWXVUAYrF39tNIiTxtQx4,16319
 codeer_cli/commands/check.py,sha256=lTxolx1mIJ8jldPhJ5FXqie9nbCLVOO-sDPOHTSy1-w,3817
-codeer_cli/commands/eval_cmd.py,sha256=Zlt_YIA3D5-Y8WL1HZ6PxZgL-TcwFTDoXvO2gv7yfOM,71989
+codeer_cli/commands/eval_cmd.py,sha256=S-ydzWZppPLMCu2i_HOK0OW5ZaoebxcLO--WDYOIuAk,73548
 codeer_cli/commands/history.py,sha256=2p9VuPwyexr7Q2FFIBSuv9hNzn7VdIMa-AtvtP-8LvI,17253
 codeer_cli/commands/kb.py,sha256=kVEinBVM6NN8_0djOqIQErh46dLmArwFngXMNzvGeAI,28345
 codeer_cli/commands/model.py,sha256=91LDo_YeHi3POZV7KTCkoCcgtK5p87LwrHX1lZgfCps,1698
 codeer_cli/commands/profile.py,sha256=IdlXC_6cqobsfN3JRrAnt-1OgBUsIFneS9QtR4Un6Kc,6521
-codeer_cli-0.1.12.dist-info/METADATA,sha256=CFrY9ndSNGm4K5UC0A9tZyCXAQPOgWMO5CcGMMNRda0,7885
-codeer_cli-0.1.12.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-codeer_cli-0.1.12.dist-info/entry_points.txt,sha256=-nXIrlm5SR5r7gg3y8AS0tN66MwmvNHsrlwLNQNGD50,47
-codeer_cli-0.1.12.dist-info/RECORD,,
+codeer_cli-0.1.14.dist-info/METADATA,sha256=Dyw4s9TmhHF1Jsn9X0NSN9H-qouENxBsxIMRC4d_gb4,8547
+codeer_cli-0.1.14.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+codeer_cli-0.1.14.dist-info/entry_points.txt,sha256=-nXIrlm5SR5r7gg3y8AS0tN66MwmvNHsrlwLNQNGD50,47
+codeer_cli-0.1.14.dist-info/RECORD,,

{codeer_cli-0.1.12.dist-info → codeer_cli-0.1.14.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.31.0
+Generator: hatchling 1.32.0
 Root-Is-Purelib: true
 Tag: py3-none-any