synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/cli/commands/train/verifier_validation.py

@@ -0,0 +1,235 @@
+"""
+Validation logic for verifier/rubric configuration from TOML.
+
+This module validates and normalizes verifier/rubric config, removing all dead fields
+and ensuring only the fields actually used by the backend are present.
+"""
+
+from __future__ import annotations
+
+from collections.abc import MutableMapping
+from typing import Any, Optional, Tuple
+
+from pydantic import ValidationError
+
+from .errors import InvalidRubricConfigError, InvalidVerifierConfigError
+from .verifier_schemas import (
+    RubricConfig,
+    RubricWeightsConfig,
+    VerifierConfig,
+    VerifierOptionsConfig,
+)
+
+__all__ = [
+    "validate_verifier_config",
+    "validate_rubric_config",
+    "extract_and_validate_verifier_rubric",
+]
+
+# Dead fields that should trigger deprecation warnings
+DEPRECATED_RUBRIC_FIELDS = {
+    "model",
+    "api_base",
+    "api_key_env",
+    "event",
+    "outcome",
+}
+
+DEPRECATED_VERIFIER_FIELDS = {
+    "type",
+    "timeout_s",  # Moved to verifier.options.timeout_s
+}
+
+DEPRECATED_VERIFIER_OPTIONS_FIELDS = {
+    "max_concurrency",
+    "tracks",
+}
+
+
+def _reject_deprecated_fields(
+    section: str,
+    fields: set[str],
+    present_fields: set[str],
+    error_cls: type[Exception],
+) -> None:
+    deprecated_present = fields & present_fields
+    if deprecated_present:
+        field_list = ", ".join(sorted(deprecated_present))
+        raise error_cls(
+            detail=f"[{section}] contains deprecated fields that are not supported: {field_list}."
+        )
+
+
+def validate_rubric_config(config: MutableMapping[str, Any]) -> RubricConfig:
+    """
+    Validate and normalize rubric configuration from TOML.
+    
+    Args:
+        config: Raw [rubric] section from TOML
+        
+    Returns:
+        Validated RubricConfig instance
+        
+    Raises:
+        InvalidRubricConfigError: If validation fails
+    """
+    if not config:
+        # Default: rubric disabled
+        return RubricConfig(enabled=False)
+    
+    config_dict = dict(config)
+    
+    _reject_deprecated_fields(
+        "rubric",
+        DEPRECATED_RUBRIC_FIELDS,
+        set(config_dict.keys()),
+        InvalidRubricConfigError,
+    )
+    
+    if "event" in config_dict:
+        raise InvalidRubricConfigError(
+            detail="[rubric.event] is not supported. Use [verifier.options.rubric_overrides] instead."
+        )
+    
+    if "outcome" in config_dict:
+        raise InvalidRubricConfigError(
+            detail="[rubric.outcome] is not supported. Use [verifier.options.rubric_overrides] instead."
+        )
+    
+    # Extract only valid fields
+    enabled = config_dict.get("enabled", False)
+    weights_dict = config_dict.get("weights", {})
+    
+    # Validate using Pydantic
+    try:
+        if not isinstance(weights_dict, dict):
+            raise ValueError("[rubric.weights] must be a dictionary")
+        
+        weights = RubricWeightsConfig(**weights_dict)
+        return RubricConfig(enabled=enabled, weights=weights)
+    
+    except ValidationError as exc:
+        errors = []
+        for error in exc.errors():
+            loc = ".".join(str(x) for x in error["loc"])
+            msg = error["msg"]
+            errors.append(f"  • rubric.{loc}: {msg}")
+        raise InvalidRubricConfigError(
+            detail="Rubric validation failed:\n" + "\n".join(errors)
+        ) from exc
+    except Exception as exc:
+        raise InvalidRubricConfigError(
+            detail=f"Rubric validation failed: {exc}"
+        ) from exc
+
+
+def validate_verifier_config(config: MutableMapping[str, Any]) -> Optional[VerifierConfig]:
+    """
+    Validate and normalize verifier configuration from TOML.
+    
+    Args:
+        config: Raw [verifier] section from TOML
+        
+    Returns:
+        Validated VerifierConfig instance, or None if not present
+        
+    Raises:
+        InvalidVerifierConfigError: If validation fails
+    """
+    if not config:
+        return None
+    
+    config_dict = dict(config)
+    
+    _reject_deprecated_fields(
+        "verifier",
+        DEPRECATED_VERIFIER_FIELDS,
+        set(config_dict.keys()),
+        InvalidVerifierConfigError,
+    )
+    
+    # Extract verifier.options (required)
+    options_dict = config_dict.get("options")
+    if not options_dict:
+        raise InvalidVerifierConfigError(
+            detail="[verifier.options] section is required when [verifier] is present"
+        )
+    
+    if not isinstance(options_dict, dict):
+        raise InvalidVerifierConfigError(
+            detail="[verifier.options] must be a dictionary"
+        )
+    
+    _reject_deprecated_fields(
+        "verifier.options",
+        DEPRECATED_VERIFIER_OPTIONS_FIELDS,
+        set(options_dict.keys()),
+        InvalidVerifierConfigError,
+    )
+    
+    # Validate using Pydantic
+    try:
+        options = VerifierOptionsConfig(**options_dict)
+        return VerifierConfig(options=options)
+    
+    except ValidationError as exc:
+        errors = []
+        for error in exc.errors():
+            loc = ".".join(str(x) for x in error["loc"])
+            msg = error["msg"]
+            errors.append(f"  • verifier.options.{loc}: {msg}")
+        raise InvalidVerifierConfigError(
+            detail="Verifier validation failed:\n" + "\n".join(errors)
+        ) from exc
+    except Exception as exc:
+        raise InvalidVerifierConfigError(
+            detail=f"Verifier validation failed: {exc}"
+        ) from exc
+
+
+def extract_and_validate_verifier_rubric(
+    toml_config: MutableMapping[str, Any]
+) -> Tuple[RubricConfig, Optional[VerifierConfig]]:
+    """
+    Extract and validate verifier/rubric config from full TOML config.
+    
+    Args:
+        toml_config: Full TOML configuration dict
+        
+    Returns:
+        Tuple of (validated_rubric, validated_verifier_or_none)
+        
+    Raises:
+        InvalidRubricConfigError: If rubric validation fails
+        InvalidVerifierConfigError: If verifier validation fails
+    """
+    rubric_dict = toml_config.get("rubric", {})
+    verifier_dict = toml_config.get("verifier", {})
+    
+    # Validate rubric
+    rubric_config = validate_rubric_config(rubric_dict)
+    
+    # Validate verifier (if present)
+    verifier_config = validate_verifier_config(verifier_dict) if verifier_dict else None
+    
+    if rubric_config.enabled and not verifier_config:
+        raise InvalidVerifierConfigError(
+            detail="[rubric].enabled=true requires a [verifier] section."
+        )
+    
+    if rubric_config.enabled and verifier_config:
+        weights = rubric_config.weights
+        options = verifier_config.options
+        
+        if weights.event > 0 and not options.event:
+            raise InvalidVerifierConfigError(
+                detail="[rubric.weights].event > 0 requires [verifier.options].event=true."
+            )
+        
+        if weights.outcome > 0 and not options.outcome:
+            raise InvalidVerifierConfigError(
+                detail="[rubric.weights].outcome > 0 requires [verifier.options].outcome=true."
+            )
+    
+    return rubric_config, verifier_config
+

synth_ai/cli/demo_apps/demo_task_apps/math/config.toml

@@ -24,7 +24,6 @@ evaluation_episodes = 50
 
 [training]
 max_turns = 1
-ops = ["agent", "env"]
 batch_size = 2
 group_size = 16
 reward_positive = 1.0

synth_ai/cli/demo_apps/demo_task_apps/math/modal_task_app.py

@@ -480,9 +480,6 @@ def fastapi_app():
         data = request if isinstance(request, dict) else {}
         env = data.get("env") if isinstance(data, dict) else {}
         policy = data.get("policy") if isinstance(data, dict) else {}
-        ops = data.get("ops") if isinstance(data, dict) else []
-        if not isinstance(ops, list):
-            ops = []
         env_name = (env or {}).get("env_name") or "math"  # type: ignore[misc]
         policy_cfg = (policy or {}).get("config") or {}  # type: ignore[misc]
         model = policy_cfg.get("model")  # type: ignore[misc]
@@ -730,13 +727,12 @@ def fastapi_app():
             ],
             "branches": {},
             "metrics": {
-                "episode_returns": [total_reward],
-                "mean_return": float(total_reward),
+                "episode_rewards": [total_reward],
+                "reward_mean": float(total_reward),
                 "num_steps": len(steps),
                 "num_episodes": 1,
             },
             "aborted": False,
-            "ops_executed": len(steps),
         }
 
     return api

synth_ai/cli/demo_apps/math/config.toml

@@ -26,7 +26,6 @@ evaluation_episodes = 50
 
 [training]
 max_turns = 1
-ops = ["agent", "env"]
 batch_size = 2
 group_size = 16
 reward_positive = 1.0

synth_ai/cli/demo_apps/math/modal_task_app.py

@@ -469,9 +469,6 @@ def fastapi_app():
         data = request if isinstance(request, dict) else {}
         env = data.get("env") if isinstance(data, dict) else {}
         policy = data.get("policy") if isinstance(data, dict) else {}
-        ops = data.get("ops") if isinstance(data, dict) else []
-        if not isinstance(ops, list):
-            ops = []
         env_name = (env or {}).get("env_name") or "math"  # type: ignore[misc]
         policy_cfg = (policy or {}).get("config") or {}  # type: ignore[misc]
         model = policy_cfg.get("model")  # type: ignore[misc]
@@ -690,13 +687,12 @@ def fastapi_app():
             ],
             "branches": {},
             "metrics": {
-                "episode_returns": [total_reward],
-                "mean_return": float(total_reward),
+                "episode_rewards": [total_reward],
+                "reward_mean": float(total_reward),
                 "num_steps": len(steps),
                 "num_episodes": 1,
             },
             "aborted": False,
-            "ops_executed": len(steps),
         }
 
     return api

synth_ai/cli/demo_apps/mipro/task_app.py

@@ -5,7 +5,6 @@ import inspect
 import json
 import os
 import socket
-import uuid
 from collections.abc import Iterable, Sequence
 from pathlib import Path
 from typing import Any, Mapping, cast
@@ -21,8 +20,6 @@ from synth_ai.sdk.task.contracts import (
     RolloutMetrics,
     RolloutRequest,
     RolloutResponse,
-    RolloutStep,
-    RolloutTrajectory,
     TaskInfo,
 )
 from synth_ai.sdk.task.datasets import TaskDatasetRegistry, TaskDatasetSpec
@@ -34,6 +31,10 @@ from synth_ai.sdk.task.server import (
     create_task_app,
     run_task_app,
 )
+from synth_ai.sdk.task.trace_correlation_helpers import (
+    build_trace_payload,
+    extract_trace_correlation_id,
+)
 from synth_ai.sdk.task.vendors import normalize_vendor_keys
 
 # Dataset configuration
@@ -593,60 +594,37 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
             flush=True,
         )
 
-    step = RolloutStep(
-        obs=observation,
-        tool_calls=tool_calls,
-        reward=reward,
-        done=True,
-        info=info_payload,
-    )
-
     inference_url = (request.policy.config or {}).get("inference_url")
-    trajectory = RolloutTrajectory(  # type: ignore[call-overload]
-        env_id=f"banking77::{sample['split']}::{sample['index']}",
-        policy_id=request.policy.policy_id or request.policy.policy_name or "policy",
-        steps=[step],
-        final={"observation": observation, "reward": reward},  # type: ignore[arg-type]
-        length=1,
-        inference_url=str(inference_url or ""),
-    )
 
     metrics = RolloutMetrics(
-        episode_returns=[reward],
-        mean_return=reward,
-        num_steps=1,
-        num_episodes=1,
-        outcome_score=reward,
-        events_score=reward,
-        details={"correct": is_correct},
+        outcome_reward=reward,
+        details={"predicted": predicted_intent, "expected": expected_intent},
     )
-
-    trace_payload = None
-    include_trace = bool(
-        (request.record and getattr(request.record, "return_trace", False))
-        or os.getenv("TASKAPP_TRACING_ENABLED")
+    policy_config = request.policy.config or {}
+    trace_correlation_id = extract_trace_correlation_id(
+        policy_config=policy_config,
+        inference_url=str(inference_url or ""),
+        mode=request.mode,
+    )
+    trace_metadata = {
+        "env": "banking77",
+        "split": sample["split"],
+        "index": sample["index"],
+        "correct": is_correct,
+    }
+    trace_payload = build_trace_payload(
+        messages=rendered_messages,
+        response=response_json if isinstance(response_json, dict) else None,
+        correlation_id=trace_correlation_id,
+        metadata=trace_metadata,
     )
-    if include_trace:
-        trace_payload = {
-            "session_id": str(uuid.uuid4()),
-            "events_count": 1,
-            "decision_rewards": [reward],
-            "metadata": {
-                "env": "banking77",
-                "split": sample["split"],
-                "index": sample["index"],
-                "correct": is_correct,
-            },
-        }
 
     return RolloutResponse(
         run_id=request.run_id,
-        trajectories=[trajectory],
-        branches={},
         metrics=metrics,
-        aborted=False,
-        ops_executed=2,
         trace=trace_payload,
+        trace_correlation_id=trace_correlation_id,
+        inference_url=str(inference_url or ""),
     )
 
 

synth_ai/cli/lib/apps/task_app.py

@@ -144,7 +144,7 @@ def _validate_rollout_payload(payload: Any) -> None:
     if not isinstance(trajectories, list):
         raise ValueError(
             f"`/rollout` response field 'trajectories' must be a list, got {type(trajectories).__name__}. "
-            f"Make sure your rollout executor returns a proper RolloutResponse with a list of RolloutTrajectory objects."
+            f"Make sure your rollout executor returns a proper RolloutResponse with a v3 trace payload."
         )
 
     # Ensure trajectories list is not empty (training will fail if it's empty)
@@ -265,27 +265,27 @@ def _validate_rollout_payload(payload: Any) -> None:
         )
 
     # Metrics can be either:
-    # 1. Full RolloutMetrics with episode_returns (list), mean_return, num_steps
-    # 2. Simple dict with scalar values (episode_returns as float, mean_return, num_steps)
-    required_metrics_fields = ["episode_returns", "mean_return", "num_steps"]
+    # 1. Full RolloutMetrics with episode_rewards (list), reward_mean, num_steps
+    # 2. Simple dict with scalar values (episode_rewards as float, reward_mean, num_steps)
+    required_metrics_fields = ["episode_rewards", "reward_mean", "num_steps"]
     for field in required_metrics_fields:
         if field not in metrics:
             raise ValueError(
                 f"`/rollout` metrics missing required field '{field}'. "
-                f"Metrics must include: episode_returns, mean_return, and num_steps."
+                f"Metrics must include: episode_rewards, reward_mean, and num_steps."
             )
 
-    # Validate types - episode_returns can be either a list or a scalar
-    episode_returns = metrics.get("episode_returns")
-    if not isinstance(episode_returns, list | int | float):
+    # Validate types - episode_rewards can be either a list or a scalar
+    episode_rewards = metrics.get("episode_rewards")
+    if not isinstance(episode_rewards, list | int | float):
         raise ValueError(
-            f"`/rollout` metrics.episode_returns must be a list or number, got {type(episode_returns).__name__}"
+            f"`/rollout` metrics.episode_rewards must be a list or number, got {type(episode_rewards).__name__}"
         )
 
-    mean_return = metrics.get("mean_return")
-    if not isinstance(mean_return, int | float):
+    reward_mean = metrics.get("reward_mean")
+    if not isinstance(reward_mean, int | float):
         raise ValueError(
-            f"`/rollout` metrics.mean_return must be a number, got {type(mean_return).__name__}"
+            f"`/rollout` metrics.reward_mean must be a number, got {type(reward_mean).__name__}"
         )
 
     num_steps = metrics.get("num_steps")
@@ -388,7 +388,6 @@ def test_route_contracts(app: ASGIApp) -> None:
                             "assert_proxy": True,   # Backend always sets this for prompt learning
                             "proxy_only": True,     # Backend always sets this for prompt learning
                         },
-                        "ops": ["agent", "env"],  # Critical: training sends this
                         "record": {"trajectories": True},
                         "mode": "eval",
                     }

synth_ai/cli/lib/task_app_discovery.py

@@ -307,8 +307,8 @@ def _extract_app_id(node: ast.Call) -> str | None:
 
 def _is_register_task_app_call(node: ast.Call) -> bool:
     func = node.func
-    return (isinstance(func, ast.Name) and func.id == "register_task_app") or (
-        isinstance(func, ast.Attribute) and func.attr == "register_task_app"
+    return (isinstance(func, ast.Name) and func.id in {"register_task_app", "register_local_api"}) or (
+        isinstance(func, ast.Attribute) and func.attr in {"register_task_app", "register_local_api"}
     )
 
 
@@ -316,10 +316,10 @@ def _extract_register_app_id(node: ast.Call) -> str | None:
     for kw in node.keywords:
         if kw.arg == "entry" and isinstance(kw.value, ast.Call):
             entry_call = kw.value
-            if isinstance(entry_call.func, ast.Name) and entry_call.func.id == "TaskAppEntry":
+            if isinstance(entry_call.func, ast.Name) and entry_call.func.id in {"TaskAppEntry", "LocalAPIEntry"}:
                 for entry_kw in entry_call.keywords:
                     if (
-                        entry_kw.arg == "app_id"
+                        entry_kw.arg in {"app_id", "api_id"}
                         and isinstance(entry_kw.value, ast.Constant)
                         and isinstance(entry_kw.value.value, str)
                     ):
@@ -535,7 +535,7 @@ def _has_modal_support_in_file(path: Path) -> bool:
                     entry_call = kw.value
                     if (
                         isinstance(entry_call.func, ast.Name)
-                        and entry_call.func.id == "TaskAppEntry"
+                        and entry_call.func.id in {"TaskAppEntry", "LocalAPIEntry"}
                     ):
                         for entry_kw in entry_call.keywords:
                             if entry_kw.arg == "modal" and isinstance(entry_kw.value, ast.Call):
@@ -562,7 +562,7 @@ def _extract_modal_config_from_file(path: Path) -> ModalDeploymentConfig | None:
                     entry_call = kw.value
                     if (
                         isinstance(entry_call.func, ast.Name)
-                        and entry_call.func.id == "TaskAppEntry"
+                        and entry_call.func.id in {"TaskAppEntry", "LocalAPIEntry"}
                     ):
                         for entry_kw in entry_call.keywords:
                             if entry_kw.arg == "modal" and isinstance(entry_kw.value, ast.Call):

synth_ai/cli/lib/train_cfgs.py

@@ -6,8 +6,8 @@ from typing import Any, Dict, List, Literal, Tuple
 from synth_ai.cli.lib.prompts import ctx_print
 from synth_ai.core.paths import is_hidden_path, validate_file_type
 
-# Train config types: prompt optimization, reinforcement learning, supervised fine-tuning, ADAS, context learning
-TrainType = Literal["prompt", "rl", "sft", "adas", "context_learning"]
+# Train config types: prompt optimization, reinforcement learning, supervised fine-tuning, graph opt, context learning
+TrainType = Literal["prompt", "rl", "sft", "graphgen", "context_learning"]
 
 
 def get_type(config: Dict[str, Any]) -> TrainType | None:
@@ -17,9 +17,9 @@ def get_type(config: Dict[str, Any]) -> TrainType | None:
     if "prompt_learning" in config:
         return "prompt"
 
-    # Graph / ADAS jobs use a dedicated [graph] (or [adas]) section.
-    if isinstance(config.get("graph"), dict) or isinstance(config.get("adas"), dict):
-        return "adas"
+    # Graph Opt jobs use a dedicated [graph] section.
+    if isinstance(config.get("graph"), dict):
+        return "graphgen"
 
     algorithm = config.get("algorithm")
     algo_type = None
@@ -221,14 +221,14 @@ def validate_rl_cfg(cfg: Dict[str, Any]) -> None:
     return None
 
 
-def validate_adas_cfg(cfg: Dict[str, Any], *, path: Path) -> None:
-    """Validate a graph/ADAS TOML config.
+def validate_graph_cfg(cfg: Dict[str, Any], *, path: Path) -> None:
+    """Validate a graph opt TOML config.
 
     Uses the SDK validator so backend and CLI stay in sync.
     """
     from synth_ai.sdk.api.train.graph_validators import validate_graph_job_section
 
-    section = cfg.get("graph") or cfg.get("adas") or {}
+    section = cfg.get("graph") or {}
     validate_graph_job_section(section, base_dir=path.parent.resolve())
 
 
@@ -262,8 +262,8 @@ def validate_train_cfg(path: Path, discovery: bool = False) -> TrainType:
             validate_rl_cfg(cfg)
         case "sft":
             validate_sft_cfg(cfg)
-        case "adas":
-            validate_adas_cfg(cfg, path=path)
+        case "graphgen":
+            validate_graph_cfg(cfg, path=path)
     print_pass()
 
     return train_type

synth_ai/cli/task_apps/__init__.py

@@ -4,10 +4,17 @@ Commands for managing Synth task apps - local serving, Modal deployment,
 validation, and discovery.
 """
 
+from __future__ import annotations
+
+import importlib
+
 from synth_ai.cli.task_apps.commands import (
     AppChoice,
     TaskAppEntryType,
+    _find_modal_executable,
+    _is_modal_shim,
     _markov_message_from_dict,
+    _modal_command_prefix,
     register,
     serve_command,
     task_app_group,
@@ -22,5 +29,9 @@ __all__ = [
     "task_app_group",
     "serve_command",
     "register",
+    "_find_modal_executable",
+    "_is_modal_shim",
+    "_modal_command_prefix",
     "_markov_message_from_dict",
+    "importlib",
 ]

synth_ai/cli/task_apps/commands.py

@@ -33,7 +33,6 @@ except Exception:  # pragma: no cover - fallback
 import click
 from click.exceptions import Abort
 
-from synth_ai.cli.commands.eval import core as eval_core
 from synth_ai.cli.commands.filter import core as filter_core
 
 # Tracing imports - make conditional for optional dependencies
@@ -569,20 +568,20 @@ def _extract_app_id(node: ast.Call) -> str | None:
 
 def _is_register_task_app_call(node: ast.Call) -> bool:
     func = node.func
-    return (isinstance(func, ast.Name) and func.id == "register_task_app") or (
-        isinstance(func, ast.Attribute) and func.attr == "register_task_app"
+    return (isinstance(func, ast.Name) and func.id in {"register_task_app", "register_local_api"}) or (
+        isinstance(func, ast.Attribute) and func.attr in {"register_task_app", "register_local_api"}
     )
 
 
 def _extract_register_app_id(node: ast.Call) -> str | None:
-    # Look for entry=TaskAppEntry(app_id="...", ...)
+    # Look for entry=TaskAppEntry(app_id="...") or entry=LocalAPIEntry(api_id="...")
     for kw in node.keywords:
         if kw.arg == "entry" and isinstance(kw.value, ast.Call):
             entry_call = kw.value
-            if isinstance(entry_call.func, ast.Name) and entry_call.func.id == "TaskAppEntry":
+            if isinstance(entry_call.func, ast.Name) and entry_call.func.id in {"TaskAppEntry", "LocalAPIEntry"}:
                 for entry_kw in entry_call.keywords:
                     if (
-                        entry_kw.arg == "app_id"
+                        entry_kw.arg in {"app_id", "api_id"}
                         and isinstance(entry_kw.value, ast.Constant)
                         and isinstance(entry_kw.value.value, str)
                     ):
@@ -865,7 +864,7 @@ def _has_modal_support_in_file(path: Path) -> bool:
                         entry_call = kw.value
                         if (
                             isinstance(entry_call.func, ast.Name)
-                            and entry_call.func.id == "TaskAppEntry"
+                            and entry_call.func.id in {"TaskAppEntry", "LocalAPIEntry"}
                         ):
                             for entry_kw in entry_call.keywords:
                                 if entry_kw.arg == "modal" and isinstance(entry_kw.value, ast.Call):
@@ -895,7 +894,7 @@ def _extract_modal_config_from_file(path: Path) -> ModalDeploymentConfigType | N
                         entry_call = kw.value
                         if (
                             isinstance(entry_call.func, ast.Name)
-                            and entry_call.func.id == "TaskAppEntry"
+                            and entry_call.func.id in {"TaskAppEntry", "LocalAPIEntry"}
                         ):
                             for entry_kw in entry_call.keywords:
                                 if entry_kw.arg == "modal" and isinstance(entry_kw.value, ast.Call):
@@ -3140,14 +3139,7 @@ def fastapi_app():
 def register(cli: click.Group) -> None:
     cli.add_command(serve_command)
     cli.add_command(task_app_group)
-    cli.add_command(eval_command)
     cli.add_command(filter_command)
 
 
-eval_command = eval_core.command
-
 filter_command = filter_core.command
-
-
-def register_eval(cli: click.Group) -> None:
-    cli.add_command(eval_command)