uipath 2.1.107__py3-none-any.whl → 2.1.109__py3-none-any.whl

uipath/_cli/__init__.py

@@ -4,6 +4,7 @@ import sys
 import click
 
 from ._utils._common import add_cwd_to_path, load_environment_variables
+from .cli_add import add as add
 from .cli_auth import auth as auth
 from .cli_debug import debug as debug  # type: ignore
 from .cli_deploy import deploy as deploy  # type: ignore
@@ -16,6 +17,7 @@ from .cli_pack import pack as pack  # type: ignore
 from .cli_publish import publish as publish  # type: ignore
 from .cli_pull import pull as pull  # type: ignore
 from .cli_push import push as push  # type: ignore
+from .cli_register import register as register  # type: ignore
 from .cli_run import run as run  # type: ignore
 
 
@@ -75,4 +77,6 @@ cli.add_command(push)
 cli.add_command(pull)
 cli.add_command(eval)
 cli.add_command(dev)
+cli.add_command(add)
+cli.add_command(register)
 cli.add_command(debug)

uipath/_cli/_evals/_console_progress_reporter.py

@@ -7,6 +7,7 @@ from rich.console import Console
 from rich.rule import Rule
 from rich.table import Table
 
+from uipath._cli._evals._models._evaluation_set import AnyEvaluator
 from uipath._events._event_bus import EventBus
 from uipath._events._events import (
     EvalRunCreatedEvent,
@@ -15,7 +16,6 @@ from uipath._events._events import (
     EvalSetRunUpdatedEvent,
     EvaluationEvents,
 )
-from uipath.eval.evaluators import BaseEvaluator
 from uipath.eval.models import ScoreType
 
 logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ class ConsoleProgressReporter:
 
     def __init__(self):
         self.console = Console()
-        self.evaluators: Dict[str, BaseEvaluator[Any]] = {}
+        self.evaluators: Dict[str, AnyEvaluator] = {}
         self.display_started = False
         self.eval_results_by_name: Dict[str, list[Any]] = {}
 

uipath/_cli/_evals/_evaluator_factory.py

@@ -1,21 +1,69 @@
+import importlib.util
+import sys
+from pathlib import Path
 from typing import Any, Dict
 
 from pydantic import TypeAdapter
 
+from uipath._cli._evals._helpers import try_extract_file_and_class_name  # type: ignore
+from uipath._cli._evals._models._evaluation_set import AnyEvaluator
 from uipath._cli._evals._models._evaluator import (
     EqualsEvaluatorParams,
-    Evaluator,
+    EvaluatorConfig,
     JsonSimilarityEvaluatorParams,
+    LegacyEvaluator,
     LLMEvaluatorParams,
     TrajectoryEvaluatorParams,
 )
 from uipath._cli._evals._models._evaluator_base_params import EvaluatorBaseParams
 from uipath.eval.evaluators import (
     BaseEvaluator,
+    LegacyBaseEvaluator,
+    LegacyExactMatchEvaluator,
+    LegacyJsonSimilarityEvaluator,
+    LegacyLlmAsAJudgeEvaluator,
+    LegacyTrajectoryEvaluator,
+)
+from uipath.eval.evaluators.base_evaluator import BaseEvaluatorConfig
+from uipath.eval.evaluators.contains_evaluator import (
+    ContainsEvaluator,
+    ContainsEvaluatorConfig,
+)
+from uipath.eval.evaluators.exact_match_evaluator import (
     ExactMatchEvaluator,
+    ExactMatchEvaluatorConfig,
+)
+from uipath.eval.evaluators.json_similarity_evaluator import (
     JsonSimilarityEvaluator,
-    LlmAsAJudgeEvaluator,
-    TrajectoryEvaluator,
+    JsonSimilarityEvaluatorConfig,
+)
+from uipath.eval.evaluators.llm_judge_output_evaluator import (
+    LLMJudgeOutputEvaluator,
+    LLMJudgeOutputEvaluatorConfig,
+    LLMJudgeStrictJSONSimilarityOutputEvaluator,
+    LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig,
+)
+from uipath.eval.evaluators.llm_judge_trajectory_evaluator import (
+    LLMJudgeTrajectoryEvaluator,
+    LLMJudgeTrajectoryEvaluatorConfig,
+    LLMJudgeTrajectorySimulationEvaluator,
+    LLMJudgeTrajectorySimulationEvaluatorConfig,
+)
+from uipath.eval.evaluators.tool_call_args_evaluator import (
+    ToolCallArgsEvaluator,
+    ToolCallArgsEvaluatorConfig,
+)
+from uipath.eval.evaluators.tool_call_count_evaluator import (
+    ToolCallCountEvaluator,
+    ToolCallCountEvaluatorConfig,
+)
+from uipath.eval.evaluators.tool_call_order_evaluator import (
+    ToolCallOrderEvaluator,
+    ToolCallOrderEvaluatorConfig,
+)
+from uipath.eval.evaluators.tool_call_output_evaluator import (
+    ToolCallOutputEvaluator,
+    ToolCallOutputEvaluatorConfig,
 )
 
 
@@ -23,7 +71,252 @@ class EvaluatorFactory:
     """Factory class for creating evaluator instances based on configuration."""
 
     @classmethod
-    def create_evaluator(cls, data: Dict[str, Any]) -> BaseEvaluator[Any]:
+    def create_evaluator(cls, data: Dict[str, Any]) -> AnyEvaluator:
+        if data.get("version", None) == "1.0":
+            return cls._create_evaluator_internal(data)
+        return cls._create_legacy_evaluator_internal(data)
+
+    @staticmethod
+    def _create_evaluator_internal(
+        data: Dict[str, Any],
+    ) -> BaseEvaluator[Any, Any, Any]:
+        # check custom evaluator
+        evaluator_schema = data.get("evaluatorSchema", "")
+        success, file_path, class_name = try_extract_file_and_class_name(
+            evaluator_schema
+        )
+        if success:
+            return EvaluatorFactory._create_coded_evaluator_internal(
+                data, file_path, class_name
+            )
+
+        # use built-in evaluators
+        config: BaseEvaluatorConfig[Any] = TypeAdapter(EvaluatorConfig).validate_python(
+            data
+        )
+        match config:
+            case ContainsEvaluatorConfig():
+                return EvaluatorFactory._create_contains_evaluator(data)
+            case ExactMatchEvaluatorConfig():
+                return EvaluatorFactory._create_exact_match_evaluator(data)
+            case JsonSimilarityEvaluatorConfig():
+                return EvaluatorFactory._create_json_similarity_evaluator(data)
+            case LLMJudgeOutputEvaluatorConfig():
+                return EvaluatorFactory._create_llm_judge_output_evaluator(data)
+            case LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig():
+                return EvaluatorFactory._create_llm_judge_strict_json_similarity_output_evaluator(
+                    data
+                )
+            case LLMJudgeTrajectoryEvaluatorConfig():
+                return EvaluatorFactory._create_trajectory_evaluator(data)
+            case ToolCallArgsEvaluatorConfig():
+                return EvaluatorFactory._create_tool_call_args_evaluator(data)
+            case ToolCallCountEvaluatorConfig():
+                return EvaluatorFactory._create_tool_call_count_evaluator(data)
+            case ToolCallOrderEvaluatorConfig():
+                return EvaluatorFactory._create_tool_call_order_evaluator(data)
+            case ToolCallOutputEvaluatorConfig():
+                return EvaluatorFactory._create_tool_call_output_evaluator(data)
+            case LLMJudgeTrajectorySimulationEvaluatorConfig():
+                return (
+                    EvaluatorFactory._create_llm_judge_simulation_trajectory_evaluator(
+                        data
+                    )
+                )
+            case _:
+                raise ValueError(f"Unknown evaluator configuration: {config}")
+
+    @staticmethod
+    def _create_contains_evaluator(data: Dict[str, Any]) -> ContainsEvaluator:
+        evaluator_id = data.get("id")
+        if not evaluator_id or not isinstance(evaluator_id, str):
+            raise ValueError("Evaluator 'id' must be a non-empty string")
+        return ContainsEvaluator(
+            id=evaluator_id,
+            config=data.get("evaluatorConfig"),
+        )  # type: ignore
+
+    @staticmethod
+    def _create_coded_evaluator_internal(
+        data: Dict[str, Any], file_path_str: str, class_name: str
+    ) -> BaseEvaluator[Any, Any, Any]:
+        """Create a coded evaluator by dynamically loading from a Python file.
+
+        Args:
+            data: Dictionary containing evaluator configuration with evaluatorTypeId
+                  in format "file://path/to/file.py:ClassName"
+
+        Returns:
+            Instance of the dynamically loaded evaluator class
+
+        Raises:
+            ValueError: If file or class cannot be loaded, or if the class is not a BaseEvaluator subclass
+        """
+        file_path = Path(file_path_str)
+        if not file_path.is_absolute():
+            if not file_path.exists():
+                file_path = (
+                    Path.cwd() / "evals" / "evaluators" / "custom" / file_path_str
+                )
+
+        if not file_path.exists():
+            raise ValueError(
+                f"Evaluator file not found: {file_path}. "
+                f"Make sure the file exists in evals/evaluators/custom/"
+            )
+
+        module_name = f"_custom_evaluator_{file_path.stem}_{id(data)}"
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        if spec is None or spec.loader is None:
+            raise ValueError(f"Could not load module from {file_path}")
+
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        try:
+            spec.loader.exec_module(module)
+        except Exception as e:
+            raise ValueError(
+                f"Error executing module from {file_path}: {str(e)}"
+            ) from e
+
+        # Get the class from the module
+        if not hasattr(module, class_name):
+            raise ValueError(
+                f"Class '{class_name}' not found in {file_path}. "
+                f"Available classes: {[name for name in dir(module) if not name.startswith('_')]}"
+            )
+
+        evaluator_class = getattr(module, class_name)
+
+        if not isinstance(evaluator_class, type) or not issubclass(
+            evaluator_class, BaseEvaluator
+        ):
+            raise ValueError(
+                f"Class '{class_name}' must be a subclass of BaseEvaluator"
+            )
+
+        evaluator_id = data.get("id")
+        if not evaluator_id or not isinstance(evaluator_id, str):
+            raise ValueError("Evaluator 'id' must be a non-empty string")
+        return evaluator_class(
+            id=evaluator_id,
+            config=data.get("evaluatorConfig", {}),
+        )  # type: ignore
+
+    @staticmethod
+    def _create_exact_match_evaluator(
+        data: Dict[str, Any],
+    ) -> ExactMatchEvaluator:
+        return TypeAdapter(ExactMatchEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_json_similarity_evaluator(
+        data: Dict[str, Any],
+    ) -> JsonSimilarityEvaluator:
+        return TypeAdapter(JsonSimilarityEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_llm_judge_output_evaluator(
+        data: Dict[str, Any],
+    ) -> LLMJudgeOutputEvaluator:
+        return TypeAdapter(LLMJudgeOutputEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_llm_judge_strict_json_similarity_output_evaluator(
+        data: Dict[str, Any],
+    ) -> LLMJudgeStrictJSONSimilarityOutputEvaluator:
+        return TypeAdapter(LLMJudgeStrictJSONSimilarityOutputEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_trajectory_evaluator(
+        data: Dict[str, Any],
+    ) -> LLMJudgeTrajectoryEvaluator:
+        return TypeAdapter(LLMJudgeTrajectoryEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_tool_call_args_evaluator(
+        data: Dict[str, Any],
+    ) -> ToolCallArgsEvaluator:
+        return TypeAdapter(ToolCallArgsEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_tool_call_count_evaluator(
+        data: Dict[str, Any],
+    ) -> ToolCallCountEvaluator:
+        return TypeAdapter(ToolCallCountEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_tool_call_order_evaluator(
+        data: Dict[str, Any],
+    ) -> ToolCallOrderEvaluator:
+        return TypeAdapter(ToolCallOrderEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_tool_call_output_evaluator(
+        data: Dict[str, Any],
+    ) -> ToolCallOutputEvaluator:
+        return TypeAdapter(ToolCallOutputEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_llm_judge_simulation_trajectory_evaluator(
+        data: Dict[str, Any],
+    ) -> LLMJudgeTrajectorySimulationEvaluator:
+        return TypeAdapter(LLMJudgeTrajectorySimulationEvaluator).validate_python(
+            {
+                "id": data.get("id"),
+                "config": data.get("evaluatorConfig"),
+            }
+        )
+
+    @staticmethod
+    def _create_legacy_evaluator_internal(
+        data: Dict[str, Any],
+    ) -> LegacyBaseEvaluator[Any]:
         """Create an evaluator instance from configuration data.
 
         Args:
@@ -35,46 +328,38 @@ class EvaluatorFactory:
         Raises:
             ValueError: If category is unknown or required fields are missing
         """
-        # Extract common fields
-        name = data.get("name", "")
-        if not name:
-            raise ValueError("Evaluator configuration must include 'name' field")
-        id = data.get("id", "")
-        if not id:
-            raise ValueError("Evaluator configuration must include 'id' field")
-
-        params: EvaluatorBaseParams = TypeAdapter(Evaluator).validate_python(data)
+        params: EvaluatorBaseParams = TypeAdapter(LegacyEvaluator).validate_python(data)
 
         match params:
             case EqualsEvaluatorParams():
-                return EvaluatorFactory._create_exact_match_evaluator(params)
+                return EvaluatorFactory._create_legacy_exact_match_evaluator(params)
             case JsonSimilarityEvaluatorParams():
-                return EvaluatorFactory._create_json_similarity_evaluator(params)
+                return EvaluatorFactory._create_legacy_json_similarity_evaluator(params)
             case LLMEvaluatorParams():
-                return EvaluatorFactory._create_llm_as_judge_evaluator(params)
+                return EvaluatorFactory._create_legacy_llm_as_judge_evaluator(params)
             case TrajectoryEvaluatorParams():
-                return EvaluatorFactory._create_trajectory_evaluator(params)
+                return EvaluatorFactory._create_legacy_trajectory_evaluator(params)
             case _:
                 raise ValueError(f"Unknown evaluator category: {params}")
 
     @staticmethod
-    def _create_exact_match_evaluator(
+    def _create_legacy_exact_match_evaluator(
         params: EqualsEvaluatorParams,
-    ) -> ExactMatchEvaluator:
+    ) -> LegacyExactMatchEvaluator:
         """Create a deterministic evaluator."""
-        return ExactMatchEvaluator(**params.model_dump())
+        return LegacyExactMatchEvaluator(**params.model_dump())
 
     @staticmethod
-    def _create_json_similarity_evaluator(
+    def _create_legacy_json_similarity_evaluator(
         params: JsonSimilarityEvaluatorParams,
-    ) -> JsonSimilarityEvaluator:
+    ) -> LegacyJsonSimilarityEvaluator:
         """Create a deterministic evaluator."""
-        return JsonSimilarityEvaluator(**params.model_dump())
+        return LegacyJsonSimilarityEvaluator(**params.model_dump())
 
     @staticmethod
-    def _create_llm_as_judge_evaluator(
+    def _create_legacy_llm_as_judge_evaluator(
         params: LLMEvaluatorParams,
-    ) -> LlmAsAJudgeEvaluator:
+    ) -> LegacyLlmAsAJudgeEvaluator:
         """Create an LLM-as-a-judge evaluator."""
         if not params.prompt:
             raise ValueError("LLM evaluator must include 'prompt' field")
@@ -86,12 +371,12 @@ class EvaluatorFactory:
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return LlmAsAJudgeEvaluator(**params.model_dump())
+        return LegacyLlmAsAJudgeEvaluator(**params.model_dump())
 
     @staticmethod
-    def _create_trajectory_evaluator(
+    def _create_legacy_trajectory_evaluator(
         params: TrajectoryEvaluatorParams,
-    ) -> TrajectoryEvaluator:
+    ) -> LegacyTrajectoryEvaluator:
         """Create a trajectory evaluator."""
         if not params.prompt:
             raise ValueError("Trajectory evaluator must include 'prompt' field")
@@ -103,4 +388,4 @@ class EvaluatorFactory:
                 "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
             )
 
-        return TrajectoryEvaluator(**params.model_dump())
+        return LegacyTrajectoryEvaluator(**params.model_dump())

uipath/_cli/_evals/_helpers.py

@@ -0,0 +1,194 @@
+# type: ignore
+import ast
+import importlib.util
+import json
+import logging
+import re
+import sys
+from pathlib import Path
+from typing import Any, Optional
+
+import click
+
+from uipath._cli._utils._console import ConsoleLogger
+from uipath._utils.constants import CUSTOM_EVALUATOR_PREFIX
+
+logger = logging.getLogger(__name__)
+console = ConsoleLogger().get_instance()
+
+
+def try_extract_file_and_class_name(text: str) -> tuple[bool, str, str]:
+    if text.startswith(CUSTOM_EVALUATOR_PREFIX):
+        file_and_class = text[len(CUSTOM_EVALUATOR_PREFIX) :]
+        if ":" not in file_and_class:
+            raise ValueError(
+                f"evaluatorSchema must include class name after ':' - got: {text}"
+            )
+        file_path_str, class_name = file_and_class.rsplit(":", 1)
+
+        return True, file_path_str, class_name
+    return False, "", ""
+
+
+def to_kebab_case(text: str) -> str:
+    return re.sub(r"(?<!^)(?=[A-Z])", "-", text).lower()
+
+
+def find_evaluator_file(filename: str) -> Optional[Path]:
+    """Find the evaluator file in evals/evaluators/custom folder."""
+    custom_evaluators_path = Path.cwd() / "evals" / "evaluators" / "custom"
+
+    if not custom_evaluators_path.exists():
+        return None
+
+    file_path = custom_evaluators_path / filename
+    if file_path.exists():
+        return file_path
+
+    return None
+
+
+def find_base_evaluator_class(file_path: Path) -> Optional[str]:
+    """Parse the Python file and find the class that inherits from BaseEvaluator."""
+    try:
+        with open(file_path, "r") as f:
+            tree = ast.parse(f.read(), filename=str(file_path))
+
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef):
+                for base in node.bases:
+                    if isinstance(base, ast.Name) and base.id == "BaseEvaluator":
+                        return node.name
+                    elif isinstance(base, ast.Subscript):
+                        if (
+                            isinstance(base.value, ast.Name)
+                            and base.value.id == "BaseEvaluator"
+                        ):
+                            return node.name
+
+        return None
+    except Exception as e:
+        logger.error(f"Error parsing file: {e}")
+        return None
+
+
+def load_evaluator_class(file_path: Path, class_name: str) -> Optional[type]:
+    """Dynamically load the evaluator class from the file."""
+    try:
+        parent_dir = str(file_path.parent)
+        if parent_dir not in sys.path:
+            sys.path.insert(0, parent_dir)
+
+        spec = importlib.util.spec_from_file_location("custom_evaluator", file_path)
+        if spec is None or spec.loader is None:
+            return None
+
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        if hasattr(module, class_name):
+            return getattr(module, class_name)
+
+        return None
+    except Exception as e:
+        logger.error(f"Error loading class: {e}")
+        return None
+    finally:
+        # Remove from sys.path
+        if parent_dir in sys.path:
+            sys.path.remove(parent_dir)
+
+
+def generate_evaluator_config(evaluator_class: type, class_name: str) -> dict[str, Any]:
+    """Generate the evaluator config from the class."""
+    try:
+        config_type = evaluator_class._extract_config_type()
+        config_instance = config_type()
+        config_dict = config_instance.model_dump(by_alias=True, exclude_none=False)
+
+        return config_dict
+    except Exception as e:
+        console.error(f"Error inferring evaluator config: {e}")
+
+
+def register_evaluator(filename: str) -> tuple[str, str]:
+    """Infers the schema and types of a custom evaluator.
+
+    Returns:
+        tuple[str, str]:
+            - The first string is the path to the python evaluator file.
+            - The second string is the evaluator type that corresponds to the schema file.
+    """
+    if not filename.endswith(".py"):
+        filename = filename + ".py"
+    file_path = find_evaluator_file(filename)
+    if file_path is None:
+        console.error(f"Could not find '{filename}' in evals/evaluators/custom folder")
+
+    relative_path = f"evals/evaluators/custom/{filename}"
+    console.info(
+        f"Found custom evaluator file: {click.style(relative_path, fg='cyan')}"
+    )
+
+    class_name = find_base_evaluator_class(file_path)
+    if class_name is None:
+        console.error(
+            f"Could not find a class inheriting from BaseEvaluator in {filename}"
+        )
+
+    console.info(f"Found custom evaluator class: {click.style(class_name, fg='cyan')}")
+
+    evaluator_class = load_evaluator_class(file_path, class_name)
+    if evaluator_class is None:
+        console.error(f"Could not load class {class_name} from {filename}")
+
+    try:
+        evaluator_id = evaluator_class.get_evaluator_id()
+    except Exception as e:
+        console.error(f"Error getting evaluator ID: {e}")
+
+    evaluator_config = generate_evaluator_config(evaluator_class, class_name)
+    evaluator_json_type = evaluator_class.generate_json_type()
+
+    evaluators_dir = Path.cwd() / "evals" / "evaluators"
+    evaluators_dir.mkdir(parents=True, exist_ok=True)
+
+    evaluator_types_dir = evaluators_dir / "custom" / "types"
+    evaluator_types_dir.mkdir(parents=True, exist_ok=True)
+
+    kebab_class_name = to_kebab_case(class_name)
+    output_file_evaluator_types = kebab_class_name + "-types.json"
+    evaluator_types_output_path = (
+        evaluators_dir / "custom" / "types" / output_file_evaluator_types
+    )
+
+    with open(evaluator_types_output_path, "w") as f:
+        json.dump(evaluator_json_type, f, indent=2)
+
+    relative_output_path = (
+        f"evals/evaluators/custom/types/{output_file_evaluator_types}"
+    )
+    console.success(
+        f"Generated evaluator types: {click.style(relative_output_path, fg='cyan')}"
+    )
+
+    output = {
+        "version": "1.0",
+        "id": evaluator_id,
+        "evaluatorTypeId": f"{CUSTOM_EVALUATOR_PREFIX}types/{output_file_evaluator_types}",
+        "evaluatorSchema": f"{CUSTOM_EVALUATOR_PREFIX}{filename}:{class_name}",
+        "description": evaluator_class.__doc__,
+        "evaluatorConfig": evaluator_config,
+    }
+
+    output_file_evaluator_spec = kebab_class_name + ".json"
+    evaluator_spec_output_path = evaluators_dir / output_file_evaluator_spec
+    with open(evaluator_spec_output_path, "w") as f:
+        json.dump(output, f, indent=2)
+
+    relative_output_path = f"evals/evaluators/{output_file_evaluator_spec}"
+    console.success(
+        f"Generated evaluator spec: {click.style(relative_output_path, fg='cyan')}"
+    )
+
+    return str(file_path), str(evaluator_types_output_path)