uipath 2.1.20__tar.gz → 2.1.21__tar.gz

{uipath-2.1.20 → uipath-2.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.20
+Version: 2.1.21
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.20 → uipath-2.1.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.20"
+version = "2.1.21"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

{uipath-2.1.20 → uipath-2.1.21}/src/uipath/_cli/_evals/_evaluators/_evaluator_factory.py

@@ -107,6 +107,10 @@ class EvaluatorFactory:
         model = data.get("model", "")
         if not model:
             raise ValueError("LLM evaluator must include 'model' field")
+        if model == "same-as-agent":
+            raise ValueError(
+                "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
+            )
 
         return LlmAsAJudgeEvaluator.from_params(
             base_params,

{uipath-2.1.20 → uipath-2.1.21}/src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -36,6 +36,16 @@ class EvaluationSet(BaseModel):
     createdAt: str
     updatedAt: str
 
+    def extract_selected_evals(self, eval_ids) -> None:
+        selected_evals: list[EvaluationItem] = []
+        for evaluation in self.evaluations:
+            if evaluation.id in eval_ids:
+                selected_evals.append(evaluation)
+                eval_ids.remove(evaluation.id)
+        if len(eval_ids) > 0:
+            raise ValueError("Unknown evaluation ids: {}".format(eval_ids))
+        self.evaluations = selected_evals
+
 
 class EvaluationStatus(IntEnum):
     PENDING = 0

{uipath-2.1.20 → uipath-2.1.21}/src/uipath/_cli/_evals/evaluation_service.py

@@ -33,6 +33,7 @@ class EvaluationService:
         self,
         entrypoint: Optional[str] = None,
         eval_set_path: Optional[str | Path] = None,
+        eval_ids: Optional[List[str]] = None,
         workers: int = 8,
         report_progress: bool = True,
     ):
@@ -47,10 +48,10 @@ class EvaluationService:
         self.entrypoint, self.eval_set_path = self._resolve_paths(
             entrypoint, eval_set_path
         )
-        self.eval_set = self._load_eval_set()
+        self._eval_set = self._load_eval_set(eval_ids)
         self._evaluators = self._load_evaluators()
-        self.num_workers = workers
-        self.results_lock = asyncio.Lock()
+        self._num_workers = workers
+        self._results_lock = asyncio.Lock()
         self._progress_manager: Optional[EvaluationProgressManager] = None
         self._report_progress = report_progress
         self._progress_reporter: Optional[ProgressReporter] = None
@@ -169,9 +170,9 @@ class EvaluationService:
         if self._report_progress:
             agent_snapshot = self._extract_agent_snapshot()
             self._progress_reporter = ProgressReporter(
-                eval_set_id=self.eval_set.id,
+                eval_set_id=self._eval_set.id,
                 agent_snapshot=agent_snapshot,
-                no_of_evals=len(self.eval_set.evaluations),
+                no_of_evals=len(self._eval_set.evaluations),
                 evaluators=self._evaluators,
             )
 
@@ -215,12 +216,12 @@ class EvaluationService:
 
         # Create results file
         timestamp = datetime.now(timezone.utc).strftime("%M-%H-%d-%m-%Y")
-        eval_set_name = self.eval_set.name
+        eval_set_name = self._eval_set.name
         self.result_file = results_dir / f"eval-{eval_set_name}-{timestamp}.json"
 
         initial_results = EvaluationSetResult(
-            eval_set_id=self.eval_set.id,
-            eval_set_name=self.eval_set.name,
+            eval_set_id=self._eval_set.id,
+            eval_set_name=self._eval_set.name,
             results=[],
             average_score=0.0,
         )
@@ -228,7 +229,7 @@ class EvaluationService:
         with open(self.result_file, "w", encoding="utf-8") as f:
             f.write(initial_results.model_dump_json(indent=2))
 
-    def _load_eval_set(self) -> EvaluationSet:
+    def _load_eval_set(self, eval_ids: Optional[List[str]] = None) -> EvaluationSet:
         """Load the evaluation set from file.
 
         Returns:
@@ -236,13 +237,16 @@ class EvaluationService:
         """
         with open(self.eval_set_path, "r", encoding="utf-8") as f:
             data = json.load(f)
-            return EvaluationSet(**data)
+        eval_set = EvaluationSet(**data)
+        if eval_ids:
+            eval_set.extract_selected_evals(eval_ids)
+        return eval_set
 
     def _load_evaluators(self) -> List[EvaluatorBase]:
         """Load evaluators referenced by the evaluation set."""
         evaluators = []
         evaluators_dir = self.eval_set_path.parent.parent / "evaluators"
-        evaluator_refs = set(self.eval_set.evaluatorRefs)
+        evaluator_refs = set(self._eval_set.evaluatorRefs)
         found_evaluator_ids = set()
 
         # Load evaluators from JSON files
@@ -252,14 +256,9 @@ class EvaluationService:
                 evaluator_id = data.get("id")
 
                 if evaluator_id in evaluator_refs:
-                    try:
-                        evaluator = EvaluatorFactory.create_evaluator(data)
-                        evaluators.append(evaluator)
-                        found_evaluator_ids.add(evaluator_id)
-                    except Exception as e:
-                        console.warning(
-                            f"Failed to create evaluator {evaluator_id}: {str(e)}"
-                        )
+                    evaluator = EvaluatorFactory.create_evaluator(data)
+                    evaluators.append(evaluator)
+                    found_evaluator_ids.add(evaluator_id)
 
         # Check if all referenced evaluators were found
         missing_evaluators = evaluator_refs - found_evaluator_ids
@@ -276,7 +275,7 @@ class EvaluationService:
         Args:
             results: List of evaluation results to write
         """
-        async with self.results_lock:
+        async with self._results_lock:
             # Read current results
             with open(self.result_file, "r", encoding="utf-8") as f:
                 current_results = EvaluationSetResult.model_validate_json(f.read())
@@ -473,11 +472,11 @@ class EvaluationService:
         Args:
             task_queue: The asyncio queue to add tasks to
         """
-        for eval_item in self.eval_set.evaluations:
+        for eval_item in self._eval_set.evaluations:
             await task_queue.put(eval_item.model_dump())
 
         # Add sentinel values to signal workers to stop
-        for _ in range(self.num_workers):
+        for _ in range(self._num_workers):
             await task_queue.put(None)
 
     async def _consumer_task(
@@ -517,7 +516,7 @@ class EvaluationService:
     async def run_evaluation(self) -> None:
         """Run the evaluation set using multiple worker tasks."""
         console.info(
-            f"Starting evaluating {click.style(self.eval_set.name, fg='cyan')} evaluation set..."
+            f"Starting evaluating {click.style(self._eval_set.name, fg='cyan')} evaluation set..."
         )
 
         if self._report_progress and self._progress_reporter:
@@ -526,7 +525,7 @@ class EvaluationService:
         # Prepare items for progress tracker
         progress_items = [
             {"id": eval_item.id, "name": eval_item.name}
-            for eval_item in self.eval_set.evaluations
+            for eval_item in self._eval_set.evaluations
         ]
 
         with console.evaluation_progress(progress_items) as progress_manager:
@@ -539,7 +538,7 @@ class EvaluationService:
             producer = asyncio.create_task(self._producer_task(task_queue))
 
             consumers = []
-            for worker_id in range(self.num_workers):
+            for worker_id in range(self._num_workers):
                 consumer = asyncio.create_task(
                     self._consumer_task(
                         task_queue, worker_id, results_queue, sw_progress_reporter_queue

{uipath-2.1.20 → uipath-2.1.21}/src/uipath/_cli/cli_eval.py

@@ -1,7 +1,8 @@
 # type: ignore
+import ast
 import asyncio
 import os
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
 
 import click
 from dotenv import load_dotenv
@@ -15,9 +16,18 @@ console = ConsoleLogger()
 load_dotenv(override=True)
 
 
+class LiteralOption(click.Option):
+    def type_cast_value(self, ctx, value):
+        try:
+            return ast.literal_eval(value)
+        except Exception as e:
+            raise click.BadParameter(value) from e
+
+
 def eval_agent(
     entrypoint: Optional[str] = None,
     eval_set: Optional[str] = None,
+    eval_ids: Optional[List[str]] = None,
     workers: int = 8,
     no_report: bool = False,
     **kwargs,
@@ -27,6 +37,7 @@ def eval_agent(
     Args:
         entrypoint: Path to the agent script to evaluate (optional, will auto-discover if not provided)
         eval_set: Path to the evaluation set JSON file (optional, will auto-discover if not provided)
+        eval_ids: Optional list of evaluation IDs
         workers: Number of parallel workers for running evaluations
         no_report: Do not report the evaluation results
         **kwargs: Additional arguments for future extensibility
@@ -41,8 +52,13 @@ def eval_agent(
         if workers < 1:
             return False, "Number of workers must be at least 1", None
 
+        print("EVAL SET")
+        print(eval_set)
+        if eval_set is not None and len(eval_set) == 0:
+            return False, "Evaluation set must not be empty", None
+
         service = EvaluationService(
-            entrypoint, eval_set, workers, report_progress=not no_report
+            entrypoint, eval_set, eval_ids, workers, report_progress=not no_report
         )
         asyncio.run(service.run_evaluation())
 
@@ -55,6 +71,7 @@ def eval_agent(
 @click.command()
 @click.argument("entrypoint", required=False)
 @click.argument("eval_set", required=False)
+@click.option("--eval-ids", cls=LiteralOption, default="[]")
 @click.option(
     "--no-report",
     is_flag=True,
@@ -69,20 +86,28 @@ def eval_agent(
 )
 @track(when=lambda *_a, **_kw: os.getenv(ENV_JOB_ID) is None)
 def eval(
-    entrypoint: Optional[str], eval_set: Optional[str], no_report: bool, workers: int
+    entrypoint: Optional[str],
+    eval_set: Optional[str],
+    eval_ids: List[str],
+    no_report: bool,
+    workers: int,
 ) -> None:
     """Run an evaluation set against the agent.
 
     Args:
         entrypoint: Path to the agent script to evaluate (optional, will auto-discover if not specified)
         eval_set: Path to the evaluation set JSON file (optional, will auto-discover if not specified)
+        eval_ids: Optional list of evaluation IDs
         workers: Number of parallel workers for running evaluations
         no_report: Do not report the evaluation results
     """
     success, error_message, info_message = eval_agent(
-        entrypoint=entrypoint, eval_set=eval_set, workers=workers, no_report=no_report
+        entrypoint=entrypoint,
+        eval_set=eval_set,
+        eval_ids=eval_ids,
+        workers=workers,
+        no_report=no_report,
     )
-
     if error_message:
         console.error(error_message)
         click.get_current_context().exit(1)

{uipath-2.1.20 → uipath-2.1.21}/src/uipath/_cli/cli_invoke.py

@@ -70,7 +70,6 @@ def invoke(
         url = f"{base_url}/orchestrator_/odata/Jobs/UiPath.Server.Configuration.OData.StartJobs"
         _, personal_workspace_folder_id = get_personal_workspace_info(base_url, token)
         project_name, project_version = _read_project_details()
-
         if not personal_workspace_folder_id:
             console.error(
                 "No personal workspace found for user. Please try reauthenticating."

{uipath-2.1.20 → uipath-2.1.21}/uv.lock

@@ -2029,7 +2029,7 @@ wheels = [
 
 [[package]]
 name = "uipath"
-version = "2.1.20"
+version = "2.1.21"
 source = { editable = "." }
 dependencies = [
     { name = "azure-monitor-opentelemetry" },