uipath 2.1.20__py3-none-any.whl → 2.1.21__py3-none-any.whl

uipath/_cli/_evals/_evaluators/_evaluator_factory.py

@@ -107,6 +107,10 @@ class EvaluatorFactory:
         model = data.get("model", "")
         if not model:
             raise ValueError("LLM evaluator must include 'model' field")
+        if model == "same-as-agent":
+            raise ValueError(
+                "'same-as-agent' model option is not supported by coded agents evaluations. Please select a specific model for the evaluator."
+            )
 
         return LlmAsAJudgeEvaluator.from_params(
             base_params,

uipath/_cli/_evals/_models/_evaluation_set.py

@@ -36,6 +36,16 @@ class EvaluationSet(BaseModel):
     createdAt: str
     updatedAt: str
 
+    def extract_selected_evals(self, eval_ids) -> None:
+        selected_evals: list[EvaluationItem] = []
+        for evaluation in self.evaluations:
+            if evaluation.id in eval_ids:
+                selected_evals.append(evaluation)
+                eval_ids.remove(evaluation.id)
+        if len(eval_ids) > 0:
+            raise ValueError("Unknown evaluation ids: {}".format(eval_ids))
+        self.evaluations = selected_evals
+
 
 class EvaluationStatus(IntEnum):
     PENDING = 0

uipath/_cli/_evals/evaluation_service.py

@@ -33,6 +33,7 @@ class EvaluationService:
         self,
         entrypoint: Optional[str] = None,
         eval_set_path: Optional[str | Path] = None,
+        eval_ids: Optional[List[str]] = None,
         workers: int = 8,
         report_progress: bool = True,
     ):
@@ -47,10 +48,10 @@ class EvaluationService:
         self.entrypoint, self.eval_set_path = self._resolve_paths(
             entrypoint, eval_set_path
         )
-        self.eval_set = self._load_eval_set()
+        self._eval_set = self._load_eval_set(eval_ids)
         self._evaluators = self._load_evaluators()
-        self.num_workers = workers
-        self.results_lock = asyncio.Lock()
+        self._num_workers = workers
+        self._results_lock = asyncio.Lock()
         self._progress_manager: Optional[EvaluationProgressManager] = None
         self._report_progress = report_progress
         self._progress_reporter: Optional[ProgressReporter] = None
@@ -169,9 +170,9 @@ class EvaluationService:
         if self._report_progress:
             agent_snapshot = self._extract_agent_snapshot()
             self._progress_reporter = ProgressReporter(
-                eval_set_id=self.eval_set.id,
+                eval_set_id=self._eval_set.id,
                 agent_snapshot=agent_snapshot,
-                no_of_evals=len(self.eval_set.evaluations),
+                no_of_evals=len(self._eval_set.evaluations),
                 evaluators=self._evaluators,
             )
 
@@ -215,12 +216,12 @@ class EvaluationService:
 
         # Create results file
         timestamp = datetime.now(timezone.utc).strftime("%M-%H-%d-%m-%Y")
-        eval_set_name = self.eval_set.name
+        eval_set_name = self._eval_set.name
         self.result_file = results_dir / f"eval-{eval_set_name}-{timestamp}.json"
 
         initial_results = EvaluationSetResult(
-            eval_set_id=self.eval_set.id,
-            eval_set_name=self.eval_set.name,
+            eval_set_id=self._eval_set.id,
+            eval_set_name=self._eval_set.name,
             results=[],
             average_score=0.0,
         )
@@ -228,7 +229,7 @@ class EvaluationService:
         with open(self.result_file, "w", encoding="utf-8") as f:
             f.write(initial_results.model_dump_json(indent=2))
 
-    def _load_eval_set(self) -> EvaluationSet:
+    def _load_eval_set(self, eval_ids: Optional[List[str]] = None) -> EvaluationSet:
         """Load the evaluation set from file.
 
         Returns:
@@ -236,13 +237,16 @@ class EvaluationService:
         """
         with open(self.eval_set_path, "r", encoding="utf-8") as f:
             data = json.load(f)
-            return EvaluationSet(**data)
+        eval_set = EvaluationSet(**data)
+        if eval_ids:
+            eval_set.extract_selected_evals(eval_ids)
+        return eval_set
 
     def _load_evaluators(self) -> List[EvaluatorBase]:
         """Load evaluators referenced by the evaluation set."""
         evaluators = []
         evaluators_dir = self.eval_set_path.parent.parent / "evaluators"
-        evaluator_refs = set(self.eval_set.evaluatorRefs)
+        evaluator_refs = set(self._eval_set.evaluatorRefs)
         found_evaluator_ids = set()
 
         # Load evaluators from JSON files
@@ -252,14 +256,9 @@ class EvaluationService:
                 evaluator_id = data.get("id")
 
                 if evaluator_id in evaluator_refs:
-                    try:
-                        evaluator = EvaluatorFactory.create_evaluator(data)
-                        evaluators.append(evaluator)
-                        found_evaluator_ids.add(evaluator_id)
-                    except Exception as e:
-                        console.warning(
-                            f"Failed to create evaluator {evaluator_id}: {str(e)}"
-                        )
+                    evaluator = EvaluatorFactory.create_evaluator(data)
+                    evaluators.append(evaluator)
+                    found_evaluator_ids.add(evaluator_id)
 
         # Check if all referenced evaluators were found
         missing_evaluators = evaluator_refs - found_evaluator_ids
@@ -276,7 +275,7 @@ class EvaluationService:
         Args:
             results: List of evaluation results to write
         """
-        async with self.results_lock:
+        async with self._results_lock:
             # Read current results
             with open(self.result_file, "r", encoding="utf-8") as f:
                 current_results = EvaluationSetResult.model_validate_json(f.read())
@@ -473,11 +472,11 @@ class EvaluationService:
         Args:
             task_queue: The asyncio queue to add tasks to
         """
-        for eval_item in self.eval_set.evaluations:
+        for eval_item in self._eval_set.evaluations:
             await task_queue.put(eval_item.model_dump())
 
         # Add sentinel values to signal workers to stop
-        for _ in range(self.num_workers):
+        for _ in range(self._num_workers):
             await task_queue.put(None)
 
     async def _consumer_task(
@@ -517,7 +516,7 @@ class EvaluationService:
     async def run_evaluation(self) -> None:
         """Run the evaluation set using multiple worker tasks."""
         console.info(
-            f"Starting evaluating {click.style(self.eval_set.name, fg='cyan')} evaluation set..."
+            f"Starting evaluating {click.style(self._eval_set.name, fg='cyan')} evaluation set..."
         )
 
         if self._report_progress and self._progress_reporter:
@@ -526,7 +525,7 @@ class EvaluationService:
         # Prepare items for progress tracker
         progress_items = [
             {"id": eval_item.id, "name": eval_item.name}
-            for eval_item in self.eval_set.evaluations
+            for eval_item in self._eval_set.evaluations
         ]
 
         with console.evaluation_progress(progress_items) as progress_manager:
@@ -539,7 +538,7 @@ class EvaluationService:
             producer = asyncio.create_task(self._producer_task(task_queue))
 
             consumers = []
-            for worker_id in range(self.num_workers):
+            for worker_id in range(self._num_workers):
                 consumer = asyncio.create_task(
                     self._consumer_task(
                         task_queue, worker_id, results_queue, sw_progress_reporter_queue

uipath/_cli/cli_eval.py

@@ -1,7 +1,8 @@
 # type: ignore
+import ast
 import asyncio
 import os
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
 
 import click
 from dotenv import load_dotenv
@@ -15,9 +16,18 @@ console = ConsoleLogger()
 load_dotenv(override=True)
 
 
+class LiteralOption(click.Option):
+    def type_cast_value(self, ctx, value):
+        try:
+            return ast.literal_eval(value)
+        except Exception as e:
+            raise click.BadParameter(value) from e
+
+
 def eval_agent(
     entrypoint: Optional[str] = None,
     eval_set: Optional[str] = None,
+    eval_ids: Optional[List[str]] = None,
     workers: int = 8,
     no_report: bool = False,
     **kwargs,
@@ -27,6 +37,7 @@ def eval_agent(
     Args:
         entrypoint: Path to the agent script to evaluate (optional, will auto-discover if not provided)
         eval_set: Path to the evaluation set JSON file (optional, will auto-discover if not provided)
+        eval_ids: Optional list of evaluation IDs
         workers: Number of parallel workers for running evaluations
         no_report: Do not report the evaluation results
         **kwargs: Additional arguments for future extensibility
@@ -41,8 +52,13 @@ def eval_agent(
         if workers < 1:
             return False, "Number of workers must be at least 1", None
 
+        print("EVAL SET")
+        print(eval_set)
+        if eval_set is not None and len(eval_set) == 0:
+            return False, "Evaluation set must not be empty", None
+
         service = EvaluationService(
-            entrypoint, eval_set, workers, report_progress=not no_report
+            entrypoint, eval_set, eval_ids, workers, report_progress=not no_report
         )
         asyncio.run(service.run_evaluation())
 
@@ -55,6 +71,7 @@ def eval_agent(
 @click.command()
 @click.argument("entrypoint", required=False)
 @click.argument("eval_set", required=False)
+@click.option("--eval-ids", cls=LiteralOption, default="[]")
 @click.option(
     "--no-report",
     is_flag=True,
@@ -69,20 +86,28 @@ def eval_agent(
 )
 @track(when=lambda *_a, **_kw: os.getenv(ENV_JOB_ID) is None)
 def eval(
-    entrypoint: Optional[str], eval_set: Optional[str], no_report: bool, workers: int
+    entrypoint: Optional[str],
+    eval_set: Optional[str],
+    eval_ids: List[str],
+    no_report: bool,
+    workers: int,
 ) -> None:
     """Run an evaluation set against the agent.
 
     Args:
         entrypoint: Path to the agent script to evaluate (optional, will auto-discover if not specified)
         eval_set: Path to the evaluation set JSON file (optional, will auto-discover if not specified)
+        eval_ids: Optional list of evaluation IDs
         workers: Number of parallel workers for running evaluations
         no_report: Do not report the evaluation results
     """
     success, error_message, info_message = eval_agent(
-        entrypoint=entrypoint, eval_set=eval_set, workers=workers, no_report=no_report
+        entrypoint=entrypoint,
+        eval_set=eval_set,
+        eval_ids=eval_ids,
+        workers=workers,
+        no_report=no_report,
     )
-
     if error_message:
         console.error(error_message)
         click.get_current_context().exit(1)

uipath/_cli/cli_invoke.py

@@ -70,7 +70,6 @@ def invoke(
         url = f"{base_url}/orchestrator_/odata/Jobs/UiPath.Server.Configuration.OData.StartJobs"
         _, personal_workspace_folder_id = get_personal_workspace_info(base_url, token)
         project_name, project_version = _read_project_details()
-
         if not personal_workspace_folder_id:
             console.error(
                 "No personal workspace found for user. Please try reauthenticating."

{uipath-2.1.20.dist-info → uipath-2.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.20
+Version: 2.1.21
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.20.dist-info → uipath-2.1.21.dist-info}/RECORD

@@ -8,9 +8,9 @@ uipath/_cli/README.md,sha256=GLtCfbeIKZKNnGTCsfSVqRQ27V1btT1i2bSAyW_xZl4,474
 uipath/_cli/__init__.py,sha256=oG0oTrb60qfIncJ0EcGsytBYxAVbepcBlOkqBKQlsJM,2104
 uipath/_cli/cli_auth.py,sha256=RUSBHfmqhBtITrx52FeXMlVCuNyo8vrjTdjEhmM1Khw,6734
 uipath/_cli/cli_deploy.py,sha256=KPCmQ0c_NYD5JofSDao5r6QYxHshVCRxlWDVnQvlp5w,645
-uipath/_cli/cli_eval.py,sha256=z0ER8pN5rJyINcSr1tM75HbSlmZXtx96YtqDvDI6zHk,2945
+uipath/_cli/cli_eval.py,sha256=INkfaZKadShtFOrVfTNM7K2kjXV-cwIqsOfIEYqDSGc,3656
 uipath/_cli/cli_init.py,sha256=jksza6bHfh4z1nKyEJBEEZlkO37yZoCz_FJWq_RPhWI,6093
-uipath/_cli/cli_invoke.py,sha256=FurosrZNGlmANIrplKWhw3EQ1b46ph5Z2rPwVaYJgmc,4001
+uipath/_cli/cli_invoke.py,sha256=4Oc6CM21Y24b_5I2MAqu-TffZL1aOKQwxfBmC6mPR8o,4000
 uipath/_cli/cli_new.py,sha256=9378NYUBc9j-qKVXV7oja-jahfJhXBg8zKVyaon7ctY,2102
 uipath/_cli/cli_pack.py,sha256=NmwZTfwZ2fURiHyiX1BM0juAtBOjPB1Jmcpu-rD7p-4,11025
 uipath/_cli/cli_publish.py,sha256=QT17JTClAyLve6ZjB-WvQaJ-j4DdmNneV_eDRyXjeeQ,6578
@@ -29,18 +29,18 @@ uipath/_cli/_auth/auth_config.json,sha256=UnAhdum8phjuZaZKE5KLp0IcPCbIltDEU1M_G8
 uipath/_cli/_auth/index.html,sha256=_Q2OtqPfapG_6vumbQYqtb2PfFe0smk7TlGERKEBvB4,22518
 uipath/_cli/_auth/localhost.crt,sha256=oGl9oLLOiouHubAt39B4zEfylFvKEtbtr_43SIliXJc,1226
 uipath/_cli/_auth/localhost.key,sha256=X31VYXD8scZtmGA837dGX5l6G-LXHLo5ItWJhZXaz3c,1679
-uipath/_cli/_evals/evaluation_service.py,sha256=VVxZxoCJoB2SUhej_c0DzC9AlnIlWMKnug7z5weNSoE,22077
+uipath/_cli/_evals/evaluation_service.py,sha256=zqYRB-tZpTTFqMctjIpEli3joIlmrz3dCVZsxekxIps,22053
 uipath/_cli/_evals/progress_reporter.py,sha256=m1Dio1vG-04nFTFz5ijM_j1dhudlgOzQukmTkkg6wS4,11490
 uipath/_cli/_evals/_evaluators/__init__.py,sha256=jD7KNLjbsUpsESFXX11eW2MEPXDNuPp2-t-IPB-inlM,734
 uipath/_cli/_evals/_evaluators/_deterministic_evaluator_base.py,sha256=BTl0puBjp9iCsU3YFfYWqk4TOz4iE19O3q1-dK6qUOI,1723
 uipath/_cli/_evals/_evaluators/_evaluator_base.py,sha256=knHUwYFt0gMG1uJhq5TGEab6M_YevxX019yT3yYwZsw,3787
-uipath/_cli/_evals/_evaluators/_evaluator_factory.py,sha256=RJtCuFREZ8Ijlldpa0521poZLmcR7vTU3WyYOmhJOkc,4688
+uipath/_cli/_evals/_evaluators/_evaluator_factory.py,sha256=cURShn17X6BW-_G3rknJXWtlgpeh5UdioLUV6oGCGAU,4912
 uipath/_cli/_evals/_evaluators/_exact_match_evaluator.py,sha256=lvEtAitrZy9myoZLMXLqlBWBPX06Msu67kuFMGSbikM,1319
 uipath/_cli/_evals/_evaluators/_json_similarity_evaluator.py,sha256=HpmkvuwU4Az3IIqFVLUmDvzkqb21pFMxY0sg2biZOMM,7093
 uipath/_cli/_evals/_evaluators/_llm_as_judge_evaluator.py,sha256=nSLZ29xWqALEI53ifr79JPXjyx0T4sr7p-4NygwgAio,6594
 uipath/_cli/_evals/_evaluators/_trajectory_evaluator.py,sha256=dnogQTOskpI4_cNF0Ge3hBceJJocvOgxBWAwaCWnzB0,1595
 uipath/_cli/_evals/_models/__init__.py,sha256=Ewjp3u2YeTH2MmzY9LWf7EIbAoIf_nW9fMYbj7pGlPs,420
-uipath/_cli/_evals/_models/_evaluation_set.py,sha256=UIapFwn_Ti9zHUIcL3xyHDcLZ4lq4sHJ3JXLvY5OYI0,1080
+uipath/_cli/_evals/_models/_evaluation_set.py,sha256=tVHykSget-G3sOCs9bSchMYUTpFqzXVlYYbY8L9SI0c,1518
 uipath/_cli/_evals/_models/_evaluators.py,sha256=l57NEVyYmzSKuoIXuGkE94Br01hAMg35fiS2MlTkaQM,2115
 uipath/_cli/_push/sw_file_handler.py,sha256=tRE9n68xv0r20ulwOyALHtYwzbjGneiASwzNm8xtBN0,16372
 uipath/_cli/_runtime/_contracts.py,sha256=WlpaiQAMWCo-JFHjee35Klf49A3GsKjOU1Mf2IpUGHY,16033
@@ -115,8 +115,8 @@ uipath/tracing/_traced.py,sha256=qeVDrds2OUnpdUIA0RhtF0kg2dlAZhyC1RRkI-qivTM,185
 uipath/tracing/_utils.py,sha256=ZeensQexnw69jVcsVrGyED7mPlAU-L1agDGm6_1A3oc,10388
 uipath/utils/__init__.py,sha256=VD-KXFpF_oWexFg6zyiWMkxl2HM4hYJMIUDZ1UEtGx0,105
 uipath/utils/_endpoints_manager.py,sha256=iRTl5Q0XAm_YgcnMcJOXtj-8052sr6jpWuPNz6CgT0Q,8408
-uipath-2.1.20.dist-info/METADATA,sha256=CMJ4e0Xkauc6EZhmTQW6kVrQ3mnhWjZte2atCM4JyyE,6367
-uipath-2.1.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-uipath-2.1.20.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
-uipath-2.1.20.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
-uipath-2.1.20.dist-info/RECORD,,
+uipath-2.1.21.dist-info/METADATA,sha256=O_9m-ZcdrpDqGwxFfOtVQDN9TvtCtuaCB33jpMuzLnY,6367
+uipath-2.1.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+uipath-2.1.21.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
+uipath-2.1.21.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
+uipath-2.1.21.dist-info/RECORD,,