uipath 2.1.110__py3-none-any.whl → 2.1.111__py3-none-any.whl

uipath/_cli/_evals/_models/_evaluation_set.py

@@ -142,6 +142,10 @@ class LegacyEvaluationItem(BaseModel):
         default=None,
         alias="mockingStrategy",
     )
+    input_mocking_strategy: Optional[InputMockingStrategy] = Field(
+        default=None,
+        alias="inputMockingStrategy",
+    )
 
 
 class EvaluationSet(BaseModel):

uipath/_cli/_evals/_runtime.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import logging
+import os
 import uuid
 from collections import defaultdict
 from pathlib import Path
@@ -186,6 +187,13 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         if self.context.eval_set is None:
             raise ValueError("eval_set must be provided for evaluation runs")
 
+        # Get entrypoint from a temporary runtime
+        temp_context = self.factory.new_context(
+            entrypoint=self.context.entrypoint, runtime_dir=os.getcwd()
+        )
+        temp_runtime = self.factory.from_context(temp_context)
+        self.entrypoint = await temp_runtime.get_entrypoint()
+
         event_bus = self.event_bus
 
         # Load eval set (path is already resolved in cli_eval.py)
@@ -330,6 +338,10 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         evaluators: List[AnyEvaluator],
         event_bus: EventBus,
     ) -> EvaluationRunResult:
+        # Generate LLM-based input if input_mocking_strategy is defined
+        if eval_item.input_mocking_strategy:
+            eval_item = await self._generate_input_for_eval(eval_item)
+
         execution_id = str(uuid.uuid4())
 
         set_execution_context(eval_item, self.span_collector, execution_id)
@@ -462,12 +474,10 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
         return evaluation_run_results
 
     async def _generate_input_for_eval(
-        self, eval_item: EvaluationItem
-    ) -> EvaluationItem:
+        self, eval_item: AnyEvaluationItem
+    ) -> AnyEvaluationItem:
         """Use LLM to generate a mock input for an evaluation item."""
-        # TODO(bai): get the input schema from agent definition, once it is available there.
-        input_schema: dict[str, Any] = {}
-        generated_input = await generate_llm_input(eval_item, input_schema)
+        generated_input = await generate_llm_input(eval_item, self.entrypoint.input)
         updated_eval_item = eval_item.model_copy(update={"inputs": generated_input})
         return updated_eval_item
 

uipath/_cli/_evals/mocks/input_mocker.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from typing import Any, Dict
 
 from uipath import UiPath
-from uipath._cli._evals._models._evaluation_set import EvaluationItem
+from uipath._cli._evals._models._evaluation_set import AnyEvaluationItem
 from uipath.tracing._traced import traced
 
 from .mocker import UiPathInputMockingError
@@ -54,20 +54,30 @@ OUTPUT: ONLY the simulated agent input in the exact format of the INPUT_SCHEMA i
 
 @traced(name="__mocker__")
 async def generate_llm_input(
-    evaluation_item: EvaluationItem,
+    evaluation_item: AnyEvaluationItem,
     input_schema: Dict[str, Any],
 ) -> Dict[str, Any]:
     """Generate synthetic input using an LLM based on the evaluation context."""
     try:
         llm = UiPath().llm
 
+        # Ensure additionalProperties is set for strict mode compatibility
+        if "additionalProperties" not in input_schema:
+            input_schema["additionalProperties"] = False
+
+        expected_output = (
+            getattr(evaluation_item, "evaluation_criterias", None)
+            or getattr(evaluation_item, "expected_output", None)
+            or {}
+        )
+
         prompt = get_input_mocking_prompt(
             input_schema=json.dumps(input_schema, indent=2),
             input_generation_instructions=evaluation_item.input_mocking_strategy.prompt
             if evaluation_item.input_mocking_strategy
             else "",
             expected_behavior=evaluation_item.expected_agent_behavior or "",
-            expected_output=json.dumps(evaluation_item.evaluation_criterias, indent=2),
+            expected_output=json.dumps(expected_output, indent=2),
         )
 
         response_format = {

{uipath-2.1.110.dist-info → uipath-2.1.111.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.110
+Version: 2.1.111
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.110.dist-info → uipath-2.1.111.dist-info}/RECORD

@@ -54,9 +54,9 @@ uipath/_cli/_evals/_evaluate.py,sha256=yRVhZ6uV58EV5Fv5X_K6425ZGsseQslnLe6FpIKy-
 uipath/_cli/_evals/_evaluator_factory.py,sha256=gPF9fRMZBOUPnJSM1fzQyXGHMGYQw_0VmHv-JOGbZf4,14348
 uipath/_cli/_evals/_helpers.py,sha256=dYHgkWxy2fOuqqZDtOKWKsZ1Ri4dn8qMnuB6DE-1MUk,6661
 uipath/_cli/_evals/_progress_reporter.py,sha256=CinS0S7vqHDyEp7cU87eARebEvQWwkX7H7fanSqIHxo,27385
-uipath/_cli/_evals/_runtime.py,sha256=ymRWmHlDfHmgcAgvjehTTYd7q3UhUwM6RDB43LhAGQQ,23037
+uipath/_cli/_evals/_runtime.py,sha256=t2owFdxDPUaZsOhy4_zx-BrMfClsdLgaVaplAMatM8Q,23423
 uipath/_cli/_evals/_span_collection.py,sha256=RoKoeDFG2XODdlgI27ionCjU7LLD_C0LJJ3gu0wab10,779
-uipath/_cli/_evals/_models/_evaluation_set.py,sha256=8mkAL3BvvGIYz8VrVczVq-QlNcCFPUnCC13Kj-gtjwk,7136
+uipath/_cli/_evals/_models/_evaluation_set.py,sha256=7P6zIkgerGKHXL6rD1YHXFFWpyxCUpNu7AX71bAaNoE,7270
 uipath/_cli/_evals/_models/_evaluator.py,sha256=UXrN103gHJFw3MtVWlGwViQWAo2cICRR-n357zL6wTA,9369
 uipath/_cli/_evals/_models/_evaluator_base_params.py,sha256=8i7Ir70IjaNOINTHMTXVXsKB4koYf3BCR8Vh2cyrBQI,406
 uipath/_cli/_evals/_models/_exceptions.py,sha256=-oXLTDa4ab9Boa34ZxuUrCezf8ajIGrIEUVwZnmBASE,195
@@ -64,7 +64,7 @@ uipath/_cli/_evals/_models/_mocks.py,sha256=mlD9qvdZNniuKxzY_ttJtwLVFvKGvvIukYvy
 uipath/_cli/_evals/_models/_output.py,sha256=AuqmSoyMuoN79UVR8J8OQ92Pqik0P_l2fegxEV-lSB0,7026
 uipath/_cli/_evals/_models/_sw_reporting.py,sha256=tSBLQFAdOIun8eP0vsqt56K6bmCZz_uMaWI3hskg_24,536
 uipath/_cli/_evals/mocks/__init__.py,sha256=2WXwAy_oZw5bKp6L0HB13QygCJeftOB_Bget0AI6Gik,32
-uipath/_cli/_evals/mocks/input_mocker.py,sha256=z8vcgg2Sq-Obash4l3Jph10b74kKl59vz125Z3WVBtQ,4088
+uipath/_cli/_evals/mocks/input_mocker.py,sha256=DtI5mqLKlg_QAvoPXUUbsLX5ptvPaA8MM8v7CPOWd_M,4450
 uipath/_cli/_evals/mocks/llm_mocker.py,sha256=p9TSl2awjgVorlJcghynFhhHNl-KsLaPPvsKSwFZVt8,7523
 uipath/_cli/_evals/mocks/mocker.py,sha256=VXCxuRAPqUK40kRUXpPmXZRckd7GrOY5ZzIYDe-NNfo,910
 uipath/_cli/_evals/mocks/mocker_factory.py,sha256=A8XzDhJRKddUrGWmtp2FABd6d86VFdzAl0rG4_OD5ss,817
@@ -224,8 +224,8 @@ uipath/tracing/_utils.py,sha256=zMjiKjNpSN3YQNEU4-u5AAvPtUsi8QuEqNLya89jfAU,1446
 uipath/utils/__init__.py,sha256=VD-KXFpF_oWexFg6zyiWMkxl2HM4hYJMIUDZ1UEtGx0,105
 uipath/utils/_endpoints_manager.py,sha256=tnF_FiCx8qI2XaJDQgYkMN_gl9V0VqNR1uX7iawuLp8,8230
 uipath/utils/dynamic_schema.py,sha256=w0u_54MoeIAB-mf3GmwX1A_X8_HDrRy6p998PvX9evY,3839
-uipath-2.1.110.dist-info/METADATA,sha256=kECcmHAYumtR_e3ktndmcdtaQbiNuacXEn9AJ9p_93I,6626
-uipath-2.1.110.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-uipath-2.1.110.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
-uipath-2.1.110.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
-uipath-2.1.110.dist-info/RECORD,,
+uipath-2.1.111.dist-info/METADATA,sha256=VJa6HaV9Xw0fWJy4saCYS5viF9gUsYDItjo_5uXmais,6626
+uipath-2.1.111.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+uipath-2.1.111.dist-info/entry_points.txt,sha256=9C2_29U6Oq1ExFu7usihR-dnfIVNSKc-0EFbh0rskB4,43
+uipath-2.1.111.dist-info/licenses/LICENSE,sha256=-KBavWXepyDjimmzH5fVAsi-6jNVpIKFc2kZs0Ri4ng,1058
+uipath-2.1.111.dist-info/RECORD,,