uipath 2.1.107__tar.gz → 2.1.109__tar.gz

{uipath-2.1.107 → uipath-2.1.109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uipath
-Version: 2.1.107
+Version: 2.1.109
 Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
 Project-URL: Homepage, https://uipath.com
 Project-URL: Repository, https://github.com/UiPath/uipath-python

{uipath-2.1.107 → uipath-2.1.109}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.107"
+version = "2.1.109"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

uipath-2.1.109/samples/calculator/README.md

@@ -0,0 +1,27 @@
+# Simple UiPath Coded Agents
+
+This is a simple, standalone Coded Agent which does not require external dependencies.
+
+After initialization, execute the agent using this sample command:
+```
+uipath run main.py '{"a": 0, "b": 1, "operator": "+"}'
+```
+
+# Run evaluations
+```
+uipath eval .\main.py .\evals\eval-sets\default.json --no-report --output-file output.json
+```
+
+# Add and register custom evaluator
+
+1. (Optional) Add a new evaluator -> can be created manually in the evals/custom-evaluators directory
+```
+uipath add evaluator my_custom_evaluator
+```
+2. Implement the logic
+
+3. Register the evaluator
+```
+uipath register evaluator my_custom_evaluator
+```
+4. Apply it to any dataset

uipath-2.1.109/samples/calculator/evals/eval-sets/default.json

@@ -0,0 +1,84 @@
+{
+  "version": "1.0",
+  "id": "NewSchemaSampleEval",
+  "name": "New Schema Sample Evaluation",
+  "evaluatorRefs": [
+    "ContainsEvaluator",
+    "ExactMatchEvaluator",
+    "JsonSimilarityEvaluator",
+    "LLMJudgeOutputEvaluator",
+    "LLMJudgeStrictJSONSimilarityOutputEvaluator",
+    "TrajectoryEvaluator",
+    "CorrectOperatorEvaluator"
+  ],
+  "evaluations": [
+    {
+      "id": "default",
+      "name": "Add",
+      "inputs": {
+        "a": 1,
+        "b": 4,
+        "operator": "+"
+      },
+      "evaluationCriterias": {
+        "ContainsEvaluator": null,
+        "ExactMatchEvaluator": null,
+        "JsonSimilarityEvaluator": null,
+        "LLMJudgeOutputEvaluator": null,
+        "LLMJudgeStrictJSONSimilarityOutputEvaluator": null,
+        "TrajectoryEvaluator": null,
+        "CorrectOperatorEvaluator": null
+      }
+    },
+    {
+      "id": "override",
+      "name": "Multiply",
+      "inputs": {
+        "a": 2,
+        "b": 4,
+        "operator": "*"
+      },
+      "evaluationCriterias": {
+        "ContainsEvaluator": {
+          "searchText": "8"
+        },
+        "CorrectOperatorEvaluator": {
+          "operator": "*"
+        },
+        "ExactMatchEvaluator": {
+          "expectedOutput": {
+            "result": "8.0"
+          }
+        },
+        "JsonSimilarityEvaluator": {
+          "expectedOutput": {
+            "result": 8.0
+          }
+        },
+        "LLMJudgeOutputEvaluator": {
+          "expectedOutput": {
+            "result": 8.0
+          }
+        },
+        "LLMJudgeStrictJSONSimilarityOutputEvaluator": {
+          "expectedOutput": {
+            "result": 8.0
+          }
+        },
+        "TrajectoryEvaluator": {
+          "expectedAgentBehavior": "The agent should correctly parse the multiply operation, perform the calculation 2 * 4, and return the result 8.0"
+        }
+      }
+    },
+    {
+      "id": "skip",
+      "name": "Skip denial code check",
+      "inputs": {
+        "a": 1,
+        "b": 1,
+        "operator": "+"
+      },
+      "evaluationCriterias": {}
+    }
+  ]
+}

uipath-2.1.107/samples/calculator/evals/eval-sets/default.json → uipath-2.1.109/samples/calculator/evals/eval-sets/legacy.json

@@ -5,14 +5,22 @@
   "batchSize": 10,
   "evaluatorRefs": [
     "equality",
-    "llm-as-a-judge"
+    "llm-as-a-judge",
+    "json-similarity",
+    "trajectory"
   ],
   "evaluations": [
     {
       "id": "test-addition",
       "name": "Test Addition",
-      "inputs": {"a": 1, "b": 1, "operator":  "+"},
-      "expectedOutput": {"result": 2},
+      "inputs": {
+        "a": 1,
+        "b": 1,
+        "operator": "+"
+      },
+      "expectedOutput": {
+        "result": 2
+      },
       "expectedAgentBehavior": "",
       "evalSetId": "default-eval-set-id",
       "createdAt": "2025-09-04T18:54:58.378Z",
@@ -21,8 +29,14 @@
     {
       "id": "test-random-addition-using-mockito",
       "name": "Test Random Addition Using Mockito",
-      "inputs": {"a": 1, "b": 1, "operator":  "random"},
-      "expectedOutput": {"result": 2},
+      "inputs": {
+        "a": 1,
+        "b": 1,
+        "operator": "random"
+      },
+      "expectedOutput": {
+        "result": 2
+      },
       "expectedAgentBehavior": "",
       "mockingStrategy": {
         "type": "mockito",
@@ -36,7 +50,9 @@
             "then": [
               {
                 "type": "return",
-                "value": {"result": "+"}
+                "value": {
+                  "result": "+"
+                }
               }
             ]
           }
@@ -49,13 +65,23 @@
     {
       "id": "test-random-addition-using-llm",
       "name": "Test Random Addition Using LLM",
-      "inputs": {"a": 1, "b": 1, "operator":  "random"},
-      "expectedOutput": {"result": 2},
+      "inputs": {
+        "a": 1,
+        "b": 1,
+        "operator": "random"
+      },
+      "expectedOutput": {
+        "result": 2
+      },
       "expectedAgentBehavior": "",
       "mockingStrategy": {
         "type": "llm",
         "prompt": "The random operator is '+'.",
-        "toolsToSimulate": [{"name":  "get_random_operator"}],
+        "toolsToSimulate": [
+          {
+            "name": "get_random_operator"
+          }
+        ],
         "model": {
           "model": "gpt-4o-mini-2024-07-18",
           "temperature": 0

uipath-2.1.109/samples/calculator/evals/evaluators/contains.json

@@ -0,0 +1,15 @@
+{
+  "version": "1.0",
+  "id": "ContainsEvaluator",
+  "description": "Checks if the response text includes the expected denial code.",
+  "evaluatorTypeId": "uipath-contains",
+  "evaluatorConfig": {
+    "name": "ContainsEvaluator",
+    "targetOutputKey": "result",
+    "negated": false,
+    "ignoreCase": false,
+    "defaultEvaluationCriteria": {
+      "searchText": "5"
+    }
+  }
+}

uipath-2.1.109/samples/calculator/evals/evaluators/correct-operator-evaluator.json

@@ -0,0 +1,14 @@
+{
+  "version": "1.0",
+  "id": "CorrectOperatorEvaluator",
+  "evaluatorTypeId": "file://types/correct-operator-evaluator-types.json",
+  "evaluatorSchema": "file://correct_operator.py:CorrectOperatorEvaluator",
+  "description": "A custom evaluator that checks if the correct operator is being used by the agent ",
+  "evaluatorConfig": {
+    "name": "CorrectOperatorEvaluator",
+    "defaultEvaluationCriteria": {
+      "operator": "+"
+    },
+    "negated": false
+  }
+}

uipath-2.1.109/samples/calculator/evals/evaluators/custom/correct_operator.py

@@ -0,0 +1,46 @@
+import json
+
+from uipath.eval.evaluators import BaseEvaluator, BaseEvaluationCriteria, BaseEvaluatorConfig
+from uipath.eval.models import AgentExecution, EvaluationResult, NumericEvaluationResult
+from opentelemetry.sdk.trace import ReadableSpan
+
+class CorrectOperatorEvaluationCriteria(BaseEvaluationCriteria):
+    """Evaluation criteria for the contains evaluator."""
+
+    operator: str
+
+class CorrectOperatorEvaluatorConfig(BaseEvaluatorConfig[CorrectOperatorEvaluationCriteria]):
+    """Configuration for the contains evaluator."""
+
+    name: str = "CorrectOperatorEvaluator"
+    negated: bool = False
+    default_evaluation_criteria: CorrectOperatorEvaluationCriteria = CorrectOperatorEvaluationCriteria(operator="+")
+
+class CorrectOperatorEvaluator(BaseEvaluator[CorrectOperatorEvaluationCriteria, CorrectOperatorEvaluatorConfig, type(None)]):
+    """A custom evaluator that checks if the correct operator is being used by the agent """
+
+    def extract_operator_from_spans(self, agent_trace: list[ReadableSpan]) -> str:
+        for span in agent_trace:
+            if span.name == "track_operator":
+                if span.attributes:
+                    input_value_as_str = span.attributes.get("input.value", "{}")
+                    assert isinstance(input_value_as_str, str)
+                    input_value = json.loads(input_value_as_str)
+                    return input_value.get("operator")
+        raise Exception(f"No 'track_operator' span found")
+
+
+    @classmethod
+    def get_evaluator_id(cls) -> str:
+        return "CorrectOperatorEvaluator"
+
+
+    async def evaluate(self, agent_execution: AgentExecution, evaluation_criteria: CorrectOperatorEvaluationCriteria) -> EvaluationResult:
+        actual_operator = self.extract_operator_from_spans(agent_execution.agent_trace)
+        print(actual_operator)
+        is_expected_operator = evaluation_criteria.operator == actual_operator
+        if self.evaluator_config.negated:
+            is_expected_operator = not is_expected_operator
+        return NumericEvaluationResult(
+            score=float(is_expected_operator),
+        )

uipath-2.1.109/samples/calculator/evals/evaluators/custom/types/correct-operator-evaluator-types.json

@@ -0,0 +1,57 @@
+{
+  "evaluatorTypeId": "CorrectOperatorEvaluator",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "CorrectOperatorEvaluationCriteria": {
+        "description": "Evaluation criteria for the contains evaluator.",
+        "properties": {
+          "operator": {
+            "title": "Operator",
+            "type": "string"
+          }
+        },
+        "required": [
+          "operator"
+        ],
+        "title": "CorrectOperatorEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the contains evaluator.",
+    "properties": {
+      "name": {
+        "default": "CorrectOperatorEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "defaultEvaluationCriteria": {
+        "$ref": "#/$defs/CorrectOperatorEvaluationCriteria",
+        "default": {
+          "operator": "+"
+        }
+      },
+      "negated": {
+        "default": false,
+        "title": "Negated",
+        "type": "boolean"
+      }
+    },
+    "title": "CorrectOperatorEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Evaluation criteria for the contains evaluator.",
+    "properties": {
+      "operator": {
+        "title": "Operator",
+        "type": "string"
+      }
+    },
+    "required": [
+      "operator"
+    ],
+    "title": "CorrectOperatorEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {}
+}

uipath-2.1.109/samples/calculator/evals/evaluators/exact-match.json

@@ -0,0 +1,17 @@
+{
+  "version": "1.0",
+  "id": "ExactMatchEvaluator",
+  "description": "Checks if the response text exactly matches the expected value.",
+  "evaluatorTypeId": "uipath-exact-match",
+  "evaluatorConfig": {
+    "name": "ExactMatchEvaluator",
+    "targetOutputKey": "result",
+    "negated": false,
+    "ignoreCase": false,
+    "defaultEvaluationCriteria": {
+      "expectedOutput": {
+        "result": "5.0"
+      }
+    }
+  }
+}

uipath-2.1.109/samples/calculator/evals/evaluators/json-similarity.json

@@ -0,0 +1,15 @@
+{
+  "version": "1.0",
+  "id": "JsonSimilarityEvaluator",
+  "description": "Checks if the response JSON is similar to the expected JSON structure.",
+  "evaluatorTypeId": "uipath-json-similarity",
+  "evaluatorConfig": {
+    "name": "JsonSimilarityEvaluator",
+    "targetOutputKey": "*",
+    "defaultEvaluationCriteria": {
+      "expectedOutput": {
+        "result": 5.0
+      }
+    }
+  }
+}

uipath-2.1.109/samples/calculator/evals/evaluators/legacy-json-similarity.json

@@ -0,0 +1,11 @@
+{
+    "fileName": "json-similarity.json",
+    "id": "json-similarity",
+    "name": "JSON Similarity Evaluator",
+    "description": "An evaluator that compares JSON structures with tolerance for numeric and string differences.",
+    "category": 0,
+    "type": 6,
+    "targetOutputKey": "*",
+    "createdAt": "2025-06-26T17:45:39.651Z",
+    "updatedAt": "2025-06-26T17:45:39.651Z"
+}

uipath-2.1.109/samples/calculator/evals/evaluators/legacy-trajectory.json

@@ -0,0 +1,13 @@
+{
+  "fileName": "trajectory.json",
+  "id": "trajectory",
+  "name": "Trajectory Evaluator",
+  "description": "An evaluator that analyzes the execution trajectory and decision sequence taken by the agent.",
+  "category": 3,
+  "type": 7,
+  "prompt": "Evaluate the agent's execution trajectory based on the expected behavior.\n\nExpected Agent Behavior: {{ExpectedAgentBehavior}}\nAgent Run History: {{AgentRunHistory}}\n\nProvide a score from 0-100 based on how well the agent followed the expected trajectory.",
+  "model": "gpt-4o-mini",
+  "targetOutputKey": "*",
+  "createdAt": "2025-06-26T17:45:39.651Z",
+  "updatedAt": "2025-06-26T17:45:39.651Z"
+}

uipath-2.1.109/samples/calculator/evals/evaluators/llm-judge-semantic-similarity.json

@@ -0,0 +1,18 @@
+{
+  "version": "1.0",
+  "id": "LLMJudgeOutputEvaluator",
+  "description": "Uses an LLM to judge semantic similarity between expected and actual output.",
+  "evaluatorTypeId": "uipath-llm-judge-output-semantic-similarity",
+  "evaluatorConfig": {
+    "name": "LLMJudgeOutputEvaluator",
+    "targetOutputKey": "*",
+    "model": "gpt-4.1-2025-04-14",
+    "prompt": "Compare the following outputs and evaluate their semantic similarity.\n\nActual Output: {{ActualOutput}}\nExpected Output: {{ExpectedOutput}}\n\nProvide a score from 0-100 where 100 means semantically identical and 0 means completely different.",
+    "temperature": 0.0,
+    "defaultEvaluationCriteria": {
+      "expectedOutput": {
+        "result": 5.0
+      }
+    }
+  }
+}

uipath-2.1.109/samples/calculator/evals/evaluators/llm-judge-strict-json-similarity.json

@@ -0,0 +1,18 @@
+{
+  "version": "1.0",
+  "id": "LLMJudgeStrictJSONSimilarityOutputEvaluator",
+  "description": "Uses an LLM to judge strict JSON similarity between expected and actual output.",
+  "evaluatorTypeId": "uipath-llm-judge-output-strict-json-similarity",
+  "evaluatorConfig": {
+    "name": "LLMJudgeStrictJSONSimilarityOutputEvaluator",
+    "targetOutputKey": "*",
+    "model": "gpt-4.1-2025-04-14",
+    "prompt": "Compare the following JSON outputs for strict structural similarity.\n\nActual Output: {{ActualOutput}}\nExpected Output: {{ExpectedOutput}}\n\nEvaluate if the JSON structure and values match precisely. Provide a score from 0-100 where 100 means exact match and 0 means completely different.",
+    "temperature": 0.0,
+    "defaultEvaluationCriteria": {
+      "expectedOutput": {
+        "result": 5.0
+      }
+    }
+  }
+}

uipath-2.1.109/samples/calculator/evals/evaluators/trajectory.json

@@ -0,0 +1,15 @@
+{
+  "version": "1.0",
+  "id": "TrajectoryEvaluator",
+  "description": "Evaluates the agent's execution trajectory and decision sequence.",
+  "evaluatorTypeId": "uipath-llm-judge-trajectory-similarity",
+  "evaluatorConfig": {
+    "name": "TrajectoryEvaluator",
+    "model": "gpt-4.1-2025-04-14",
+    "prompt": "Evaluate the agent's execution trajectory based on the expected behavior.\n\nExpected Agent Behavior: {{ExpectedAgentBehavior}}\nAgent Run History: {{AgentRunHistory}}\n\nProvide a score from 0-100 based on how well the agent followed the expected trajectory.",
+    "temperature": 0.0,
+    "defaultEvaluationCriteria": {
+      "expectedAgentBehavior": "The agent should correctly perform the calculation and return the result."
+    }
+  }
+}

{uipath-2.1.107 → uipath-2.1.109}/samples/calculator/main.py

@@ -1,11 +1,11 @@
+import logging
 import random
+from enum import Enum
 
 from pydantic.dataclasses import dataclass
-from enum import Enum
 
-from uipath.eval.mocks import mockable, ExampleCall
+from uipath.eval.mocks import ExampleCall, mockable
 from uipath.tracing import traced
-import logging
 
 logger = logging.getLogger(__name__)
 
@@ -39,6 +39,9 @@ async def get_random_operator() -> Wrapper:
     """Get a random operator."""
     return Wrapper(result=random.choice([Operator.ADD, Operator.SUBTRACT, Operator.MULTIPLY, Operator.DIVIDE]))
 
+@traced(name="track_operator")
+def track_operator(operator: Operator):
+    pass
 
 @traced()
 async def main(input: CalculatorInput) -> CalculatorOutput:
@@ -46,6 +49,7 @@ async def main(input: CalculatorInput) -> CalculatorOutput:
         operator = (await get_random_operator()).result
     else:
         operator = input.operator
+    track_operator(operator)
     match operator:
         case Operator.ADD: result = input.a + input.b
         case Operator.SUBTRACT: result = input.a - input.b