uipath 2.1.108__py3-none-any.whl → 2.1.109__py3-none-any.whl

uipath/eval/evaluators_types/JsonSimilarityEvaluator.json

@@ -0,0 +1,81 @@
+{
+  "evaluatorTypeId": "uipath-json-similarity",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "OutputEvaluationCriteria": {
+        "description": "Base class for all output evaluation criteria.",
+        "properties": {
+          "expected_output": {
+            "anyOf": [
+              {
+                "additionalProperties": true,
+                "type": "object"
+              },
+              {
+                "type": "string"
+              }
+            ],
+            "title": "Expected Output"
+          }
+        },
+        "required": [
+          "expected_output"
+        ],
+        "title": "OutputEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the json similarity evaluator.",
+    "properties": {
+      "name": {
+        "default": "JsonSimilarityEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/OutputEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "target_output_key": {
+        "default": "*",
+        "description": "Key to extract output from agent execution",
+        "title": "Target Output Key",
+        "type": "string"
+      }
+    },
+    "title": "JsonSimilarityEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Base class for all output evaluation criteria.",
+    "properties": {
+      "expected_output": {
+        "anyOf": [
+          {
+            "additionalProperties": true,
+            "type": "object"
+          },
+          {
+            "type": "string"
+          }
+        ],
+        "title": "Expected Output"
+      }
+    },
+    "required": [
+      "expected_output"
+    ],
+    "title": "OutputEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {
+    "type": "string"
+  }
+}

uipath/eval/evaluators_types/LLMJudgeOutputEvaluator.json

@@ -0,0 +1,110 @@
+{
+  "evaluatorTypeId": "uipath-llm-judge-output-semantic-similarity",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "OutputEvaluationCriteria": {
+        "description": "Base class for all output evaluation criteria.",
+        "properties": {
+          "expected_output": {
+            "anyOf": [
+              {
+                "additionalProperties": true,
+                "type": "object"
+              },
+              {
+                "type": "string"
+              }
+            ],
+            "title": "Expected Output"
+          }
+        },
+        "required": [
+          "expected_output"
+        ],
+        "title": "OutputEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the LLM judge output evaluator.",
+    "properties": {
+      "name": {
+        "default": "LLMJudgeOutputEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/OutputEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "prompt": {
+        "default": "As an expert evaluator, analyze the semantic similarity of these JSON contents to determine a score from 0-100. Focus on comparing the meaning and contextual equivalence of corresponding fields, accounting for alternative valid expressions, synonyms, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nExpectedOutput:\n{{ExpectedOutput}}\n----\nActualOutput:\n{{ActualOutput}}",
+        "title": "Prompt",
+        "type": "string"
+      },
+      "model": {
+        "title": "Model",
+        "type": "string"
+      },
+      "temperature": {
+        "default": 0.0,
+        "title": "Temperature",
+        "type": "number"
+      },
+      "max_tokens": {
+        "anyOf": [
+          {
+            "type": "integer"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null,
+        "title": "Max Tokens"
+      },
+      "target_output_key": {
+        "default": "*",
+        "description": "Key to extract output from agent execution",
+        "title": "Target Output Key",
+        "type": "string"
+      }
+    },
+    "required": [
+      "model"
+    ],
+    "title": "LLMJudgeOutputEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Base class for all output evaluation criteria.",
+    "properties": {
+      "expected_output": {
+        "anyOf": [
+          {
+            "additionalProperties": true,
+            "type": "object"
+          },
+          {
+            "type": "string"
+          }
+        ],
+        "title": "Expected Output"
+      }
+    },
+    "required": [
+      "expected_output"
+    ],
+    "title": "OutputEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {
+    "type": "string"
+  }
+}

uipath/eval/evaluators_types/LLMJudgeSimulationTrajectoryEvaluator.json

@@ -0,0 +1,88 @@
+{
+  "evaluatorTypeId": "uipath-llm-judge-trajectory-simulation",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "TrajectoryEvaluationCriteria": {
+        "description": "Evaluation criteria for trajectory-based evaluations.",
+        "properties": {
+          "expected_agent_behavior": {
+            "title": "Expected Agent Behavior",
+            "type": "string"
+          }
+        },
+        "required": [
+          "expected_agent_behavior"
+        ],
+        "title": "TrajectoryEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the llm judge simulation trajectory evaluator.",
+    "properties": {
+      "name": {
+        "default": "LLMJudgeSimulationEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/TrajectoryEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "prompt": {
+        "default": "As an expert evaluator, determine how well the agent did on a scale of 0-100. Focus on if the simulation was successful and if the agent behaved according to the expected output accounting for alternative valid expressions, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nAgentInput:\n{{UserOrSyntheticInput}}\n----\nSimulationInstructions:\n{{SimulationInstructions}}\n----\nExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n----\nAgentRunHistory:\n{{AgentRunHistory}}\n",
+        "title": "Prompt",
+        "type": "string"
+      },
+      "model": {
+        "title": "Model",
+        "type": "string"
+      },
+      "temperature": {
+        "default": 0.0,
+        "title": "Temperature",
+        "type": "number"
+      },
+      "max_tokens": {
+        "anyOf": [
+          {
+            "type": "integer"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null,
+        "title": "Max Tokens"
+      }
+    },
+    "required": [
+      "model"
+    ],
+    "title": "LLMJudgeSimulationEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Evaluation criteria for trajectory-based evaluations.",
+    "properties": {
+      "expected_agent_behavior": {
+        "title": "Expected Agent Behavior",
+        "type": "string"
+      }
+    },
+    "required": [
+      "expected_agent_behavior"
+    ],
+    "title": "TrajectoryEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {
+    "type": "string"
+  }
+}

uipath/eval/evaluators_types/LLMJudgeStrictJSONSimilarityOutputEvaluator.json

@@ -0,0 +1,110 @@
+{
+  "evaluatorTypeId": "uipath-llm-judge-output-strict-json-similarity",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "OutputEvaluationCriteria": {
+        "description": "Base class for all output evaluation criteria.",
+        "properties": {
+          "expected_output": {
+            "anyOf": [
+              {
+                "additionalProperties": true,
+                "type": "object"
+              },
+              {
+                "type": "string"
+              }
+            ],
+            "title": "Expected Output"
+          }
+        },
+        "required": [
+          "expected_output"
+        ],
+        "title": "OutputEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the LLM judge strict JSON similarity output evaluator.",
+    "properties": {
+      "name": {
+        "default": "LLMJudgeStrictJSONSimilarityOutputEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/OutputEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "prompt": {
+        "default": "ExpectedOutput (ground truth):\n{{ExpectedOutput}}\n\nActualOutput (model answer):\n{{ActualOutput}}",
+        "title": "Prompt",
+        "type": "string"
+      },
+      "model": {
+        "title": "Model",
+        "type": "string"
+      },
+      "temperature": {
+        "default": 0.0,
+        "title": "Temperature",
+        "type": "number"
+      },
+      "max_tokens": {
+        "anyOf": [
+          {
+            "type": "integer"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null,
+        "title": "Max Tokens"
+      },
+      "target_output_key": {
+        "default": "*",
+        "description": "Key to extract output from agent execution",
+        "title": "Target Output Key",
+        "type": "string"
+      }
+    },
+    "required": [
+      "model"
+    ],
+    "title": "LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Base class for all output evaluation criteria.",
+    "properties": {
+      "expected_output": {
+        "anyOf": [
+          {
+            "additionalProperties": true,
+            "type": "object"
+          },
+          {
+            "type": "string"
+          }
+        ],
+        "title": "Expected Output"
+      }
+    },
+    "required": [
+      "expected_output"
+    ],
+    "title": "OutputEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {
+    "type": "string"
+  }
+}

uipath/eval/evaluators_types/LLMJudgeTrajectoryEvaluator.json

@@ -0,0 +1,88 @@
+{
+  "evaluatorTypeId": "uipath-llm-judge-trajectory-similarity",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "TrajectoryEvaluationCriteria": {
+        "description": "Evaluation criteria for trajectory-based evaluations.",
+        "properties": {
+          "expected_agent_behavior": {
+            "title": "Expected Agent Behavior",
+            "type": "string"
+          }
+        },
+        "required": [
+          "expected_agent_behavior"
+        ],
+        "title": "TrajectoryEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the llm judge trajectory evaluator.",
+    "properties": {
+      "name": {
+        "default": "LLMJudgeTrajectoryEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/TrajectoryEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "prompt": {
+        "default": "As an expert evaluator, determine how well the agent performed on a scale of 0-100. Focus on whether the agent's actions and outputs matched the expected behavior, while allowing for alternative valid expressions and reasonable variations in language. Maintain high standards for accuracy and completeness. Provide your score with a brief and clear justification explaining your reasoning.\n----\nAgentInput:\n{{UserOrSyntheticInput}}\n----\nExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n----\nAgentRunHistory:\n{{AgentRunHistory}}\n",
+        "title": "Prompt",
+        "type": "string"
+      },
+      "model": {
+        "title": "Model",
+        "type": "string"
+      },
+      "temperature": {
+        "default": 0.0,
+        "title": "Temperature",
+        "type": "number"
+      },
+      "max_tokens": {
+        "anyOf": [
+          {
+            "type": "integer"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null,
+        "title": "Max Tokens"
+      }
+    },
+    "required": [
+      "model"
+    ],
+    "title": "LLMJudgeTrajectoryEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "description": "Evaluation criteria for trajectory-based evaluations.",
+    "properties": {
+      "expected_agent_behavior": {
+        "title": "Expected Agent Behavior",
+        "type": "string"
+      }
+    },
+    "required": [
+      "expected_agent_behavior"
+    ],
+    "title": "TrajectoryEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {
+    "type": "string"
+  }
+}

uipath/eval/evaluators_types/ToolCallArgsEvaluator.json

@@ -0,0 +1,131 @@
+{
+  "evaluatorTypeId": "uipath-tool-call-args",
+  "evaluatorConfigSchema": {
+    "$defs": {
+      "ToolCall": {
+        "description": "Represents a tool call with its arguments.",
+        "properties": {
+          "name": {
+            "title": "Name",
+            "type": "string"
+          },
+          "args": {
+            "additionalProperties": true,
+            "title": "Args",
+            "type": "object"
+          }
+        },
+        "required": [
+          "name",
+          "args"
+        ],
+        "title": "ToolCall",
+        "type": "object"
+      },
+      "ToolCallArgsEvaluationCriteria": {
+        "description": "Evaluation criteria for the tool call order evaluator.",
+        "properties": {
+          "tool_calls": {
+            "items": {
+              "$ref": "#/$defs/ToolCall"
+            },
+            "title": "Tool Calls",
+            "type": "array"
+          }
+        },
+        "required": [
+          "tool_calls"
+        ],
+        "title": "ToolCallArgsEvaluationCriteria",
+        "type": "object"
+      }
+    },
+    "description": "Configuration for the tool call count evaluator.",
+    "properties": {
+      "name": {
+        "default": "ToolCallArgsEvaluator",
+        "title": "Name",
+        "type": "string"
+      },
+      "default_evaluation_criteria": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/ToolCallArgsEvaluationCriteria"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "default": null
+      },
+      "strict": {
+        "default": false,
+        "title": "Strict",
+        "type": "boolean"
+      },
+      "subset": {
+        "default": false,
+        "title": "Subset",
+        "type": "boolean"
+      }
+    },
+    "title": "ToolCallArgsEvaluatorConfig",
+    "type": "object"
+  },
+  "evaluationCriteriaSchema": {
+    "$defs": {
+      "ToolCall": {
+        "description": "Represents a tool call with its arguments.",
+        "properties": {
+          "name": {
+            "title": "Name",
+            "type": "string"
+          },
+          "args": {
+            "additionalProperties": true,
+            "title": "Args",
+            "type": "object"
+          }
+        },
+        "required": [
+          "name",
+          "args"
+        ],
+        "title": "ToolCall",
+        "type": "object"
+      }
+    },
+    "description": "Evaluation criteria for the tool call order evaluator.",
+    "properties": {
+      "tool_calls": {
+        "items": {
+          "$ref": "#/$defs/ToolCall"
+        },
+        "title": "Tool Calls",
+        "type": "array"
+      }
+    },
+    "required": [
+      "tool_calls"
+    ],
+    "title": "ToolCallArgsEvaluationCriteria",
+    "type": "object"
+  },
+  "justificationSchema": {
+    "description": "Justification for the tool call args evaluator.",
+    "properties": {
+      "explained_tool_calls_args": {
+        "additionalProperties": {
+          "type": "string"
+        },
+        "title": "Explained Tool Calls Args",
+        "type": "object"
+      }
+    },
+    "required": [
+      "explained_tool_calls_args"
+    ],
+    "title": "ToolCallArgsEvaluatorJustification",
+    "type": "object"
+  }
+}