uipath 2.1.74__tar.gz → 2.1.75__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (335) hide show
  1. {uipath-2.1.74 → uipath-2.1.75}/PKG-INFO +1 -1
  2. {uipath-2.1.74 → uipath-2.1.75}/pyproject.toml +1 -1
  3. {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/evals/eval-sets/default.json +2 -1
  4. uipath-2.1.75/samples/calculator/evals/evaluators/llm-as-a-judge.json +13 -0
  5. uipath-2.1.75/src/uipath/_cli/_evals/_console_progress_reporter.py +241 -0
  6. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_output.py +2 -0
  7. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_progress_reporter.py +11 -5
  8. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_runtime.py +117 -50
  9. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_contracts.py +4 -2
  10. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_logging.py +29 -0
  11. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_eval.py +5 -5
  12. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/middlewares.py +3 -3
  13. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_events/_events.py +2 -0
  14. {uipath-2.1.74 → uipath-2.1.75}/uv.lock +1 -1
  15. {uipath-2.1.74 → uipath-2.1.75}/.cursorrules +0 -0
  16. {uipath-2.1.74 → uipath-2.1.75}/.editorconfig +0 -0
  17. {uipath-2.1.74 → uipath-2.1.75}/.gitattributes +0 -0
  18. {uipath-2.1.74 → uipath-2.1.75}/.github/labeler.yml +0 -0
  19. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/auto-label.yml +0 -0
  20. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/cd.yml +0 -0
  21. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/ci.yml +0 -0
  22. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/commitlint.yml +0 -0
  23. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/integration_tests.yml +0 -0
  24. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/lint.yml +0 -0
  25. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/publish-dev.yml +0 -0
  26. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/publish-docs.yml +0 -0
  27. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/slack.yml +0 -0
  28. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/test-uipath-langchain.yml +0 -0
  29. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/test-uipath-llamaindex.yml +0 -0
  30. {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/test.yml +0 -0
  31. {uipath-2.1.74 → uipath-2.1.75}/.gitignore +0 -0
  32. {uipath-2.1.74 → uipath-2.1.75}/.pre-commit-config.yaml +0 -0
  33. {uipath-2.1.74 → uipath-2.1.75}/.python-version +0 -0
  34. {uipath-2.1.74 → uipath-2.1.75}/.vscode/extensions.json +0 -0
  35. {uipath-2.1.74 → uipath-2.1.75}/.vscode/launch.json +0 -0
  36. {uipath-2.1.74 → uipath-2.1.75}/.vscode/settings.json +0 -0
  37. {uipath-2.1.74 → uipath-2.1.75}/CONTRIBUTING.md +0 -0
  38. {uipath-2.1.74 → uipath-2.1.75}/LICENSE +0 -0
  39. {uipath-2.1.74 → uipath-2.1.75}/README.md +0 -0
  40. {uipath-2.1.74 → uipath-2.1.75}/docs/CONTRIBUTING.md +0 -0
  41. {uipath-2.1.74 → uipath-2.1.75}/docs/FAQ.md +0 -0
  42. {uipath-2.1.74 → uipath-2.1.75}/docs/assets/env-preparation-failed-dark.png +0 -0
  43. {uipath-2.1.74 → uipath-2.1.75}/docs/assets/env-preparation-failed-light.png +0 -0
  44. {uipath-2.1.74 → uipath-2.1.75}/docs/assets/favicon.png +0 -0
  45. {uipath-2.1.74 → uipath-2.1.75}/docs/assets/logo-dark.svg +0 -0
  46. {uipath-2.1.74 → uipath-2.1.75}/docs/assets/logo-light.svg +0 -0
  47. {uipath-2.1.74 → uipath-2.1.75}/docs/cli/index.md +0 -0
  48. {uipath-2.1.74 → uipath-2.1.75}/docs/core/actions.md +0 -0
  49. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_dark.gif +0 -0
  50. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_light.gif +0 -0
  51. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_secret_dark.png +0 -0
  52. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_secret_light.png +0 -0
  53. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/copy_path_dark.png +0 -0
  54. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/copy_path_light.png +0 -0
  55. {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets.md +0 -0
  56. {uipath-2.1.74 → uipath-2.1.75}/docs/core/attachments.md +0 -0
  57. {uipath-2.1.74 → uipath-2.1.75}/docs/core/buckets.md +0 -0
  58. {uipath-2.1.74 → uipath-2.1.75}/docs/core/connections.md +0 -0
  59. {uipath-2.1.74 → uipath-2.1.75}/docs/core/context_grounding.md +0 -0
  60. {uipath-2.1.74 → uipath-2.1.75}/docs/core/documents.md +0 -0
  61. {uipath-2.1.74 → uipath-2.1.75}/docs/core/documents_models.md +0 -0
  62. {uipath-2.1.74 → uipath-2.1.75}/docs/core/environment_variables.md +0 -0
  63. {uipath-2.1.74 → uipath-2.1.75}/docs/core/getting_started.md +0 -0
  64. {uipath-2.1.74 → uipath-2.1.75}/docs/core/jobs.md +0 -0
  65. {uipath-2.1.74 → uipath-2.1.75}/docs/core/llm_gateway.md +0 -0
  66. {uipath-2.1.74 → uipath-2.1.75}/docs/core/processes.md +0 -0
  67. {uipath-2.1.74 → uipath-2.1.75}/docs/core/queues.md +0 -0
  68. {uipath-2.1.74 → uipath-2.1.75}/docs/core/traced.md +0 -0
  69. {uipath-2.1.74 → uipath-2.1.75}/docs/hooks.py +0 -0
  70. {uipath-2.1.74 → uipath-2.1.75}/docs/index.md +0 -0
  71. {uipath-2.1.74 → uipath-2.1.75}/docs/javascripts/extra.js +0 -0
  72. {uipath-2.1.74 → uipath-2.1.75}/docs/overrides/main.html +0 -0
  73. {uipath-2.1.74 → uipath-2.1.75}/docs/overrides/partials/actions.html +0 -0
  74. {uipath-2.1.74 → uipath-2.1.75}/docs/overrides/partials/logo.html +0 -0
  75. {uipath-2.1.74 → uipath-2.1.75}/docs/release_policy.md +0 -0
  76. {uipath-2.1.74 → uipath-2.1.75}/docs/sample_images/google-ADK-agent/agent-output.png +0 -0
  77. {uipath-2.1.74 → uipath-2.1.75}/docs/stylesheets/extra.css +0 -0
  78. {uipath-2.1.74 → uipath-2.1.75}/justfile +0 -0
  79. {uipath-2.1.74 → uipath-2.1.75}/mkdocs.yml +0 -0
  80. {uipath-2.1.74 → uipath-2.1.75}/py.typed +0 -0
  81. {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/README.md +0 -0
  82. {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/evals/evaluators/equality.json +0 -0
  83. {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/main.py +0 -0
  84. {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/pyproject.toml +0 -0
  85. {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/.python-version +0 -0
  86. {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/README.md +0 -0
  87. {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/main.py +0 -0
  88. {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/pyproject.toml +0 -0
  89. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/.env.example +0 -0
  90. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/README.md +0 -0
  91. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/input.json +0 -0
  92. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/multi_tool_agent/__init__.py +0 -0
  93. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/multi_tool_agent/agent.py +0 -0
  94. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/pyproject.toml +0 -0
  95. {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/uv.lock +0 -0
  96. {uipath-2.1.74 → uipath-2.1.75}/scripts/debug_test.py +0 -0
  97. {uipath-2.1.74 → uipath-2.1.75}/scripts/lint_httpx_client.py +0 -0
  98. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/__init__.py +0 -0
  99. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/README.md +0 -0
  100. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/__init__.py +0 -0
  101. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_auth_server.py +0 -0
  102. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_auth_service.py +0 -0
  103. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_client_credentials.py +0 -0
  104. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_models.py +0 -0
  105. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_oidc_utils.py +0 -0
  106. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_portal_service.py +0 -0
  107. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_url_utils.py +0 -0
  108. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_utils.py +0 -0
  109. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/auth_config.json +0 -0
  110. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/index.html +0 -0
  111. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/localhost.crt +0 -0
  112. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/localhost.key +0 -0
  113. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/__init__.py +0 -0
  114. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_chat.py +0 -0
  115. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_details.py +0 -0
  116. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_history.py +0 -0
  117. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_json_input.py +0 -0
  118. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_new.py +0 -0
  119. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_models/_execution.py +0 -0
  120. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_models/_messages.py +0 -0
  121. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_styles/terminal.tcss +0 -0
  122. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_utils/_chat.py +0 -0
  123. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_utils/_exporter.py +0 -0
  124. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_utils/_logger.py +0 -0
  125. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_evaluator_factory.py +0 -0
  126. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_evaluation_set.py +0 -0
  127. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_evaluator.py +0 -0
  128. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_evaluator_base_params.py +0 -0
  129. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_sw_reporting.py +0 -0
  130. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_trajectory_span.py +0 -0
  131. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/__init__.py +0 -0
  132. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/llm_mocker.py +0 -0
  133. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mocker.py +0 -0
  134. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mocker_factory.py +0 -0
  135. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mockito_mocker.py +0 -0
  136. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mocks.py +0 -0
  137. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_push/sw_file_handler.py +0 -0
  138. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_escalation.py +0 -0
  139. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_hitl.py +0 -0
  140. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_runtime.py +0 -0
  141. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_script_executor.py +0 -0
  142. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/.psmdcp.template +0 -0
  143. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/.rels.template +0 -0
  144. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/[Content_Types].xml.template +0 -0
  145. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/main.py.template +0 -0
  146. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/package.nuspec.template +0 -0
  147. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_common.py +0 -0
  148. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_console.py +0 -0
  149. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_constants.py +0 -0
  150. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_debug.py +0 -0
  151. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_eval_set.py +0 -0
  152. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_folders.py +0 -0
  153. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_input_args.py +0 -0
  154. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_parse_ast.py +0 -0
  155. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_processes.py +0 -0
  156. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_project_files.py +0 -0
  157. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_studio_project.py +0 -0
  158. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_tracing.py +0 -0
  159. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_uv_helpers.py +0 -0
  160. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_auth.py +0 -0
  161. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_deploy.py +0 -0
  162. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_dev.py +0 -0
  163. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_init.py +0 -0
  164. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_invoke.py +0 -0
  165. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_new.py +0 -0
  166. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_pack.py +0 -0
  167. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_publish.py +0 -0
  168. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_pull.py +0 -0
  169. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_push.py +0 -0
  170. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_run.py +0 -0
  171. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/spinner.py +0 -0
  172. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_config.py +0 -0
  173. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_events/__init__.py +0 -0
  174. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_events/_event_bus.py +0 -0
  175. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_execution_context.py +0 -0
  176. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_folder_context.py +0 -0
  177. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_resources/AGENTS.md +0 -0
  178. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/__init__.py +0 -0
  179. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/_base_service.py +0 -0
  180. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/actions_service.py +0 -0
  181. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/api_client.py +0 -0
  182. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/assets_service.py +0 -0
  183. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/attachments_service.py +0 -0
  184. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/buckets_service.py +0 -0
  185. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/connections_service.py +0 -0
  186. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/context_grounding_service.py +0 -0
  187. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/documents_service.py +0 -0
  188. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/entities_service.py +0 -0
  189. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/folder_service.py +0 -0
  190. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/jobs_service.py +0 -0
  191. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/llm_gateway_service.py +0 -0
  192. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/processes_service.py +0 -0
  193. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/queues_service.py +0 -0
  194. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_uipath.py +0 -0
  195. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/__init__.py +0 -0
  196. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_endpoint.py +0 -0
  197. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_infer_bindings.py +0 -0
  198. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_logs.py +0 -0
  199. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_read_overwrites.py +0 -0
  200. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_request_override.py +0 -0
  201. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_request_spec.py +0 -0
  202. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_ssl_context.py +0 -0
  203. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_url.py +0 -0
  204. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_user_agent.py +0 -0
  205. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/constants.py +0 -0
  206. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/_utils.py +0 -0
  207. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/__init__.py +0 -0
  208. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/async_stream.py +0 -0
  209. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/citation.py +0 -0
  210. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/content.py +0 -0
  211. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/conversation.py +0 -0
  212. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/event.py +0 -0
  213. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/exchange.py +0 -0
  214. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/message.py +0 -0
  215. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/meta.py +0 -0
  216. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/tool.py +0 -0
  217. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/models/agent.py +0 -0
  218. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/_helpers/__init__.py +0 -0
  219. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/_helpers/helpers.py +0 -0
  220. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/__init__.py +0 -0
  221. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/base_evaluator.py +0 -0
  222. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/deterministic_evaluator_base.py +0 -0
  223. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/exact_match_evaluator.py +0 -0
  224. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/json_similarity_evaluator.py +0 -0
  225. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/llm_as_judge_evaluator.py +0 -0
  226. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/trajectory_evaluator.py +0 -0
  227. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/models/__init__.py +0 -0
  228. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/models/models.py +0 -0
  229. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/__init__.py +0 -0
  230. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/action_schema.py +0 -0
  231. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/actions.py +0 -0
  232. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/assets.py +0 -0
  233. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/attachment.py +0 -0
  234. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/buckets.py +0 -0
  235. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/connections.py +0 -0
  236. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/context_grounding.py +0 -0
  237. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/context_grounding_index.py +0 -0
  238. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/documents.py +0 -0
  239. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/entities.py +0 -0
  240. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/errors.py +0 -0
  241. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/exceptions.py +0 -0
  242. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/interrupt_models.py +0 -0
  243. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/job.py +0 -0
  244. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/llm_gateway.py +0 -0
  245. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/processes.py +0 -0
  246. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/queues.py +0 -0
  247. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/py.typed +0 -0
  248. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/telemetry/__init__.py +0 -0
  249. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/telemetry/_constants.py +0 -0
  250. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/telemetry/_track.py +0 -0
  251. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/__init__.py +0 -0
  252. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/_otel_exporters.py +0 -0
  253. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/_traced.py +0 -0
  254. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/_utils.py +0 -0
  255. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/utils/__init__.py +0 -0
  256. {uipath-2.1.74 → uipath-2.1.75}/src/uipath/utils/_endpoints_manager.py +0 -0
  257. {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/main.py +0 -0
  258. {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/pyproject.toml +0 -0
  259. {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/run.sh +0 -0
  260. {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/src/assert.py +0 -0
  261. {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/main.py +0 -0
  262. {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/pyproject.toml +0 -0
  263. {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/run.sh +0 -0
  264. {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/src/assert.py +0 -0
  265. {uipath-2.1.74 → uipath-2.1.75}/testcases/common/validate_output.sh +0 -0
  266. {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/agent.mermaid +0 -0
  267. {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/langgraph.json +0 -0
  268. {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/pyproject.toml +0 -0
  269. {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/run.sh +0 -0
  270. {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/src/assert.py +0 -0
  271. {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/src/main.py +0 -0
  272. {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/agent.mermaid +0 -0
  273. {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/llama_index.json +0 -0
  274. {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/pyproject.toml +0 -0
  275. {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/run.sh +0 -0
  276. {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/src/assert.py +0 -0
  277. {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/src/main.py +0 -0
  278. {uipath-2.1.74 → uipath-2.1.75}/tests/__init__.py +0 -0
  279. {uipath-2.1.74 → uipath-2.1.75}/tests/agent/models/test_agent.py +0 -0
  280. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/conftest.py +0 -0
  281. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/eval/mocks/test_mocks.py +0 -0
  282. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/evaluators/test_json_similarity_evaluator.py +0 -0
  283. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/bindings_script.py +0 -0
  284. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/pyproject.toml +0 -0
  285. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/simple_script.py +0 -0
  286. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/uipath-mock.json +0 -0
  287. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/uipath-simple-script-mock.json +0 -0
  288. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_auth.py +0 -0
  289. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_hitl.py +0 -0
  290. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_init.py +0 -0
  291. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_init_agents_md.py +0 -0
  292. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_input_args.py +0 -0
  293. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_invoke.py +0 -0
  294. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_new.py +0 -0
  295. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_pack.py +0 -0
  296. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_portal_service_ensure_valid_token.py +0 -0
  297. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_portal_service_refresh_token.py +0 -0
  298. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_publish.py +0 -0
  299. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_pull.py +0 -0
  300. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_push.py +0 -0
  301. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_run.py +0 -0
  302. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_utils.py +0 -0
  303. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/utils/common.py +0 -0
  304. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/utils/project_details.py +0 -0
  305. {uipath-2.1.74 → uipath-2.1.75}/tests/cli/utils/uipath_json.py +0 -0
  306. {uipath-2.1.74 → uipath-2.1.75}/tests/conftest.py +0 -0
  307. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/conftest.py +0 -0
  308. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_actions_service.py +0 -0
  309. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_api_client.py +0 -0
  310. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_assets_service.py +0 -0
  311. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_attachments_service.py +0 -0
  312. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_base_service.py +0 -0
  313. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_buckets_service.py +0 -0
  314. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_connections_service.py +0 -0
  315. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_context_grounding_service.py +0 -0
  316. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_documents_service.py +0 -0
  317. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_entities_service.py +0 -0
  318. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_folder_service.py +0 -0
  319. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_jobs_service.py +0 -0
  320. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_llm_integration.py +0 -0
  321. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_llm_schema_cleanup.py +0 -0
  322. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_llm_service.py +0 -0
  323. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_processes_service.py +0 -0
  324. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_queues_service.py +0 -0
  325. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_uipath_llm_integration.py +0 -0
  326. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/tests_data/documents_service/create_validation_action_response.json +0 -0
  327. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/tests_data/documents_service/extraction_response.json +0 -0
  328. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/tests_data/documents_service/validated_result.json +0 -0
  329. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/test_bindings_inference.py +0 -0
  330. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/test_config.py +0 -0
  331. {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/test_overwrites.py +0 -0
  332. {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_otel_exporters.py +0 -0
  333. {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_span_utils.py +0 -0
  334. {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_traced.py +0 -0
  335. {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_tracing_manager.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: uipath
3
- Version: 2.1.74
3
+ Version: 2.1.75
4
4
  Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
5
5
  Project-URL: Homepage, https://uipath.com
6
6
  Project-URL: Repository, https://github.com/UiPath/uipath-python
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "uipath"
3
- version = "2.1.74"
3
+ version = "2.1.75"
4
4
  description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
5
5
  readme = { file = "README.md", content-type = "text/markdown" }
6
6
  requires-python = ">=3.10"
@@ -4,7 +4,8 @@
4
4
  "name": "Basic Calculator Evaluation Set",
5
5
  "batchSize": 10,
6
6
  "evaluatorRefs": [
7
- "equality"
7
+ "equality",
8
+ "llm-as-a-judge"
8
9
  ],
9
10
  "evaluations": [
10
11
  {
@@ -0,0 +1,13 @@
1
+ {
2
+ "fileName": "llm-as-a-judge.json",
3
+ "id": "llm-as-a-judge",
4
+ "name": "LLMAsAJudge Evaluator",
5
+ "description": "An evaluator that judges the agent based on it's run history and expected behavior",
6
+ "category": 3,
7
+ "type": 7,
8
+ "prompt": "As an expert evaluator, determine how well the agent did on a scale of 0-100. Focus on if the simulation was successful and if the agent behaved according to the expected output accounting for alternative valid expressions, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nUserOrSyntheticInputGivenToAgent:\n{{UserOrSyntheticInput}}\n----\nSimulationInstructions:\n{{SimulationInstructions}}\n----\nExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n----\nAgentRunHistory:\n{{AgentRunHistory}}\n",
9
+ "targetOutputKey": "*",
10
+ "model":"gpt-4.1-2025-04-14",
11
+ "createdAt": "2025-08-21T15:12:58.695Z",
12
+ "updatedAt": "2025-08-21T15:12:58.695Z"
13
+ }
@@ -0,0 +1,241 @@
1
+ """Console progress reporter for evaluation runs with line-by-line output."""
2
+
3
+ import logging
4
+ from typing import Any, Dict
5
+
6
+ from rich.console import Console
7
+ from rich.rule import Rule
8
+ from rich.table import Table
9
+
10
+ from uipath._events._event_bus import EventBus
11
+ from uipath._events._events import (
12
+ EvalRunCreatedEvent,
13
+ EvalRunUpdatedEvent,
14
+ EvalSetRunCreatedEvent,
15
+ EvalSetRunUpdatedEvent,
16
+ EvaluationEvents,
17
+ )
18
+ from uipath.eval.evaluators import BaseEvaluator
19
+ from uipath.eval.models import ScoreType
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+
24
+ class ConsoleProgressReporter:
25
+ """Handles displaying evaluation progress to the console with line-by-line output."""
26
+
27
+ def __init__(self):
28
+ self.console = Console()
29
+ self.evaluators: Dict[str, BaseEvaluator[Any]] = {}
30
+ self.display_started = False
31
+ self.eval_results_by_name: Dict[str, list[Any]] = {}
32
+
33
+ def _convert_score_to_numeric(self, eval_result) -> float:
34
+ """Convert evaluation result score to numeric value."""
35
+ score_value = eval_result.result.score
36
+ if eval_result.result.score_type == ScoreType.BOOLEAN:
37
+ score_value = 100 if score_value else 0
38
+ return score_value
39
+
40
+ def _get_evaluator_name(self, evaluator_id: str) -> str:
41
+ """Get evaluator name from ID, with fallback."""
42
+ return self.evaluators.get(
43
+ evaluator_id,
44
+ type(
45
+ "obj",
46
+ (object,),
47
+ {"name": f"Evaluator {evaluator_id[:8]}"},
48
+ )(),
49
+ ).name
50
+
51
+ def _display_successful_evaluation(self, eval_name: str, eval_results) -> None:
52
+ """Display results for a successful evaluation."""
53
+ from rich.text import Text
54
+
55
+ if eval_results:
56
+ result = Text()
57
+ result.append("▌", style="bold green")
58
+ result.append(" ", style="")
59
+ result.append(eval_name, style="bold green")
60
+ self.console.print(result)
61
+ table = Table(show_header=False, box=None, padding=(0, 2, 0, 2))
62
+
63
+ for eval_result in eval_results:
64
+ evaluator_name = self._get_evaluator_name(eval_result.evaluator_id)
65
+ score_value = self._convert_score_to_numeric(eval_result)
66
+ table.add_row(
67
+ f"{evaluator_name}", f"[bold cyan]{score_value:.1f}[/bold cyan]"
68
+ )
69
+
70
+ self.console.print(table)
71
+ else:
72
+ result = Text()
73
+ result.append(" ✓ ", style="bold green")
74
+ result.append(eval_name, style="bold white")
75
+ result.append(" - No evaluators", style="dim")
76
+ self.console.print(result)
77
+
78
+ def _extract_error_message(self, eval_item_payload) -> str:
79
+ """Extract clean error message from evaluation item."""
80
+ if hasattr(eval_item_payload, "_error_message"):
81
+ error_message = getattr(eval_item_payload, "_error_message", None)
82
+ if error_message:
83
+ return str(error_message) or "Execution failed"
84
+ return "Execution failed"
85
+
86
+ def _display_failed_evaluation(self, eval_name: str, error_msg: str) -> None:
87
+ """Display results for a failed evaluation."""
88
+ from rich.text import Text
89
+
90
+ result = Text()
91
+ result.append(" ✗ ", style="bold red")
92
+ result.append(eval_name, style="bold white")
93
+ self.console.print(result)
94
+
95
+ error_text = Text()
96
+ error_text.append(" ", style="")
97
+ error_text.append(error_msg, style="red")
98
+ self.console.print(error_text)
99
+
100
+ def start_display(self):
101
+ """Start the display."""
102
+ if not self.display_started:
103
+ self.console.print()
104
+ self.console.print("→ [bold]Running Evaluations[/bold]")
105
+ self.console.print()
106
+ self.display_started = True
107
+
108
+ async def handle_create_eval_set_run(self, payload: EvalSetRunCreatedEvent) -> None:
109
+ """Handle evaluation set run creation."""
110
+ try:
111
+ self.evaluators = {eval.id: eval for eval in payload.evaluators}
112
+ except Exception as e:
113
+ logger.error(f"Failed to handle create eval set run event: {e}")
114
+
115
+ async def handle_create_eval_run(self, payload: EvalRunCreatedEvent) -> None:
116
+ """Handle individual evaluation run creation."""
117
+ try:
118
+ if not self.display_started:
119
+ self.start_display()
120
+
121
+ self.console.print(f" ○ [dim]{payload.eval_item.name}[/dim] - Running...")
122
+ except Exception as e:
123
+ logger.error(f"Failed to handle create eval run event: {e}")
124
+
125
+ async def handle_update_eval_run(self, payload: EvalRunUpdatedEvent) -> None:
126
+ """Handle evaluation run updates."""
127
+ try:
128
+ if payload.success:
129
+ # Store results for final display
130
+ self.eval_results_by_name[payload.eval_item.name] = payload.eval_results
131
+ self._display_successful_evaluation(
132
+ payload.eval_item.name, payload.eval_results
133
+ )
134
+ else:
135
+ error_msg = self._extract_error_message(payload.eval_item)
136
+ self._display_failed_evaluation(payload.eval_item.name, error_msg)
137
+
138
+ logs = payload.logs
139
+
140
+ self.console.print(
141
+ Rule(
142
+ f"[dim italic]Execution Logs: {payload.eval_item.name}[/dim italic]",
143
+ style="dim",
144
+ align="center",
145
+ )
146
+ )
147
+
148
+ if len(logs) > 0:
149
+ for record in logs:
150
+ log_line = f" [dim]{record.getMessage()}[/dim]"
151
+ self.console.print(log_line)
152
+ else:
153
+ self.console.print(" [dim italic]No execution logs[/dim italic]")
154
+
155
+ self.console.print(Rule(style="dim"))
156
+ except Exception as e:
157
+ logger.error(f"Console reporter error: {e}")
158
+
159
+ async def handle_update_eval_set_run(self, payload: EvalSetRunUpdatedEvent) -> None:
160
+ """Handle evaluation set run completion."""
161
+ try:
162
+ self.final_results = payload.evaluator_scores
163
+ self.display_final_results()
164
+ except Exception as e:
165
+ logger.error(f"Console reporter error: {e}")
166
+
167
+ def display_final_results(self):
168
+ """Display final results summary."""
169
+ self.console.print()
170
+
171
+ if hasattr(self, "final_results") and self.final_results:
172
+ from rich.table import Table
173
+
174
+ # Group evaluators by ID to organize display
175
+ evaluator_ids = list(self.final_results.keys())
176
+
177
+ # Print title
178
+ self.console.print("[bold]Evaluation Results[/bold]")
179
+ self.console.print()
180
+
181
+ # Create single summary table
182
+ summary_table = Table(show_header=True, padding=(0, 2))
183
+ summary_table.add_column("Evaluation", style="cyan")
184
+
185
+ # Add column for each evaluator
186
+ for evaluator_id in evaluator_ids:
187
+ evaluator_name = self._get_evaluator_name(evaluator_id)
188
+ summary_table.add_column(evaluator_name, justify="right")
189
+
190
+ # Add row for each evaluation
191
+ for eval_name, eval_results in self.eval_results_by_name.items():
192
+ row_values = [eval_name]
193
+
194
+ # Get score for each evaluator
195
+ for evaluator_id in evaluator_ids:
196
+ score_found = False
197
+ for eval_result in eval_results:
198
+ if eval_result.evaluator_id == evaluator_id:
199
+ score_value = self._convert_score_to_numeric(eval_result)
200
+ row_values.append(f"{score_value:.1f}")
201
+ score_found = True
202
+ break
203
+
204
+ if not score_found:
205
+ row_values.append("-")
206
+
207
+ summary_table.add_row(*row_values)
208
+
209
+ # Add separator row before average
210
+ summary_table.add_section()
211
+
212
+ # Add average row
213
+ avg_row_values = ["[bold]Average[/bold]"]
214
+ for evaluator_id in evaluator_ids:
215
+ avg_score = self.final_results[evaluator_id]
216
+ avg_row_values.append(f"[bold]{avg_score:.1f}[/bold]")
217
+
218
+ summary_table.add_row(*avg_row_values)
219
+
220
+ self.console.print(summary_table)
221
+ self.console.print()
222
+ else:
223
+ self.console.print(
224
+ "→ [bold green]All evaluations completed successfully![/bold green]"
225
+ )
226
+ self.console.print()
227
+
228
+ async def subscribe_to_eval_runtime_events(self, event_bus: EventBus) -> None:
229
+ """Subscribe to evaluation runtime events."""
230
+ event_bus.subscribe(
231
+ EvaluationEvents.CREATE_EVAL_SET_RUN, self.handle_create_eval_set_run
232
+ )
233
+ event_bus.subscribe(
234
+ EvaluationEvents.CREATE_EVAL_RUN, self.handle_create_eval_run
235
+ )
236
+ event_bus.subscribe(
237
+ EvaluationEvents.UPDATE_EVAL_RUN, self.handle_update_eval_run
238
+ )
239
+ event_bus.subscribe(
240
+ EvaluationEvents.UPDATE_EVAL_SET_RUN, self.handle_update_eval_set_run
241
+ )
@@ -1,3 +1,4 @@
1
+ import logging
1
2
  from typing import List, Optional
2
3
 
3
4
  from opentelemetry.sdk.trace import ReadableSpan
@@ -15,6 +16,7 @@ class UiPathEvalRunExecutionOutput(BaseModel):
15
16
 
16
17
  execution_time: float
17
18
  spans: list[ReadableSpan]
19
+ logs: list[logging.LogRecord]
18
20
  result: UiPathRuntimeResult
19
21
 
20
22
 
@@ -7,6 +7,7 @@ import os
7
7
  from typing import Any, Dict, List
8
8
 
9
9
  from opentelemetry import trace
10
+ from rich.console import Console
10
11
 
11
12
  from uipath import UiPath
12
13
  from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationStatus
@@ -68,6 +69,7 @@ class StudioWebProgressReporter:
68
69
 
69
70
  self._client = uipath.api_client
70
71
  self._console = console_logger
72
+ self._rich_console = Console()
71
73
  self._project_id = os.getenv("UIPATH_PROJECT_ID", None)
72
74
  if not self._project_id:
73
75
  logger.warning(
@@ -79,6 +81,10 @@ class StudioWebProgressReporter:
79
81
  self.evaluator_scores: Dict[str, List[float]] = {}
80
82
  self.eval_run_ids: Dict[str, str] = {}
81
83
 
84
+ def _format_error_message(self, error: Exception, context: str) -> None:
85
+ """Helper method to format and display error messages consistently."""
86
+ self._rich_console.print(f" • \u26a0 [dim]{context}: {error}[/dim]")
87
+
82
88
  @gracefully_handle_errors
83
89
  async def create_eval_set_run(
84
90
  self,
@@ -182,7 +188,7 @@ class StudioWebProgressReporter:
182
188
  logger.debug(f"Created eval set run with ID: {eval_set_run_id}")
183
189
 
184
190
  except Exception as e:
185
- logger.error(f"Failed to handle create eval set run event: {e}")
191
+ self._format_error_message(e, "StudioWeb create eval set run error")
186
192
 
187
193
  async def handle_create_eval_run(self, payload: EvalRunCreatedEvent) -> None:
188
194
  try:
@@ -197,7 +203,7 @@ class StudioWebProgressReporter:
197
203
  logger.warning("Cannot create eval run: eval_set_run_id not available")
198
204
 
199
205
  except Exception as e:
200
- logger.error(f"Failed to handle create eval run event: {e}")
206
+ self._format_error_message(e, "StudioWeb create eval run error")
201
207
 
202
208
  async def handle_update_eval_run(self, payload: EvalRunUpdatedEvent) -> None:
203
209
  try:
@@ -238,7 +244,7 @@ class StudioWebProgressReporter:
238
244
  logger.debug(f"Updated eval run with ID: {eval_run_id}")
239
245
 
240
246
  except Exception as e:
241
- logger.error(f"Failed to handle update eval run event: {e}")
247
+ self._format_error_message(e, "StudioWeb reporting error")
242
248
 
243
249
  async def handle_update_eval_set_run(self, payload: EvalSetRunUpdatedEvent) -> None:
244
250
  try:
@@ -254,7 +260,7 @@ class StudioWebProgressReporter:
254
260
  )
255
261
 
256
262
  except Exception as e:
257
- logger.error(f"Failed to handle update eval set run event: {e}")
263
+ self._format_error_message(e, "StudioWeb update eval set run error")
258
264
 
259
265
  async def subscribe_to_eval_runtime_events(self, event_bus: EventBus) -> None:
260
266
  event_bus.subscribe(
@@ -270,7 +276,7 @@ class StudioWebProgressReporter:
270
276
  EvaluationEvents.UPDATE_EVAL_SET_RUN, self.handle_update_eval_set_run
271
277
  )
272
278
 
273
- logger.info("StudioWeb progress reporter subscribed to evaluation events")
279
+ logger.debug("StudioWeb progress reporter subscribed to evaluation events")
274
280
 
275
281
  def _extract_agent_snapshot(self, entrypoint: str) -> StudioWebAgentSnapshot:
276
282
  try:
@@ -1,4 +1,6 @@
1
1
  import json
2
+ import logging
3
+ import uuid
2
4
  from collections import defaultdict
3
5
  from pathlib import Path
4
6
  from time import time
@@ -25,6 +27,7 @@ from .._runtime._contracts import (
25
27
  UiPathRuntimeResult,
26
28
  UiPathRuntimeStatus,
27
29
  )
30
+ from .._runtime._logging import ExecutionLogHandler
28
31
  from .._utils._eval_set import EvalHelpers
29
32
  from ._evaluator_factory import EvaluatorFactory
30
33
  from ._models._evaluation_set import EvaluationItem, EvaluationSet
@@ -72,6 +75,33 @@ class ExecutionSpanExporter(SpanExporter):
72
75
  self.clear()
73
76
 
74
77
 
78
+ class ExecutionLogsExporter:
79
+ """Custom exporter that stores multiple execution log handlers."""
80
+
81
+ def __init__(self):
82
+ self._log_handlers: dict[str, ExecutionLogHandler] = {}
83
+
84
+ def register(self, execution_id: str, handler: ExecutionLogHandler) -> None:
85
+ self._log_handlers[execution_id] = handler
86
+
87
+ def get_logs(self, execution_id: str) -> list[logging.LogRecord]:
88
+ """Clear stored spans for one or all executions."""
89
+ log_handler = self._log_handlers.get(execution_id)
90
+ return log_handler.buffer if log_handler else []
91
+
92
+ def clear(self, execution_id: Optional[str] = None) -> None:
93
+ """Clear stored spans for one or all executions."""
94
+ if execution_id:
95
+ self._log_handlers.pop(execution_id, None)
96
+ else:
97
+ self._log_handlers.clear()
98
+
99
+ def flush_logs(self, execution_id: str, target_handler: logging.Handler) -> None:
100
+ log_handler = self._log_handlers.get(execution_id)
101
+ if log_handler:
102
+ log_handler.flush_execution_logs(target_handler)
103
+
104
+
75
105
  class UiPathEvalContext(UiPathRuntimeContext):
76
106
  """Context used for evaluation runs."""
77
107
 
@@ -96,6 +126,8 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
96
126
  self.event_bus: EventBus = event_bus
97
127
  self.span_exporter: ExecutionSpanExporter = ExecutionSpanExporter()
98
128
  self.factory.add_span_exporter(self.span_exporter)
129
+ self.logs_exporter: ExecutionLogsExporter = ExecutionLogsExporter()
130
+ self.execution_id = str(uuid.uuid4())
99
131
 
100
132
  @classmethod
101
133
  def from_eval_context(
@@ -110,9 +142,6 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
110
142
  if self.context.eval_set is None:
111
143
  raise ValueError("eval_set must be provided for evaluation runs")
112
144
 
113
- if not self.context.execution_id:
114
- raise ValueError("execution_id must be provided for evaluation runs")
115
-
116
145
  event_bus = self.event_bus
117
146
 
118
147
  evaluation_set = EvalHelpers.load_eval_set(
@@ -126,7 +155,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
126
155
  await event_bus.publish(
127
156
  EvaluationEvents.CREATE_EVAL_SET_RUN,
128
157
  EvalSetRunCreatedEvent(
129
- execution_id=self.context.execution_id,
158
+ execution_id=self.execution_id,
130
159
  entrypoint=self.context.entrypoint or "",
131
160
  eval_set_id=evaluation_set.id,
132
161
  no_of_evals=len(evaluation_set.evaluations),
@@ -142,7 +171,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
142
171
  await event_bus.publish(
143
172
  EvaluationEvents.CREATE_EVAL_RUN,
144
173
  EvalRunCreatedEvent(
145
- execution_id=self.context.execution_id,
174
+ execution_id=self.execution_id,
146
175
  eval_item=eval_item,
147
176
  ),
148
177
  )
@@ -153,60 +182,87 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
153
182
 
154
183
  results.evaluation_set_results.append(evaluation_run_results)
155
184
 
156
- agent_execution_output = await self.execute_runtime(eval_item)
157
- evaluation_item_results: list[EvalItemResult] = []
158
-
159
- for evaluator in evaluators:
160
- evaluation_result = await self.run_evaluator(
161
- evaluator=evaluator,
162
- execution_output=agent_execution_output,
163
- eval_item=eval_item,
164
- )
185
+ try:
186
+ agent_execution_output = await self.execute_runtime(eval_item)
187
+ evaluation_item_results: list[EvalItemResult] = []
188
+
189
+ for evaluator in evaluators:
190
+ evaluation_result = await self.run_evaluator(
191
+ evaluator=evaluator,
192
+ execution_output=agent_execution_output,
193
+ eval_item=eval_item,
194
+ )
165
195
 
166
- dto_result = EvaluationResultDto.from_evaluation_result(
167
- evaluation_result
168
- )
169
- evaluator_counts[evaluator.id] += 1
170
- count = evaluator_counts[evaluator.id]
171
- evaluator_averages[evaluator.id] += (
172
- dto_result.score - evaluator_averages[evaluator.id]
173
- ) / count
174
-
175
- evaluation_run_results.evaluation_run_results.append(
176
- EvaluationRunResultDto(
177
- evaluator_name=evaluator.name,
178
- result=dto_result,
196
+ dto_result = EvaluationResultDto.from_evaluation_result(
197
+ evaluation_result
179
198
  )
180
- )
181
- evaluation_item_results.append(
182
- EvalItemResult(
183
- evaluator_id=evaluator.id,
184
- result=evaluation_result,
199
+ evaluator_counts[evaluator.id] += 1
200
+ count = evaluator_counts[evaluator.id]
201
+ evaluator_averages[evaluator.id] += (
202
+ dto_result.score - evaluator_averages[evaluator.id]
203
+ ) / count
204
+
205
+ evaluation_run_results.evaluation_run_results.append(
206
+ EvaluationRunResultDto(
207
+ evaluator_name=evaluator.name,
208
+ result=dto_result,
209
+ )
210
+ )
211
+ evaluation_item_results.append(
212
+ EvalItemResult(
213
+ evaluator_id=evaluator.id,
214
+ result=evaluation_result,
215
+ )
185
216
  )
186
- )
187
-
188
- evaluation_run_results.compute_average_score()
189
217
 
190
- await event_bus.publish(
191
- EvaluationEvents.UPDATE_EVAL_RUN,
192
- EvalRunUpdatedEvent(
193
- execution_id=self.context.execution_id,
194
- eval_item=eval_item,
195
- eval_results=evaluation_item_results,
196
- success=not agent_execution_output.result.error,
197
- agent_output=agent_execution_output.result.output,
198
- agent_execution_time=agent_execution_output.execution_time,
199
- spans=agent_execution_output.spans,
200
- ),
201
- wait_for_completion=False,
202
- )
218
+ evaluation_run_results.compute_average_score()
219
+
220
+ await event_bus.publish(
221
+ EvaluationEvents.UPDATE_EVAL_RUN,
222
+ EvalRunUpdatedEvent(
223
+ execution_id=self.execution_id,
224
+ eval_item=eval_item,
225
+ eval_results=evaluation_item_results,
226
+ success=not agent_execution_output.result.error,
227
+ agent_output=agent_execution_output.result.output,
228
+ agent_execution_time=agent_execution_output.execution_time,
229
+ spans=agent_execution_output.spans,
230
+ logs=agent_execution_output.logs,
231
+ ),
232
+ wait_for_completion=False,
233
+ )
234
+ except Exception as e:
235
+ error_msg = str(e)
236
+ eval_item._error_message = error_msg # type: ignore[attr-defined]
237
+
238
+ for evaluator in evaluators:
239
+ evaluator_counts[evaluator.id] += 1
240
+ count = evaluator_counts[evaluator.id]
241
+ evaluator_averages[evaluator.id] += (
242
+ 0.0 - evaluator_averages[evaluator.id]
243
+ ) / count
244
+
245
+ await event_bus.publish(
246
+ EvaluationEvents.UPDATE_EVAL_RUN,
247
+ EvalRunUpdatedEvent(
248
+ execution_id=self.execution_id,
249
+ eval_item=eval_item,
250
+ eval_results=[],
251
+ success=False,
252
+ agent_output={},
253
+ agent_execution_time=0.0,
254
+ spans=[],
255
+ logs=[],
256
+ ),
257
+ wait_for_completion=False,
258
+ )
203
259
 
204
260
  results.compute_average_score()
205
261
 
206
262
  await event_bus.publish(
207
263
  EvaluationEvents.UPDATE_EVAL_SET_RUN,
208
264
  EvalSetRunUpdatedEvent(
209
- execution_id=self.context.execution_id,
265
+ execution_id=self.execution_id,
210
266
  evaluator_scores=evaluator_averages,
211
267
  ),
212
268
  wait_for_completion=False,
@@ -221,10 +277,12 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
221
277
  async def execute_runtime(
222
278
  self, eval_item: EvaluationItem
223
279
  ) -> UiPathEvalRunExecutionOutput:
280
+ eval_item_id = eval_item.id
224
281
  runtime_context: C = self.factory.new_context(
225
- execution_id=eval_item.id,
282
+ execution_id=eval_item_id,
226
283
  input_json=eval_item.inputs,
227
284
  is_eval_run=True,
285
+ log_handler=self._setup_execution_logging(eval_item_id),
228
286
  )
229
287
  attributes = {
230
288
  "evalId": eval_item.id,
@@ -247,15 +305,24 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
247
305
  spans = self.span_exporter.get_spans(runtime_context.execution_id)
248
306
  self.span_exporter.clear(runtime_context.execution_id)
249
307
 
308
+ logs = self.logs_exporter.get_logs(runtime_context.execution_id)
309
+ self.logs_exporter.clear(runtime_context.execution_id)
310
+
250
311
  if result is None:
251
312
  raise ValueError("Execution result cannot be None for eval runs")
252
313
 
253
314
  return UiPathEvalRunExecutionOutput(
254
315
  execution_time=end_time - start_time,
255
316
  spans=spans,
317
+ logs=logs,
256
318
  result=result,
257
319
  )
258
320
 
321
+ def _setup_execution_logging(self, eval_item_id: str) -> ExecutionLogHandler:
322
+ execution_log_handler = ExecutionLogHandler(eval_item_id)
323
+ self.logs_exporter.register(eval_item_id, execution_log_handler)
324
+ return execution_log_handler
325
+
259
326
  async def run_evaluator(
260
327
  self,
261
328
  evaluator: BaseEvaluator[Any],
@@ -519,7 +519,8 @@ class UiPathBaseRuntime(ABC):
519
519
 
520
520
  await self.validate()
521
521
 
522
- # Intercept all stdout/stderr/logs and write them to a file (runtime/evals), stdout (debug)
522
+ # Intercept all stdout/stderr/logs
523
+ # write to file (runtime) or stdout (debug)
523
524
  self.logs_interceptor = LogsInterceptor(
524
525
  min_level=self.context.logs_min_level,
525
526
  dir=self.context.runtime_dir,
@@ -649,7 +650,7 @@ class UiPathBaseRuntime(ABC):
649
650
  raise
650
651
  finally:
651
652
  # Restore original logging
652
- if self.logs_interceptor:
653
+ if hasattr(self, "logs_interceptor"):
653
654
  self.logs_interceptor.teardown()
654
655
 
655
656
  await self.cleanup()
@@ -698,6 +699,7 @@ class UiPathRuntimeFactory(Generic[T, C]):
698
699
  self.context_generator = context_generator
699
700
  self.tracer_provider: TracerProvider = TracerProvider()
700
701
  self.tracer_span_processors: List[SpanProcessor] = []
702
+ self.logs_exporter: Optional[Any] = None
701
703
  trace.set_tracer_provider(self.tracer_provider)
702
704
 
703
705
  def add_span_exporter(