uipath 2.1.74__tar.gz → 2.1.75__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {uipath-2.1.74 → uipath-2.1.75}/PKG-INFO +1 -1
- {uipath-2.1.74 → uipath-2.1.75}/pyproject.toml +1 -1
- {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/evals/eval-sets/default.json +2 -1
- uipath-2.1.75/samples/calculator/evals/evaluators/llm-as-a-judge.json +13 -0
- uipath-2.1.75/src/uipath/_cli/_evals/_console_progress_reporter.py +241 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_output.py +2 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_progress_reporter.py +11 -5
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_runtime.py +117 -50
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_contracts.py +4 -2
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_logging.py +29 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_eval.py +5 -5
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/middlewares.py +3 -3
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_events/_events.py +2 -0
- {uipath-2.1.74 → uipath-2.1.75}/uv.lock +1 -1
- {uipath-2.1.74 → uipath-2.1.75}/.cursorrules +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.editorconfig +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.gitattributes +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/labeler.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/auto-label.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/cd.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/ci.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/commitlint.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/integration_tests.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/lint.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/publish-dev.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/publish-docs.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/slack.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/test-uipath-langchain.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/test-uipath-llamaindex.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.github/workflows/test.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.gitignore +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.pre-commit-config.yaml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.python-version +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.vscode/extensions.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.vscode/launch.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/.vscode/settings.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/CONTRIBUTING.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/LICENSE +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/README.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/CONTRIBUTING.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/FAQ.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/assets/env-preparation-failed-dark.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/assets/env-preparation-failed-light.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/assets/favicon.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/assets/logo-dark.svg +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/assets/logo-light.svg +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/cli/index.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/actions.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_dark.gif +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_light.gif +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_secret_dark.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/cloud_env_var_secret_light.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/copy_path_dark.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets/copy_path_light.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/assets.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/attachments.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/buckets.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/connections.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/context_grounding.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/documents.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/documents_models.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/environment_variables.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/getting_started.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/jobs.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/llm_gateway.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/processes.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/queues.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/core/traced.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/hooks.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/index.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/javascripts/extra.js +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/overrides/main.html +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/overrides/partials/actions.html +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/overrides/partials/logo.html +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/release_policy.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/sample_images/google-ADK-agent/agent-output.png +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/docs/stylesheets/extra.css +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/justfile +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/mkdocs.yml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/py.typed +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/README.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/evals/evaluators/equality.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/main.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/calculator/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/.python-version +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/README.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/main.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/event-trigger/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/.env.example +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/README.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/input.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/multi_tool_agent/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/multi_tool_agent/agent.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/samples/google-ADK-agent/uv.lock +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/scripts/debug_test.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/scripts/lint_httpx_client.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/README.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_auth_server.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_auth_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_client_credentials.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_models.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_oidc_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_portal_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_url_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/auth_config.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/index.html +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/localhost.crt +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_auth/localhost.key +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_chat.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_details.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_history.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_json_input.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_components/_new.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_models/_execution.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_models/_messages.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_styles/terminal.tcss +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_utils/_chat.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_utils/_exporter.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_dev/_terminal/_utils/_logger.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_evaluator_factory.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_evaluation_set.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_evaluator_base_params.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_sw_reporting.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/_models/_trajectory_span.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/llm_mocker.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mocker.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mocker_factory.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mockito_mocker.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_evals/mocks/mocks.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_push/sw_file_handler.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_escalation.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_hitl.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_runtime.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_runtime/_script_executor.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/.psmdcp.template +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/.rels.template +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/[Content_Types].xml.template +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/main.py.template +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_templates/package.nuspec.template +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_common.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_console.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_constants.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_debug.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_eval_set.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_folders.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_input_args.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_parse_ast.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_processes.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_project_files.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_studio_project.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_tracing.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/_utils/_uv_helpers.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_auth.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_deploy.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_dev.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_init.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_invoke.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_new.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_pack.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_publish.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_pull.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_push.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/cli_run.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_cli/spinner.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_config.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_events/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_events/_event_bus.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_execution_context.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_folder_context.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_resources/AGENTS.md +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/_base_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/actions_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/api_client.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/assets_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/attachments_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/buckets_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/connections_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/context_grounding_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/documents_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/entities_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/folder_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/jobs_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/llm_gateway_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/processes_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_services/queues_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_uipath.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_endpoint.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_infer_bindings.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_logs.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_read_overwrites.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_request_override.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_request_spec.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_ssl_context.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_url.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/_user_agent.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/_utils/constants.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/async_stream.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/citation.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/content.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/conversation.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/event.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/exchange.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/message.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/meta.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/conversation/tool.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/agent/models/agent.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/_helpers/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/_helpers/helpers.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/base_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/deterministic_evaluator_base.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/exact_match_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/json_similarity_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/llm_as_judge_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/evaluators/trajectory_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/models/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/eval/models/models.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/action_schema.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/actions.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/assets.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/attachment.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/buckets.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/connections.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/context_grounding.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/context_grounding_index.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/documents.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/entities.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/errors.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/exceptions.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/interrupt_models.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/job.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/llm_gateway.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/processes.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/models/queues.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/py.typed +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/telemetry/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/telemetry/_constants.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/telemetry/_track.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/_otel_exporters.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/_traced.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/tracing/_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/utils/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/src/uipath/utils/_endpoints_manager.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/main.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/run.sh +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/apicalls-testcase/src/assert.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/main.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/run.sh +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/basic-testcase/src/assert.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/common/validate_output.sh +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/agent.mermaid +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/langgraph.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/run.sh +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/src/assert.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/langchain-cross/src/main.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/agent.mermaid +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/llama_index.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/run.sh +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/src/assert.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/testcases/llamaindex-cross/src/main.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/__init__.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/agent/models/test_agent.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/conftest.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/eval/mocks/test_mocks.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/evaluators/test_json_similarity_evaluator.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/bindings_script.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/pyproject.toml +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/simple_script.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/uipath-mock.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/mocks/uipath-simple-script-mock.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_auth.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_hitl.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_init.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_init_agents_md.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_input_args.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_invoke.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_new.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_pack.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_portal_service_ensure_valid_token.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_portal_service_refresh_token.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_publish.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_pull.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_push.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_run.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/test_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/utils/common.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/utils/project_details.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/cli/utils/uipath_json.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/conftest.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/conftest.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_actions_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_api_client.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_assets_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_attachments_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_base_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_buckets_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_connections_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_context_grounding_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_documents_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_entities_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_folder_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_jobs_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_llm_integration.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_llm_schema_cleanup.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_llm_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_processes_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_queues_service.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/test_uipath_llm_integration.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/tests_data/documents_service/create_validation_action_response.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/tests_data/documents_service/extraction_response.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/services/tests_data/documents_service/validated_result.json +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/test_bindings_inference.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/test_config.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/sdk/test_overwrites.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_otel_exporters.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_span_utils.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_traced.py +0 -0
- {uipath-2.1.74 → uipath-2.1.75}/tests/tracing/test_tracing_manager.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: uipath
|
3
|
-
Version: 2.1.
|
3
|
+
Version: 2.1.75
|
4
4
|
Summary: Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools.
|
5
5
|
Project-URL: Homepage, https://uipath.com
|
6
6
|
Project-URL: Repository, https://github.com/UiPath/uipath-python
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[project]
|
2
2
|
name = "uipath"
|
3
|
-
version = "2.1.
|
3
|
+
version = "2.1.75"
|
4
4
|
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
|
5
5
|
readme = { file = "README.md", content-type = "text/markdown" }
|
6
6
|
requires-python = ">=3.10"
|
@@ -0,0 +1,13 @@
|
|
1
|
+
{
|
2
|
+
"fileName": "llm-as-a-judge.json",
|
3
|
+
"id": "llm-as-a-judge",
|
4
|
+
"name": "LLMAsAJudge Evaluator",
|
5
|
+
"description": "An evaluator that judges the agent based on it's run history and expected behavior",
|
6
|
+
"category": 3,
|
7
|
+
"type": 7,
|
8
|
+
"prompt": "As an expert evaluator, determine how well the agent did on a scale of 0-100. Focus on if the simulation was successful and if the agent behaved according to the expected output accounting for alternative valid expressions, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.\n----\nUserOrSyntheticInputGivenToAgent:\n{{UserOrSyntheticInput}}\n----\nSimulationInstructions:\n{{SimulationInstructions}}\n----\nExpectedAgentBehavior:\n{{ExpectedAgentBehavior}}\n----\nAgentRunHistory:\n{{AgentRunHistory}}\n",
|
9
|
+
"targetOutputKey": "*",
|
10
|
+
"model":"gpt-4.1-2025-04-14",
|
11
|
+
"createdAt": "2025-08-21T15:12:58.695Z",
|
12
|
+
"updatedAt": "2025-08-21T15:12:58.695Z"
|
13
|
+
}
|
@@ -0,0 +1,241 @@
|
|
1
|
+
"""Console progress reporter for evaluation runs with line-by-line output."""
|
2
|
+
|
3
|
+
import logging
|
4
|
+
from typing import Any, Dict
|
5
|
+
|
6
|
+
from rich.console import Console
|
7
|
+
from rich.rule import Rule
|
8
|
+
from rich.table import Table
|
9
|
+
|
10
|
+
from uipath._events._event_bus import EventBus
|
11
|
+
from uipath._events._events import (
|
12
|
+
EvalRunCreatedEvent,
|
13
|
+
EvalRunUpdatedEvent,
|
14
|
+
EvalSetRunCreatedEvent,
|
15
|
+
EvalSetRunUpdatedEvent,
|
16
|
+
EvaluationEvents,
|
17
|
+
)
|
18
|
+
from uipath.eval.evaluators import BaseEvaluator
|
19
|
+
from uipath.eval.models import ScoreType
|
20
|
+
|
21
|
+
logger = logging.getLogger(__name__)
|
22
|
+
|
23
|
+
|
24
|
+
class ConsoleProgressReporter:
|
25
|
+
"""Handles displaying evaluation progress to the console with line-by-line output."""
|
26
|
+
|
27
|
+
def __init__(self):
|
28
|
+
self.console = Console()
|
29
|
+
self.evaluators: Dict[str, BaseEvaluator[Any]] = {}
|
30
|
+
self.display_started = False
|
31
|
+
self.eval_results_by_name: Dict[str, list[Any]] = {}
|
32
|
+
|
33
|
+
def _convert_score_to_numeric(self, eval_result) -> float:
|
34
|
+
"""Convert evaluation result score to numeric value."""
|
35
|
+
score_value = eval_result.result.score
|
36
|
+
if eval_result.result.score_type == ScoreType.BOOLEAN:
|
37
|
+
score_value = 100 if score_value else 0
|
38
|
+
return score_value
|
39
|
+
|
40
|
+
def _get_evaluator_name(self, evaluator_id: str) -> str:
|
41
|
+
"""Get evaluator name from ID, with fallback."""
|
42
|
+
return self.evaluators.get(
|
43
|
+
evaluator_id,
|
44
|
+
type(
|
45
|
+
"obj",
|
46
|
+
(object,),
|
47
|
+
{"name": f"Evaluator {evaluator_id[:8]}"},
|
48
|
+
)(),
|
49
|
+
).name
|
50
|
+
|
51
|
+
def _display_successful_evaluation(self, eval_name: str, eval_results) -> None:
|
52
|
+
"""Display results for a successful evaluation."""
|
53
|
+
from rich.text import Text
|
54
|
+
|
55
|
+
if eval_results:
|
56
|
+
result = Text()
|
57
|
+
result.append("▌", style="bold green")
|
58
|
+
result.append(" ", style="")
|
59
|
+
result.append(eval_name, style="bold green")
|
60
|
+
self.console.print(result)
|
61
|
+
table = Table(show_header=False, box=None, padding=(0, 2, 0, 2))
|
62
|
+
|
63
|
+
for eval_result in eval_results:
|
64
|
+
evaluator_name = self._get_evaluator_name(eval_result.evaluator_id)
|
65
|
+
score_value = self._convert_score_to_numeric(eval_result)
|
66
|
+
table.add_row(
|
67
|
+
f"{evaluator_name}", f"[bold cyan]{score_value:.1f}[/bold cyan]"
|
68
|
+
)
|
69
|
+
|
70
|
+
self.console.print(table)
|
71
|
+
else:
|
72
|
+
result = Text()
|
73
|
+
result.append(" ✓ ", style="bold green")
|
74
|
+
result.append(eval_name, style="bold white")
|
75
|
+
result.append(" - No evaluators", style="dim")
|
76
|
+
self.console.print(result)
|
77
|
+
|
78
|
+
def _extract_error_message(self, eval_item_payload) -> str:
|
79
|
+
"""Extract clean error message from evaluation item."""
|
80
|
+
if hasattr(eval_item_payload, "_error_message"):
|
81
|
+
error_message = getattr(eval_item_payload, "_error_message", None)
|
82
|
+
if error_message:
|
83
|
+
return str(error_message) or "Execution failed"
|
84
|
+
return "Execution failed"
|
85
|
+
|
86
|
+
def _display_failed_evaluation(self, eval_name: str, error_msg: str) -> None:
|
87
|
+
"""Display results for a failed evaluation."""
|
88
|
+
from rich.text import Text
|
89
|
+
|
90
|
+
result = Text()
|
91
|
+
result.append(" ✗ ", style="bold red")
|
92
|
+
result.append(eval_name, style="bold white")
|
93
|
+
self.console.print(result)
|
94
|
+
|
95
|
+
error_text = Text()
|
96
|
+
error_text.append(" ", style="")
|
97
|
+
error_text.append(error_msg, style="red")
|
98
|
+
self.console.print(error_text)
|
99
|
+
|
100
|
+
def start_display(self):
|
101
|
+
"""Start the display."""
|
102
|
+
if not self.display_started:
|
103
|
+
self.console.print()
|
104
|
+
self.console.print("→ [bold]Running Evaluations[/bold]")
|
105
|
+
self.console.print()
|
106
|
+
self.display_started = True
|
107
|
+
|
108
|
+
async def handle_create_eval_set_run(self, payload: EvalSetRunCreatedEvent) -> None:
|
109
|
+
"""Handle evaluation set run creation."""
|
110
|
+
try:
|
111
|
+
self.evaluators = {eval.id: eval for eval in payload.evaluators}
|
112
|
+
except Exception as e:
|
113
|
+
logger.error(f"Failed to handle create eval set run event: {e}")
|
114
|
+
|
115
|
+
async def handle_create_eval_run(self, payload: EvalRunCreatedEvent) -> None:
|
116
|
+
"""Handle individual evaluation run creation."""
|
117
|
+
try:
|
118
|
+
if not self.display_started:
|
119
|
+
self.start_display()
|
120
|
+
|
121
|
+
self.console.print(f" ○ [dim]{payload.eval_item.name}[/dim] - Running...")
|
122
|
+
except Exception as e:
|
123
|
+
logger.error(f"Failed to handle create eval run event: {e}")
|
124
|
+
|
125
|
+
async def handle_update_eval_run(self, payload: EvalRunUpdatedEvent) -> None:
|
126
|
+
"""Handle evaluation run updates."""
|
127
|
+
try:
|
128
|
+
if payload.success:
|
129
|
+
# Store results for final display
|
130
|
+
self.eval_results_by_name[payload.eval_item.name] = payload.eval_results
|
131
|
+
self._display_successful_evaluation(
|
132
|
+
payload.eval_item.name, payload.eval_results
|
133
|
+
)
|
134
|
+
else:
|
135
|
+
error_msg = self._extract_error_message(payload.eval_item)
|
136
|
+
self._display_failed_evaluation(payload.eval_item.name, error_msg)
|
137
|
+
|
138
|
+
logs = payload.logs
|
139
|
+
|
140
|
+
self.console.print(
|
141
|
+
Rule(
|
142
|
+
f"[dim italic]Execution Logs: {payload.eval_item.name}[/dim italic]",
|
143
|
+
style="dim",
|
144
|
+
align="center",
|
145
|
+
)
|
146
|
+
)
|
147
|
+
|
148
|
+
if len(logs) > 0:
|
149
|
+
for record in logs:
|
150
|
+
log_line = f" [dim]{record.getMessage()}[/dim]"
|
151
|
+
self.console.print(log_line)
|
152
|
+
else:
|
153
|
+
self.console.print(" [dim italic]No execution logs[/dim italic]")
|
154
|
+
|
155
|
+
self.console.print(Rule(style="dim"))
|
156
|
+
except Exception as e:
|
157
|
+
logger.error(f"Console reporter error: {e}")
|
158
|
+
|
159
|
+
async def handle_update_eval_set_run(self, payload: EvalSetRunUpdatedEvent) -> None:
|
160
|
+
"""Handle evaluation set run completion."""
|
161
|
+
try:
|
162
|
+
self.final_results = payload.evaluator_scores
|
163
|
+
self.display_final_results()
|
164
|
+
except Exception as e:
|
165
|
+
logger.error(f"Console reporter error: {e}")
|
166
|
+
|
167
|
+
def display_final_results(self):
|
168
|
+
"""Display final results summary."""
|
169
|
+
self.console.print()
|
170
|
+
|
171
|
+
if hasattr(self, "final_results") and self.final_results:
|
172
|
+
from rich.table import Table
|
173
|
+
|
174
|
+
# Group evaluators by ID to organize display
|
175
|
+
evaluator_ids = list(self.final_results.keys())
|
176
|
+
|
177
|
+
# Print title
|
178
|
+
self.console.print("[bold]Evaluation Results[/bold]")
|
179
|
+
self.console.print()
|
180
|
+
|
181
|
+
# Create single summary table
|
182
|
+
summary_table = Table(show_header=True, padding=(0, 2))
|
183
|
+
summary_table.add_column("Evaluation", style="cyan")
|
184
|
+
|
185
|
+
# Add column for each evaluator
|
186
|
+
for evaluator_id in evaluator_ids:
|
187
|
+
evaluator_name = self._get_evaluator_name(evaluator_id)
|
188
|
+
summary_table.add_column(evaluator_name, justify="right")
|
189
|
+
|
190
|
+
# Add row for each evaluation
|
191
|
+
for eval_name, eval_results in self.eval_results_by_name.items():
|
192
|
+
row_values = [eval_name]
|
193
|
+
|
194
|
+
# Get score for each evaluator
|
195
|
+
for evaluator_id in evaluator_ids:
|
196
|
+
score_found = False
|
197
|
+
for eval_result in eval_results:
|
198
|
+
if eval_result.evaluator_id == evaluator_id:
|
199
|
+
score_value = self._convert_score_to_numeric(eval_result)
|
200
|
+
row_values.append(f"{score_value:.1f}")
|
201
|
+
score_found = True
|
202
|
+
break
|
203
|
+
|
204
|
+
if not score_found:
|
205
|
+
row_values.append("-")
|
206
|
+
|
207
|
+
summary_table.add_row(*row_values)
|
208
|
+
|
209
|
+
# Add separator row before average
|
210
|
+
summary_table.add_section()
|
211
|
+
|
212
|
+
# Add average row
|
213
|
+
avg_row_values = ["[bold]Average[/bold]"]
|
214
|
+
for evaluator_id in evaluator_ids:
|
215
|
+
avg_score = self.final_results[evaluator_id]
|
216
|
+
avg_row_values.append(f"[bold]{avg_score:.1f}[/bold]")
|
217
|
+
|
218
|
+
summary_table.add_row(*avg_row_values)
|
219
|
+
|
220
|
+
self.console.print(summary_table)
|
221
|
+
self.console.print()
|
222
|
+
else:
|
223
|
+
self.console.print(
|
224
|
+
"→ [bold green]All evaluations completed successfully![/bold green]"
|
225
|
+
)
|
226
|
+
self.console.print()
|
227
|
+
|
228
|
+
async def subscribe_to_eval_runtime_events(self, event_bus: EventBus) -> None:
|
229
|
+
"""Subscribe to evaluation runtime events."""
|
230
|
+
event_bus.subscribe(
|
231
|
+
EvaluationEvents.CREATE_EVAL_SET_RUN, self.handle_create_eval_set_run
|
232
|
+
)
|
233
|
+
event_bus.subscribe(
|
234
|
+
EvaluationEvents.CREATE_EVAL_RUN, self.handle_create_eval_run
|
235
|
+
)
|
236
|
+
event_bus.subscribe(
|
237
|
+
EvaluationEvents.UPDATE_EVAL_RUN, self.handle_update_eval_run
|
238
|
+
)
|
239
|
+
event_bus.subscribe(
|
240
|
+
EvaluationEvents.UPDATE_EVAL_SET_RUN, self.handle_update_eval_set_run
|
241
|
+
)
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import logging
|
1
2
|
from typing import List, Optional
|
2
3
|
|
3
4
|
from opentelemetry.sdk.trace import ReadableSpan
|
@@ -15,6 +16,7 @@ class UiPathEvalRunExecutionOutput(BaseModel):
|
|
15
16
|
|
16
17
|
execution_time: float
|
17
18
|
spans: list[ReadableSpan]
|
19
|
+
logs: list[logging.LogRecord]
|
18
20
|
result: UiPathRuntimeResult
|
19
21
|
|
20
22
|
|
@@ -7,6 +7,7 @@ import os
|
|
7
7
|
from typing import Any, Dict, List
|
8
8
|
|
9
9
|
from opentelemetry import trace
|
10
|
+
from rich.console import Console
|
10
11
|
|
11
12
|
from uipath import UiPath
|
12
13
|
from uipath._cli._evals._models._evaluation_set import EvaluationItem, EvaluationStatus
|
@@ -68,6 +69,7 @@ class StudioWebProgressReporter:
|
|
68
69
|
|
69
70
|
self._client = uipath.api_client
|
70
71
|
self._console = console_logger
|
72
|
+
self._rich_console = Console()
|
71
73
|
self._project_id = os.getenv("UIPATH_PROJECT_ID", None)
|
72
74
|
if not self._project_id:
|
73
75
|
logger.warning(
|
@@ -79,6 +81,10 @@ class StudioWebProgressReporter:
|
|
79
81
|
self.evaluator_scores: Dict[str, List[float]] = {}
|
80
82
|
self.eval_run_ids: Dict[str, str] = {}
|
81
83
|
|
84
|
+
def _format_error_message(self, error: Exception, context: str) -> None:
|
85
|
+
"""Helper method to format and display error messages consistently."""
|
86
|
+
self._rich_console.print(f" • \u26a0 [dim]{context}: {error}[/dim]")
|
87
|
+
|
82
88
|
@gracefully_handle_errors
|
83
89
|
async def create_eval_set_run(
|
84
90
|
self,
|
@@ -182,7 +188,7 @@ class StudioWebProgressReporter:
|
|
182
188
|
logger.debug(f"Created eval set run with ID: {eval_set_run_id}")
|
183
189
|
|
184
190
|
except Exception as e:
|
185
|
-
|
191
|
+
self._format_error_message(e, "StudioWeb create eval set run error")
|
186
192
|
|
187
193
|
async def handle_create_eval_run(self, payload: EvalRunCreatedEvent) -> None:
|
188
194
|
try:
|
@@ -197,7 +203,7 @@ class StudioWebProgressReporter:
|
|
197
203
|
logger.warning("Cannot create eval run: eval_set_run_id not available")
|
198
204
|
|
199
205
|
except Exception as e:
|
200
|
-
|
206
|
+
self._format_error_message(e, "StudioWeb create eval run error")
|
201
207
|
|
202
208
|
async def handle_update_eval_run(self, payload: EvalRunUpdatedEvent) -> None:
|
203
209
|
try:
|
@@ -238,7 +244,7 @@ class StudioWebProgressReporter:
|
|
238
244
|
logger.debug(f"Updated eval run with ID: {eval_run_id}")
|
239
245
|
|
240
246
|
except Exception as e:
|
241
|
-
|
247
|
+
self._format_error_message(e, "StudioWeb reporting error")
|
242
248
|
|
243
249
|
async def handle_update_eval_set_run(self, payload: EvalSetRunUpdatedEvent) -> None:
|
244
250
|
try:
|
@@ -254,7 +260,7 @@ class StudioWebProgressReporter:
|
|
254
260
|
)
|
255
261
|
|
256
262
|
except Exception as e:
|
257
|
-
|
263
|
+
self._format_error_message(e, "StudioWeb update eval set run error")
|
258
264
|
|
259
265
|
async def subscribe_to_eval_runtime_events(self, event_bus: EventBus) -> None:
|
260
266
|
event_bus.subscribe(
|
@@ -270,7 +276,7 @@ class StudioWebProgressReporter:
|
|
270
276
|
EvaluationEvents.UPDATE_EVAL_SET_RUN, self.handle_update_eval_set_run
|
271
277
|
)
|
272
278
|
|
273
|
-
logger.
|
279
|
+
logger.debug("StudioWeb progress reporter subscribed to evaluation events")
|
274
280
|
|
275
281
|
def _extract_agent_snapshot(self, entrypoint: str) -> StudioWebAgentSnapshot:
|
276
282
|
try:
|
@@ -1,4 +1,6 @@
|
|
1
1
|
import json
|
2
|
+
import logging
|
3
|
+
import uuid
|
2
4
|
from collections import defaultdict
|
3
5
|
from pathlib import Path
|
4
6
|
from time import time
|
@@ -25,6 +27,7 @@ from .._runtime._contracts import (
|
|
25
27
|
UiPathRuntimeResult,
|
26
28
|
UiPathRuntimeStatus,
|
27
29
|
)
|
30
|
+
from .._runtime._logging import ExecutionLogHandler
|
28
31
|
from .._utils._eval_set import EvalHelpers
|
29
32
|
from ._evaluator_factory import EvaluatorFactory
|
30
33
|
from ._models._evaluation_set import EvaluationItem, EvaluationSet
|
@@ -72,6 +75,33 @@ class ExecutionSpanExporter(SpanExporter):
|
|
72
75
|
self.clear()
|
73
76
|
|
74
77
|
|
78
|
+
class ExecutionLogsExporter:
|
79
|
+
"""Custom exporter that stores multiple execution log handlers."""
|
80
|
+
|
81
|
+
def __init__(self):
|
82
|
+
self._log_handlers: dict[str, ExecutionLogHandler] = {}
|
83
|
+
|
84
|
+
def register(self, execution_id: str, handler: ExecutionLogHandler) -> None:
|
85
|
+
self._log_handlers[execution_id] = handler
|
86
|
+
|
87
|
+
def get_logs(self, execution_id: str) -> list[logging.LogRecord]:
|
88
|
+
"""Clear stored spans for one or all executions."""
|
89
|
+
log_handler = self._log_handlers.get(execution_id)
|
90
|
+
return log_handler.buffer if log_handler else []
|
91
|
+
|
92
|
+
def clear(self, execution_id: Optional[str] = None) -> None:
|
93
|
+
"""Clear stored spans for one or all executions."""
|
94
|
+
if execution_id:
|
95
|
+
self._log_handlers.pop(execution_id, None)
|
96
|
+
else:
|
97
|
+
self._log_handlers.clear()
|
98
|
+
|
99
|
+
def flush_logs(self, execution_id: str, target_handler: logging.Handler) -> None:
|
100
|
+
log_handler = self._log_handlers.get(execution_id)
|
101
|
+
if log_handler:
|
102
|
+
log_handler.flush_execution_logs(target_handler)
|
103
|
+
|
104
|
+
|
75
105
|
class UiPathEvalContext(UiPathRuntimeContext):
|
76
106
|
"""Context used for evaluation runs."""
|
77
107
|
|
@@ -96,6 +126,8 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
96
126
|
self.event_bus: EventBus = event_bus
|
97
127
|
self.span_exporter: ExecutionSpanExporter = ExecutionSpanExporter()
|
98
128
|
self.factory.add_span_exporter(self.span_exporter)
|
129
|
+
self.logs_exporter: ExecutionLogsExporter = ExecutionLogsExporter()
|
130
|
+
self.execution_id = str(uuid.uuid4())
|
99
131
|
|
100
132
|
@classmethod
|
101
133
|
def from_eval_context(
|
@@ -110,9 +142,6 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
110
142
|
if self.context.eval_set is None:
|
111
143
|
raise ValueError("eval_set must be provided for evaluation runs")
|
112
144
|
|
113
|
-
if not self.context.execution_id:
|
114
|
-
raise ValueError("execution_id must be provided for evaluation runs")
|
115
|
-
|
116
145
|
event_bus = self.event_bus
|
117
146
|
|
118
147
|
evaluation_set = EvalHelpers.load_eval_set(
|
@@ -126,7 +155,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
126
155
|
await event_bus.publish(
|
127
156
|
EvaluationEvents.CREATE_EVAL_SET_RUN,
|
128
157
|
EvalSetRunCreatedEvent(
|
129
|
-
execution_id=self.
|
158
|
+
execution_id=self.execution_id,
|
130
159
|
entrypoint=self.context.entrypoint or "",
|
131
160
|
eval_set_id=evaluation_set.id,
|
132
161
|
no_of_evals=len(evaluation_set.evaluations),
|
@@ -142,7 +171,7 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
142
171
|
await event_bus.publish(
|
143
172
|
EvaluationEvents.CREATE_EVAL_RUN,
|
144
173
|
EvalRunCreatedEvent(
|
145
|
-
execution_id=self.
|
174
|
+
execution_id=self.execution_id,
|
146
175
|
eval_item=eval_item,
|
147
176
|
),
|
148
177
|
)
|
@@ -153,60 +182,87 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
153
182
|
|
154
183
|
results.evaluation_set_results.append(evaluation_run_results)
|
155
184
|
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
185
|
+
try:
|
186
|
+
agent_execution_output = await self.execute_runtime(eval_item)
|
187
|
+
evaluation_item_results: list[EvalItemResult] = []
|
188
|
+
|
189
|
+
for evaluator in evaluators:
|
190
|
+
evaluation_result = await self.run_evaluator(
|
191
|
+
evaluator=evaluator,
|
192
|
+
execution_output=agent_execution_output,
|
193
|
+
eval_item=eval_item,
|
194
|
+
)
|
165
195
|
|
166
|
-
|
167
|
-
|
168
|
-
)
|
169
|
-
evaluator_counts[evaluator.id] += 1
|
170
|
-
count = evaluator_counts[evaluator.id]
|
171
|
-
evaluator_averages[evaluator.id] += (
|
172
|
-
dto_result.score - evaluator_averages[evaluator.id]
|
173
|
-
) / count
|
174
|
-
|
175
|
-
evaluation_run_results.evaluation_run_results.append(
|
176
|
-
EvaluationRunResultDto(
|
177
|
-
evaluator_name=evaluator.name,
|
178
|
-
result=dto_result,
|
196
|
+
dto_result = EvaluationResultDto.from_evaluation_result(
|
197
|
+
evaluation_result
|
179
198
|
)
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
199
|
+
evaluator_counts[evaluator.id] += 1
|
200
|
+
count = evaluator_counts[evaluator.id]
|
201
|
+
evaluator_averages[evaluator.id] += (
|
202
|
+
dto_result.score - evaluator_averages[evaluator.id]
|
203
|
+
) / count
|
204
|
+
|
205
|
+
evaluation_run_results.evaluation_run_results.append(
|
206
|
+
EvaluationRunResultDto(
|
207
|
+
evaluator_name=evaluator.name,
|
208
|
+
result=dto_result,
|
209
|
+
)
|
210
|
+
)
|
211
|
+
evaluation_item_results.append(
|
212
|
+
EvalItemResult(
|
213
|
+
evaluator_id=evaluator.id,
|
214
|
+
result=evaluation_result,
|
215
|
+
)
|
185
216
|
)
|
186
|
-
)
|
187
|
-
|
188
|
-
evaluation_run_results.compute_average_score()
|
189
217
|
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
218
|
+
evaluation_run_results.compute_average_score()
|
219
|
+
|
220
|
+
await event_bus.publish(
|
221
|
+
EvaluationEvents.UPDATE_EVAL_RUN,
|
222
|
+
EvalRunUpdatedEvent(
|
223
|
+
execution_id=self.execution_id,
|
224
|
+
eval_item=eval_item,
|
225
|
+
eval_results=evaluation_item_results,
|
226
|
+
success=not agent_execution_output.result.error,
|
227
|
+
agent_output=agent_execution_output.result.output,
|
228
|
+
agent_execution_time=agent_execution_output.execution_time,
|
229
|
+
spans=agent_execution_output.spans,
|
230
|
+
logs=agent_execution_output.logs,
|
231
|
+
),
|
232
|
+
wait_for_completion=False,
|
233
|
+
)
|
234
|
+
except Exception as e:
|
235
|
+
error_msg = str(e)
|
236
|
+
eval_item._error_message = error_msg # type: ignore[attr-defined]
|
237
|
+
|
238
|
+
for evaluator in evaluators:
|
239
|
+
evaluator_counts[evaluator.id] += 1
|
240
|
+
count = evaluator_counts[evaluator.id]
|
241
|
+
evaluator_averages[evaluator.id] += (
|
242
|
+
0.0 - evaluator_averages[evaluator.id]
|
243
|
+
) / count
|
244
|
+
|
245
|
+
await event_bus.publish(
|
246
|
+
EvaluationEvents.UPDATE_EVAL_RUN,
|
247
|
+
EvalRunUpdatedEvent(
|
248
|
+
execution_id=self.execution_id,
|
249
|
+
eval_item=eval_item,
|
250
|
+
eval_results=[],
|
251
|
+
success=False,
|
252
|
+
agent_output={},
|
253
|
+
agent_execution_time=0.0,
|
254
|
+
spans=[],
|
255
|
+
logs=[],
|
256
|
+
),
|
257
|
+
wait_for_completion=False,
|
258
|
+
)
|
203
259
|
|
204
260
|
results.compute_average_score()
|
205
261
|
|
206
262
|
await event_bus.publish(
|
207
263
|
EvaluationEvents.UPDATE_EVAL_SET_RUN,
|
208
264
|
EvalSetRunUpdatedEvent(
|
209
|
-
execution_id=self.
|
265
|
+
execution_id=self.execution_id,
|
210
266
|
evaluator_scores=evaluator_averages,
|
211
267
|
),
|
212
268
|
wait_for_completion=False,
|
@@ -221,10 +277,12 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
221
277
|
async def execute_runtime(
|
222
278
|
self, eval_item: EvaluationItem
|
223
279
|
) -> UiPathEvalRunExecutionOutput:
|
280
|
+
eval_item_id = eval_item.id
|
224
281
|
runtime_context: C = self.factory.new_context(
|
225
|
-
execution_id=
|
282
|
+
execution_id=eval_item_id,
|
226
283
|
input_json=eval_item.inputs,
|
227
284
|
is_eval_run=True,
|
285
|
+
log_handler=self._setup_execution_logging(eval_item_id),
|
228
286
|
)
|
229
287
|
attributes = {
|
230
288
|
"evalId": eval_item.id,
|
@@ -247,15 +305,24 @@ class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
|
|
247
305
|
spans = self.span_exporter.get_spans(runtime_context.execution_id)
|
248
306
|
self.span_exporter.clear(runtime_context.execution_id)
|
249
307
|
|
308
|
+
logs = self.logs_exporter.get_logs(runtime_context.execution_id)
|
309
|
+
self.logs_exporter.clear(runtime_context.execution_id)
|
310
|
+
|
250
311
|
if result is None:
|
251
312
|
raise ValueError("Execution result cannot be None for eval runs")
|
252
313
|
|
253
314
|
return UiPathEvalRunExecutionOutput(
|
254
315
|
execution_time=end_time - start_time,
|
255
316
|
spans=spans,
|
317
|
+
logs=logs,
|
256
318
|
result=result,
|
257
319
|
)
|
258
320
|
|
321
|
+
def _setup_execution_logging(self, eval_item_id: str) -> ExecutionLogHandler:
|
322
|
+
execution_log_handler = ExecutionLogHandler(eval_item_id)
|
323
|
+
self.logs_exporter.register(eval_item_id, execution_log_handler)
|
324
|
+
return execution_log_handler
|
325
|
+
|
259
326
|
async def run_evaluator(
|
260
327
|
self,
|
261
328
|
evaluator: BaseEvaluator[Any],
|
@@ -519,7 +519,8 @@ class UiPathBaseRuntime(ABC):
|
|
519
519
|
|
520
520
|
await self.validate()
|
521
521
|
|
522
|
-
# Intercept all stdout/stderr/logs
|
522
|
+
# Intercept all stdout/stderr/logs
|
523
|
+
# write to file (runtime) or stdout (debug)
|
523
524
|
self.logs_interceptor = LogsInterceptor(
|
524
525
|
min_level=self.context.logs_min_level,
|
525
526
|
dir=self.context.runtime_dir,
|
@@ -649,7 +650,7 @@ class UiPathBaseRuntime(ABC):
|
|
649
650
|
raise
|
650
651
|
finally:
|
651
652
|
# Restore original logging
|
652
|
-
if self
|
653
|
+
if hasattr(self, "logs_interceptor"):
|
653
654
|
self.logs_interceptor.teardown()
|
654
655
|
|
655
656
|
await self.cleanup()
|
@@ -698,6 +699,7 @@ class UiPathRuntimeFactory(Generic[T, C]):
|
|
698
699
|
self.context_generator = context_generator
|
699
700
|
self.tracer_provider: TracerProvider = TracerProvider()
|
700
701
|
self.tracer_span_processors: List[SpanProcessor] = []
|
702
|
+
self.logs_exporter: Optional[Any] = None
|
701
703
|
trace.set_tracer_provider(self.tracer_provider)
|
702
704
|
|
703
705
|
def add_span_exporter(
|