eval-studio-client 1.2.5__py3-none-any.whl → 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (306) hide show
  1. eval_studio_client/api/__init__.py +65 -0
  2. eval_studio_client/api/api/__init__.py +3 -0
  3. eval_studio_client/api/api/dashboard_report_service_api.py +292 -0
  4. eval_studio_client/api/api/dashboard_service_api.py +16 -16
  5. eval_studio_client/api/api/dashboard_test_case_annotation_service_api.py +611 -0
  6. eval_studio_client/api/api/document_service_api.py +16 -16
  7. eval_studio_client/api/api/evaluation_service_api.py +12 -12
  8. eval_studio_client/api/api/evaluator_service_api.py +16 -16
  9. eval_studio_client/api/api/leaderboard_report_service_api.py +304 -17
  10. eval_studio_client/api/api/leaderboard_service_api.py +554 -16
  11. eval_studio_client/api/api/leaderboard_test_case_annotation_service_api.py +611 -0
  12. eval_studio_client/api/api/model_service_api.py +16 -16
  13. eval_studio_client/api/api/operation_service_api.py +821 -17
  14. eval_studio_client/api/api/perturbator_service_api.py +22 -22
  15. eval_studio_client/api/api/test_case_service_api.py +300 -16
  16. eval_studio_client/api/api/test_class_service_api.py +16 -16
  17. eval_studio_client/api/api/test_service_api.py +285 -16
  18. eval_studio_client/api/api/workflow_node_service_api.py +16 -16
  19. eval_studio_client/api/api/workflow_service_api.py +16 -16
  20. eval_studio_client/api/docs/AdversarialInputsServiceTestAdversarialInputsRobustnessRequest.md +2 -1
  21. eval_studio_client/api/docs/DashboardReportServiceApi.md +75 -0
  22. eval_studio_client/api/docs/DashboardServiceApi.md +5 -5
  23. eval_studio_client/api/docs/DashboardTestCaseAnnotationServiceApi.md +149 -0
  24. eval_studio_client/api/docs/DocumentServiceApi.md +5 -5
  25. eval_studio_client/api/docs/EvaluationServiceApi.md +4 -4
  26. eval_studio_client/api/docs/EvaluatorServiceApi.md +5 -5
  27. eval_studio_client/api/docs/LeaderboardReportServiceApi.md +75 -5
  28. eval_studio_client/api/docs/LeaderboardServiceApi.md +141 -5
  29. eval_studio_client/api/docs/LeaderboardTestCaseAnnotationServiceApi.md +149 -0
  30. eval_studio_client/api/docs/ModelServiceApi.md +5 -5
  31. eval_studio_client/api/docs/OperationServiceApi.md +215 -8
  32. eval_studio_client/api/docs/PerturbatorServiceApi.md +7 -7
  33. eval_studio_client/api/docs/RequiredTheDashboardTestCaseAnnotationToUpdate.md +35 -0
  34. eval_studio_client/api/docs/RequiredTheLeaderboardTestCaseAnnotationToUpdate.md +35 -0
  35. eval_studio_client/api/docs/RequiredTheLeaderboardToUpdate.md +1 -0
  36. eval_studio_client/api/docs/RequiredTheOperationToFinalize.md +1 -0
  37. eval_studio_client/api/docs/RequiredTheOperationToUpdate.md +1 -0
  38. eval_studio_client/api/docs/TestCaseServiceApi.md +75 -5
  39. eval_studio_client/api/docs/TestCaseServiceAppendTestCasesRequest.md +30 -0
  40. eval_studio_client/api/docs/TestClassServiceApi.md +5 -5
  41. eval_studio_client/api/docs/TestServiceApi.md +73 -5
  42. eval_studio_client/api/docs/V1ActualOutputMeta.md +30 -0
  43. eval_studio_client/api/docs/V1ActualOutputMetaDiff.md +36 -0
  44. eval_studio_client/api/docs/V1AgentChatActivityDiagram.md +31 -0
  45. eval_studio_client/api/docs/V1AgentChatActivityDiagramEdge.md +32 -0
  46. eval_studio_client/api/docs/V1AgentChatActivityDiagramNode.md +32 -0
  47. eval_studio_client/api/docs/V1AgentChatActivityDiagramRow.md +30 -0
  48. eval_studio_client/api/docs/V1AgentChatScriptUsage.md +33 -0
  49. eval_studio_client/api/docs/V1AgentChatScriptsBarChart.md +30 -0
  50. eval_studio_client/api/docs/V1AgentChatToolUsage.md +33 -0
  51. eval_studio_client/api/docs/V1AgentChatToolsBarChart.md +30 -0
  52. eval_studio_client/api/docs/V1AllMetricScores.md +29 -0
  53. eval_studio_client/api/docs/V1AppendTestCasesResponse.md +29 -0
  54. eval_studio_client/api/docs/V1BatchCreateLeaderboardsWithoutCacheRequest.md +31 -0
  55. eval_studio_client/api/docs/V1BatchCreateLeaderboardsWithoutCacheResponse.md +29 -0
  56. eval_studio_client/api/docs/V1BatchMarkOperationSeenByCreatorResponse.md +29 -0
  57. eval_studio_client/api/docs/V1CmpLeaderboardReportsRequest.md +33 -0
  58. eval_studio_client/api/docs/V1CmpLeaderboardReportsResponse.md +29 -0
  59. eval_studio_client/api/docs/V1ComparisonItem.md +36 -0
  60. eval_studio_client/api/docs/V1ComparisonMetricScore.md +30 -0
  61. eval_studio_client/api/docs/V1ComparisonResult.md +31 -0
  62. eval_studio_client/api/docs/V1ComparisonSummary.md +31 -0
  63. eval_studio_client/api/docs/V1CreateEvaluationRequest.md +1 -0
  64. eval_studio_client/api/docs/V1CreateTestFromTestCasesRequest.md +32 -0
  65. eval_studio_client/api/docs/V1CreateTestFromTestCasesResponse.md +29 -0
  66. eval_studio_client/api/docs/V1DashboardReport.md +31 -0
  67. eval_studio_client/api/docs/V1DashboardReportResult.md +39 -0
  68. eval_studio_client/api/docs/V1DashboardTestCaseAnnotation.md +36 -0
  69. eval_studio_client/api/docs/V1DataFragment.md +31 -0
  70. eval_studio_client/api/docs/V1DeepCompareLeaderboardsRequest.md +33 -0
  71. eval_studio_client/api/docs/V1DeepCompareLeaderboardsResponse.md +29 -0
  72. eval_studio_client/api/docs/V1DiffItem.md +36 -0
  73. eval_studio_client/api/docs/V1EvaluationType.md +12 -0
  74. eval_studio_client/api/docs/V1FlippedMetric.md +31 -0
  75. eval_studio_client/api/docs/V1GetDashboardReportResponse.md +29 -0
  76. eval_studio_client/api/docs/V1HumanDecision.md +12 -0
  77. eval_studio_client/api/docs/V1Info.md +1 -0
  78. eval_studio_client/api/docs/V1Leaderboard.md +1 -0
  79. eval_studio_client/api/docs/V1LeaderboardCmpReport.md +30 -0
  80. eval_studio_client/api/docs/V1LeaderboardComparisonItem.md +31 -0
  81. eval_studio_client/api/docs/V1LeaderboardInfo.md +30 -0
  82. eval_studio_client/api/docs/V1LeaderboardReportActualOutputMeta.md +6 -3
  83. eval_studio_client/api/docs/V1LeaderboardReportResult.md +11 -8
  84. eval_studio_client/api/docs/V1LeaderboardReportResultView.md +12 -0
  85. eval_studio_client/api/docs/V1LeaderboardTestCaseAnnotation.md +36 -0
  86. eval_studio_client/api/docs/V1ListDashboardTestCaseAnnotationsResponse.md +29 -0
  87. eval_studio_client/api/docs/V1ListLeaderboardTestCaseAnnotationsResponse.md +29 -0
  88. eval_studio_client/api/docs/V1ListOperationsResponse.md +1 -0
  89. eval_studio_client/api/docs/V1ListUnseenOperationsResponse.md +30 -0
  90. eval_studio_client/api/docs/V1MarkOperationSeenByCreatorResponse.md +29 -0
  91. eval_studio_client/api/docs/V1Metric.md +30 -0
  92. eval_studio_client/api/docs/V1MetricAverage.md +36 -0
  93. eval_studio_client/api/docs/V1MetricMeta.md +40 -0
  94. eval_studio_client/api/docs/V1MetricScore.md +1 -1
  95. eval_studio_client/api/docs/V1MetricScores.md +1 -1
  96. eval_studio_client/api/docs/V1ModelType.md +1 -1
  97. eval_studio_client/api/docs/V1ModelsComparisons.md +32 -0
  98. eval_studio_client/api/docs/V1ModelsComparisonsMetrics.md +33 -0
  99. eval_studio_client/api/docs/V1ModelsOverview.md +34 -0
  100. eval_studio_client/api/docs/V1Operation.md +1 -0
  101. eval_studio_client/api/docs/V1OperationView.md +12 -0
  102. eval_studio_client/api/docs/V1RetrievedContextDiff.md +36 -0
  103. eval_studio_client/api/docs/V1Stats.md +2 -0
  104. eval_studio_client/api/docs/V1TechnicalMetrics.md +30 -0
  105. eval_studio_client/api/docs/V1TechnicalMetricsDetail.md +33 -0
  106. eval_studio_client/api/docs/V1TestCaseLeaderboardItem.md +31 -0
  107. eval_studio_client/api/docs/V1TestCaseRelationshipInfo.md +31 -0
  108. eval_studio_client/api/docs/V1TestCaseResult.md +48 -0
  109. eval_studio_client/api/docs/V1TextSimilarityMetric.md +12 -0
  110. eval_studio_client/api/docs/V1UpdateDashboardTestCaseAnnotationResponse.md +29 -0
  111. eval_studio_client/api/docs/V1UpdateLeaderboardTestCaseAnnotationResponse.md +29 -0
  112. eval_studio_client/api/docs/WorkflowNodeServiceApi.md +5 -5
  113. eval_studio_client/api/docs/WorkflowServiceApi.md +5 -5
  114. eval_studio_client/api/models/__init__.py +62 -0
  115. eval_studio_client/api/models/adversarial_inputs_service_test_adversarial_inputs_robustness_request.py +17 -2
  116. eval_studio_client/api/models/required_the_dashboard_test_case_annotation_to_update.py +108 -0
  117. eval_studio_client/api/models/required_the_leaderboard_test_case_annotation_to_update.py +108 -0
  118. eval_studio_client/api/models/required_the_leaderboard_to_update.py +5 -2
  119. eval_studio_client/api/models/required_the_operation_to_finalize.py +6 -2
  120. eval_studio_client/api/models/required_the_operation_to_update.py +6 -2
  121. eval_studio_client/api/models/test_case_service_append_test_cases_request.py +89 -0
  122. eval_studio_client/api/models/v1_actual_output_meta.py +97 -0
  123. eval_studio_client/api/models/v1_actual_output_meta_diff.py +101 -0
  124. eval_studio_client/api/models/v1_agent_chat_activity_diagram.py +109 -0
  125. eval_studio_client/api/models/v1_agent_chat_activity_diagram_edge.py +97 -0
  126. eval_studio_client/api/models/v1_agent_chat_activity_diagram_node.py +97 -0
  127. eval_studio_client/api/models/v1_agent_chat_activity_diagram_row.py +97 -0
  128. eval_studio_client/api/models/v1_agent_chat_script_usage.py +101 -0
  129. eval_studio_client/api/models/v1_agent_chat_scripts_bar_chart.py +102 -0
  130. eval_studio_client/api/models/v1_agent_chat_tool_usage.py +101 -0
  131. eval_studio_client/api/models/v1_agent_chat_tools_bar_chart.py +102 -0
  132. eval_studio_client/api/models/v1_all_metric_scores.py +87 -0
  133. eval_studio_client/api/models/v1_append_test_cases_response.py +95 -0
  134. eval_studio_client/api/models/v1_batch_create_leaderboards_without_cache_request.py +99 -0
  135. eval_studio_client/api/models/v1_batch_create_leaderboards_without_cache_response.py +91 -0
  136. eval_studio_client/api/models/v1_batch_mark_operation_seen_by_creator_response.py +95 -0
  137. eval_studio_client/api/models/v1_cmp_leaderboard_reports_request.py +96 -0
  138. eval_studio_client/api/models/v1_cmp_leaderboard_reports_response.py +91 -0
  139. eval_studio_client/api/models/v1_comparison_item.py +130 -0
  140. eval_studio_client/api/models/v1_comparison_metric_score.py +89 -0
  141. eval_studio_client/api/models/v1_comparison_result.py +120 -0
  142. eval_studio_client/api/models/v1_comparison_summary.py +91 -0
  143. eval_studio_client/api/models/v1_create_evaluation_request.py +5 -2
  144. eval_studio_client/api/models/v1_create_test_from_test_cases_request.py +93 -0
  145. eval_studio_client/api/models/v1_create_test_from_test_cases_response.py +91 -0
  146. eval_studio_client/api/models/v1_dashboard_report.py +109 -0
  147. eval_studio_client/api/models/v1_dashboard_report_result.py +139 -0
  148. eval_studio_client/api/models/v1_dashboard_test_case_annotation.py +112 -0
  149. eval_studio_client/api/models/v1_data_fragment.py +91 -0
  150. eval_studio_client/api/models/v1_deep_compare_leaderboards_request.py +96 -0
  151. eval_studio_client/api/models/v1_deep_compare_leaderboards_response.py +91 -0
  152. eval_studio_client/api/models/v1_diff_item.py +137 -0
  153. eval_studio_client/api/models/v1_evaluation_type.py +39 -0
  154. eval_studio_client/api/models/v1_flipped_metric.py +91 -0
  155. eval_studio_client/api/models/v1_get_dashboard_report_response.py +91 -0
  156. eval_studio_client/api/models/v1_human_decision.py +38 -0
  157. eval_studio_client/api/models/v1_info.py +4 -2
  158. eval_studio_client/api/models/v1_leaderboard.py +5 -2
  159. eval_studio_client/api/models/v1_leaderboard_cmp_report.py +93 -0
  160. eval_studio_client/api/models/v1_leaderboard_comparison_item.py +91 -0
  161. eval_studio_client/api/models/v1_leaderboard_info.py +97 -0
  162. eval_studio_client/api/models/v1_leaderboard_report_actual_output_meta.py +23 -9
  163. eval_studio_client/api/models/v1_leaderboard_report_result.py +21 -10
  164. eval_studio_client/api/models/v1_leaderboard_report_result_view.py +38 -0
  165. eval_studio_client/api/models/v1_leaderboard_test_case_annotation.py +112 -0
  166. eval_studio_client/api/models/v1_list_dashboard_test_case_annotations_response.py +95 -0
  167. eval_studio_client/api/models/v1_list_leaderboard_test_case_annotations_response.py +95 -0
  168. eval_studio_client/api/models/v1_list_operations_response.py +5 -3
  169. eval_studio_client/api/models/v1_list_unseen_operations_response.py +97 -0
  170. eval_studio_client/api/models/v1_mark_operation_seen_by_creator_response.py +91 -0
  171. eval_studio_client/api/models/v1_metric.py +89 -0
  172. eval_studio_client/api/models/v1_metric_average.py +101 -0
  173. eval_studio_client/api/models/v1_metric_meta.py +109 -0
  174. eval_studio_client/api/models/v1_metric_score.py +6 -1
  175. eval_studio_client/api/models/v1_metric_scores.py +1 -1
  176. eval_studio_client/api/models/v1_model_type.py +2 -1
  177. eval_studio_client/api/models/v1_models_comparisons.py +93 -0
  178. eval_studio_client/api/models/v1_models_comparisons_metrics.py +103 -0
  179. eval_studio_client/api/models/v1_models_overview.py +97 -0
  180. eval_studio_client/api/models/v1_operation.py +6 -2
  181. eval_studio_client/api/models/v1_operation_view.py +38 -0
  182. eval_studio_client/api/models/v1_retrieved_context_diff.py +101 -0
  183. eval_studio_client/api/models/v1_stats.py +16 -2
  184. eval_studio_client/api/models/v1_technical_metrics.py +96 -0
  185. eval_studio_client/api/models/v1_technical_metrics_detail.py +95 -0
  186. eval_studio_client/api/models/v1_test_case_leaderboard_item.py +91 -0
  187. eval_studio_client/api/models/v1_test_case_relationship_info.py +91 -0
  188. eval_studio_client/api/models/v1_test_case_result.py +157 -0
  189. eval_studio_client/api/models/v1_text_similarity_metric.py +39 -0
  190. eval_studio_client/api/models/v1_update_dashboard_test_case_annotation_response.py +91 -0
  191. eval_studio_client/api/models/v1_update_leaderboard_test_case_annotation_response.py +91 -0
  192. eval_studio_client/api/models/v1_workflow_node_type.py +1 -0
  193. eval_studio_client/api/models/v1_workflow_type.py +1 -0
  194. eval_studio_client/api/test/test_adversarial_inputs_service_test_adversarial_inputs_robustness_request.py +6 -0
  195. eval_studio_client/api/test/test_dashboard_report_service_api.py +37 -0
  196. eval_studio_client/api/test/test_dashboard_test_case_annotation_service_api.py +43 -0
  197. eval_studio_client/api/test/test_leaderboard_report_service_api.py +6 -0
  198. eval_studio_client/api/test/test_leaderboard_service_api.py +12 -0
  199. eval_studio_client/api/test/test_leaderboard_test_case_annotation_service_api.py +43 -0
  200. eval_studio_client/api/test/test_operation_service_api.py +18 -0
  201. eval_studio_client/api/test/test_required_the_dashboard_test_case_annotation_to_update.py +57 -0
  202. eval_studio_client/api/test/test_required_the_leaderboard_test_case_annotation_to_update.py +57 -0
  203. eval_studio_client/api/test/test_required_the_leaderboard_to_update.py +2 -1
  204. eval_studio_client/api/test/test_required_the_operation_to_finalize.py +2 -1
  205. eval_studio_client/api/test/test_required_the_operation_to_update.py +2 -1
  206. eval_studio_client/api/test/test_test_case_service_api.py +6 -0
  207. eval_studio_client/api/test/test_test_case_service_append_test_cases_request.py +52 -0
  208. eval_studio_client/api/test/test_test_service_api.py +6 -0
  209. eval_studio_client/api/test/test_v1_abort_operation_response.py +2 -1
  210. eval_studio_client/api/test/test_v1_actual_output_meta.py +61 -0
  211. eval_studio_client/api/test/test_v1_actual_output_meta_diff.py +66 -0
  212. eval_studio_client/api/test/test_v1_agent_chat_activity_diagram.py +65 -0
  213. eval_studio_client/api/test/test_v1_agent_chat_activity_diagram_edge.py +53 -0
  214. eval_studio_client/api/test/test_v1_agent_chat_activity_diagram_node.py +53 -0
  215. eval_studio_client/api/test/test_v1_agent_chat_activity_diagram_row.py +56 -0
  216. eval_studio_client/api/test/test_v1_agent_chat_script_usage.py +54 -0
  217. eval_studio_client/api/test/test_v1_agent_chat_scripts_bar_chart.py +57 -0
  218. eval_studio_client/api/test/test_v1_agent_chat_tool_usage.py +54 -0
  219. eval_studio_client/api/test/test_v1_agent_chat_tools_bar_chart.py +57 -0
  220. eval_studio_client/api/test/test_v1_all_metric_scores.py +53 -0
  221. eval_studio_client/api/test/test_v1_append_test_cases_response.py +74 -0
  222. eval_studio_client/api/test/test_v1_batch_create_leaderboards_request.py +2 -1
  223. eval_studio_client/api/test/test_v1_batch_create_leaderboards_response.py +2 -1
  224. eval_studio_client/api/test/test_v1_batch_create_leaderboards_without_cache_request.py +120 -0
  225. eval_studio_client/api/test/test_v1_batch_create_leaderboards_without_cache_response.py +72 -0
  226. eval_studio_client/api/test/test_v1_batch_delete_leaderboards_response.py +2 -1
  227. eval_studio_client/api/test/test_v1_batch_get_leaderboards_response.py +2 -1
  228. eval_studio_client/api/test/test_v1_batch_get_operations_response.py +2 -1
  229. eval_studio_client/api/test/test_v1_batch_import_leaderboard_response.py +2 -1
  230. eval_studio_client/api/test/test_v1_batch_mark_operation_seen_by_creator_response.py +74 -0
  231. eval_studio_client/api/test/test_v1_cmp_leaderboard_reports_request.py +55 -0
  232. eval_studio_client/api/test/test_v1_cmp_leaderboard_reports_response.py +255 -0
  233. eval_studio_client/api/test/test_v1_comparison_item.py +233 -0
  234. eval_studio_client/api/test/test_v1_comparison_metric_score.py +52 -0
  235. eval_studio_client/api/test/test_v1_comparison_result.py +258 -0
  236. eval_studio_client/api/test/test_v1_comparison_summary.py +53 -0
  237. eval_studio_client/api/test/test_v1_create_evaluation_request.py +2 -1
  238. eval_studio_client/api/test/test_v1_create_leaderboard_request.py +2 -1
  239. eval_studio_client/api/test/test_v1_create_leaderboard_response.py +2 -1
  240. eval_studio_client/api/test/test_v1_create_leaderboard_without_cache_response.py +2 -1
  241. eval_studio_client/api/test/test_v1_create_test_from_test_cases_request.py +54 -0
  242. eval_studio_client/api/test/test_v1_create_test_from_test_cases_response.py +68 -0
  243. eval_studio_client/api/test/test_v1_dashboard_report.py +142 -0
  244. eval_studio_client/api/test/test_v1_dashboard_report_result.py +72 -0
  245. eval_studio_client/api/test/test_v1_dashboard_test_case_annotation.py +58 -0
  246. eval_studio_client/api/test/test_v1_data_fragment.py +57 -0
  247. eval_studio_client/api/test/test_v1_deep_compare_leaderboards_request.py +55 -0
  248. eval_studio_client/api/test/test_v1_deep_compare_leaderboards_response.py +255 -0
  249. eval_studio_client/api/test/test_v1_delete_leaderboard_response.py +2 -1
  250. eval_studio_client/api/test/test_v1_diff_item.py +226 -0
  251. eval_studio_client/api/test/test_v1_evaluation_type.py +33 -0
  252. eval_studio_client/api/test/test_v1_finalize_operation_response.py +2 -1
  253. eval_studio_client/api/test/test_v1_flipped_metric.py +53 -0
  254. eval_studio_client/api/test/test_v1_generate_test_cases_response.py +2 -1
  255. eval_studio_client/api/test/test_v1_get_dashboard_report_response.py +143 -0
  256. eval_studio_client/api/test/test_v1_get_info_response.py +4 -1
  257. eval_studio_client/api/test/test_v1_get_leaderboard_report_response.py +39 -2
  258. eval_studio_client/api/test/test_v1_get_leaderboard_response.py +2 -1
  259. eval_studio_client/api/test/test_v1_get_operation_response.py +2 -1
  260. eval_studio_client/api/test/test_v1_get_stats_response.py +3 -1
  261. eval_studio_client/api/test/test_v1_human_decision.py +33 -0
  262. eval_studio_client/api/test/test_v1_import_leaderboard_response.py +2 -1
  263. eval_studio_client/api/test/test_v1_import_test_cases_from_library_response.py +2 -1
  264. eval_studio_client/api/test/test_v1_info.py +4 -1
  265. eval_studio_client/api/test/test_v1_leaderboard.py +2 -1
  266. eval_studio_client/api/test/test_v1_leaderboard_cmp_report.py +254 -0
  267. eval_studio_client/api/test/test_v1_leaderboard_comparison_item.py +53 -0
  268. eval_studio_client/api/test/test_v1_leaderboard_info.py +57 -0
  269. eval_studio_client/api/test/test_v1_leaderboard_report.py +39 -2
  270. eval_studio_client/api/test/test_v1_leaderboard_report_actual_output_meta.py +33 -1
  271. eval_studio_client/api/test/test_v1_leaderboard_report_result.py +39 -2
  272. eval_studio_client/api/test/test_v1_leaderboard_report_result_view.py +33 -0
  273. eval_studio_client/api/test/test_v1_leaderboard_test_case_annotation.py +58 -0
  274. eval_studio_client/api/test/test_v1_list_dashboard_test_case_annotations_response.py +61 -0
  275. eval_studio_client/api/test/test_v1_list_leaderboard_test_case_annotations_response.py +61 -0
  276. eval_studio_client/api/test/test_v1_list_leaderboards_response.py +2 -1
  277. eval_studio_client/api/test/test_v1_list_most_recent_leaderboards_response.py +2 -1
  278. eval_studio_client/api/test/test_v1_list_operations_response.py +4 -2
  279. eval_studio_client/api/test/test_v1_list_unseen_operations_response.py +75 -0
  280. eval_studio_client/api/test/test_v1_mark_operation_seen_by_creator_response.py +72 -0
  281. eval_studio_client/api/test/test_v1_metric.py +52 -0
  282. eval_studio_client/api/test/test_v1_metric_average.py +58 -0
  283. eval_studio_client/api/test/test_v1_metric_meta.py +66 -0
  284. eval_studio_client/api/test/test_v1_models_comparisons.py +54 -0
  285. eval_studio_client/api/test/test_v1_models_comparisons_metrics.py +65 -0
  286. eval_studio_client/api/test/test_v1_models_overview.py +60 -0
  287. eval_studio_client/api/test/test_v1_operation.py +2 -1
  288. eval_studio_client/api/test/test_v1_operation_view.py +33 -0
  289. eval_studio_client/api/test/test_v1_process_workflow_node_response.py +2 -1
  290. eval_studio_client/api/test/test_v1_retrieved_context_diff.py +66 -0
  291. eval_studio_client/api/test/test_v1_stats.py +3 -1
  292. eval_studio_client/api/test/test_v1_technical_metrics.py +62 -0
  293. eval_studio_client/api/test/test_v1_technical_metrics_detail.py +55 -0
  294. eval_studio_client/api/test/test_v1_test_case_leaderboard_item.py +53 -0
  295. eval_studio_client/api/test/test_v1_test_case_relationship_info.py +53 -0
  296. eval_studio_client/api/test/test_v1_test_case_result.py +106 -0
  297. eval_studio_client/api/test/test_v1_text_similarity_metric.py +33 -0
  298. eval_studio_client/api/test/test_v1_update_dashboard_test_case_annotation_response.py +59 -0
  299. eval_studio_client/api/test/test_v1_update_leaderboard_response.py +2 -1
  300. eval_studio_client/api/test/test_v1_update_leaderboard_test_case_annotation_response.py +59 -0
  301. eval_studio_client/api/test/test_v1_update_operation_response.py +2 -1
  302. eval_studio_client/gen/openapiv2/eval_studio.swagger.json +2340 -210
  303. eval_studio_client/models.py +18 -6
  304. {eval_studio_client-1.2.5.dist-info → eval_studio_client-1.3.0.dist-info}/METADATA +2 -2
  305. {eval_studio_client-1.2.5.dist-info → eval_studio_client-1.3.0.dist-info}/RECORD +306 -111
  306. {eval_studio_client-1.2.5.dist-info → eval_studio_client-1.3.0.dist-info}/WHEEL +0 -0
@@ -16,8 +16,11 @@ from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt
16
16
  from typing import Any, Dict, List, Optional, Tuple, Union
17
17
  from typing_extensions import Annotated
18
18
 
19
- from pydantic import Field, field_validator
19
+ from pydantic import Field, StrictStr, field_validator
20
+ from typing import Optional
20
21
  from typing_extensions import Annotated
22
+ from eval_studio_client.api.models.v1_cmp_leaderboard_reports_request import V1CmpLeaderboardReportsRequest
23
+ from eval_studio_client.api.models.v1_cmp_leaderboard_reports_response import V1CmpLeaderboardReportsResponse
21
24
  from eval_studio_client.api.models.v1_get_leaderboard_report_response import V1GetLeaderboardReportResponse
22
25
 
23
26
  from eval_studio_client.api.api_client import ApiClient, RequestSerialized
@@ -38,10 +41,278 @@ class LeaderboardReportServiceApi:
38
41
  self.api_client = api_client
39
42
 
40
43
 
44
+ @validate_call
45
+ def leaderboard_report_service_cmp_leaderboard_reports(
46
+ self,
47
+ body: V1CmpLeaderboardReportsRequest,
48
+ _request_timeout: Union[
49
+ None,
50
+ Annotated[StrictFloat, Field(gt=0)],
51
+ Tuple[
52
+ Annotated[StrictFloat, Field(gt=0)],
53
+ Annotated[StrictFloat, Field(gt=0)]
54
+ ]
55
+ ] = None,
56
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
57
+ _content_type: Optional[StrictStr] = None,
58
+ _headers: Optional[Dict[StrictStr, Any]] = None,
59
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
60
+ ) -> V1CmpLeaderboardReportsResponse:
61
+ """leaderboard_report_service_cmp_leaderboard_reports
62
+
63
+
64
+ :param body: (required)
65
+ :type body: V1CmpLeaderboardReportsRequest
66
+ :param _request_timeout: timeout setting for this request. If one
67
+ number provided, it will be total request
68
+ timeout. It can also be a pair (tuple) of
69
+ (connection, read) timeouts.
70
+ :type _request_timeout: int, tuple(int, int), optional
71
+ :param _request_auth: set to override the auth_settings for an a single
72
+ request; this effectively ignores the
73
+ authentication in the spec for a single request.
74
+ :type _request_auth: dict, optional
75
+ :param _content_type: force content-type for the request.
76
+ :type _content_type: str, Optional
77
+ :param _headers: set to override the headers for a single
78
+ request; this effectively ignores the headers
79
+ in the spec for a single request.
80
+ :type _headers: dict, optional
81
+ :param _host_index: set to override the host_index for a single
82
+ request; this effectively ignores the host_index
83
+ in the spec for a single request.
84
+ :type _host_index: int, optional
85
+ :return: Returns the result object.
86
+ """ # noqa: E501
87
+
88
+ _param = self._leaderboard_report_service_cmp_leaderboard_reports_serialize(
89
+ body=body,
90
+ _request_auth=_request_auth,
91
+ _content_type=_content_type,
92
+ _headers=_headers,
93
+ _host_index=_host_index
94
+ )
95
+
96
+ _response_types_map: Dict[str, Optional[str]] = {
97
+ '200': "V1CmpLeaderboardReportsResponse",
98
+ }
99
+ response_data = self.api_client.call_api(
100
+ *_param,
101
+ _request_timeout=_request_timeout
102
+ )
103
+ response_data.read()
104
+ return self.api_client.response_deserialize(
105
+ response_data=response_data,
106
+ response_types_map=_response_types_map,
107
+ ).data
108
+
109
+
110
+ @validate_call
111
+ def leaderboard_report_service_cmp_leaderboard_reports_with_http_info(
112
+ self,
113
+ body: V1CmpLeaderboardReportsRequest,
114
+ _request_timeout: Union[
115
+ None,
116
+ Annotated[StrictFloat, Field(gt=0)],
117
+ Tuple[
118
+ Annotated[StrictFloat, Field(gt=0)],
119
+ Annotated[StrictFloat, Field(gt=0)]
120
+ ]
121
+ ] = None,
122
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
123
+ _content_type: Optional[StrictStr] = None,
124
+ _headers: Optional[Dict[StrictStr, Any]] = None,
125
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
126
+ ) -> ApiResponse[V1CmpLeaderboardReportsResponse]:
127
+ """leaderboard_report_service_cmp_leaderboard_reports
128
+
129
+
130
+ :param body: (required)
131
+ :type body: V1CmpLeaderboardReportsRequest
132
+ :param _request_timeout: timeout setting for this request. If one
133
+ number provided, it will be total request
134
+ timeout. It can also be a pair (tuple) of
135
+ (connection, read) timeouts.
136
+ :type _request_timeout: int, tuple(int, int), optional
137
+ :param _request_auth: set to override the auth_settings for an a single
138
+ request; this effectively ignores the
139
+ authentication in the spec for a single request.
140
+ :type _request_auth: dict, optional
141
+ :param _content_type: force content-type for the request.
142
+ :type _content_type: str, Optional
143
+ :param _headers: set to override the headers for a single
144
+ request; this effectively ignores the headers
145
+ in the spec for a single request.
146
+ :type _headers: dict, optional
147
+ :param _host_index: set to override the host_index for a single
148
+ request; this effectively ignores the host_index
149
+ in the spec for a single request.
150
+ :type _host_index: int, optional
151
+ :return: Returns the result object.
152
+ """ # noqa: E501
153
+
154
+ _param = self._leaderboard_report_service_cmp_leaderboard_reports_serialize(
155
+ body=body,
156
+ _request_auth=_request_auth,
157
+ _content_type=_content_type,
158
+ _headers=_headers,
159
+ _host_index=_host_index
160
+ )
161
+
162
+ _response_types_map: Dict[str, Optional[str]] = {
163
+ '200': "V1CmpLeaderboardReportsResponse",
164
+ }
165
+ response_data = self.api_client.call_api(
166
+ *_param,
167
+ _request_timeout=_request_timeout
168
+ )
169
+ response_data.read()
170
+ return self.api_client.response_deserialize(
171
+ response_data=response_data,
172
+ response_types_map=_response_types_map,
173
+ )
174
+
175
+
176
+ @validate_call
177
+ def leaderboard_report_service_cmp_leaderboard_reports_without_preload_content(
178
+ self,
179
+ body: V1CmpLeaderboardReportsRequest,
180
+ _request_timeout: Union[
181
+ None,
182
+ Annotated[StrictFloat, Field(gt=0)],
183
+ Tuple[
184
+ Annotated[StrictFloat, Field(gt=0)],
185
+ Annotated[StrictFloat, Field(gt=0)]
186
+ ]
187
+ ] = None,
188
+ _request_auth: Optional[Dict[StrictStr, Any]] = None,
189
+ _content_type: Optional[StrictStr] = None,
190
+ _headers: Optional[Dict[StrictStr, Any]] = None,
191
+ _host_index: Annotated[StrictInt, Field(ge=0, le=0)] = 0,
192
+ ) -> RESTResponseType:
193
+ """leaderboard_report_service_cmp_leaderboard_reports
194
+
195
+
196
+ :param body: (required)
197
+ :type body: V1CmpLeaderboardReportsRequest
198
+ :param _request_timeout: timeout setting for this request. If one
199
+ number provided, it will be total request
200
+ timeout. It can also be a pair (tuple) of
201
+ (connection, read) timeouts.
202
+ :type _request_timeout: int, tuple(int, int), optional
203
+ :param _request_auth: set to override the auth_settings for an a single
204
+ request; this effectively ignores the
205
+ authentication in the spec for a single request.
206
+ :type _request_auth: dict, optional
207
+ :param _content_type: force content-type for the request.
208
+ :type _content_type: str, Optional
209
+ :param _headers: set to override the headers for a single
210
+ request; this effectively ignores the headers
211
+ in the spec for a single request.
212
+ :type _headers: dict, optional
213
+ :param _host_index: set to override the host_index for a single
214
+ request; this effectively ignores the host_index
215
+ in the spec for a single request.
216
+ :type _host_index: int, optional
217
+ :return: Returns the result object.
218
+ """ # noqa: E501
219
+
220
+ _param = self._leaderboard_report_service_cmp_leaderboard_reports_serialize(
221
+ body=body,
222
+ _request_auth=_request_auth,
223
+ _content_type=_content_type,
224
+ _headers=_headers,
225
+ _host_index=_host_index
226
+ )
227
+
228
+ _response_types_map: Dict[str, Optional[str]] = {
229
+ '200': "V1CmpLeaderboardReportsResponse",
230
+ }
231
+ response_data = self.api_client.call_api(
232
+ *_param,
233
+ _request_timeout=_request_timeout
234
+ )
235
+ return response_data.response
236
+
237
+
238
+ def _leaderboard_report_service_cmp_leaderboard_reports_serialize(
239
+ self,
240
+ body,
241
+ _request_auth,
242
+ _content_type,
243
+ _headers,
244
+ _host_index,
245
+ ) -> RequestSerialized:
246
+
247
+ _host = None
248
+
249
+ _collection_formats: Dict[str, str] = {
250
+ }
251
+
252
+ _path_params: Dict[str, str] = {}
253
+ _query_params: List[Tuple[str, str]] = []
254
+ _header_params: Dict[str, Optional[str]] = _headers or {}
255
+ _form_params: List[Tuple[str, str]] = []
256
+ _files: Dict[str, Union[str, bytes]] = {}
257
+ _body_params: Optional[bytes] = None
258
+
259
+ # process the path parameters
260
+ # process the query parameters
261
+ # process the header parameters
262
+ # process the form parameters
263
+ # process the body parameter
264
+ if body is not None:
265
+ _body_params = body
266
+
267
+
268
+ # set the HTTP header `Accept`
269
+ _header_params['Accept'] = self.api_client.select_header_accept(
270
+ [
271
+ 'application/json'
272
+ ]
273
+ )
274
+
275
+ # set the HTTP header `Content-Type`
276
+ if _content_type:
277
+ _header_params['Content-Type'] = _content_type
278
+ else:
279
+ _default_content_type = (
280
+ self.api_client.select_header_content_type(
281
+ [
282
+ 'application/json'
283
+ ]
284
+ )
285
+ )
286
+ if _default_content_type is not None:
287
+ _header_params['Content-Type'] = _default_content_type
288
+
289
+ # authentication setting
290
+ _auth_settings: List[str] = [
291
+ ]
292
+
293
+ return self.api_client.param_serialize(
294
+ method='POST',
295
+ resource_path='/v1/leaderboardReports:compare',
296
+ path_params=_path_params,
297
+ query_params=_query_params,
298
+ header_params=_header_params,
299
+ body=_body_params,
300
+ post_params=_form_params,
301
+ files=_files,
302
+ auth_settings=_auth_settings,
303
+ collection_formats=_collection_formats,
304
+ _host=_host,
305
+ _request_auth=_request_auth
306
+ )
307
+
308
+
309
+
310
+
41
311
  @validate_call
42
312
  def leaderboard_report_service_get_leaderboard_report(
43
313
  self,
44
- name_3: Annotated[str, Field(strict=True, description="Required. The name of the Leaderboard to retrieve.")],
314
+ name_4: Annotated[str, Field(strict=True, description="Required. The name of the Leaderboard to retrieve.")],
315
+ view: Annotated[Optional[StrictStr], Field(description="Optional. The view of the leaderboard report results to return. Defaults to LEADERBOARD_REPORT_RESULT_VIEW_FULL. - LEADERBOARD_REPORT_RESULT_VIEW_UNSPECIFIED: Default value. The basic view with essential fields. - LEADERBOARD_REPORT_RESULT_VIEW_FULL: Full view with all fields populated. - LEADERBOARD_REPORT_RESULT_VIEW_SUMMARY: Summary view with only key fields.")] = None,
45
316
  _request_timeout: Union[
46
317
  None,
47
318
  Annotated[StrictFloat, Field(gt=0)],
@@ -58,8 +329,10 @@ class LeaderboardReportServiceApi:
58
329
  """leaderboard_report_service_get_leaderboard_report
59
330
 
60
331
 
61
- :param name_3: Required. The name of the Leaderboard to retrieve. (required)
62
- :type name_3: str
332
+ :param name_4: Required. The name of the Leaderboard to retrieve. (required)
333
+ :type name_4: str
334
+ :param view: Optional. The view of the leaderboard report results to return. Defaults to LEADERBOARD_REPORT_RESULT_VIEW_FULL. - LEADERBOARD_REPORT_RESULT_VIEW_UNSPECIFIED: Default value. The basic view with essential fields. - LEADERBOARD_REPORT_RESULT_VIEW_FULL: Full view with all fields populated. - LEADERBOARD_REPORT_RESULT_VIEW_SUMMARY: Summary view with only key fields.
335
+ :type view: str
63
336
  :param _request_timeout: timeout setting for this request. If one
64
337
  number provided, it will be total request
65
338
  timeout. It can also be a pair (tuple) of
@@ -83,7 +356,8 @@ class LeaderboardReportServiceApi:
83
356
  """ # noqa: E501
84
357
 
85
358
  _param = self._leaderboard_report_service_get_leaderboard_report_serialize(
86
- name_3=name_3,
359
+ name_4=name_4,
360
+ view=view,
87
361
  _request_auth=_request_auth,
88
362
  _content_type=_content_type,
89
363
  _headers=_headers,
@@ -107,7 +381,8 @@ class LeaderboardReportServiceApi:
107
381
  @validate_call
108
382
  def leaderboard_report_service_get_leaderboard_report_with_http_info(
109
383
  self,
110
- name_3: Annotated[str, Field(strict=True, description="Required. The name of the Leaderboard to retrieve.")],
384
+ name_4: Annotated[str, Field(strict=True, description="Required. The name of the Leaderboard to retrieve.")],
385
+ view: Annotated[Optional[StrictStr], Field(description="Optional. The view of the leaderboard report results to return. Defaults to LEADERBOARD_REPORT_RESULT_VIEW_FULL. - LEADERBOARD_REPORT_RESULT_VIEW_UNSPECIFIED: Default value. The basic view with essential fields. - LEADERBOARD_REPORT_RESULT_VIEW_FULL: Full view with all fields populated. - LEADERBOARD_REPORT_RESULT_VIEW_SUMMARY: Summary view with only key fields.")] = None,
111
386
  _request_timeout: Union[
112
387
  None,
113
388
  Annotated[StrictFloat, Field(gt=0)],
@@ -124,8 +399,10 @@ class LeaderboardReportServiceApi:
124
399
  """leaderboard_report_service_get_leaderboard_report
125
400
 
126
401
 
127
- :param name_3: Required. The name of the Leaderboard to retrieve. (required)
128
- :type name_3: str
402
+ :param name_4: Required. The name of the Leaderboard to retrieve. (required)
403
+ :type name_4: str
404
+ :param view: Optional. The view of the leaderboard report results to return. Defaults to LEADERBOARD_REPORT_RESULT_VIEW_FULL. - LEADERBOARD_REPORT_RESULT_VIEW_UNSPECIFIED: Default value. The basic view with essential fields. - LEADERBOARD_REPORT_RESULT_VIEW_FULL: Full view with all fields populated. - LEADERBOARD_REPORT_RESULT_VIEW_SUMMARY: Summary view with only key fields.
405
+ :type view: str
129
406
  :param _request_timeout: timeout setting for this request. If one
130
407
  number provided, it will be total request
131
408
  timeout. It can also be a pair (tuple) of
@@ -149,7 +426,8 @@ class LeaderboardReportServiceApi:
149
426
  """ # noqa: E501
150
427
 
151
428
  _param = self._leaderboard_report_service_get_leaderboard_report_serialize(
152
- name_3=name_3,
429
+ name_4=name_4,
430
+ view=view,
153
431
  _request_auth=_request_auth,
154
432
  _content_type=_content_type,
155
433
  _headers=_headers,
@@ -173,7 +451,8 @@ class LeaderboardReportServiceApi:
173
451
  @validate_call
174
452
  def leaderboard_report_service_get_leaderboard_report_without_preload_content(
175
453
  self,
176
- name_3: Annotated[str, Field(strict=True, description="Required. The name of the Leaderboard to retrieve.")],
454
+ name_4: Annotated[str, Field(strict=True, description="Required. The name of the Leaderboard to retrieve.")],
455
+ view: Annotated[Optional[StrictStr], Field(description="Optional. The view of the leaderboard report results to return. Defaults to LEADERBOARD_REPORT_RESULT_VIEW_FULL. - LEADERBOARD_REPORT_RESULT_VIEW_UNSPECIFIED: Default value. The basic view with essential fields. - LEADERBOARD_REPORT_RESULT_VIEW_FULL: Full view with all fields populated. - LEADERBOARD_REPORT_RESULT_VIEW_SUMMARY: Summary view with only key fields.")] = None,
177
456
  _request_timeout: Union[
178
457
  None,
179
458
  Annotated[StrictFloat, Field(gt=0)],
@@ -190,8 +469,10 @@ class LeaderboardReportServiceApi:
190
469
  """leaderboard_report_service_get_leaderboard_report
191
470
 
192
471
 
193
- :param name_3: Required. The name of the Leaderboard to retrieve. (required)
194
- :type name_3: str
472
+ :param name_4: Required. The name of the Leaderboard to retrieve. (required)
473
+ :type name_4: str
474
+ :param view: Optional. The view of the leaderboard report results to return. Defaults to LEADERBOARD_REPORT_RESULT_VIEW_FULL. - LEADERBOARD_REPORT_RESULT_VIEW_UNSPECIFIED: Default value. The basic view with essential fields. - LEADERBOARD_REPORT_RESULT_VIEW_FULL: Full view with all fields populated. - LEADERBOARD_REPORT_RESULT_VIEW_SUMMARY: Summary view with only key fields.
475
+ :type view: str
195
476
  :param _request_timeout: timeout setting for this request. If one
196
477
  number provided, it will be total request
197
478
  timeout. It can also be a pair (tuple) of
@@ -215,7 +496,8 @@ class LeaderboardReportServiceApi:
215
496
  """ # noqa: E501
216
497
 
217
498
  _param = self._leaderboard_report_service_get_leaderboard_report_serialize(
218
- name_3=name_3,
499
+ name_4=name_4,
500
+ view=view,
219
501
  _request_auth=_request_auth,
220
502
  _content_type=_content_type,
221
503
  _headers=_headers,
@@ -234,7 +516,8 @@ class LeaderboardReportServiceApi:
234
516
 
235
517
  def _leaderboard_report_service_get_leaderboard_report_serialize(
236
518
  self,
237
- name_3,
519
+ name_4,
520
+ view,
238
521
  _request_auth,
239
522
  _content_type,
240
523
  _headers,
@@ -254,9 +537,13 @@ class LeaderboardReportServiceApi:
254
537
  _body_params: Optional[bytes] = None
255
538
 
256
539
  # process the path parameters
257
- if name_3 is not None:
258
- _path_params['name_3'] = name_3
540
+ if name_4 is not None:
541
+ _path_params['name_4'] = name_4
259
542
  # process the query parameters
543
+ if view is not None:
544
+
545
+ _query_params.append(('view', view))
546
+
260
547
  # process the header parameters
261
548
  # process the form parameters
262
549
  # process the body parameter
@@ -276,7 +563,7 @@ class LeaderboardReportServiceApi:
276
563
 
277
564
  return self.api_client.param_serialize(
278
565
  method='GET',
279
- resource_path='/v1/{name_3}',
566
+ resource_path='/v1/{name_4}',
280
567
  path_params=_path_params,
281
568
  query_params=_query_params,
282
569
  header_params=_header_params,