eval-studio-client 1.2.4a2__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eval_studio_client/api/__init__.py +65 -0
- eval_studio_client/api/api/__init__.py +3 -0
- eval_studio_client/api/api/dashboard_report_service_api.py +292 -0
- eval_studio_client/api/api/dashboard_service_api.py +16 -16
- eval_studio_client/api/api/dashboard_test_case_annotation_service_api.py +611 -0
- eval_studio_client/api/api/document_service_api.py +16 -16
- eval_studio_client/api/api/evaluation_service_api.py +12 -12
- eval_studio_client/api/api/evaluator_service_api.py +16 -16
- eval_studio_client/api/api/leaderboard_report_service_api.py +304 -17
- eval_studio_client/api/api/leaderboard_service_api.py +554 -16
- eval_studio_client/api/api/leaderboard_test_case_annotation_service_api.py +611 -0
- eval_studio_client/api/api/model_service_api.py +16 -16
- eval_studio_client/api/api/operation_service_api.py +821 -17
- eval_studio_client/api/api/perturbator_service_api.py +22 -22
- eval_studio_client/api/api/test_case_service_api.py +300 -16
- eval_studio_client/api/api/test_class_service_api.py +16 -16
- eval_studio_client/api/api/test_service_api.py +285 -16
- eval_studio_client/api/api/workflow_node_service_api.py +16 -16
- eval_studio_client/api/api/workflow_service_api.py +16 -16
- eval_studio_client/api/docs/AdversarialInputsServiceTestAdversarialInputsRobustnessRequest.md +2 -1
- eval_studio_client/api/docs/DashboardReportServiceApi.md +75 -0
- eval_studio_client/api/docs/DashboardServiceApi.md +5 -5
- eval_studio_client/api/docs/DashboardTestCaseAnnotationServiceApi.md +149 -0
- eval_studio_client/api/docs/DocumentServiceApi.md +5 -5
- eval_studio_client/api/docs/EvaluationServiceApi.md +4 -4
- eval_studio_client/api/docs/EvaluatorServiceApi.md +5 -5
- eval_studio_client/api/docs/LeaderboardReportServiceApi.md +75 -5
- eval_studio_client/api/docs/LeaderboardServiceApi.md +141 -5
- eval_studio_client/api/docs/LeaderboardTestCaseAnnotationServiceApi.md +149 -0
- eval_studio_client/api/docs/ModelServiceApi.md +5 -5
- eval_studio_client/api/docs/OperationServiceApi.md +215 -8
- eval_studio_client/api/docs/PerturbatorServiceApi.md +7 -7
- eval_studio_client/api/docs/RequiredTheDashboardTestCaseAnnotationToUpdate.md +35 -0
- eval_studio_client/api/docs/RequiredTheLeaderboardTestCaseAnnotationToUpdate.md +35 -0
- eval_studio_client/api/docs/RequiredTheLeaderboardToUpdate.md +1 -0
- eval_studio_client/api/docs/RequiredTheOperationToFinalize.md +1 -0
- eval_studio_client/api/docs/RequiredTheOperationToUpdate.md +1 -0
- eval_studio_client/api/docs/TestCaseServiceApi.md +75 -5
- eval_studio_client/api/docs/TestCaseServiceAppendTestCasesRequest.md +30 -0
- eval_studio_client/api/docs/TestClassServiceApi.md +5 -5
- eval_studio_client/api/docs/TestServiceApi.md +73 -5
- eval_studio_client/api/docs/V1ActualOutputMeta.md +30 -0
- eval_studio_client/api/docs/V1ActualOutputMetaDiff.md +36 -0
- eval_studio_client/api/docs/V1AgentChatActivityDiagram.md +31 -0
- eval_studio_client/api/docs/V1AgentChatActivityDiagramEdge.md +32 -0
- eval_studio_client/api/docs/V1AgentChatActivityDiagramNode.md +32 -0
- eval_studio_client/api/docs/V1AgentChatActivityDiagramRow.md +30 -0
- eval_studio_client/api/docs/V1AgentChatScriptUsage.md +33 -0
- eval_studio_client/api/docs/V1AgentChatScriptsBarChart.md +30 -0
- eval_studio_client/api/docs/V1AgentChatToolUsage.md +33 -0
- eval_studio_client/api/docs/V1AgentChatToolsBarChart.md +30 -0
- eval_studio_client/api/docs/V1AllMetricScores.md +29 -0
- eval_studio_client/api/docs/V1AppendTestCasesResponse.md +29 -0
- eval_studio_client/api/docs/V1BatchCreateLeaderboardsWithoutCacheRequest.md +31 -0
- eval_studio_client/api/docs/V1BatchCreateLeaderboardsWithoutCacheResponse.md +29 -0
- eval_studio_client/api/docs/V1BatchMarkOperationSeenByCreatorResponse.md +29 -0
- eval_studio_client/api/docs/V1CmpLeaderboardReportsRequest.md +33 -0
- eval_studio_client/api/docs/V1CmpLeaderboardReportsResponse.md +29 -0
- eval_studio_client/api/docs/V1ComparisonItem.md +36 -0
- eval_studio_client/api/docs/V1ComparisonMetricScore.md +30 -0
- eval_studio_client/api/docs/V1ComparisonResult.md +31 -0
- eval_studio_client/api/docs/V1ComparisonSummary.md +31 -0
- eval_studio_client/api/docs/V1CreateEvaluationRequest.md +1 -0
- eval_studio_client/api/docs/V1CreateTestFromTestCasesRequest.md +32 -0
- eval_studio_client/api/docs/V1CreateTestFromTestCasesResponse.md +29 -0
- eval_studio_client/api/docs/V1DashboardReport.md +31 -0
- eval_studio_client/api/docs/V1DashboardReportResult.md +39 -0
- eval_studio_client/api/docs/V1DashboardTestCaseAnnotation.md +36 -0
- eval_studio_client/api/docs/V1DataFragment.md +31 -0
- eval_studio_client/api/docs/V1DeepCompareLeaderboardsRequest.md +33 -0
- eval_studio_client/api/docs/V1DeepCompareLeaderboardsResponse.md +29 -0
- eval_studio_client/api/docs/V1DiffItem.md +36 -0
- eval_studio_client/api/docs/V1EvaluationType.md +12 -0
- eval_studio_client/api/docs/V1FlippedMetric.md +31 -0
- eval_studio_client/api/docs/V1GetDashboardReportResponse.md +29 -0
- eval_studio_client/api/docs/V1HumanDecision.md +12 -0
- eval_studio_client/api/docs/V1Info.md +1 -0
- eval_studio_client/api/docs/V1Leaderboard.md +1 -0
- eval_studio_client/api/docs/V1LeaderboardCmpReport.md +30 -0
- eval_studio_client/api/docs/V1LeaderboardComparisonItem.md +31 -0
- eval_studio_client/api/docs/V1LeaderboardInfo.md +30 -0
- eval_studio_client/api/docs/V1LeaderboardReportActualOutputMeta.md +6 -3
- eval_studio_client/api/docs/V1LeaderboardReportResult.md +11 -8
- eval_studio_client/api/docs/V1LeaderboardReportResultView.md +12 -0
- eval_studio_client/api/docs/V1LeaderboardTestCaseAnnotation.md +36 -0
- eval_studio_client/api/docs/V1ListDashboardTestCaseAnnotationsResponse.md +29 -0
- eval_studio_client/api/docs/V1ListLeaderboardTestCaseAnnotationsResponse.md +29 -0
- eval_studio_client/api/docs/V1ListOperationsResponse.md +1 -0
- eval_studio_client/api/docs/V1ListUnseenOperationsResponse.md +30 -0
- eval_studio_client/api/docs/V1MarkOperationSeenByCreatorResponse.md +29 -0
- eval_studio_client/api/docs/V1Metric.md +30 -0
- eval_studio_client/api/docs/V1MetricAverage.md +36 -0
- eval_studio_client/api/docs/V1MetricMeta.md +40 -0
- eval_studio_client/api/docs/V1MetricScore.md +1 -1
- eval_studio_client/api/docs/V1MetricScores.md +1 -1
- eval_studio_client/api/docs/V1ModelType.md +1 -1
- eval_studio_client/api/docs/V1ModelsComparisons.md +32 -0
- eval_studio_client/api/docs/V1ModelsComparisonsMetrics.md +33 -0
- eval_studio_client/api/docs/V1ModelsOverview.md +34 -0
- eval_studio_client/api/docs/V1Operation.md +1 -0
- eval_studio_client/api/docs/V1OperationView.md +12 -0
- eval_studio_client/api/docs/V1RetrievedContextDiff.md +36 -0
- eval_studio_client/api/docs/V1Stats.md +2 -0
- eval_studio_client/api/docs/V1TechnicalMetrics.md +30 -0
- eval_studio_client/api/docs/V1TechnicalMetricsDetail.md +33 -0
- eval_studio_client/api/docs/V1TestCaseLeaderboardItem.md +31 -0
- eval_studio_client/api/docs/V1TestCaseRelationshipInfo.md +31 -0
- eval_studio_client/api/docs/V1TestCaseResult.md +48 -0
- eval_studio_client/api/docs/V1TextSimilarityMetric.md +12 -0
- eval_studio_client/api/docs/V1UpdateDashboardTestCaseAnnotationResponse.md +29 -0
- eval_studio_client/api/docs/V1UpdateLeaderboardTestCaseAnnotationResponse.md +29 -0
- eval_studio_client/api/docs/WorkflowNodeServiceApi.md +5 -5
- eval_studio_client/api/docs/WorkflowServiceApi.md +5 -5
- eval_studio_client/api/models/__init__.py +62 -0
- eval_studio_client/api/models/adversarial_inputs_service_test_adversarial_inputs_robustness_request.py +17 -2
- eval_studio_client/api/models/required_the_dashboard_test_case_annotation_to_update.py +108 -0
- eval_studio_client/api/models/required_the_leaderboard_test_case_annotation_to_update.py +108 -0
- eval_studio_client/api/models/required_the_leaderboard_to_update.py +5 -2
- eval_studio_client/api/models/required_the_operation_to_finalize.py +6 -2
- eval_studio_client/api/models/required_the_operation_to_update.py +6 -2
- eval_studio_client/api/models/test_case_service_append_test_cases_request.py +89 -0
- eval_studio_client/api/models/v1_actual_output_meta.py +97 -0
- eval_studio_client/api/models/v1_actual_output_meta_diff.py +101 -0
- eval_studio_client/api/models/v1_agent_chat_activity_diagram.py +109 -0
- eval_studio_client/api/models/v1_agent_chat_activity_diagram_edge.py +97 -0
- eval_studio_client/api/models/v1_agent_chat_activity_diagram_node.py +97 -0
- eval_studio_client/api/models/v1_agent_chat_activity_diagram_row.py +97 -0
- eval_studio_client/api/models/v1_agent_chat_script_usage.py +101 -0
- eval_studio_client/api/models/v1_agent_chat_scripts_bar_chart.py +102 -0
- eval_studio_client/api/models/v1_agent_chat_tool_usage.py +101 -0
- eval_studio_client/api/models/v1_agent_chat_tools_bar_chart.py +102 -0
- eval_studio_client/api/models/v1_all_metric_scores.py +87 -0
- eval_studio_client/api/models/v1_append_test_cases_response.py +95 -0
- eval_studio_client/api/models/v1_batch_create_leaderboards_without_cache_request.py +99 -0
- eval_studio_client/api/models/v1_batch_create_leaderboards_without_cache_response.py +91 -0
- eval_studio_client/api/models/v1_batch_mark_operation_seen_by_creator_response.py +95 -0
- eval_studio_client/api/models/v1_cmp_leaderboard_reports_request.py +96 -0
- eval_studio_client/api/models/v1_cmp_leaderboard_reports_response.py +91 -0
- eval_studio_client/api/models/v1_comparison_item.py +130 -0
- eval_studio_client/api/models/v1_comparison_metric_score.py +89 -0
- eval_studio_client/api/models/v1_comparison_result.py +120 -0
- eval_studio_client/api/models/v1_comparison_summary.py +91 -0
- eval_studio_client/api/models/v1_create_evaluation_request.py +5 -2
- eval_studio_client/api/models/v1_create_test_from_test_cases_request.py +93 -0
- eval_studio_client/api/models/v1_create_test_from_test_cases_response.py +91 -0
- eval_studio_client/api/models/v1_dashboard_report.py +109 -0
- eval_studio_client/api/models/v1_dashboard_report_result.py +139 -0
- eval_studio_client/api/models/v1_dashboard_test_case_annotation.py +112 -0
- eval_studio_client/api/models/v1_data_fragment.py +91 -0
- eval_studio_client/api/models/v1_deep_compare_leaderboards_request.py +96 -0
- eval_studio_client/api/models/v1_deep_compare_leaderboards_response.py +91 -0
- eval_studio_client/api/models/v1_diff_item.py +137 -0
- eval_studio_client/api/models/v1_evaluation_type.py +39 -0
- eval_studio_client/api/models/v1_flipped_metric.py +91 -0
- eval_studio_client/api/models/v1_get_dashboard_report_response.py +91 -0
- eval_studio_client/api/models/v1_human_decision.py +38 -0
- eval_studio_client/api/models/v1_info.py +4 -2
- eval_studio_client/api/models/v1_leaderboard.py +5 -2
- eval_studio_client/api/models/v1_leaderboard_cmp_report.py +93 -0
- eval_studio_client/api/models/v1_leaderboard_comparison_item.py +91 -0
- eval_studio_client/api/models/v1_leaderboard_info.py +97 -0
- eval_studio_client/api/models/v1_leaderboard_report_actual_output_meta.py +23 -9
- eval_studio_client/api/models/v1_leaderboard_report_result.py +21 -10
- eval_studio_client/api/models/v1_leaderboard_report_result_view.py +38 -0
- eval_studio_client/api/models/v1_leaderboard_test_case_annotation.py +112 -0
- eval_studio_client/api/models/v1_list_dashboard_test_case_annotations_response.py +95 -0
- eval_studio_client/api/models/v1_list_leaderboard_test_case_annotations_response.py +95 -0
- eval_studio_client/api/models/v1_list_operations_response.py +5 -3
- eval_studio_client/api/models/v1_list_unseen_operations_response.py +97 -0
- eval_studio_client/api/models/v1_mark_operation_seen_by_creator_response.py +91 -0
- eval_studio_client/api/models/v1_metric.py +89 -0
- eval_studio_client/api/models/v1_metric_average.py +101 -0
- eval_studio_client/api/models/v1_metric_meta.py +109 -0
- eval_studio_client/api/models/v1_metric_score.py +6 -1
- eval_studio_client/api/models/v1_metric_scores.py +1 -1
- eval_studio_client/api/models/v1_model_type.py +2 -1
- eval_studio_client/api/models/v1_models_comparisons.py +93 -0
- eval_studio_client/api/models/v1_models_comparisons_metrics.py +103 -0
- eval_studio_client/api/models/v1_models_overview.py +97 -0
- eval_studio_client/api/models/v1_operation.py +6 -2
- eval_studio_client/api/models/v1_operation_view.py +38 -0
- eval_studio_client/api/models/v1_retrieved_context_diff.py +101 -0
- eval_studio_client/api/models/v1_stats.py +16 -2
- eval_studio_client/api/models/v1_technical_metrics.py +96 -0
- eval_studio_client/api/models/v1_technical_metrics_detail.py +95 -0
- eval_studio_client/api/models/v1_test_case_leaderboard_item.py +91 -0
- eval_studio_client/api/models/v1_test_case_relationship_info.py +91 -0
- eval_studio_client/api/models/v1_test_case_result.py +157 -0
- eval_studio_client/api/models/v1_text_similarity_metric.py +39 -0
- eval_studio_client/api/models/v1_update_dashboard_test_case_annotation_response.py +91 -0
- eval_studio_client/api/models/v1_update_leaderboard_test_case_annotation_response.py +91 -0
- eval_studio_client/api/models/v1_workflow_node_type.py +1 -0
- eval_studio_client/api/models/v1_workflow_type.py +1 -0
- eval_studio_client/api/test/test_adversarial_inputs_service_test_adversarial_inputs_robustness_request.py +6 -0
- eval_studio_client/api/test/test_dashboard_report_service_api.py +37 -0
- eval_studio_client/api/test/test_dashboard_test_case_annotation_service_api.py +43 -0
- eval_studio_client/api/test/test_leaderboard_report_service_api.py +6 -0
- eval_studio_client/api/test/test_leaderboard_service_api.py +12 -0
- eval_studio_client/api/test/test_leaderboard_test_case_annotation_service_api.py +43 -0
- eval_studio_client/api/test/test_operation_service_api.py +18 -0
- eval_studio_client/api/test/test_required_the_dashboard_test_case_annotation_to_update.py +57 -0
- eval_studio_client/api/test/test_required_the_leaderboard_test_case_annotation_to_update.py +57 -0
- eval_studio_client/api/test/test_required_the_leaderboard_to_update.py +2 -1
- eval_studio_client/api/test/test_required_the_operation_to_finalize.py +2 -1
- eval_studio_client/api/test/test_required_the_operation_to_update.py +2 -1
- eval_studio_client/api/test/test_test_case_service_api.py +6 -0
- eval_studio_client/api/test/test_test_case_service_append_test_cases_request.py +52 -0
- eval_studio_client/api/test/test_test_service_api.py +6 -0
- eval_studio_client/api/test/test_v1_abort_operation_response.py +2 -1
- eval_studio_client/api/test/test_v1_actual_output_meta.py +61 -0
- eval_studio_client/api/test/test_v1_actual_output_meta_diff.py +66 -0
- eval_studio_client/api/test/test_v1_agent_chat_activity_diagram.py +65 -0
- eval_studio_client/api/test/test_v1_agent_chat_activity_diagram_edge.py +53 -0
- eval_studio_client/api/test/test_v1_agent_chat_activity_diagram_node.py +53 -0
- eval_studio_client/api/test/test_v1_agent_chat_activity_diagram_row.py +56 -0
- eval_studio_client/api/test/test_v1_agent_chat_script_usage.py +54 -0
- eval_studio_client/api/test/test_v1_agent_chat_scripts_bar_chart.py +57 -0
- eval_studio_client/api/test/test_v1_agent_chat_tool_usage.py +54 -0
- eval_studio_client/api/test/test_v1_agent_chat_tools_bar_chart.py +57 -0
- eval_studio_client/api/test/test_v1_all_metric_scores.py +53 -0
- eval_studio_client/api/test/test_v1_append_test_cases_response.py +74 -0
- eval_studio_client/api/test/test_v1_batch_create_leaderboards_request.py +2 -1
- eval_studio_client/api/test/test_v1_batch_create_leaderboards_response.py +2 -1
- eval_studio_client/api/test/test_v1_batch_create_leaderboards_without_cache_request.py +120 -0
- eval_studio_client/api/test/test_v1_batch_create_leaderboards_without_cache_response.py +72 -0
- eval_studio_client/api/test/test_v1_batch_delete_leaderboards_response.py +2 -1
- eval_studio_client/api/test/test_v1_batch_get_leaderboards_response.py +2 -1
- eval_studio_client/api/test/test_v1_batch_get_operations_response.py +2 -1
- eval_studio_client/api/test/test_v1_batch_import_leaderboard_response.py +2 -1
- eval_studio_client/api/test/test_v1_batch_mark_operation_seen_by_creator_response.py +74 -0
- eval_studio_client/api/test/test_v1_cmp_leaderboard_reports_request.py +55 -0
- eval_studio_client/api/test/test_v1_cmp_leaderboard_reports_response.py +255 -0
- eval_studio_client/api/test/test_v1_comparison_item.py +233 -0
- eval_studio_client/api/test/test_v1_comparison_metric_score.py +52 -0
- eval_studio_client/api/test/test_v1_comparison_result.py +258 -0
- eval_studio_client/api/test/test_v1_comparison_summary.py +53 -0
- eval_studio_client/api/test/test_v1_create_evaluation_request.py +2 -1
- eval_studio_client/api/test/test_v1_create_leaderboard_request.py +2 -1
- eval_studio_client/api/test/test_v1_create_leaderboard_response.py +2 -1
- eval_studio_client/api/test/test_v1_create_leaderboard_without_cache_response.py +2 -1
- eval_studio_client/api/test/test_v1_create_test_from_test_cases_request.py +54 -0
- eval_studio_client/api/test/test_v1_create_test_from_test_cases_response.py +68 -0
- eval_studio_client/api/test/test_v1_dashboard_report.py +142 -0
- eval_studio_client/api/test/test_v1_dashboard_report_result.py +72 -0
- eval_studio_client/api/test/test_v1_dashboard_test_case_annotation.py +58 -0
- eval_studio_client/api/test/test_v1_data_fragment.py +57 -0
- eval_studio_client/api/test/test_v1_deep_compare_leaderboards_request.py +55 -0
- eval_studio_client/api/test/test_v1_deep_compare_leaderboards_response.py +255 -0
- eval_studio_client/api/test/test_v1_delete_leaderboard_response.py +2 -1
- eval_studio_client/api/test/test_v1_diff_item.py +226 -0
- eval_studio_client/api/test/test_v1_evaluation_type.py +33 -0
- eval_studio_client/api/test/test_v1_finalize_operation_response.py +2 -1
- eval_studio_client/api/test/test_v1_flipped_metric.py +53 -0
- eval_studio_client/api/test/test_v1_generate_test_cases_response.py +2 -1
- eval_studio_client/api/test/test_v1_get_dashboard_report_response.py +143 -0
- eval_studio_client/api/test/test_v1_get_info_response.py +4 -1
- eval_studio_client/api/test/test_v1_get_leaderboard_report_response.py +39 -2
- eval_studio_client/api/test/test_v1_get_leaderboard_response.py +2 -1
- eval_studio_client/api/test/test_v1_get_operation_response.py +2 -1
- eval_studio_client/api/test/test_v1_get_stats_response.py +3 -1
- eval_studio_client/api/test/test_v1_human_decision.py +33 -0
- eval_studio_client/api/test/test_v1_import_leaderboard_response.py +2 -1
- eval_studio_client/api/test/test_v1_import_test_cases_from_library_response.py +2 -1
- eval_studio_client/api/test/test_v1_info.py +4 -1
- eval_studio_client/api/test/test_v1_leaderboard.py +2 -1
- eval_studio_client/api/test/test_v1_leaderboard_cmp_report.py +254 -0
- eval_studio_client/api/test/test_v1_leaderboard_comparison_item.py +53 -0
- eval_studio_client/api/test/test_v1_leaderboard_info.py +57 -0
- eval_studio_client/api/test/test_v1_leaderboard_report.py +39 -2
- eval_studio_client/api/test/test_v1_leaderboard_report_actual_output_meta.py +33 -1
- eval_studio_client/api/test/test_v1_leaderboard_report_result.py +39 -2
- eval_studio_client/api/test/test_v1_leaderboard_report_result_view.py +33 -0
- eval_studio_client/api/test/test_v1_leaderboard_test_case_annotation.py +58 -0
- eval_studio_client/api/test/test_v1_list_dashboard_test_case_annotations_response.py +61 -0
- eval_studio_client/api/test/test_v1_list_leaderboard_test_case_annotations_response.py +61 -0
- eval_studio_client/api/test/test_v1_list_leaderboards_response.py +2 -1
- eval_studio_client/api/test/test_v1_list_most_recent_leaderboards_response.py +2 -1
- eval_studio_client/api/test/test_v1_list_operations_response.py +4 -2
- eval_studio_client/api/test/test_v1_list_unseen_operations_response.py +75 -0
- eval_studio_client/api/test/test_v1_mark_operation_seen_by_creator_response.py +72 -0
- eval_studio_client/api/test/test_v1_metric.py +52 -0
- eval_studio_client/api/test/test_v1_metric_average.py +58 -0
- eval_studio_client/api/test/test_v1_metric_meta.py +66 -0
- eval_studio_client/api/test/test_v1_models_comparisons.py +54 -0
- eval_studio_client/api/test/test_v1_models_comparisons_metrics.py +65 -0
- eval_studio_client/api/test/test_v1_models_overview.py +60 -0
- eval_studio_client/api/test/test_v1_operation.py +2 -1
- eval_studio_client/api/test/test_v1_operation_view.py +33 -0
- eval_studio_client/api/test/test_v1_process_workflow_node_response.py +2 -1
- eval_studio_client/api/test/test_v1_retrieved_context_diff.py +66 -0
- eval_studio_client/api/test/test_v1_stats.py +3 -1
- eval_studio_client/api/test/test_v1_technical_metrics.py +62 -0
- eval_studio_client/api/test/test_v1_technical_metrics_detail.py +55 -0
- eval_studio_client/api/test/test_v1_test_case_leaderboard_item.py +53 -0
- eval_studio_client/api/test/test_v1_test_case_relationship_info.py +53 -0
- eval_studio_client/api/test/test_v1_test_case_result.py +106 -0
- eval_studio_client/api/test/test_v1_text_similarity_metric.py +33 -0
- eval_studio_client/api/test/test_v1_update_dashboard_test_case_annotation_response.py +59 -0
- eval_studio_client/api/test/test_v1_update_leaderboard_response.py +2 -1
- eval_studio_client/api/test/test_v1_update_leaderboard_test_case_annotation_response.py +59 -0
- eval_studio_client/api/test/test_v1_update_operation_response.py +2 -1
- eval_studio_client/gen/openapiv2/eval_studio.swagger.json +2340 -210
- eval_studio_client/models.py +18 -6
- {eval_studio_client-1.2.4a2.dist-info → eval_studio_client-1.3.0.dist-info}/METADATA +2 -2
- {eval_studio_client-1.2.4a2.dist-info → eval_studio_client-1.3.0.dist-info}/RECORD +306 -111
- {eval_studio_client-1.2.4a2.dist-info → eval_studio_client-1.3.0.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from datetime import datetime
|
|
21
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictStr
|
|
22
|
+
from typing import Any, ClassVar, Dict, List, Optional
|
|
23
|
+
from typing import Optional, Set
|
|
24
|
+
from typing_extensions import Self
|
|
25
|
+
|
|
26
|
+
class V1DashboardTestCaseAnnotation(BaseModel):
|
|
27
|
+
"""
|
|
28
|
+
V1DashboardTestCaseAnnotation
|
|
29
|
+
""" # noqa: E501
|
|
30
|
+
name: Optional[StrictStr] = None
|
|
31
|
+
create_time: Optional[datetime] = Field(default=None, description="Output only. Timestamp when the DashboardTestCaseAnnotation was created.", alias="createTime")
|
|
32
|
+
creator: Optional[StrictStr] = Field(default=None, description="Output only. Name of the user or service that requested creation of the DashboardTestCaseAnnotation.")
|
|
33
|
+
update_time: Optional[datetime] = Field(default=None, description="Output only. Optional. Timestamp when the DashboardTestCaseAnnotation was last updated.", alias="updateTime")
|
|
34
|
+
updater: Optional[StrictStr] = Field(default=None, description="Output only. Optional. Name of the user or service that requested update of the DashboardTestCaseAnnotation.")
|
|
35
|
+
parent: Optional[StrictStr] = Field(default=None, description="Parent Dashboard Test Case resource name. e.g.: \"dashboards/<UUID>/testCases/<UUID>\".")
|
|
36
|
+
key: Optional[StrictStr] = Field(default=None, description="Immutable. Annotation key.")
|
|
37
|
+
value: Optional[Dict[str, Any]] = Field(default=None, description="Annotation value.")
|
|
38
|
+
__properties: ClassVar[List[str]] = ["name", "createTime", "creator", "updateTime", "updater", "parent", "key", "value"]
|
|
39
|
+
|
|
40
|
+
model_config = ConfigDict(
|
|
41
|
+
populate_by_name=True,
|
|
42
|
+
validate_assignment=True,
|
|
43
|
+
protected_namespaces=(),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def to_str(self) -> str:
|
|
48
|
+
"""Returns the string representation of the model using alias"""
|
|
49
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
50
|
+
|
|
51
|
+
def to_json(self) -> str:
|
|
52
|
+
"""Returns the JSON representation of the model using alias"""
|
|
53
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
54
|
+
return json.dumps(self.to_dict())
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
58
|
+
"""Create an instance of V1DashboardTestCaseAnnotation from a JSON string"""
|
|
59
|
+
return cls.from_dict(json.loads(json_str))
|
|
60
|
+
|
|
61
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
62
|
+
"""Return the dictionary representation of the model using alias.
|
|
63
|
+
|
|
64
|
+
This has the following differences from calling pydantic's
|
|
65
|
+
`self.model_dump(by_alias=True)`:
|
|
66
|
+
|
|
67
|
+
* `None` is only added to the output dict for nullable fields that
|
|
68
|
+
were set at model initialization. Other fields with value `None`
|
|
69
|
+
are ignored.
|
|
70
|
+
* OpenAPI `readOnly` fields are excluded.
|
|
71
|
+
* OpenAPI `readOnly` fields are excluded.
|
|
72
|
+
* OpenAPI `readOnly` fields are excluded.
|
|
73
|
+
* OpenAPI `readOnly` fields are excluded.
|
|
74
|
+
* OpenAPI `readOnly` fields are excluded.
|
|
75
|
+
"""
|
|
76
|
+
excluded_fields: Set[str] = set([
|
|
77
|
+
"name",
|
|
78
|
+
"create_time",
|
|
79
|
+
"creator",
|
|
80
|
+
"update_time",
|
|
81
|
+
"updater",
|
|
82
|
+
])
|
|
83
|
+
|
|
84
|
+
_dict = self.model_dump(
|
|
85
|
+
by_alias=True,
|
|
86
|
+
exclude=excluded_fields,
|
|
87
|
+
exclude_none=True,
|
|
88
|
+
)
|
|
89
|
+
return _dict
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
93
|
+
"""Create an instance of V1DashboardTestCaseAnnotation from a dict"""
|
|
94
|
+
if obj is None:
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
if not isinstance(obj, dict):
|
|
98
|
+
return cls.model_validate(obj, strict=False)
|
|
99
|
+
|
|
100
|
+
_obj = cls.model_validate({
|
|
101
|
+
"name": obj.get("name"),
|
|
102
|
+
"createTime": obj.get("createTime"),
|
|
103
|
+
"creator": obj.get("creator"),
|
|
104
|
+
"updateTime": obj.get("updateTime"),
|
|
105
|
+
"updater": obj.get("updater"),
|
|
106
|
+
"parent": obj.get("parent"),
|
|
107
|
+
"key": obj.get("key"),
|
|
108
|
+
"value": obj.get("value")
|
|
109
|
+
}, strict=False)
|
|
110
|
+
return _obj
|
|
111
|
+
|
|
112
|
+
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional, Union
|
|
22
|
+
from typing import Optional, Set
|
|
23
|
+
from typing_extensions import Self
|
|
24
|
+
|
|
25
|
+
class V1DataFragment(BaseModel):
|
|
26
|
+
"""
|
|
27
|
+
V1DataFragment
|
|
28
|
+
""" # noqa: E501
|
|
29
|
+
text: Optional[StrictStr] = Field(default=None, description="Text content.")
|
|
30
|
+
metrics: Optional[Dict[str, Union[StrictFloat, StrictInt]]] = Field(default=None, description="Metrics for this fragment.")
|
|
31
|
+
meta: Optional[Dict[str, StrictStr]] = Field(default=None, description="Additional metadata.")
|
|
32
|
+
__properties: ClassVar[List[str]] = ["text", "metrics", "meta"]
|
|
33
|
+
|
|
34
|
+
model_config = ConfigDict(
|
|
35
|
+
populate_by_name=True,
|
|
36
|
+
validate_assignment=True,
|
|
37
|
+
protected_namespaces=(),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def to_str(self) -> str:
|
|
42
|
+
"""Returns the string representation of the model using alias"""
|
|
43
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
44
|
+
|
|
45
|
+
def to_json(self) -> str:
|
|
46
|
+
"""Returns the JSON representation of the model using alias"""
|
|
47
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
48
|
+
return json.dumps(self.to_dict())
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
52
|
+
"""Create an instance of V1DataFragment from a JSON string"""
|
|
53
|
+
return cls.from_dict(json.loads(json_str))
|
|
54
|
+
|
|
55
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
56
|
+
"""Return the dictionary representation of the model using alias.
|
|
57
|
+
|
|
58
|
+
This has the following differences from calling pydantic's
|
|
59
|
+
`self.model_dump(by_alias=True)`:
|
|
60
|
+
|
|
61
|
+
* `None` is only added to the output dict for nullable fields that
|
|
62
|
+
were set at model initialization. Other fields with value `None`
|
|
63
|
+
are ignored.
|
|
64
|
+
"""
|
|
65
|
+
excluded_fields: Set[str] = set([
|
|
66
|
+
])
|
|
67
|
+
|
|
68
|
+
_dict = self.model_dump(
|
|
69
|
+
by_alias=True,
|
|
70
|
+
exclude=excluded_fields,
|
|
71
|
+
exclude_none=True,
|
|
72
|
+
)
|
|
73
|
+
return _dict
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
77
|
+
"""Create an instance of V1DataFragment from a dict"""
|
|
78
|
+
if obj is None:
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
if not isinstance(obj, dict):
|
|
82
|
+
return cls.model_validate(obj, strict=False)
|
|
83
|
+
|
|
84
|
+
_obj = cls.model_validate({
|
|
85
|
+
"text": obj.get("text"),
|
|
86
|
+
"metrics": obj.get("metrics"),
|
|
87
|
+
"meta": obj.get("meta")
|
|
88
|
+
}, strict=False)
|
|
89
|
+
return _obj
|
|
90
|
+
|
|
91
|
+
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictStr
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional
|
|
22
|
+
from eval_studio_client.api.models.v1_text_similarity_metric import V1TextSimilarityMetric
|
|
23
|
+
from typing import Optional, Set
|
|
24
|
+
from typing_extensions import Self
|
|
25
|
+
|
|
26
|
+
class V1DeepCompareLeaderboardsRequest(BaseModel):
|
|
27
|
+
"""
|
|
28
|
+
V1DeepCompareLeaderboardsRequest
|
|
29
|
+
""" # noqa: E501
|
|
30
|
+
leaderboard_baseline_name: Optional[StrictStr] = Field(default=None, description="Required. The resource name of the baseline leaderboard to compare against.", alias="leaderboardBaselineName")
|
|
31
|
+
leaderboard_current_name: Optional[StrictStr] = Field(default=None, description="Required. The resource name of the current leaderboard to compare.", alias="leaderboardCurrentName")
|
|
32
|
+
text_similarity_metric: Optional[V1TextSimilarityMetric] = Field(default=None, alias="textSimilarityMetric")
|
|
33
|
+
llm_model_baseline_name: Optional[StrictStr] = Field(default=None, description="Required. The baseline LLM model name to compare.", alias="llmModelBaselineName")
|
|
34
|
+
llm_model_current_name: Optional[StrictStr] = Field(default=None, description="Required. The current LLM model name to compare.", alias="llmModelCurrentName")
|
|
35
|
+
__properties: ClassVar[List[str]] = ["leaderboardBaselineName", "leaderboardCurrentName", "textSimilarityMetric", "llmModelBaselineName", "llmModelCurrentName"]
|
|
36
|
+
|
|
37
|
+
model_config = ConfigDict(
|
|
38
|
+
populate_by_name=True,
|
|
39
|
+
validate_assignment=True,
|
|
40
|
+
protected_namespaces=(),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def to_str(self) -> str:
|
|
45
|
+
"""Returns the string representation of the model using alias"""
|
|
46
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
47
|
+
|
|
48
|
+
def to_json(self) -> str:
|
|
49
|
+
"""Returns the JSON representation of the model using alias"""
|
|
50
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
51
|
+
return json.dumps(self.to_dict())
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
55
|
+
"""Create an instance of V1DeepCompareLeaderboardsRequest from a JSON string"""
|
|
56
|
+
return cls.from_dict(json.loads(json_str))
|
|
57
|
+
|
|
58
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
59
|
+
"""Return the dictionary representation of the model using alias.
|
|
60
|
+
|
|
61
|
+
This has the following differences from calling pydantic's
|
|
62
|
+
`self.model_dump(by_alias=True)`:
|
|
63
|
+
|
|
64
|
+
* `None` is only added to the output dict for nullable fields that
|
|
65
|
+
were set at model initialization. Other fields with value `None`
|
|
66
|
+
are ignored.
|
|
67
|
+
"""
|
|
68
|
+
excluded_fields: Set[str] = set([
|
|
69
|
+
])
|
|
70
|
+
|
|
71
|
+
_dict = self.model_dump(
|
|
72
|
+
by_alias=True,
|
|
73
|
+
exclude=excluded_fields,
|
|
74
|
+
exclude_none=True,
|
|
75
|
+
)
|
|
76
|
+
return _dict
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
80
|
+
"""Create an instance of V1DeepCompareLeaderboardsRequest from a dict"""
|
|
81
|
+
if obj is None:
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
if not isinstance(obj, dict):
|
|
85
|
+
return cls.model_validate(obj, strict=False)
|
|
86
|
+
|
|
87
|
+
_obj = cls.model_validate({
|
|
88
|
+
"leaderboardBaselineName": obj.get("leaderboardBaselineName"),
|
|
89
|
+
"leaderboardCurrentName": obj.get("leaderboardCurrentName"),
|
|
90
|
+
"textSimilarityMetric": obj.get("textSimilarityMetric"),
|
|
91
|
+
"llmModelBaselineName": obj.get("llmModelBaselineName"),
|
|
92
|
+
"llmModelCurrentName": obj.get("llmModelCurrentName")
|
|
93
|
+
}, strict=False)
|
|
94
|
+
return _obj
|
|
95
|
+
|
|
96
|
+
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional
|
|
22
|
+
from eval_studio_client.api.models.v1_leaderboard_cmp_report import V1LeaderboardCmpReport
|
|
23
|
+
from typing import Optional, Set
|
|
24
|
+
from typing_extensions import Self
|
|
25
|
+
|
|
26
|
+
class V1DeepCompareLeaderboardsResponse(BaseModel):
|
|
27
|
+
"""
|
|
28
|
+
V1DeepCompareLeaderboardsResponse
|
|
29
|
+
""" # noqa: E501
|
|
30
|
+
leaderboard_cmp_report: Optional[V1LeaderboardCmpReport] = Field(default=None, alias="leaderboardCmpReport")
|
|
31
|
+
__properties: ClassVar[List[str]] = ["leaderboardCmpReport"]
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(
|
|
34
|
+
populate_by_name=True,
|
|
35
|
+
validate_assignment=True,
|
|
36
|
+
protected_namespaces=(),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def to_str(self) -> str:
|
|
41
|
+
"""Returns the string representation of the model using alias"""
|
|
42
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
43
|
+
|
|
44
|
+
def to_json(self) -> str:
|
|
45
|
+
"""Returns the JSON representation of the model using alias"""
|
|
46
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
47
|
+
return json.dumps(self.to_dict())
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
51
|
+
"""Create an instance of V1DeepCompareLeaderboardsResponse from a JSON string"""
|
|
52
|
+
return cls.from_dict(json.loads(json_str))
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
55
|
+
"""Return the dictionary representation of the model using alias.
|
|
56
|
+
|
|
57
|
+
This has the following differences from calling pydantic's
|
|
58
|
+
`self.model_dump(by_alias=True)`:
|
|
59
|
+
|
|
60
|
+
* `None` is only added to the output dict for nullable fields that
|
|
61
|
+
were set at model initialization. Other fields with value `None`
|
|
62
|
+
are ignored.
|
|
63
|
+
"""
|
|
64
|
+
excluded_fields: Set[str] = set([
|
|
65
|
+
])
|
|
66
|
+
|
|
67
|
+
_dict = self.model_dump(
|
|
68
|
+
by_alias=True,
|
|
69
|
+
exclude=excluded_fields,
|
|
70
|
+
exclude_none=True,
|
|
71
|
+
)
|
|
72
|
+
# override the default output from pydantic by calling `to_dict()` of leaderboard_cmp_report
|
|
73
|
+
if self.leaderboard_cmp_report:
|
|
74
|
+
_dict['leaderboardCmpReport'] = self.leaderboard_cmp_report.to_dict()
|
|
75
|
+
return _dict
|
|
76
|
+
|
|
77
|
+
@classmethod
|
|
78
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
79
|
+
"""Create an instance of V1DeepCompareLeaderboardsResponse from a dict"""
|
|
80
|
+
if obj is None:
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
if not isinstance(obj, dict):
|
|
84
|
+
return cls.model_validate(obj, strict=False)
|
|
85
|
+
|
|
86
|
+
_obj = cls.model_validate({
|
|
87
|
+
"leaderboardCmpReport": V1LeaderboardCmpReport.from_dict(obj["leaderboardCmpReport"]) if obj.get("leaderboardCmpReport") is not None else None
|
|
88
|
+
}, strict=False)
|
|
89
|
+
return _obj
|
|
90
|
+
|
|
91
|
+
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictStr
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional
|
|
22
|
+
from eval_studio_client.api.models.v1_comparison_item import V1ComparisonItem
|
|
23
|
+
from eval_studio_client.api.models.v1_comparison_summary import V1ComparisonSummary
|
|
24
|
+
from eval_studio_client.api.models.v1_models_comparisons import V1ModelsComparisons
|
|
25
|
+
from eval_studio_client.api.models.v1_models_comparisons_metrics import V1ModelsComparisonsMetrics
|
|
26
|
+
from eval_studio_client.api.models.v1_models_overview import V1ModelsOverview
|
|
27
|
+
from eval_studio_client.api.models.v1_technical_metrics import V1TechnicalMetrics
|
|
28
|
+
from eval_studio_client.api.models.v1_test_case_leaderboard_item import V1TestCaseLeaderboardItem
|
|
29
|
+
from typing import Optional, Set
|
|
30
|
+
from typing_extensions import Self
|
|
31
|
+
|
|
32
|
+
class V1DiffItem(BaseModel):
|
|
33
|
+
"""
|
|
34
|
+
V1DiffItem
|
|
35
|
+
""" # noqa: E501
|
|
36
|
+
diff_key: Optional[StrictStr] = Field(default=None, description="Unique key identifying the diff (format: \"baseline_id|current_id\").", alias="diffKey")
|
|
37
|
+
items: Optional[List[V1ComparisonItem]] = Field(default=None, description="List of comparison items.")
|
|
38
|
+
summary: Optional[V1ComparisonSummary] = None
|
|
39
|
+
models_overview: Optional[V1ModelsOverview] = Field(default=None, alias="modelsOverview")
|
|
40
|
+
models_comparisons: Optional[V1ModelsComparisons] = Field(default=None, alias="modelsComparisons")
|
|
41
|
+
models_comparisons_metrics: Optional[V1ModelsComparisonsMetrics] = Field(default=None, alias="modelsComparisonsMetrics")
|
|
42
|
+
technical_metrics: Optional[V1TechnicalMetrics] = Field(default=None, alias="technicalMetrics")
|
|
43
|
+
test_cases_leaderboard: Optional[List[V1TestCaseLeaderboardItem]] = Field(default=None, description="Test cases leaderboard.", alias="testCasesLeaderboard")
|
|
44
|
+
__properties: ClassVar[List[str]] = ["diffKey", "items", "summary", "modelsOverview", "modelsComparisons", "modelsComparisonsMetrics", "technicalMetrics", "testCasesLeaderboard"]
|
|
45
|
+
|
|
46
|
+
model_config = ConfigDict(
|
|
47
|
+
populate_by_name=True,
|
|
48
|
+
validate_assignment=True,
|
|
49
|
+
protected_namespaces=(),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def to_str(self) -> str:
|
|
54
|
+
"""Returns the string representation of the model using alias"""
|
|
55
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
56
|
+
|
|
57
|
+
def to_json(self) -> str:
|
|
58
|
+
"""Returns the JSON representation of the model using alias"""
|
|
59
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
60
|
+
return json.dumps(self.to_dict())
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
64
|
+
"""Create an instance of V1DiffItem from a JSON string"""
|
|
65
|
+
return cls.from_dict(json.loads(json_str))
|
|
66
|
+
|
|
67
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
68
|
+
"""Return the dictionary representation of the model using alias.
|
|
69
|
+
|
|
70
|
+
This has the following differences from calling pydantic's
|
|
71
|
+
`self.model_dump(by_alias=True)`:
|
|
72
|
+
|
|
73
|
+
* `None` is only added to the output dict for nullable fields that
|
|
74
|
+
were set at model initialization. Other fields with value `None`
|
|
75
|
+
are ignored.
|
|
76
|
+
"""
|
|
77
|
+
excluded_fields: Set[str] = set([
|
|
78
|
+
])
|
|
79
|
+
|
|
80
|
+
_dict = self.model_dump(
|
|
81
|
+
by_alias=True,
|
|
82
|
+
exclude=excluded_fields,
|
|
83
|
+
exclude_none=True,
|
|
84
|
+
)
|
|
85
|
+
# override the default output from pydantic by calling `to_dict()` of each item in items (list)
|
|
86
|
+
_items = []
|
|
87
|
+
if self.items:
|
|
88
|
+
for _item_items in self.items:
|
|
89
|
+
if _item_items:
|
|
90
|
+
_items.append(_item_items.to_dict())
|
|
91
|
+
_dict['items'] = _items
|
|
92
|
+
# override the default output from pydantic by calling `to_dict()` of summary
|
|
93
|
+
if self.summary:
|
|
94
|
+
_dict['summary'] = self.summary.to_dict()
|
|
95
|
+
# override the default output from pydantic by calling `to_dict()` of models_overview
|
|
96
|
+
if self.models_overview:
|
|
97
|
+
_dict['modelsOverview'] = self.models_overview.to_dict()
|
|
98
|
+
# override the default output from pydantic by calling `to_dict()` of models_comparisons
|
|
99
|
+
if self.models_comparisons:
|
|
100
|
+
_dict['modelsComparisons'] = self.models_comparisons.to_dict()
|
|
101
|
+
# override the default output from pydantic by calling `to_dict()` of models_comparisons_metrics
|
|
102
|
+
if self.models_comparisons_metrics:
|
|
103
|
+
_dict['modelsComparisonsMetrics'] = self.models_comparisons_metrics.to_dict()
|
|
104
|
+
# override the default output from pydantic by calling `to_dict()` of technical_metrics
|
|
105
|
+
if self.technical_metrics:
|
|
106
|
+
_dict['technicalMetrics'] = self.technical_metrics.to_dict()
|
|
107
|
+
# override the default output from pydantic by calling `to_dict()` of each item in test_cases_leaderboard (list)
|
|
108
|
+
_items = []
|
|
109
|
+
if self.test_cases_leaderboard:
|
|
110
|
+
for _item_test_cases_leaderboard in self.test_cases_leaderboard:
|
|
111
|
+
if _item_test_cases_leaderboard:
|
|
112
|
+
_items.append(_item_test_cases_leaderboard.to_dict())
|
|
113
|
+
_dict['testCasesLeaderboard'] = _items
|
|
114
|
+
return _dict
|
|
115
|
+
|
|
116
|
+
@classmethod
|
|
117
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
118
|
+
"""Create an instance of V1DiffItem from a dict"""
|
|
119
|
+
if obj is None:
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
if not isinstance(obj, dict):
|
|
123
|
+
return cls.model_validate(obj, strict=False)
|
|
124
|
+
|
|
125
|
+
_obj = cls.model_validate({
|
|
126
|
+
"diffKey": obj.get("diffKey"),
|
|
127
|
+
"items": [V1ComparisonItem.from_dict(_item) for _item in obj["items"]] if obj.get("items") is not None else None,
|
|
128
|
+
"summary": V1ComparisonSummary.from_dict(obj["summary"]) if obj.get("summary") is not None else None,
|
|
129
|
+
"modelsOverview": V1ModelsOverview.from_dict(obj["modelsOverview"]) if obj.get("modelsOverview") is not None else None,
|
|
130
|
+
"modelsComparisons": V1ModelsComparisons.from_dict(obj["modelsComparisons"]) if obj.get("modelsComparisons") is not None else None,
|
|
131
|
+
"modelsComparisonsMetrics": V1ModelsComparisonsMetrics.from_dict(obj["modelsComparisonsMetrics"]) if obj.get("modelsComparisonsMetrics") is not None else None,
|
|
132
|
+
"technicalMetrics": V1TechnicalMetrics.from_dict(obj["technicalMetrics"]) if obj.get("technicalMetrics") is not None else None,
|
|
133
|
+
"testCasesLeaderboard": [V1TestCaseLeaderboardItem.from_dict(_item) for _item in obj["testCasesLeaderboard"]] if obj.get("testCasesLeaderboard") is not None else None
|
|
134
|
+
}, strict=False)
|
|
135
|
+
return _obj
|
|
136
|
+
|
|
137
|
+
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import json
|
|
17
|
+
from enum import Enum
|
|
18
|
+
from typing_extensions import Self
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class V1EvaluationType(str, Enum):
|
|
22
|
+
"""
|
|
23
|
+
- EVALUATION_TYPE_UNSPECIFIED: Unspecified type. - EVALUATION_TYPE_LLM: LLM Evaluation, where evaluated model doesn't retrieve any context. - EVALUATION_TYPE_RAG: RAG Evaluation, where evaluated model retrieves context from a given collection. - EVALUATION_TYPE_AGENT: Agent Evaluation, where evaluated model is an agent that can interact with tools.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
"""
|
|
27
|
+
allowed enum values
|
|
28
|
+
"""
|
|
29
|
+
EVALUATION_TYPE_UNSPECIFIED = 'EVALUATION_TYPE_UNSPECIFIED'
|
|
30
|
+
EVALUATION_TYPE_LLM = 'EVALUATION_TYPE_LLM'
|
|
31
|
+
EVALUATION_TYPE_RAG = 'EVALUATION_TYPE_RAG'
|
|
32
|
+
EVALUATION_TYPE_AGENT = 'EVALUATION_TYPE_AGENT'
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def from_json(cls, json_str: str) -> Self:
|
|
36
|
+
"""Create an instance of V1EvaluationType from a JSON string"""
|
|
37
|
+
return cls(json.loads(json_str))
|
|
38
|
+
|
|
39
|
+
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
ai/h2o/eval_studio/v1/insight.proto
|
|
5
|
+
|
|
6
|
+
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
|
|
7
|
+
|
|
8
|
+
The version of the OpenAPI document: version not set
|
|
9
|
+
Generated by OpenAPI Generator (https://openapi-generator.tech)
|
|
10
|
+
|
|
11
|
+
Do not edit the class manually.
|
|
12
|
+
""" # noqa: E501
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
import pprint
|
|
17
|
+
import re # noqa: F401
|
|
18
|
+
import json
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr
|
|
21
|
+
from typing import Any, ClassVar, Dict, List, Optional, Union
|
|
22
|
+
from typing import Optional, Set
|
|
23
|
+
from typing_extensions import Self
|
|
24
|
+
|
|
25
|
+
class V1FlippedMetric(BaseModel):
|
|
26
|
+
"""
|
|
27
|
+
V1FlippedMetric
|
|
28
|
+
""" # noqa: E501
|
|
29
|
+
metric_name: Optional[StrictStr] = Field(default=None, description="Metric name.", alias="metricName")
|
|
30
|
+
baseline_value: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Baseline value.", alias="baselineValue")
|
|
31
|
+
current_value: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Current value.", alias="currentValue")
|
|
32
|
+
__properties: ClassVar[List[str]] = ["metricName", "baselineValue", "currentValue"]
|
|
33
|
+
|
|
34
|
+
model_config = ConfigDict(
|
|
35
|
+
populate_by_name=True,
|
|
36
|
+
validate_assignment=True,
|
|
37
|
+
protected_namespaces=(),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def to_str(self) -> str:
|
|
42
|
+
"""Returns the string representation of the model using alias"""
|
|
43
|
+
return pprint.pformat(self.model_dump(by_alias=True))
|
|
44
|
+
|
|
45
|
+
def to_json(self) -> str:
|
|
46
|
+
"""Returns the JSON representation of the model using alias"""
|
|
47
|
+
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
|
|
48
|
+
return json.dumps(self.to_dict())
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_json(cls, json_str: str) -> Optional[Self]:
|
|
52
|
+
"""Create an instance of V1FlippedMetric from a JSON string"""
|
|
53
|
+
return cls.from_dict(json.loads(json_str))
|
|
54
|
+
|
|
55
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
56
|
+
"""Return the dictionary representation of the model using alias.
|
|
57
|
+
|
|
58
|
+
This has the following differences from calling pydantic's
|
|
59
|
+
`self.model_dump(by_alias=True)`:
|
|
60
|
+
|
|
61
|
+
* `None` is only added to the output dict for nullable fields that
|
|
62
|
+
were set at model initialization. Other fields with value `None`
|
|
63
|
+
are ignored.
|
|
64
|
+
"""
|
|
65
|
+
excluded_fields: Set[str] = set([
|
|
66
|
+
])
|
|
67
|
+
|
|
68
|
+
_dict = self.model_dump(
|
|
69
|
+
by_alias=True,
|
|
70
|
+
exclude=excluded_fields,
|
|
71
|
+
exclude_none=True,
|
|
72
|
+
)
|
|
73
|
+
return _dict
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
|
|
77
|
+
"""Create an instance of V1FlippedMetric from a dict"""
|
|
78
|
+
if obj is None:
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
if not isinstance(obj, dict):
|
|
82
|
+
return cls.model_validate(obj, strict=False)
|
|
83
|
+
|
|
84
|
+
_obj = cls.model_validate({
|
|
85
|
+
"metricName": obj.get("metricName"),
|
|
86
|
+
"baselineValue": obj.get("baselineValue"),
|
|
87
|
+
"currentValue": obj.get("currentValue")
|
|
88
|
+
}, strict=False)
|
|
89
|
+
return _obj
|
|
90
|
+
|
|
91
|
+
|