eval-studio-client 1.2.5__py3-none-any.whl → 1.3.0a1__py3-none-any.whl

eval_studio_client/api/test/test_v1_diff_item.py

@@ -0,0 +1,226 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_diff_item import V1DiffItem
+
+class TestV1DiffItem(unittest.TestCase):
+    """V1DiffItem unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1DiffItem:
+        """Test V1DiffItem
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1DiffItem`
+        """
+        model = V1DiffItem()
+        if include_optional:
+            return V1DiffItem(
+                diff_key = '',
+                items = [
+                    eval_studio_client.api.models.a_single_comparison_item_showing_differences_between_baseline_and_current.A single comparison item showing differences between baseline and current(
+                        question = '', 
+                        diff_flipped_metrics = [
+                            eval_studio_client.api.models.flipped_metric_information.Flipped metric information(
+                                metric_name = '', 
+                                baseline_value = 1.337, 
+                                current_value = 1.337, )
+                            ], 
+                        baseline_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                            key = '', 
+                            input = '', 
+                            corpus = [
+                                ''
+                                ], 
+                            context = [
+                                ''
+                                ], 
+                            categories = [
+                                ''
+                                ], 
+                            relationships = [
+                                eval_studio_client.api.models.test_case_relationship_information.Test case relationship information(
+                                    type = '', 
+                                    target = '', 
+                                    target_type = '', )
+                                ], 
+                            expected_output = '', 
+                            output_constraints = [
+                                ''
+                                ], 
+                            output_condition = '', 
+                            actual_output = '', 
+                            actual_duration = 1.337, 
+                            cost = 1.337, 
+                            model_key = '', 
+                            test_key = '', 
+                            test_case_key = '', 
+                            metrics = [
+                                eval_studio_client.api.models.metric_information.Metric information(
+                                    key = '', 
+                                    value = 1.337, )
+                                ], 
+                            metrics_meta = {
+                                'key' : ''
+                                }, 
+                            actual_output_meta = [
+                                eval_studio_client.api.models.actual_output_metadata.Actual output metadata(
+                                    tokenization = '', 
+                                    data = [
+                                        eval_studio_client.api.models.data_fragment.Data fragment(
+                                            text = '', 
+                                            meta = {
+                                                'key' : ''
+                                                }, )
+                                        ], )
+                                ], 
+                            metric_scores = [
+                                eval_studio_client.api.models.comparison_metric_score_information_(specific_to_comparison_reports).Comparison metric score information (specific to comparison reports)(
+                                    metric_name = '', 
+                                    metric_score = 1.337, )
+                                ], 
+                            result_error_message = '', ), 
+                        baseline_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                            sentences = [
+                                ''
+                                ], 
+                            sentences_count = 56, 
+                            common_sentences = [
+                                ''
+                                ], 
+                            common_count = 56, 
+                            unique_sentences = [
+                                ''
+                                ], 
+                            unique_count = 56, 
+                            identical = True, 
+                            sentence_similarity = {
+                                'key' : 1.337
+                                }, ), 
+                        baseline_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                            chunks = [
+                                ''
+                                ], 
+                            chunks_count = 56, 
+                            common_chunks = [
+                                ''
+                                ], 
+                            common_count = 56, 
+                            unique_chunks = [
+                                ''
+                                ], 
+                            unique_count = 56, 
+                            identical = True, 
+                            chunk_similarity = {
+                                'key' : 1.337
+                                }, ), 
+                        current_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                            key = '', 
+                            input = '', 
+                            expected_output = '', 
+                            output_condition = '', 
+                            actual_output = '', 
+                            actual_duration = 1.337, 
+                            cost = 1.337, 
+                            model_key = '', 
+                            test_key = '', 
+                            test_case_key = '', 
+                            result_error_message = '', ), 
+                        current_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                            sentences_count = 56, 
+                            common_count = 56, 
+                            unique_count = 56, 
+                            identical = True, ), 
+                        current_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                            chunks_count = 56, 
+                            common_count = 56, 
+                            unique_count = 56, 
+                            identical = True, ), )
+                    ],
+                summary = eval_studio_client.api.models.comparison_summary.Comparison summary(
+                    recommendation_winner = '', 
+                    recommendation = '', 
+                    recommendation_confidence = '', ),
+                models_overview = eval_studio_client.api.models.models_overview.Models overview(
+                    baseline_model_key = '', 
+                    current_model_key = '', 
+                    baseline_model_name = '', 
+                    baseline_collection_id = [
+                        ''
+                        ], 
+                    current_model_name = '', 
+                    current_collection_id = [
+                        ''
+                        ], ),
+                models_comparisons = eval_studio_client.api.models.models_comparison_statistics.Models comparison statistics(
+                    test_case_ranks_baseline = 56, 
+                    test_case_ranks_current = 56, 
+                    test_case_wins_baseline = 56, 
+                    test_case_wins_current = 56, ),
+                models_comparisons_metrics = eval_studio_client.api.models.detailed_metrics_comparisons.Detailed metrics comparisons(
+                    metrics_ranks_baseline = 1.337, 
+                    metrics_ranks_current = 1.337, 
+                    metrics_wins_baseline = 56, 
+                    metrics_wins_current = 56, 
+                    metrics_averages = [
+                        eval_studio_client.api.models.metric_average_comparison.Metric average comparison(
+                            metric_key = '', 
+                            baseline_avg = 1.337, 
+                            current_avg = 1.337, 
+                            diff = 1.337, 
+                            baseline_better_wins = 56, 
+                            current_better_wins = 56, 
+                            baseline_rank_avg = 1.337, 
+                            current_rank_avg = 1.337, )
+                        ], ),
+                technical_metrics = eval_studio_client.api.models.technical_metrics_for_model_performance.Technical metrics for model performance(
+                    baseline = eval_studio_client.api.models.technical_metrics_detail.Technical metrics detail(
+                        cost_sum = 1.337, 
+                        duration_sum = 1.337, 
+                        duration_min = 1.337, 
+                        duration_max = 1.337, 
+                        duration_avg = 1.337, ), 
+                    current = eval_studio_client.api.models.technical_metrics_detail.Technical metrics detail(
+                        cost_sum = 1.337, 
+                        duration_sum = 1.337, 
+                        duration_min = 1.337, 
+                        duration_max = 1.337, 
+                        duration_avg = 1.337, ), ),
+                test_cases_leaderboard = [
+                    eval_studio_client.api.models.test_case_leaderboard_item.Test case leaderboard item(
+                        wins = 56, 
+                        question = '', 
+                        changed_metrics_count = 56, )
+                    ]
+            )
+        else:
+            return V1DiffItem(
+        )
+        """
+
+    def testV1DiffItem(self):
+        """Test V1DiffItem"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_evaluation_type.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_evaluation_type import V1EvaluationType
+
+class TestV1EvaluationType(unittest.TestCase):
+    """V1EvaluationType unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testV1EvaluationType(self):
+        """Test V1EvaluationType"""
+        # inst = V1EvaluationType()
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_finalize_operation_response.py

@@ -55,7 +55,8 @@ class TestV1FinalizeOperationResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1FinalizeOperationResponse(

eval_studio_client/api/test/test_v1_flipped_metric.py

@@ -0,0 +1,53 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_flipped_metric import V1FlippedMetric
+
+class TestV1FlippedMetric(unittest.TestCase):
+    """V1FlippedMetric unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1FlippedMetric:
+        """Test V1FlippedMetric
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1FlippedMetric`
+        """
+        model = V1FlippedMetric()
+        if include_optional:
+            return V1FlippedMetric(
+                metric_name = '',
+                baseline_value = 1.337,
+                current_value = 1.337
+            )
+        else:
+            return V1FlippedMetric(
+        )
+        """
+
+    def testV1FlippedMetric(self):
+        """Test V1FlippedMetric"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_generate_test_cases_response.py

@@ -55,7 +55,8 @@ class TestV1GenerateTestCasesResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1GenerateTestCasesResponse(

eval_studio_client/api/test/test_v1_get_dashboard_report_response.py

@@ -0,0 +1,143 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_get_dashboard_report_response import V1GetDashboardReportResponse
+
+class TestV1GetDashboardReportResponse(unittest.TestCase):
+    """V1GetDashboardReportResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1GetDashboardReportResponse:
+        """Test V1GetDashboardReportResponse
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1GetDashboardReportResponse`
+        """
+        model = V1GetDashboardReportResponse()
+        if include_optional:
+            return V1GetDashboardReportResponse(
+                dashboard_report = eval_studio_client.api.models.v1_dashboard_report.v1DashboardReport(
+                    results = [
+                        eval_studio_client.api.models.v1_dashboard_report_result.v1DashboardReportResult(
+                            key = '', 
+                            input = '', 
+                            expected_output = '', 
+                            actual_output = '', 
+                            model_key = '', 
+                            test_case_key = '', 
+                            metrics = {
+                                'key' : eval_studio_client.api.models.v1_metric_scores.v1MetricScores(
+                                    scores = [
+                                        eval_studio_client.api.models.v1_metric_score.v1MetricScore(
+                                            key = '', 
+                                            value = 1.337, )
+                                        ], )
+                                }, 
+                            result_error_map = {
+                                'key' : ''
+                                }, 
+                            human_decision = 'HUMAN_DECISION_UNSPECIFIED', 
+                            comment = '', 
+                            annotations = {
+                                'key' : None
+                                }, )
+                        ], 
+                    evaluator = [
+                        eval_studio_client.api.models.v1_leaderboard_report_evaluator.v1LeaderboardReportEvaluator(
+                            id = '', 
+                            name = '', 
+                            display_name = '', 
+                            tagline = '', 
+                            description = '', 
+                            brief_description = '', 
+                            model_types = [
+                                ''
+                                ], 
+                            can_explain = [
+                                ''
+                                ], 
+                            explanation_scopes = [
+                                ''
+                                ], 
+                            explanations = [
+                                eval_studio_client.api.models.v1_leaderboard_report_explanation.v1LeaderboardReportExplanation(
+                                    explanation_type = '', 
+                                    name = '', 
+                                    category = '', 
+                                    scope = '', 
+                                    has_local = '', 
+                                    formats = [
+                                        ''
+                                        ], )
+                                ], 
+                            parameters = [
+                                eval_studio_client.api.models.v1_leaderboard_report_evaluator_parameter.v1LeaderboardReportEvaluatorParameter(
+                                    name = '', 
+                                    description = '', 
+                                    comment = '', 
+                                    type = '', 
+                                    predefined = [
+                                        None
+                                        ], 
+                                    tags = [
+                                        ''
+                                        ], 
+                                    min = 1.337, 
+                                    max = 1.337, 
+                                    category = '', )
+                                ], 
+                            keywords = [
+                                ''
+                                ], 
+                            metrics_meta = [
+                                eval_studio_client.api.models.v1_leaderboard_report_metrics_meta_entry.v1LeaderboardReportMetricsMetaEntry(
+                                    key = '', 
+                                    display_name = '', 
+                                    data_type = '', 
+                                    display_value = '', 
+                                    description = '', 
+                                    value_range = [
+                                        1.337
+                                        ], 
+                                    value_enum = [
+                                        ''
+                                        ], 
+                                    higher_is_better = True, 
+                                    threshold = 1.337, 
+                                    is_primary_metric = True, 
+                                    parent_metric = '', 
+                                    exclude = True, )
+                                ], )
+                        ], )
+            )
+        else:
+            return V1GetDashboardReportResponse(
+        )
+        """
+
+    def testV1GetDashboardReportResponse(self):
+        """Test V1GetDashboardReportResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_get_info_response.py

@@ -51,7 +51,10 @@ class TestV1GetInfoResponse(unittest.TestCase):
                     h2o_cloud_url = '', 
                     public_instance = True, 
                     sharing_enabled = True, 
-                    experimental_features_enabled = True, )
+                    experimental_features_enabled = True, 
+                    model_type_allowlist = [
+                        ''
+                        ], )
             )
         else:
             return V1GetInfoResponse(

eval_studio_client/api/test/test_v1_get_leaderboard_report_response.py

@@ -77,8 +77,45 @@ class TestV1GetLeaderboardReportResponse(unittest.TestCase):
                                     data = [
                                         eval_studio_client.api.models.v1_leaderboard_report_actual_output_data.v1LeaderboardReportActualOutputData(
                                             text = '', )
-                                        ], )
-                                ], )
+                                        ], 
+                                    agent_chat_activity_diagram = eval_studio_client.api.models.v1_agent_chat_activity_diagram.v1AgentChatActivityDiagram(
+                                        rows = [
+                                            eval_studio_client.api.models.v1_agent_chat_activity_diagram_row.v1AgentChatActivityDiagramRow(
+                                                nodes = [
+                                                    eval_studio_client.api.models.v1_agent_chat_activity_diagram_node.v1AgentChatActivityDiagramNode(
+                                                        id = '', 
+                                                        role = '', 
+                                                        label = '', )
+                                                    ], )
+                                            ], 
+                                        edges = [
+                                            eval_studio_client.api.models.v1_agent_chat_activity_diagram_edge.v1AgentChatActivityDiagramEdge(
+                                                from = '', 
+                                                to = '', 
+                                                label = '', )
+                                            ], ), 
+                                    agent_chat_tools_bar_chart = eval_studio_client.api.models.v1_agent_chat_tools_bar_chart.v1AgentChatToolsBarChart(
+                                        tools = {
+                                            'key' : eval_studio_client.api.models.v1_agent_chat_tool_usage.v1AgentChatToolUsage(
+                                                name = '', 
+                                                success_count = 56, 
+                                                failure_count = 56, 
+                                                total_count = 56, )
+                                            }, ), 
+                                    agent_chat_scripts_bar_chart = eval_studio_client.api.models.v1_agent_chat_scripts_bar_chart.v1AgentChatScriptsBarChart(
+                                        scripts = {
+                                            'key' : eval_studio_client.api.models.v1_agent_chat_script_usage.v1AgentChatScriptUsage(
+                                                name = '', 
+                                                success_count = 56, 
+                                                failure_count = 56, 
+                                                total_count = 56, )
+                                            }, ), )
+                                ], 
+                            human_decision = 'HUMAN_DECISION_UNSPECIFIED', 
+                            comment = '', 
+                            annotations = {
+                                'key' : None
+                                }, )
                         ], 
                     models = [
                         eval_studio_client.api.models.v1_leaderboard_report_model.v1LeaderboardReportModel(

eval_studio_client/api/test/test_v1_get_leaderboard_response.py

@@ -98,7 +98,8 @@ class TestV1GetLeaderboardResponse(unittest.TestCase):
                     h2ogpte_collection = '', 
                     type = 'LEADERBOARD_TYPE_UNSPECIFIED', 
                     demo = True, 
-                    test_lab = '', )
+                    test_lab = '', 
+                    evaluation_type = 'EVALUATION_TYPE_UNSPECIFIED', )
             )
         else:
             return V1GetLeaderboardResponse(

eval_studio_client/api/test/test_v1_get_operation_response.py

@@ -55,7 +55,8 @@ class TestV1GetOperationResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1GetOperationResponse(

eval_studio_client/api/test/test_v1_get_stats_response.py

@@ -38,7 +38,9 @@ class TestV1GetStatsResponse(unittest.TestCase):
                 stats = eval_studio_client.api.models.v1_stats.v1Stats(
                     topic_modeling_pending_jobs = '', 
                     test_validation_pending_jobs = '', 
-                    failure_clustering_pending_jobs = '', )
+                    failure_clustering_pending_jobs = '', 
+                    test_case_import_pending_jobs = '', 
+                    evaluation_worker_queue_length = '', )
             )
         else:
             return V1GetStatsResponse(

eval_studio_client/api/test/test_v1_human_decision.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_human_decision import V1HumanDecision
+
+class TestV1HumanDecision(unittest.TestCase):
+    """V1HumanDecision unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testV1HumanDecision(self):
+        """Test V1HumanDecision"""
+        # inst = V1HumanDecision()
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_import_leaderboard_response.py

@@ -55,7 +55,8 @@ class TestV1ImportLeaderboardResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1ImportLeaderboardResponse(

eval_studio_client/api/test/test_v1_import_test_cases_from_library_response.py

@@ -55,7 +55,8 @@ class TestV1ImportTestCasesFromLibraryResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1ImportTestCasesFromLibraryResponse(

eval_studio_client/api/test/test_v1_info.py

@@ -50,7 +50,10 @@ class TestV1Info(unittest.TestCase):
                 h2o_cloud_url = '',
                 public_instance = True,
                 sharing_enabled = True,
-                experimental_features_enabled = True
+                experimental_features_enabled = True,
+                model_type_allowlist = [
+                    ''
+                    ]
             )
         else:
             return V1Info(

eval_studio_client/api/test/test_v1_leaderboard.py

@@ -100,7 +100,8 @@ class TestV1Leaderboard(unittest.TestCase):
                 h2ogpte_collection = '',
                 type = 'LEADERBOARD_TYPE_UNSPECIFIED',
                 demo = True,
-                test_lab = ''
+                test_lab = '',
+                evaluation_type = 'EVALUATION_TYPE_UNSPECIFIED'
             )
         else:
             return V1Leaderboard(