eval-studio-client 1.2.4a2__py3-none-any.whl → 1.3.0__py3-none-any.whl

eval_studio_client/api/test/test_v1_comparison_result.py

@@ -0,0 +1,258 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_comparison_result import V1ComparisonResult
+
+class TestV1ComparisonResult(unittest.TestCase):
+    """V1ComparisonResult unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1ComparisonResult:
+        """Test V1ComparisonResult
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1ComparisonResult`
+        """
+        model = V1ComparisonResult()
+        if include_optional:
+            return V1ComparisonResult(
+                diffs = [
+                    eval_studio_client.api.models.a_single_diff_item_comparing_two_leaderboards.A single diff item comparing two leaderboards(
+                        diff_key = '', 
+                        items = [
+                            eval_studio_client.api.models.a_single_comparison_item_showing_differences_between_baseline_and_current.A single comparison item showing differences between baseline and current(
+                                question = '', 
+                                diff_flipped_metrics = [
+                                    eval_studio_client.api.models.flipped_metric_information.Flipped metric information(
+                                        metric_name = '', 
+                                        baseline_value = 1.337, 
+                                        current_value = 1.337, )
+                                    ], 
+                                baseline_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                                    key = '', 
+                                    input = '', 
+                                    corpus = [
+                                        ''
+                                        ], 
+                                    context = [
+                                        ''
+                                        ], 
+                                    categories = [
+                                        ''
+                                        ], 
+                                    relationships = [
+                                        eval_studio_client.api.models.test_case_relationship_information.Test case relationship information(
+                                            type = '', 
+                                            target = '', 
+                                            target_type = '', )
+                                        ], 
+                                    expected_output = '', 
+                                    output_constraints = [
+                                        ''
+                                        ], 
+                                    output_condition = '', 
+                                    actual_output = '', 
+                                    actual_duration = 1.337, 
+                                    cost = 1.337, 
+                                    model_key = '', 
+                                    test_key = '', 
+                                    test_case_key = '', 
+                                    metrics = [
+                                        eval_studio_client.api.models.metric_information.Metric information(
+                                            key = '', 
+                                            value = 1.337, )
+                                        ], 
+                                    metrics_meta = {
+                                        'key' : ''
+                                        }, 
+                                    actual_output_meta = [
+                                        eval_studio_client.api.models.actual_output_metadata.Actual output metadata(
+                                            tokenization = '', 
+                                            data = [
+                                                eval_studio_client.api.models.data_fragment.Data fragment(
+                                                    text = '', 
+                                                    meta = {
+                                                        'key' : ''
+                                                        }, )
+                                                ], )
+                                        ], 
+                                    metric_scores = [
+                                        eval_studio_client.api.models.comparison_metric_score_information_(specific_to_comparison_reports).Comparison metric score information (specific to comparison reports)(
+                                            metric_name = '', 
+                                            metric_score = 1.337, )
+                                        ], 
+                                    result_error_message = '', ), 
+                                baseline_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                                    sentences = [
+                                        ''
+                                        ], 
+                                    sentences_count = 56, 
+                                    common_sentences = [
+                                        ''
+                                        ], 
+                                    common_count = 56, 
+                                    unique_sentences = [
+                                        ''
+                                        ], 
+                                    unique_count = 56, 
+                                    identical = True, 
+                                    sentence_similarity = {
+                                        'key' : 1.337
+                                        }, ), 
+                                baseline_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                                    chunks = [
+                                        ''
+                                        ], 
+                                    chunks_count = 56, 
+                                    common_chunks = [
+                                        ''
+                                        ], 
+                                    common_count = 56, 
+                                    unique_chunks = [
+                                        ''
+                                        ], 
+                                    unique_count = 56, 
+                                    identical = True, 
+                                    chunk_similarity = {
+                                        'key' : 1.337
+                                        }, ), 
+                                current_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                                    key = '', 
+                                    input = '', 
+                                    expected_output = '', 
+                                    output_condition = '', 
+                                    actual_output = '', 
+                                    actual_duration = 1.337, 
+                                    cost = 1.337, 
+                                    model_key = '', 
+                                    test_key = '', 
+                                    test_case_key = '', 
+                                    result_error_message = '', ), 
+                                current_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                                    sentences_count = 56, 
+                                    common_count = 56, 
+                                    unique_count = 56, 
+                                    identical = True, ), 
+                                current_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                                    chunks_count = 56, 
+                                    common_count = 56, 
+                                    unique_count = 56, 
+                                    identical = True, ), )
+                            ], 
+                        summary = eval_studio_client.api.models.comparison_summary.Comparison summary(
+                            recommendation_winner = '', 
+                            recommendation = '', 
+                            recommendation_confidence = '', ), 
+                        models_overview = eval_studio_client.api.models.models_overview.Models overview(
+                            baseline_model_key = '', 
+                            current_model_key = '', 
+                            baseline_model_name = '', 
+                            baseline_collection_id = [
+                                ''
+                                ], 
+                            current_model_name = '', 
+                            current_collection_id = [
+                                ''
+                                ], ), 
+                        models_comparisons = eval_studio_client.api.models.models_comparison_statistics.Models comparison statistics(
+                            test_case_ranks_baseline = 56, 
+                            test_case_ranks_current = 56, 
+                            test_case_wins_baseline = 56, 
+                            test_case_wins_current = 56, ), 
+                        models_comparisons_metrics = eval_studio_client.api.models.detailed_metrics_comparisons.Detailed metrics comparisons(
+                            metrics_ranks_baseline = 1.337, 
+                            metrics_ranks_current = 1.337, 
+                            metrics_wins_baseline = 56, 
+                            metrics_wins_current = 56, 
+                            metrics_averages = [
+                                eval_studio_client.api.models.metric_average_comparison.Metric average comparison(
+                                    metric_key = '', 
+                                    baseline_avg = 1.337, 
+                                    current_avg = 1.337, 
+                                    diff = 1.337, 
+                                    baseline_better_wins = 56, 
+                                    current_better_wins = 56, 
+                                    baseline_rank_avg = 1.337, 
+                                    current_rank_avg = 1.337, )
+                                ], ), 
+                        technical_metrics = eval_studio_client.api.models.technical_metrics_for_model_performance.Technical metrics for model performance(
+                            baseline = eval_studio_client.api.models.technical_metrics_detail.Technical metrics detail(
+                                cost_sum = 1.337, 
+                                duration_sum = 1.337, 
+                                duration_min = 1.337, 
+                                duration_max = 1.337, 
+                                duration_avg = 1.337, ), 
+                            current = eval_studio_client.api.models.technical_metrics_detail.Technical metrics detail(
+                                cost_sum = 1.337, 
+                                duration_sum = 1.337, 
+                                duration_min = 1.337, 
+                                duration_max = 1.337, 
+                                duration_avg = 1.337, ), ), 
+                        test_cases_leaderboard = [
+                            eval_studio_client.api.models.test_case_leaderboard_item.Test case leaderboard item(
+                                wins = 56, 
+                                question = '', 
+                                changed_metrics_count = 56, )
+                            ], )
+                    ],
+                leaderboards = [
+                    eval_studio_client.api.models.leaderboard_information.Leaderboard information(
+                        key = '', 
+                        items = [
+                            eval_studio_client.api.models.leaderboard_comparison_item.Leaderboard comparison item(
+                                wins = 56, 
+                                question = '', 
+                                changed_metrics_count = 56, )
+                            ], )
+                    ],
+                metrics_meta = {
+                    'key' : eval_studio_client.api.models.metric_metadata.Metric metadata(
+                        key = '', 
+                        display_name = '', 
+                        data_type = '', 
+                        display_value = '', 
+                        description = '', 
+                        value_range = [
+                            1.337
+                            ], 
+                        value_enum = [
+                            ''
+                            ], 
+                        higher_is_better = True, 
+                        threshold = 1.337, 
+                        is_primary_metric = True, 
+                        parent_metric = '', 
+                        exclude = True, )
+                    }
+            )
+        else:
+            return V1ComparisonResult(
+        )
+        """
+
+    def testV1ComparisonResult(self):
+        """Test V1ComparisonResult"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_comparison_summary.py

@@ -0,0 +1,53 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_comparison_summary import V1ComparisonSummary
+
+class TestV1ComparisonSummary(unittest.TestCase):
+    """V1ComparisonSummary unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1ComparisonSummary:
+        """Test V1ComparisonSummary
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1ComparisonSummary`
+        """
+        model = V1ComparisonSummary()
+        if include_optional:
+            return V1ComparisonSummary(
+                recommendation_winner = '',
+                recommendation = '',
+                recommendation_confidence = ''
+            )
+        else:
+            return V1ComparisonSummary(
+        )
+        """
+
+    def testV1ComparisonSummary(self):
+        """Test V1ComparisonSummary"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_create_evaluation_request.py

@@ -113,7 +113,8 @@ class TestV1CreateEvaluationRequest(unittest.TestCase):
                     api_key = '', 
                     type = 'MODEL_TYPE_UNSPECIFIED', 
                     parameters = '', 
-                    demo = True, )
+                    demo = True, ),
+                evaluation_type = 'EVALUATION_TYPE_UNSPECIFIED'
             )
         else:
             return V1CreateEvaluationRequest(

eval_studio_client/api/test/test_v1_create_leaderboard_request.py

@@ -98,7 +98,8 @@ class TestV1CreateLeaderboardRequest(unittest.TestCase):
                     h2ogpte_collection = '', 
                     type = 'LEADERBOARD_TYPE_UNSPECIFIED', 
                     demo = True, 
-                    test_lab = '', )
+                    test_lab = '', 
+                    evaluation_type = 'EVALUATION_TYPE_UNSPECIFIED', )
             )
         else:
             return V1CreateLeaderboardRequest(

eval_studio_client/api/test/test_v1_create_leaderboard_response.py

@@ -55,7 +55,8 @@ class TestV1CreateLeaderboardResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1CreateLeaderboardResponse(

eval_studio_client/api/test/test_v1_create_leaderboard_without_cache_response.py

@@ -55,7 +55,8 @@ class TestV1CreateLeaderboardWithoutCacheResponse(unittest.TestCase):
                                 'key' : None
                                 }
                             ], ), 
-                    response = , )
+                    response = , 
+                    seen_by_creator_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), )
             )
         else:
             return V1CreateLeaderboardWithoutCacheResponse(

eval_studio_client/api/test/test_v1_create_test_from_test_cases_request.py

@@ -0,0 +1,54 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_create_test_from_test_cases_request import V1CreateTestFromTestCasesRequest
+
+class TestV1CreateTestFromTestCasesRequest(unittest.TestCase):
+    """V1CreateTestFromTestCasesRequest unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1CreateTestFromTestCasesRequest:
+        """Test V1CreateTestFromTestCasesRequest
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1CreateTestFromTestCasesRequest`
+        """
+        model = V1CreateTestFromTestCasesRequest()
+        if include_optional:
+            return V1CreateTestFromTestCasesRequest(
+                tests_json = '',
+                url = '',
+                test_display_name = '',
+                test_description = ''
+            )
+        else:
+            return V1CreateTestFromTestCasesRequest(
+        )
+        """
+
+    def testV1CreateTestFromTestCasesRequest(self):
+        """Test V1CreateTestFromTestCasesRequest"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_create_test_from_test_cases_response.py

@@ -0,0 +1,68 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_create_test_from_test_cases_response import V1CreateTestFromTestCasesResponse
+
+class TestV1CreateTestFromTestCasesResponse(unittest.TestCase):
+    """V1CreateTestFromTestCasesResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1CreateTestFromTestCasesResponse:
+        """Test V1CreateTestFromTestCasesResponse
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1CreateTestFromTestCasesResponse`
+        """
+        model = V1CreateTestFromTestCasesResponse()
+        if include_optional:
+            return V1CreateTestFromTestCasesResponse(
+                test = eval_studio_client.api.models.v1_test.v1Test(
+                    name = '', 
+                    create_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    creator = '', 
+                    update_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    updater = '', 
+                    delete_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
+                    deleter = '', 
+                    display_name = '', 
+                    description = '', 
+                    documents = [
+                        ''
+                        ], 
+                    tags = [
+                        ''
+                        ], 
+                    demo = True, 
+                    type = 'TEST_TYPE_UNSPECIFIED', )
+            )
+        else:
+            return V1CreateTestFromTestCasesResponse(
+        )
+        """
+
+    def testV1CreateTestFromTestCasesResponse(self):
+        """Test V1CreateTestFromTestCasesResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_dashboard_report.py

@@ -0,0 +1,142 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_dashboard_report import V1DashboardReport
+
+class TestV1DashboardReport(unittest.TestCase):
+    """V1DashboardReport unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1DashboardReport:
+        """Test V1DashboardReport
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1DashboardReport`
+        """
+        model = V1DashboardReport()
+        if include_optional:
+            return V1DashboardReport(
+                results = [
+                    eval_studio_client.api.models.v1_dashboard_report_result.v1DashboardReportResult(
+                        key = '', 
+                        input = '', 
+                        expected_output = '', 
+                        actual_output = '', 
+                        model_key = '', 
+                        test_case_key = '', 
+                        metrics = {
+                            'key' : eval_studio_client.api.models.v1_metric_scores.v1MetricScores(
+                                scores = [
+                                    eval_studio_client.api.models.v1_metric_score.v1MetricScore(
+                                        key = '', 
+                                        value = 1.337, )
+                                    ], )
+                            }, 
+                        result_error_map = {
+                            'key' : ''
+                            }, 
+                        human_decision = 'HUMAN_DECISION_UNSPECIFIED', 
+                        comment = '', 
+                        annotations = {
+                            'key' : None
+                            }, )
+                    ],
+                evaluator = [
+                    eval_studio_client.api.models.v1_leaderboard_report_evaluator.v1LeaderboardReportEvaluator(
+                        id = '', 
+                        name = '', 
+                        display_name = '', 
+                        tagline = '', 
+                        description = '', 
+                        brief_description = '', 
+                        model_types = [
+                            ''
+                            ], 
+                        can_explain = [
+                            ''
+                            ], 
+                        explanation_scopes = [
+                            ''
+                            ], 
+                        explanations = [
+                            eval_studio_client.api.models.v1_leaderboard_report_explanation.v1LeaderboardReportExplanation(
+                                explanation_type = '', 
+                                name = '', 
+                                category = '', 
+                                scope = '', 
+                                has_local = '', 
+                                formats = [
+                                    ''
+                                    ], )
+                            ], 
+                        parameters = [
+                            eval_studio_client.api.models.v1_leaderboard_report_evaluator_parameter.v1LeaderboardReportEvaluatorParameter(
+                                name = '', 
+                                description = '', 
+                                comment = '', 
+                                type = '', 
+                                predefined = [
+                                    None
+                                    ], 
+                                tags = [
+                                    ''
+                                    ], 
+                                min = 1.337, 
+                                max = 1.337, 
+                                category = '', )
+                            ], 
+                        keywords = [
+                            ''
+                            ], 
+                        metrics_meta = [
+                            eval_studio_client.api.models.v1_leaderboard_report_metrics_meta_entry.v1LeaderboardReportMetricsMetaEntry(
+                                key = '', 
+                                display_name = '', 
+                                data_type = '', 
+                                display_value = '', 
+                                description = '', 
+                                value_range = [
+                                    1.337
+                                    ], 
+                                value_enum = [
+                                    ''
+                                    ], 
+                                higher_is_better = True, 
+                                threshold = 1.337, 
+                                is_primary_metric = True, 
+                                parent_metric = '', 
+                                exclude = True, )
+                            ], )
+                    ]
+            )
+        else:
+            return V1DashboardReport(
+        )
+        """
+
+    def testV1DashboardReport(self):
+        """Test V1DashboardReport"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()