eval-studio-client 1.2.5__py3-none-any.whl → 1.3.0a1__py3-none-any.whl

eval_studio_client/api/test/test_v1_cmp_leaderboard_reports_response.py

@@ -0,0 +1,255 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_cmp_leaderboard_reports_response import V1CmpLeaderboardReportsResponse
+
+class TestV1CmpLeaderboardReportsResponse(unittest.TestCase):
+    """V1CmpLeaderboardReportsResponse unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1CmpLeaderboardReportsResponse:
+        """Test V1CmpLeaderboardReportsResponse
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1CmpLeaderboardReportsResponse`
+        """
+        model = V1CmpLeaderboardReportsResponse()
+        if include_optional:
+            return V1CmpLeaderboardReportsResponse(
+                leaderboard_cmp_report = eval_studio_client.api.models.v1_leaderboard_cmp_report.v1LeaderboardCmpReport(
+                    summary = '', 
+                    comparison_result = eval_studio_client.api.models.complete_comparison_result_structure.Complete comparison result structure(
+                        diffs = [
+                            eval_studio_client.api.models.a_single_diff_item_comparing_two_leaderboards.A single diff item comparing two leaderboards(
+                                diff_key = '', 
+                                items = [
+                                    eval_studio_client.api.models.a_single_comparison_item_showing_differences_between_baseline_and_current.A single comparison item showing differences between baseline and current(
+                                        question = '', 
+                                        diff_flipped_metrics = [
+                                            eval_studio_client.api.models.flipped_metric_information.Flipped metric information(
+                                                metric_name = '', 
+                                                baseline_value = 1.337, 
+                                                current_value = 1.337, )
+                                            ], 
+                                        baseline_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                                            key = '', 
+                                            input = '', 
+                                            corpus = [
+                                                ''
+                                                ], 
+                                            context = [
+                                                ''
+                                                ], 
+                                            categories = [
+                                                ''
+                                                ], 
+                                            relationships = [
+                                                eval_studio_client.api.models.test_case_relationship_information.Test case relationship information(
+                                                    type = '', 
+                                                    target = '', 
+                                                    target_type = '', )
+                                                ], 
+                                            expected_output = '', 
+                                            output_constraints = [
+                                                ''
+                                                ], 
+                                            output_condition = '', 
+                                            actual_output = '', 
+                                            actual_duration = 1.337, 
+                                            cost = 1.337, 
+                                            model_key = '', 
+                                            test_key = '', 
+                                            test_case_key = '', 
+                                            metrics = [
+                                                eval_studio_client.api.models.metric_information.Metric information(
+                                                    key = '', 
+                                                    value = 1.337, )
+                                                ], 
+                                            metrics_meta = {
+                                                'key' : ''
+                                                }, 
+                                            actual_output_meta = [
+                                                eval_studio_client.api.models.actual_output_metadata.Actual output metadata(
+                                                    tokenization = '', 
+                                                    data = [
+                                                        eval_studio_client.api.models.data_fragment.Data fragment(
+                                                            text = '', 
+                                                            meta = {
+                                                                'key' : ''
+                                                                }, )
+                                                        ], )
+                                                ], 
+                                            metric_scores = [
+                                                eval_studio_client.api.models.comparison_metric_score_information_(specific_to_comparison_reports).Comparison metric score information (specific to comparison reports)(
+                                                    metric_name = '', 
+                                                    metric_score = 1.337, )
+                                                ], 
+                                            result_error_message = '', ), 
+                                        baseline_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                                            sentences = [
+                                                ''
+                                                ], 
+                                            sentences_count = 56, 
+                                            common_sentences = [
+                                                ''
+                                                ], 
+                                            common_count = 56, 
+                                            unique_sentences = [
+                                                ''
+                                                ], 
+                                            unique_count = 56, 
+                                            identical = True, 
+                                            sentence_similarity = {
+                                                'key' : 1.337
+                                                }, ), 
+                                        baseline_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                                            chunks = [
+                                                ''
+                                                ], 
+                                            chunks_count = 56, 
+                                            common_chunks = [
+                                                ''
+                                                ], 
+                                            common_count = 56, 
+                                            unique_chunks = [
+                                                ''
+                                                ], 
+                                            unique_count = 56, 
+                                            identical = True, 
+                                            chunk_similarity = {
+                                                'key' : 1.337
+                                                }, ), 
+                                        current_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                                            key = '', 
+                                            input = '', 
+                                            expected_output = '', 
+                                            output_condition = '', 
+                                            actual_output = '', 
+                                            actual_duration = 1.337, 
+                                            cost = 1.337, 
+                                            model_key = '', 
+                                            test_key = '', 
+                                            test_case_key = '', 
+                                            result_error_message = '', ), 
+                                        current_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                                            sentences_count = 56, 
+                                            common_count = 56, 
+                                            unique_count = 56, 
+                                            identical = True, ), 
+                                        current_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                                            chunks_count = 56, 
+                                            common_count = 56, 
+                                            unique_count = 56, 
+                                            identical = True, ), )
+                                    ], 
+                                summary = eval_studio_client.api.models.comparison_summary.Comparison summary(
+                                    recommendation_winner = '', 
+                                    recommendation = '', 
+                                    recommendation_confidence = '', ), 
+                                models_overview = eval_studio_client.api.models.models_overview.Models overview(
+                                    baseline_model_key = '', 
+                                    current_model_key = '', 
+                                    baseline_model_name = '', 
+                                    baseline_collection_id = [
+                                        ''
+                                        ], 
+                                    current_model_name = '', 
+                                    current_collection_id = [
+                                        ''
+                                        ], ), 
+                                models_comparisons = eval_studio_client.api.models.models_comparison_statistics.Models comparison statistics(
+                                    test_case_ranks_baseline = 56, 
+                                    test_case_ranks_current = 56, 
+                                    test_case_wins_baseline = 56, 
+                                    test_case_wins_current = 56, ), 
+                                models_comparisons_metrics = eval_studio_client.api.models.detailed_metrics_comparisons.Detailed metrics comparisons(
+                                    metrics_ranks_baseline = 1.337, 
+                                    metrics_ranks_current = 1.337, 
+                                    metrics_wins_baseline = 56, 
+                                    metrics_wins_current = 56, 
+                                    metrics_averages = [
+                                        eval_studio_client.api.models.metric_average_comparison.Metric average comparison(
+                                            metric_key = '', 
+                                            baseline_avg = 1.337, 
+                                            current_avg = 1.337, 
+                                            diff = 1.337, 
+                                            baseline_better_wins = 56, 
+                                            current_better_wins = 56, 
+                                            baseline_rank_avg = 1.337, 
+                                            current_rank_avg = 1.337, )
+                                        ], ), 
+                                technical_metrics = eval_studio_client.api.models.technical_metrics_for_model_performance.Technical metrics for model performance(
+                                    baseline = eval_studio_client.api.models.technical_metrics_detail.Technical metrics detail(
+                                        cost_sum = 1.337, 
+                                        duration_sum = 1.337, 
+                                        duration_min = 1.337, 
+                                        duration_max = 1.337, 
+                                        duration_avg = 1.337, ), 
+                                    current = eval_studio_client.api.models.technical_metrics_detail.Technical metrics detail(
+                                        cost_sum = 1.337, 
+                                        duration_sum = 1.337, 
+                                        duration_min = 1.337, 
+                                        duration_max = 1.337, 
+                                        duration_avg = 1.337, ), ), 
+                                test_cases_leaderboard = [
+                                    eval_studio_client.api.models.test_case_leaderboard_item.Test case leaderboard item(
+                                        wins = 56, 
+                                        question = '', 
+                                        changed_metrics_count = 56, )
+                                    ], )
+                            ], 
+                        leaderboards = [
+                            eval_studio_client.api.models.leaderboard_information.Leaderboard information(
+                                key = '', )
+                            ], 
+                        metrics_meta = {
+                            'key' : eval_studio_client.api.models.metric_metadata.Metric metadata(
+                                key = '', 
+                                display_name = '', 
+                                data_type = '', 
+                                display_value = '', 
+                                description = '', 
+                                value_range = [
+                                    1.337
+                                    ], 
+                                value_enum = [
+                                    ''
+                                    ], 
+                                higher_is_better = True, 
+                                threshold = 1.337, 
+                                is_primary_metric = True, 
+                                parent_metric = '', 
+                                exclude = True, )
+                            }, ), )
+            )
+        else:
+            return V1CmpLeaderboardReportsResponse(
+        )
+        """
+
+    def testV1CmpLeaderboardReportsResponse(self):
+        """Test V1CmpLeaderboardReportsResponse"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_comparison_item.py

@@ -0,0 +1,233 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_comparison_item import V1ComparisonItem
+
+class TestV1ComparisonItem(unittest.TestCase):
+    """V1ComparisonItem unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1ComparisonItem:
+        """Test V1ComparisonItem
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1ComparisonItem`
+        """
+        model = V1ComparisonItem()
+        if include_optional:
+            return V1ComparisonItem(
+                question = '',
+                diff_flipped_metrics = [
+                    eval_studio_client.api.models.flipped_metric_information.Flipped metric information(
+                        metric_name = '', 
+                        baseline_value = 1.337, 
+                        current_value = 1.337, )
+                    ],
+                baseline_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                    key = '', 
+                    input = '', 
+                    corpus = [
+                        ''
+                        ], 
+                    context = [
+                        ''
+                        ], 
+                    categories = [
+                        ''
+                        ], 
+                    relationships = [
+                        eval_studio_client.api.models.test_case_relationship_information.Test case relationship information(
+                            type = '', 
+                            target = '', 
+                            target_type = '', )
+                        ], 
+                    expected_output = '', 
+                    output_constraints = [
+                        ''
+                        ], 
+                    output_condition = '', 
+                    actual_output = '', 
+                    actual_duration = 1.337, 
+                    cost = 1.337, 
+                    model_key = '', 
+                    test_key = '', 
+                    test_case_key = '', 
+                    metrics = [
+                        eval_studio_client.api.models.metric_information.Metric information(
+                            key = '', 
+                            value = 1.337, )
+                        ], 
+                    metrics_meta = {
+                        'key' : ''
+                        }, 
+                    actual_output_meta = [
+                        eval_studio_client.api.models.actual_output_metadata.Actual output metadata(
+                            tokenization = '', 
+                            data = [
+                                eval_studio_client.api.models.data_fragment.Data fragment(
+                                    text = '', 
+                                    meta = {
+                                        'key' : ''
+                                        }, )
+                                ], )
+                        ], 
+                    metric_scores = [
+                        eval_studio_client.api.models.comparison_metric_score_information_(specific_to_comparison_reports).Comparison metric score information (specific to comparison reports)(
+                            metric_name = '', 
+                            metric_score = 1.337, )
+                        ], 
+                    result_error_message = '', ),
+                baseline_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                    sentences = [
+                        ''
+                        ], 
+                    sentences_count = 56, 
+                    common_sentences = [
+                        ''
+                        ], 
+                    common_count = 56, 
+                    unique_sentences = [
+                        ''
+                        ], 
+                    unique_count = 56, 
+                    identical = True, 
+                    sentence_similarity = {
+                        'key' : 1.337
+                        }, ),
+                baseline_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                    chunks = [
+                        ''
+                        ], 
+                    chunks_count = 56, 
+                    common_chunks = [
+                        ''
+                        ], 
+                    common_count = 56, 
+                    unique_chunks = [
+                        ''
+                        ], 
+                    unique_count = 56, 
+                    identical = True, 
+                    chunk_similarity = {
+                        'key' : 1.337
+                        }, ),
+                current_test_case_result = eval_studio_client.api.models.test_case_result.Test case result(
+                    key = '', 
+                    input = '', 
+                    corpus = [
+                        ''
+                        ], 
+                    context = [
+                        ''
+                        ], 
+                    categories = [
+                        ''
+                        ], 
+                    relationships = [
+                        eval_studio_client.api.models.test_case_relationship_information.Test case relationship information(
+                            type = '', 
+                            target = '', 
+                            target_type = '', )
+                        ], 
+                    expected_output = '', 
+                    output_constraints = [
+                        ''
+                        ], 
+                    output_condition = '', 
+                    actual_output = '', 
+                    actual_duration = 1.337, 
+                    cost = 1.337, 
+                    model_key = '', 
+                    test_key = '', 
+                    test_case_key = '', 
+                    metrics = [
+                        eval_studio_client.api.models.metric_information.Metric information(
+                            key = '', 
+                            value = 1.337, )
+                        ], 
+                    metrics_meta = {
+                        'key' : ''
+                        }, 
+                    actual_output_meta = [
+                        eval_studio_client.api.models.actual_output_metadata.Actual output metadata(
+                            tokenization = '', 
+                            data = [
+                                eval_studio_client.api.models.data_fragment.Data fragment(
+                                    text = '', 
+                                    meta = {
+                                        'key' : ''
+                                        }, )
+                                ], )
+                        ], 
+                    metric_scores = [
+                        eval_studio_client.api.models.comparison_metric_score_information_(specific_to_comparison_reports).Comparison metric score information (specific to comparison reports)(
+                            metric_name = '', 
+                            metric_score = 1.337, )
+                        ], 
+                    result_error_message = '', ),
+                current_diff_actual_output_meta = eval_studio_client.api.models.actual_output_metadata_diff.Actual output metadata diff(
+                    sentences = [
+                        ''
+                        ], 
+                    sentences_count = 56, 
+                    common_sentences = [
+                        ''
+                        ], 
+                    common_count = 56, 
+                    unique_sentences = [
+                        ''
+                        ], 
+                    unique_count = 56, 
+                    identical = True, 
+                    sentence_similarity = {
+                        'key' : 1.337
+                        }, ),
+                current_diff_retrieved_context = eval_studio_client.api.models.retrieved_context_diff.Retrieved context diff(
+                    chunks = [
+                        ''
+                        ], 
+                    chunks_count = 56, 
+                    common_chunks = [
+                        ''
+                        ], 
+                    common_count = 56, 
+                    unique_chunks = [
+                        ''
+                        ], 
+                    unique_count = 56, 
+                    identical = True, 
+                    chunk_similarity = {
+                        'key' : 1.337
+                        }, )
+            )
+        else:
+            return V1ComparisonItem(
+        )
+        """
+
+    def testV1ComparisonItem(self):
+        """Test V1ComparisonItem"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()

eval_studio_client/api/test/test_v1_comparison_metric_score.py

@@ -0,0 +1,52 @@
+# coding: utf-8
+
+"""
+    ai/h2o/eval_studio/v1/insight.proto
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+    The version of the OpenAPI document: version not set
+    Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+    Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from eval_studio_client.api.models.v1_comparison_metric_score import V1ComparisonMetricScore
+
+class TestV1ComparisonMetricScore(unittest.TestCase):
+    """V1ComparisonMetricScore unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> V1ComparisonMetricScore:
+        """Test V1ComparisonMetricScore
+            include_option is a boolean, when False only required
+            params are included, when True both required and
+            optional params are included """
+        # uncomment below to create an instance of `V1ComparisonMetricScore`
+        """
+        model = V1ComparisonMetricScore()
+        if include_optional:
+            return V1ComparisonMetricScore(
+                metric_name = '',
+                metric_score = 1.337
+            )
+        else:
+            return V1ComparisonMetricScore(
+        )
+        """
+
+    def testV1ComparisonMetricScore(self):
+        """Test V1ComparisonMetricScore"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+if __name__ == '__main__':
+    unittest.main()