spl2-testing-framework 1.3.0__tar.gz → 1.4.0__tar.gz

{spl2_testing_framework-1.3.0 → spl2_testing_framework-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.3.0
+Version: 1.4.0
 Summary: SPL2 Testing Framework
 Home-page: https://github.com/splunk/spl2-testing-framework
 Keywords: SPL2 testing framework
@@ -131,6 +131,7 @@ It is possible to pass more options to the command, which works also with pytest
 Additionally, the following options are supported:
 
 * `--ignore_empty_strings` - to ignore empty strings in the results
+* `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the

{spl2_testing_framework-1.3.0 → spl2_testing_framework-1.4.0}/README.md

@@ -99,6 +99,7 @@ It is possible to pass more options to the command, which works also with pytest
 Additionally, the following options are supported:
 
 * `--ignore_empty_strings` - to ignore empty strings in the results
+* `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the

{spl2_testing_framework-1.3.0 → spl2_testing_framework-1.4.0}/pyproject.toml

@@ -16,7 +16,7 @@
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.3.0"
+version = "1.4.0"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"

{spl2_testing_framework-1.3.0 → spl2_testing_framework-1.4.0}/spl2_testing_framework/conftest.py

@@ -47,6 +47,7 @@ BOX_TEST = "box_test"
 SINGLE_SPL2_FILE = "single_spl2_file"
 
 IGNORE_EMPTY_STRINGS = "ignore_empty_strings"
+IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL = "ignore_additional_fields_in_actual"
 CREATE_COMPARISON_SHEET = "create_comparison_sheet"
 PERFORMANCE_CHECK = "performance_check"
 TEST_TYPE = "type"
@@ -76,6 +77,7 @@ def pytest_addoption(parser):
     parser.addoption("--sample_delimiter", action="store", default="\n")
     parser.addoption(f"--{CREATE_COMPARISON_SHEET}", action="store_true", default=False)
     parser.addoption(f"--{IGNORE_EMPTY_STRINGS}", action="store_true")
+    parser.addoption(f"--{IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL}", action="store_true")
     parser.addoption(f"--{LIMIT_TESTS}", action="store", type=int, default=None)
 
 
@@ -249,3 +251,7 @@ def bearer_token_authentication(config):
 def _set_results_options(metafunc):
     ignore_empty_strings = metafunc.config.getoption(IGNORE_EMPTY_STRINGS)
     Results._ignore_empty_strings = ignore_empty_strings
+    ignore_fields_in_actual = metafunc.config.getoption(
+        IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL
+    )
+    Results._ignore_additional_fields_in_actual = ignore_fields_in_actual

{spl2_testing_framework-1.3.0 → spl2_testing_framework-1.4.0}/spl2_testing_framework/tools/results.py

@@ -24,6 +24,7 @@ import pytest
 
 class Results(UserDict):
     _ignore_empty_strings = False
+    _ignore_additional_fields_in_actual = False
     """Class responsible for parsing and storing results from Splunk, IP and CLI.
     It's necessary to parse nested jsons and other python types to strings in order to compare them
     """
@@ -121,6 +122,53 @@ class Results(UserDict):
         if self._ignore_empty_strings:
             self.data = self._remove_field_by_value(self.data, "")
 
+    @staticmethod
+    def _remove_extra_fields(actual_data, expected_data):
+        """Remove fields from actual_data that are not present in expected_data.
+        This is useful when actual results may have additional fields that should be ignored in comparison.
+        Works recursively for nested dicts and lists.
+        """
+        if isinstance(expected_data, dict) and isinstance(actual_data, dict):
+            # Get the keys that are in actual but not in expected
+            keys_to_remove = set(actual_data.keys()) - set(expected_data.keys())
+            for key in keys_to_remove:
+                actual_data.pop(key, None)
+
+            # Recursively process nested structures
+            for key in actual_data.keys():
+                if key in expected_data:
+                    if isinstance(actual_data[key], dict) and isinstance(
+                        expected_data[key], dict
+                    ):
+                        Results._remove_extra_fields(
+                            actual_data[key], expected_data[key]
+                        )
+                    elif isinstance(actual_data[key], list) and isinstance(
+                        expected_data[key], list
+                    ):
+                        # For lists, we need to handle them carefully
+                        if len(actual_data[key]) > 0 and len(expected_data[key]) > 0:
+                            if isinstance(actual_data[key][0], dict) and isinstance(
+                                expected_data[key][0], dict
+                            ):
+                                # If both are lists of dicts, process each element
+                                for i in range(
+                                    min(len(actual_data[key]), len(expected_data[key]))
+                                ):
+                                    Results._remove_extra_fields(
+                                        actual_data[key][i], expected_data[key][i]
+                                    )
+
+        return actual_data
+
+    def remove_fields_not_in_expected(self, expected):
+        """Remove fields from this Results object that are not present in the expected Results object.
+        This is the public method to be called when comparing results.
+        """
+        if self._ignore_additional_fields_in_actual and isinstance(expected, Results):
+            self.data = self._remove_extra_fields(self.data, expected.data)
+        return self
+
     def __lt__(self, other):
         if "_raw" in self.data:
             s = self.data.get("_raw")

{spl2_testing_framework-1.3.0 → spl2_testing_framework-1.4.0}/spl2_testing_framework/tools/spl2test_runner.py

@@ -131,6 +131,11 @@ class SPL2TestRunner:
         perf.check_performance(output_result)
 
         if box_test.output:
+            # Remove fields from actual results that are not in expected results if configured
+            for i, actual_result in enumerate(output_result):
+                if i < len(box_test.output):
+                    actual_result.remove_fields_not_in_expected(box_test.output[i])
+
             if self.create_comparison_sheet:
                 self.comp_obj.create_comparison_sheet(output_result, box_test)
             assert sorted(output_result) == sorted(box_test.output)
@@ -141,6 +146,11 @@ class SPL2TestRunner:
             _LOGGER.info(f"Output check skipped (expected empty): {box_test.name}")
 
         if box_test.metrics:
+            # Remove fields from actual metrics that are not in expected metrics if configured
+            for i, actual_metric in enumerate(metrics_result):
+                if i < len(box_test.metrics):
+                    actual_metric.remove_fields_not_in_expected(box_test.metrics[i])
+
             assert sorted(metrics_result) == sorted(box_test.metrics)
             _LOGGER.info(f"Metric check passed: {box_test.name}")
             _LOGGER.debug("Received: \n%s", str(metrics_result))