spl2-testing-framework 1.2.2__tar.gz → 1.4.0__tar.gz

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.2.2
+Version: 1.4.0
 Summary: SPL2 Testing Framework
 Home-page: https://github.com/splunk/spl2-testing-framework
 Keywords: SPL2 testing framework
@@ -131,6 +131,7 @@ It is possible to pass more options to the command, which works also with pytest
 Additionally, the following options are supported:
 
 * `--ignore_empty_strings` - to ignore empty strings in the results
+* `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
@@ -180,6 +181,8 @@ It is possible to pass more options to the command, which works also with pytest
   the samples in module.json file of the corresponding template file
 * `--sample_delimiter` - Separator for separating the samples provided in the sample file. If not provided, it will use
   newline as a default separator.
+* `--limit_tests` - Limit number of tests to execute from the unit, sample file or module.json. If not provided, all will be executed.
+  This can be used for quick testing of spl2 files during development.
 
 * ... and much more, whatever is supported by pytest
 

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/README.md

@@ -99,6 +99,7 @@ It is possible to pass more options to the command, which works also with pytest
 Additionally, the following options are supported:
 
 * `--ignore_empty_strings` - to ignore empty strings in the results
+* `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
@@ -148,6 +149,8 @@ It is possible to pass more options to the command, which works also with pytest
   the samples in module.json file of the corresponding template file
 * `--sample_delimiter` - Separator for separating the samples provided in the sample file. If not provided, it will use
   newline as a default separator.
+* `--limit_tests` - Limit number of tests to execute from the unit, sample file or module.json. If not provided, all will be executed.
+  This can be used for quick testing of spl2 files during development.
 
 * ... and much more, whatever is supported by pytest
 

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/pyproject.toml

@@ -16,7 +16,7 @@
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.2.2"
+version = "1.4.0"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/spl2_testing_framework/conftest.py

@@ -47,10 +47,12 @@ BOX_TEST = "box_test"
 SINGLE_SPL2_FILE = "single_spl2_file"
 
 IGNORE_EMPTY_STRINGS = "ignore_empty_strings"
+IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL = "ignore_additional_fields_in_actual"
 CREATE_COMPARISON_SHEET = "create_comparison_sheet"
 PERFORMANCE_CHECK = "performance_check"
 TEST_TYPE = "type"
 TEST_DIR = "test_dir"
+LIMIT_TESTS = "limit_tests"
 
 import logging
 
@@ -62,7 +64,7 @@ _LOGGER = logging.getLogger(__name__)
 
 
 def pytest_addoption(parser):
-    parser.addoption(f"--{TEST_TYPE}", action="store", default="local")
+    parser.addoption(f"--{TEST_TYPE}", action="store", default="cli")
     parser.addoption(f"--{TEST_DIR}", action="store", default=".")
     parser.addoption(
         f"--{PERFORMANCE_CHECK}",
@@ -75,6 +77,8 @@ def pytest_addoption(parser):
     parser.addoption("--sample_delimiter", action="store", default="\n")
     parser.addoption(f"--{CREATE_COMPARISON_SHEET}", action="store_true", default=False)
     parser.addoption(f"--{IGNORE_EMPTY_STRINGS}", action="store_true")
+    parser.addoption(f"--{IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL}", action="store_true")
+    parser.addoption(f"--{LIMIT_TESTS}", action="store", type=int, default=None)
 
 
 def pytest_generate_tests(metafunc):
@@ -82,11 +86,12 @@ def pytest_generate_tests(metafunc):
     --test_dir option"""
     test_dir = metafunc.config.getoption(TEST_DIR)
     performance_check = metafunc.config.getoption(PERFORMANCE_CHECK)
+    limit_tests = metafunc.config.getoption(LIMIT_TESTS)
     _set_results_options(metafunc)
 
     if BOX_TEST in metafunc.fixturenames:
         _LOGGER.info("Collecting Box Tests")
-        module_tests = BoxTestDiscovery(test_dir, performance_check)
+        module_tests = BoxTestDiscovery(test_dir, performance_check, limit_tests)
         module_tests.discover_tests()
         metafunc.parametrize(
             "%s" % BOX_TEST, module_tests.tests, ids=module_tests.tests.get_ids()
@@ -105,6 +110,7 @@ def pytest_generate_tests(metafunc):
             sample_file,
             sample_delimiter,
             performance_check=performance_check,
+            limit_tests=limit_tests,
         )
         try:
             spl2_pipelines.discover_tests()
@@ -118,7 +124,7 @@ def pytest_generate_tests(metafunc):
 
     if UNIT_TEST in metafunc.fixturenames:
         _LOGGER.info("Collecting Unit Tests")
-        unit_tests = UTDiscovery(test_dir)
+        unit_tests = UTDiscovery(test_dir, limit_tests)
         unit_tests.discover_tests()
         metafunc.parametrize(
             UNIT_TEST, unit_tests.tests, ids=unit_tests.tests.get_ids()
@@ -245,3 +251,7 @@ def bearer_token_authentication(config):
 def _set_results_options(metafunc):
     ignore_empty_strings = metafunc.config.getoption(IGNORE_EMPTY_STRINGS)
     Results._ignore_empty_strings = ignore_empty_strings
+    ignore_fields_in_actual = metafunc.config.getoption(
+        IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL
+    )
+    Results._ignore_additional_fields_in_actual = ignore_fields_in_actual

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/spl2_testing_framework/tools/results.py

@@ -24,6 +24,7 @@ import pytest
 
 class Results(UserDict):
     _ignore_empty_strings = False
+    _ignore_additional_fields_in_actual = False
     """Class responsible for parsing and storing results from Splunk, IP and CLI.
     It's necessary to parse nested jsons and other python types to strings in order to compare them
     """
@@ -121,6 +122,53 @@ class Results(UserDict):
         if self._ignore_empty_strings:
             self.data = self._remove_field_by_value(self.data, "")
 
+    @staticmethod
+    def _remove_extra_fields(actual_data, expected_data):
+        """Remove fields from actual_data that are not present in expected_data.
+        This is useful when actual results may have additional fields that should be ignored in comparison.
+        Works recursively for nested dicts and lists.
+        """
+        if isinstance(expected_data, dict) and isinstance(actual_data, dict):
+            # Get the keys that are in actual but not in expected
+            keys_to_remove = set(actual_data.keys()) - set(expected_data.keys())
+            for key in keys_to_remove:
+                actual_data.pop(key, None)
+
+            # Recursively process nested structures
+            for key in actual_data.keys():
+                if key in expected_data:
+                    if isinstance(actual_data[key], dict) and isinstance(
+                        expected_data[key], dict
+                    ):
+                        Results._remove_extra_fields(
+                            actual_data[key], expected_data[key]
+                        )
+                    elif isinstance(actual_data[key], list) and isinstance(
+                        expected_data[key], list
+                    ):
+                        # For lists, we need to handle them carefully
+                        if len(actual_data[key]) > 0 and len(expected_data[key]) > 0:
+                            if isinstance(actual_data[key][0], dict) and isinstance(
+                                expected_data[key][0], dict
+                            ):
+                                # If both are lists of dicts, process each element
+                                for i in range(
+                                    min(len(actual_data[key]), len(expected_data[key]))
+                                ):
+                                    Results._remove_extra_fields(
+                                        actual_data[key][i], expected_data[key][i]
+                                    )
+
+        return actual_data
+
+    def remove_fields_not_in_expected(self, expected):
+        """Remove fields from this Results object that are not present in the expected Results object.
+        This is the public method to be called when comparing results.
+        """
+        if self._ignore_additional_fields_in_actual and isinstance(expected, Results):
+            self.data = self._remove_extra_fields(self.data, expected.data)
+        return self
+
     def __lt__(self, other):
         if "_raw" in self.data:
             s = self.data.get("_raw")

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/spl2_testing_framework/tools/spl2test_runner.py

@@ -131,6 +131,11 @@ class SPL2TestRunner:
         perf.check_performance(output_result)
 
         if box_test.output:
+            # Remove fields from actual results that are not in expected results if configured
+            for i, actual_result in enumerate(output_result):
+                if i < len(box_test.output):
+                    actual_result.remove_fields_not_in_expected(box_test.output[i])
+
             if self.create_comparison_sheet:
                 self.comp_obj.create_comparison_sheet(output_result, box_test)
             assert sorted(output_result) == sorted(box_test.output)
@@ -141,6 +146,11 @@ class SPL2TestRunner:
             _LOGGER.info(f"Output check skipped (expected empty): {box_test.name}")
 
         if box_test.metrics:
+            # Remove fields from actual metrics that are not in expected metrics if configured
+            for i, actual_metric in enumerate(metrics_result):
+                if i < len(box_test.metrics):
+                    actual_metric.remove_fields_not_in_expected(box_test.metrics[i])
+
             assert sorted(metrics_result) == sorted(box_test.metrics)
             _LOGGER.info(f"Metric check passed: {box_test.name}")
             _LOGGER.debug("Received: \n%s", str(metrics_result))

{spl2_testing_framework-1.2.2 → spl2_testing_framework-1.4.0}/spl2_testing_framework/tools/test_discovery.py

@@ -41,9 +41,10 @@ class SamplesNotFoundException(Exception):
 class TestDiscovery(abc.ABC):
     """Base class for test discovery classes"""
 
-    def __init__(self, path: str, performance_check: str):
+    def __init__(self, path: str, performance_check: str, limit_tests: int = None):
         self.path = pathlib.Path(path)
         self._performance_check = performance_check
+        self.limit_tests = limit_tests
         self.tests = TestSuite()
 
     def _read_tests(self, test_name: str) -> None:
@@ -56,8 +57,8 @@ class TestDiscovery(abc.ABC):
 class BoxTestDiscovery(TestDiscovery):
     """Discovers tests for box tests, looking into module.test.json files"""
 
-    def __init__(self, path: str, performance_check: str):
-        super().__init__(path, performance_check)
+    def __init__(self, path: str, performance_check: str, limit_tests: int = None):
+        super().__init__(path, performance_check, limit_tests)
 
     def discover_tests(self) -> None:
         self._read_tests("module.test.json")
@@ -70,7 +71,13 @@ class BoxTestDiscovery(TestDiscovery):
 
         for module_path, content in self._parsed_modules.items():
             path = module_path.parent
+            tests_added = 0
             for test in content:
+                if self.limit_tests is not None and tests_added >= self.limit_tests:
+                    _LOGGER.info(
+                        f"Limit of {self.limit_tests} tests reached for {module_path}. Stopping."
+                    )
+                    break
                 name = test["filename"].replace(".spl2", "")
                 test_input = test["test"]["source"]
                 code_module_path = path / test["filename"]
@@ -92,6 +99,7 @@ class BoxTestDiscovery(TestDiscovery):
                 )
                 test.meta["performance"] = self._performance_check
                 self.tests.add(test)
+                tests_added += 1
 
 
 class SingleSPL2Discovery(TestDiscovery):
@@ -104,8 +112,9 @@ class SingleSPL2Discovery(TestDiscovery):
         sample_file=None,
         sample_delimiter="\n",
         performance_check="no",
+        limit_tests: int = None,
     ):
-        super().__init__(path, performance_check)
+        super().__init__(path, performance_check, limit_tests)
         self.template_file = template_file
         self.sample_file = sample_file
         self.sample_delimiter = sample_delimiter
@@ -188,8 +197,8 @@ class SingleSPL2Discovery(TestDiscovery):
 class UTDiscovery(TestDiscovery):
     """Discovers tests for unit tests, looking into .test.spl2 files"""
 
-    def __init__(self, path: str):
-        super().__init__(path, performance_check="no")
+    def __init__(self, path: str, limit_tests: int = None):
+        super().__init__(path, performance_check="no", limit_tests=limit_tests)
 
     def discover_tests(self) -> None:
         self._read_tests("*.test.spl2")
@@ -208,7 +217,13 @@ class UTDiscovery(TestDiscovery):
             code_module_name = file_name.replace(".test.", ".")
             code_module_path = path.parent / code_module_name
 
+            tests_added = 0
             for test in tests:
+                if self.limit_tests is not None and tests_added >= self.limit_tests:
+                    _LOGGER.info(
+                        f"Limit of {self.limit_tests} tests reached for {file}. Stopping."
+                    )
+                    break
                 self.tests.add(
                     UnitTest(
                         name=test,
@@ -218,3 +233,4 @@ class UTDiscovery(TestDiscovery):
                         code_module_path=code_module_path,
                     )
                 )
+                tests_added += 1