spl2-testing-framework 1.1.1__tar.gz → 1.2.0__tar.gz

{spl2_testing_framework-1.1.1 → spl2_testing_framework-1.2.0}/PKG-INFO

@@ -1,18 +1,26 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.1.1
+Version: 1.2.0
 Summary: SPL2 Testing Framework
 Home-page: https://cd.splunkdev.com/data-availability/spl2-content-tools
 Keywords: SPL2 testing framework
 Author: mgazda-splunk
 Author-email: mgazda@splunk.com
-Requires-Python: >=3.7,<4.0
+Requires-Python: >=3.7.1,<4.0.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: numpy (>=1.21,<1.22) ; python_version < "3.8"
+Requires-Dist: numpy (>=1.21,<1.24) ; python_version >= "3.8" and python_version < "3.10"
+Requires-Dist: numpy (>=1.25,<2.0) ; python_version >= "3.10" and python_version < "3.13"
+Requires-Dist: numpy (>=1.26,<2.0) ; python_version >= "3.13"
+Requires-Dist: openpyxl (>=3.1.2,<4.0.0)
+Requires-Dist: pandas (==1.3.5) ; python_version < "3.8"
+Requires-Dist: pandas (==1.5.3) ; python_version >= "3.8" and python_version < "3.10"
+Requires-Dist: pandas (>=2.0.3,<3.0.0) ; python_version >= "3.10" and python_version < "3.13"
+Requires-Dist: pandas (>=2.1.0,<3.0.0) ; python_version >= "3.13"
 Requires-Dist: pytest (==7.4.4)
 Requires-Dist: pytest-subtests (==0.13.1)
 Requires-Dist: pytest-xdist (==3.5.0)
@@ -123,6 +131,7 @@ It is possible to pass more options to the command, which works also with pytest
 Additionally, the following options are supported:
 
 * `--ignore_empty_strings` - to ignore empty strings in the results
+* `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.

{spl2_testing_framework-1.1.1 → spl2_testing_framework-1.2.0}/README.md

@@ -99,6 +99,7 @@ It is possible to pass more options to the command, which works also with pytest
 Additionally, the following options are supported:
 
 * `--ignore_empty_strings` - to ignore empty strings in the results
+* `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.

{spl2_testing_framework-1.1.1 → spl2_testing_framework-1.2.0}/pyproject.toml

@@ -16,7 +16,7 @@
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.1.1"
+version = "1.2.0"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"
@@ -33,13 +33,26 @@ spl2_tests_run = "spl2_testing_framework.test_runner:run"
 single_spl2_file_run = "spl2_testing_framework.single_spl2_file_runner:run"
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.7.1"
 pytest = "7.4.4"
 pytest-xdist = "3.5.0"
 pytest-subtests = "0.13.1"
 requests = "^2.31.0"
 tabulate = "0.9.0"
 urllib3 = "^1.26.20"
+pandas = [
+    { version = "1.3.5", python = "<3.8" },
+    { version = "1.5.3", python = ">=3.8,<3.10" },
+    { version = "^2.0.3", python = ">=3.10,<3.13" },
+    { version = "^2.1.0", python = ">=3.13" }
+]
+numpy = [
+    { version = ">=1.21,<1.22", python = "<3.8" },
+    { version = ">=1.21,<1.24", python = ">=3.8,<3.10" },
+    { version = ">=1.25,<2.0", python = ">=3.10,<3.13" },
+    { version = ">=1.26,<2.0", python = ">=3.13" }
+]
+openpyxl = "^3.1.2"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "*"

{spl2_testing_framework-1.1.1 → spl2_testing_framework-1.2.0}/spl2_testing_framework/conftest.py

@@ -47,6 +47,7 @@ BOX_TEST = "box_test"
 SINGLE_SPL2_FILE = "single_spl2_file"
 
 IGNORE_EMPTY_STRINGS = "ignore_empty_strings"
+CREATE_COMPARISON_SHEET = "create_comparison_sheet"
 PERFORMANCE_CHECK = "performance_check"
 TEST_TYPE = "type"
 TEST_DIR = "test_dir"
@@ -72,6 +73,7 @@ def pytest_addoption(parser):
     parser.addoption("--template_file", action="store", default=None)
     parser.addoption("--sample_file", action="store", default=None)
     parser.addoption("--sample_delimiter", action="store", default="\n")
+    parser.addoption(f"--{CREATE_COMPARISON_SHEET}", action="store_true", default=False)
     parser.addoption(f"--{IGNORE_EMPTY_STRINGS}", action="store_true")
 
 
@@ -123,9 +125,13 @@ def pytest_generate_tests(metafunc):
         )
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def spl2_test_runner(
-    request, config, basic_authentication, bearer_token_authentication
+    request,
+    config,
+    basic_authentication,
+    bearer_token_authentication,
+    run_configurations,
 ):
     """This fixture is used to create a test runner object based on the test type specified by the --type option
     3 types of tests are supported: 'splunk', 'cloud', 'cli'
@@ -146,10 +152,20 @@ def spl2_test_runner(
             )
         )
 
-    x = SPL2TestRunner(search_client)
+    x = SPL2TestRunner(search_client, run_configurations)
     return x
 
 
+@pytest.fixture(scope="session")
+def run_configurations(request):
+    """Returns all the configuration options in one object (dict). Add more as necessary."""
+    return {
+        CREATE_COMPARISON_SHEET: request.config.getoption(
+            f"--{CREATE_COMPARISON_SHEET}"
+        ),
+    }
+
+
 @pytest.fixture(scope="session")
 def config():
     """This fixture is used to read the configuration from the environment variables or from the spl2_test_config.json

spl2_testing_framework-1.2.0/spl2_testing_framework/tools/create_diff_sheet.py

@@ -0,0 +1,228 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import pandas as pd
+from openpyxl import load_workbook
+from openpyxl.styles import PatternFill, Border, Side, Alignment
+import json, os
+from typing import Union, List, Dict
+import logging
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class ComparisonSheet:
+    """Class to create a comparison Excel sheet between expected and actual data."""
+
+    def __init__(self):
+        self.actual_json_list = []
+        self.expected_json_list = []
+        self.expected_path = "comparison_box_test/expected.json"
+        self.actual_path = "comparison_box_test/actual.json"
+        self.output_path = "comparison_box_test/comparison.xlsx"
+        os.makedirs("comparison_box_test", exist_ok=True)
+
+    def create_comparison_sheet(self, output_result, box_test):
+        """
+        1. Creates a comparison_box_test directory.
+        2. Then creates actual.json, expected.json and comparison.xlsx sheet in that folder.
+
+        Parameters:
+            output_result (list): The output result got from the execution.
+            box_test (object): Box test object.
+
+        """
+        self.actual_json_list.append(output_result[0])
+        self.expected_json_list.append(box_test.output[0])
+        with open(self.expected_path, "w") as f:
+            json.dump([r.__dict__ for r in self.expected_json_list], f, indent=2)
+        with open(self.actual_path, "w") as f:
+            json.dump([r.__dict__ for r in self.actual_json_list], f, indent=2)
+        self._create_comparison_sheet()
+
+    def _create_comparison_sheet(self):
+        """
+        Creates a comparison Excel sheet between expected and actual data.
+        """
+
+        _LOGGER.info("Starting comparison generation...")
+
+        expected_list = self._load_json(self.expected_path)
+        actual_list = self._load_json(self.actual_path)
+        df = self._compare_events(expected_list, actual_list)
+        df.to_excel(self.output_path, index=False)
+
+        self._apply_excel_styles(self.output_path)
+
+        _LOGGER.info(f"✅ Comparison sheet created at: {self.output_path}")
+
+    def _load_json(self, json_input: Union[str, list]) -> List[Dict]:
+        """Load JSON data from a file path or return list directly."""
+        if isinstance(json_input, str):
+            with open(json_input) as f:
+                return json.load(f)
+        return json_input
+
+    def _compare_events(self, list1: List[Dict], list2: List[Dict]) -> pd.DataFrame:
+        """Compare expected and actual events, returning a formatted DataFrame."""
+
+        self.fields = []
+        self.expected_vals = []
+        self.actual_vals = []
+        self.problems = []
+        self.solutions = []
+
+        for idx, (event_expected, event_actual) in enumerate(
+            zip(list1, list2), start=1
+        ):
+            dict_expected = event_expected.get("data", {})
+            dict_actual = event_actual.get("data", {})
+
+            all_fields = sorted(set(dict_expected.keys()) | set(dict_actual.keys()))
+            equal_fields = []
+
+            for field in all_fields:
+                exp_val = dict_expected.get(field)
+                act_val = dict_actual.get(field)
+
+                if exp_val is None and act_val is not None:
+                    self._add_row(
+                        field,
+                        exp_val,
+                        act_val,
+                        problem="Extracted additionally from SPL2 pipeline",
+                        solution="Possible to ignore",
+                    )
+                elif act_val is None and exp_val is not None:
+                    self._add_row(
+                        field,
+                        exp_val,
+                        act_val,
+                        problem="Missing from SPL2 pipeline. Need to fix.",
+                        solution="",
+                    )
+                elif exp_val != act_val:
+                    self._add_row(
+                        field,
+                        exp_val,
+                        act_val,
+                        problem="Value mismatch",
+                        solution="Check this in SPL2 pipeline and fix if needed.",
+                    )
+                else:
+                    if field == "_raw":
+                        self._add_row(field, exp_val, act_val)
+                    else:
+                        equal_fields.append(field)
+
+            if equal_fields:
+                self._add_row(
+                    field="Equal fields for this event",
+                    exp_val=", ".join(equal_fields),
+                    act_val="",
+                )
+
+            self._add_row(
+                field="All Expected Fields", exp_val=dict_expected, act_val=""
+            )
+            self._add_row(field="All Actual Fields", exp_val=dict_actual, act_val="")
+            self._add_row(field="", exp_val="", act_val="")
+
+        return pd.DataFrame(
+            {
+                "Field": self.fields,
+                "Expected": self.expected_vals,
+                "Actual": self.actual_vals,
+                "Problem": self.problems,
+                "Solution": self.solutions,
+            }
+        )
+
+    def _add_row(self, field, exp_val, act_val, problem="", solution=""):
+        """Append a single comparison row to the running lists."""
+        self.fields.append(field)
+        self.expected_vals.append(exp_val)
+        self.actual_vals.append(act_val)
+        self.problems.append(problem)
+        self.solutions.append(solution)
+
+    def _apply_excel_styles(self, output_path: str) -> None:
+        """Apply Excel formatting and highlighting."""
+        wb = load_workbook(output_path)
+        ws = wb.active
+
+        diff_fill = PatternFill(
+            start_color="FFFF00", end_color="FFFF00", fill_type="solid"
+        )
+        missing_fill = PatternFill(
+            start_color="FF9999", end_color="FF9999", fill_type="solid"
+        )
+        equal_fill = PatternFill(
+            start_color="CCFFCC", end_color="CCFFCC", fill_type="solid"
+        )
+
+        thin_border = Border(
+            left=Side(style="thin", color="000000"),
+            right=Side(style="thin", color="000000"),
+            top=Side(style="thin", color="000000"),
+            bottom=Side(style="thin", color="000000"),
+        )
+
+        alignment = Alignment(horizontal="left", vertical="center", wrap_text=True)
+
+        for r, row in enumerate(
+            ws.iter_rows(
+                min_row=1, max_row=ws.max_row, min_col=1, max_col=ws.max_column
+            ),
+            start=1,
+        ):
+            ws.row_dimensions[r].height = 15
+
+            field_name = row[0].value
+            expected_val = row[1].value if len(row) > 1 else None
+            actual_val = row[2].value if len(row) > 2 else None
+
+            for cell in row:
+                cell.border = thin_border
+                cell.alignment = alignment
+
+            if r > 1:  # skip header row
+                if expected_val is None and actual_val not in (None, ""):
+                    row[1].fill = missing_fill
+                elif actual_val is None and expected_val not in (None, ""):
+                    row[2].fill = missing_fill
+                elif expected_val != actual_val and field_name not in (
+                    "",
+                    "Equal fields for this event",
+                    "All Expected Fields",
+                    "All Actual Fields",
+                ):
+                    row[1].fill = diff_fill
+                    row[2].fill = diff_fill
+
+                if field_name in (
+                    "All Expected Fields",
+                    "All Actual Fields",
+                    "Equal fields for this event",
+                ):
+                    for cell in row:
+                        cell.fill = equal_fill
+
+        for col in ["A", "B", "C", "D", "E"]:
+            ws.column_dimensions[col].width = 50
+        ws.freeze_panes = "B2"
+
+        wb.save(output_path)

{spl2_testing_framework-1.1.1 → spl2_testing_framework-1.2.0}/spl2_testing_framework/tools/spl2test_runner.py

@@ -21,6 +21,8 @@ from tabulate import tabulate
 from spl2_testing_framework.tools.performance import PerformanceCheck
 from spl2_testing_framework.tools.test_types import UnitTest, BoxTest, SingleSPL2
 from spl2_testing_framework.tools.utils import _make_functions_visible_for_testing
+from spl2_testing_framework.tools.create_diff_sheet import ComparisonSheet
+
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -28,8 +30,10 @@ _LOGGER = logging.getLogger(__name__)
 class SPL2TestRunner:
     """Main class responsible for running tests"""
 
-    def __init__(self, search_client):
+    def __init__(self, search_client, run_configurations):
         self._search_client = search_client
+        self.create_comparison_sheet = run_configurations.get("create_comparison_sheet")
+        self.comp_obj = ComparisonSheet()
 
     def run_single_spl2_file(self, single_spl2_file: SingleSPL2) -> None:
         """Run a single spl2 file and print the output"""
@@ -127,6 +131,8 @@ class SPL2TestRunner:
         perf.check_performance(output_result)
 
         if box_test.output:
+            if self.create_comparison_sheet:
+                self.comp_obj.create_comparison_sheet(output_result, box_test)
             assert sorted(output_result) == sorted(box_test.output)
             _LOGGER.info(f"Output check passed: {box_test.name}")
             _LOGGER.debug("Received: \n%s", str(output_result))