spl2-testing-framework 1.7.2__tar.gz → 1.7.3__tar.gz

{spl2_testing_framework-1.7.2 → spl2_testing_framework-1.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.7.2
+Version: 1.7.3
 Summary: SPL2 Testing Framework
 Home-page: https://github.com/splunk/spl2-testing-framework
 Keywords: SPL2 testing framework

{spl2_testing_framework-1.7.2 → spl2_testing_framework-1.7.3}/pyproject.toml

@@ -15,7 +15,7 @@
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.7.2"
+version = "1.7.3"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"

{spl2_testing_framework-1.7.2 → spl2_testing_framework-1.7.3}/spl2_testing_framework/tools/results.py

@@ -13,12 +13,11 @@
 # limitations under the License.
 
 
+import ast
 import json
 from collections import UserDict
 from datetime import datetime, timezone
-from os import remove
 from typing import Any
-import pytest
 
 
 class Results(UserDict):
@@ -38,16 +37,34 @@ class Results(UserDict):
         self._remove_timestamps_if_necessary()
         self._remove_all_empty_strings_from_dicts_if_necessary()
 
+    @staticmethod
+    def _coerce_string_if_json_or_python_literal(value: str) -> Any:
+        """If ``value`` looks like JSON or a Python ``list``/``dict`` literal, parse it.
+
+        Test JSON and Splunk sometimes embed ``list``/``dict`` as strings; JSON only
+        accepts double quotes, so single-quoted Python reprs need ``literal_eval``.
+        """
+        candidate = value.strip()
+        if not (candidate.startswith("[") or candidate.startswith("{")):
+            return value
+        try:
+            return json.loads(candidate)
+        except Exception:
+            try:
+                parsed = ast.literal_eval(candidate)
+            except (ValueError, SyntaxError):
+                return value
+            if isinstance(parsed, (list, dict)):
+                return parsed
+            return value
+
     @staticmethod
     def _cast_types(data: Any):
         """Casts everything into dicts / lists / strings.
         This is necessary because Splunk, IP and CLI often return different types of data.
         """
-        if isinstance(data, str) and (data.startswith("[") or data.startswith("{")):
-            try:
-                data = json.loads(data)
-            except Exception:
-                pass  # If it fails, it's not a json, so it's just a normal string
+        if isinstance(data, str):
+            data = Results._coerce_string_if_json_or_python_literal(data)
 
         if isinstance(data, list):
             data = [Results._cast_types(item) for item in data]
@@ -55,7 +72,7 @@ class Results(UserDict):
             for key, value in data.items():
                 data[key] = Results._cast_types(value)
                 if key == "_time":
-                    data[key] = Results._convert_timestamp(value)
+                    data[key] = Results._convert_timestamp(data[key])
         if isinstance(data, (int, float)):
             data = str(data)
 
@@ -66,7 +83,7 @@ class Results(UserDict):
         """We need to convert timestamp, as there are different formats returned by cloud / CLI / splunk"""
         try:
             ts = datetime.fromisoformat(timestamp)
-        except ValueError:
+        except (ValueError, TypeError):
             ts = datetime.fromtimestamp(float(timestamp), timezone.utc)
 
         return ts.isoformat()

{spl2_testing_framework-1.7.2 → spl2_testing_framework-1.7.3}/spl2_testing_framework/tools/spl2test_runner.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+import copy
+import json
 import logging
 import pytest
 
@@ -20,6 +22,7 @@ from tabulate import tabulate
 
 from spl2_testing_framework.tools.create_diff_sheet import ComparisonSheet
 from spl2_testing_framework.tools.performance import PerformanceCheck
+from spl2_testing_framework.tools.results import Results
 from spl2_testing_framework.tools.search_clients.cli_search_client import (
     CLISearchClient,
 )
@@ -187,7 +190,7 @@ class SPL2TestRunner:
         _LOGGER.info("Reduction rate for this event: %s", reduction_rate)
 
         if self.check_splunk_results:
-            self._check_splunk_results(box_test, job)
+            self._check_splunk_results(box_test, output_result)
         else:
             if box_test.output:
                 # Remove fields from actual results that are not in expected results if configured
@@ -222,6 +225,23 @@ class SPL2TestRunner:
     def _row_as_dict(row):
         return row.data if hasattr(row, "data") else dict(row)
 
+    @staticmethod
+    def _tree_copy_for_results(d: dict) -> dict:
+        """Clone ``d`` so ``Results`` normalization cannot mutate caller data.
+
+        ``Results._cast_types`` updates dicts in place. Splunk-shaped rows are
+        JSON-like; cloning via :func:`json.dumps` / :func:`json.loads` is
+        typically faster than :func:`copy.deepcopy`. Non-JSON values use
+        ``default=str`` (same idea as serializing for HEC). Falls back to
+        deepcopy only if JSON cloning fails.
+        """
+        if not d:
+            return {}
+        try:
+            return json.loads(json.dumps(d, default=str))
+        except (TypeError, ValueError):
+            return copy.deepcopy(d)
+
     @staticmethod
     def _strip_ignored_fields_for_splunk_compare(
         row: dict, ignore_keys: frozenset
@@ -239,27 +259,44 @@ class SPL2TestRunner:
                 context=f"box_test={box_test.name!r}",
             ) from e
 
-    def _check_splunk_results(self, box_test, job):
-        output_result = job.result.get("destination", [])
+    def _check_splunk_results(self, box_test, destination_rows):
+        """Use the same destination rows as ``run_box_test`` (from ``get_job_results``), not ``job.result``."""
         if self.check_splunk_results == "CIM":
             if not box_test.cim_fields:
                 pytest.skip("CIM output not available for this test")
 
-            for event in output_result:
-                processed_event = self._splunk_process_event(box_test, event)
-                validate_compatibility(processed_event, box_test.cim_fields)
+            for event in destination_rows:
+                processed_event = self._splunk_process_event(
+                    box_test, self._row_as_dict(event)
+                )
+                # Same normalization as in-memory box compares (Results._cast_types).
+                actual_cmp = Results(self._tree_copy_for_results(processed_event)).data
+                expected_cmp = Results(
+                    self._tree_copy_for_results(box_test.cim_fields)
+                ).data
+                validate_compatibility(actual_cmp, expected_cmp)
         elif self.check_splunk_results == "TA":
+            if not box_test.output:
+                pytest.skip("TA expected output not available for this test")
+
             ignore_spec = box_test.ignore_fields_in_splunk_check
             ignore_keys = frozenset(ignore_spec.keys())
-            for event, expected in zip(output_result, box_test.output):
-                processed_event = self._splunk_process_event(box_test, event)
-                actual = self._strip_ignored_fields_for_splunk_compare(
-                    processed_event, ignore_keys
+            for event, expected in zip(destination_rows, box_test.output):
+                processed_event = self._splunk_process_event(
+                    box_test, self._row_as_dict(event)
+                )
+                expected_data = self._row_as_dict(expected)
+                # Splunk adds many search-time fields; mirror in-memory compare by
+                # dropping keys not present on the expected row before asserting.
+                actual_dict = dict(processed_event)
+                Results._remove_extra_fields(actual_dict, expected_data)
+                actual_cmp = self._strip_ignored_fields_for_splunk_compare(
+                    actual_dict, ignore_keys
                 )
                 expected_cmp = self._strip_ignored_fields_for_splunk_compare(
-                    self._row_as_dict(expected), ignore_keys
+                    expected_data, ignore_keys
                 )
-                assert actual == expected_cmp, (
+                assert actual_cmp == expected_cmp, (
                     f"TA Splunk compare mismatch for {box_test.name}; "
                     f"ignored_fields={ignore_spec!r}"
                 )

{spl2_testing_framework-1.7.2 → spl2_testing_framework-1.7.3}/spl2_testing_framework/tools/test_discovery.py

@@ -245,9 +245,9 @@ class BoxTestDiscovery(TestDiscovery):
                     Results(x)
                     for x in test_block.get("expected_destination_result", [])
                 ]
-                cim_fields = test_block.get("expected_cim_fields", {}).get(
-                    "cim_fields", {}
-                )
+                cim_fields = Results(
+                    test_block.get("expected_cim_fields", {}).get("cim_fields", {})
+                ).data
                 metrics = [
                     Metrics(x)
                     for x in test_block.get("expected_metrics_destination_result", [])

{spl2_testing_framework-1.7.2 → spl2_testing_framework-1.7.3}/spl2_testing_framework/tools/utils.py

@@ -68,31 +68,58 @@ def calculate_reduction_rate(original_event, reduced_event):
 
 
 def validate_compatibility(actual, expected):
-    """Check that every field in ``expected`` exists in ``actual`` with the same value.
+    """Check that every field in *expected* exists in *actual* with the same value.
 
     Order matches the usual test convention ``assert actual == expected`` (observed
-    value first, reference second). For CIM checks, ``actual`` is the Splunk-processed
-    event and ``expected`` is ``box_test.cim_fields``.
+    value first, reference second). For Splunk CIM checks, the runner passes both
+    sides through :class:`~spl2_testing_framework.tools.results.Results` first so
+    numeric and other types match pipeline/CLI normalization.
+    Extra fields in *actual* that are not in *expected* are silently ignored,
+    so callers only need to specify the subset of fields they care about.
+
+    Raises ``AssertionError`` with a human-readable summary listing every
+    mismatched and every missing field in a single message.
     """
-    extracted_but_wrong = {}
-    not_extracted = {}
-    for field, value in expected.items():
-        if field in actual:
-            if actual[field] == value:
-                _LOGGER.info(f"FIELD {field} in reduced event")
-                continue
-            else:
-                extracted_but_wrong[field] = value
-                _LOGGER.error(f"FIELD {field} in reduced event has wrong value")
-
+    wrong_value = {}
+    missing = {}
+
+    for field, expected_val in expected.items():
+        if field not in actual:
+            missing[field] = expected_val
+            _LOGGER.error("FIELD %s is missing from actual event", field)
+            continue
+
+        actual_val = actual[field]
+        if actual_val == expected_val:
+            _LOGGER.info("FIELD %s OK", field)
         else:
-            not_extracted[field] = value
-            _LOGGER.error(f"FIELD {field} is not present in the reduced event")
+            wrong_value[field] = {"actual": actual_val, "expected": expected_val}
+            _LOGGER.error(
+                "FIELD %s mismatch: actual=%r, expected=%r",
+                field,
+                actual_val,
+                expected_val,
+            )
 
-    _LOGGER.info(f"Checking wrong extractions")
-    assert extracted_but_wrong == {}
-    _LOGGER.info(f"Checking missing extractions")
-    assert not_extracted == {}
+    message = _format_compatibility_errors(wrong_value, missing)
+    assert message is None, message
+
+
+def _format_compatibility_errors(wrong_value, missing):
+    """Return a formatted error string, or ``None`` when there are no errors."""
+    sections = []
+    if wrong_value:
+        lines = [
+            f"  {field}: actual={d['actual']!r}, expected={d['expected']!r}"
+            for field, d in wrong_value.items()
+        ]
+        sections.append("Wrong values:\n" + "\n".join(lines))
+    if missing:
+        lines = [f"  {field}: expected={val!r}" for field, val in missing.items()]
+        sections.append("Missing fields:\n" + "\n".join(lines))
+    if not sections:
+        return None
+    return "\n" + "\n".join(sections)
 
 
 _QUOTED_STRING_OR_BARE_KEY = re.compile(