spl2-testing-framework 1.0.0__py3-none-any.whl

spl2_testing_framework/tools/spl2test_runner.py

@@ -0,0 +1,143 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import logging
+
+from tabulate import tabulate
+
+from spl2_testing_framework.tools.performance import PerformanceCheck
+from spl2_testing_framework.tools.test_types import UnitTest, BoxTest, SingleSPL2
+from spl2_testing_framework.tools.utils import _make_functions_visible_for_testing
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class SPL2TestRunner:
+    """Main class responsible for running tests"""
+
+    def __init__(self, search_client):
+        self._search_client = search_client
+
+    def run_single_spl2_file(self, single_spl2_file: SingleSPL2) -> None:
+        """Run a single spl2 file and print the output"""
+        code_to_test = _make_functions_visible_for_testing(
+            single_spl2_file.code_module_path
+        )
+        perf = PerformanceCheck(single_spl2_file)
+        code_to_test = perf.apply_code_transformations(code_to_test)
+
+        job = self._search_client.create_simple_job(
+            single_spl2_file.input,
+            single_spl2_file.code_module_path.stem,
+            code_to_test,
+            single_spl2_file.name,
+        )
+        self._search_client.run_job(job)
+        job_result = self._search_client.get_job_results(job)
+        output_result = job_result.get("destination", [])
+
+        perf.check_performance(output_result)
+
+        self.print_results(output_result, "RESULTS")
+        metrics_result = job_result.get("metrics_destination", [])
+        if len(metrics_result) > 0:
+            self.print_results(metrics_result, "METRICS RESULTS")
+
+    def print_results(self, results, type):
+        """Prints the results in cli and log file"""
+        separator = "#"
+        _LOGGER.info(f"\n\n{separator * 50} {type} {separator * 50}\n\n")
+        for item in results:
+            table = tabulate(item.items(), headers=["Field", "Value"], tablefmt="plain")
+            _LOGGER.info("\n" + "-" * 20 + "\n")
+            _LOGGER.info("\n" + table + "\n")
+            _LOGGER.info("\n" + "-" * 20 + "\n")
+
+    def run_unit_test(self, template_test: UnitTest) -> None:
+        """Run a unit test"""
+        code_to_test = _make_functions_visible_for_testing(
+            template_test.code_module_path
+        )
+
+        job = self._search_client.create_job(
+            template_test.content,
+            template_test.code_module_path.stem,
+            code_to_test,
+            template_test.name,
+        )
+
+        self._search_client.run_job(job)
+        job_results = self._search_client.get_job_results(job)
+
+        errors = []
+
+        for result in job_results[template_test.name]:
+            for case_result in result["_testResults"]["results"]:
+                received, expected = (
+                    case_result.get("received", None),
+                    case_result.get("expected", None),
+                )
+                _LOGGER.debug("Check details: %s", case_result)
+
+                if expected == received:
+                    _LOGGER.info(
+                        f'Check passed: {template_test.file_name}/{template_test.name}/{case_result["test"]}'
+                    )
+                else:
+                    errors.append(
+                        {
+                            "TEST_STATEMENT": case_result["test"],
+                            "ACTUAL": received,
+                            "EXPECTED": expected,
+                        }
+                    )
+
+        assert errors == [], f"Assertion errors: \n{errors}"
+
+    def run_box_test(self, box_test: BoxTest) -> None:
+        """Run a box test"""
+        code_to_test = _make_functions_visible_for_testing(box_test.code_module_path)
+
+        perf = PerformanceCheck(box_test)
+        code_to_test = perf.apply_code_transformations(code_to_test)
+
+        job = self._search_client.create_simple_job(
+            box_test.input, box_test.code_module_path.stem, code_to_test, box_test.name
+        )
+
+        self._search_client.run_job(job)
+        job_result = self._search_client.get_job_results(job)
+
+        output_result = job_result.get("destination", [])
+        metrics_result = job_result.get("metrics_destination", [])
+
+        perf.check_performance(output_result)
+
+        if box_test.output:
+            assert sorted(output_result) == sorted(box_test.output)
+            _LOGGER.info(f"Output check passed: {box_test.name}")
+            _LOGGER.debug("Received: \n%s", str(output_result))
+            _LOGGER.debug("Expected: \n%s", str(box_test.output))
+        else:
+            _LOGGER.info(f"Output check skipped (expected empty): {box_test.name}")
+
+        if box_test.metrics:
+            assert sorted(metrics_result) == sorted(box_test.metrics)
+            _LOGGER.info(f"Metric check passed: {box_test.name}")
+            _LOGGER.debug("Received: \n%s", str(metrics_result))
+            _LOGGER.debug("Expected: \n%s", str(box_test.metrics))
+        else:
+            _LOGGER.info(f"Metric check skipped (expected empty): {box_test.name}")

spl2_testing_framework/tools/test_discovery.py

@@ -0,0 +1,220 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import abc
+import json
+import logging
+import os
+import pathlib
+import re
+
+from spl2_testing_framework.tools.results import Metrics, Results
+from spl2_testing_framework.tools.test_types import (
+    BoxTest,
+    SingleSPL2,
+    TestSuite,
+    UnitTest,
+)
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class SamplesNotFoundException(Exception):
+    """An Exception to be raised if samples are not found"""
+
+    pass
+
+
+class TestDiscovery(abc.ABC):
+    """Base class for test discovery classes"""
+
+    def __init__(self, path: str, performance_check: str):
+        self.path = pathlib.Path(path)
+        self._performance_check = performance_check
+        self.tests = TestSuite()
+
+    def _read_tests(self, test_name: str) -> None:
+        tests_paths = self.path.rglob(test_name)
+        self._raw_tests = {
+            x: pathlib.Path(x).read_text(encoding="utf-8") for x in tests_paths
+        }
+
+
+class BoxTestDiscovery(TestDiscovery):
+    """Discovers tests for box tests, looking into module.test.json files"""
+
+    def __init__(self, path: str, performance_check: str):
+        super().__init__(path, performance_check)
+
+    def discover_tests(self) -> None:
+        self._read_tests("module.test.json")
+        self._parse_tests()
+
+    def _parse_tests(self) -> None:
+        self._parsed_modules = {
+            x: json.loads(content) for x, content in self._raw_tests.items()
+        }
+
+        for module_path, content in self._parsed_modules.items():
+            path = module_path.parent
+            for test in content:
+                name = test["filename"].replace(".spl2", "")
+                test_input = test["test"]["source"]
+                code_module_path = path / test["filename"]
+                output = [
+                    Results(x)
+                    for x in test["test"].get("expected_destination_result", [])
+                ]
+                metrics = [
+                    Metrics(x)
+                    for x in test["test"].get("expected_metrics_destination_result", [])
+                ]
+                test = BoxTest(
+                    name=name,
+                    input=test_input,
+                    output=output,
+                    metrics=metrics,
+                    module_path=module_path,
+                    code_module_path=code_module_path,
+                )
+                test.meta["performance"] = self._performance_check
+                self.tests.add(test)
+
+
+class SingleSPL2Discovery(TestDiscovery):
+    """Discovers the template file and its samples"""
+
+    def __init__(
+        self,
+        path: str,
+        template_file,
+        sample_file=None,
+        sample_delimiter="\n",
+        performance_check="no",
+    ):
+        super().__init__(path, performance_check)
+        self.template_file = template_file
+        self.sample_file = sample_file
+        self.sample_delimiter = sample_delimiter
+
+    def discover_tests(self) -> None:
+        if not self.sample_file:
+            self._read_tests("module.json")
+        self._parse_tests()
+
+    def _parse_tests(self) -> None:
+        name = None
+        code_module_path = None
+        if self.sample_file:
+            _LOGGER.info("Reading samples from provided sample file.")
+            events = self._read_samples_from_file(delimiter=self.sample_delimiter)
+            name = self.template_file.replace(".spl2", "")
+            code_module_path = next(self.path.rglob(self.template_file), None)
+        else:
+            _LOGGER.info("Reading samples from corresponding module.json file")
+            self._parsed_modules = {
+                x: json.loads(content) for x, content in self._raw_tests.items()
+            }
+            events = None
+            for module_path, content in self._parsed_modules.items():
+                path = module_path.parent
+                break_outer = False
+                for test in content:
+                    if test["filename"] == os.path.basename(self.template_file):
+                        name = test["filename"].replace(".spl2", "")
+                        code_module_path = path / test["filename"]
+                        events = test["context"]["events"]
+                        if isinstance(events, str):
+                            events = events.split("\r\n")
+                        elif isinstance(events, list):
+                            events = [event["_raw"] for event in events]
+                        else:
+                            _LOGGER.error("Invalid events found.")
+                            raise SamplesNotFoundException(
+                                "No valid sample events found. Exiting.."
+                            )
+                        break_outer = True
+                        break
+                if break_outer:
+                    break
+        if not events:
+            _LOGGER.error("No events found.")
+            raise SamplesNotFoundException("No valid sample events found. Exiting..")
+        _LOGGER.debug(f"Events collected: {events}")
+        test_input = self._parse_and_append_events(events)
+
+        test = SingleSPL2(
+            name=name,
+            input=test_input,
+            code_module_path=code_module_path,
+        )
+        test.meta["performance"] = self._performance_check
+
+        self.tests.add(test)
+
+    def _read_samples_from_file(self, delimiter):
+        with open(self.sample_file) as file:
+            events = filter(lambda ev: len(ev) > 0, file.read().split(delimiter))
+            updated_events = []
+            for event in events:
+                event = "".join([delimiter, event])
+                updated_events.append(event)
+            final_events = [
+                event.encode().decode("unicode_escape") for event in updated_events
+            ]
+        return final_events
+
+    def _parse_and_append_events(self, events):
+        formatted_events = ",".join(
+            [f"{{_raw: {json.dumps(event)} }}" for event in events]
+        )
+        template_module = f"[{formatted_events}]"
+        return template_module
+
+
+class UTDiscovery(TestDiscovery):
+    """Discovers tests for unit tests, looking into .test.spl2 files"""
+
+    def __init__(self, path: str):
+        super().__init__(path, performance_check="no")
+
+    def discover_tests(self) -> None:
+        self._read_tests("*.test.spl2")
+        self._parse_tests()
+
+    def _parse_tests(self) -> None:
+        self.parsed = {
+            filename: re.findall(r"\$?(.*?__test)", test_file_content)
+            for filename, test_file_content in self._raw_tests.items()
+        }
+
+        for file, tests in self.parsed.items():
+            path = pathlib.Path(file)
+            file_name = path.name
+            content = path.read_text(encoding="utf-8")
+            code_module_name = file_name.replace(".test.", ".")
+            code_module_path = path.parent / code_module_name
+
+            for test in tests:
+                self.tests.add(
+                    UnitTest(
+                        name=test,
+                        file_name=file_name,
+                        file_path=path,
+                        content=content,
+                        code_module_path=code_module_path,
+                    )
+                )

spl2_testing_framework/tools/test_types.py

@@ -0,0 +1,84 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import logging
+from collections import UserDict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Iterator
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@dataclass
+class Test:
+    name: str
+    meta: dict = field(default_factory=dict, init=False)
+
+    @property
+    def test_id(self) -> str:
+        return self.name
+
+
+@dataclass
+class SingleSPL2(Test):
+    input: str
+    code_module_path: Path
+
+
+@dataclass
+class BoxTest(Test):
+    input: str
+    output: list
+    module_path: Path
+    code_module_path: Path
+    metrics: list = field(default_factory=list)
+
+
+@dataclass
+class UnitTest(Test):
+    file_path: Path
+    file_name: str
+    content: str
+    code_module_path: Path
+
+    @property
+    def test_id(self) -> str:
+        return self.file_name + "__" + self.name
+
+
+@dataclass
+class TestSuite(UserDict):
+    tests: Dict[str, Test] = field(default_factory=dict)
+
+    def add(self, test: Test) -> None:
+        if test.test_id not in self.tests:
+            self.tests[test.test_id] = test
+        else:
+            _LOGGER.warning(
+                f"Test {test.test_id} already exists, name will be modified"
+            )
+            test_name = self.__calculate_new_name(test.test_id)
+            self.tests[test_name] = test
+
+    def __calculate_new_name(self, name: str) -> str:
+        return name + "__" + str(len(self.tests))
+
+    def __iter__(self):
+        return iter(self.tests.values())
+
+    def get_ids(self) -> Iterator[str]:
+        return iter(self.tests)

spl2_testing_framework/tools/utils.py

@@ -0,0 +1,52 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import pathlib
+import re
+from functools import lru_cache
+
+
+def get_framework_root() -> pathlib.Path:
+    framework_root = pathlib.Path(__file__).parents[1]
+    return framework_root
+
+
+def _make_functions_visible_for_testing(code_module_path):
+    code_module = code_module_path.read_text(encoding="utf-8")
+    functions = re.findall(r"^function\s+([^(]+)\(", code_module, flags=re.M)
+    for function in functions:
+        if not re.search(r"export\s" + function, code_module):
+            code_module = re.sub(
+                r"^function\s+" + function,
+                f"export function {function}",
+                code_module,
+                flags=re.M,
+            )
+    return code_module
+
+
+@lru_cache(None)
+def read_assertions_module():
+    root_folder = get_framework_root()
+    assertions_file = root_folder / "spl2_utils" / "assertions.spl2"
+    return assertions_file.read_text(encoding="utf-8")
+
+
+@lru_cache(None)
+def read_commands_modules():
+    root_folder = get_framework_root()
+    commands_module = root_folder / "spl2_utils" / "logs_to_metrics.spl2"
+    return commands_module.read_text(encoding="utf-8")