spl2-testing-framework 1.0.0__py3-none-any.whl

spl2_testing_framework/tools/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

spl2_testing_framework/tools/jobs/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

spl2_testing_framework/tools/jobs/cli_job.py

@@ -0,0 +1,72 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from .job import Job
+
+
+class CLIAssertionJob(Job):
+    """A job that is executed on CLI in unit tests."""
+
+    def create(self) -> "CLIAssertionJob":
+        """Creates a job. To run the job use run_job() method."""
+
+        cli = "spl2-processor-cli"
+        command = [
+            cli,
+            "test",
+            "-s",
+            self.test_name,
+        ]
+
+        input_data = "\n\n".join(
+            [
+                (  # for assertion tests we don't add assertions module
+                    self.assertions_module
+                    if not self.code_module_name == "assertions"
+                    else ""
+                ),
+                self.commands_module,
+                self.code_module_content,
+                self.test_module,
+            ]
+        )
+        self.job_content = {"command": command, "input_data": input_data}
+        return self
+
+
+class CLISimpleJob(Job):
+    """A job that is executed on CLI in box tests."""
+
+    def __init__(
+        self, source: str, code_module_name: str, code_module_content, test_name
+    ):
+        code_module_content += f"\n$source = from {str(source)};"
+        super().__init__("", code_module_name, code_module_content, test_name)
+
+    def create(self) -> "CLISimpleJob":
+        """Creates a job. To run the job use run_job() method."""
+
+        cli = "spl2-processor-cli"
+        command = [cli, "test", "-s", "pipeline"]
+
+        input_data = "\n\n".join(
+            [
+                self.commands_module,
+                self.code_module_content,
+            ]
+        )
+        self.job_content = {"command": command, "input_data": input_data}
+        return self

spl2_testing_framework/tools/jobs/cloud_job.py

@@ -0,0 +1,107 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from requests import Response
+
+from .job import Job
+
+
+class CloudJob(Job):
+    """A job that is executed on the cloud tenant in unit tests."""
+
+    def create(self):
+        """Creates a job. To run the job use run_job() method."""
+        self.job_content = {
+            "wipModules": {
+                "gdi.addons.spl2_content.dmx_utilities.assertions": {
+                    "name": "assertions",
+                    "definition": str(self.assertions_module),
+                    "namespace": "/gdi.addons.spl2_content.dmx_utilities.assertions",
+                },
+                self.code_module_name: {
+                    "name": self.code_module_name,
+                    "definition": str(self.code_module_content),
+                },
+            },
+            "module": self.test_module,
+            "namespace": "",
+            "queryParameters": {
+                "defaults": {
+                    "earliest": "-30m@m",
+                    "latest": "now",
+                    "profile": "ingestProcessor",
+                    "collectFieldSummary": True,
+                    "collectTimeBuckets": True,
+                    "extractFields": "all",
+                    "enablePreview": True,
+                    "allowSideEffects": False,
+                },
+                self.test_name: {"saveAllResults": True},
+            },
+        }
+
+        return self
+
+    def _assign_ids_from_response(self, response: Response, metrics=False) -> None:
+        if not response.ok:
+            raise Exception(response.status_code, response.content)
+
+        self.ids[self.test_name] = response.json()["queryParameters"][self.test_name][
+            "sid"
+        ]
+
+
+class CloudSimpleJob(Job):
+    """A job that is executed on the cloud tenant in box tests."""
+
+    def __init__(
+        self, source: str, code_module_name: str, code_module_content, test_name
+    ):
+        code_module_content += f"\n$source = from {str(source)};"
+        super().__init__("", code_module_name, code_module_content, test_name)
+
+    def create(self) -> "CloudSimpleJob":
+        """Creates a job. To run the job use run_job() method."""
+        self.job_content = {
+            "wipModules": {},
+            "module": self.code_module_content,
+            "namespace": "shared.pipelines",
+            "queryParameters": {
+                "defaults": {
+                    "earliest": "-30m@m",
+                    "latest": "now",
+                    "profile": "ingestProcessor",
+                    "collectFieldSummary": True,
+                    "collectTimeBuckets": True,
+                    "extractFields": "all",
+                    "enablePreview": True,
+                    "allowSideEffects": False,
+                },
+                "pipeline": {"saveAllResults": False},
+            },
+        }
+
+        return self
+
+    def _assign_ids_from_response(self, response: Response, metrics=False) -> None:
+        if not response.ok:
+            raise Exception(response.status_code, response.content)
+
+        for metric in ["destination", "metrics_destination"]:
+            try:
+                self.ids[metric] = response.json()["queryParameters"][metric]["sid"]
+            except KeyError:
+                pass

spl2_testing_framework/tools/jobs/job.py

@@ -0,0 +1,65 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from requests import Response
+
+from ..utils import read_assertions_module, read_commands_modules
+
+
+class Job:
+    """A job that can be executed in the testing framework.
+    Every pipeline that is tested in the framework is represented by a job,
+    It contains tested code, test code, some helper code modules (assertions and commands)
+    and job results"""
+
+    def __init__(
+        self,
+        test_content: str,
+        code_module_name: str,
+        code_module_content: str,
+        test_name: str,
+    ):
+        self._result = {}
+        self._ids = {}
+        self.assertions_module = read_assertions_module()
+        self.commands_module = read_commands_modules()
+        self.job_content = None
+
+        self.test_module = f"import * from {code_module_name}\n{test_content}"
+        self.code_module_name = code_module_name
+        self.code_module_content = code_module_content
+        self.test_name = test_name
+
+    @property
+    def ids(self) -> dict:
+        """IDs of jobs run on the Splunk server, IP/EP or CLI"""
+        return self._ids
+
+    @ids.setter
+    def ids(self, value: dict):
+        self._ids = value
+
+    def _assign_ids_from_response(self, response: Response):
+        raise NotImplementedError()
+
+    @property
+    def result(self) -> dict:
+        return self._result
+
+    @result.setter
+    def result(self, value: dict):
+        """Results of jobs run on the Splunk server, IP/EP or CLI"""
+        self._result = value

spl2_testing_framework/tools/jobs/splunk_job.py

@@ -0,0 +1,112 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from requests import Response
+
+from .job import Job
+
+
+class SplunkAssertionJob(Job):
+    """A job that is executed on the Splunk server in unit tests."""
+
+    def create(self) -> "SplunkAssertionJob":
+        """Creates a job. To run the job use run_job() method."""
+        self.job_content = {
+            "wipModules": {
+                "assertions": {
+                    "name": "assertions",
+                    "definition": str(self.assertions_module),
+                    "namespace": "/gdi.addons.spl2_content.dmx_utilities.assertions",
+                },
+                "commands": {
+                    "name": "commands",
+                    "definition": str(self.commands_module),
+                    "namespace": "commands",
+                },
+                self.code_module_name: {
+                    "name": self.code_module_name,
+                    "definition": str(self.code_module_content),
+                },
+            },
+            "module": self.test_module,
+            "namespace": "",
+            "queryParameters": {
+                "defaults": {
+                    "earliest": "-30m@m",
+                    "latest": "now",
+                    "profile": "ingestProcessor",
+                    "collectFieldSummary": True,
+                    "collectTimeBuckets": True,
+                    "extractFields": "all",
+                    "enablePreview": True,
+                    "allowSideEffects": False,
+                },
+                self.test_name: {"saveAllResults": True},
+            },
+        }
+
+        return self
+
+    def _assign_ids_from_response(self, response: Response) -> None:
+        if not response.ok:
+            raise Exception(response.status_code, response.content)
+
+        self.ids = {x["name"]: x["sid"] for x in response.json() if x["sid"]}
+
+
+class SplunkSimpleJob(Job):
+    """A job that is executed on the Splunk server in box tests."""
+
+    def __init__(
+        self, source: str, code_module_name: str, code_module_content, test_name
+    ):
+        code_module_content += f"\n$source = from {str(source)};"
+        super().__init__("", code_module_name, code_module_content, test_name)
+
+    def create(self) -> "SplunkSimpleJob":
+        """Creates a job. To run the job use run_job() method."""
+        self.job_content = {
+            "wipModules": {
+                "commands": {
+                    "name": "commands",
+                    "definition": str(self.commands_module),
+                    "namespace": "commands",
+                },
+            },
+            "module": self.code_module_content,
+            "namespace": "",
+            "queryParameters": {
+                "defaults": {
+                    "earliest": "-30m@m",
+                    "latest": "now",
+                    "profile": None,
+                    "collectFieldSummary": True,
+                    "collectTimeBuckets": True,
+                    "extractFields": "all",
+                    "enablePreview": True,
+                    "allowSideEffects": False,
+                },
+                "pipeline": {"saveAllResults": False},
+            },
+        }
+
+        return self
+
+    def _assign_ids_from_response(self, response: Response, metrics=False) -> None:
+        if not response.ok:
+            raise Exception(response.status_code, response.content)
+
+        self.ids = {x["name"]: x["sid"] for x in response.json() if x["sid"]}

spl2_testing_framework/tools/performance.py

@@ -0,0 +1,252 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import logging
+import os
+import re
+from copy import copy
+from datetime import datetime
+from itertools import groupby
+from typing import Tuple, List, Dict
+
+from spl2_testing_framework.tools.test_types import Test
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class PerformanceCheck:
+    """
+    Class responsible for adding timestamps and validating performance of SPL2 pipelines
+    For basic usecase (_performance_check_type == "time") it's showing the time of spl2 pipeline execution
+    For _performance_check_type == "detailed_time" it's printing pipeline code with added execution time for every
+        line, which consists code statement. The result of "detailed_time" execution is also saved to the text file.
+    """
+
+    OUTPUT_DIR = "PERF_CHECK"
+    PIPELINE_PATTERN = re.compile(
+        r"(?P<fullmatch>"
+        r"^\s*?\$pipeline\s*?=\s*?\|?\s*?from\s*?\$source\s*?"  # first line of pipeline
+        r"(.|\n)*?"  # anything in between
+        r"(\|\s*?into\s*?\$destination)"  # last line of pipeline definition
+        r")",
+        flags=re.M | re.VERBOSE,
+    )
+
+    FUNCTION_PATTERN = re.compile(
+        r"^(?P<fullmatch>"
+        r"(export\s+)?function\s+(?P<f_name>\S+)\((?P<source>\S+):"  # function header
+        r".*{\n\s+return\s+(\|\s+)?from\s+(?P=source)\n*"  # return statement
+        r"(?P<statements>(.|\n)*?)"  # anything before closing bracket
+        r"(?P<closing_bracket>^})"  # closing bracket [ASSUMED IT'S IN THE NEW LINE]
+        r")",
+        flags=re.M | re.VERBOSE,
+    )
+
+    STATEMENT_PATTERN = re.compile(r"^(?P<lx>\s*?\|.*)$", re.M)
+
+    def __init__(self, test: Test):
+        self._test = test
+        self._performance_check_type = test.meta.get("performance", None)
+        self.code_module = None
+
+    def apply_code_transformations(self, code_module):
+        """Apply code transformations necessary for measuring performance.
+        Depending on perfromance_check_type (assigned to instance) it's adding code
+        for calculating timestamps to spl2 pipeline"""
+        if self._performance_check_type == "time":
+            code_module = self._performance_time_of_execution_(code_module)
+        elif self._performance_check_type == "detailed_time":
+            code_module = self._performance_timestamps(code_module)
+
+        self.code_module = code_module
+        return code_module
+
+    def check_performance(self, output_result):
+        """Check performance basing on timestamps in the output.
+        Depending on performance_check_type (assigned to instance) it's calculating time of execution
+        of spl2 pipeline or injecting timestamp values into the pipeline code"""
+        if self._performance_check_type == "time":
+            return self._check_time_of_execution(output_result)
+        elif self._performance_check_type == "detailed_time":
+            return self._check_profiling(output_result)
+
+    def _check_time_of_execution(self, output_result):
+        """Log time of execution to stdout and cleanup results."""
+        _LOGGER.info("Checking time of execution... ")
+
+        for event_no, single_output in enumerate(output_result):
+            time_of_execution = single_output.get("_timestamp_perf_exec", None)
+            _LOGGER.info(
+                f"Time of pipeline execution for event no {event_no}: {float(time_of_execution) * 1000}ms"
+            )
+
+        self._cleanup_result(output_result)
+
+    @staticmethod
+    def __find_group(timestamp):
+        """Finds a group of timestamps basing on function name, as their name consists it"""
+        m = re.search(r"_timestamp_(.+?)_\d{3}", timestamp[0])
+        return m.group(1)
+
+    @staticmethod
+    def __calculate_diff_for_timestamps(timestamps: List[List]) -> List[Tuple]:
+        """Calculate a time diff between consecutive timestamps and the relative"""
+        diffs = []
+        ts = sorted(
+            timestamps, key=lambda x: x[0]
+        )  # sort by name, to have the order correct
+        time_of_func_exec = ts[-1][1] - ts[0][1]
+        for start, end in zip(ts[:-1], ts[1:]):
+            diff: float = end[1] - start[1]
+            relative_time: float = diff / time_of_func_exec
+            diffs.append((start[0], diff, relative_time))
+
+        return diffs
+
+    def _check_profiling(self, output_result: List[Dict[str, str]]) -> List:
+        """Add timestamp values to pipeline code and cleanup results.
+        Timestamps are modified to be relative to 0 - which is the start of the pipeline.
+        """
+        _LOGGER.info("Checking profiling results... ")
+        outputs = []
+        for event_no, single_output in enumerate(output_result):
+            perf = [
+                [field, float(value)]
+                for field, value in single_output.items()
+                if field.startswith("_timestamp_")
+            ]
+
+            calculated_diffs = []
+            for _, v in groupby(perf, self.__find_group):
+                calculated_diffs.extend(self.__calculate_diff_for_timestamps(list(v)))
+
+            output = copy(self.code_module)
+
+            for timestamp_name, value, relative in calculated_diffs:
+                output = re.sub(
+                    rf"\| eval {timestamp_name}=time\(\)",
+                    f">>> {(value * 1000):.6f}ms [{relative:.2%}] <<< ",
+                    output,
+                )
+
+            output = re.sub(r"\| eval _timestamp_\w+_\d{3}=time\(\)", r"", output)
+
+            _LOGGER.info(output)
+
+            self._save_output(output, event_no)
+            outputs.append(output)
+
+        self._cleanup_result(output_result)
+        return outputs
+
+    def _save_output(self, output: str, event_number: int) -> None:
+        """Save output to text file"""
+        os.makedirs(self.OUTPUT_DIR, exist_ok=True)
+        filename = f"{self.OUTPUT_DIR}/performance_{self._test.name}_{event_number}_{datetime.now()}.txt"
+
+        with open(filename, "w") as f:
+            f.write(output)
+
+    @staticmethod
+    def _cleanup_result(output_result: List) -> List:
+        """Remove all timestamps from output not to brake assertions"""
+        for single_result in output_result:
+            for k in list(single_result.keys()):
+                if k.startswith("_timestamp_perf"):
+                    del single_result[k]
+
+        return output_result
+
+    @staticmethod
+    def _performance_time_of_execution_(code_module: str) -> str:
+        """Injects 3 timestamps to code - for beginning and end of pipeline execution
+        and also the difference of these 2.
+        """
+        code_module = re.sub(
+            r"(^\s*?\$pipeline\s*?=\s*?\|?\s*?from\s*?\$source\s*?)",
+            r"\1 | eval _timestamp_perf_start=time()",
+            code_module,
+            flags=re.M,
+        )
+
+        code_module = re.sub(
+            r"(\|\s*?into\s*?\$destination)",
+            r"| eval _timestamp_perf_stop = time()"
+            r"| eval _timestamp_perf_exec = _timestamp_perf_stop - _timestamp_perf_start \1",
+            code_module,
+            flags=re.M,
+        )
+
+        return code_module
+
+    def _performance_timestamps(self, code_module: str) -> str:
+        """Injects time of execution just before every command (In practise in every line which starts from "|")"""
+
+        functions = self.__parse_functions(code_module)
+        pipeline = self.__parse_pipeline(code_module)
+
+        functions.append(pipeline)
+
+        for f in functions:
+            new_function = self.__add_timestamps_to_function(f)
+            code_module = code_module.replace(f["fullmatch"], new_function)
+
+        return code_module
+
+    def __parse_functions(self, code_module: str) -> List[Dict]:
+        """Parse all functions from code module. The finding contains groups necessary for parsing it
+        and adding timestamps"""
+        functions = [m.groupdict() for m in self.FUNCTION_PATTERN.finditer(code_module)]
+        return functions
+
+    def __parse_pipeline(self, code_module: str) -> Dict:
+        """Parse pipeline definition from code module. The finding contains groups necessary for parsing it
+        and adding timestamps"""
+        pipeline = [m.groupdict() for m in self.PIPELINE_PATTERN.finditer(code_module)][
+            0
+        ]
+        pipeline[
+            "f_name"
+        ] = "__main_pipeline__"  # just to make it working with function timestamp parser
+        pipeline["closing_bracket"] = "NOTHING_THERE"  # ^^^
+
+        return pipeline
+
+    def __add_timestamps_to_function(self, f: Dict) -> str:
+        """Add timestamps to all functions (and also correctly parsed pipeline definition)
+        Timestamps are parametrized using function name and numbered consecutively"""
+        new_function = self.STATEMENT_PATTERN.sub(
+            rf"| eval _time_{f['f_name']}_placeholder=time() \g<lx>", f["fullmatch"]
+        )
+
+        new_function = re.sub(
+            rf"^{f['closing_bracket']}",
+            rf"| eval _time_{f['f_name']}_placeholder=time() {f['closing_bracket']}",
+            new_function,
+            flags=re.M,
+        )
+
+        func_timestamps = re.findall(rf"_time_{f['f_name']}_placeholder", new_function)
+
+        for n, timestamp in enumerate(func_timestamps):
+            new_function = re.sub(
+                timestamp,
+                rf"_timestamp_perf{f['f_name']}_{n:03d}",
+                new_function,
+                count=1,
+            )
+
+        return new_function