spl2-testing-framework 1.0.0b1__py3-none-any.whl

spl2_testing_framework/tools/results.py

@@ -0,0 +1,135 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import json
+from collections import UserDict
+from datetime import datetime, timezone
+from typing import Any
+
+
+class Results(UserDict):
+    """Class responsible for parsing and storing results from Splunk, IP and CLI.
+    It's necessary to parse nested jsons and other python types to strings in order to compare them
+    """
+
+    def __init__(self, dictionary: dict = None, **kwargs):
+        super().__init__(dictionary, **kwargs)
+        self.data = self._cast_types(self.data)
+        self.data = self._remove_all_string_nulls_from_dicts(
+            self.data
+        )  # TODO -> remove
+
+        self._remove_timestamps_if_necessary()
+
+    @staticmethod
+    def _cast_types(data: Any):
+        """Casts everything into dicts / lists / strings.
+        This is necessary because Splunk, IP and CLI often return different types of data.
+        """
+        if isinstance(data, str) and (data.startswith("[") or data.startswith("{")):
+            try:
+                data = json.loads(data)
+            except Exception:
+                pass  # If it fails, it's not a json, so it's just a normal string
+
+        if isinstance(data, list):
+            data = [Results._cast_types(item) for item in data]
+        if isinstance(data, dict):
+            for key, value in data.items():
+                data[key] = Results._cast_types(value)
+                if key == "_time":
+                    data[key] = Results._convert_timestamp(value)
+        if isinstance(data, (int, float)):
+            data = str(data)
+
+        return data
+
+    @staticmethod
+    def _convert_timestamp(timestamp):
+        """We need to convert timestamp, as there are different formats returned by cloud / CLI / splunk"""
+        try:
+            ts = datetime.fromisoformat(timestamp)
+        except ValueError:
+            ts = datetime.fromtimestamp(float(timestamp), timezone.utc)
+
+        return ts.isoformat()
+
+    def __hash__(self):
+        return hash(json.dumps(self.data, sort_keys=True))
+
+    def _remove_timestamps_if_necessary(self):
+        """Compatibility workaround
+        Because IP is adding _has_time field and automatically adds timestamp=now() if there is no timestamp,
+        but CLI doesn't, we need to remove it from comparison.
+        """
+
+        if self.data.get("_has_time", "1") == "0":
+            self.data.pop("_has_time", None)
+            self.data.pop("_time", None)
+        else:
+            self.data.pop("_has_time", None)
+
+    @staticmethod
+    def _remove_all_string_nulls_from_dicts(
+        data,
+    ):  # TODO -> remove, it's just a workaround for different behaviour of splunk, IP and CLI
+        """Because of different behaviour of Splunk, IP and CLI, we need to remove all "null" values,
+        which are returned by CLI
+        """
+        for k, v in list(
+            data.items()
+        ):  # list(...) is to avoid changing dict size during iteration
+            if v == "null":
+                del data[k]
+            if isinstance(v, dict):
+                Results._remove_all_string_nulls_from_dicts(v)
+            if isinstance(v, list):
+                for item in v[:]:
+                    if isinstance(item, dict):
+                        Results._remove_all_string_nulls_from_dicts(item)
+
+        return data
+
+    def __lt__(self, other):
+        if "_raw" in self.data:
+            s = self.data.get("_raw")
+            o = other.data.get("_raw")
+        else:
+            s = self.data.get("name")  # just a "workaround" for log_to_metrics func
+            o = other.data.get("name")
+        return json.dumps(s, sort_keys=True) < json.dumps(o, sort_keys=True)
+
+
+class Metrics(Results):
+    """Class responsible for storing metrics from Splunk, IP and CLI.
+    Its main responsibility is to remove  metric_ prefix from keys in order to compare
+    actual metric results with expected ones.
+    """
+
+    def __init__(self, dictionary: dict = None, **kwargs):
+        if not dictionary:
+            dictionary = {}
+        modified_dict = self._remove_metric_prefix(dictionary)
+        super().__init__(modified_dict, **kwargs)
+
+    @staticmethod
+    def _remove_metric_prefix(dictionary: dict = None):
+        new_data = {}
+        for key, value in dictionary.items():
+            if key.startswith("metric_"):
+                new_key = key[7:]
+                new_data[new_key] = value
+        return new_data

spl2_testing_framework/tools/search_clients/__init__.py

@@ -0,0 +1,28 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

spl2_testing_framework/tools/search_clients/cli_search_client.py

@@ -0,0 +1,87 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+
+import json
+import logging
+import subprocess
+from ..jobs.cli_job import CLIAssertionJob, CLISimpleJob
+from ..jobs.job import Job
+from .search_client import SearchClient
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class CLISearchClient(SearchClient):
+    """Search client responsible for running jobs on CLI."""
+
+    jobs_dict = {}
+
+    @staticmethod
+    def create_job(
+        test_module: str, code_file_name: str, code_to_test: str, test_statement: str
+    ) -> Job:
+        """Creates a job for unit test run."""
+        return CLIAssertionJob(
+            test_module, code_file_name, code_to_test, test_statement
+        ).create()
+
+    @staticmethod
+    def create_simple_job(
+        source: str, code_module_name: str, code_module_content, test_name
+    ) -> Job:
+        """Creates a job for box test run."""
+
+        return CLISimpleJob(
+            source, code_module_name, code_module_content, test_name
+        ).create()
+
+    def run_job(self, job):
+        """Run a job on CLI."""
+        result = subprocess.run(
+            job.job_content["command"],
+            input=job.job_content["input_data"],  # Pass data to stdin
+            text=True,  # Encode input_data as string
+            capture_output=True,
+        )
+
+        if result.stderr:
+            raise RuntimeError(result.stderr)
+
+        job.result = json.loads(result.stdout)
+        return
+
+    def get_job_results(self, job: Job) -> dict:
+        """Get results of a job."""
+        tmp = {  # remove job_ prefix
+            k.replace("job_", ""): v for k, v in job.result.items()
+        }  # TODO for python>3.9 use str.remove_prefix
+
+        results = self._cast_results(tmp)
+        return results

spl2_testing_framework/tools/search_clients/cloud_search_client.py

@@ -0,0 +1,84 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+#
+#
+
+import time
+import requests
+from requests import Response
+
+from ..jobs.job import Job
+from ..jobs.cloud_job import CloudJob, CloudSimpleJob
+from .search_client import HTTPSSearchClient
+
+
+class CloudSearchClient(HTTPSSearchClient):
+    """Search client responsible for running jobs on cloud tenant."""
+
+    DISPATCH_SERVICE = "search/v3alpha1/dispatch"
+    JOBS_SERVICE = "search/v3alpha1/jobs"
+
+    def _get_job_results_url(self, job_id: str) -> str:
+        return f"{self.jobs_url}/{job_id}/results"
+
+    @staticmethod
+    def create_job(
+        test_module: str, code_file_name: str, code_to_test: str, test_statement: str
+    ) -> Job:
+        """Creates a job for unit test run."""
+        return CloudJob(
+            test_module, code_file_name, code_to_test, test_statement
+        ).create()
+
+    @staticmethod
+    def create_simple_job(
+        source: str, code_module_name: str, code_module_content, test_name
+    ) -> Job:
+        """Creates a job for box test run."""
+        return CloudSimpleJob(
+            source, code_module_name, code_module_content, test_name
+        ).create()
+
+    def _wait_for_job_status(self, job_id: str) -> Response:
+        time.sleep(2)
+
+        while True:
+            response = requests.get(
+                self._get_job_details_url(job_id), verify=False, auth=self.auth
+            )
+            if response.status_code != 200:
+                if "GET_PIPELINE_NOT_FOUND" in response.text:
+                    continue
+                raise Exception(response.text)
+            if response.json()["status"] == "failed":
+                raise Exception(f"Job failed, id: {job_id}")
+            if response.json()["status"] == "done":
+                return response
+            time.sleep(1)

spl2_testing_framework/tools/search_clients/search_client.py

@@ -0,0 +1,150 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+#
+#
+
+import json
+import time
+from typing import Dict, Union, List
+
+import requests
+from requests import Response
+
+from ..jobs.job import Job
+from ..results import Results, Metrics
+from abc import ABC, abstractmethod
+
+
+class SearchClient(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def run_job(self, job: Job):
+        pass
+
+    @abstractmethod
+    def get_job_results(self, job: Job) -> Dict[str, Results]:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def create_job(*args, **kwargs):
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def create_simple_job(*args, **kwargs):
+        pass
+
+    @staticmethod
+    def _cast_results(results: dict) -> Dict[str, List[Union[Results, Metrics]]]:
+        """Cast results to Results / Metrics instances"""
+        rs = {}
+        for key, value in results.items():
+            if key == "metrics_destination":
+                rs[key] = [Metrics(x) for x in value]
+            else:
+                rs[key] = [Results(x) for x in value]
+
+        return rs
+
+
+class HTTPSSearchClient(SearchClient):
+    """Base class for HTTP based search clients."""
+
+    @property
+    @abstractmethod
+    def DISPATCH_SERVICE(self):
+        pass
+
+    @property
+    @abstractmethod
+    def JOBS_SERVICE(self):
+        pass
+
+    def __init__(self, url: str, auth=None):
+        super().__init__()
+        self._base_url = url
+        self.auth = auth
+
+    def run_job(self, job: Job) -> Response:
+        """Run a job."""
+        response = requests.post(
+            self.dispatch_url,
+            data=json.dumps(job.job_content),
+            verify=False,
+            auth=self.auth,
+        )
+
+        job._assign_ids_from_response(response)
+
+        return response
+
+    @property
+    def url(self) -> str:
+        """Base URL of a Splunk instance / cloud tenant."""
+        return self._base_url
+
+    @property
+    def dispatch_url(self) -> str:
+        """URL of the dispatch service."""
+        return f"{self.url}/{self.DISPATCH_SERVICE}"
+
+    @property
+    def jobs_url(self) -> str:
+        """URL of the job service."""
+        return f"{self.url}/{self.JOBS_SERVICE}"
+
+    def _get_job_details_url(self, job_id: str) -> str:
+        return f"{self.jobs_url}/{job_id}"
+
+    def get_job_results(self, job: Job) -> Dict[str, Union[Results, Metrics]]:
+        """Get results of a job."""
+        for job_name, job_id in job.ids.items():
+            self._wait_for_job_status(job_id)
+
+            job_url = self._get_job_results_url(job_id)
+
+            response = requests.get(job_url, verify=False, auth=self.auth)
+            while response.status_code == 204:
+                time.sleep(1)
+                response = requests.get(job_url, verify=False, auth=self.auth)
+            if response.status_code != 200:
+                raise Exception(response.status_code, response.content)
+
+            job.result[job_name] = response.json()["results"]
+
+        return self._cast_results(job.result)
+
+    @staticmethod
+    @abstractmethod
+    def create_job(*args, **kwargs):
+        pass

spl2_testing_framework/tools/search_clients/splunk_search_client.py

@@ -0,0 +1,81 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+#
+#
+
+import time
+import requests
+from requests import Response
+
+from ..jobs.job import Job
+from ..jobs.splunk_job import SplunkAssertionJob, SplunkSimpleJob
+from .search_client import HTTPSSearchClient
+
+
+class SplunkSearchClient(HTTPSSearchClient):
+    """Search client responsible for running jobs on Splunk instance"""
+
+    DISPATCH_SERVICE = "servicesNS/admin/-/search/spl2-module-dispatch"
+    JOBS_SERVICE = "servicesNS/admin/-/search/jobs"
+
+    @staticmethod
+    def create_job(
+        test_module: str, code_file_name: str, code_to_test: str, test_statement: str
+    ) -> Job:
+        """Creates a job for unit test run."""
+        return SplunkAssertionJob(
+            test_module, code_file_name, code_to_test, test_statement
+        ).create()
+
+    @staticmethod
+    def create_simple_job(
+        source: str, code_module_name: str, code_module_content, test_name
+    ) -> Job:
+        """Creates a job for box test run."""
+        return SplunkSimpleJob(
+            source, code_module_name, code_module_content, test_name
+        ).create()
+
+    def _get_job_results_url(self, job_id: str) -> str:
+        return f"{self.jobs_url}/{job_id}/results?output_mode=json"
+
+    def _wait_for_job_status(self, job_id: str) -> Response:
+        job_url = self._get_job_details_url(job_id)
+
+        counter = 0
+        while True:
+            response = requests.get(job_url, verify=False, auth=self.auth)
+            if response.status_code != 200:
+                counter += 1
+            if response.status_code == 200:
+                return response
+            time.sleep(1)
+            if counter == 5:
+                raise Exception(response.text)