spl2-testing-framework 1.0.0__py3-none-any.whl

spl2_testing_framework/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

spl2_testing_framework/conftest.py

@@ -0,0 +1,217 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import json
+import os
+
+import pytest
+
+pytest.register_assert_rewrite("spl2_testing_framework")
+
+from requests.auth import AuthBase, HTTPBasicAuth
+
+from spl2_testing_framework.tools.search_clients.cli_search_client import (
+    CLISearchClient,
+)
+from spl2_testing_framework.tools.search_clients.cloud_search_client import (
+    CloudSearchClient,
+)
+from spl2_testing_framework.tools.search_clients.splunk_search_client import (
+    SplunkSearchClient,
+)
+from spl2_testing_framework.tools.spl2test_runner import SPL2TestRunner
+from spl2_testing_framework.tools.test_discovery import (
+    BoxTestDiscovery,
+    SamplesNotFoundException,
+    SingleSPL2Discovery,
+    UTDiscovery,
+)
+
+UNIT_TEST = "unit_test"
+BOX_TEST = "box_test"
+SINGLE_SPL2_FILE = "single_spl2_file"
+
+import logging
+
+from .logger_manager import setup_logging
+
+setup_logging()
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def pytest_addoption(parser):
+    parser.addoption("--type", action="store", default="local")
+    parser.addoption("--test_dir", action="store", default=".")
+    parser.addoption(
+        "--performance_check",
+        action="store",
+        default="no",
+        choices=["no", "time", "detailed_time"],
+    )
+    parser.addoption("--template_file", action="store", default=None)
+    parser.addoption("--sample_file", action="store", default=None)
+    parser.addoption("--sample_delimiter", action="store", default="\n")
+
+
+def pytest_generate_tests(metafunc):
+    """This function is used for dynamic generation of tests based on the test files in the directory specified by the
+    --test_dir option"""
+    test_dir = metafunc.config.getoption("test_dir")
+    performance_check = metafunc.config.getoption("performance_check")
+
+    if BOX_TEST in metafunc.fixturenames:
+        _LOGGER.info("Collecting Box Tests")
+        module_tests = BoxTestDiscovery(test_dir, performance_check)
+        module_tests.discover_tests()
+        metafunc.parametrize(
+            "%s" % BOX_TEST, module_tests.tests, ids=module_tests.tests.get_ids()
+        )
+
+    if SINGLE_SPL2_FILE in metafunc.fixturenames:
+        _LOGGER.info("Collecting the single spl2 file")
+        template_file = metafunc.config.getoption("template_file")
+        if template_file is None:
+            pytest.skip("No template file found to execute. Exiting..")
+        sample_file = metafunc.config.getoption("sample_file")
+        sample_delimiter = metafunc.config.getoption("sample_delimiter")
+        spl2_pipelines = SingleSPL2Discovery(
+            test_dir,
+            template_file,
+            sample_file,
+            sample_delimiter,
+            performance_check=performance_check,
+        )
+        try:
+            spl2_pipelines.discover_tests()
+        except SamplesNotFoundException as err:
+            pytest.exit(err)
+        metafunc.parametrize(
+            "%s" % SINGLE_SPL2_FILE,
+            spl2_pipelines.tests,
+            ids=spl2_pipelines.tests.get_ids(),
+        )
+
+    if UNIT_TEST in metafunc.fixturenames:
+        _LOGGER.info("Collecting Unit Tests")
+        unit_tests = UTDiscovery(test_dir)
+        unit_tests.discover_tests()
+        metafunc.parametrize(
+            UNIT_TEST, unit_tests.tests, ids=unit_tests.tests.get_ids()
+        )
+
+
+@pytest.fixture()
+def spl2_test_runner(
+    request, config, basic_authentication, bearer_token_authentication
+):
+    """This fixture is used to create a test runner object based on the test type specified by the --type option
+    3 types of tests are supported: 'splunk', 'cloud', 'cli'
+    """
+    test_type = request.config.option.type.lower()
+    if test_type == "splunk":
+        url = f"{config['host']}:{config['port']}"
+        search_client = SplunkSearchClient(url=url, auth=basic_authentication)
+    elif test_type == "cli":
+        search_client = CLISearchClient()
+    elif test_type == "cloud":
+        url = f"https://{config['tenant']}.api.{config['cloud_instance']}/{config['tenant']}"
+        search_client = CloudSearchClient(url=url, auth=bearer_token_authentication)
+    else:
+        raise Exception(
+            "Unknown test type: {}. Supported tests types: 'splunk', 'cloud', 'cli'".format(
+                test_type
+            )
+        )
+
+    x = SPL2TestRunner(search_client)
+    return x
+
+
+@pytest.fixture(scope="session")
+def config():
+    """This fixture is used to read the configuration from the environment variables or from the spl2_test_config.json
+    Configuration set using environment variables will be overwritten by settings from spl2_test_config.json file,
+    however empty values in spl2_test_config.json will be ignored.
+    """
+    host = os.environ.get("SPL2_TF_HOST", None)
+    port = os.environ.get("SPL2_TF_PORT", None)
+    user = os.environ.get("SPL2_TF_USER", None)
+    password = os.environ.get("SPL2_TF_PASSWORD", None)
+    bearer_token = os.environ.get("SPL2_TF_BEARER_TOKEN", None)
+    tenant = os.environ.get("SPL2_TF_TENANT", None)
+    cloud_instance = os.environ.get("SPL2_TF_CLOUD_INSTANCE", None)
+
+    try:
+        with open("spl2_test_config.json", "rb") as f:
+            conf_file = json.load(fp=f)
+    except FileNotFoundError:
+        from spl2_testing_framework.tools.utils import get_framework_root
+
+        with open(f"{get_framework_root()}/spl2_test_config.json", "rb") as f:
+            conf_file = json.load(fp=f)
+
+    if conf_file.get("user", None):
+        user = conf_file["user"]
+
+    if conf_file.get("host", None):
+        host = conf_file["host"]
+
+    if conf_file.get("port", None):
+        port = conf_file["port"]
+
+    if conf_file.get("password", None):
+        password = conf_file["password"]
+
+    if conf_file.get("bearer_token", None):
+        bearer_token = conf_file["bearer_token"]
+
+    if conf_file.get("tenant", None):
+        tenant = conf_file["tenant"]
+
+    if conf_file.get("cloud_instance", None):
+        cloud_instance = conf_file["cloud_instance"]
+
+    return {
+        "host": host,
+        "port": port,
+        "user": user,
+        "password": password,
+        "bearer_token": bearer_token,
+        "tenant": tenant,
+        "cloud_instance": cloud_instance,
+    }
+
+
+@pytest.fixture(scope="session")
+def basic_authentication(config):
+    return HTTPBasicAuth(config.get("user", ""), config.get("password", ""))
+
+
+@pytest.fixture(scope="session")
+def bearer_token_authentication(config):
+    class HTTPBearerTokenAuth(AuthBase):
+        """Attaches HTTP Basic Authentication to the given Request object."""
+
+        def __init__(self, token):
+            self.token = token
+
+        def __call__(self, r):
+            r.headers["Authorization"] = f"Bearer {self.token}"
+            return r
+
+    token = config.get("bearer_token", "")
+    return HTTPBearerTokenAuth(token)

spl2_testing_framework/launch_remotely_spl2.py

@@ -0,0 +1,85 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import argparse
+import json
+
+from spl2_testing_framework.tools.modules_reader import get_template_file
+from spl2_testing_framework.tools.payload_functions import create_pipeline_job
+from spl2_testing_framework.tools.cloud_search_client import CloudSearchClient
+
+
+class Spl2Runner:
+    def _run_job_and_get_response(self, job):
+        response = self._search_client.run_job(job)
+        if response.status_code == 401:
+            print(
+                f"Unauthorised, may need to refresh token. Additional info: {response.text}"
+            )
+            exit(-1)
+        if response.status_code != 201:
+            print(f"Message: {response.text}. Status code: {response.status_code}.")
+        return response
+
+    def main(self):
+        parser = argparse.ArgumentParser("spl2_launcher")
+        parser.add_argument(
+            "spl2file",
+            help="Name of file to launch like: `pan_traffic_reduction.spl2`",
+            type=str,
+        )
+        parser.add_argument(
+            "-s",
+            "--source",
+            help="$source definition if it is not in the code.",
+            type=str,
+            default=None,
+        )
+        parser.add_argument(
+            "-e", "--events", help="Events definition txt file.", type=str, default=None
+        )
+        args = parser.parse_args()
+        self._search_client = CloudSearchClient()
+        template_module = get_template_file(args.spl2file).read_text(encoding="utf-8")
+        if args.source is not None:
+            template_module = template_module + f"\n$source = from {args.source};"
+        if args.events is not None:
+            events = list(
+                filter(lambda ev: len(ev) > 0, open(args.events).read().split("\n"))
+            )
+            template_module = template_module + f"\n$source = from ["
+            for event in events:
+                template_module = template_module + f"{{_raw: {json.dumps(event)} }}"
+                if event != events[-1]:
+                    template_module = template_module + ","
+            template_module = template_module + "];"
+        job = create_pipeline_job(
+            template_module, "ingestProcessor", False, None
+        )  # allways launch in IP profile to use IP runner for tests
+        response = self._run_job_and_get_response(job)
+        # TODO check both destinations
+        destination_job_sid = response.json()["queryParameters"]["destination"]["sid"]
+
+        self._search_client.wait_for_job_status(destination_job_sid)
+
+        job_result = self._search_client.get_job_results(destination_job_sid)
+
+        print(json.dumps(job_result, indent=2))
+
+
+if __name__ == "__main__":
+    spl2runner = Spl2Runner()
+    spl2runner.main()

spl2_testing_framework/logger_manager.py

@@ -0,0 +1,26 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import logging
+
+
+def setup_logging():
+    logging.basicConfig(
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=[
+            logging.FileHandler("spl2_testing_framework.log"),
+        ],
+    )

spl2_testing_framework/pytest.ini.sample

@@ -0,0 +1,16 @@
+[pytest]
+
+log_cli = true
+log-cli-level=INFO
+log_level = INFO
+
+
+addopts =
+    -rA
+    --verbose
+    -k <single_spl2_file|unit_test|box_test> ; Used to run a specific kind of test
+    --type=<cloud|cli|local> ; Any preferable method can be used to run the tests
+    --test_dir=<test_directory> ; Directory in which your tests and spl2 templates are kept
+    --template_file=<template_file_name> ; Name of the .spl2 file for which you want to get the results. Only used when running single_spl2_test_runner.py
+    --sample_file=<sample_file_name> ; Path to the sample file. Only used when running single_spl2_test_runner.py
+    --sample_delimiter=<sample_file_delimiter> ; Delimiter to separate events in sample_file. Only used when running single_spl2_test_runner.py

spl2_testing_framework/single_spl2_file_runner.py

@@ -0,0 +1,52 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import argparse
+import logging
+import sys
+
+import pytest
+
+from spl2_testing_framework.tools.test_types import SingleSPL2
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def test_single_spl2_file(spl2_test_runner, single_spl2_file: SingleSPL2):
+    _LOGGER.info(f"Executing single SPL2 File: '{single_spl2_file.name}'")
+    _LOGGER.debug(f"Single SPL2 File Details: {single_spl2_file}")
+    spl2_test_runner.run_single_spl2_file(single_spl2_file)
+
+
+def run():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("type")
+
+    arguments, other = parser.parse_known_args()
+
+    args = [__file__, "--type", arguments.type]
+    args.extend(other)
+
+    # INFO log level is required for result printing
+    if "--log-cli-level" not in args:
+        args.append("--log-cli-level")
+        args.append("INFO")
+
+    sys.exit(pytest.main(args))
+
+
+if __name__ == "__main__":
+    run()

spl2_testing_framework/spl2_test_config.json

@@ -0,0 +1,9 @@
+{
+  "host": "",
+  "cloud_instance": "",
+  "tenant": "",
+  "bearer_token": "",
+  "port": "",
+  "user": "",
+  "password": ""
+}

spl2_testing_framework/spl2_utils/assertions.spl2

@@ -0,0 +1,157 @@
+/**
+#  SPDX-FileCopyrightText: 2021 Splunk, Inc. <sales@splunk.com>
+#  SPDX-License-Identifier: LicenseRef-Splunk-8-2021
+ */
+
+/**
+ * Utility function to compare a result with an expected value.
+ * 
+ * @param $source events
+ * @param $expected_value value the test expects
+ * @param $test_field name of field that has the result to check
+ * @param $test_name optional, name the test to make it easier to parse results with multiple tests
+ * @param $expected_field optional, expected value is in this field
+
+ */
+export function expect($source: dataset, $expected_value: any="", $test_field: any=_raw, $test_name: string="test", $expected_field: any=expected): dataset {
+    return from $source
+        | eval _testExpectedField = if(isnotnull($expected_field), $expected_field, $expected_value)
+        | eval _testResult = if($test_field == _testExpectedField, "true", "false")
+        | _testAddResult $test_name _testResult _testExpectedField, $test_field
+        | fields - _testExpectedField, _testResult
+}
+
+/**
+ * Test utility to ensure a given field exists on an event.
+ *
+ * @param $source events
+ * @param $field_name the field name to ensure exists
+ */
+export function expect_field($source: dataset, $field_name: any): dataset {
+    return | from $source
+        | eval _expect = if(isnotnull(_expect), _expect, "true")
+        | eval _original_event_json = tojson()
+        | eval _expectHasField=if(isnotnull(json_extract_exact(_original_event_json, $field_name)), "true", "false")
+        | eval _result=if(_expectHasField == _expect, "true", "false")
+        | _testAddResult name="Has field: ${$field_name}" result=_result expected=_expectHasField received=_expectHasField
+        | fields - _expectHasField, _original_event_json, _result
+}
+
+/**
+ * Test utility to ensure given fields exists on an event.
+ * 
+ * _expect field is not considered
+ *
+ * @param $source events
+ * @param $field_names the field name that are required
+ * @param $delimeter delimeter for the field names
+ */
+export function expect_fields($source: dataset, $field_names: string): dataset {
+    $delimiter = ","
+  return | from $source
+    | eval _expect = if(isnotnull(_expect),_expect, "false")
+    // Convert the original event into json
+    | eval _original_event_json = tojson()
+    // Remove the _expect field
+    | eval _original_event_json=json_delete(_original_event_json, "_expect")
+    // Set the expected fields
+    | eval  _expected_fields = mv_to_json_array(split($field_names, $delimiter))
+    // Test all the expected fields
+    | eval _test_result = json_object("result", "true", "received", ""),
+       _test_result = reduce(
+           _expected_fields,
+           _test_result,
+           ($results, $field) -> if(isnotnull(json_extract_exact(_original_event_json, $field)),
+               json_set($results, "", {"result": json_extract($results, "result"), "received": json_extract_exact($results, "received")+$field+": true\n"}),
+               json_set($results, "", {"result": "false", "received": json_extract($results, "received")+$field+": false\n"})
+           )
+       )
+    | eval _result=if(_test_result.result == _expect, "true", "false")      
+    // Create a test result
+    | _testAddResult name="expect_fields" result=_result expected=_expect received=_test_result.result
+    | fields - _original_event_json, _expected_fields, _test_result, _result
+ }
+
+
+
+/**
+ * Utility to ensure a given field exists on an event.
+ *
+ * @param $source events
+ * @param $field_names comma seperated list of fields to validate
+ * @param $delimeter the delimeter to split the fields list
+ * @param $ignore_testResults ignore the _testResult field if it exists. Useful when chaining tests.
+ */
+export function expect_exact_fields($source: dataset, $field_names: string, $delimiter: string=",", $ignore_testResults: boolean=true): dataset {
+    return | from $source
+       | eval _expect = if(isnotnull(_expect),_expect, "false")
+        // Convert the original event into json
+        | eval _original_event_json = tojson()
+        // Remove the _expect field
+        | eval _original_event_json=json_delete(_original_event_json, "_expect")
+        // Get and sort the names of all the fields in the event into a MV field
+        | eval _original_event_fields = json_keys(_original_event_json)
+        // Get and sort all of the expected fields into a MV field
+        | eval _expected_fields = split($field_names, $delimiter)
+
+        // Match the expected fields against the event fields
+        | eval _test_result = json_object("fields", json_object()),
+            _test_result = reduce(
+                mv_to_json_array(_expected_fields),
+                _test_result,
+                ($results, $field) ->
+                    json_set($results, "fields."+$field,
+                        if(isnotnull(json_extract_exact(_original_event_json, $field)), "present", "missing")
+                    )
+            )
+        // Match event fields against expected fields to find unexpected fields
+        | eval _test_result = reduce(
+                _original_event_fields,
+                _test_result,
+                ($results, $field) ->
+                    json_set($results, "fields."+$field,
+                        if(in($field, _expected_fields), "present", "not expected")
+                    )
+            )
+        // Determine the test result
+        | eval _result = "true", 
+          _result = reduce(
+                json_keys(_test_result.fields),
+                _result,
+                ($results, $field) ->
+                    if(json_extract_exact(_test_result.fields, $field) != "present", "false", _result)
+            )        
+
+        | eval _output=if(_result == _expect, "true", "false")   
+        | _testAddResult name="expect_exact_fields", result=_output, expected=_expect, received=_result
+        | fields - _original_event_json, _original_event_fields, _expected_fields, _result, _test_result, _output
+}
+
+
+// Test result object
+type testResult = {
+    test: string,
+    result: string,
+    expected: any,
+    received: any
+};
+
+/**
+ * Utility function to add a test result object to the test result field.
+ *
+ * @param $source events
+ * @param $name test name
+ * @param $result test result
+ * @param $expected expected result of the test
+ * @param $received the received result of the test
+ */
+function _testAddResult($source: dataset, $name: string, $result: any, $expected: any, $received: any): dataset {
+    return from $source
+        // Create the test result object
+        | eval _testResultObject = json_object("result", $result, "test", $name, "expected", $expected, "received", $received)
+        | eval _testResults = if(isnotnull(_testResults),
+            json_append(_testResults, "results", json_extract(_testResultObject)),
+            json_object("results", json_array(json_extract(_testResultObject)))
+        )
+        | fields - _testResultObject
+}

spl2_testing_framework/spl2_utils/logs_to_metrics.spl2

@@ -0,0 +1,36 @@
+/**
+#  SPDX-FileCopyrightText: 2021 Splunk, Inc. <sales@splunk.com>
+#  SPDX-License-Identifier: LicenseRef-Splunk-8-2021
+ */
+
+type metrict=string where $value in (["gauge", "counter", "cumulative", "histogram"]);
+
+/**
+ * This is a MOCK implementation of logs_to_metrics function. It is meant to be used just for testing.
+ *
+ * @param $source events
+ * @param $time time of the metric
+ * @param $name name of the metric
+ * @param $value value of the metric
+ * @param $metrictype type of the metric
+ * @param $dimensions dimensions of the metric
+ */
+function logs_to_metrics($source, $time:time, $name:string, $value:number, $metrictype:metrict = "gauge", $dimensions:object = {}): dataset {
+	return from $source
+	| eval _has_time=if(isnotnull($time), 1, 0)
+    | eval _time=if(_has_time=1, $time, time())
+    | eval _time=floor(_time / 10) * 10
+	| select _time as _time,
+            $name as metric_name,
+            $value as metric_value,
+            $metrictype as metric_type,
+            $dimensions as dimensions,
+            source as source,
+            sourcetype as sourcetype,
+            index as index,
+            host as host,
+            _has_time as _has_time
+	| fields - _raw
+}
+
+export logs_to_metrics;

spl2_testing_framework/test_runner.py

@@ -0,0 +1,59 @@
+#
+#   Copyright 2025 Splunk Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import argparse
+import sys
+from logging import getLogger
+
+import pytest
+
+from spl2_testing_framework.tools.test_types import BoxTest
+
+_LOGGER = getLogger(__name__)
+
+
+def test_box_tests(spl2_test_runner, box_test: BoxTest):
+    _LOGGER.info("Running box test: %s from \n %s", box_test.name, box_test.module_path)
+    _LOGGER.debug("Test details: %s", box_test)
+
+    spl2_test_runner.run_box_test(box_test)
+
+
+def test_spl2_unit_tests(spl2_test_runner, unit_test):
+    _LOGGER.info("Running unit test: %s", unit_test.name)
+    _LOGGER.debug("Test details: %s", unit_test)
+
+    spl2_test_runner.run_unit_test(unit_test)
+
+
+def run():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("type")
+
+    arguments, other = parser.parse_known_args()
+
+    args = [__file__, "--type", arguments.type]
+
+    if "--log-cli-level" not in args:
+        args.append("--log-cli-level")
+        args.append("INFO")
+
+    args.extend(other)
+    sys.exit(pytest.main(args))
+
+
+if __name__ == "__main__":
+    run()