spl2-testing-framework 1.4.0__tar.gz → 1.5.1__tar.gz

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.4.0
+Version: 1.5.1
 Summary: SPL2 Testing Framework
 Home-page: https://github.com/splunk/spl2-testing-framework
 Keywords: SPL2 testing framework
@@ -133,6 +133,7 @@ Additionally, the following options are supported:
 * `--ignore_empty_strings` - to ignore empty strings in the results
 * `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
+* `--cli_bench` - when running tests with CLI, this option enables benchmarking mode by adding `-b` and `-n` flags to the spl2-processor-cli command. The value specifies the number of events to test (e.g., `--cli_bench=1000` will add `-b -n 1000` to the CLI command). **In bench mode, all tests always succeed and CLI output is printed to logs** - this is useful for performance testing and benchmarking. This is only applicable when using `--type cli`.
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.
@@ -204,3 +205,11 @@ Content of this file will also be printed to stdout.
 
 This checks can be applied only to box tests, as assertions which are used in unit tests may impact spl2 pipeline
 performance. 
+
+## Format all files
+To format all files run:
+
+```bash
+black spl2_testing_framework tests
+```
+

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/README.md

@@ -101,6 +101,7 @@ Additionally, the following options are supported:
 * `--ignore_empty_strings` - to ignore empty strings in the results
 * `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
+* `--cli_bench` - when running tests with CLI, this option enables benchmarking mode by adding `-b` and `-n` flags to the spl2-processor-cli command. The value specifies the number of events to test (e.g., `--cli_bench=1000` will add `-b -n 1000` to the CLI command). **In bench mode, all tests always succeed and CLI output is printed to logs** - this is useful for performance testing and benchmarking. This is only applicable when using `--type cli`.
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.
@@ -171,4 +172,11 @@ command ("|")
 Content of this file will also be printed to stdout.
 
 This checks can be applied only to box tests, as assertions which are used in unit tests may impact spl2 pipeline
-performance. 
+performance. 
+
+## Format all files
+To format all files run:
+
+```bash
+black spl2_testing_framework tests
+```

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/pyproject.toml

@@ -1,22 +1,21 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.4.0"
+version = "1.5.1"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"

spl2_testing_framework-1.5.1/spl2_testing_framework/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/conftest.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import json
 import os
@@ -53,6 +52,7 @@ PERFORMANCE_CHECK = "performance_check"
 TEST_TYPE = "type"
 TEST_DIR = "test_dir"
 LIMIT_TESTS = "limit_tests"
+CLI_BENCH = "cli_bench"
 
 import logging
 
@@ -79,6 +79,7 @@ def pytest_addoption(parser):
     parser.addoption(f"--{IGNORE_EMPTY_STRINGS}", action="store_true")
     parser.addoption(f"--{IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL}", action="store_true")
     parser.addoption(f"--{LIMIT_TESTS}", action="store", type=int, default=None)
+    parser.addoption(f"--{CLI_BENCH}", action="store", type=int, default=None)
 
 
 def pytest_generate_tests(metafunc):
@@ -147,7 +148,8 @@ def spl2_test_runner(
         url = f"{config['host']}:{config['port']}"
         search_client = SplunkSearchClient(url=url, auth=basic_authentication)
     elif test_type == "cli":
-        search_client = CLISearchClient()
+        cli_bench = run_configurations.get(CLI_BENCH)
+        search_client = CLISearchClient(cli_bench=cli_bench)
     elif test_type == "cloud":
         url = f"https://{config['tenant']}.api.{config['cloud_instance']}/{config['tenant']}"
         search_client = CloudSearchClient(url=url, auth=bearer_token_authentication)
@@ -169,6 +171,7 @@ def run_configurations(request):
         CREATE_COMPARISON_SHEET: request.config.getoption(
             f"--{CREATE_COMPARISON_SHEET}"
         ),
+        CLI_BENCH: request.config.getoption(f"--{CLI_BENCH}"),
     }
 
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/launch_remotely_spl2.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import argparse
 import json

spl2_testing_framework-1.5.1/spl2_testing_framework/logger_manager.py

@@ -0,0 +1,25 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+
+
+def setup_logging():
+    logging.basicConfig(
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=[
+            logging.FileHandler("spl2_testing_framework.log"),
+        ],
+    )

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/single_spl2_file_runner.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import argparse
 import logging

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/test_runner.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import argparse
 import sys

spl2_testing_framework-1.5.1/spl2_testing_framework/tools/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/tools/create_diff_sheet.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import pandas as pd
 from openpyxl import load_workbook

spl2_testing_framework-1.5.1/spl2_testing_framework/tools/jobs/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/tools/jobs/cli_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from .job import Job
 
@@ -51,10 +50,16 @@ class CLISimpleJob(Job):
     """A job that is executed on CLI in box tests."""
 
     def __init__(
-        self, source: str, code_module_name: str, code_module_content, test_name
+        self,
+        source: str,
+        code_module_name: str,
+        code_module_content,
+        test_name,
+        cli_bench=None,
     ):
         code_module_content += f"\n$source = from {str(source)};"
         super().__init__("", code_module_name, code_module_content, test_name)
+        self.cli_bench = cli_bench
 
     def create(self) -> "CLISimpleJob":
         """Creates a job. To run the job use run_job() method."""
@@ -62,6 +67,10 @@ class CLISimpleJob(Job):
         cli = "spl2-processor-cli"
         command = [cli, "test", "-s", "pipeline"]
 
+        # Add bench parameters if cli_bench is specified
+        if self.cli_bench is not None:
+            command.extend(["-b", "-n", str(self.cli_bench)])
+
         input_data = "\n\n".join(
             [
                 self.commands_module,

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/tools/jobs/cloud_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from requests import Response
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/tools/jobs/job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from requests import Response
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/tools/jobs/splunk_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from requests import Response
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.1}/spl2_testing_framework/tools/jobs/test_cloud_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import unittest
 from unittest.mock import MagicMock