spl2-testing-framework 1.4.0__tar.gz → 1.5.0__tar.gz

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.4.0
+Version: 1.5.0
 Summary: SPL2 Testing Framework
 Home-page: https://github.com/splunk/spl2-testing-framework
 Keywords: SPL2 testing framework
@@ -133,6 +133,7 @@ Additionally, the following options are supported:
 * `--ignore_empty_strings` - to ignore empty strings in the results
 * `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
+* `--cli_bench` - when running tests with CLI, this option enables benchmarking mode by adding `-b` and `-n` flags to the spl2-processor-cli command. The value specifies the number of events to test (e.g., `--cli_bench=1000` will add `-b -n 1000` to the CLI command). **In bench mode, all tests always succeed and CLI output is printed to logs** - this is useful for performance testing and benchmarking. This is only applicable when using `--type cli`.
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.
@@ -204,3 +205,10 @@ Content of this file will also be printed to stdout.
 
 This checks can be applied only to box tests, as assertions which are used in unit tests may impact spl2 pipeline
 performance. 
+
+## Format all files
+To format all files, run:
+
+```bash
+black spl2_testing_framework tests
+```

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/README.md

@@ -101,6 +101,7 @@ Additionally, the following options are supported:
 * `--ignore_empty_strings` - to ignore empty strings in the results
 * `--ignore_additional_fields_in_actual` - to ignore fields present in actual results but not in expected results (useful when actual results contain extra fields that should not affect comparison)
 * `--create_comparison_sheet` - to create a comparison sheet in `comparison_box_test` folder using actual and expected outputs (works only when running box tests)
+* `--cli_bench` - when running tests with CLI, this option enables benchmarking mode by adding `-b` and `-n` flags to the spl2-processor-cli command. The value specifies the number of events to test (e.g., `--cli_bench=1000` will add `-b -n 1000` to the CLI command). **In bench mode, all tests always succeed and CLI output is printed to logs** - this is useful for performance testing and benchmarking. This is only applicable when using `--type cli`.
 
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.
@@ -171,4 +172,11 @@ command ("|")
 Content of this file will also be printed to stdout.
 
 This checks can be applied only to box tests, as assertions which are used in unit tests may impact spl2 pipeline
-performance. 
+performance. 
+
+## Format all files
+To format all files, run:
+
+```bash
+black spl2_testing_framework tests
+```

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/pyproject.toml

@@ -1,22 +1,21 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.4.0"
+version = "1.5.0"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"

spl2_testing_framework-1.5.0/spl2_testing_framework/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/conftest.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import json
 import os
@@ -53,6 +52,7 @@ PERFORMANCE_CHECK = "performance_check"
 TEST_TYPE = "type"
 TEST_DIR = "test_dir"
 LIMIT_TESTS = "limit_tests"
+CLI_BENCH = "cli_bench"
 
 import logging
 
@@ -79,6 +79,7 @@ def pytest_addoption(parser):
     parser.addoption(f"--{IGNORE_EMPTY_STRINGS}", action="store_true")
     parser.addoption(f"--{IGNORE_ADDITIONAL_FIELDS_IN_ACTUAL}", action="store_true")
     parser.addoption(f"--{LIMIT_TESTS}", action="store", type=int, default=None)
+    parser.addoption(f"--{CLI_BENCH}", action="store", type=int, default=None)
 
 
 def pytest_generate_tests(metafunc):
@@ -147,7 +148,8 @@ def spl2_test_runner(
         url = f"{config['host']}:{config['port']}"
         search_client = SplunkSearchClient(url=url, auth=basic_authentication)
     elif test_type == "cli":
-        search_client = CLISearchClient()
+        cli_bench = run_configurations.get(CLI_BENCH)
+        search_client = CLISearchClient(cli_bench=cli_bench)
     elif test_type == "cloud":
         url = f"https://{config['tenant']}.api.{config['cloud_instance']}/{config['tenant']}"
         search_client = CloudSearchClient(url=url, auth=bearer_token_authentication)
@@ -169,6 +171,7 @@ def run_configurations(request):
         CREATE_COMPARISON_SHEET: request.config.getoption(
             f"--{CREATE_COMPARISON_SHEET}"
         ),
+        CLI_BENCH: request.config.getoption(f"--{CLI_BENCH}"),
     }
 
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/launch_remotely_spl2.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import argparse
 import json

spl2_testing_framework-1.5.0/spl2_testing_framework/logger_manager.py

@@ -0,0 +1,25 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+
+
+def setup_logging():
+    logging.basicConfig(
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        handlers=[
+            logging.FileHandler("spl2_testing_framework.log"),
+        ],
+    )

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/single_spl2_file_runner.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import argparse
 import logging

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/test_runner.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import argparse
 import sys

spl2_testing_framework-1.5.0/spl2_testing_framework/tools/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/create_diff_sheet.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import pandas as pd
 from openpyxl import load_workbook

spl2_testing_framework-1.5.0/spl2_testing_framework/tools/jobs/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/jobs/cli_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from .job import Job
 
@@ -51,10 +50,16 @@ class CLISimpleJob(Job):
     """A job that is executed on CLI in box tests."""
 
     def __init__(
-        self, source: str, code_module_name: str, code_module_content, test_name
+        self,
+        source: str,
+        code_module_name: str,
+        code_module_content,
+        test_name,
+        cli_bench=None,
     ):
         code_module_content += f"\n$source = from {str(source)};"
         super().__init__("", code_module_name, code_module_content, test_name)
+        self.cli_bench = cli_bench
 
     def create(self) -> "CLISimpleJob":
         """Creates a job. To run the job use run_job() method."""
@@ -62,6 +67,10 @@ class CLISimpleJob(Job):
         cli = "spl2-processor-cli"
         command = [cli, "test", "-s", "pipeline"]
 
+        # Add bench parameters if cli_bench is specified
+        if self.cli_bench is not None:
+            command.extend(["-b", "-n", str(self.cli_bench)])
+
         input_data = "\n\n".join(
             [
                 self.commands_module,

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/jobs/cloud_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from requests import Response
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/jobs/job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from requests import Response
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/jobs/splunk_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 from requests import Response
 

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/jobs/test_cloud_job.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import unittest
 from unittest.mock import MagicMock

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/performance.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import logging
 import os

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/results.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import json
 from collections import UserDict

spl2_testing_framework-1.5.0/spl2_testing_framework/tools/search_clients/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

spl2_testing_framework-1.5.0/spl2_testing_framework/tools/search_clients/cli_search_client.py

@@ -0,0 +1,109 @@
+# Copyright 2026 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import logging
+import subprocess
+
+from .search_client import SearchClient
+from ..jobs.cli_job import CLIAssertionJob, CLISimpleJob
+from ..jobs.job import Job
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class CLISearchClient(SearchClient):
+    """Search client responsible for running jobs on CLI."""
+
+    jobs_dict = {}
+
+    def __init__(self, cli_bench=None):
+        """Initialize CLISearchClient with optional bench parameter.
+
+        Args:
+            cli_bench: Number of events to use for benchmarking (adds -b and -n flags)
+        """
+        self.cli_bench = cli_bench
+
+    @staticmethod
+    def create_job(
+        test_module: str, code_file_name: str, code_to_test: str, test_statement: str
+    ) -> Job:
+        """Creates a job for unit test run."""
+        return CLIAssertionJob(
+            test_module, code_file_name, code_to_test, test_statement
+        ).create()
+
+    def create_simple_job(
+        self, source: str, code_module_name: str, code_module_content, test_name
+    ) -> Job:
+        """Creates a job for box test run."""
+
+        return CLISimpleJob(
+            source, code_module_name, code_module_content, test_name, self.cli_bench
+        ).create()
+
+    def run_job(self, job):
+        """Run a job on CLI."""
+        result = subprocess.run(
+            job.job_content["command"],
+            input=job.job_content["input_data"],  # Pass data to stdin
+            text=True,  # Encode input_data as string
+            capture_output=True,
+        )
+
+        # When cli_bench is enabled, print output and always succeed
+        if self.cli_bench is not None:
+            # Set empty result to avoid json parsing errors
+            job.result = {}
+
+            self._print_bench_results(job, result)
+            return
+
+        # Normal mode - raise errors as before
+        if result.stderr:
+            raise RuntimeError(result.stderr)
+
+        job.result = json.loads(result.stdout)
+        return
+
+    def _print_bench_results(self, job, result):
+        """Print benchmark results to logs."""
+        _LOGGER.info("=" * 80)
+        _LOGGER.info("CLI BENCH MODE - Test: %s", job.test_name)
+        _LOGGER.info("=" * 80)
+
+        if result.stdout:
+            _LOGGER.info("STDOUT:\n%s", result.stdout)
+
+        if result.stderr:
+            _LOGGER.info("STDERR:\n%s", result.stderr)
+
+        _LOGGER.info("=" * 80)
+        _LOGGER.info("Return Code: %s", result.returncode)
+        _LOGGER.info("=" * 80)
+
+    def get_job_results(self, job: Job) -> dict:
+        """Get results of a job."""
+        # When cli_bench is enabled, return empty results to skip validation
+        if self.cli_bench is not None:
+            return {}
+
+        tmp = {  # remove job_ prefix
+            k.replace("job_", ""): v for k, v in job.result.items()
+        }  # TODO for python>3.9 use str.remove_prefix
+
+        results = self._cast_results(tmp)
+        return results

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/search_clients/cloud_search_client.py

@@ -1,31 +1,16 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 #
 #

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/search_clients/search_client.py

@@ -1,31 +1,16 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 #
 #

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/search_clients/splunk_search_client.py

@@ -1,31 +1,16 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 #
 #

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/spl2test_runner.py

@@ -1,28 +1,29 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import logging
 
 from tabulate import tabulate
 
+from spl2_testing_framework.tools.create_diff_sheet import ComparisonSheet
 from spl2_testing_framework.tools.performance import PerformanceCheck
+from spl2_testing_framework.tools.search_clients.cli_search_client import (
+    CLISearchClient,
+)
 from spl2_testing_framework.tools.test_types import UnitTest, BoxTest, SingleSPL2
 from spl2_testing_framework.tools.utils import _make_functions_visible_for_testing
-from spl2_testing_framework.tools.create_diff_sheet import ComparisonSheet
-
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -50,6 +51,16 @@ class SPL2TestRunner:
             single_spl2_file.name,
         )
         self._search_client.run_job(job)
+        # Check if cli_bench is enabled - if so, skip all assertions
+        if (
+            isinstance(self._search_client, CLISearchClient)
+            and self._search_client.cli_bench is not None
+        ):
+            _LOGGER.info(
+                f"CLI Bench Mode: Test {single_spl2_file.name} completed - skipping assertions"
+            )
+            return
+
         job_result = self._search_client.get_job_results(job)
         output_result = job_result.get("destination", [])
 
@@ -84,6 +95,17 @@ class SPL2TestRunner:
         )
 
         self._search_client.run_job(job)
+
+        # Check if cli_bench is enabled - if so, skip all assertions
+        if (
+            isinstance(self._search_client, CLISearchClient)
+            and self._search_client.cli_bench is not None
+        ):
+            _LOGGER.info(
+                f"CLI Bench Mode: Test {template_test.name} completed - skipping assertions"
+            )
+            return
+
         job_results = self._search_client.get_job_results(job)
 
         errors = []
@@ -123,6 +145,17 @@ class SPL2TestRunner:
         )
 
         self._search_client.run_job(job)
+
+        # Check if cli_bench is enabled - if so, skip all assertions
+        if (
+            isinstance(self._search_client, CLISearchClient)
+            and self._search_client.cli_bench is not None
+        ):
+            _LOGGER.info(
+                f"CLI Bench Mode: Test {box_test.name} completed - skipping assertions"
+            )
+            return
+
         job_result = self._search_client.get_job_results(job)
 
         output_result = job_result.get("destination", [])

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/test_discovery.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import abc
 import json

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/test_types.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import logging
 from collections import UserDict

{spl2_testing_framework-1.4.0 → spl2_testing_framework-1.5.0}/spl2_testing_framework/tools/utils.py

@@ -1,18 +1,17 @@
+# Copyright 2026 Splunk Inc.
 #
-#   Copyright 2025 Splunk Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 import pathlib
 import re

spl2_testing_framework-1.4.0/spl2_testing_framework/__init__.py

@@ -1,15 +0,0 @@
-#
-#   Copyright 2025 Splunk Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#

spl2_testing_framework-1.4.0/spl2_testing_framework/logger_manager.py

@@ -1,26 +0,0 @@
-#
-#   Copyright 2025 Splunk Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-
-import logging
-
-
-def setup_logging():
-    logging.basicConfig(
-        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-        handlers=[
-            logging.FileHandler("spl2_testing_framework.log"),
-        ],
-    )

spl2_testing_framework-1.4.0/spl2_testing_framework/tools/__init__.py

@@ -1,15 +0,0 @@
-#
-#   Copyright 2025 Splunk Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#

spl2_testing_framework-1.4.0/spl2_testing_framework/tools/jobs/__init__.py

@@ -1,15 +0,0 @@
-#
-#   Copyright 2025 Splunk Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#

spl2_testing_framework-1.4.0/spl2_testing_framework/tools/search_clients/__init__.py

@@ -1,28 +0,0 @@
-#
-#   Copyright 2025 Splunk Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#

spl2_testing_framework-1.4.0/spl2_testing_framework/tools/search_clients/cli_search_client.py

@@ -1,87 +0,0 @@
-#
-#   Copyright 2025 Splunk Inc.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-
-
-import json
-import logging
-import subprocess
-from ..jobs.cli_job import CLIAssertionJob, CLISimpleJob
-from ..jobs.job import Job
-from .search_client import SearchClient
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class CLISearchClient(SearchClient):
-    """Search client responsible for running jobs on CLI."""
-
-    jobs_dict = {}
-
-    @staticmethod
-    def create_job(
-        test_module: str, code_file_name: str, code_to_test: str, test_statement: str
-    ) -> Job:
-        """Creates a job for unit test run."""
-        return CLIAssertionJob(
-            test_module, code_file_name, code_to_test, test_statement
-        ).create()
-
-    @staticmethod
-    def create_simple_job(
-        source: str, code_module_name: str, code_module_content, test_name
-    ) -> Job:
-        """Creates a job for box test run."""
-
-        return CLISimpleJob(
-            source, code_module_name, code_module_content, test_name
-        ).create()
-
-    def run_job(self, job):
-        """Run a job on CLI."""
-        result = subprocess.run(
-            job.job_content["command"],
-            input=job.job_content["input_data"],  # Pass data to stdin
-            text=True,  # Encode input_data as string
-            capture_output=True,
-        )
-
-        if result.stderr:
-            raise RuntimeError(result.stderr)
-
-        job.result = json.loads(result.stdout)
-        return
-
-    def get_job_results(self, job: Job) -> dict:
-        """Get results of a job."""
-        tmp = {  # remove job_ prefix
-            k.replace("job_", ""): v for k, v in job.result.items()
-        }  # TODO for python>3.9 use str.remove_prefix
-
-        results = self._cast_results(tmp)
-        return results