spl2-testing-framework 1.0.1__tar.gz → 1.1.0b1__tar.gz

{spl2_testing_framework-1.0.1 → spl2_testing_framework-1.1.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spl2-testing-framework
-Version: 1.0.1
+Version: 1.1.0b1
 Summary: SPL2 Testing Framework
 Home-page: https://cd.splunkdev.com/data-availability/spl2-content-tools
 Keywords: SPL2 testing framework
@@ -120,6 +120,10 @@ It is possible to pass more options to the command, which works also with pytest
 * `-pdb` - to enter debugger on failure
 * ... and much more, whatever is supported by pytest
 
+Additionally, the following options are supported:
+
+* `--ignore_empty_strings` - to ignore empty strings in the results
+
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.
 

{spl2_testing_framework-1.0.1 → spl2_testing_framework-1.1.0b1}/README.md

@@ -96,6 +96,10 @@ It is possible to pass more options to the command, which works also with pytest
 * `-pdb` - to enter debugger on failure
 * ... and much more, whatever is supported by pytest
 
+Additionally, the following options are supported:
+
+* `--ignore_empty_strings` - to ignore empty strings in the results
+
 Note: The `pytest.ini.sample` file allows you to define command parameters. Just update the configurations, rename the
 file by removing the `.sample` extension, and execute the command.
 

{spl2_testing_framework-1.0.1 → spl2_testing_framework-1.1.0b1}/pyproject.toml

@@ -16,7 +16,7 @@
 
 [tool.poetry]
 name = "spl2-testing-framework"
-version = "1.0.1"
+version = "1.1.0-beta.1"
 description = "SPL2 Testing Framework"
 authors = ["mgazda-splunk <mgazda@splunk.com>"]
 readme = "README.md"

{spl2_testing_framework-1.0.1 → spl2_testing_framework-1.1.0b1}/spl2_testing_framework/conftest.py

@@ -19,6 +19,8 @@ import os
 
 import pytest
 
+from .tools.results import Results
+
 pytest.register_assert_rewrite("spl2_testing_framework")
 
 from requests.auth import AuthBase, HTTPBasicAuth
@@ -44,6 +46,11 @@ UNIT_TEST = "unit_test"
 BOX_TEST = "box_test"
 SINGLE_SPL2_FILE = "single_spl2_file"
 
+IGNORE_EMPTY_STRINGS = "ignore_empty_strings"
+PERFORMANCE_CHECK = "performance_check"
+TEST_TYPE = "type"
+TEST_DIR = "test_dir"
+
 import logging
 
 from .logger_manager import setup_logging
@@ -54,10 +61,10 @@ _LOGGER = logging.getLogger(__name__)
 
 
 def pytest_addoption(parser):
-    parser.addoption("--type", action="store", default="local")
-    parser.addoption("--test_dir", action="store", default=".")
+    parser.addoption(f"--{TEST_TYPE}", action="store", default="local")
+    parser.addoption(f"--{TEST_DIR}", action="store", default=".")
     parser.addoption(
-        "--performance_check",
+        f"--{PERFORMANCE_CHECK}",
         action="store",
         default="no",
         choices=["no", "time", "detailed_time"],
@@ -65,13 +72,15 @@ def pytest_addoption(parser):
     parser.addoption("--template_file", action="store", default=None)
     parser.addoption("--sample_file", action="store", default=None)
     parser.addoption("--sample_delimiter", action="store", default="\n")
+    parser.addoption(f"--{IGNORE_EMPTY_STRINGS}", action="store_true")
 
 
 def pytest_generate_tests(metafunc):
     """This function is used for dynamic generation of tests based on the test files in the directory specified by the
     --test_dir option"""
-    test_dir = metafunc.config.getoption("test_dir")
-    performance_check = metafunc.config.getoption("performance_check")
+    test_dir = metafunc.config.getoption(TEST_DIR)
+    performance_check = metafunc.config.getoption(PERFORMANCE_CHECK)
+    _set_results_options(metafunc)
 
     if BOX_TEST in metafunc.fixturenames:
         _LOGGER.info("Collecting Box Tests")
@@ -121,7 +130,7 @@ def spl2_test_runner(
     """This fixture is used to create a test runner object based on the test type specified by the --type option
     3 types of tests are supported: 'splunk', 'cloud', 'cli'
     """
-    test_type = request.config.option.type.lower()
+    test_type = request.config.getoption(TEST_TYPE).lower()
     if test_type == "splunk":
         url = f"{config['host']}:{config['port']}"
         search_client = SplunkSearchClient(url=url, auth=basic_authentication)
@@ -215,3 +224,8 @@ def bearer_token_authentication(config):
 
     token = config.get("bearer_token", "")
     return HTTPBearerTokenAuth(token)
+
+
+def _set_results_options(metafunc):
+    ignore_empty_strings = metafunc.config.getoption(IGNORE_EMPTY_STRINGS)
+    Results._ignore_empty_strings = ignore_empty_strings

{spl2_testing_framework-1.0.1 → spl2_testing_framework-1.1.0b1}/spl2_testing_framework/tools/results.py

@@ -17,10 +17,13 @@
 import json
 from collections import UserDict
 from datetime import datetime, timezone
+from os import remove
 from typing import Any
+import pytest
 
 
 class Results(UserDict):
+    _ignore_empty_strings = False
     """Class responsible for parsing and storing results from Splunk, IP and CLI.
     It's necessary to parse nested jsons and other python types to strings in order to compare them
     """
@@ -33,6 +36,7 @@ class Results(UserDict):
         )  # TODO -> remove
 
         self._remove_timestamps_if_necessary()
+        self._remove_all_empty_strings_from_dicts_if_necessary()
 
     @staticmethod
     def _cast_types(data: Any):
@@ -83,26 +87,40 @@ class Results(UserDict):
             self.data.pop("_has_time", None)
 
     @staticmethod
-    def _remove_all_string_nulls_from_dicts(
-        data,
-    ):  # TODO -> remove, it's just a workaround for different behaviour of splunk, IP and CLI
+    def _remove_field_by_value(data, value):
+        # TODO -> remove, it's just a workaround for different behaviour of splunk, IP and CLI
         """Because of different behaviour of Splunk, IP and CLI, we need to remove all "null" values,
         which are returned by CLI
         """
         for k, v in list(
             data.items()
         ):  # list(...) is to avoid changing dict size during iteration
-            if v == "null":
+            if v == value:
                 del data[k]
             if isinstance(v, dict):
-                Results._remove_all_string_nulls_from_dicts(v)
+                Results._remove_field_by_value(v, value)
             if isinstance(v, list):
                 for item in v[:]:
                     if isinstance(item, dict):
-                        Results._remove_all_string_nulls_from_dicts(item)
+                        Results._remove_field_by_value(item, value)
 
         return data
 
+    def _remove_all_string_nulls_from_dicts(self, data):
+        # TODO -> remove, it's just a workaround for different behaviour of splunk, IP and CLI
+        """Because of different behaviour of Splunk, IP and CLI, we need to remove all "null" values,
+        which are returned by CLI
+        """
+        return self._remove_field_by_value(data, "null")
+
+    def _remove_all_empty_strings_from_dicts_if_necessary(self):
+        """If ignore_empty_strings is True, remove all empty strings from dicts
+        This is necessary because for some cases we may want to ignore empty strings in comparison
+        (e.g. when a field is not set, it may be returned as empty string)
+        """
+        if self._ignore_empty_strings:
+            self.data = self._remove_field_by_value(self.data, "")
+
     def __lt__(self, other):
         if "_raw" in self.data:
             s = self.data.get("_raw")

{spl2_testing_framework-1.0.1 → spl2_testing_framework-1.1.0b1}/spl2_testing_framework/tools/search_clients/cloud_search_client.py

@@ -76,9 +76,11 @@ class CloudSearchClient(HTTPSSearchClient):
             if response.status_code != 200:
                 if "GET_PIPELINE_NOT_FOUND" in response.text:
                     continue
-                raise Exception(response.text)
+                raise Exception(f"Job failed, id: {job_id}", response.text)
             if response.json()["status"] == "failed":
-                raise Exception(f"Job failed, id: {job_id}")
+                raise Exception(
+                    f"Job failed, id: {job_id}", response.status_code, response.text
+                )
             if response.json()["status"] == "done":
                 return response
             time.sleep(1)