contentctl 5.2.0__py3-none-any.whl → 5.3.0__py3-none-any.whl

contentctl/actions/build.py

@@ -1,19 +1,14 @@
+import datetime
+import json
+import pathlib
 import shutil
-
 from dataclasses import dataclass
 
 from contentctl.input.director import DirectorOutputDto
+from contentctl.objects.config import build
+from contentctl.output.api_json_output import ApiJsonOutput
 from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
-from contentctl.output.api_json_output import ApiJsonOutput
-from contentctl.output.data_source_writer import DataSourceWriter
-from contentctl.objects.lookup import CSVLookup, Lookup_Type
-import pathlib
-import json
-import datetime
-import uuid
-
-from contentctl.objects.config import build
 
 
 @dataclass(frozen=True)
@@ -28,39 +23,6 @@ class Build:
             updated_conf_files: set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
 
-            # Construct a path to a YML that does not actually exist.
-            # We mock this "fake" path since the YML does not exist.
-            # This ensures the checking for the existence of the CSV is correct
-            data_sources_fake_yml_path = (
-                input_dto.config.getPackageDirectoryPath()
-                / "lookups"
-                / "data_sources.yml"
-            )
-
-            # Construct a special lookup whose CSV is created at runtime and
-            # written directly into the lookups folder. We will delete this after a build,
-            # assuming that it is successful.
-            data_sources_lookup_csv_path = (
-                input_dto.config.getPackageDirectoryPath()
-                / "lookups"
-                / "data_sources.csv"
-            )
-
-            DataSourceWriter.writeDataSourceCsv(
-                input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
-            )
-            input_dto.director_output_dto.addContentToDictMappings(
-                CSVLookup.model_construct(
-                    name="data_sources",
-                    id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
-                    version=1,
-                    author=input_dto.config.app.author_name,
-                    date=datetime.date.today(),
-                    description="A lookup file that will contain the data source objects for detections.",
-                    lookup_type=Lookup_Type.csv,
-                    file_path=data_sources_fake_yml_path,
-                )
-            )
             updated_conf_files.update(conf_output.writeHeaders())
             updated_conf_files.update(
                 conf_output.writeLookups(input_dto.director_output_dto.lookups)

contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -1,7 +1,16 @@
+import concurrent.futures
+import datetime
+import signal
+import traceback
+from dataclasses import dataclass
 from typing import List, Union
-from contentctl.objects.config import test, test_servers, Container, Infrastructure
+
+import docker
+from pydantic import BaseModel
+
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
     DetectionTestingInfrastructure,
+    DetectionTestingManagerOutputDto,
 )
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
     DetectionTestingInfrastructureContainer,
@@ -9,24 +18,12 @@ from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfras
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
     DetectionTestingInfrastructureServer,
 )
-import signal
-import datetime
-
-# from queue import Queue
-from dataclasses import dataclass
-
-# import threading
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
-    DetectionTestingManagerOutputDto,
-)
 from contentctl.actions.detection_testing.views.DetectionTestingView import (
     DetectionTestingView,
 )
-from contentctl.objects.enums import PostTestBehavior
-from pydantic import BaseModel
+from contentctl.objects.config import Container, Infrastructure, test, test_servers
 from contentctl.objects.detection import Detection
-import concurrent.futures
-import docker
+from contentctl.objects.enums import PostTestBehavior
 
 
 @dataclass(frozen=False)
@@ -63,12 +60,14 @@ class DetectionTestingManager(BaseModel):
                 # a newline '\r\n' which will cause that wait to stop
                 print("*******************************")
                 print(
-                    "If testing is paused and you are debugging a detection, you MUST hit CTRL-D at the prompt to complete shutdown."
+                    "If testing is paused and you are debugging a detection, you MUST hit CTRL-D "
+                    "at the prompt to complete shutdown."
                 )
                 print("*******************************")
 
         signal.signal(signal.SIGINT, sigint_handler)
 
+        # TODO (#337): futures can be hard to maintain/debug; let's consider alternatives
         with (
             concurrent.futures.ThreadPoolExecutor(
                 max_workers=len(self.input_dto.config.test_instances),
@@ -80,10 +79,19 @@ class DetectionTestingManager(BaseModel):
                 max_workers=len(self.input_dto.config.test_instances),
             ) as view_shutdowner,
         ):
+            # Capture any errors for reporting at the end after all threads have been gathered
+            errors: dict[str, list[Exception]] = {
+                "INSTANCE SETUP ERRORS": [],
+                "TESTING ERRORS": [],
+                "ERRORS DURING VIEW SHUTDOWN": [],
+                "ERRORS DURING VIEW EXECUTION": [],
+            }
+
             # Start all the views
             future_views = {
                 view_runner.submit(view.setup): view for view in self.input_dto.views
             }
+
             # Configure all the instances
             future_instances_setup = {
                 instance_pool.submit(instance.setup): instance
@@ -96,7 +104,11 @@ class DetectionTestingManager(BaseModel):
                     future.result()
                 except Exception as e:
                     self.output_dto.terminate = True
-                    print(f"Error setting up container: {str(e)}")
+                    # Output the traceback if we encounter errors in verbose mode
+                    if self.input_dto.config.verbose:
+                        tb = traceback.format_exc()
+                        print(tb)
+                    errors["INSTANCE SETUP ERRORS"].append(e)
 
             # Start and wait for all tests to run
             if not self.output_dto.terminate:
@@ -111,7 +123,11 @@ class DetectionTestingManager(BaseModel):
                         future.result()
                     except Exception as e:
                         self.output_dto.terminate = True
-                        print(f"Error running in container: {str(e)}")
+                        # Output the traceback if we encounter errors in verbose mode
+                        if self.input_dto.config.verbose:
+                            tb = traceback.format_exc()
+                            print(tb)
+                        errors["TESTING ERRORS"].append(e)
 
             self.output_dto.terminate = True
 
@@ -123,14 +139,34 @@ class DetectionTestingManager(BaseModel):
                 try:
                     future.result()
                 except Exception as e:
-                    print(f"Error stopping view: {str(e)}")
+                    # Output the traceback if we encounter errors in verbose mode
+                    if self.input_dto.config.verbose:
+                        tb = traceback.format_exc()
+                        print(tb)
+                    errors["ERRORS DURING VIEW SHUTDOWN"].append(e)
 
             # Wait for original view-related threads to complete
             for future in concurrent.futures.as_completed(future_views):
                 try:
                     future.result()
                 except Exception as e:
-                    print(f"Error running container: {str(e)}")
+                    # Output the traceback if we encounter errors in verbose mode
+                    if self.input_dto.config.verbose:
+                        tb = traceback.format_exc()
+                        print(tb)
+                    errors["ERRORS DURING VIEW EXECUTION"].append(e)
+
+            # Log any errors
+            for error_type in errors:
+                if len(errors[error_type]) > 0:
+                    print()
+                    print(f"[{error_type}]:")
+                    for error in errors[error_type]:
+                        print(f"\t❌ {str(error)}")
+                        if isinstance(error, ExceptionGroup):
+                            for suberror in error.exceptions:  # type: ignore
+                                print(f"\t\t❌ {str(suberror)}")  # type: ignore
+                    print()
 
         return self.output_dto
 
@@ -154,12 +190,15 @@ class DetectionTestingManager(BaseModel):
                     )
                     if len(parts) != 2:
                         raise Exception(
-                            f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
-                            f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'"
+                            "Expected to find a name:tag in "
+                            f"{self.input_dto.config.container_settings.full_image_path}, "
+                            f"but instead found {parts}. Note that this path MUST include the "
+                            "tag, which is separated by ':'"
                         )
 
                     print(
-                        f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...",
+                        "Getting the latest version of the container image "
+                        f"[{self.input_dto.config.container_settings.full_image_path}]...",
                         end="",
                         flush=True,
                     )
@@ -168,7 +207,8 @@ class DetectionTestingManager(BaseModel):
                     break
                 except Exception as e:
                     raise Exception(
-                        f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
+                        "Failed to pull docker container image "
+                        f"[{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
                     )
 
         already_staged_container_files = False

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -11,13 +11,20 @@ from shutil import copyfile
 from ssl import SSLEOFError, SSLZeroReturnError
 from sys import stdout
 from tempfile import TemporaryDirectory, mktemp
-from typing import Optional, Union
+from typing import Callable, Optional, Union
 
 import requests  # type: ignore
 import splunklib.client as client  # type: ignore
-import splunklib.results
 import tqdm  # type: ignore
-from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, dataclasses
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    computed_field,
+    dataclasses,
+)
+from semantic_version import Version
 from splunklib.binding import HTTPError  # type: ignore
 from splunklib.results import JSONResultsReader, Message  # type: ignore
 from urllib3 import disable_warnings
@@ -31,7 +38,8 @@ from contentctl.actions.detection_testing.progress_bar import (
 from contentctl.helper.utils import Utils
 from contentctl.objects.base_test import BaseTest
 from contentctl.objects.base_test_result import TestResultStatus
-from contentctl.objects.config import Infrastructure, test_common
+from contentctl.objects.config import All, Infrastructure, test_common
+from contentctl.objects.content_versioning_service import ContentVersioningService
 from contentctl.objects.correlation_search import CorrelationSearch, PbarData
 from contentctl.objects.detection import Detection
 from contentctl.objects.enums import AnalyticsType, PostTestBehavior
@@ -42,6 +50,9 @@ from contentctl.objects.test_group import TestGroup
 from contentctl.objects.unit_test import UnitTest
 from contentctl.objects.unit_test_result import UnitTestResult
 
+# The app name of ES; needed to check ES version
+ES_APP_NAME = "SplunkEnterpriseSecuritySuite"
+
 
 class SetupTestGroupResults(BaseModel):
     exception: Union[Exception, None] = None
@@ -136,21 +147,27 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
         )
 
         self.start_time = time.time()
+
+        # Init the list of setup functions we always need
+        primary_setup_functions: list[
+            tuple[Callable[[], None | client.Service], str]
+        ] = [
+            (self.start, "Starting"),
+            (self.get_conn, "Waiting for App Installation"),
+            (self.configure_conf_file_datamodels, "Configuring Datamodels"),
+            (self.create_replay_index, f"Create index '{self.sync_obj.replay_index}'"),
+            (self.get_all_indexes, "Getting all indexes from server"),
+            (self.check_for_es_install, "Checking for ES Install"),
+            (self.configure_imported_roles, "Configuring Roles"),
+            (self.configure_delete_indexes, "Configuring Indexes"),
+            (self.configure_hec, "Configuring HEC"),
+            (self.wait_for_ui_ready, "Finishing Primary Setup"),
+        ]
+
+        # Execute and report on each setup function
         try:
-            for func, msg in [
-                (self.start, "Starting"),
-                (self.get_conn, "Waiting for App Installation"),
-                (self.configure_conf_file_datamodels, "Configuring Datamodels"),
-                (
-                    self.create_replay_index,
-                    f"Create index '{self.sync_obj.replay_index}'",
-                ),
-                (self.get_all_indexes, "Getting all indexes from server"),
-                (self.configure_imported_roles, "Configuring Roles"),
-                (self.configure_delete_indexes, "Configuring Indexes"),
-                (self.configure_hec, "Configuring HEC"),
-                (self.wait_for_ui_ready, "Finishing Setup"),
-            ]:
+            # Run the primary setup functions
+            for func, msg in primary_setup_functions:
                 self.format_pbar_string(
                     TestReportingType.SETUP,
                     self.get_name(),
@@ -160,18 +177,114 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
                 func()
                 self.check_for_teardown()
 
+            # Run any setup functions only applicable to content versioning validation
+            if self.should_test_content_versioning:
+                self.pbar.write(
+                    self.format_pbar_string(
+                        TestReportingType.SETUP,
+                        self.get_name(),
+                        "Beginning Content Versioning Validation...",
+                        set_pbar=False,
+                    )
+                )
+                for func, msg in self.content_versioning_service.setup_functions:
+                    self.format_pbar_string(
+                        TestReportingType.SETUP,
+                        self.get_name(),
+                        msg,
+                        update_sync_status=True,
+                    )
+                    func()
+                    self.check_for_teardown()
+
         except Exception as e:
-            self.pbar.write(str(e))
+            msg = f"[{self.get_name()}]: {str(e)}"
             self.finish()
-            return
+            if isinstance(e, ExceptionGroup):
+                raise ExceptionGroup(msg, e.exceptions) from e  # type: ignore
+            raise Exception(msg) from e
 
-        self.format_pbar_string(
-            TestReportingType.SETUP, self.get_name(), "Finished Setup!"
+        self.pbar.write(
+            self.format_pbar_string(
+                TestReportingType.SETUP,
+                self.get_name(),
+                "Finished Setup!",
+                set_pbar=False,
+            )
         )
 
     def wait_for_ui_ready(self):
         self.get_conn()
 
+    @computed_field
+    @property
+    def content_versioning_service(self) -> ContentVersioningService:
+        """
+        A computed field returning a handle to the content versioning service, used by ES to
+        version detections. We use this model to validate that all detections have been installed
+        compatibly with ES versioning.
+
+        :return:  a handle to the content versioning service on the instance
+        :rtype: :class:`contentctl.objects.content_versioning_service.ContentVersioningService`
+        """
+        return ContentVersioningService(
+            global_config=self.global_config,
+            infrastructure=self.infrastructure,
+            service=self.get_conn(),
+            detections=self.sync_obj.inputQueue,
+        )
+
+    @property
+    def should_test_content_versioning(self) -> bool:
+        """
+        Indicates whether we should test content versioning. Content versioning
+        should be tested when integration testing is enabled, the mode is all, and ES is at least
+        version 8.0.0.
+
+        :return: a bool indicating whether we should test content versioning
+        :rtype: bool
+        """
+        es_version = self.es_version
+        return (
+            self.global_config.enable_integration_testing
+            and isinstance(self.global_config.mode, All)
+            and es_version is not None
+            and es_version >= Version("8.0.0")
+        )
+
+    @property
+    def es_version(self) -> Version | None:
+        """
+        Returns the version of Enterprise Security installed on the instance; None if not installed.
+
+        :return: the version of ES, as a semver aware object
+        :rtype: :class:`semantic_version.Version`
+        """
+        if not self.es_installed:
+            return None
+        return Version(self.get_conn().apps[ES_APP_NAME]["version"])  # type: ignore
+
+    @property
+    def es_installed(self) -> bool:
+        """
+        Indicates whether ES is installed on the instance.
+
+        :return: a bool indicating whether ES is installed or not
+        :rtype: bool
+        """
+        return ES_APP_NAME in self.get_conn().apps
+
+    def check_for_es_install(self) -> None:
+        """
+        Validating function which raises an error if Enterprise Security is not installed and
+        integration testing is enabled.
+        """
+        if not self.es_installed and self.global_config.enable_integration_testing:
+            raise Exception(
+                "Enterprise Security does not appear to be installed on this instance and "
+                "integration testing is enabled."
+            )
+
     def configure_hec(self):
         self.hec_channel = str(uuid.uuid4())
         try:
@@ -298,14 +411,14 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
         imported_roles: list[str] = ["user", "power", "can_delete"],
         enterprise_security_roles: list[str] = ["ess_admin", "ess_analyst", "ess_user"],
     ):
-        try:
-            # Set which roles should be configured. For Enterprise Security/Integration Testing,
-            # we must add some extra foles.
-            if self.global_config.enable_integration_testing:
-                roles = imported_roles + enterprise_security_roles
-            else:
-                roles = imported_roles
+        # Set which roles should be configured. For Enterprise Security/Integration Testing,
+        # we must add some extra foles.
+        if self.global_config.enable_integration_testing:
+            roles = imported_roles + enterprise_security_roles
+        else:
+            roles = imported_roles
 
+        try:
             self.get_conn().roles.post(
                 self.infrastructure.splunk_app_username,
                 imported_roles=roles,
@@ -314,16 +427,9 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
             )
             return
         except Exception as e:
-            self.pbar.write(
-                f"The following role(s) do not exist:'{enterprise_security_roles}: {str(e)}"
-            )
-
-        self.get_conn().roles.post(
-            self.infrastructure.splunk_app_username,
-            imported_roles=imported_roles,
-            srchIndexesAllowed=";".join(self.all_indexes_on_server),
-            srchIndexesDefault=self.sync_obj.replay_index,
-        )
+            msg = f"Error configuring roles: {str(e)}"
+            self.pbar.write(msg)
+            raise Exception(msg) from e
 
     def configure_delete_indexes(self):
         endpoint = "/services/properties/authorize/default/deleteIndexesAllowed"
@@ -1170,8 +1276,6 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
                     # on a field.  In this case, the field will appear but will not contain any values
                     current_empty_fields: set[str] = set()
 
-                    # TODO (cmcginley): @ljstella is this something we're keeping for testing as
-                    #   well?
                     for field in full_rba_field_set:
                         if result.get(field, "null") == "null":
                             if field in risk_object_fields_set:
@@ -1257,7 +1361,7 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
                 job = self.get_conn().jobs.create(splunk_search, **kwargs)
                 results_stream = job.results(output_mode="json")
                 # TODO: should we be doing something w/ this reader?
-                _ = splunklib.results.JSONResultsReader(results_stream)
+                _ = JSONResultsReader(results_stream)
 
             except Exception as e:
                 raise (

contentctl/actions/detection_testing/views/DetectionTestingView.py

@@ -4,14 +4,13 @@ from typing import Any
 
 from pydantic import BaseModel
 
-from contentctl.objects.config import test_common
-
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
     DetectionTestingManagerOutputDto,
 )
 from contentctl.helper.utils import Utils
-from contentctl.objects.enums import DetectionStatus
 from contentctl.objects.base_test_result import TestResultStatus
+from contentctl.objects.config import test_common
+from contentctl.objects.enums import ContentStatus
 
 
 class DetectionTestingView(BaseModel, abc.ABC):
@@ -117,11 +116,11 @@ class DetectionTestingView(BaseModel, abc.ABC):
                 total_skipped += 1
 
             # Aggregate production status metrics
-            if detection.status == DetectionStatus.production:
+            if detection.status == ContentStatus.production:
                 total_production += 1
-            elif detection.status == DetectionStatus.experimental:
+            elif detection.status == ContentStatus.experimental:
                 total_experimental += 1
-            elif detection.status == DetectionStatus.deprecated:
+            elif detection.status == ContentStatus.deprecated:
                 total_deprecated += 1
 
             # Check if the detection is manual_test

contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py

@@ -47,6 +47,8 @@ class DetectionTestingViewCLI(DetectionTestingView, arbitrary_types_allowed=True
         while True:
             summary = self.getSummaryObject()
 
+            # TODO (#338): there's a 1-off error here I think (we show one more than we
+            #   actually have during testing)
             total = len(
                 summary.get("tested_detections", [])
                 + summary.get("untested_detections", [])

contentctl/actions/initialize.py

@@ -1,7 +1,21 @@
-import shutil
 import os
 import pathlib
+import shutil
+
+from contentctl.objects.baseline import Baseline
 from contentctl.objects.config import test
+from contentctl.objects.dashboard import Dashboard
+from contentctl.objects.data_source import DataSource
+from contentctl.objects.deployment import Deployment
+from contentctl.objects.detection import Detection
+from contentctl.objects.investigation import Investigation
+from contentctl.objects.lookup import Lookup
+from contentctl.objects.macro import Macro
+from contentctl.objects.playbook import Playbook
+from contentctl.objects.removed_security_content_object import (
+    RemovedSecurityContentObject,
+)
+from contentctl.objects.story import Story
 from contentctl.output.yml_writer import YmlWriter
 
 
@@ -13,21 +27,33 @@ class Initialize:
 
         YmlWriter.writeYmlFile(str(config.path / "contentctl.yml"), config.model_dump())
 
-        # Create the following empty directories:
+        # Create the following empty directories. Each type of content,
+        # even if you don't have any of that type of content, need its own directory to exist.
+        for contentType in [
+            Detection,
+            Playbook,
+            Story,
+            DataSource,
+            Investigation,
+            Macro,
+            Lookup,
+            Dashboard,
+            Baseline,
+            Deployment,
+            RemovedSecurityContentObject,
+        ]:
+            contentType.containing_folder().mkdir(exist_ok=False, parents=True)
+
+        # Some other directories that do not map directly to a piece of content also must exist
+
         for emptyDir in [
-            "lookups",
-            "baselines",
-            "data_sources",
             "docs",
             "reporting",
-            "investigations",
             "detections/application",
             "detections/cloud",
             "detections/endpoint",
             "detections/network",
             "detections/web",
-            "macros",
-            "stories",
         ]:
             # Throw an error if this directory already exists
             (config.path / emptyDir).mkdir(exist_ok=False, parents=True)
@@ -60,7 +86,7 @@ class Initialize:
             source_directory = pathlib.Path(os.path.dirname(__file__)) / templateDir
             target_directory = config.path / targetDir
             # Throw an exception if the target exists
-            shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
+            shutil.copytree(source_directory, target_directory, dirs_exist_ok=True)
 
         # Create a README.md file.  Note that this is the README.md for the repository, not the
         # one which will actually be packaged into the app. That is located in the app_template folder.

contentctl/actions/release_notes.py

@@ -1,10 +1,12 @@
-from contentctl.objects.config import release_notes
-from git import Repo
-import re
-import yaml
 import pathlib
+import re
 from typing import List, Union
 
+import yaml
+from git import Repo
+
+from contentctl.objects.config import release_notes
+
 
 class ReleaseNotes:
     def create_notes(
@@ -171,13 +173,13 @@ class ReleaseNotes:
 
     def release_notes(self, config: release_notes) -> None:
         ### Remove hard coded path
-        directories = [
-            "detections/",
-            "stories/",
-            "macros/",
-            "lookups/",
-            "playbooks/",
-            "ssa_detections/",
+        directories: list[pathlib.Path] = [
+            config.path / "detections",
+            config.path / "stories",
+            config.path / "macros",
+            config.path / "lookups",
+            config.path / "playbooks",
+            config.path / "ssa_detections",
         ]
 
         repo = Repo(config.path)
@@ -229,7 +231,7 @@ class ReleaseNotes:
             file_path = pathlib.Path(diff.a_path)
 
             # Check if the file is in the specified directories
-            if any(str(file_path).startswith(directory) for directory in directories):
+            if any(file_path.is_relative_to(directory) for directory in directories):
                 # Check if a file is Modified
                 if diff.change_type == "M":
                     modified_files.append(file_path)