contentctl 4.4.7__py3-none-any.whl → 5.0.0__py3-none-any.whl

contentctl/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.0'
+__version__ = "0.1.0"

contentctl/actions/build.py

@@ -1,91 +1,136 @@
-import sys
 import shutil
-import os
 
 from dataclasses import dataclass
 
-from contentctl.objects.enums import SecurityContentProduct, SecurityContentType
-from contentctl.input.director import Director, DirectorOutputDto
+from contentctl.input.director import DirectorOutputDto
 from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
 from contentctl.output.api_json_output import ApiJsonOutput
 from contentctl.output.data_source_writer import DataSourceWriter
-from contentctl.objects.lookup import Lookup
+from contentctl.objects.lookup import CSVLookup, Lookup_Type
 import pathlib
 import json
 import datetime
-from typing import Union
+import uuid
 
 from contentctl.objects.config import build
 
+
 @dataclass(frozen=True)
 class BuildInputDto:
     director_output_dto: DirectorOutputDto
-    config:build
+    config: build
 
 
 class Build:
-
-
-
     def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
         if input_dto.config.build_app:
-
-            updated_conf_files:set[pathlib.Path] = set()
+            updated_conf_files: set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
 
-            # Construct a special lookup whose CSV is created at runtime and
-            # written directly into the output folder. It is created with model_construct,
-            # not model_validate, because the CSV does not exist yet.
-            data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv"
-            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path)
-            input_dto.director_output_dto.addContentToDictMappings(Lookup.model_construct(description= "A lookup file that will contain the data source objects for detections.", 
-                                                                        filename=data_sources_lookup_csv_path, 
-                                                                        name="data_sources"))
+            # Construct a path to a YML that does not actually exist.
+            # We mock this "fake" path since the YML does not exist.
+            # This ensures the checking for the existence of the CSV is correct
+            data_sources_fake_yml_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "data_sources.yml"
+            )
 
+            # Construct a special lookup whose CSV is created at runtime and
+            # written directly into the lookups folder. We will delete this after a build,
+            # assuming that it is successful.
+            data_sources_lookup_csv_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "data_sources.csv"
+            )
+
+            DataSourceWriter.writeDataSourceCsv(
+                input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
+            )
+            input_dto.director_output_dto.addContentToDictMappings(
+                CSVLookup.model_construct(
+                    name="data_sources",
+                    id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
+                    version=1,
+                    author=input_dto.config.app.author_name,
+                    date=datetime.date.today(),
+                    description="A lookup file that will contain the data source objects for detections.",
+                    lookup_type=Lookup_Type.csv,
+                    file_path=data_sources_fake_yml_path,
+                )
+            )
             updated_conf_files.update(conf_output.writeHeaders())
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.stories, SecurityContentType.stories))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.baselines, SecurityContentType.baselines))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.investigations, SecurityContentType.investigations))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards))
+            updated_conf_files.update(
+                conf_output.writeLookups(input_dto.director_output_dto.lookups)
+            )
+            updated_conf_files.update(
+                conf_output.writeDetections(input_dto.director_output_dto.detections)
+            )
+            updated_conf_files.update(
+                conf_output.writeStories(input_dto.director_output_dto.stories)
+            )
+            updated_conf_files.update(
+                conf_output.writeBaselines(input_dto.director_output_dto.baselines)
+            )
+            updated_conf_files.update(
+                conf_output.writeInvestigations(
+                    input_dto.director_output_dto.investigations
+                )
+            )
+            updated_conf_files.update(
+                conf_output.writeMacros(input_dto.director_output_dto.macros)
+            )
+            updated_conf_files.update(
+                conf_output.writeDashboards(input_dto.director_output_dto.dashboards)
+            )
             updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
-            
 
-            
-            #Ensure that the conf file we just generated/update is syntactically valid
+            # Ensure that the conf file we just generated/update is syntactically valid
             for conf_file in updated_conf_files:
-                ConfWriter.validateConfFile(conf_file) 
-                
+                ConfWriter.validateConfFile(conf_file)
+
             conf_output.packageApp()
 
-            print(f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}")
-        
+            print(
+                f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}"
+            )
 
-        if input_dto.config.build_api:    
+        if input_dto.config.build_api:
             shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True)
             input_dto.config.getAPIPath().mkdir(parents=True)
-            api_json_output = ApiJsonOutput()
-            for output_objects, output_type in [(input_dto.director_output_dto.detections, SecurityContentType.detections),
-                                                (input_dto.director_output_dto.stories, SecurityContentType.stories),
-                                                (input_dto.director_output_dto.baselines, SecurityContentType.baselines),
-                                                (input_dto.director_output_dto.investigations, SecurityContentType.investigations),
-                                                (input_dto.director_output_dto.lookups, SecurityContentType.lookups),
-                                                (input_dto.director_output_dto.macros, SecurityContentType.macros),
-                                                (input_dto.director_output_dto.deployments, SecurityContentType.deployments)]:
-                api_json_output.writeObjects(output_objects, input_dto.config.getAPIPath(), input_dto.config.app.label, output_type )
-            
-           
-            
-            #create version file for sse api
-            version_file = input_dto.config.getAPIPath()/"version.json"
-            utc_time = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0,tzinfo=None).isoformat()
-            version_dict = {"version":{"name":f"v{input_dto.config.app.version}","published_at": f"{utc_time}Z"  }}
-            with open(version_file,"w") as version_f:
-                json.dump(version_dict,version_f)
-            
-            print(f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}")
-
-        return input_dto.director_output_dto
+            api_json_output = ApiJsonOutput(
+                input_dto.config.getAPIPath(), input_dto.config.app.label
+            )
+            api_json_output.writeDetections(input_dto.director_output_dto.detections)
+            api_json_output.writeStories(input_dto.director_output_dto.stories)
+            api_json_output.writeBaselines(input_dto.director_output_dto.baselines)
+            api_json_output.writeInvestigations(
+                input_dto.director_output_dto.investigations
+            )
+            api_json_output.writeLookups(input_dto.director_output_dto.lookups)
+            api_json_output.writeMacros(input_dto.director_output_dto.macros)
+            api_json_output.writeDeployments(input_dto.director_output_dto.deployments)
+
+            # create version file for sse api
+            version_file = input_dto.config.getAPIPath() / "version.json"
+            utc_time = (
+                datetime.datetime.now(datetime.timezone.utc)
+                .replace(microsecond=0, tzinfo=None)
+                .isoformat()
+            )
+            version_dict = {
+                "version": {
+                    "name": f"v{input_dto.config.app.version}",
+                    "published_at": f"{utc_time}Z",
+                }
+            }
+            with open(version_file, "w") as version_f:
+                json.dump(version_dict, version_f)
+
+            print(
+                f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}"
+            )
+
+        return input_dto.director_output_dto

contentctl/actions/deploy_acs.py

@@ -4,52 +4,57 @@ import pprint
 
 
 class Deploy:
-    def execute(self, config: deploy_acs, appinspect_token:str) -> None:
-        
-        #The following common headers are used by both Clasic and Victoria
+    def execute(self, config: deploy_acs, appinspect_token: str) -> None:
+        # The following common headers are used by both Clasic and Victoria
         headers = {
-            'Authorization': f'Bearer {config.splunk_cloud_jwt_token}',
-            'ACS-Legal-Ack': 'Y'
+            "Authorization": f"Bearer {config.splunk_cloud_jwt_token}",
+            "ACS-Legal-Ack": "Y",
         }
         try:
-            
-            with open(config.getPackageFilePath(include_version=False),'rb') as app_data:
-                #request_data = app_data.read()
+            with open(
+                config.getPackageFilePath(include_version=False), "rb"
+            ) as app_data:
+                # request_data = app_data.read()
                 if config.stack_type == StackType.classic:
                     # Classic instead uses a form to store token and package
                     # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
                     address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
-                    
-                    form_data = {
-                        'token': (None, appinspect_token),
-                        'package': app_data
-                    }
-                    res = post(address, headers=headers, files = form_data)
+
+                    form_data = {"token": (None, appinspect_token), "package": app_data}
+                    res = post(address, headers=headers, files=form_data)
                 elif config.stack_type == StackType.victoria:
                     # Victoria uses the X-Splunk-Authorization Header
                     # It also uses --data-binary for the app content
                     # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
-                    headers.update({'X-Splunk-Authorization':  appinspect_token})
+                    headers.update({"X-Splunk-Authorization": appinspect_token})
                     address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
                     res = post(address, headers=headers, data=app_data.read())
                 else:
                     raise Exception(f"Unsupported stack type: '{config.stack_type}'")
         except Exception as e:
-            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}")
-        
+            raise Exception(
+                f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}"
+            )
+
         try:
             # Request went through and completed, but may have returned a non-successful error code.
             # This likely includes a more verbose response describing the error
             res.raise_for_status()
             print(res.json())
-        except Exception as e:
+        except Exception:
             try:
                 error_text = res.json()
-            except Exception as e:
+            except Exception:
                 error_text = "No error text - request failed"
             formatted_error_text = pprint.pformat(error_text)
-            print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
-                  "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.")
-            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
-        
-        print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")
+            print(
+                "While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
+                "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase."
+            )
+            raise Exception(
+                f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}"
+            )
+
+        print(
+            f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!"
+        )

contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -1,26 +1,29 @@
-from typing import List,Union
-from contentctl.objects.config import test, test_servers, Container,Infrastructure
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer
-from urllib.parse import urlparse
-from copy import deepcopy
-from contentctl.objects.enums import DetectionTestingTargetInfrastructure
+from typing import List, Union
+from contentctl.objects.config import test, test_servers, Container, Infrastructure
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
+    DetectionTestingInfrastructure,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
+    DetectionTestingInfrastructureContainer,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
+    DetectionTestingInfrastructureServer,
+)
 import signal
 import datetime
+
 # from queue import Queue
 from dataclasses import dataclass
+
 # import threading
-import ctypes
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
-    DetectionTestingInfrastructure,
     DetectionTestingManagerOutputDto,
 )
 from contentctl.actions.detection_testing.views.DetectionTestingView import (
     DetectionTestingView,
 )
 from contentctl.objects.enums import PostTestBehavior
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from contentctl.objects.detection import Detection
 import concurrent.futures
 import docker
@@ -28,7 +31,7 @@ import docker
 
 @dataclass(frozen=False)
 class DetectionTestingManagerInputDto:
-    config: Union[test,test_servers]
+    config: Union[test, test_servers]
     detections: List[Detection]
     views: list[DetectionTestingView]
 
@@ -65,15 +68,18 @@ class DetectionTestingManager(BaseModel):
                 print("*******************************")
 
         signal.signal(signal.SIGINT, sigint_handler)
-        
-        with concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.config.test_instances),
-        ) as instance_pool, concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.views)
-        ) as view_runner, concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.config.test_instances),
-        ) as view_shutdowner:
 
+        with (
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.config.test_instances),
+            ) as instance_pool,
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.views)
+            ) as view_runner,
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.config.test_instances),
+            ) as view_shutdowner,
+        ):
             # Start all the views
             future_views = {
                 view_runner.submit(view.setup): view for view in self.input_dto.views
@@ -87,7 +93,7 @@ class DetectionTestingManager(BaseModel):
             # Wait for all instances to be set up
             for future in concurrent.futures.as_completed(future_instances_setup):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     self.output_dto.terminate = True
                     print(f"Error setting up container: {str(e)}")
@@ -102,7 +108,7 @@ class DetectionTestingManager(BaseModel):
                 # Wait for execution to finish
                 for future in concurrent.futures.as_completed(future_instances_execute):
                     try:
-                        result = future.result()
+                        future.result()
                     except Exception as e:
                         self.output_dto.terminate = True
                         print(f"Error running in container: {str(e)}")
@@ -115,34 +121,43 @@ class DetectionTestingManager(BaseModel):
             }
             for future in concurrent.futures.as_completed(future_views_shutdowner):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     print(f"Error stopping view: {str(e)}")
 
             # Wait for original view-related threads to complete
             for future in concurrent.futures.as_completed(future_views):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     print(f"Error running container: {str(e)}")
 
         return self.output_dto
 
     def create_DetectionTestingInfrastructureObjects(self):
-        #Make sure that, if we need to, we pull the appropriate container
+        # Make sure that, if we need to, we pull the appropriate container
         for infrastructure in self.input_dto.config.test_instances:
-            if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+            if isinstance(self.input_dto.config, test) and isinstance(
+                infrastructure, Container
+            ):
                 try:
                     client = docker.from_env()
-                except Exception as e:
-                    raise Exception("Unable to connect to docker.  Are you sure that docker is running on this host?")
+                except Exception:
+                    raise Exception(
+                        "Unable to connect to docker.  Are you sure that docker is running on this host?"
+                    )
                 try:
-                    
-                    parts = self.input_dto.config.container_settings.full_image_path.split(':')
+                    parts = (
+                        self.input_dto.config.container_settings.full_image_path.split(
+                            ":"
+                        )
+                    )
                     if len(parts) != 2:
-                        raise Exception(f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
-                                        f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'")
-                    
+                        raise Exception(
+                            f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
+                            f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'"
+                        )
+
                     print(
                         f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...",
                         end="",
@@ -152,12 +167,15 @@ class DetectionTestingManager(BaseModel):
                     print("done!")
                     break
                 except Exception as e:
-                    raise Exception(f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}")
+                    raise Exception(
+                        f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
+                    )
 
         already_staged_container_files = False
         for infrastructure in self.input_dto.config.test_instances:
-
-            if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+            if isinstance(self.input_dto.config, test) and isinstance(
+                infrastructure, Container
+            ):
                 # Stage the files in the apps dir so that they can be passed directly to
                 # subsequent containers. Do this here, instead of inside each container, to
                 # avoid duplicate downloads/moves/copies
@@ -167,18 +185,24 @@ class DetectionTestingManager(BaseModel):
 
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureContainer(
-                        global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
+                        global_config=self.input_dto.config,
+                        infrastructure=infrastructure,
+                        sync_obj=self.output_dto,
                     )
                 )
 
-            elif (isinstance(self.input_dto.config, test_servers) and isinstance(infrastructure, Infrastructure)):
+            elif isinstance(self.input_dto.config, test_servers) and isinstance(
+                infrastructure, Infrastructure
+            ):
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureServer(
-                        global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
+                        global_config=self.input_dto.config,
+                        infrastructure=infrastructure,
+                        sync_obj=self.output_dto,
                     )
                 )
 
             else:
-
-                raise Exception(f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}")
-                
+                raise Exception(
+                    f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}"
+                )