contentctl 5.0.0a2__py3-none-any.whl → 5.0.0a3__py3-none-any.whl

contentctl/helper/utils.py

@@ -12,6 +12,7 @@ import tqdm
 from math import ceil
 
 from typing import TYPE_CHECKING
+
 if TYPE_CHECKING:
     from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.security_content_object import SecurityContentObject
@@ -24,26 +25,29 @@ ALWAYS_PULL = True
 class Utils:
     @staticmethod
     def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]:
-        listOfFiles:list[pathlib.Path] = []
+        listOfFiles: list[pathlib.Path] = []
         base_path = pathlib.Path(path)
         if not base_path.exists():
             return listOfFiles
-        for (dirpath, dirnames, filenames) in os.walk(path):
+        for dirpath, dirnames, filenames in os.walk(path):
             for file in filenames:
                 if file.endswith(".yml"):
                     listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
-    
+
         return sorted(listOfFiles)
-    
+
     @staticmethod
-    def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExtensions:list[str]=[".yml"], fileExtensionsToReturn:list[str]=[".yml"]) -> list[pathlib.Path]:
-    
+    def get_security_content_files_from_directory(
+        path: pathlib.Path,
+        allowedFileExtensions: list[str] = [".yml"],
+        fileExtensionsToReturn: list[str] = [".yml"],
+    ) -> list[pathlib.Path]:
         """
         Get all of the Security Content Object Files rooted in a given directory.  These will almost
         certain be YML files, but could be other file types as specified by the user
 
         Args:
-            path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed. 
+            path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed.
             allowedFileExtensions (set[str], optional): File extensions which are allowed to be  present in this directory.  In most cases, we do not want to allow the presence of non-YML files. Defaults to [".yml"].
             fileExtensionsToReturn (set[str], optional): Filenames with extensions that should be returned from this function. For example, the lookups/ directory contains YML, CSV, and MLMODEL directories, but only the YMLs are Security Content Objects for constructing Lookyps. Defaults to[".yml"].
 
@@ -56,14 +60,18 @@ class Utils:
             list[pathlib.Path]: list of files with an extension in fileExtensionsToReturn found in path
         """
         if not set(fileExtensionsToReturn).issubset(set(allowedFileExtensions)):
-            raise Exception(f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not")
-        
+            raise Exception(
+                f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not"
+            )
+
         if not path.exists() or not path.is_dir():
-            raise Exception(f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory")
-        
-        allowedFiles:list[pathlib.Path] = []
-        erroneousFiles:list[pathlib.Path] = []
-        #Get every single file extension 
+            raise Exception(
+                f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory"
+            )
+
+        allowedFiles: list[pathlib.Path] = []
+        erroneousFiles: list[pathlib.Path] = []
+        # Get every single file extension
         for filePath in path.glob("**/*.*"):
             if filePath.suffix in allowedFileExtensions:
                 # Yes these are allowed
@@ -73,58 +81,75 @@ class Utils:
                 erroneousFiles.append(filePath)
 
         if len(erroneousFiles):
-            raise Exception(f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}")
-        
+            raise Exception(
+                f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}"
+            )
+
         # There were no errorneous files, so return the requested files
-        return sorted([filePath for filePath in allowedFiles if filePath.suffix in fileExtensionsToReturn])
+        return sorted(
+            [
+                filePath
+                for filePath in allowedFiles
+                if filePath.suffix in fileExtensionsToReturn
+            ]
+        )
 
     @staticmethod
-    def get_all_yml_files_from_directory_one_layer_deep(path: str) -> list[pathlib.Path]:
+    def get_all_yml_files_from_directory_one_layer_deep(
+        path: str,
+    ) -> list[pathlib.Path]:
         listOfFiles: list[pathlib.Path] = []
         base_path = pathlib.Path(path)
         if not base_path.exists():
             return listOfFiles
         # Check the base directory
         for item in base_path.iterdir():
-            if item.is_file() and item.suffix == '.yml':
+            if item.is_file() and item.suffix == ".yml":
                 listOfFiles.append(item)
         # Check one subfolder level deep
         for subfolder in base_path.iterdir():
             if subfolder.is_dir() and subfolder.name != "cim":
                 for item in subfolder.iterdir():
-                    if item.is_file() and item.suffix == '.yml':
+                    if item.is_file() and item.suffix == ".yml":
                         listOfFiles.append(item)
         return sorted(listOfFiles)
 
-
     @staticmethod
-    def add_id(id_dict:dict[str, list[pathlib.Path]], obj:SecurityContentObject, path:pathlib.Path) -> None:     
+    def add_id(
+        id_dict: dict[str, list[pathlib.Path]],
+        obj: SecurityContentObject,
+        path: pathlib.Path,
+    ) -> None:
         if hasattr(obj, "id"):
             obj_id = obj.id
             if obj_id in id_dict:
                 id_dict[obj_id].append(path)
             else:
                 id_dict[obj_id] = [path]
+
     # Otherwise, no ID so nothing to add....
 
     @staticmethod
-    def check_ids_for_duplicates(id_dict:dict[str, list[pathlib.Path]])->list[Tuple[pathlib.Path,  ValueError]]:
-        validation_errors:list[Tuple[pathlib.Path,  ValueError]] = []
-        
+    def check_ids_for_duplicates(
+        id_dict: dict[str, list[pathlib.Path]],
+    ) -> list[Tuple[pathlib.Path, ValueError]]:
+        validation_errors: list[Tuple[pathlib.Path, ValueError]] = []
+
         for key, values in id_dict.items():
             if len(values) > 1:
                 error_file_path = pathlib.Path("MULTIPLE")
-                all_files = '\n\t'.join(str(pathlib.Path(p)) for p in values)
-                exception = ValueError(f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}")
+                all_files = "\n\t".join(str(pathlib.Path(p)) for p in values)
+                exception = ValueError(
+                    f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}"
+                )
                 validation_errors.append((error_file_path, exception))
-                
+
         return validation_errors
 
     @staticmethod
     def validate_git_hash(
         repo_path: str, repo_url: str, commit_hash: str, branch_name: Union[str, None]
     ) -> bool:
-
         # Get a list of all branches
         repo = git.Repo(repo_path)
         if commit_hash is None:
@@ -141,14 +166,14 @@ class Utils:
             # Note, of course, that a hash can be in 0, 1, more branches!
             for branch_string in all_branches_containing_hash:
                 if branch_string.split(" ")[0] == "*" and (
-                    branch_string.split(" ")[-1] == branch_name or branch_name == None
+                    branch_string.split(" ")[-1] == branch_name or branch_name is None
                 ):
                     # Yes, the hash exists in the branch (or branch_name was None and it existed in at least one branch)!
                     return True
             # If we get here, it does not exist in the given branch
             raise (Exception("Does not exist in branch"))
 
-        except Exception as e:
+        except Exception:
             if branch_name is None:
                 branch_name = "ANY_BRANCH"
             if ALWAYS_PULL:
@@ -251,7 +276,6 @@ class Utils:
     def verify_file_exists(
         file_path: str, verbose_print=False, timeout_seconds: int = 10
     ) -> None:
-
         try:
             if pathlib.Path(file_path).is_file():
                 # This is a file and we know it exists
@@ -261,18 +285,13 @@ class Utils:
 
         # Try to make a head request to verify existence of the file
         try:
-            
             req = requests.head(
                 file_path, timeout=timeout_seconds, verify=True, allow_redirects=True
             )
             if req.status_code > 400:
                 raise (Exception(f"Return code={req.status_code}"))
         except Exception as e:
-            raise (
-                Exception(
-                    f"HTTP Resolution Failed: {str(e)}"
-                )
-            )
+            raise (Exception(f"HTTP Resolution Failed: {str(e)}"))
 
     @staticmethod
     def copy_local_file(
@@ -376,7 +395,7 @@ class Utils:
             )
 
         try:
-            download_start_time = default_timer()
+            default_timer()
             bytes_written = 0
             file_to_download = requests.get(file_path, stream=True)
             file_to_download.raise_for_status()

contentctl/input/director.py

@@ -29,7 +29,7 @@ from contentctl.helper.utils import Utils
 
 @dataclass
 class DirectorOutputDto:
-    # Atomic Tests are first because parsing them 
+    # Atomic Tests are first because parsing them
     # is far quicker than attack_enrichment
     atomic_enrichment: AtomicEnrichment
     attack_enrichment: AttackEnrichment
@@ -50,20 +50,20 @@ class DirectorOutputDto:
 
     def addContentToDictMappings(self, content: SecurityContentObject):
         content_name = content.name
-        
-        
+
         if content_name in self.name_to_content_map:
             raise ValueError(
                 f"Duplicate name '{content_name}' with paths:\n"
                 f" - {content.file_path}\n"
                 f" - {self.name_to_content_map[content_name].file_path}"
             )
-    
+
         if content.id in self.uuid_to_content_map:
             raise ValueError(
                 f"Duplicate id '{content.id}' with paths:\n"
                 f" - {content.file_path}\n"
-                f" - {self.uuid_to_content_map[content.id].file_path}")
+                f" - {self.uuid_to_content_map[content.id].file_path}"
+            )
 
         if isinstance(content, Lookup):
             self.lookups.append(content)
@@ -82,7 +82,7 @@ class DirectorOutputDto:
         elif isinstance(content, Detection):
             self.detections.append(content)
         elif isinstance(content, Dashboard):
-            self.dashboards.append(content)            
+            self.dashboards.append(content)
 
         elif isinstance(content, DataSource):
             self.data_sources.append(content)
@@ -93,7 +93,7 @@ class DirectorOutputDto:
         self.uuid_to_content_map[content.id] = content
 
 
-class Director():
+class Director:
     input_dto: validate
     output_dto: DirectorOutputDto
 
@@ -112,13 +112,18 @@ class Director():
         self.createSecurityContent(SecurityContentType.playbooks)
         self.createSecurityContent(SecurityContentType.detections)
         self.createSecurityContent(SecurityContentType.dashboards)
-        
-        from contentctl.objects.abstract_security_content_objects.detection_abstract import MISSING_SOURCES
+
+        from contentctl.objects.abstract_security_content_objects.detection_abstract import (
+            MISSING_SOURCES,
+        )
+
         if len(MISSING_SOURCES) > 0:
             missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES)))
-            print("WARNING: The following data_sources have been used in detections, but are not yet defined.\n"
-                  "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 "
-                  f"{missing_sources_string}")
+            print(
+                "WARNING: The following data_sources have been used in detections, but are not yet defined.\n"
+                "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 "
+                f"{missing_sources_string}"
+            )
         else:
             print("No missing data_sources!")
 
@@ -133,18 +138,20 @@ class Director():
             SecurityContentType.playbooks,
             SecurityContentType.detections,
             SecurityContentType.data_sources,
-            SecurityContentType.dashboards
+            SecurityContentType.dashboards,
         ]:
             files = Utils.get_all_yml_files_from_directory(
                 os.path.join(self.input_dto.path, str(contentType.name))
             )
-            security_content_files = [
-                f for f in files 
-            ]
+            security_content_files = [f for f in files]
         else:
-            raise (Exception(f"Cannot createSecurityContent for unknown product {contentType}."))
+            raise (
+                Exception(
+                    f"Cannot createSecurityContent for unknown product {contentType}."
+                )
+            )
 
-        validation_errors:list[tuple[Path,ValueError]] = []
+        validation_errors: list[tuple[Path, ValueError]] = []
 
         already_ran = False
         progress_percent = 0
@@ -156,41 +163,67 @@ class Director():
                 modelDict = YmlReader.load_file(file)
 
                 if contentType == SecurityContentType.lookups:
-                    lookup = LookupAdapter.validate_python(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
-                    #lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
+                    lookup = LookupAdapter.validate_python(
+                        modelDict,
+                        context={
+                            "output_dto": self.output_dto,
+                            "config": self.input_dto,
+                        },
+                    )
+                    # lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
                     self.output_dto.addContentToDictMappings(lookup)
-                
+
                 elif contentType == SecurityContentType.macros:
-                    macro = Macro.model_validate(modelDict, context={"output_dto":self.output_dto})
+                    macro = Macro.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
                     self.output_dto.addContentToDictMappings(macro)
-                
+
                 elif contentType == SecurityContentType.deployments:
-                    deployment = Deployment.model_validate(modelDict, context={"output_dto":self.output_dto})
+                    deployment = Deployment.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
                     self.output_dto.addContentToDictMappings(deployment)
 
                 elif contentType == SecurityContentType.playbooks:
-                    playbook = Playbook.model_validate(modelDict, context={"output_dto":self.output_dto})
-                    self.output_dto.addContentToDictMappings(playbook)                  
-                
+                    playbook = Playbook.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
+                    self.output_dto.addContentToDictMappings(playbook)
+
                 elif contentType == SecurityContentType.baselines:
-                    baseline = Baseline.model_validate(modelDict, context={"output_dto":self.output_dto})
+                    baseline = Baseline.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
                     self.output_dto.addContentToDictMappings(baseline)
-                
+
                 elif contentType == SecurityContentType.investigations:
-                    investigation = Investigation.model_validate(modelDict, context={"output_dto":self.output_dto})
+                    investigation = Investigation.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
                     self.output_dto.addContentToDictMappings(investigation)
 
                 elif contentType == SecurityContentType.stories:
-                    story = Story.model_validate(modelDict, context={"output_dto":self.output_dto})
+                    story = Story.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
                     self.output_dto.addContentToDictMappings(story)
-            
+
                 elif contentType == SecurityContentType.detections:
-                    detection = Detection.model_validate(modelDict, context={"output_dto":self.output_dto, "app":self.input_dto.app})
+                    detection = Detection.model_validate(
+                        modelDict,
+                        context={
+                            "output_dto": self.output_dto,
+                            "app": self.input_dto.app,
+                        },
+                    )
                     self.output_dto.addContentToDictMappings(detection)
-                
+
                 elif contentType == SecurityContentType.dashboards:
-                        dashboard = Dashboard.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(dashboard)
+                    dashboard = Dashboard.model_validate(
+                        modelDict, context={"output_dto": self.output_dto}
+                    )
+                    self.output_dto.addContentToDictMappings(dashboard)
 
                 elif contentType == SecurityContentType.data_sources:
                     data_source = DataSource.model_validate(
@@ -237,4 +270,3 @@ class Director():
             raise Exception(
                 f"The following {len(validation_errors)} error(s) were found during validation:\n\n{errors_string}\n\nVALIDATION FAILED"
             )
-

contentctl/input/new_content_questions.py

@@ -3,9 +3,8 @@ from contentctl.objects.enums import DataSource
 
 
 class NewContentQuestions:
-
     @classmethod
-    def get_questions_detection(cls) -> list[dict[str,Any]]:
+    def get_questions_detection(cls) -> list[dict[str, Any]]:
         questions = [
             {
                 "type": "text",
@@ -14,22 +13,16 @@ class NewContentQuestions:
                 "default": "Powershell Encoded Command",
             },
             {
-                'type': 'select',
-                'message': 'what kind of detection is this',
-                'name': 'detection_kind',
-                'choices': [
-                    'endpoint',
-                    'cloud',
-                    'application',
-                    'network',
-                    'web'
-                ],
-                'default': 'endpoint'
+                "type": "select",
+                "message": "what kind of detection is this",
+                "name": "detection_kind",
+                "choices": ["endpoint", "cloud", "application", "network", "web"],
+                "default": "endpoint",
             },
             {
-                'type': 'text',
-                'message': 'enter author name',
-                'name': 'detection_author',
+                "type": "text",
+                "message": "enter author name",
+                "name": "detection_author",
             },
             {
                 "type": "select",
@@ -46,12 +39,11 @@ class NewContentQuestions:
                 "default": "TTP",
             },
             {
-                'type': 'checkbox',
-                'message': 'Your data source',
-                'name': 'data_sources',
-                #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory
-                'choices': sorted(DataSource._value2member_map_ )
-                
+                "type": "checkbox",
+                "message": "Your data source",
+                "name": "data_sources",
+                # In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory
+                "choices": sorted(DataSource._value2member_map_),
             },
             {
                 "type": "text",
@@ -66,24 +58,24 @@ class NewContentQuestions:
                 "default": "T1003.002",
             },
             {
-                'type': 'select',
-                'message': 'security_domain for detection',
-                'name': 'security_domain',
-                'choices': [
-                    'access',
-                    'endpoint',
-                    'network',
-                    'threat',
-                    'identity',
-                    'audit'
+                "type": "select",
+                "message": "security_domain for detection",
+                "name": "security_domain",
+                "choices": [
+                    "access",
+                    "endpoint",
+                    "network",
+                    "threat",
+                    "identity",
+                    "audit",
                 ],
-                'default': 'endpoint'
+                "default": "endpoint",
             },
         ]
         return questions
 
     @classmethod
-    def get_questions_story(cls)-> list[dict[str,Any]]:
+    def get_questions_story(cls) -> list[dict[str, Any]]:
         questions = [
             {
                 "type": "text",

contentctl/input/yml_reader.py

@@ -3,36 +3,43 @@ import yaml
 import sys
 import pathlib
 
-class YmlReader():
 
+class YmlReader:
     @staticmethod
-    def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING:bool=False) -> Dict[str,Any]:
+    def load_file(
+        file_path: pathlib.Path,
+        add_fields: bool = True,
+        STRICT_YML_CHECKING: bool = False,
+    ) -> Dict[str, Any]:
         try:
-            file_handler = open(file_path, 'r', encoding="utf-8")
-            
-            # The following code can help diagnose issues with duplicate keys or 
+            file_handler = open(file_path, "r", encoding="utf-8")
+
+            # The following code can help diagnose issues with duplicate keys or
             # poorly-formatted but still "compliant" YML.  This code should be
-            # enabled manually for debugging purposes. As such, strictyaml 
+            # enabled manually for debugging purposes. As such, strictyaml
             # library is intentionally excluded from the contentctl requirements
 
             if STRICT_YML_CHECKING:
                 import strictyaml
+
                 try:
-                    strictyaml.dirty_load(file_handler.read(), allow_flow_style = True)
+                    strictyaml.dirty_load(file_handler.read(), allow_flow_style=True)
                     file_handler.seek(0)
                 except Exception as e:
                     print(f"Error loading YML file {file_path}: {str(e)}")
                     sys.exit(1)
             try:
-                #Ideally we should use 
-                # from contentctl.actions.new_content import NewContent 
-                # and use NewContent.UPDATE_PREFIX, 
+                # Ideally we should use
+                # from contentctl.actions.new_content import NewContent
+                # and use NewContent.UPDATE_PREFIX,
                 # but there is a circular dependency right now which makes that difficult.
                 # We have instead hardcoded UPDATE_PREFIX
                 UPDATE_PREFIX = "__UPDATE__"
                 data = file_handler.read()
                 if UPDATE_PREFIX in data:
-                    raise Exception(f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required.")
+                    raise Exception(
+                        f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required."
+                    )
                 yml_obj = yaml.load(data, Loader=yaml.CSafeLoader)
             except yaml.YAMLError as exc:
                 print(exc)
@@ -41,12 +48,10 @@ class YmlReader():
         except OSError as exc:
             print(exc)
             sys.exit(1)
-        
-        if add_fields == False:
+
+        if add_fields is False:
             return yml_obj
-        
-        
-        yml_obj['file_path'] = str(file_path)
-    
+
+        yml_obj["file_path"] = str(file_path)
 
         return yml_obj