contentctl 4.4.7__py3-none-any.whl → 5.0.0__py3-none-any.whl

contentctl/actions/test.py

@@ -1,11 +1,9 @@
 from dataclasses import dataclass
 from typing import List
 
-from contentctl.objects.config import test_common
-from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType
+from contentctl.objects.config import test_common, Selected, Changes
 from contentctl.objects.detection import Detection
 
-from contentctl.input.director import DirectorOutputDto
 
 from contentctl.actions.detection_testing.DetectionTestingManager import (
     DetectionTestingManager,
@@ -41,7 +39,7 @@ MAXIMUM_CONFIGURATION_TIME_SECONDS = 600
 class TestInputDto:
     detections: List[Detection]
     config: test_common
-    
+
 
 class Test:
     def filter_tests(self, input_dto: TestInputDto) -> None:
@@ -52,7 +50,7 @@ class Test:
         Args:
             input_dto (TestInputDto): A configuration of the test and all of the
             tests to be run.
-        """        
+        """
 
         if not input_dto.config.enable_integration_testing:
             # Skip all integraiton tests if integration testing is not enabled:
@@ -61,7 +59,6 @@ class Test:
                     if isinstance(test, IntegrationTest):
                         test.skip("TEST SKIPPED: Skipping all integration tests")
 
-        
     def execute(self, input_dto: TestInputDto) -> bool:
         output_dto = DetectionTestingManagerOutputDto()
 
@@ -78,10 +75,9 @@ class Test:
             input_dto=manager_input_dto, output_dto=output_dto
         )
 
-        mode = input_dto.config.getModeName()
         if len(input_dto.detections) == 0:
             print(
-                f"With Detection Testing Mode '{mode}', there were [0] detections found to test."
+                f"With Detection Testing Mode '{input_dto.config.mode.mode_name}', there were [0] detections found to test."
                 "\nAs such, we will quit immediately."
             )
             # Directly call stop so that the summary.yml will be generated. Of course it will not
@@ -89,10 +85,21 @@ class Test:
             # detections were tested.
             file.stop()
         else:
-            print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections")
-            if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]:
-                files_string = '\n- '.join(
-                    [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections]
+            print(
+                f"MODE: [{input_dto.config.mode.mode_name}] - Test [{len(input_dto.detections)}] detections"
+            )
+            if isinstance(input_dto.config.mode, Selected) or isinstance(
+                input_dto.config.mode, Changes
+            ):
+                files_string = "\n- ".join(
+                    [
+                        str(
+                            pathlib.Path(detection.file_path).relative_to(
+                                input_dto.config.path
+                            )
+                        )
+                        for detection in input_dto.detections
+                    ]
                 )
                 print(f"Detections:\n- {files_string}")
 
@@ -103,43 +110,41 @@ class Test:
             summary_results = file.getSummaryObject()
             summary = summary_results.get("summary", {})
 
-            print(f"Test Summary (mode: {summary.get('mode','Error')})")
-            print(f"\tSuccess                      : {summary.get('success',False)}")
-            print(
-                f"\tSuccess Rate                 : {summary.get('success_rate','ERROR')}"
-            )
+            print(f"Test Summary (mode: {summary.get('mode', 'Error')})")
+            print(f"\tSuccess                      : {summary.get('success', False)}")
             print(
-                f"\tTotal Detections             : {summary.get('total_detections','ERROR')}"
+                f"\tSuccess Rate                 : {summary.get('success_rate', 'ERROR')}"
             )
             print(
-                f"\tTotal Tested Detections      : {summary.get('total_tested_detections','ERROR')}"
+                f"\tTotal Detections             : {summary.get('total_detections', 'ERROR')}"
             )
             print(
-                f"\t  Passed Detections          : {summary.get('total_pass','ERROR')}"
+                f"\tTotal Tested Detections      : {summary.get('total_tested_detections', 'ERROR')}"
             )
             print(
-                f"\t  Failed Detections          : {summary.get('total_fail','ERROR')}"
+                f"\t  Passed Detections          : {summary.get('total_pass', 'ERROR')}"
             )
             print(
-                f"\tSkipped Detections           : {summary.get('total_skipped','ERROR')}"
+                f"\t  Failed Detections          : {summary.get('total_fail', 'ERROR')}"
             )
             print(
-                "\tProduction Status            :"
+                f"\tSkipped Detections           : {summary.get('total_skipped', 'ERROR')}"
             )
+            print("\tProduction Status            :")
             print(
-                f"\t  Production Detections      : {summary.get('total_production','ERROR')}"
+                f"\t  Production Detections      : {summary.get('total_production', 'ERROR')}"
             )
             print(
-                f"\t  Experimental Detections    : {summary.get('total_experimental','ERROR')}"
+                f"\t  Experimental Detections    : {summary.get('total_experimental', 'ERROR')}"
             )
             print(
-                f"\t  Deprecated Detections      : {summary.get('total_deprecated','ERROR')}"
+                f"\t  Deprecated Detections      : {summary.get('total_deprecated', 'ERROR')}"
             )
             print(
-                f"\tManually Tested Detections : {summary.get('total_manual','ERROR')}"
+                f"\tManually Tested Detections : {summary.get('total_manual', 'ERROR')}"
             )
             print(
-                f"\tUntested Detections          : {summary.get('total_untested','ERROR')}"
+                f"\tUntested Detections          : {summary.get('total_untested', 'ERROR')}"
             )
             print(f"\tTest Results File            : {file.getOutputFilePath()}")
             print(

contentctl/actions/validate.py

@@ -1,4 +1,3 @@
-
 import pathlib
 
 from contentctl.input.director import Director, DirectorOutputDto
@@ -6,6 +5,7 @@ from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.objects.atomic import AtomicEnrichment
+from contentctl.objects.lookup import FileBackedLookup
 from contentctl.helper.utils import Utils
 from contentctl.objects.data_source import DataSource
 from contentctl.helper.splunk_app import SplunkApp
@@ -26,7 +26,7 @@ class Validate:
             [],
             [],
             [],
-            []
+            [],
         )
 
         director = Director(director_output_dto)
@@ -34,51 +34,69 @@ class Validate:
         self.ensure_no_orphaned_files_in_lookups(input_dto.path, director_output_dto)
         if input_dto.data_source_TA_validation:
             self.validate_latest_TA_information(director_output_dto.data_sources)
-            
+
         return director_output_dto
 
-    
-    def ensure_no_orphaned_files_in_lookups(self, repo_path:pathlib.Path, director_output_dto:DirectorOutputDto):
+    def ensure_no_orphaned_files_in_lookups(
+        self, repo_path: pathlib.Path, director_output_dto: DirectorOutputDto
+    ):
         """
         This function ensures that only files which are relevant to lookups are included in the lookups folder.
         This means that a file must be either:
         1. A lookup YML (.yml)
         2. A lookup CSV (.csv) which is referenced by a YML
         3. A lookup MLMODEL (.mlmodel) which is referenced by a YML.
-        
+
         All other files, includes CSV and MLMODEL files which are NOT
         referenced by a YML, will generate an exception from this function.
-        
+
         Args:
             repo_path (pathlib.Path): path to the root of the app
             director_output_dto (DirectorOutputDto): director object with all constructed content
 
         Raises:
-            Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel 
-            files in this directory. Additionally, an exception will be raised if there 
-            exists one or more .csv or .mlmodel files that are not referenced by at least 1 
-            detection .yml file in this directory. 
+            Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel
+            files in this directory. Additionally, an exception will be raised if there
+            exists one or more .csv or .mlmodel files that are not referenced by at least 1
+            detection .yml file in this directory.
             This avoids having additional, unused files in this directory that may be copied into
             the app when it is built (which can cause appinspect errors or larger app size.)
-        """        
-        lookupsDirectory = repo_path/"lookups"
-        
+        """
+        lookupsDirectory = repo_path / "lookups"
+
         # Get all of the files referneced by Lookups
-        usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if lookup.filename is not None] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None]
+        usedLookupFiles: list[pathlib.Path] = [
+            lookup.filename
+            for lookup in director_output_dto.lookups
+            if isinstance(lookup, FileBackedLookup)
+        ] + [
+            lookup.file_path
+            for lookup in director_output_dto.lookups
+            if lookup.file_path is not None
+        ]
 
         # Get all of the mlmodel and csv files in the lookups directory
-        csvAndMlmodelFiles  = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"])
-        
+        csvAndMlmodelFiles = Utils.get_security_content_files_from_directory(
+            lookupsDirectory,
+            allowedFileExtensions=[".yml", ".csv", ".mlmodel"],
+            fileExtensionsToReturn=[".csv", ".mlmodel"],
+        )
+
         # Generate an exception of any csv or mlmodel files exist but are not used
-        unusedLookupFiles:list[pathlib.Path] = [testFile for testFile in csvAndMlmodelFiles if testFile not in usedLookupFiles]
+        unusedLookupFiles: list[pathlib.Path] = [
+            testFile
+            for testFile in csvAndMlmodelFiles
+            if testFile not in usedLookupFiles
+        ]
         if len(unusedLookupFiles) > 0:
-            raise Exception(f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}")
+            raise Exception(
+                f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}"
+            )
         return
-    
 
     def validate_latest_TA_information(self, data_sources: list[DataSource]) -> None:
         validated_TAs: list[tuple[str, str]] = []
-        errors:list[str] = []
+        errors: list[str] = []
         print("----------------------")
         print("Validating latest TA:")
         print("----------------------")
@@ -89,22 +107,25 @@ class Validate:
                     continue
                 if supported_TA.url is not None:
                     validated_TAs.append(ta_identifier)
-                    uid = int(str(supported_TA.url).rstrip('/').split("/")[-1])
+                    uid = int(str(supported_TA.url).rstrip("/").split("/")[-1])
                     try:
                         splunk_app = SplunkApp(app_uid=uid)
                         if splunk_app.latest_version != supported_TA.version:
-                            errors.append(f"Version mismatch in '{data_source.file_path}' supported TA '{supported_TA.name}'"
-                                          f"\n  Latest version on Splunkbase    : {splunk_app.latest_version}"
-                                          f"\n  Version specified in data source: {supported_TA.version}")
+                            errors.append(
+                                f"Version mismatch in '{data_source.file_path}' supported TA '{supported_TA.name}'"
+                                f"\n  Latest version on Splunkbase    : {splunk_app.latest_version}"
+                                f"\n  Version specified in data source: {supported_TA.version}"
+                            )
                     except Exception as e:
-                        errors.append(f"Error processing checking version of TA {supported_TA.name}: {str(e)}")
-                            
+                        errors.append(
+                            f"Error processing checking version of TA {supported_TA.name}: {str(e)}"
+                        )
+
         if len(errors) > 0:
-            errorString = '\n\n'.join(errors)
-            raise Exception(f"[{len(errors)}] or more TA versions are out of date or have other errors."
-                            f"Please update the following data sources with the latest versions of "
-                            f"their supported tas:\n\n{errorString}")
+            errorString = "\n\n".join(errors)
+            raise Exception(
+                f"[{len(errors)}] or more TA versions are out of date or have other errors."
+                f"Please update the following data sources with the latest versions of "
+                f"their supported tas:\n\n{errorString}"
+            )
         print("All TA versions are up to date.")
-        
-
-

contentctl/api.py

@@ -5,42 +5,48 @@ from contentctl.objects.config import test_common, test, test_servers
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.input.director import DirectorOutputDto
 
-def config_from_file(path:Path=Path("contentctl.yml"), config: dict[str,Any]={}, 
-                   configType:Type[Union[test,test_servers]]=test)->test_common:
-    
+
+def config_from_file(
+    path: Path = Path("contentctl.yml"),
+    config: dict[str, Any] = {},
+    configType: Type[Union[test, test_servers]] = test,
+) -> test_common:
     """
     Fetch a configuration object that can be used for a number of different contentctl
-    operations including validate, build, inspect, test, and test_servers. A file will 
+    operations including validate, build, inspect, test, and test_servers. A file will
     be used as the basis for constructing the configuration.
 
     Args:
-        path (Path, optional): Relative or absolute path to a contentctl config file. 
+        path (Path, optional): Relative or absolute path to a contentctl config file.
         Defaults to Path("contentctl.yml"), which is the default name and location (in the current directory)
         of the configuration files which are automatically generated for contentctl.
         config (dict[], optional): Dictionary of values to override values read from the YML
         path passed as the first argument. Defaults to {}, an empty dict meaning that nothing
-        will be overwritten 
-        configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. 
+        will be overwritten
+        configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate.
         This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test.
     Returns:
         test_common: Returns a complete contentctl test_common configuration. Note that this configuration
         will have all applicable field for validate and build as well, but can also be used for easily
-        construction a test or test_servers object.  
-    """    
+        construction a test or test_servers object.
+    """
 
     try:
         yml_dict = YmlReader.load_file(path, add_fields=False)
-        
-        
+
     except Exception as e:
-        raise Exception(f"Failed to load contentctl configuration from file '{path}': {str(e)}")
-    
+        raise Exception(
+            f"Failed to load contentctl configuration from file '{path}': {str(e)}"
+        )
+
     # Apply settings that have been overridden from the ones in the file
     try:
         yml_dict.update(config)
     except Exception as e:
-        raise Exception(f"Failed updating dictionary of values read from file '{path}'"
-                        f" with the dictionary of arguments passed: {str(e)}")
+        raise Exception(
+            f"Failed updating dictionary of values read from file '{path}'"
+            f" with the dictionary of arguments passed: {str(e)}"
+        )
 
     # The function below will throw its own descriptive exception if it fails
     configObject = config_from_dict(yml_dict, configType=configType)
@@ -48,13 +54,12 @@ def config_from_file(path:Path=Path("contentctl.yml"), config: dict[str,Any]={},
     return configObject
 
 
-
-
-def config_from_dict(config: dict[str,Any]={}, 
-                   configType:Type[Union[test,test_servers]]=test)->test_common:
+def config_from_dict(
+    config: dict[str, Any] = {}, configType: Type[Union[test, test_servers]] = test
+) -> test_common:
     """
     Fetch a configuration object that can be used for a number of different contentctl
-    operations including validate, build, inspect, test, and test_servers. A dict will 
+    operations including validate, build, inspect, test, and test_servers. A dict will
     be used as the basis for constructing the configuration.
 
     Args:
@@ -63,29 +68,30 @@ def config_from_dict(config: dict[str,Any]={},
         values.  Note that based on default values in the contentctl/objects/config.py
         file, this may raise an exception.  If so, please set appropriate default values
         in the file above or supply those values via this argument.
-        configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. 
+        configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate.
         This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test.
     Returns:
         test_common: Returns a complete contentctl test_common configuration. Note that this configuration
         will have all applicable field for validate and build as well, but can also be used for easily
-        construction a test or test_servers object.  
-    """    
+        construction a test or test_servers object.
+    """
     try:
         test_object = configType.model_validate(config)
     except Exception as e:
         raise Exception(f"Failed to load contentctl configuration from dict:\n{str(e)}")
-    
+
     return test_object
 
 
-def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,Any])->test_common:
-    
+def update_config(
+    config: Union[test, test_servers], **key_value_updates: dict[str, Any]
+) -> test_common:
     """Update any relevant keys in a config file with the specified values.
     Full validation will be performed after this update and descriptive errors
     will be produced
 
     Args:
-        config (test_common): A previously-constructed test_common object.  This can be 
+        config (test_common): A previously-constructed test_common object.  This can be
         build using the configFromDict or configFromFile functions.
         key_value_updates (kwargs, optional): Additional keyword/argument pairs to update
         arbitrary fields in the configuration.
@@ -101,37 +107,40 @@ def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,
 
     # Force validation of assignment since doing so via arbitrary dict can be error prone
     # Also, ensure that we do not try to add fields that are not part of the model
-    config_copy.model_config.update({'validate_assignment': True, 'extra': 'forbid'})
+    config_copy.model_config.update({"validate_assignment": True, "extra": "forbid"})
 
-    
-    
     # Collect any errors that may occur
-    errors:list[Exception] = []
-    
-    # We need to do this one by one because the extra:forbid argument does not appear to 
+    errors: list[Exception] = []
+
+    # We need to do this one by one because the extra:forbid argument does not appear to
     # be respected at this time.
     for key, value in key_value_updates.items():
         try:
-            setattr(config_copy,key,value)
+            setattr(config_copy, key, value)
         except Exception as e:
             errors.append(e)
     if len(errors) > 0:
-        errors_string = '\n'.join([str(e) for e in errors])
+        errors_string = "\n".join([str(e) for e in errors])
         raise Exception(f"Error(s) updaitng configuration:\n{errors_string}")
-    
+
     return config_copy
-    
 
 
-def content_to_dict(director:DirectorOutputDto)->dict[str,list[dict[str,Any]]]:
-    output_dict:dict[str,list[dict[str,Any]]] = {}
-    for contentType in ['detections','stories','baselines','investigations',
-                        'playbooks','macros','lookups','deployments',]:
-        
+def content_to_dict(director: DirectorOutputDto) -> dict[str, list[dict[str, Any]]]:
+    output_dict: dict[str, list[dict[str, Any]]] = {}
+    for contentType in [
+        "detections",
+        "stories",
+        "baselines",
+        "investigations",
+        "playbooks",
+        "macros",
+        "lookups",
+        "deployments",
+    ]:
         output_dict[contentType] = []
-        t:list[SecurityContentObject] = getattr(director,contentType)
-        
+        t: list[SecurityContentObject] = getattr(director, contentType)
+
         for item in t:
             output_dict[contentType].append(item.model_dump())
     return output_dict
-