contentctl 4.4.6__py3-none-any.whl → 5.0.0a0__py3-none-any.whl

contentctl/actions/test.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import List
 
-from contentctl.objects.config import test_common
+from contentctl.objects.config import test_common, Selected, Changes
 from contentctl.objects.enums import DetectionTestingMode, DetectionStatus, AnalyticsType
 from contentctl.objects.detection import Detection
 
@@ -78,10 +78,9 @@ class Test:
             input_dto=manager_input_dto, output_dto=output_dto
         )
 
-        mode = input_dto.config.getModeName()
         if len(input_dto.detections) == 0:
             print(
-                f"With Detection Testing Mode '{mode}', there were [0] detections found to test."
+                f"With Detection Testing Mode '{input_dto.config.mode.mode_name}', there were [0] detections found to test."
                 "\nAs such, we will quit immediately."
             )
             # Directly call stop so that the summary.yml will be generated. Of course it will not
@@ -89,8 +88,8 @@ class Test:
             # detections were tested.
             file.stop()
         else:
-            print(f"MODE: [{mode}] - Test [{len(input_dto.detections)}] detections")
-            if mode in [DetectionTestingMode.changes.value, DetectionTestingMode.selected.value]:
+            print(f"MODE: [{input_dto.config.mode.mode_name}] - Test [{len(input_dto.detections)}] detections")
+            if isinstance(input_dto.config.mode,  Selected) or isinstance(input_dto.config.mode, Changes):
                 files_string = '\n- '.join(
                     [str(pathlib.Path(detection.file_path).relative_to(input_dto.config.path)) for detection in input_dto.detections]
                 )

contentctl/actions/validate.py

@@ -6,6 +6,7 @@ from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.objects.atomic import AtomicEnrichment
+from contentctl.objects.lookup import FileBackedLookup
 from contentctl.helper.utils import Utils
 from contentctl.objects.data_source import DataSource
 from contentctl.helper.splunk_app import SplunkApp
@@ -64,7 +65,7 @@ class Validate:
         lookupsDirectory = repo_path/"lookups"
         
         # Get all of the files referneced by Lookups
-        usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if lookup.filename is not None] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None]
+        usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if isinstance(lookup, FileBackedLookup)] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None]
 
         # Get all of the mlmodel and csv files in the lookups directory
         csvAndMlmodelFiles  = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"])

contentctl/contentctl.py

@@ -1,31 +1,39 @@
-import traceback
+import pathlib
 import sys
+import traceback
 import warnings
-import pathlib
+
 import tyro
 
-from contentctl.actions.initialize import Initialize
-from contentctl.objects.config import init, validate, build,  new, deploy_acs, test, test_servers, inspect, report, test_common, release_notes
-from contentctl.actions.validate import Validate
-from contentctl.actions.new_content import NewContent
+from contentctl.actions.build import Build, BuildInputDto, DirectorOutputDto
+from contentctl.actions.deploy_acs import Deploy
 from contentctl.actions.detection_testing.GitService import GitService
-from contentctl.actions.build import (
-     BuildInputDto,
-     DirectorOutputDto,
-     Build,
-)
-from contentctl.actions.test import Test
-from contentctl.actions.test import TestInputDto
-from contentctl.actions.reporting import ReportingInputDto, Reporting
+from contentctl.actions.initialize import Initialize
 from contentctl.actions.inspect import Inspect
-from contentctl.input.yml_reader import YmlReader
-from contentctl.actions.deploy_acs import Deploy
+from contentctl.actions.new_content import NewContent
 from contentctl.actions.release_notes import ReleaseNotes
+from contentctl.actions.reporting import Reporting, ReportingInputDto
+from contentctl.actions.test import Test, TestInputDto
+from contentctl.actions.validate import Validate
+from contentctl.input.yml_reader import YmlReader
+from contentctl.objects.config import (
+    build,
+    deploy_acs,
+    init,
+    inspect,
+    new,
+    release_notes,
+    report,
+    test,
+    test_common,
+    test_servers,
+    validate,
+)
 
 # def print_ascii_art():
 #     print(
 #         """
-# Running Splunk Security Content Control Tool (contentctl) 
+# Running Splunk Security Content Control Tool (contentctl)
 # ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
 # ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢶⠛⡇⠀⠀⠀⠀⠀⠀⣠⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
 # ⠀⠀⠀⠀⠀⠀⠀⠀⣀⠼⠖⠛⠋⠉⠉⠓⠢⣴⡻⣾⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
@@ -53,114 +61,137 @@ from contentctl.actions.release_notes import ReleaseNotes
 #     )
 
 
-
-
-def init_func(config:test):    
+def init_func(config: test):
     Initialize().execute(config)
 
 
-def validate_func(config:validate)->DirectorOutputDto:
+def validate_func(config: validate) -> DirectorOutputDto:
     validate = Validate()
     return validate.execute(config)
 
-def report_func(config:report)->None:
+
+def report_func(config: report) -> None:
     # First, perform validation. Remember that the validate
     # configuration is actually a subset of the build configuration
     director_output_dto = validate_func(config)
-    
-    r = Reporting() 
-    return r.execute(ReportingInputDto(director_output_dto=director_output_dto, 
-                                       config=config))
-    
 
-def build_func(config:build)->DirectorOutputDto:
+    r = Reporting()
+    return r.execute(
+        ReportingInputDto(director_output_dto=director_output_dto, config=config)
+    )
+
+
+def build_func(config: build) -> DirectorOutputDto:
     # First, perform validation. Remember that the validate
     # configuration is actually a subset of the build configuration
     director_output_dto = validate_func(config)
     builder = Build()
     return builder.execute(BuildInputDto(director_output_dto, config))
 
-def inspect_func(config:inspect)->str:
-    #Make sure that we have built the most recent version of the app
+
+def inspect_func(config: inspect) -> str:
+    # Make sure that we have built the most recent version of the app
     _ = build_func(config)
     inspect_token = Inspect().execute(config)
     return inspect_token
-    
 
-def release_notes_func(config:release_notes)->None:
+
+def release_notes_func(config: release_notes) -> None:
     ReleaseNotes().release_notes(config)
 
-def new_func(config:new):
-    NewContent().execute(config)
 
+def new_func(config: new):
+    NewContent().execute(config)
 
 
-def deploy_acs_func(config:deploy_acs):
+def deploy_acs_func(config: deploy_acs):
     print("Building and inspecting app...")
     token = inspect_func(config)
     print("App successfully built and inspected.")
     print("Deploying app...")
     Deploy().execute(config, token)
 
-def test_common_func(config:test_common):
+
+def test_common_func(config: test_common):
     if type(config) == test:
-        #construct the container Infrastructure objects
+        # construct the container Infrastructure objects
         config.getContainerInfrastructureObjects()
-        #otherwise, they have already been passed as servers
+        # otherwise, they have already been passed as servers
 
     director_output_dto = build_func(config)
-    gitServer = GitService(director=director_output_dto,config=config)
+    gitServer = GitService(director=director_output_dto, config=config)
     detections_to_test = gitServer.getContent()
 
-    
-
     test_input_dto = TestInputDto(detections_to_test, config)
-    
+
     t = Test()
     t.filter_tests(test_input_dto)
-    
+
     if config.plan_only:
-        #Emit the test plan and quit. Do not actually run the test
-        config.dumpCICDPlanAndQuit(gitServer.getHash(),test_input_dto.detections)
-        return 
-    
+        # Emit the test plan and quit. Do not actually run the test
+        config.dumpCICDPlanAndQuit(gitServer.getHash(), test_input_dto.detections)
+        return
+
     success = t.execute(test_input_dto)
-    
+
     if success:
-        #Everything passed!
+        # Everything passed!
         print("All tests have run successfully or been marked as 'skipped'")
         return
     raise Exception("There was at least one unsuccessful test")
 
+
+CONTENTCTL_5_WARNING = """
+*****************************************************************************
+WARNING - THIS IS AN ALPHA BUILD OF CONTENTCTL 5.
+THERE HAVE BEEN NUMEROUS CHANGES IN CONTENTCTL (ESPECIALLY TO YML FORMATS). 
+YOU ALMOST CERTAINLY DO NOT WANT TO USE THIS BUILD.
+IF YOU ENCOUNTER ERRORS, PLEASE USE THE LATEST CURRENTYLY SUPPORTED RELEASE: 
+
+CONTENTCTL==4.4.7 
+
+YOU HAVE BEEN WARNED!
+*****************************************************************************
+"""
+
+
 def main():
+    print(CONTENTCTL_5_WARNING)
     try:
         configFile = pathlib.Path("contentctl.yml")
-        
+
         # We MUST load a config (with testing info) object so that we can
         # properly construct the command line, including 'contentctl test' parameters.
         if not configFile.is_file():
-            if "init" not in sys.argv and "--help" not in sys.argv and "-h" not in sys.argv:
-                raise Exception(f"'{configFile}' not found in the current directory.\n"
-                                "Please ensure you are in the correct directory or run 'contentctl init' to create a new content pack.")
-            
+            if (
+                "init" not in sys.argv
+                and "--help" not in sys.argv
+                and "-h" not in sys.argv
+            ):
+                raise Exception(
+                    f"'{configFile}' not found in the current directory.\n"
+                    "Please ensure you are in the correct directory or run 'contentctl init' to create a new content pack."
+                )
+
             if "--help" in sys.argv or "-h" in sys.argv:
-                print("Warning - contentctl.yml is missing from this directory. The configuration values showed at the default and are informational only.\n"
-                      "Please ensure that contentctl.yml exists by manually creating it or running 'contentctl init'")
+                print(
+                    "Warning - contentctl.yml is missing from this directory. The configuration values showed at the default and are informational only.\n"
+                    "Please ensure that contentctl.yml exists by manually creating it or running 'contentctl init'"
+                )
             # Otherwise generate a stub config file.
             # It will be used during init workflow
 
             t = test()
             config_obj = t.model_dump()
-            
+
         else:
-            #The file exists, so load it up!
-            config_obj = YmlReader().load_file(configFile)
+            # The file exists, so load it up!
+            config_obj = YmlReader().load_file(configFile, add_fields=False)
             t = test.model_validate(config_obj)
     except Exception as e:
         print(f"Error validating 'contentctl.yml':\n{str(e)}")
         sys.exit(1)
-        
-    
+
     # For ease of generating the constructor, we want to allow construction
     # of an object from default values WITHOUT requiring all fields to be declared
     # with defaults OR in the config file. As such, we construct the model rather
@@ -169,22 +200,19 @@ def main():
 
     models = tyro.extras.subcommand_type_from_defaults(
         {
-            "init":init.model_validate(config_obj),
+            "init": init.model_validate(config_obj),
             "validate": validate.model_validate(config_obj),
             "report": report.model_validate(config_obj),
-            "build":build.model_validate(config_obj),
+            "build": build.model_validate(config_obj),
             "inspect": inspect.model_construct(**t.__dict__),
-            "new":new.model_validate(config_obj),
-            "test":test.model_validate(config_obj),
-            "test_servers":test_servers.model_construct(**t.__dict__),
+            "new": new.model_validate(config_obj),
+            "test": test.model_validate(config_obj),
+            "test_servers": test_servers.model_construct(**t.__dict__),
             "release_notes": release_notes.model_construct(**config_obj),
-            "deploy_acs": deploy_acs.model_construct(**t.__dict__)
+            "deploy_acs": deploy_acs.model_construct(**t.__dict__),
         }
     )
-    
-
 
-   
     config = None
     try:
         # Since some model(s) were constructed and not model_validated, we have to catch
@@ -192,7 +220,6 @@ def main():
         with warnings.catch_warnings(action="ignore"):
             config = tyro.cli(models)
 
-        
         if type(config) == init:
             t.__dict__.update(config.__dict__)
             init_func(t)
@@ -219,21 +246,29 @@ def main():
         print(e)
         sys.exit(1)
     except Exception as e:
+        print(CONTENTCTL_5_WARNING)
+
         if config is None:
-            print("There was a serious issue where the config file could not be created.\n"
-                  "The entire stack trace is provided below (please include it if filing a bug report).\n")
+            print(
+                "There was a serious issue where the config file could not be created.\n"
+                "The entire stack trace is provided below (please include it if filing a bug report).\n"
+            )
             traceback.print_exc()
         elif config.verbose:
-            print("Verbose error logging is ENABLED.\n"
-                  "The entire stack trace has been provided below (please include it if filing a bug report):\n")
+            print(
+                "Verbose error logging is ENABLED.\n"
+                "The entire stack trace has been provided below (please include it if filing a bug report):\n"
+            )
             traceback.print_exc()
         else:
-            print("Verbose error logging is DISABLED.\n"
-                  "Please use the --verbose command line argument if you need more context for your error or file a bug report.")
+            print(
+                "Verbose error logging is DISABLED.\n"
+                "Please use the --verbose command line argument if you need more context for your error or file a bug report."
+            )
             print(e)
-            
+
         sys.exit(1)
 
 
 if __name__ == "__main__":
-    main()
+    main()

contentctl/helper/utils.py

@@ -247,20 +247,6 @@ class Utils:
 
         return hash
 
-    # @staticmethod
-    # def check_required_fields(
-    #     thisField: str, definedFields: dict, requiredFields: list[str]
-    # ):
-    #     missing_fields = [
-    #         field for field in requiredFields if field not in definedFields
-    #     ]
-    #     if len(missing_fields) > 0:
-    #         raise (
-    #             ValueError(
-    #                 f"Could not validate - please resolve other errors resulting in missing fields {missing_fields}"
-    #             )
-    #         )
-
     @staticmethod
     def verify_file_exists(
         file_path: str, verbose_print=False, timeout_seconds: int = 10

contentctl/input/director.py

@@ -14,7 +14,7 @@ from contentctl.objects.investigation import Investigation
 from contentctl.objects.playbook import Playbook
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.macro import Macro
-from contentctl.objects.lookup import Lookup
+from contentctl.objects.lookup import LookupAdapter, Lookup
 from contentctl.objects.atomic import AtomicEnrichment
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.data_source import DataSource
@@ -58,13 +58,12 @@ class DirectorOutputDto:
                 f" - {content.file_path}\n"
                 f" - {self.name_to_content_map[content_name].file_path}"
             )
-        
+    
         if content.id in self.uuid_to_content_map:
             raise ValueError(
                 f"Duplicate id '{content.id}' with paths:\n"
                 f" - {content.file_path}\n"
-                f" - {self.uuid_to_content_map[content.id].file_path}"
-            )
+                f" - {self.uuid_to_content_map[content.id].file_path}")
 
         if isinstance(content, Lookup):
             self.lookups.append(content)
@@ -157,7 +156,8 @@ class Director():
                 modelDict = YmlReader.load_file(file)
 
                 if contentType == SecurityContentType.lookups:
-                    lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
+                    lookup = LookupAdapter.validate_python(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
+                    #lookup = Lookup.model_validate(modelDict, context={"output_dto":self.output_dto, "config":self.input_dto})
                     self.output_dto.addContentToDictMappings(lookup)
                 
                 elif contentType == SecurityContentType.macros:

contentctl/input/new_content_questions.py

@@ -48,7 +48,7 @@ class NewContentQuestions:
             {
                 'type': 'checkbox',
                 'message': 'Your data source',
-                'name': 'data_source',
+                'name': 'data_sources',
                 #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory
                 'choices': sorted(DataSource._value2member_map_ )
                 
@@ -57,7 +57,7 @@ class NewContentQuestions:
                 "type": "text",
                 "message": "enter search (spl)",
                 "name": "detection_search",
-                "default": "| UPDATE_SPL",
+                "default": "| __UPDATE__ SPL",
             },
             {
                 "type": "text",

contentctl/input/yml_reader.py

@@ -1,15 +1,12 @@
 from typing import Dict, Any
-
 import yaml
-
-
 import sys
 import pathlib
 
 class YmlReader():
 
     @staticmethod
-    def load_file(file_path: pathlib.Path, add_fields=True, STRICT_YML_CHECKING=False) -> Dict[str,Any]:
+    def load_file(file_path: pathlib.Path, add_fields:bool=True, STRICT_YML_CHECKING:bool=False) -> Dict[str,Any]:
         try:
             file_handler = open(file_path, 'r', encoding="utf-8")
             
@@ -27,8 +24,16 @@ class YmlReader():
                     print(f"Error loading YML file {file_path}: {str(e)}")
                     sys.exit(1)
             try:
-                #yml_obj = list(yaml.safe_load_all(file_handler))[0]
-                yml_obj = yaml.load(file_handler, Loader=yaml.CSafeLoader)
+                #Ideally we should use 
+                # from contentctl.actions.new_content import NewContent 
+                # and use NewContent.UPDATE_PREFIX, 
+                # but there is a circular dependency right now which makes that difficult.
+                # We have instead hardcoded UPDATE_PREFIX
+                UPDATE_PREFIX = "__UPDATE__"
+                data = file_handler.read()
+                if UPDATE_PREFIX in data:
+                    raise Exception(f"The file {file_path} contains the value '{UPDATE_PREFIX}'. Please fill out any unpopulated fields as required.")
+                yml_obj = yaml.load(data, Loader=yaml.CSafeLoader)
             except yaml.YAMLError as exc:
                 print(exc)
                 sys.exit(1)