contentctl 4.1.4__py3-none-any.whl → 4.2.0__py3-none-any.whl

contentctl/actions/build.py

@@ -10,6 +10,8 @@ from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
 from contentctl.output.ba_yml_output import BAYmlOutput
 from contentctl.output.api_json_output import ApiJsonOutput
+from contentctl.output.data_source_writer import DataSourceWriter
+from contentctl.objects.lookup import Lookup
 import pathlib
 import json
 import datetime
@@ -28,9 +30,20 @@ class Build:
 
 
     def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
-        if input_dto.config.build_app:    
+        if input_dto.config.build_app:
+
             updated_conf_files:set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
+
+            # Construct a special lookup whose CSV is created at runtime and
+            # written directly into the output folder. It is created with model_construct,
+            # not model_validate, because the CSV does not exist yet.
+            data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv"
+            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path)
+            input_dto.director_output_dto.addContentToDictMappings(Lookup.model_construct(description= "A lookup file that will contain the data source objects for detections.", 
+                                                                        filename=data_sources_lookup_csv_path, 
+                                                                        name="data_sources"))
+
             updated_conf_files.update(conf_output.writeHeaders())
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.stories, SecurityContentType.stories))

contentctl/actions/detection_testing/GitService.py

@@ -111,11 +111,19 @@ class GitService(BaseModel):
                                 raise Exception(f"More than 1 Lookup reference the modified CSV file '{decoded_path}': {[l.file_path for l in matched ]}")
                             else:
                                 updatedLookup = matched[0]
+                        elif decoded_path.suffix == ".mlmodel":
+                            # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
+                            # this time, we will ignore this change.
+                            updatedLookup = None
+                            
+
                         else:
-                            raise Exception(f"Error getting lookup object for file {str(decoded_path)}")
+                            raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
+                                            "Only files ending in .csv, .yml, or .mlmodel are supported in this "
+                                            "directory. This file must be removed from the lookups/ directory.")
                         
-                        if updatedLookup not in updated_lookups:
-                            # It is possible that both th CSV and YML have been modified for the same lookup,
+                        if updatedLookup is not None and updatedLookup not in updated_lookups:
+                            # It is possible that both the CSV and YML have been modified for the same lookup,
                             # and we do not want to add it twice. 
                             updated_lookups.append(updatedLookup)
 

contentctl/actions/initialize.py

@@ -28,6 +28,7 @@ class Initialize:
             ('../templates/app_template/', 'app_template'),
             ('../templates/deployments/', 'deployments'),
             ('../templates/detections/', 'detections'),
+            ('../templates/data_sources/', 'data_sources'),
             ('../templates/macros/','macros'),
             ('../templates/stories/', 'stories'),
         ]:

contentctl/actions/validate.py

@@ -1,20 +1,11 @@
-import sys
 
-from dataclasses import dataclass
-
-from pydantic import ValidationError
-from typing import Union
-
-from contentctl.objects.enums import SecurityContentProduct
-from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
-    SecurityContentObject_Abstract,
-)
+import pathlib
 from contentctl.input.director import Director, DirectorOutputDto
-
 from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.objects.atomic import AtomicTest
+from contentctl.helper.utils import Utils
 
 
 class Validate:
@@ -37,43 +28,48 @@ class Validate:
             [],
             [],
             [],
-            [],
         )
 
         director = Director(director_output_dto)
         director.execute(input_dto)
+        self.ensure_no_orphaned_files_in_lookups(input_dto.path, director_output_dto)
         return director_output_dto
 
-    def validate_duplicate_uuids(
-        self, security_content_objects: list[SecurityContentObject_Abstract]
-    ):
-        all_uuids = set()
-        duplicate_uuids = set()
-        for elem in security_content_objects:
-            if elem.id in all_uuids:
-                # The uuid has been found more than once
-                duplicate_uuids.add(elem.id)
-            else:
-                # This is the first time the uuid has been found
-                all_uuids.add(elem.id)
-
-        if len(duplicate_uuids) == 0:
-            return
+    
+    def ensure_no_orphaned_files_in_lookups(self, repo_path:pathlib.Path, director_output_dto:DirectorOutputDto):
+        """
+        This function ensures that only files which are relevant to lookups are included in the lookups folder.
+        This means that a file must be either:
+        1. A lookup YML (.yml)
+        2. A lookup CSV (.csv) which is referenced by a YML
+        3. A lookup MLMODEL (.mlmodel) which is referenced by a YML.
+        
+        All other files, includes CSV and MLMODEL files which are NOT
+        referenced by a YML, will generate an exception from this function.
+        
+        Args:
+            repo_path (pathlib.Path): path to the root of the app
+            director_output_dto (DirectorOutputDto): director object with all constructed content
 
-        # At least once duplicate uuid has been found. Enumerate all
-        # the pieces of content that use duplicate uuids
-        duplicate_messages = []
-        for uuid in duplicate_uuids:
-            duplicate_uuid_content = [
-                str(content.file_path)
-                for content in security_content_objects
-                if content.id in duplicate_uuids
-            ]
-            duplicate_messages.append(
-                f"Duplicate UUID [{uuid}] in {duplicate_uuid_content}"
-            )
+        Raises:
+            Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel 
+            files in this directory. Additionally, an exception will be raised if there 
+            exists one or more .csv or .mlmodel files that are not referenced by at least 1 
+            detection .yml file in this directory. 
+            This avoids having additional, unused files in this directory that may be copied into
+            the app when it is built (which can cause appinspect errors or larger app size.)
+        """        
+        lookupsDirectory = repo_path/"lookups"
+        
+        # Get all of the files referneced by Lookups
+        usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if lookup.filename is not None] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None]
 
-        raise ValueError(
-            "ERROR: Duplicate ID(s) found in objects:\n"
-            + "\n - ".join(duplicate_messages)
-        )
+        # Get all of the mlmodel and csv files in the lookups directory
+        csvAndMlmodelFiles  = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"])
+        
+        # Generate an exception of any csv or mlmodel files exist but are not used
+        unusedLookupFiles:list[pathlib.Path] = [testFile for testFile in csvAndMlmodelFiles if testFile not in usedLookupFiles]
+        if len(unusedLookupFiles) > 0:
+            raise Exception(f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}")
+        return
+    

contentctl/helper/utils.py

@@ -34,6 +34,49 @@ class Utils:
                     listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
     
         return sorted(listOfFiles)
+    
+    @staticmethod
+    def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExtensions:list[str]=[".yml"], fileExtensionsToReturn:list[str]=[".yml"]) -> list[pathlib.Path]:
+    
+        """
+        Get all of the Security Content Object Files rooted in a given directory.  These will almost
+        certain be YML files, but could be other file types as specified by the user
+
+        Args:
+            path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed. 
+            allowedFileExtensions (set[str], optional): File extensions which are allowed to be  present in this directory.  In most cases, we do not want to allow the presence of non-YML files. Defaults to [".yml"].
+            fileExtensionsToReturn (set[str], optional): Filenames with extensions that should be returned from this function. For example, the lookups/ directory contains YML, CSV, and MLMODEL directories, but only the YMLs are Security Content Objects for constructing Lookyps. Defaults to[".yml"].
+
+        Raises:
+            Exception: Will raise an exception if allowedFileExtensions is not a subset of fileExtensionsToReturn.
+            Exception: Will raise an exception if the path passed to the function does not exist or is not a directory
+            Exception: Will raise an exception if there are any files rooted in the directory which are not in allowedFileExtensions
+
+        Returns:
+            list[pathlib.Path]: list of files with an extension in fileExtensionsToReturn found in path
+        """
+        if not set(fileExtensionsToReturn).issubset(set(allowedFileExtensions)):
+            raise Exception(f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not")
+        
+        if not path.exists() or not path.is_dir():
+            raise Exception(f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory")
+        
+        allowedFiles:list[pathlib.Path] = []
+        erroneousFiles:list[pathlib.Path] = []
+        #Get every single file extension 
+        for filePath in path.glob("**/*.*"):
+            if filePath.suffix in allowedFileExtensions:
+                # Yes these are allowed
+                allowedFiles.append(filePath)
+            else:
+                # No these have not been allowed
+                erroneousFiles.append(filePath)
+
+        if len(erroneousFiles):
+            raise Exception(f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}")
+        
+        # There were no errorneous files, so return the requested files
+        return sorted([filePath for filePath in allowedFiles if filePath.suffix in fileExtensionsToReturn])
 
     @staticmethod
     def get_all_yml_files_from_directory_one_layer_deep(path: str) -> list[pathlib.Path]:

contentctl/input/director.py

@@ -58,7 +58,6 @@ class DirectorOutputDto:
     deployments: list[Deployment]
     ssa_detections: list[SSADetection]
     data_sources: list[DataSource]
-    event_sources: list[EventSource]
     name_to_content_map: dict[str, SecurityContentObject] = field(default_factory=dict)
     uuid_to_content_map: dict[UUID, SecurityContentObject] = field(default_factory=dict)
 
@@ -68,17 +67,19 @@ class DirectorOutputDto:
             # Since SSA detections may have the same name as ESCU detection,
             # for this function we prepend 'SSA ' to the name.
             content_name = f"SSA {content_name}"
+        
         if content_name in self.name_to_content_map:
             raise ValueError(
                 f"Duplicate name '{content_name}' with paths:\n"
                 f" - {content.file_path}\n"
                 f" - {self.name_to_content_map[content_name].file_path}"
             )
-        elif content.id in self.uuid_to_content_map:
+        
+        if content.id in self.uuid_to_content_map:
             raise ValueError(
                 f"Duplicate id '{content.id}' with paths:\n"
                 f" - {content.file_path}\n"
-                f" - {self.name_to_content_map[content_name].file_path}"
+                f" - {self.uuid_to_content_map[content.id].file_path}"
             )
 
         if isinstance(content, Lookup):
@@ -99,9 +100,10 @@ class DirectorOutputDto:
             self.detections.append(content)
         elif isinstance(content, SSADetection):
             self.ssa_detections.append(content)
+        elif isinstance(content, DataSource):
+            self.data_sources.append(content)
         else:
-             raise Exception(f"Unknown security content type: {type(content)}")
-
+            raise Exception(f"Unknown security content type: {type(content)}")
 
         self.name_to_content_map[content_name] = content
         self.uuid_to_content_map[content.id] = content
@@ -124,41 +126,27 @@ class Director():
         self.createSecurityContent(SecurityContentType.stories)
         self.createSecurityContent(SecurityContentType.baselines)
         self.createSecurityContent(SecurityContentType.investigations)
-        self.createSecurityContent(SecurityContentType.event_sources)
         self.createSecurityContent(SecurityContentType.data_sources)
         self.createSecurityContent(SecurityContentType.playbooks)
         self.createSecurityContent(SecurityContentType.detections)
         self.createSecurityContent(SecurityContentType.ssa_detections)
 
+        
+        from contentctl.objects.abstract_security_content_objects.detection_abstract import MISSING_SOURCES
+        if len(MISSING_SOURCES) > 0:
+            missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES)))
+            print("WARNING: The following data_sources have been used in detections, but are not yet defined.\n"
+                  "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 "
+                  f"{missing_sources_string}")
+        else:
+            print("No missing data_sources!")
+
     def createSecurityContent(self, contentType: SecurityContentType) -> None:
         if contentType == SecurityContentType.ssa_detections:
             files = Utils.get_all_yml_files_from_directory(
                 os.path.join(self.input_dto.path, "ssa_detections")
             )
             security_content_files = [f for f in files if f.name.startswith("ssa___")]
-
-        elif contentType == SecurityContentType.data_sources:
-            security_content_files = (
-                Utils.get_all_yml_files_from_directory_one_layer_deep(
-                    os.path.join(self.input_dto.path, "data_sources")
-                )
-            )
-
-        elif contentType == SecurityContentType.event_sources:
-            security_content_files = Utils.get_all_yml_files_from_directory(
-                os.path.join(self.input_dto.path, "data_sources", "cloud", "event_sources")
-            )
-            security_content_files.extend(
-                Utils.get_all_yml_files_from_directory(
-                    os.path.join(self.input_dto.path, "data_sources", "endpoint", "event_sources")
-                )
-            )
-            security_content_files.extend(
-                Utils.get_all_yml_files_from_directory(
-                    os.path.join(self.input_dto.path, "data_sources", "network", "event_sources")
-                )
-            )
-
         elif contentType in [
             SecurityContentType.deployments,
             SecurityContentType.lookups,
@@ -168,6 +156,7 @@ class Director():
             SecurityContentType.investigations,
             SecurityContentType.playbooks,
             SecurityContentType.detections,
+            SecurityContentType.data_sources,
         ]:
             files = Utils.get_all_yml_files_from_directory(
                 os.path.join(self.input_dto.path, str(contentType.name))
@@ -190,54 +179,48 @@ class Director():
                 modelDict = YmlReader.load_file(file)
 
                 if contentType == SecurityContentType.lookups:
-                        lookup = Lookup.model_validate(modelDict,context={"output_dto":self.output_dto, "config":self.input_dto})
-                        self.output_dto.addContentToDictMappings(lookup)
+                    lookup = Lookup.model_validate(modelDict,context={"output_dto":self.output_dto, "config":self.input_dto})
+                    self.output_dto.addContentToDictMappings(lookup)
                 
                 elif contentType == SecurityContentType.macros:
-                        macro = Macro.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(macro)
+                    macro = Macro.model_validate(modelDict,context={"output_dto":self.output_dto})
+                    self.output_dto.addContentToDictMappings(macro)
                 
                 elif contentType == SecurityContentType.deployments:
-                        deployment = Deployment.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(deployment)
+                    deployment = Deployment.model_validate(modelDict,context={"output_dto":self.output_dto})
+                    self.output_dto.addContentToDictMappings(deployment)
 
                 elif contentType == SecurityContentType.playbooks:
-                        playbook = Playbook.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(playbook)                  
+                    playbook = Playbook.model_validate(modelDict,context={"output_dto":self.output_dto})
+                    self.output_dto.addContentToDictMappings(playbook)                  
                 
                 elif contentType == SecurityContentType.baselines:
-                        baseline = Baseline.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(baseline)
+                    baseline = Baseline.model_validate(modelDict,context={"output_dto":self.output_dto})
+                    self.output_dto.addContentToDictMappings(baseline)
                 
                 elif contentType == SecurityContentType.investigations:
-                        investigation = Investigation.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(investigation)
+                    investigation = Investigation.model_validate(modelDict,context={"output_dto":self.output_dto})
+                    self.output_dto.addContentToDictMappings(investigation)
 
                 elif contentType == SecurityContentType.stories:
-                        story = Story.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.addContentToDictMappings(story)
+                    story = Story.model_validate(modelDict,context={"output_dto":self.output_dto})
+                    self.output_dto.addContentToDictMappings(story)
             
                 elif contentType == SecurityContentType.detections:
-                        detection = Detection.model_validate(modelDict,context={"output_dto":self.output_dto, "app":self.input_dto.app})
-                        self.output_dto.addContentToDictMappings(detection)
+                    detection = Detection.model_validate(modelDict,context={"output_dto":self.output_dto, "app":self.input_dto.app})
+                    self.output_dto.addContentToDictMappings(detection)
 
                 elif contentType == SecurityContentType.ssa_detections:
-                        self.constructSSADetection(self.ssa_detection_builder, self.output_dto,str(file))
-                        ssa_detection = self.ssa_detection_builder.getObject()
-                        if ssa_detection.status in [DetectionStatus.production.value, DetectionStatus.validation.value]:
-                            self.output_dto.addContentToDictMappings(ssa_detection)
+                    self.constructSSADetection(self.ssa_detection_builder, self.output_dto,str(file))
+                    ssa_detection = self.ssa_detection_builder.getObject()
+                    if ssa_detection.status in [DetectionStatus.production.value, DetectionStatus.validation.value]:
+                        self.output_dto.addContentToDictMappings(ssa_detection)
                 
                 elif contentType == SecurityContentType.data_sources:
                     data_source = DataSource.model_validate(
                         modelDict, context={"output_dto": self.output_dto}
                     )
-                    self.output_dto.data_sources.append(data_source)
-
-                elif contentType == SecurityContentType.event_sources:
-                    event_source = EventSource.model_validate(
-                        modelDict, context={"output_dto": self.output_dto}
-                    )
-                    self.output_dto.event_sources.append(event_source)
+                    self.output_dto.addContentToDictMappings(data_source)
 
                 else:
                     raise Exception(f"Unsupported type: [{contentType}]")

contentctl/input/yml_reader.py

@@ -40,6 +40,8 @@ class YmlReader():
         if add_fields == False:
             return yml_obj
         
+        
         yml_obj['file_path'] = str(file_path)
+    
 
         return yml_obj

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -22,12 +22,14 @@ from contentctl.objects.deployment import Deployment
 from contentctl.objects.unit_test import UnitTest
 from contentctl.objects.test_group import TestGroup
 from contentctl.objects.integration_test import IntegrationTest
-
+from contentctl.objects.event_source import EventSource
+from contentctl.objects.data_source import DataSource
 
 #from contentctl.objects.playbook import Playbook
-from contentctl.objects.enums import DataSource,ProvidingTechnology
+from contentctl.objects.enums import ProvidingTechnology
 from contentctl.enrichments.cve_enrichment import CveEnrichmentObj
 
+MISSING_SOURCES:set[str] = set()
 
 class Detection_Abstract(SecurityContentObject):
     model_config = ConfigDict(use_enum_values=True)
@@ -35,12 +37,11 @@ class Detection_Abstract(SecurityContentObject):
     #contentType: SecurityContentType = SecurityContentType.detections
     type: AnalyticsType = Field(...)
     status: DetectionStatus = Field(...)
-    data_source: Optional[List[str]] = None
+    data_source: list[str] = []
     tags: DetectionTags = Field(...)
     search: Union[str, dict[str,Any]] = Field(...)
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
-    data_source_objects: Optional[List[DataSource]] = None
 
     enabled_by_default: bool = False
     file_path: FilePath = Field(...)
@@ -53,6 +54,8 @@ class Detection_Abstract(SecurityContentObject):
     # A list of groups of tests, relying on the same data
     test_groups: Union[list[TestGroup], None] = Field(None,validate_default=True)
 
+    data_source_objects: list[DataSource] = []
+
 
     @field_validator("search", mode="before")
     @classmethod
@@ -138,6 +141,7 @@ class Detection_Abstract(SecurityContentObject):
         else:
             return []
     
+
     @computed_field
     @property
     def source(self)->str:
@@ -161,10 +165,12 @@ class Detection_Abstract(SecurityContentObject):
         annotations_dict["type"] = self.type
         #annotations_dict["version"] = self.version
 
+        annotations_dict["data_source"] = self.data_source
+
         #The annotations object is a superset of the mappings object.
         # So start with the mapping object.
         annotations_dict.update(self.mappings)
-
+        
         #Make sure that the results are sorted for readability/easier diffs
         return dict(sorted(annotations_dict.items(), key=lambda item: item[0]))
         
@@ -384,23 +390,37 @@ class Detection_Abstract(SecurityContentObject):
                 raise ValueError(f"Error, failed to replace detection reference in Baseline '{baseline.name}' to detection '{self.name}'")             
             baseline.tags.detections = new_detections
 
-        self.data_source_objects = []
-        for data_source_obj in director.data_sources:
-            for detection_data_source in self.data_source:
-                if data_source_obj.name in detection_data_source:
-                    self.data_source_objects.append(data_source_obj)
-
-        # Remove duplicate data source objects based on their 'name' property
-        unique_data_sources = {}
-        for data_source_obj in self.data_source_objects:
-            if data_source_obj.name not in unique_data_sources:
-                unique_data_sources[data_source_obj.name] = data_source_obj
-        self.data_source_objects = list(unique_data_sources.values())
+        # Data source may be defined 1 on each line, OR they may be defined as
+        # SOUCE_1 AND ANOTHERSOURCE AND A_THIRD_SOURCE
+        # if more than 1 data source is required for a detection (for example, because it includes a join)
+        # Parse and update the list to resolve individual names and remove potential duplicates
+        updated_data_source_names:set[str] = set()
+        
+        for ds in self.data_source:
+            split_data_sources = {d.strip() for d in ds.split('AND')}
+            updated_data_source_names.update(split_data_sources)
+        
+        sources = sorted(list(updated_data_source_names))
+        
+        matched_data_sources:list[DataSource] = []
+        missing_sources:list[str] = []
+        for source in sources:
+            try:
+                matched_data_sources += DataSource.mapNamesToSecurityContentObjects([source], director)
+            except Exception as data_source_mapping_exception:
+                # We gobble this up and add it to a global set so that we
+                # can print it ONCE at the end of the build of datasources.
+                # This will be removed later as per the note below
+                MISSING_SOURCES.add(source)
+                
+        if len(missing_sources) > 0:
+            # This will be changed to ValueError when we have a complete list of data sources
+            print(f"WARNING: The following exception occurred when mapping the data_source field to DataSource objects:{missing_sources}")
+        
+        self.data_source_objects = matched_data_sources
 
         for story in self.tags.analytic_story:
-            story.detections.append(self)
-            story.data_sources.extend(self.data_source_objects)
-
+            story.detections.append(self)            
         return self
 
     
@@ -424,14 +444,16 @@ class Detection_Abstract(SecurityContentObject):
             raise ValueError("Error, baselines are constructed automatically at runtime.  Please do not include this field.")
 
         
-        name:Union[str,dict] = info.data.get("name",None)
+        name:Union[str,None] = info.data.get("name",None)
         if name is None:
             raise ValueError("Error, cannot get Baselines because the Detection does not have a 'name' defined.")
-        
+         
         director:DirectorOutputDto = info.context.get("output_dto",None)
         baselines:List[Baseline] = []
         for baseline in director.baselines:
-            if name in baseline.tags.detections:
+            # This matching is a bit strange, because baseline.tags.detections starts as a list of strings, but 
+            # is eventually updated to a list of Detections as we construct all of the detection objects. 
+            if name in [detection_name for detection_name in baseline.tags.detections if isinstance(detection_name,str)]:
                 baselines.append(baseline)
 
         return baselines

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -125,9 +125,9 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
         errors:list[str] = []
         if len(missing_objects) > 0:
             errors.append(f"Failed to find the following '{cls.__name__}': {missing_objects}")
-        if len(missing_objects) > 0:
+        if len(mistyped_objects) > 0:
             for mistyped_object in mistyped_objects:
-                errors.append(f"'{mistyped_object.name}' expected to have type '{type(Self)}', but actually had type '{type(mistyped_object)}'")
+                errors.append(f"'{mistyped_object.name}' expected to have type '{cls}', but actually had type '{type(mistyped_object)}'")
         
         if len(errors) > 0:
             error_string = "\n  - ".join(errors)
@@ -194,6 +194,33 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
 
     def __str__(self)->str:
         return(self.__repr__())
+
+    def __lt__(self, other:object)->bool:
+        if not isinstance(other,SecurityContentObject_Abstract):
+            raise Exception(f"SecurityContentObject can only be compared to each other, not to {type(other)}")
+        return self.name < other.name 
+
+    def __eq__(self, other:object)->bool: 
+        if not isinstance(other,SecurityContentObject_Abstract):
+            raise Exception(f"SecurityContentObject can only be compared to each other, not to {type(other)}")
+        
+        if id(self) == id(other) and self.name == other.name and self.id == other.id:
+            # Yes, this is the same object
+            return True
+        
+        elif id(self) == id(other) or self.name == other.name or self.id == other.id:
+            raise Exception("Attempted to compare two SecurityContentObjects, but their fields indicate they were not globally unique:"
+                            f"\n\tid(obj1)  : {id(self)}"
+                            f"\n\tid(obj2)  : {id(other)}"
+                            f"\n\tobj1.name : {self.name}"
+                            f"\n\tobj2.name : {other.name}"
+                            f"\n\tobj1.id   : {self.id}"
+                            f"\n\tobj2.id   : {other.id}")
+        else:
+            return False
+    
+    def __hash__(self) -> NonNegativeInt:
+        return id(self)
         
 
     

contentctl/objects/data_source.py

@@ -1,28 +1,42 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from typing import Optional, Any
+from pydantic import Field, FilePath, model_serializer
+from contentctl.objects.security_content_object import SecurityContentObject
+from contentctl.objects.event_source import EventSource
 
+class DataSource(SecurityContentObject):
+    source: str = Field(...)
+    sourcetype: str = Field(...)
+    separator: Optional[str] = None
+    configuration: Optional[str] = None
+    supported_TA: Optional[list] = None
+    fields: Optional[list] = None
+    field_mappings: Optional[list] = None
+    convert_to_log_source: Optional[list] = None
+    example_log: Optional[str] = None
 
-class DataSource(BaseModel):
-    name: str
-    id: str
-    author: str
-    source: str
-    sourcetype: str
-    separator: str = None
-    configuration: str = None
-    supported_TA: dict
-    event_names: list = None
-    event_sources: list = None
-    fields: list = None
-    example_log: str = None
 
-    def model_post_init(self, ctx:dict[str,Any]):
-        context = ctx.get("output_dto")
+    @model_serializer
+    def serialize_model(self):
+        #Call serializer for parent
+        super_fields = super().serialize_model()
         
-        if self.event_names:
-            self.event_sources = []
-            for event_source in context.event_sources:
-                if any(event['event_name'] == event_source.event_name for event in self.event_names):
-                    self.event_sources.append(event_source)
-
-        return self
+        #All fields custom to this model
+        model:dict[str,Any] = {
+            "source": self.source,
+            "sourcetype": self.sourcetype,
+            "separator": self.separator,
+            "configuration": self.configuration,
+            "supported_TA": self.supported_TA,
+            "fields": self.fields,
+            "field_mappings": self.field_mappings,
+            "convert_to_log_source": self.convert_to_log_source,
+            "example_log":self.example_log
+        }
+        
+        
+        #Combine fields from this model with fields from parent
+        super_fields.update(model)
+        
+        #return the model
+        return super_fields

contentctl/objects/enums.py

@@ -56,7 +56,6 @@ class SecurityContentType(enum.Enum):
     unit_tests = 9
     ssa_detections = 10
     data_sources = 11
-    event_sources = 12
 
 # Bringing these changes back in line will take some time after
 # the initial merge is complete

contentctl/objects/event_source.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from typing import Union, Optional, List
+from pydantic import BaseModel, Field
 
+from contentctl.objects.security_content_object import SecurityContentObject
 
-class EventSource(BaseModel):
-    event_name: str
-    fields: list[str]
-    field_mappings: list[dict] = None
-    convert_to_log_source: list[dict] = None
-    example_log: str = None
+class EventSource(SecurityContentObject):
+    fields: Optional[list[str]] = None
+    field_mappings: Optional[list[dict]] = None
+    convert_to_log_source: Optional[list[dict]] = None
+    example_log: Optional[str] = None

contentctl/objects/lookup.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer
 from typing import TYPE_CHECKING, Optional, Any, Union
 import re
+import csv
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
     from contentctl.objects.config import validate
@@ -61,15 +62,53 @@ class Lookup(SecurityContentObject):
                 raise ValueError("config required for constructing lookup filename, but it was not")
         return data
 
-    @field_validator('filename')
-    @classmethod
-    def lookup_file_valid(cls, v: Union[FilePath,None], info: ValidationInfo):
-        if not v:
-            return v
-        if not (v.name.endswith(".csv") or v.name.endswith(".mlmodel")):
-            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{v}'")
 
-        return v
+    def model_post_init(self, ctx:dict[str,Any]):
+        if not self.filename:
+            return
+        import pathlib
+        filenamePath = pathlib.Path(self.filename)
+        
+        if filenamePath.suffix not in [".csv", ".mlmodel"]:
+            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{filenamePath}'")
+        
+        
+
+        if filenamePath.suffix == ".mlmodel":
+            # Do not need any additional checks for an mlmodel file
+            return
+
+        # https://docs.python.org/3/library/csv.html#csv.DictReader
+        # Column Names (fieldnames) determine by the number of columns in the first row.
+        # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception
+        # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception.    
+        csv_errors:list[str] = []
+        with open(filenamePath, "r") as csv_fp:
+            RESTKEY = "extra_fields_in_a_row"
+            csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY)            
+            if csv_dict.fieldnames is None:
+                raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t"
+                                 "Unable to read fieldnames from CSV. Is the CSV empty?\n"
+                                 "  Please try opening the file with a CSV Editor to ensure that it is correct.")
+            # Remember that row 1 has the headers and we do not iterate over it in the loop below
+            # CSVs are typically indexed starting a row 1 for the header.
+            for row_index, data_row in enumerate(csv_dict):
+                row_index+=2
+                if len(data_row.get(RESTKEY,[])) > 0:
+                    csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns,"
+                                      f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY,[]))}].")
+                
+                for column_index, column_name in enumerate(data_row):
+                    if data_row[column_name] is None:
+                        csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, "
+                                          f"but instead had [{column_index}].")
+        if len(csv_errors) > 0:
+            err_string = '\n\t'.join(csv_errors)
+            raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t{err_string}\n"
+                             f"  Please try opening the file with a CSV Editor to ensure that it is correct.")
+    
+        return
+    
         
     @field_validator('match_type')
     @classmethod

contentctl/objects/story.py

@@ -33,7 +33,18 @@ class Story(SecurityContentObject):
     detections:List[Detection] = []
     investigations: List[Investigation] = []
     baselines: List[Baseline] = []
-    data_sources: List[DataSource] = []
+    
+    
+    @computed_field
+    @property
+    def data_sources(self)-> list[DataSource]:
+        # Only add a data_source if it does not already exist in the story
+        data_source_objects:set[DataSource] = set()
+        for detection in self.detections:
+            data_source_objects.update(set(detection.data_source_objects))
+        
+        return sorted(list(data_source_objects))
+
 
     def storyAndInvestigationNamesWithApp(self, app_name:str)->List[str]:
         return [f"{app_name} - {name} - Rule" for name in self.detection_names] + \
@@ -141,7 +152,3 @@ class Story(SecurityContentObject):
     def baseline_names(self)->List[str]:        
         return [baseline.name for baseline in self.baselines]
     
-   
-    
-    
- 

contentctl/output/data_source_writer.py

@@ -0,0 +1,40 @@
+import csv
+from contentctl.objects.data_source import DataSource
+from contentctl.objects.event_source import EventSource
+from typing import List
+import pathlib
+
+class DataSourceWriter:
+
+    @staticmethod
+    def writeDataSourceCsv(data_source_objects: List[DataSource], file_path: pathlib.Path):
+        with open(file_path, mode='w', newline='') as file:
+            writer = csv.writer(file)
+            # Write the header
+            writer.writerow([
+                "name", "id", "author", "source", "sourcetype", "separator", 
+                "supported_TA_name", "supported_TA_version", "supported_TA_url",
+                "description"
+            ])
+            # Write the data
+            for data_source in data_source_objects:
+                if data_source.supported_TA and isinstance(data_source.supported_TA, list) and len(data_source.supported_TA) > 0:
+                    supported_TA_name = data_source.supported_TA[0].get('name', '')
+                    supported_TA_version = data_source.supported_TA[0].get('version', '')
+                    supported_TA_url = data_source.supported_TA[0].get('url', '')
+                else:
+                    supported_TA_name = ''
+                    supported_TA_version = ''
+                    supported_TA_url = ''
+                writer.writerow([
+                    data_source.name,
+                    data_source.id,
+                    data_source.author,
+                    data_source.source,
+                    data_source.sourcetype,
+                    data_source.separator,
+                    supported_TA_name,
+                    supported_TA_version,
+                    supported_TA_url,
+                    data_source.description,
+                ])

contentctl/templates/data_sources/sysmon_eventid_1.yml

@@ -0,0 +1,171 @@
+name: Sysmon EventID 1
+id: b375f4d1-d7ca-4bc0-9103-294825c0af17
+version: 1
+date: '2024-07-18'
+author: Patrick Bareiss, Splunk
+description: Data source object for Sysmon EventID 1
+source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational
+sourcetype: xmlwineventlog
+separator: EventID
+supported_TA:
+- name: Splunk Add-on for Sysmon
+  url: https://splunkbase.splunk.com/app/5709/
+  version: 4.0.0
+fields:
+- _time
+- Channel
+- CommandLine
+- Company
+- Computer
+- CurrentDirectory
+- Description
+- EventChannel
+- EventCode
+- EventData_Xml
+- EventDescription
+- EventID
+- EventRecordID
+- FileVersion
+- Guid
+- Hashes
+- IMPHASH
+- Image
+- IntegrityLevel
+- Keywords
+- Level
+- LogonGuid
+- LogonId
+- MD5
+- Name
+- Opcode
+- OriginalFileName
+- ParentCommandLine
+- ParentImage
+- ParentProcessGuid
+- ParentProcessId
+- ProcessGuid
+- ProcessID
+- ProcessId
+- Product
+- RecordID
+- RecordNumber
+- RuleName
+- SHA256
+- SecurityID
+- SystemTime
+- System_Props_Xml
+- Task
+- TerminalSessionId
+- ThreadID
+- TimeCreated
+- User
+- UserID
+- UtcTime
+- Version
+- action
+- date_hour
+- date_mday
+- date_minute
+- date_month
+- date_second
+- date_wday
+- date_year
+- date_zone
+- dest
+- dvc_nt_host
+- event_id
+- eventtype
+- host
+- id
+- index
+- linecount
+- original_file_name
+- os
+- parent_process
+- parent_process_exec
+- parent_process_guid
+- parent_process_id
+- parent_process_name
+- parent_process_path
+- process
+- process_current_directory
+- process_exec
+- process_guid
+- process_hash
+- process_id
+- process_integrity_level
+- process_name
+- process_path
+- punct
+- signature
+- signature_id
+- source
+- sourcetype
+- splunk_server
+- tag
+- tag::eventtype
+- timeendpos
+- timestartpos
+- user
+- user_id
+- vendor_product
+field_mappings:
+  - data_model: cim
+    data_set: Endpoint.Processes
+    mapping:
+      ProcessGuid: Processes.process_guid
+      ProcessId: Processes.process_id
+      Image: Processes.process_path
+      Image|endswith: Processes.process_name
+      CommandLine: Processes.process
+      CurrentDirectory: Processes.process_current_directory
+      User: Processes.user
+      IntegrityLevel: Processes.process_integrity_level
+      Hashes: Processes.process_hash
+      ParentProcessGuid: Processes.parent_process_guid
+      ParentProcessId: Processes.parent_process_id
+      ParentImage: Processes.parent_process_name
+      ParentCommandLine: Processes.parent_process
+      Computer: Processes.dest
+      OriginalFileName: Processes.original_file_name
+convert_to_log_source:
+  - data_source: Windows Event Log Security 4688
+    mapping:
+      ProcessId: NewProcessId
+      Image: NewProcessName
+      Image|endswith: NewProcessName|endswith
+      CommandLine: Process_Command_Line
+      User: SubjectUserSid
+      ParentProcessId: ProcessId
+      ParentImage: ParentProcessName
+      ParentImage|endswith: ParentProcessName|endswith
+      Computer: Computer
+      OriginalFileName: NewProcessName|endswith
+  - data_source: Crowdstrike Process
+    mapping:
+      ProcessId: RawProcessId
+      Image: ImageFileName
+      CommandLine: CommandLine
+      User: UserSid
+      ParentProcessId: ParentProcessId
+      ParentImage: ParentBaseFileName
+example_log: "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider\
+  \ Name='Microsoft-Windows-Sysmon' Guid='{5770385F-C22A-43E0-BF4C-06F5698FFBD9}'/><EventID>1</EventID><Version>5</Version><Level>4</Level><Task>1</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated\
+  \ SystemTime='2020-10-08T11:03:46.617920300Z'/><EventRecordID>4522</EventRecordID><Correlation/><Execution\
+  \ ProcessID='2912' ThreadID='3424'/><Channel>Microsoft-Windows-Sysmon/Operational</Channel><Computer>win-dc-6764986.attackrange.local</Computer><Security\
+  \ UserID='S-1-5-18'/></System><EventData><Data Name='RuleName'>-</Data><Data Name='UtcTime'>2020-10-08\
+  \ 11:03:46.615</Data><Data Name='ProcessGuid'>{96128EA2-F212-5F7E-E400-000000007F01}</Data><Data\
+  \ Name='ProcessId'>2296</Data><Data Name='Image'>C:\\Windows\\System32\\cmd.exe</Data><Data\
+  \ Name='FileVersion'>10.0.14393.0 (rs1_release.160715-1616)</Data><Data Name='Description'>Windows\
+  \ Command Processor</Data><Data Name='Product'>Microsoft\xAE Windows\xAE Operating\
+  \ System</Data><Data Name='Company'>Microsoft Corporation</Data><Data Name='OriginalFileName'>Cmd.Exe</Data><Data\
+  \ Name='CommandLine'>\"C:\\Windows\\system32\\cmd.exe\" /c \"reg save HKLM\\sam\
+  \ %%temp%%\\sam &amp; reg save HKLM\\system %%temp%%\\system &amp; reg save HKLM\\\
+  security %%temp%%\\security\" </Data><Data Name='CurrentDirectory'>C:\\Users\\ADMINI~1\\\
+  AppData\\Local\\Temp\\</Data><Data Name='User'>ATTACKRANGE\\Administrator</Data><Data\
+  \ Name='LogonGuid'>{96128EA2-F210-5F7E-ACD4-080000000000}</Data><Data Name='LogonId'>0x8d4ac</Data><Data\
+  \ Name='TerminalSessionId'>0</Data><Data Name='IntegrityLevel'>High</Data><Data\
+  \ Name='Hashes'>MD5=F4F684066175B77E0C3A000549D2922C,SHA256=935C1861DF1F4018D698E8B65ABFA02D7E9037D8F68CA3C2065B6CA165D44AD2,IMPHASH=3062ED732D4B25D1C64F084DAC97D37A</Data><Data\
+  \ Name='ParentProcessGuid'>{96128EA2-F211-5F7E-DF00-000000007F01}</Data><Data Name='ParentProcessId'>4624</Data><Data\
+  \ Name='ParentImage'>C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe</Data><Data\
+  \ Name='ParentCommandLine'>\"powershell.exe\" -noninteractive -encodedcommand WwBDAG8AbgBzAG8AbABlAF0AOgA6AEkAbgBwAHUAdABFAG4AYwBvAGQAaQBuAGcAIAA9ACAATgBlAHcALQBPAGIAagBlAGMAdAAgAFQAZQB4AHQALgBVAFQARgA4AEUAbgBjAG8AZABpAG4AZwAgACQAZgBhAGwAcwBlADsAIABJAG0AcABvAHIAdAAtAE0AbwBkAHUAbABlACAAIgBDADoAXABBAHQAbwBtAGkAYwBSAGUAZABUAGUAYQBtAFwAaQBuAHYAbwBrAGUALQBhAHQAbwBtAGkAYwByAGUAZAB0AGUAYQBtAFwASQBuAHYAbwBrAGUALQBBAHQAbwBtAGkAYwBSAGUAZABUAGUAYQBtAC4AcABzAGQAMQAiACAALQBGAG8AcgBjAGUACgBJAG4AdgBvAGsAZQAtAEEAdABvAG0AaQBjAFQAZQBzAHQAIAAiAFQAMQAwADAAMwAuADAAMAAyACIAIAAtAEMAbwBuAGYAaQByAG0AOgAkAGYAYQBsAHMAZQAgAC0AVABpAG0AZQBvAHUAdABTAGUAYwBvAG4AZABzACAAMwAwADAAIAAtAEUAeABlAGMAdQB0AGkAbwBuAEwAbwBnAFAAYQB0AGgAIABDADoAXABBAHQAbwBtAGkAYwBSAGUAZABUAGUAYQBtAFwAYQB0AGMAXwBlAHgAZQBjAHUAdABpAG8AbgAuAGMAcwB2AA==</Data></EventData></Event>"

contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml

@@ -13,7 +13,7 @@ description: The following detection identifies a 7z.exe spawned from `Rundll32.
   any files written to disk and analyze as needed. Review parallel processes for additional
   behaviors. Typically, archiving files will result in exfiltration.
 data_source:
-- Sysmon Event ID 1
+- Sysmon EventID 1
 search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time)
   as lastTime from datamodel=Endpoint.Processes where Processes.parent_process_name
   IN ("rundll32.exe", "dllhost.exe") Processes.process_name=*7z* by Processes.dest

{contentctl-4.1.4.dist-info → contentctl-4.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contentctl
-Version: 4.1.4
+Version: 4.2.0
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

{contentctl-4.1.4.dist-info → contentctl-4.2.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
 contentctl/__init__.py,sha256=IMjkMO3twhQzluVTo8Z6rE7Eg-9U79_LGKMcsWLKBkY,22
-contentctl/actions/build.py,sha256=BVc-1E63zeUQ9wWAHTC_fLNvfEK5YT3Z6_QLiE72TQs,4765
+contentctl/actions/build.py,sha256=mGm1F8jWdj547uJVSEWZBZcEyjoO4QpPKWhJOpRwR94,5739
 contentctl/actions/convert.py,sha256=0KBWLxvP1hSPXpExePqpOQPRvlQLamvPLyQqeTIWNbk,704
 contentctl/actions/deploy_acs.py,sha256=mf3uk495H1EU_LNN-TiOsYCo18HMGoEBMb6ojeTr0zw,1418
 contentctl/actions/detection_testing/DetectionTestingManager.py,sha256=zg8JasDjCpSC-yhseEyUwO8qbDJIUJbhlus9Li9ZAnA,8818
-contentctl/actions/detection_testing/GitService.py,sha256=FY_JtMi3qL-uC31Yyf7CTWZ5kLQhAMAvcj-8QXtkfPM,8386
+contentctl/actions/detection_testing/GitService.py,sha256=xNhuvK8oUoTxFlC0XBhlew9V0DO7l2hqaBMffEk5ohM,9000
 contentctl/actions/detection_testing/generate_detection_coverage_badge.py,sha256=N5mznaeErVak3mOBwsd0RDBFJO3bku0EZvpayCyU-uk,2259
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=VFhSHdw_0N6ol668hDkaj7yFjPsZqBoFNC8FKzWKICc,53141
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py,sha256=HVGWCXy0GQeBqu2cVJn5H-I8GY8rwgkkc53ilO1TfZA,6846
@@ -14,14 +14,14 @@ contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py,sha256=Mos
 contentctl/actions/detection_testing/views/DetectionTestingViewFile.py,sha256=OJgmQgoVnzy7p1MN9bDyKGUhFWKzQc6ejc4F87uZG1I,1123
 contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py,sha256=6mecacXFoTJxcHiRZSnlHos5Hca1jdedEEZfiIAhaJg,4706
 contentctl/actions/doc_gen.py,sha256=YNc1VYA0ikL1hWDHYjfEOmUkfhy8PEIdvTyC4ZLxQRY,863
-contentctl/actions/initialize.py,sha256=2h3_A68mNWcyZjbrKF-OeQXBi5p4Zu3z74K7QxEtII4,1749
+contentctl/actions/initialize.py,sha256=BRKmvLr50dnL4SnUEGM6jRfAghfZAmk0hFFWIZcKpxg,1809
 contentctl/actions/initialize_old.py,sha256=0qXbW_fNDvkcnEeL6Zpte8d-hpTu1REyzHsXOCY-YB8,9333
 contentctl/actions/inspect.py,sha256=6gVVKmV5CUUYOkNNVTMPKj9bM1uXVthgGCoFKZGDeS8,12628
 contentctl/actions/new_content.py,sha256=o5ZYBQ216RN6TnW_wRxVGJybx2SsJ7ht4PAi1dw45Yg,6076
 contentctl/actions/release_notes.py,sha256=akkFfLhsJuaPUyjsb6dLlKt9cUM-JApAjTFQMbYoXeM,13115
 contentctl/actions/reporting.py,sha256=MJEmvmoA1WnSFZEU9QM6daL_W94oOX0WXAcX1qAM2As,1583
 contentctl/actions/test.py,sha256=dx7f750_MrlvysxOmOdIro1bH0iVKF4K54TSwhvU2MU,5146
-contentctl/actions/validate.py,sha256=HnmB_qsluYr0BFHQzg0HvXGLHM4M1taBtsWt774esf8,2537
+contentctl/actions/validate.py,sha256=HfHfUTaRNx8eItociAEgQt8BEOVy9jb2yc8bKAGSsFA,3574
 contentctl/api.py,sha256=FBOpRhbBCBdjORmwe_8MPQ3PRZ6T0KrrFcfKovVFkug,6343
 contentctl/contentctl.py,sha256=Vr2cuvaPjpJpYvD9kVoYq7iD6rhLQEpTKmcGoq4emhA,10470
 contentctl/enrichments/attack_enrichment.py,sha256=EkEloG3hMmPTloPyYiVkhq3iT_BieXaJmprJ5stfyRw,6732
@@ -29,15 +29,15 @@ contentctl/enrichments/cve_enrichment.py,sha256=IzkKSdnQi3JrAUUyLpcGA_Y2g_B7latq
 contentctl/enrichments/splunk_app_enrichment.py,sha256=zDNHFLZTi2dJ1gdnh0sHkD6F1VtkblqFnhacFcCMBfc,3418
 contentctl/helper/link_validator.py,sha256=-XorhxfGtjLynEL1X4hcpRMiyemogf2JEnvLwhHq80c,7139
 contentctl/helper/logger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-contentctl/helper/utils.py,sha256=THV4ZuaeEHG6PK5JeZUkTLBmvN4fUV0Ar6opH1zXxKs,16545
+contentctl/helper/utils.py,sha256=8ICRvE7DUiNL9BK4Hw71hCLFbd3R2u86OwKeDOdaBTY,19454
 contentctl/input/backend_splunk_ba.py,sha256=Y70tJqgaUM0nzfm2SiGMof4HkhY84feqf-xnRx1xPb4,5861
-contentctl/input/director.py,sha256=BR1RvBD0U_JtHtrM3jM_FpcvaaNlME7nc-gNO4RJLM8,13323
+contentctl/input/director.py,sha256=36Licm8TV62oDk1s97Y7EFGvTKAP1ryXi7NL4BXP9kU,12603
 contentctl/input/new_content_questions.py,sha256=o4prlBoUhEMxqpZukquI9WKbzfFJfYhEF7a8m2q_BEE,5565
 contentctl/input/sigma_converter.py,sha256=ATFNW7boNngp5dmWM7Gr4rMZrUKjvKW2_qu28--FdiU,19391
 contentctl/input/ssa_detection_builder.py,sha256=dke9mPn2VQVSpYiaGWjZn3PkqVJTe58gcT2Vifv9_yc,8159
-contentctl/input/yml_reader.py,sha256=oaal24UP8rDXkCmN5I3GnIheZrsgkhbKOlzXtyhB474,1475
-contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=XhyMpghvizU37sOHKE009la1wo__EZqGdemXt9En5wc,34039
-contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py,sha256=IVr26xFrIlTvsqQoqwYl4cmcfaP9BeYt9I0QTriKmwE,8451
+contentctl/input/yml_reader.py,sha256=hyVUYhx4Ka8C618kP2D_E3sDUKEQGC6ty_QZQArHKd4,1489
+contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=gpyl-hnDNAxxMAHDMBZJMK10yZOq78Y6-ZMCStvrgQM,35354
+contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py,sha256=zpteipbBAwBe90SOmq5ky0KIztS806jnrg1vsBr5PsM,9778
 contentctl/objects/alert_action.py,sha256=E9gjCn5C31h0sN7k90KNe4agRxFFSnMW_Z-Ri_3YQss,1335
 contentctl/objects/atomic.py,sha256=a_G_iliAm86BunpAAG86aAL3LAEGpd9Crp7t7-PxYvI,8979
 contentctl/objects/base_test.py,sha256=6hCL9K-N_jJx1zLbuZQCsB93_XWj6JcGGs2PbbjzJWo,1028
@@ -47,7 +47,7 @@ contentctl/objects/baseline_tags.py,sha256=JLdlCUc_DEccMQD6f-sa2qD8pcxYiwMUT_sRZ
 contentctl/objects/config.py,sha256=tK0BY4A9Go5jp8tpOSwgczuOAyu9dMPvC0nyOHeO-74,43642
 contentctl/objects/constants.py,sha256=1LjiK9A7t0aHHkJz2mrW-DImdW1P98GPssTwmwNNI_M,3468
 contentctl/objects/correlation_search.py,sha256=B97vCt2Ew7PGgqd5Y9l6RD3DJdy51Eh7Gzkxxs2xqZ0,36891
-contentctl/objects/data_source.py,sha256=UoI1zLyrwwTtKqvXf_K-TifJEY5HaVBItR3vR4J43iw,768
+contentctl/objects/data_source.py,sha256=ThZivI3NoQybD0C0fS_f-FvhjhD5_n09IML913e1fEY,1459
 contentctl/objects/deployment.py,sha256=Qc6M4yeOvxjqFKR8sfjd4CG06AbVheTOqP1mwqo4t8s,2651
 contentctl/objects/deployment_email.py,sha256=Zu9cXZdfOP6noa_mZpiK1GrYCTgi3Mim94iLGjE674c,147
 contentctl/objects/deployment_notable.py,sha256=QhOI7HEkUuuqk0fum9SD8IpYBlbwIsJUff8s3kCKKj4,198
@@ -57,13 +57,13 @@ contentctl/objects/deployment_scheduling.py,sha256=bQjbJHNaUGdU1VAGV8-nFOHzHutbI
 contentctl/objects/deployment_slack.py,sha256=P6z8OLHDKcDWx7nbKWasqBc3dFRatGcpO2GtmxzVV8I,135
 contentctl/objects/detection.py,sha256=3W41cXf3ECjWuPqWrseqSLC3PAA7O5_nENWWM6MPK0Y,620
 contentctl/objects/detection_tags.py,sha256=nAHRuBtltx4Rsx9htPtxizRlmQOSypYysbzqn3CQZ_I,10321
-contentctl/objects/enums.py,sha256=cW-orYfVBgMdZKVS8ANAkSZ-zygbrhJZX6FP4TxNGgg,14075
-contentctl/objects/event_source.py,sha256=oOCCSQpfpSbYw6_v103I4VxwqjpXP4gsTbds06qiEa0,251
+contentctl/objects/enums.py,sha256=37v7w8xCg5j5hxP3kod0S3HQ9BY-CqZulPiwhnTtEvs,14052
+contentctl/objects/event_source.py,sha256=G9P7rtcN5hcBNQx6DG37mR3QyQufx--T6kgQGNqQuKk,415
 contentctl/objects/integration_test.py,sha256=W_VksBN_cRo7DTXdr1aLujjS9mgkEp0uvoNpmL0dVnQ,1273
 contentctl/objects/integration_test_result.py,sha256=DrIZRRlILSHGcsK_Rlm3KJLnbKPtIen8uEPFi4ZdJ8s,370
 contentctl/objects/investigation.py,sha256=JRoZxc_qi1fu_VFTRaxOc3B7zzSzCfEURsNzWPUCrtY,2620
 contentctl/objects/investigation_tags.py,sha256=nFpMRKBVBsW21YW_vy2G1lXaSARX-kfFyrPoCyE77Q8,1280
-contentctl/objects/lookup.py,sha256=P8YbzdDAj_MsTBJTEsym35zhQjiN9Eq0MlfON-qvuTM,4556
+contentctl/objects/lookup.py,sha256=TwNQqeMPeE8sfAjChxS2yDnejI2Xf3ils3_Xdgthr5c,6924
 contentctl/objects/macro.py,sha256=9nE-bxkFhtaltHOUCr0luU8jCCthmglHjhKs6Q2YzLU,2684
 contentctl/objects/mitre_attack_enrichment.py,sha256=bWrMG-Xj3knmULR5q2YZk7mloJBdQUzU1moZfEw9lQM,1073
 contentctl/objects/notable_action.py,sha256=ValkblBaG-60TF19y_vSnNzoNZ3eg48wIfr0qZxyKTA,1605
@@ -75,7 +75,7 @@ contentctl/objects/risk_object.py,sha256=yY4NmEwEKaRl4sLzCRZb1n8kdpV3HzYbQVQ1ClQ
 contentctl/objects/security_content_object.py,sha256=j8KNDwSMfZsSIzJucC3NuZo0SlFVpqHfDc6y3-YHjHI,234
 contentctl/objects/ssa_detection.py,sha256=-G6tXfVVlZgPWS64hIIy3M-aMePANAuQvdpXPlgUyUs,5873
 contentctl/objects/ssa_detection_tags.py,sha256=u8annjzo3MYZ-16wyFnuR8qJJzRa4LEhdprMIrQ47G0,5224
-contentctl/objects/story.py,sha256=GMcMAaDcX16B2mbSAzrzpVD8IMyOILAyIo2rR8jySto,4618
+contentctl/objects/story.py,sha256=FXe11LV19xJTtCgx7DKdvV9cL0gKeryUnE3yjpnDmrU,4957
 contentctl/objects/story_tags.py,sha256=0oF1OePLBxa-RQPb438tXrrfosa939CP8UbNV0_S8XY,2225
 contentctl/objects/test_group.py,sha256=Yb1sqGom6SkVL8B3czPndz8w3CK8WdwZ39V_cn0_JZQ,2600
 contentctl/objects/threat_object.py,sha256=S8B7RQFfLxN_g7yKPrDTuYhIy9JvQH3YwJ_T5LUZIa4,711
@@ -91,6 +91,7 @@ contentctl/output/attack_nav_writer.py,sha256=64ILZLmNbh2XLmbopgENkeo6t-4SRRG8xZ
 contentctl/output/ba_yml_output.py,sha256=Lrk13Q9-f71i3c0oNrT50G94PxdogG4k4-MI-rTMOAo,5950
 contentctl/output/conf_output.py,sha256=qCRT77UKNFCe4AufeBV8Uz9lkPqgpGzU1Y149RuEnis,10147
 contentctl/output/conf_writer.py,sha256=2TaCAPEtU-bMa7A2m7xOxh93PMpzIdhwiHiPLUCeCB4,8281
+contentctl/output/data_source_writer.py,sha256=55gi6toAMcjj0AxOmYMwkTHcANCfK6dezQs5aIQTW4k,1737
 contentctl/output/detection_writer.py,sha256=AzxbssNLmsNIOaYKotew5-ONoyq1cQpKSGy3pe191B0,960
 contentctl/output/doc_md_output.py,sha256=gf7osH1uSrC6js3D_I72g4uDe9TaB3tsvtqCHi5znp0,3238
 contentctl/output/finding_report_writer.py,sha256=bjJR7NAxLE8vt8uU3zSDhazQzqzOdtCsUu95lVdzU_w,3939
@@ -148,6 +149,7 @@ contentctl/templates/app_template/static/appIcon.png,sha256=jcJ1PNdkBX7Kl_y9Tf0S
 contentctl/templates/app_template/static/appIconAlt.png,sha256=uRXjoHQQjs0-BxcK-3KNBEdck1adDNTHMvV14xR4W0g,2656
 contentctl/templates/app_template/static/appIconAlt_2x.png,sha256=I0m-CPRqq7ak9NJQZGGmz6Ac4pmzFV_SonOUxOEDOFs,7442
 contentctl/templates/app_template/static/appIcon_2x.png,sha256=XEpqQzDvzuEV5StzD05XRgxwySqHHLes1hMPy2v5Vdk,3657
+contentctl/templates/data_sources/sysmon_eventid_1.yml,sha256=7PIcLr1e9Ql-wu_Dk9D4JAZs1OWDby-tY77nDDUZ1CQ,6079
 contentctl/templates/datamodels_cim.conf,sha256=RB_SCtpQG_KaC_0lKTCKexVOlEq_ShGwpGlg95aqOfs,9381
 contentctl/templates/datamodels_custom.conf,sha256=6BANthXdqg3fYpYmEqiGZnv4cWheNfXz1uQ_I1JePXc,480
 contentctl/templates/deployments/escu_default_configuration_anomaly.yml,sha256=j_H2wovWBj1EKxVwj3mMoJVQnVm-2Imt7xnB9U1Tun4,418
@@ -157,14 +159,14 @@ contentctl/templates/deployments/escu_default_configuration_hunting.yml,sha256=h
 contentctl/templates/deployments/escu_default_configuration_ttp.yml,sha256=1D-pvzaH1v3_yCZXaY6njmdvV4S2_Ak8uzzCOsnj9XY,548
 contentctl/templates/detections/application/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 contentctl/templates/detections/cloud/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml,sha256=hkN214ZOqbQPWyYrqgbOrYb4iA0DroG1AnFRhSC_m0M,3323
+contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml,sha256=-jp7CC3shnA9Te_0Zw6jLbLT8JnrVQvOfEUkNCQbCNo,3322
 contentctl/templates/detections/network/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
 contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
 contentctl/templates/stories/cobalt_strike.yml,sha256=rlaXxMN-5k8LnKBLPafBoksyMtlmsPMHPJOjTiMiZ-M,3063
-contentctl-4.1.4.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
-contentctl-4.1.4.dist-info/METADATA,sha256=gyBEu2sSknwen-oTm0WdQtCvaPgbWuJBd4jvPuel6iQ,19706
-contentctl-4.1.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-contentctl-4.1.4.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
-contentctl-4.1.4.dist-info/RECORD,,
+contentctl-4.2.0.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
+contentctl-4.2.0.dist-info/METADATA,sha256=Mwn05R0o74IKd0Z0KdayxzX3E7wJGJQSTfu4fzQxjQA,19706
+contentctl-4.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+contentctl-4.2.0.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
+contentctl-4.2.0.dist-info/RECORD,,