contentctl 4.0.4__py3-none-any.whl → 4.1.0__py3-none-any.whl

contentctl/input/director.py

@@ -5,9 +5,8 @@ from dataclasses import dataclass, field
 from pydantic import ValidationError
 from uuid import UUID
 from contentctl.input.yml_reader import YmlReader
-    
-    
-    
+
+
 from contentctl.objects.detection import Detection
 from contentctl.objects.story import Story
 
@@ -28,29 +27,69 @@ from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.objects.config import validate
 
 
-
-@dataclass()
+@dataclass
 class DirectorOutputDto:
-     # Atomic Tests are first because parsing them 
-     # is far quicker than attack_enrichment
-     atomic_tests: Union[list[AtomicTest],None]
-     attack_enrichment: AttackEnrichment
-     detections: list[Detection]
-     stories: list[Story]
-     baselines: list[Baseline]
-     investigations: list[Investigation]
-     playbooks: list[Playbook]
-     macros: list[Macro]
-     lookups: list[Lookup]
-     deployments: list[Deployment]
-     ssa_detections: list[SSADetection]
-     #cve_enrichment: CveEnrichment
-
-     name_to_content_map: dict[str, SecurityContentObject] = field(default_factory=dict)
-     uuid_to_content_map: dict[UUID, SecurityContentObject] = field(default_factory=dict)
-     
+    # Atomic Tests are first because parsing them 
+    # is far quicker than attack_enrichment
+    atomic_tests: Union[list[AtomicTest],None]
+    attack_enrichment: AttackEnrichment
+    cve_enrichment: CveEnrichment
+    detections: list[Detection]
+    stories: list[Story]
+    baselines: list[Baseline]
+    investigations: list[Investigation]
+    playbooks: list[Playbook]
+    macros: list[Macro]
+    lookups: list[Lookup]
+    deployments: list[Deployment]
+    ssa_detections: list[SSADetection]
+
+    name_to_content_map: dict[str, SecurityContentObject] = field(default_factory=dict)
+    uuid_to_content_map: dict[UUID, SecurityContentObject] = field(default_factory=dict)
+
+    def addContentToDictMappings(self, content: SecurityContentObject):
+        content_name = content.name
+        if isinstance(content, SSADetection):
+            # Since SSA detections may have the same name as ESCU detection,
+            # for this function we prepend 'SSA ' to the name.
+            content_name = f"SSA {content_name}"
+        if content_name in self.name_to_content_map:
+            raise ValueError(
+                f"Duplicate name '{content_name}' with paths:\n"
+                f" - {content.file_path}\n"
+                f" - {self.name_to_content_map[content_name].file_path}"
+            )
+        elif content.id in self.uuid_to_content_map:
+            raise ValueError(
+                f"Duplicate id '{content.id}' with paths:\n"
+                f" - {content.file_path}\n"
+                f" - {self.name_to_content_map[content_name].file_path}"
+            )
+
+        if isinstance(content, Lookup):
+            self.lookups.append(content)
+        elif isinstance(content, Macro):
+            self.macros.append(content)
+        elif isinstance(content, Deployment):
+            self.deployments.append(content)
+        elif isinstance(content, Playbook):
+            self.playbooks.append(content)
+        elif isinstance(content, Baseline):
+            self.baselines.append(content)
+        elif isinstance(content, Investigation):
+            self.investigations.append(content)
+        elif isinstance(content, Story):
+            self.stories.append(content)
+        elif isinstance(content, Detection):
+            self.detections.append(content)
+        elif isinstance(content, SSADetection):
+            self.ssa_detections.append(content)
+        else:
+             raise Exception(f"Unknown security content type: {type(content)}")
 
 
+        self.name_to_content_map[content_name] = content
+        self.uuid_to_content_map[content.id] = content
 
 
 from contentctl.input.ssa_detection_builder import SSADetectionBuilder
@@ -60,13 +99,6 @@ from contentctl.objects.enums import DetectionStatus
 from contentctl.helper.utils import Utils
 
 
-
-
-
-
-     
-
-
 class Director():
     input_dto: validate
     output_dto: DirectorOutputDto
@@ -77,27 +109,7 @@ class Director():
     def __init__(self, output_dto: DirectorOutputDto) -> None:
         self.output_dto = output_dto
         self.ssa_detection_builder = SSADetectionBuilder()
-    
-    def addContentToDictMappings(self, content:SecurityContentObject):
-         content_name = content.name
-         if isinstance(content,SSADetection):
-            # Since SSA detections may have the same name as ESCU detection,
-            # for this function we prepend 'SSA ' to the name.
-            content_name = f"SSA {content_name}"               
-         if content_name in self.output_dto.name_to_content_map:
-            raise ValueError(f"Duplicate name '{content_name}' with paths:\n"
-                             f" - {content.file_path}\n"
-                             f" - {self.output_dto.name_to_content_map[content_name].file_path}")
-         elif content.id in self.output_dto.uuid_to_content_map:
-            raise ValueError(f"Duplicate id '{content.id}' with paths:\n"
-                    f" - {content.file_path}\n"
-                    f" - {self.output_dto.name_to_content_map[content_name].file_path}")
-         
-         self.output_dto.name_to_content_map[content_name] = content 
-         self.output_dto.uuid_to_content_map[content.id] = content 
-    
         
-    
     def execute(self, input_dto: validate) -> None:
         self.input_dto = input_dto
 
@@ -146,50 +158,41 @@ class Director():
 
                 if contentType == SecurityContentType.lookups:
                         lookup = Lookup.model_validate(modelDict,context={"output_dto":self.output_dto, "config":self.input_dto})
-                        self.output_dto.lookups.append(lookup)
-                        self.addContentToDictMappings(lookup)
+                        self.output_dto.addContentToDictMappings(lookup)
                 
                 elif contentType == SecurityContentType.macros:
                         macro = Macro.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.macros.append(macro)
-                        self.addContentToDictMappings(macro)
+                        self.output_dto.addContentToDictMappings(macro)
                 
                 elif contentType == SecurityContentType.deployments:
                         deployment = Deployment.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.deployments.append(deployment)
-                        self.addContentToDictMappings(deployment)
+                        self.output_dto.addContentToDictMappings(deployment)
                 
                 elif contentType == SecurityContentType.playbooks:
                         playbook = Playbook.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.playbooks.append(playbook)  
-                        self.addContentToDictMappings(playbook)                  
+                        self.output_dto.addContentToDictMappings(playbook)                  
                 
                 elif contentType == SecurityContentType.baselines:
                         baseline = Baseline.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.baselines.append(baseline)
-                        self.addContentToDictMappings(baseline)
+                        self.output_dto.addContentToDictMappings(baseline)
                 
                 elif contentType == SecurityContentType.investigations:
                         investigation = Investigation.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.investigations.append(investigation)
-                        self.addContentToDictMappings(investigation)
+                        self.output_dto.addContentToDictMappings(investigation)
 
                 elif contentType == SecurityContentType.stories:
                         story = Story.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.stories.append(story)
-                        self.addContentToDictMappings(story)
+                        self.output_dto.addContentToDictMappings(story)
             
                 elif contentType == SecurityContentType.detections:
-                        detection = Detection.model_validate(modelDict,context={"output_dto":self.output_dto})
-                        self.output_dto.detections.append(detection)
-                        self.addContentToDictMappings(detection)
+                        detection = Detection.model_validate(modelDict,context={"output_dto":self.output_dto, "app":self.input_dto.app})
+                        self.output_dto.addContentToDictMappings(detection)
 
                 elif contentType == SecurityContentType.ssa_detections:
                         self.constructSSADetection(self.ssa_detection_builder, self.output_dto,str(file))
                         ssa_detection = self.ssa_detection_builder.getObject()
                         if ssa_detection.status in [DetectionStatus.production.value, DetectionStatus.validation.value]:
-                            self.output_dto.ssa_detections.append(ssa_detection)
-                            self.addContentToDictMappings(ssa_detection)
+                            self.output_dto.addContentToDictMappings(ssa_detection)
 
                 else:
                         raise Exception(f"Unsupported type: [{contentType}]")
@@ -228,6 +231,3 @@ class Director():
         builder.addMappings()
         builder.addUnitTest()
         builder.addRBA()
-
-
-    

contentctl/input/new_content_questions.py

@@ -27,11 +27,6 @@ class NewContentQuestions:
                 'message': 'enter author name',
                 'name': 'detection_author',
             },
-            {
-                "type": "text",
-                "message": "enter author name",
-                "name": "detection_author",
-            },
             {
                 "type": "select",
                 "message": "select a detection type",

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -26,7 +26,7 @@ from contentctl.objects.integration_test import IntegrationTest
 
 #from contentctl.objects.playbook import Playbook
 from contentctl.objects.enums import DataSource,ProvidingTechnology
-from contentctl.enrichments.cve_enrichment import CveEnrichment, CveEnrichmentObj
+from contentctl.enrichments.cve_enrichment import CveEnrichmentObj
 
 
 class Detection_Abstract(SecurityContentObject):
@@ -40,7 +40,6 @@ class Detection_Abstract(SecurityContentObject):
     search: Union[str, dict[str,Any]] = Field(...)
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
-    check_references: bool = False  
     #data_source: Optional[List[DataSource]] = None
 
     enabled_by_default: bool = False
@@ -54,6 +53,58 @@ class Detection_Abstract(SecurityContentObject):
     # A list of groups of tests, relying on the same data
     test_groups: Union[list[TestGroup], None] = Field(None,validate_default=True)
 
+
+    @field_validator("search", mode="before")
+    @classmethod
+    def validate_presence_of_filter_macro(cls, value:Union[str, dict[str,Any]], info:ValidationInfo)->Union[str, dict[str,Any]]:
+        """
+        Validates that, if required to be present, the filter macro is present with the proper name.
+        The filter macro MUST be derived from the name of the detection
+
+
+        Args:
+            value (Union[str, dict[str,Any]]): The search. It can either be a string (and should be SPL) 
+                                               or a dict, in which case it is Sigma-formatted.
+            info (ValidationInfo): The validation info can contain a number of different objects. Today it only contains the director. 
+
+        Returns:
+            Union[str, dict[str,Any]]: The search, either in sigma or SPL format.
+        """        
+        
+        if isinstance(value,dict):
+            #If the search is a dict, then it is in Sigma format so return it
+            return value
+        
+        # Otherwise, the search is SPL.
+        
+        
+        # In the future, we will may add support that makes the inclusion of the 
+        # filter macro optional or automatically generates it for searches that 
+        # do not have it. For now, continue to require that all searches have a filter macro.
+        FORCE_FILTER_MACRO = True
+        if not FORCE_FILTER_MACRO:
+            return value
+        
+        # Get the required macro name, which is derived from the search name.
+        # Note that a separate validation ensures that the file name matches the content name
+        name:Union[str,None] = info.data.get("name",None)
+        if name is None:
+            #The search was sigma formatted (or failed other validation and was None), so we will not validate macros in it
+            raise ValueError("Cannot validate filter macro, field 'name' (which is required to validate the macro) was missing from the detection YML.")
+        
+        #Get the file name without the extension. Note this is not a full path!
+        file_name = pathlib.Path(cls.contentNameToFileName(name)).stem
+        file_name_with_filter = f"`{file_name}_filter`"
+        
+        if file_name_with_filter not in value:
+            raise ValueError(f"Detection does not contain the EXACT filter macro {file_name_with_filter}. "
+                             "This filter macro MUST be present in the search. It usually placed at the end "
+                             "of the search and is useful for environment-specific filtering of False Positive or noisy results.")
+        
+        return value
+
+
+
     @field_validator("test_groups")
     @classmethod
     def validate_test_groups(cls, value:Union[None, List[TestGroup]], info:ValidationInfo) -> Union[List[TestGroup], None]:
@@ -144,17 +195,30 @@ class Detection_Abstract(SecurityContentObject):
     macros: list[Macro] = Field([],validate_default=True)
     lookups: list[Lookup] = Field([],validate_default=True)
 
-    @computed_field
-    @property
-    def cve_enrichment(self)->List[CveEnrichmentObj]:
-        raise Exception("CVE Enrichment Functionality not currently supported.  It will be re-added at a later time.")
-        enriched_cves = []
-        for cve_id in self.tags.cve:
-            print(f"\nEnriching {cve_id}\n")
-            enriched_cves.append(CveEnrichment.enrich_cve(cve_id))
+    cve_enrichment: list[CveEnrichmentObj] = Field([], validate_default=True)
+    
+    @model_validator(mode="after")
+    def cve_enrichment_func(self, info:ValidationInfo):
+        if len(self.cve_enrichment) > 0:
+            raise ValueError(f"Error, field 'cve_enrichment' should be empty and "
+                             f"dynamically populated at runtime. Instead, this field contained: {self.cve_enrichment}")
+
+        output_dto:Union[DirectorOutputDto,None]= info.context.get("output_dto",None)
+        if output_dto is None:
+            raise ValueError("Context not provided to detection model post validator")
+        
+        
+        enriched_cves:list[CveEnrichmentObj] = []
 
-        return enriched_cves
+        for cve_id in self.tags.cve:
+            try:
+                enriched_cves.append(output_dto.cve_enrichment.enrich_cve(cve_id, raise_exception_on_failure=False))
+            except Exception as e:
+                raise ValueError(f"{e}")
+        self.cve_enrichment = enriched_cves
+        return self
     
+
     splunk_app_enrichment: Optional[List[dict]] = None
     
     @computed_field
@@ -382,11 +446,11 @@ class Detection_Abstract(SecurityContentObject):
             filter_macro = Macro.model_validate({"name":filter_macro_name, 
                                                 "definition":'search *', 
                                                 "description":'Update this macro to limit the output results to filter out false positives.'})
-            director.macros.append(filter_macro)
+            director.addContentToDictMappings(filter_macro)
         
         macros_from_search = Macro.get_macros(search, director)
         
-        return  macros_from_search + [filter_macro]
+        return  macros_from_search
 
     def get_content_dependencies(self)->list[SecurityContentObject]:
         #Do this separately to satisfy type checker

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -12,6 +12,7 @@ import re
 import abc
 import uuid
 import datetime
+import pprint
 from pydantic import BaseModel, field_validator, Field, ValidationInfo, FilePath, HttpUrl, NonNegativeInt, ConfigDict, model_validator, model_serializer
 from typing import Tuple, Optional, List, Union
 import pathlib
@@ -181,6 +182,22 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
         for object in all_objects:
             name_dict[str(pathlib.Path(object.file_path))] = object
         return name_dict
+
+
+    def __repr__(self)->str:
+        # Just use the model_dump functionality that
+        # has already been written. This loses some of the
+        # richness where objects reference themselves, but
+        # is usable
+        m = self.model_dump()
+        return pprint.pformat(m, indent=3)
+
+    def __str__(self)->str:
+        return(self.__repr__())
+        
+
+    
+
     
 
     

contentctl/objects/baseline.py

@@ -31,7 +31,6 @@ class Baseline(SecurityContentObject):
     search: str = Field(..., min_length=4)
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
-    check_references: bool = False #Validation is done in order, this field must be defined first
     tags: BaselineTags = Field(...)
 
     # enrichment

contentctl/objects/config.py

@@ -154,6 +154,10 @@ class Config_Base(BaseModel):
 
     path: DirectoryPath = Field(default=DirectoryPath("."), description="The root of your app.")
     app:CustomApp = Field(default_factory=CustomApp)
+    verbose:bool = Field(default=False, description="Enable verbose error logging, including a stacktrace. "
+                         "This option makes debugging contentctl errors much easier, but produces way more "
+                         "output than is useful under most uses cases. "
+                         "Please use this flag if you are submitting a bug report or issue on GitHub.")
     
     @field_serializer('path',when_used='always')
     def serialize_path(path: DirectoryPath)->str:
@@ -269,14 +273,6 @@ class Infrastructure(BaseModel):
     instance_name: str = Field(...)
 
 
-class deploy_rest(build):
-    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
-    
-    target:Infrastructure = Infrastructure(instance_name="splunk_target_host", instance_address="localhost")
-    #This will overwrite existing content without promprting for confirmation
-    overwrite_existing_content:bool = Field(default=True, description="Overwrite existing macros and savedsearches in your enviornment")
-
-
 class Container(Infrastructure):
     model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
     instance_address:str = Field(default="localhost", description="Address of your splunk server.")

contentctl/objects/detection_tags.py

@@ -145,7 +145,7 @@ class DetectionTags(BaseModel):
     @model_validator(mode="after")
     def addAttackEnrichment(self, info:ValidationInfo):
         if len(self.mitre_attack_enrichments) > 0:
-            raise ValueError(f"Error, field 'mitre_attack_enrichment' should be empty and dynamically populated at runtime. Instead, this field contained: {str(v)}")
+            raise ValueError(f"Error, field 'mitre_attack_enrichment' should be empty and dynamically populated at runtime. Instead, this field contained: {self.mitre_attack_enrichments}")
         
         output_dto:Union[DirectorOutputDto,None]= info.context.get("output_dto",None)
         if output_dto is None:

contentctl/objects/macro.py

@@ -9,13 +9,14 @@ if TYPE_CHECKING:
 from contentctl.objects.security_content_object import SecurityContentObject
 
 
-
-MACROS_TO_IGNORE = set(["_filter", "drop_dm_object_name"])
-#Should all of the following be included as well?
-MACROS_TO_IGNORE.add("get_asset" )
-MACROS_TO_IGNORE.add("get_risk_severity")
-MACROS_TO_IGNORE.add("cim_corporate_web_domain_search")
-MACROS_TO_IGNORE.add("prohibited_processes")
+#The following macros are included in commonly-installed apps.
+#As such, we will ignore if they are missing from our app.
+#Included in 
+MACROS_TO_IGNORE = set(["drop_dm_object_name"]) # Part of CIM/Splunk_SA_CIM
+MACROS_TO_IGNORE.add("get_asset") #SA-IdentityManagement, part of Enterprise Security
+MACROS_TO_IGNORE.add("get_risk_severity") #SA-ThreatIntelligence, part of Enterprise Security
+MACROS_TO_IGNORE.add("cim_corporate_web_domain_search") #Part of CIM/Splunk_SA_CIM
+#MACROS_TO_IGNORE.add("prohibited_processes")
 
 
 class Macro(SecurityContentObject):

contentctl/objects/story_tags.py

@@ -14,6 +14,8 @@ class StoryUseCase(str,Enum):
    APPLICATION_SECURITY = "Application Security"
    SECURITY_MONITORING = "Security Monitoring"
    ADVANCED_THREAD_DETECTION = "Advanced Threat Detection"
+   INSIDER_THREAT = "Insider Threat"
+   OTHER = "Other"
 
 class StoryTags(BaseModel):
    model_config = ConfigDict(extra='forbid', use_enum_values=True)

contentctl/output/yml_writer.py

@@ -8,4 +8,42 @@ class YmlWriter:
     def writeYmlFile(file_path : str, obj : dict[Any,Any]) -> None:
 
         with open(file_path, 'w') as outfile:
-            yaml.safe_dump(obj, outfile, default_flow_style=False, sort_keys=False)
+            yaml.safe_dump(obj, outfile, default_flow_style=False, sort_keys=False)
+
+    @staticmethod
+    def writeDetection(file_path: str, obj: dict[Any,Any]) -> None:
+        output = dict()
+        output["name"] = obj["name"]
+        output["id"] = obj["id"]
+        output["version"] = obj["version"]
+        output["date"] = obj["date"]
+        output["author"] = obj["author"]
+        output["type"] = obj["type"]
+        output["status"] = obj["status"]
+        output["data_source"] = obj['data_sources']
+        output["description"] = obj["description"]
+        output["search"] = obj["search"]
+        output["how_to_implement"] = obj["how_to_implement"]
+        output["known_false_positives"] = obj["known_false_positives"]
+        output["references"] = obj["references"]
+        output["tags"] = obj["tags"]
+        output["tests"] = obj["tags"]
+
+        YmlWriter.writeYmlFile(file_path=file_path, obj=output)
+ 
+    @staticmethod
+    def writeStory(file_path: str, obj: dict[Any,Any]) -> None:
+        output = dict()
+        output['name'] = obj['name']
+        output['id'] = obj['id']
+        output['version'] = obj['version']
+        output['date'] = obj['date']
+        output['author'] = obj['author']
+        output['description'] = obj['description']
+        output['narrative'] = obj['narrative']
+        output['references'] = obj['references']
+        output['tags'] = obj['tags']
+
+        YmlWriter.writeYmlFile(file_path=file_path, obj=output)
+ 
+

{contentctl-4.0.4.dist-info → contentctl-4.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contentctl
-Version: 4.0.4
+Version: 4.1.0
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT
@@ -10,25 +10,24 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: Jinja2 (>=3.1.2,<4.0.0)
+Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
 Requires-Dist: attackcti (>=0.3.7,<0.4.0)
 Requires-Dist: bottle (>=0.12.25,<0.13.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: gitpython (>=3.1.43,<4.0.0)
 Requires-Dist: pycvesearch (>=1.2,<2.0)
-Requires-Dist: pydantic (>=2.5.1,<3.0.0)
+Requires-Dist: pydantic (>=2.7.1,<3.0.0)
 Requires-Dist: pygit2 (>=1.14.1,<2.0.0)
-Requires-Dist: pysigma (>=0.10.8,<0.11.0)
-Requires-Dist: pysigma-backend-splunk (>=1.0.3,<2.0.0)
+Requires-Dist: pysigma (>=0.11.5,<0.12.0)
+Requires-Dist: pysigma-backend-splunk (>=1.1.0,<2.0.0)
 Requires-Dist: questionary (>=2.0.1,<3.0.0)
 Requires-Dist: requests (>=2.32.2,<2.33.0)
 Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
-Requires-Dist: setuptools (>=69.5.1,<70.0.0)
+Requires-Dist: setuptools (>=69.5.1,<71.0.0)
 Requires-Dist: splunk-sdk (>=2.0.1,<3.0.0)
-Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Requires-Dist: tqdm (>=4.66.4,<5.0.0)
 Requires-Dist: tyro (>=0.8.3,<0.9.0)
-Requires-Dist: validators (>=0.22.0,<0.23.0)
 Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
 Description-Content-Type: text/markdown