contentctl 3.6.0__py3-none-any.whl → 4.0.2__py3-none-any.whl

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -1,71 +1,175 @@
 from __future__ import annotations
+from typing import TYPE_CHECKING, Self
+
+if TYPE_CHECKING:
+    from contentctl.objects.deployment import Deployment        
+    from contentctl.objects.security_content_object import SecurityContentObject
+    from contentctl.objects.config import Config
+    from contentctl.input.director import DirectorOutputDto
+    
+from contentctl.objects.enums import AnalyticsType
 import re
 import abc
-import string
-import uuid
-from datetime import datetime
-from pydantic import BaseModel, validator, ValidationError, Field
-from contentctl.objects.enums import SecurityContentType
-from typing import Tuple
-
 import uuid
+import datetime
+from pydantic import BaseModel, field_validator, Field, ValidationInfo, FilePath, HttpUrl, NonNegativeInt, ConfigDict, model_validator, model_serializer
+from typing import Tuple, Optional, List, Union
 import pathlib
+       
+
+
+
+
+NO_FILE_NAME = "NO_FILE_NAME"
+
 
-NO_FILE_BUILT_AT_RUNTIME = "NO_FILE_BUILT_AT_RUNTIME"
 class SecurityContentObject_Abstract(BaseModel, abc.ABC):
-    #contentType: SecurityContentType
-    name: str
-    author: str = "UNKNOWN_AUTHOR"
-    date: str = "1990-01-01"
-    version: int = 1
+    model_config = ConfigDict(use_enum_values=True,validate_default=True)
+    # name: str = ...
+    # author: str = Field(...,max_length=255)
+    # date: datetime.date = Field(...)
+    # version: NonNegativeInt = ...
+    # id: uuid.UUID = Field(default_factory=uuid.uuid4) #we set a default here until all content has a uuid
+    # description: str = Field(...,max_length=1000)
+    # file_path: FilePath = Field(...)
+    # references: Optional[List[HttpUrl]] = None
+    
+    name: str = Field("NO_NAME")
+    author: str = Field("Content Author",max_length=255)
+    date: datetime.date = Field(datetime.date.today())
+    version: NonNegativeInt = 1
     id: uuid.UUID = Field(default_factory=uuid.uuid4) #we set a default here until all content has a uuid
-    description: str = "UNKNOWN_DESCRIPTION"
-    file_path: str = "NO_FILE_BUILT_AT_RUNTIME"
+    description: str = Field("Enter Description Here",max_length=10000)
+    file_path: Optional[FilePath] = None
+    references: Optional[List[HttpUrl]] = None
 
-    @validator('name')
-    def name_max_length(cls, v):
-        if len(v) > 67:
-            raise ValueError('name is longer then 67 chars: ' + v)
-        return v
 
-    @validator('name')
-    def name_invalid_chars(cls, v):
-        invalidChars = set(string.punctuation.replace("-", ""))
-        if any(char in invalidChars for char in v):
-            raise ValueError('invalid chars used in name: ' + v)
-        return v
+    @model_serializer
+    def serialize_model(self):
+        return {
+            "name": self.name,
+            "author": self.author,
+            "date": str(self.date),
+            "version": self.version,
+            "id": str(self.id),
+            "description": self.description,
+            "references": [str(url) for url in self.references or []]
+        }
+
+    @staticmethod
+    def objectListToNameList(objects:list[SecurityContentObject], config:Optional[Config]=None)->list[str]:
+        return [object.getName(config) for object in objects]
+
+    # This function is overloadable by specific types if they want to redefine names, for example
+    # to have the format ESCU - NAME - Rule (config.tag - self.name - Rule)
+    def getName(self, config:Optional[Config])->str:
+        return self.name
+
+
+    @classmethod
+    def contentNameToFileName(cls, content_name:str)->str:
+        return content_name \
+            .replace(' ', '_') \
+            .replace('-','_') \
+            .replace('.','_') \
+            .replace('/','_') \
+            .lower() + ".yml"
+
+
+    @model_validator(mode="after")
+    def ensureFileNameMatchesSearchName(self):
+        file_name = self.contentNameToFileName(self.name)
+        
+        if (self.file_path is not None and file_name != self.file_path.name):
+            raise ValueError(f"The file name MUST be based off the content 'name' field:\n"\
+                            f"\t- Expected File Name: {file_name}\n"\
+                            f"\t- Actual File Name  : {self.file_path.name}")
+
+        return self
 
-    @validator('date')
-    def date_valid(cls, v, values):
-        try:
-            datetime.strptime(v, "%Y-%m-%d")
-        except:
-            raise ValueError('date is not in format YYYY-MM-DD: ' + values["name"])
+    @field_validator('file_path')
+    @classmethod
+    def file_path_valid(cls, v: Optional[pathlib.PosixPath], info: ValidationInfo):
+        if not v:
+            #It's possible that the object has no file path - for example filter macros that are created at runtime
+            return v
+        if not v.name.endswith(".yml"):
+            raise ValueError(f"All Security Content Objects must be YML files and end in .yml.  The following file does not: '{v}'")
         return v
 
-    @staticmethod
-    def free_text_field_valid(input_cls, v, values, field):
-        try:
-            v.encode('ascii')
-        except UnicodeEncodeError as e:
-            print(f"Potential Ascii encoding error in {values['name']}:{field.name} - {str(e)}")
-        except Exception as e:
-            print(f"Unknown encoding error in {values['name']}:{field.name} - {str(e)}")
+    def getReferencesListForJson(self)->List[str]:
+        return [str(url) for url in self.references or []]
         
+    @classmethod
+    def mapNamesToSecurityContentObjects(cls, v: list[str], director:Union[DirectorOutputDto,None])->list[Self]:
+        if director is not None:
+            name_map = director.name_to_content_map
+        else:
+            name_map = {}
         
-        if bool(re.search(r"[^\\]\n", v)):
-                raise ValueError(f"Unexpected newline(s) in {values['name']}:{field.name}.  Newline characters MUST be prefixed with \\")
-        return v
-    
-    
-    @validator("name", "author", 'description')
-    def description_valid(cls, v, values, field):
+
+
+        mappedObjects: list[Self] = []
+        mistyped_objects: list[SecurityContentObject_Abstract] = []
+        missing_objects: list[str] = []
+        for object_name in v:
+            found_object = name_map.get(object_name,None)
+            if not found_object:
+                missing_objects.append(object_name)
+            elif not isinstance(found_object,cls):
+                mistyped_objects.append(found_object)
+            else:
+                mappedObjects.append(found_object)
         
-        return SecurityContentObject_Abstract.free_text_field_valid(cls,v,values,field)
-    
+        errors:list[str] = []
+        if len(missing_objects) > 0:
+            errors.append(f"Failed to find the following '{cls.__name__}': {missing_objects}")
+        if len(missing_objects) > 0:
+            for mistyped_object in mistyped_objects:
+                errors.append(f"'{mistyped_object.name}' expected to have type '{type(Self)}', but actually had type '{type(mistyped_object)}'")
+        
+        if len(errors) > 0:
+            error_string = "\n  - ".join(errors)
+            raise ValueError(f"Found {len(errors)} issues when resolving references Security Content Object names:\n  - {error_string}")
+        
+        #Sort all objects sorted by name
+        return sorted(mappedObjects, key=lambda o: o.name)
+
+    @staticmethod
+    def getDeploymentFromType(typeField:Union[str,None], info:ValidationInfo)->Deployment:
+        if typeField is None:
+            raise ValueError("'type:' field is missing from YML.")
+        director: Optional[DirectorOutputDto] = info.context.get("output_dto",None)
+        if not director:
+            raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context")
+        
+        type_to_deployment_name_map = {AnalyticsType.TTP.value:"ESCU Default Configuration TTP", 
+                                       AnalyticsType.Hunting.value:"ESCU Default Configuration Hunting", 
+                                       AnalyticsType.Correlation.value: "ESCU Default Configuration Correlation", 
+                                       AnalyticsType.Anomaly.value: "ESCU Default Configuration Anomaly", 
+                                       "Baseline": "ESCU Default Configuration Baseline", 
+        }
+        converted_type_field = type_to_deployment_name_map[typeField]
+        
+        #TODO: This is clunky, but is imported here to resolve some circular import errors
+        from contentctl.objects.deployment import Deployment 
+
+        deployments = Deployment.mapNamesToSecurityContentObjects([converted_type_field], director)
+        if len(deployments) == 1:
+            return deployments[0]
+        elif len(deployments) == 0:
+            raise ValueError(f"Failed to find Deployment for type '{converted_type_field}' "\
+                             f"from  possible {[deployment.type for deployment in director.deployments]}")
+        else:
+            raise ValueError(f"Found more than 1 ({len(deployments)}) Deployment for type '{converted_type_field}' "\
+                             f"from  possible {[deployment.type for deployment in director.deployments]}")
+
+
+
 
     @staticmethod
     def get_objects_by_name(names_to_find:set[str], objects_to_search:list[SecurityContentObject_Abstract])->Tuple[list[SecurityContentObject_Abstract], set[str]]:
+        raise Exception("get_objects_by_name deprecated")
         found_objects = list(filter(lambda obj: obj.name in names_to_find, objects_to_search))
         found_names = set([obj.name for obj in found_objects])
         missing_names = names_to_find - found_names
@@ -74,10 +178,10 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
     @staticmethod
     def create_filename_to_content_dict(all_objects:list[SecurityContentObject_Abstract])->dict[str,SecurityContentObject_Abstract]:
         name_dict:dict[str,SecurityContentObject_Abstract] = dict()
-        
         for object in all_objects:
             name_dict[str(pathlib.Path(object.file_path))] = object
-        
         return name_dict
     
+
+    
     

contentctl/objects/alert_action.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+from pydantic import BaseModel, model_serializer
+from typing import Optional
+
+from contentctl.objects.deployment_email import DeploymentEmail
+from contentctl.objects.deployment_notable import DeploymentNotable
+from contentctl.objects.deployment_rba import DeploymentRBA
+from contentctl.objects.deployment_slack import DeploymentSlack
+from contentctl.objects.deployment_phantom import DeploymentPhantom
+
+class AlertAction(BaseModel):
+    email: Optional[DeploymentEmail] = None
+    notable: Optional[DeploymentNotable] = None
+    rba: Optional[DeploymentRBA] = DeploymentRBA()
+    slack: Optional[DeploymentSlack] = None
+    phantom: Optional[DeploymentPhantom] = None
+
+    
+    @model_serializer
+    def serialize_model(self):
+        #Call serializer for parent
+        model = {}
+
+        if self.email is not None:
+            raise Exception("Email not implemented")
+
+        if self.notable is not None:
+            model['notable'] = self.notable
+
+        if self.rba is not None and self.rba.enabled:
+            model['rba'] = {'enabled': "true"}
+
+        if self.slack is not None:
+            raise Exception("Slack not implemented")
+        
+        if self.phantom is not None:
+            raise Exception("Phantom not implemented")
+        
+        #return the model
+        return model

contentctl/objects/atomic.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+from contentctl.input.yml_reader import YmlReader
+from pydantic import BaseModel, model_validator, ConfigDict, FilePath, UUID4
+from typing import List, Optional, Dict, Union, Self
+import pathlib
+# We should determine if we want to use StrEnum, which is only present in Python3.11+
+# Alternatively, we can use
+# class SupportedPlatform(str, enum.Enum):        
+# or install the StrEnum library from pip
+
+from enum import StrEnum, auto
+
+
+class SupportedPlatform(StrEnum):        
+    windows = auto()
+    linux = auto()
+    macos = auto()
+    containers = auto()
+    # Because the following fields contain special characters 
+    # (which cannot be field names) we must specifiy them manually
+    google_workspace = "google-workspace"
+    iaas_gcp = "iaas:gcp"
+    iaas_azure = "iaas:azure"
+    iaas_aws = "iaas:aws"
+    azure_ad = "azure-ad"
+    office_365 = "office-365"
+    
+
+
+class InputArgumentType(StrEnum):
+    string = auto()
+    path = auto()
+    url = auto()
+    integer = auto()
+    float = auto()
+    # Cannot use auto() since the case sensitivity is important
+    # These should likely be converted in the ART repo to use the same case
+    # As the defined types above
+    String = "String"
+    Path = "Path"
+    Url = "Url"
+
+class AtomicExecutor(BaseModel):
+    name: str
+    elevation_required: Optional[bool] = False #Appears to be optional
+    command: Optional[str] = None
+    steps: Optional[str] = None
+    cleanup_command: Optional[str] = None
+
+    @model_validator(mode='after')
+    def ensure_mutually_exclusive_fields(self)->AtomicExecutor:
+        if self.command is not None and self.steps is not None:
+            raise ValueError("command and steps cannot both be defined in the executor section.  Exactly one must be defined.")
+        elif self.command is None and self.steps is None:
+            raise ValueError("Neither command nor steps were defined in the executor section.  Exactly one must be defined.")
+        return self
+    
+
+
+class InputArgument(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    description: str
+    type: InputArgumentType
+    default: Union[str,int,float,None] = None
+
+
+class DependencyExecutorType(StrEnum):
+    powershell = auto()
+    sh = auto()
+    bash = auto()
+    command_prompt = auto()
+
+class AtomicDependency(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    description: str
+    prereq_command: str
+    get_prereq_command: str
+
+class AtomicTest(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    name: str
+    auto_generated_guid: UUID4
+    description: str
+    supported_platforms: List[SupportedPlatform]
+    executor: AtomicExecutor
+    input_arguments: Optional[Dict[str,InputArgument]] = None
+    dependencies: Optional[List[AtomicDependency]] = None
+    dependency_executor_name: Optional[DependencyExecutorType] = None
+
+    @staticmethod
+    def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4)->Self:
+        return AtomicTest(name="Placeholder Atomic Test (enrichment disabled)",
+                          auto_generated_guid=auto_generated_guid,
+                          description="This is a placeholder AtomicTest. Because enrichments were not enabled, it has not been validated against the real Atomic Red Team Repo.",
+                          supported_platforms=[],
+                          executor=AtomicExecutor(name="Placeholder Executor (enrichment disabled)", 
+                                                  command="Placeholder command (enrichment disabled)"))
+    
+    @staticmethod
+    def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4)->Self:
+        return AtomicTest(name="Missing Atomic",
+                          auto_generated_guid=auto_generated_guid,
+                          description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile..",
+                          supported_platforms=[],
+                          executor=AtomicExecutor(name="Placeholder Executor (failed to find auto_generated_guid)", 
+                                                  command="Placeholder command (failed to find auto_generated_guid)"))
+
+
+    @classmethod
+    def getAtomicByAtomicGuid(cls, guid: UUID4, all_atomics:Union[List[AtomicTest],None])->AtomicTest:
+        if all_atomics is None:
+            return AtomicTest.AtomicTestWhenEnrichmentIsDisabled(guid)
+        matching_atomics = [atomic for atomic in all_atomics if atomic.auto_generated_guid == guid]
+        if len(matching_atomics) == 0:
+            raise ValueError(f"Unable to find atomic_guid {guid} in {len(all_atomics)} atomic_tests from ART Repo")
+        elif len(matching_atomics) > 1:
+            raise ValueError(f"Found {len(matching_atomics)} matching tests for atomic_guid {guid} in {len(all_atomics)} atomic_tests from ART Repo")
+        
+        return matching_atomics[0]
+    
+    @classmethod
+    def parseArtRepo(cls, repo_path:pathlib.Path)->List[AtomicFile]:
+        if not repo_path.is_dir():
+            print(f"WARNING: Atomic Red Team repo does NOT exist at {repo_path.absolute()}. You can check it out with:\n * git clone --single-branch https://github.com/redcanaryco/atomic-red-team. This will ONLY throw a validation error if you reference atomid_guids in your detection(s).")
+            return []
+        atomics_path = repo_path/"atomics"
+        if not atomics_path.is_dir():
+            print(f"WARNING: Atomic Red Team repo exists at {repo_path.absolute}, but atomics directory does NOT exist at {atomics_path.absolute()}. Was it deleted or renamed? This will ONLY throw a validation error if you reference atomid_guids in your detection(s).")
+            return []
+        
+
+        atomic_files:List[AtomicFile] = []
+        error_messages:List[str] = []
+        for obj_path in atomics_path.glob("**/T*.yaml"):
+            try:
+                atomic_files.append(cls.constructAtomicFile(obj_path))
+            except Exception as e:
+                error_messages.append(f"File [{obj_path}]\n{str(e)}")
+        if len(error_messages) > 0:
+            exceptions_string = '\n\n'.join(error_messages)
+            print(f"WARNING: The following [{len(error_messages)}] ERRORS were generated when parsing the Atomic Red Team Repo.\n"
+                   "Please raise an issue so that they can be fixed at https://github.com/redcanaryco/atomic-red-team/issues.\n"
+                   "Note that this is only a warning and contentctl will ignore Atomics contained in these files.\n"
+                  f"However, if you have written a detection that references them, 'contentctl build --enrichments' will fail:\n\n{exceptions_string}")
+        
+        return atomic_files
+    
+    @classmethod
+    def constructAtomicFile(cls, file_path:pathlib.Path)->AtomicFile:
+        yml_dict = YmlReader.load_file(file_path) 
+        atomic_file = AtomicFile.model_validate(yml_dict)
+        return atomic_file
+    
+    @classmethod
+    def getAtomicTestsFromArtRepo(cls, repo_path:pathlib.Path, enabled:bool=True)->Union[List[AtomicTest],None]:
+        # Get all the atomic files.  Note that if the ART repo is not found, we will not throw an error,
+        # but will not have any atomics. This means that if atomic_guids are referenced during validation,
+        # validation for those detections will fail
+        if not enabled:
+            return None
+        
+        atomic_files = cls.getAtomicFilesFromArtRepo(repo_path)
+            
+        atomic_tests:List[AtomicTest] = []
+        for atomic_file in atomic_files:
+            atomic_tests.extend(atomic_file.atomic_tests)
+        print(f"Found [{len(atomic_tests)}] Atomic Simulations in the Atomic Red Team Repo!")
+        return atomic_tests
+
+    
+    @classmethod
+    def getAtomicFilesFromArtRepo(cls, repo_path:pathlib.Path)->List[AtomicFile]:
+        return cls.parseArtRepo(repo_path)
+
+    
+    
+
+
+
+class AtomicFile(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    file_path: FilePath
+    attack_technique: str
+    display_name: str
+    atomic_tests: List[AtomicTest]
+
+
+
+
+# ATOMICS_PATH = pathlib.Path("./atomics")
+# atomic_objects = []
+# atomic_simulations = []
+# for obj_path in ATOMICS_PATH.glob("**/T*.yaml"):
+#     try:
+#         with open(obj_path, 'r', encoding="utf-8") as obj_handle:
+#             obj_data = yaml.load(obj_handle, Loader=yaml.CSafeLoader)
+#             atomic_obj = AtomicFile.model_validate(obj_data)
+#     except Exception as e:
+#         print(f"Error parsing object at path {obj_path}: {str(e)}")
+#         print(f"We have successfully parsed {len(atomic_objects)}, however!")
+#         sys.exit(1)
+
+#     print(f"Successfully parsed {obj_path}!")
+#     atomic_objects.append(atomic_obj)
+#     atomic_simulations += atomic_obj.atomic_tests
+
+# print(f"Successfully parsed all {len(atomic_objects)} files!")
+# print(f"Successfully parsed all {len(atomic_simulations)} simulations!")
+    
+
+        
+    

contentctl/objects/baseline.py

@@ -1,17 +1,21 @@
-import string
-import uuid
-import requests
 
-from pydantic import BaseModel, validator, ValidationError
-from dataclasses import dataclass
-from datetime import datetime
+from __future__ import annotations
+from typing import TYPE_CHECKING, Annotated, Optional, List,Any
+from pydantic import field_validator, ValidationInfo, Field, model_serializer
+if TYPE_CHECKING:
+    from contentctl.input.director import DirectorOutputDto
 
+from contentctl.objects.deployment import Deployment
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.enums import DataModel
+from contentctl.objects.enums import DataModel, AnalyticsType
 from contentctl.objects.baseline_tags import BaselineTags
-from contentctl.objects.deployment import Deployment
-from contentctl.helper.link_validator import LinkValidator
-from contentctl.objects.enums import SecurityContentType
+from contentctl.objects.enums import DeploymentType
+#from contentctl.objects.deployment import Deployment
+
+# from typing import TYPE_CHECKING
+# if TYPE_CHECKING:
+#     from contentctl.input.director import DirectorOutputDto
+
 
 class Baseline(SecurityContentObject):
     # baseline spec
@@ -21,43 +25,40 @@ class Baseline(SecurityContentObject):
     #date: str
     #author: str
     #contentType: SecurityContentType = SecurityContentType.baselines
-    type: str
-    datamodel: list
+    type: Annotated[str,Field(pattern="^Baseline$")] = Field(...)
+    datamodel: Optional[List[DataModel]] = None
     #description: str
-    search: str
-    how_to_implement: str
-    known_false_positives: str
+    search: str = Field(..., min_length=4)
+    how_to_implement: str = Field(..., min_length=4)
+    known_false_positives: str = Field(..., min_length=4)
     check_references: bool = False #Validation is done in order, this field must be defined first
-    references: list
-    tags: BaselineTags
+    tags: BaselineTags = Field(...)
 
     # enrichment
-    deployment: Deployment = None
-
+    deployment: Deployment = Field({})
 
+    @field_validator("deployment", mode="before")
+    def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment:
+        return Deployment.getDeployment(v,info)
+    
 
-
-    @validator('type')
-    def type_valid(cls, v, values):
-        if v != "Baseline":
-            raise ValueError('not valid analytics type: ' + values["name"])
-        return v
-
-    @validator('datamodel')
-    def datamodel_valid(cls, v, values):
-        for datamodel in v:
-            if datamodel not in [el.name for el in DataModel]:
-                raise ValueError('not valid data model: ' + values["name"])
-        return v
-
-    @validator('how_to_implement')
-    def encode_error(cls, v, values, field):
-        return SecurityContentObject.free_text_field_valid(cls,v,values,field)
+    @model_serializer
+    def serialize_model(self):
+        #Call serializer for parent
+        super_fields = super().serialize_model()
+        
+        #All fields custom to this model
+        model= {
+            "tags": self.tags.model_dump(),
+            "type": self.type,
+            "search": self.search,
+            "how_to_implement":self.how_to_implement,
+            "known_false_positives":self.known_false_positives,
+            "datamodel": self.datamodel,
+        }
+        
+        #Combine fields from this model with fields from parent
+        super_fields.update(model)
         
-    # @validator('references')
-    # def references_check(cls, v, values):
-    #     return LinkValidator.SecurityContentObject_validate_references(v, values)
-    @validator('search')
-    def search_validate(cls, v, values):
-        # write search validator
-        return v
+        #return the model
+        return super_fields