contentctl 4.2.2__py3-none-any.whl → 4.2.4__py3-none-any.whl

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -1,49 +1,48 @@
 from __future__ import annotations
-from typing import TYPE_CHECKING, Self
+from typing import TYPE_CHECKING, Self, Any
 
 if TYPE_CHECKING:
-    from contentctl.objects.deployment import Deployment        
+    from contentctl.objects.deployment import Deployment
     from contentctl.objects.security_content_object import SecurityContentObject
-    from contentctl.objects.config import Config
     from contentctl.input.director import DirectorOutputDto
-    
+
 from contentctl.objects.enums import AnalyticsType
-import re
 import abc
 import uuid
 import datetime
 import pprint
-from pydantic import BaseModel, field_validator, Field, ValidationInfo, FilePath, HttpUrl, NonNegativeInt, ConfigDict, model_validator, model_serializer
+from pydantic import (
+    BaseModel,
+    field_validator,
+    Field,
+    ValidationInfo,
+    FilePath,
+    HttpUrl,
+    NonNegativeInt,
+    ConfigDict,
+    model_serializer
+)
 from typing import Tuple, Optional, List, Union
 import pathlib
-       
-
-
 
 
 NO_FILE_NAME = "NO_FILE_NAME"
 
 
 class SecurityContentObject_Abstract(BaseModel, abc.ABC):
-    model_config = ConfigDict(use_enum_values=True,validate_default=True)
-    # name: str = ...
-    # author: str = Field(...,max_length=255)
-    # date: datetime.date = Field(...)
-    # version: NonNegativeInt = ...
-    # id: uuid.UUID = Field(default_factory=uuid.uuid4) #we set a default here until all content has a uuid
-    # description: str = Field(...,max_length=1000)
-    # file_path: FilePath = Field(...)
-    # references: Optional[List[HttpUrl]] = None
-    
-    name: str = Field("NO_NAME")
-    author: str = Field("Content Author",max_length=255)
+    model_config = ConfigDict(use_enum_values=True, validate_default=True)
+
+    name: str = Field(...)
+    author: str = Field("Content Author", max_length=255)
     date: datetime.date = Field(datetime.date.today())
     version: NonNegativeInt = 1
-    id: uuid.UUID = Field(default_factory=uuid.uuid4) #we set a default here until all content has a uuid
-    description: str = Field("Enter Description Here",max_length=10000)
+    id: uuid.UUID = Field(default_factory=uuid.uuid4)  # we set a default here until all content has a uuid
+    description: str = Field("Enter Description Here", max_length=10000)
     file_path: Optional[FilePath] = None
     references: Optional[List[HttpUrl]] = None
 
+    def model_post_init(self, __context: Any) -> None:
+        self.ensureFileNameMatchesSearchName()
 
     @model_serializer
     def serialize_model(self):
@@ -58,33 +57,32 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
         }
 
     @staticmethod
-    def objectListToNameList(objects:list[SecurityContentObject], config:Optional[Config]=None)->list[str]:
-        return [object.getName(config) for object in objects]
+    def objectListToNameList(objects: list[SecurityContentObject]) -> list[str]:
+        return [object.getName() for object in objects]
 
     # This function is overloadable by specific types if they want to redefine names, for example
     # to have the format ESCU - NAME - Rule (config.tag - self.name - Rule)
-    def getName(self, config:Optional[Config])->str:
+    def getName(self) -> str:
         return self.name
 
-
     @classmethod
-    def contentNameToFileName(cls, content_name:str)->str:
+    def contentNameToFileName(cls, content_name: str) -> str:
         return content_name \
             .replace(' ', '_') \
-            .replace('-','_') \
-            .replace('.','_') \
-            .replace('/','_') \
+            .replace('-', '_') \
+            .replace('.', '_') \
+            .replace('/', '_') \
             .lower() + ".yml"
 
-
-    @model_validator(mode="after")
     def ensureFileNameMatchesSearchName(self):
         file_name = self.contentNameToFileName(self.name)
-        
+
         if (self.file_path is not None and file_name != self.file_path.name):
-            raise ValueError(f"The file name MUST be based off the content 'name' field:\n"\
-                            f"\t- Expected File Name: {file_name}\n"\
-                            f"\t- Actual File Name  : {self.file_path.name}")
+            raise ValueError(
+                f"The file name MUST be based off the content 'name' field:\n"
+                f"\t- Expected File Name: {file_name}\n"
+                f"\t- Actual File Name  : {self.file_path.name}"
+            )
 
         return self
 
@@ -92,99 +90,120 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
     @classmethod
     def file_path_valid(cls, v: Optional[pathlib.PosixPath], info: ValidationInfo):
         if not v:
-            #It's possible that the object has no file path - for example filter macros that are created at runtime
+            # It's possible that the object has no file path - for example filter macros that are created at runtime
             return v
         if not v.name.endswith(".yml"):
-            raise ValueError(f"All Security Content Objects must be YML files and end in .yml.  The following file does not: '{v}'")
+            raise ValueError(
+                "All Security Content Objects must be YML files and end in .yml.  The following"
+                f" file does not: '{v}'"
+            )
         return v
 
-    def getReferencesListForJson(self)->List[str]:
+    def getReferencesListForJson(self) -> List[str]:
         return [str(url) for url in self.references or []]
-        
+
     @classmethod
-    def mapNamesToSecurityContentObjects(cls, v: list[str], director:Union[DirectorOutputDto,None])->list[Self]:
+    def mapNamesToSecurityContentObjects(cls, v: list[str], director: Union[DirectorOutputDto, None]) -> list[Self]:
         if director is not None:
             name_map = director.name_to_content_map
         else:
             name_map = {}
-        
-
 
         mappedObjects: list[Self] = []
         mistyped_objects: list[SecurityContentObject_Abstract] = []
         missing_objects: list[str] = []
         for object_name in v:
-            found_object = name_map.get(object_name,None)
+            found_object = name_map.get(object_name, None)
             if not found_object:
                 missing_objects.append(object_name)
-            elif not isinstance(found_object,cls):
+            elif not isinstance(found_object, cls):
                 mistyped_objects.append(found_object)
             else:
                 mappedObjects.append(found_object)
-        
-        errors:list[str] = []
+
+        errors: list[str] = []
         if len(missing_objects) > 0:
             errors.append(f"Failed to find the following '{cls.__name__}': {missing_objects}")
         if len(mistyped_objects) > 0:
             for mistyped_object in mistyped_objects:
-                errors.append(f"'{mistyped_object.name}' expected to have type '{cls}', but actually had type '{type(mistyped_object)}'")
-        
+                errors.append(
+                    f"'{mistyped_object.name}' expected to have type '{cls}', but actually "
+                    f"had type '{type(mistyped_object)}'"
+                )
+
         if len(errors) > 0:
             error_string = "\n  - ".join(errors)
-            raise ValueError(f"Found {len(errors)} issues when resolving references Security Content Object names:\n  - {error_string}")
-        
-        #Sort all objects sorted by name
+            raise ValueError(
+                f"Found {len(errors)} issues when resolving references Security Content Object "
+                f"names:\n  - {error_string}")
+
+        # Sort all objects sorted by name
         return sorted(mappedObjects, key=lambda o: o.name)
 
     @staticmethod
-    def getDeploymentFromType(typeField:Union[str,None], info:ValidationInfo)->Deployment:
+    def getDeploymentFromType(typeField: Union[str, None], info: ValidationInfo) -> Deployment:
         if typeField is None:
             raise ValueError("'type:' field is missing from YML.")
-        director: Optional[DirectorOutputDto] = info.context.get("output_dto",None)
+
+        if info.context is None:
+            raise ValueError("ValidationInfo.context unexpectedly null")
+
+        director: Optional[DirectorOutputDto] = info.context.get("output_dto", None)
         if not director:
             raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context")
-        
-        type_to_deployment_name_map = {AnalyticsType.TTP.value:"ESCU Default Configuration TTP", 
-                                       AnalyticsType.Hunting.value:"ESCU Default Configuration Hunting", 
-                                       AnalyticsType.Correlation.value: "ESCU Default Configuration Correlation", 
-                                       AnalyticsType.Anomaly.value: "ESCU Default Configuration Anomaly", 
-                                       "Baseline": "ESCU Default Configuration Baseline", 
+
+        type_to_deployment_name_map = {
+            AnalyticsType.TTP.value: "ESCU Default Configuration TTP",
+            AnalyticsType.Hunting.value: "ESCU Default Configuration Hunting",
+            AnalyticsType.Correlation.value: "ESCU Default Configuration Correlation",
+            AnalyticsType.Anomaly.value: "ESCU Default Configuration Anomaly",
+            "Baseline": "ESCU Default Configuration Baseline"
         }
         converted_type_field = type_to_deployment_name_map[typeField]
-        
-        #TODO: This is clunky, but is imported here to resolve some circular import errors
-        from contentctl.objects.deployment import Deployment 
+
+        # TODO: This is clunky, but is imported here to resolve some circular import errors
+        from contentctl.objects.deployment import Deployment
 
         deployments = Deployment.mapNamesToSecurityContentObjects([converted_type_field], director)
         if len(deployments) == 1:
             return deployments[0]
         elif len(deployments) == 0:
-            raise ValueError(f"Failed to find Deployment for type '{converted_type_field}' "\
-                             f"from  possible {[deployment.type for deployment in director.deployments]}")
+            raise ValueError(
+                f"Failed to find Deployment for type '{converted_type_field}' "
+                f"from  possible {[deployment.type for deployment in director.deployments]}"
+            )
         else:
-            raise ValueError(f"Found more than 1 ({len(deployments)}) Deployment for type '{converted_type_field}' "\
-                             f"from  possible {[deployment.type for deployment in director.deployments]}")
-
-
-
+            raise ValueError(
+                f"Found more than 1 ({len(deployments)}) Deployment for type '{converted_type_field}' "
+                f"from  possible {[deployment.type for deployment in director.deployments]}"
+            )
 
     @staticmethod
-    def get_objects_by_name(names_to_find:set[str], objects_to_search:list[SecurityContentObject_Abstract])->Tuple[list[SecurityContentObject_Abstract], set[str]]:
+    def get_objects_by_name(
+        names_to_find: set[str],
+        objects_to_search: list[SecurityContentObject_Abstract]
+    ) -> Tuple[list[SecurityContentObject_Abstract], set[str]]:
         raise Exception("get_objects_by_name deprecated")
         found_objects = list(filter(lambda obj: obj.name in names_to_find, objects_to_search))
         found_names = set([obj.name for obj in found_objects])
         missing_names = names_to_find - found_names
-        return found_objects,missing_names
-    
+        return found_objects, missing_names
+
     @staticmethod
-    def create_filename_to_content_dict(all_objects:list[SecurityContentObject_Abstract])->dict[str,SecurityContentObject_Abstract]:
-        name_dict:dict[str,SecurityContentObject_Abstract] = dict()
+    def create_filename_to_content_dict(
+        all_objects: list[SecurityContentObject_Abstract]
+    ) -> dict[str, SecurityContentObject_Abstract]:
+        name_dict: dict[str, SecurityContentObject_Abstract] = dict()
         for object in all_objects:
+            # If file_path is None, this function has been called on an inappropriate
+            # SecurityContentObject (e.g. filter macros that are created at runtime but have no
+            # actual file associated)
+            if object.file_path is None:
+                raise ValueError(f"SecurityContentObject is missing a file_path: {object.name}")
             name_dict[str(pathlib.Path(object.file_path))] = object
         return name_dict
 
-
-    def __repr__(self)->str:
+    def __repr__(self) -> str:
         # Just use the model_dump functionality that
         # has already been written. This loses some of the
         # richness where objects reference themselves, but
@@ -192,40 +211,35 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
         m = self.model_dump()
         return pprint.pformat(m, indent=3)
 
-    def __str__(self)->str:
-        return(self.__repr__())
+    def __str__(self) -> str:
+        return self.__repr__()
 
-    def __lt__(self, other:object)->bool:
-        if not isinstance(other,SecurityContentObject_Abstract):
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, SecurityContentObject_Abstract):
             raise Exception(f"SecurityContentObject can only be compared to each other, not to {type(other)}")
-        return self.name < other.name 
+        return self.name < other.name
 
-    def __eq__(self, other:object)->bool: 
-        if not isinstance(other,SecurityContentObject_Abstract):
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, SecurityContentObject_Abstract):
             raise Exception(f"SecurityContentObject can only be compared to each other, not to {type(other)}")
-        
+
         if id(self) == id(other) and self.name == other.name and self.id == other.id:
             # Yes, this is the same object
             return True
-        
+
         elif id(self) == id(other) or self.name == other.name or self.id == other.id:
-            raise Exception("Attempted to compare two SecurityContentObjects, but their fields indicate they were not globally unique:"
-                            f"\n\tid(obj1)  : {id(self)}"
-                            f"\n\tid(obj2)  : {id(other)}"
-                            f"\n\tobj1.name : {self.name}"
-                            f"\n\tobj2.name : {other.name}"
-                            f"\n\tobj1.id   : {self.id}"
-                            f"\n\tobj2.id   : {other.id}")
+            raise Exception(
+                "Attempted to compare two SecurityContentObjects, but their fields indicate they "
+                "were not globally unique:"
+                f"\n\tid(obj1)  : {id(self)}"
+                f"\n\tid(obj2)  : {id(other)}"
+                f"\n\tobj1.name : {self.name}"
+                f"\n\tobj2.name : {other.name}"
+                f"\n\tobj1.id   : {self.id}"
+                f"\n\tobj2.id   : {other.id}"
+            )
         else:
             return False
-    
+
     def __hash__(self) -> NonNegativeInt:
         return id(self)
-        
-
-    
-
-    
-
-    
-    

contentctl/objects/atomic.py

@@ -3,10 +3,7 @@ from contentctl.input.yml_reader import YmlReader
 from pydantic import BaseModel, model_validator, ConfigDict, FilePath, UUID4
 from typing import List, Optional, Dict, Union, Self
 import pathlib
-# We should determine if we want to use StrEnum, which is only present in Python3.11+
-# Alternatively, we can use
-# class SupportedPlatform(str, enum.Enum):        
-# or install the StrEnum library from pip
+
 
 from enum import StrEnum, auto
 
@@ -48,7 +45,7 @@ class AtomicExecutor(BaseModel):
     cleanup_command: Optional[str] = None
 
     @model_validator(mode='after')
-    def ensure_mutually_exclusive_fields(self)->AtomicExecutor:
+    def ensure_mutually_exclusive_fields(self)->Self:
         if self.command is not None and self.steps is not None:
             raise ValueError("command and steps cannot both be defined in the executor section.  Exactly one must be defined.")
         elif self.command is None and self.steps is None:
@@ -88,7 +85,7 @@ class AtomicTest(BaseModel):
     dependency_executor_name: Optional[DependencyExecutorType] = None
 
     @staticmethod
-    def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4)->Self:
+    def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4) -> AtomicTest:
         return AtomicTest(name="Placeholder Atomic Test (enrichment disabled)",
                           auto_generated_guid=auto_generated_guid,
                           description="This is a placeholder AtomicTest. Because enrichments were not enabled, it has not been validated against the real Atomic Red Team Repo.",
@@ -97,17 +94,17 @@ class AtomicTest(BaseModel):
                                                   command="Placeholder command (enrichment disabled)"))
     
     @staticmethod
-    def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4)->Self:
+    def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4) -> AtomicTest:
         return AtomicTest(name="Missing Atomic",
                           auto_generated_guid=auto_generated_guid,
-                          description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile..",
+                          description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile.",
                           supported_platforms=[],
                           executor=AtomicExecutor(name="Placeholder Executor (failed to find auto_generated_guid)", 
                                                   command="Placeholder command (failed to find auto_generated_guid)"))
 
 
     @classmethod
-    def getAtomicByAtomicGuid(cls, guid: UUID4, all_atomics:Union[List[AtomicTest],None])->AtomicTest:
+    def getAtomicByAtomicGuid(cls, guid: UUID4, all_atomics:list[AtomicTest] | None)->AtomicTest:
         if all_atomics is None:
             return AtomicTest.AtomicTestWhenEnrichmentIsDisabled(guid)
         matching_atomics = [atomic for atomic in all_atomics if atomic.auto_generated_guid == guid]
@@ -152,7 +149,7 @@ class AtomicTest(BaseModel):
         return atomic_file
     
     @classmethod
-    def getAtomicTestsFromArtRepo(cls, repo_path:pathlib.Path, enabled:bool=True)->Union[List[AtomicTest],None]:
+    def getAtomicTestsFromArtRepo(cls, repo_path:pathlib.Path, enabled:bool=True)->list[AtomicTest] | None:
         # Get all the atomic files.  Note that if the ART repo is not found, we will not throw an error,
         # but will not have any atomics. This means that if atomic_guids are referenced during validation,
         # validation for those detections will fail

contentctl/objects/base_test.py

@@ -18,7 +18,7 @@ class TestType(str, Enum):
         return self.value
 
 
-# TODO (cmcginley): enforce distinct test names w/in detections
+# TODO (#224): enforce distinct test names w/in detections
 class BaseTest(BaseModel, ABC):
     """
     A test case for a detection

contentctl/objects/base_test_result.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Union, Any
 from enum import Enum
 
 from pydantic import BaseModel
@@ -7,6 +7,8 @@ from splunklib.data import Record
 from contentctl.helper.utils import Utils
 
 
+# TODO (PEX-432): add status "UNSET" so that we can make sure the result is always of this enum
+#   type; remove mypy ignores associated w/ these typing issues once we do
 class TestResultStatus(str, Enum):
     """Enum for test status (e.g. pass/fail)"""
     # Test failed (detection did NOT fire appropriately)
@@ -26,7 +28,7 @@ class TestResultStatus(str, Enum):
         return self.value
 
 
-# TODO (cmcginley): add validator to BaseTestResult which makes a lack of exception incompatible
+# TODO (#225): add validator to BaseTestResult which makes a lack of exception incompatible
 #   with status ERROR
 class BaseTestResult(BaseModel):
     """
@@ -94,7 +96,7 @@ class BaseTestResult(BaseModel):
             "success", "exception", "message", "sid_link", "status", "duration", "wait_duration"
         ],
         job_fields: list[str] = ["search", "resultCount", "runDuration"],
-    ) -> dict:
+    ) -> dict[str, Any]:
         """
         Aggregates a dictionary summarizing the test result model
         :param model_fields: the fields of the test result to gather
@@ -102,7 +104,7 @@ class BaseTestResult(BaseModel):
         :returns: a dict summary
         """
         # Init the summary dict
-        summary_dict = {}
+        summary_dict: dict[str, Any] = {}
 
         # Grab the fields required
         for field in model_fields:
@@ -122,7 +124,7 @@ class BaseTestResult(BaseModel):
         # Grab the job content fields required
         for field in job_fields:
             if self.job_content is not None:
-                value = self.job_content.get(field, None)
+                value: Any = self.job_content.get(field, None)                                      # type: ignore
 
                 # convert runDuration to a fixed width string representation of a float
                 if field == "runDuration":

contentctl/objects/baseline_tags.py

@@ -17,6 +17,7 @@ if TYPE_CHECKING:
 class BaselineTags(BaseModel):
     analytic_story: list[Story] = Field(...)
     #deployment: Deployment = Field('SET_IN_GET_DEPLOYMENT_FUNCTION')
+    # TODO (#223): can we remove str from the possible types here?
     detections: List[Union[Detection,str]] = Field(...)
     product: list[SecurityContentProductName] = Field(...,min_length=1)
     required_fields: List[str] = Field(...,min_length=1)
@@ -42,33 +43,4 @@ class BaselineTags(BaseModel):
         
         
         #return the model
-        return model
-    
-    def replaceDetectionNameWithDetectionObject(self, detection:Detection)->bool:
-        
-        pass
-
-    
-
-
-    # @field_validator("deployment", mode="before")
-    # def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment:         
-    #     if v != 'SET_IN_GET_DEPLOYMENT_FUNCTION':
-    #         print(f"Deployment defined in YML: {v}")
-    #         return v
-        
-    #     director: Optional[DirectorOutputDto] = info.context.get("output_dto",None)
-    #     if not director:
-    #         raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context")
-        
-    #     typeField = "Baseline"
-    #     deps = [deployment for deployment in director.deployments if deployment.type == typeField]
-    #     if len(deps) == 1:
-    #         return deps[0]
-    #     elif len(deps) == 0:
-    #         raise ValueError(f"Failed to find Deployment for type '{typeField}' "\
-    #                         f"from  possible {[deployment.type for deployment in director.deployments]}")
-    #     else:
-    #         raise ValueError(f"Found more than 1 ({len(deps)}) Deployment for type '{typeField}' "\
-    #                         f"from  possible {[deployment.type for deployment in director.deployments]}")
-        
+        return model

contentctl/objects/config.py

@@ -176,6 +176,7 @@ class validate(Config_Base):
     build_app: bool = Field(default=True, description="Should an app be built and output in the build_path?")
     build_api: bool = Field(default=False, description="Should api objects be built and output in the build_path?")
     build_ssa: bool = Field(default=False, description="Should ssa objects be built and output in the build_path?")
+    data_source_TA_validation: bool = Field(default=False, description="Validate latest TA information from Splunkbase")
 
     def getAtomicRedTeamRepoPath(self, atomic_red_team_repo_name:str = "atomic-red-team"):
         return self.path/atomic_red_team_repo_name
@@ -601,13 +602,13 @@ class test_common(build):
 
 
     def getLocalAppDir(self)->pathlib.Path:
-        #docker really wants abolsute paths
+        # docker really wants absolute paths
         path = self.path / "apps"
         return path.absolute()
     
     def getContainerAppDir(self)->pathlib.Path:
-        #docker really wants abolsute paths
-        return pathlib.Path("/tmp/apps").absolute()
+        # docker really wants absolute paths
+        return pathlib.Path("/tmp/apps")
 
     def enterpriseSecurityInApps(self)->bool:
         
@@ -739,7 +740,7 @@ class test(test_common):
             if path.startswith(SPLUNKBASE_URL):
                 container_paths.append(path)
             else:
-                container_paths.append(str(self.getContainerAppDir()/pathlib.Path(path).name))
+                container_paths.append((self.getContainerAppDir()/pathlib.Path(path).name).as_posix())
         
         return ','.join(container_paths)