contentctl 3.6.0__py3-none-any.whl → 4.0.2__py3-none-any.whl

contentctl/objects/lookup.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
-
-from pydantic import BaseModel, validator, ValidationError
-from typing import Tuple
+from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer
+from typing import TYPE_CHECKING, Optional, Any, Union
 import re
+if TYPE_CHECKING:
+    from contentctl.input.director import DirectorOutputDto
+    from contentctl.objects.config import validate
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.enums import SecurityContentType
+
 
 LOOKUPS_TO_IGNORE = set(["outputlookup"])
 LOOKUPS_TO_IGNORE.add("ut_shannon_lookup") #In the URL toolbox app which is recommended for ESCU
@@ -18,39 +20,86 @@ LOOKUPS_TO_IGNORE.add("=")
 LOOKUPS_TO_IGNORE.add("other_lookups") 
 
 
-class Lookup(BaseModel):
-    #contentType: SecurityContentType = SecurityContentType.lookups
-    name: str
-    description: str
-    collection: str = None
-    fields_list: str = None
-    filename: str = None
-    default_match: str = None
-    match_type: str = None
-    min_matches: int = None
-    case_sensitive_match: str = None
-    file_path:str = None
-
-     # Macro can have different punctuatuation in it,
-    # so redefine the name validator. For now, jsut
-    # allow any characters in the macro
-    @validator('name',check_fields=False)
-    def name_invalid_chars(cls, v):
-        return v
+class Lookup(SecurityContentObject):
+    
+    collection: Optional[str] = None
+    fields_list: Optional[str] = None
+    filename: Optional[FilePath] = None
+    default_match: Optional[bool] = None
+    match_type: Optional[str] = None
+    min_matches: Optional[int] = None
+    case_sensitive_match: Optional[bool] = None
+    
+
+    @model_serializer
+    def serialize_model(self):
+        #Call parent serializer
+        super_fields = super().serialize_model()
+
+        #All fields custom to this model
+        model= {
+            "filename": self.filename.name if self.filename is not None else None,
+            "default_match": "true" if self.default_match is True else "false",
+            "match_type": self.match_type,
+            "min_matches": self.min_matches,
+            "case_sensitive_match": "true" if self.case_sensitive_match is True else "false",
+            "collection": self.collection,
+            "fields_list": self.fields_list
+        }
+        
+        #return the model
+        model.update(super_fields)
+        return model
+
+    @model_validator(mode="before")
+    def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any:
+        if data.get("filename"):
+            config:validate = info.context.get("config",None)
+            if config is not None:
+                data["filename"] = config.path / "lookups/" / data["filename"]
+            else:
+                raise ValueError("config required for constructing lookup filename, but it was not")
+        return data
 
+    @field_validator('filename')
+    @classmethod
+    def lookup_file_valid(cls, v: Union[FilePath,None], info: ValidationInfo):
+        if not v:
+            return v
+        if not (v.name.endswith(".csv") or v.name.endswith(".mlmodel")):
+            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{v}'")
 
-    # Allow long names for lookups
-    @validator('name',check_fields=False)
-    def name_max_length(cls, v):
-        #if len(v) > 67:
-        #    raise ValueError('name is longer then 67 chars: ' + v)
         return v
+        
+    @field_validator('match_type')
+    @classmethod
+    def match_type_valid(cls, v: Union[str,None], info: ValidationInfo):
+        if not v:
+            #Match type can be None and that's okay
+            return v
+
+        if not (v.startswith("WILDCARD(") or v.endswith(")")) :
+            raise ValueError(f"All match_types must take the format 'WILDCARD(field_name)'. The following file does not: '{v}'")
+        return v
+
+
+    #Ensure that exactly one of location or filename are defined
+    @model_validator(mode='after')
+    def ensure_mutually_exclusive_fields(self)->Lookup:
+        if self.filename is not None and self.collection is not None:
+            raise ValueError("filename and collection cannot be defined in the lookup file.  Exactly one must be defined.")
+        elif self.filename is None and self.collection is None:
+            raise ValueError("Neither filename nor collection were defined in the lookup file.  Exactly one must "
+                             "be defined.")
+
+
+        return self
+    
     
     @staticmethod
-    def get_lookups(text_field: str, all_lookups: list[Lookup], ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->Tuple[list[Lookup], set[str]]:
+    def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]:
         lookups_to_get = set(re.findall(r'[^output]lookup (?:update=true)?(?:append=t)?\s*([^\s]*)', text_field))
         lookups_to_ignore = set([lookup for lookup in lookups_to_get if any(to_ignore in lookups_to_get for to_ignore in ignore_lookups)])
         lookups_to_get -= lookups_to_ignore
-        found_lookups, missing_lookups = SecurityContentObject.get_objects_by_name(lookups_to_get, all_lookups)
-        return found_lookups, missing_lookups
+        return Lookup.mapNamesToSecurityContentObjects(list(lookups_to_get), director)
     

contentctl/objects/macro.py

@@ -1,12 +1,13 @@
 # Used so that we can have a staticmethod that takes the class 
 # type Macro as an argument
 from __future__ import annotations
+from typing import TYPE_CHECKING, List
 import re
-from pydantic import BaseModel, validator, ValidationError
-
+from pydantic import Field, model_serializer
+if TYPE_CHECKING:
+    from contentctl.input.director import DirectorOutputDto
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.enums import SecurityContentType
-from typing import Tuple
+
 
 
 MACROS_TO_IGNORE = set(["_filter", "drop_dm_object_name"])
@@ -17,32 +18,32 @@ MACROS_TO_IGNORE.add("cim_corporate_web_domain_search")
 MACROS_TO_IGNORE.add("prohibited_processes")
 
 
-class Macro(BaseModel):
-    #contentType: SecurityContentType = SecurityContentType.macros
-    name: str
-    definition: str
-    description: str
-    arguments: list = None
-    file_path: str = None
-
-    # Macro can have different punctuatuation in it,
-    # so redefine the name validator. For now, jsut
-    # allow any characters in the macro
-    @validator('name',check_fields=False)
-    def name_invalid_chars(cls, v):
-        return v
+class Macro(SecurityContentObject):
+    definition: str = Field(..., min_length=1)
+    arguments: List[str] = Field([])
+    
+    
 
 
-    # Allow long names for macros
-    @validator('name',check_fields=False)
-    def name_max_length(cls, v):
-        #if len(v) > 67:
-        #    raise ValueError('name is longer then 67 chars: ' + v)
-        return v
+    @model_serializer
+    def serialize_model(self):
+        #Call serializer for parent
+        super_fields = super().serialize_model()
 
+        #All fields custom to this model
+        model= {
+            "definition": self.definition,
+            "description": self.description,
+        }
+        
+        #return the model
+        model.update(super_fields)
+        
+        return model
     
     @staticmethod
-    def get_macros(text_field:str, all_macros: list[Macro], ignore_macros:set[str]=MACROS_TO_IGNORE)->Tuple[list[Macro], set[str]]:
+
+    def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[str]=MACROS_TO_IGNORE)->list[Macro]:
         #Remove any comments, allowing there to be macros (which have a single backtick) inside those comments
         #If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here````
         #then there is a small edge case where the regex below does not work properly.  If that is 
@@ -50,6 +51,7 @@ class Macro(BaseModel):
         text_field = re.sub(r"\`\`\`\`", r"` ```", text_field)
         text_field = re.sub(r"\`\`\`.*?\`\`\`", " ", text_field)
         
+
         macros_to_get = re.findall(r'`([^\s]+)`', text_field)
         #If macros take arguments, stop at the first argument.  We just want the name of the macro
         macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get])
@@ -57,24 +59,6 @@ class Macro(BaseModel):
         macros_to_ignore = set([macro for macro in macros_to_get if any(to_ignore in macro for to_ignore in ignore_macros)])
         #remove the ones that we will ignore
         macros_to_get -= macros_to_ignore
-        found_macros, missing_macros = SecurityContentObject.get_objects_by_name(macros_to_get, all_macros)
-        return found_macros, missing_macros
-
-        # found_macros = [macro for macro in all_macros if macro.name in macros_to_get]
-        
-        # missing_macros = macros_to_get - set([macro.name for macro in found_macros])
-        # missing_macros_after_ignored_macros = set()
-        # for macro in missing_macros:
-        #     found = False
-        #     for ignore in ignore_macros:
-        #         if ignore in macro:
-        #             found=True
-        #             break
-        #     if found is False:
-        #         missing_macros_after_ignored_macros.add(macro)
-
-        #return found_macros, missing_macros_after_ignored_macros
+        return Macro.mapNamesToSecurityContentObjects(list(macros_to_get), director)
         
-
-
-
+    

contentctl/objects/mitre_attack_enrichment.py

@@ -1,8 +1,32 @@
-from pydantic import BaseModel, validator, ValidationError
+from __future__ import annotations
+from pydantic import BaseModel, Field, ConfigDict
+from typing import Set,List,Annotated
+from enum import StrEnum
+
+
+class MitreTactics(StrEnum):
+    RECONNAISSANCE = "Reconnaissance"
+    RESOURCE_DEVELOPMENT = "Resource Development"
+    INITIAL_ACCESS = "Initial Access"
+    EXECUTION = "Execution"
+    PERSISTENCE = "Persistence"
+    PRIVILEGE_ESCALATION = "Privilege Escalation"
+    DEFENSE_EVASION = "Defense Evasion"
+    CREDENTIAL_ACCESS = "Credential Access"
+    DISCOVERY = "Discovery"
+    LATERAL_MOVEMENT = "Lateral Movement"
+    COLLECTION = "Collection"
+    COMMAND_AND_CONTROL = "Command And Control"
+    EXFILTRATION = "Exfiltration"
+    IMPACT = "Impact"
 
 
 class MitreAttackEnrichment(BaseModel):
-    mitre_attack_id: str
-    mitre_attack_technique: str
-    mitre_attack_tactics: list
-    mitre_attack_groups: list
+    ConfigDict(use_enum_values=True)
+    mitre_attack_id: Annotated[str, Field(pattern="^T\d{4}(.\d{3})?$")] = Field(...)
+    mitre_attack_technique: str = Field(...)
+    mitre_attack_tactics: List[MitreTactics] = Field(...)
+    mitre_attack_groups: List[str] = Field(...)
+
+    def __hash__(self) -> int:
+        return id(self)

contentctl/objects/observable.py

@@ -1,10 +1,6 @@
-import abc
-import string
-import uuid
-from typing import Literal
-from datetime import datetime
-from pydantic import BaseModel, validator, ValidationError
-from contentctl.objects.enums import SecurityContentType
+from __future__ import annotations
+from pydantic import BaseModel, validator
+
 from contentctl.objects.constants import *
 
 

contentctl/objects/playbook.py

@@ -1,36 +1,66 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING,Self
+from pydantic import model_validator, Field, FilePath
 
-import uuid
-import string
 
-from pydantic import BaseModel, validator, ValidationError
-
-from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.playbook_tags import PlaybookTag
-from contentctl.helper.link_validator import LinkValidator
-from contentctl.objects.enums import SecurityContentType
+from contentctl.objects.security_content_object import SecurityContentObject
+from contentctl.objects.enums import PlaybookType
 
 
 class Playbook(SecurityContentObject):
-    #name: str
-    #id: str
-    #version: int
-    #date: str
-    #author: str
-    #contentType: SecurityContentType = SecurityContentType.playbooks
-    type: str
-    #description: str
-    how_to_implement: str
-    playbook: str
-    check_references: bool = False #Validation is done in order, this field must be defined first
-    references: list
-    app_list: list
-    tags: PlaybookTag
-
-
-    @validator('references')
-    def references_check(cls, v, values):
-        return LinkValidator.SecurityContentObject_validate_references(v, values)
-
-    @validator('how_to_implement')
-    def encode_error(cls, v, values, field):
-        return SecurityContentObject.free_text_field_valid(cls,v,values,field)
+    type: PlaybookType = Field(...)
+    
+    # Override the type definition for filePath.
+    # This MUST be backed by a file and cannot be None
+    file_path: FilePath
+    
+    how_to_implement: str = Field(min_length=4)
+    playbook: str = Field(min_length=4)
+    app_list: list[str] = Field(...,min_length=0) 
+    tags: PlaybookTag = Field(...)
+    
+
+    
+    @model_validator(mode="after")
+    def ensureJsonAndPyFilesExist(self)->Self:
+        json_file_path = self.file_path.with_suffix(".json")
+        python_file_path = self.file_path.with_suffix(".py")
+        missing:list[str] = []
+        if not json_file_path.is_file():
+            missing.append(f"Playbook file named '{self.file_path.name}' MUST "\
+                           f"have a .json file named '{json_file_path.name}', "\
+                            "but it does not exist")
+            
+        if not python_file_path.is_file():
+            missing.append(f"Playbook file named '{self.file_path.name}' MUST "\
+                           f"have a .py file named '{python_file_path.name}', "\
+                            "but it does not exist")
+            
+        
+        if len(missing) == 0:
+            return self
+        else:
+            missing_files_string = '\n - '.join(missing)
+            raise ValueError(f"Playbook files missing:\n -{missing_files_string}")
+
+
+    #Override playbook file name checking FOR NOW
+    @model_validator(mode="after")
+    def ensureFileNameMatchesSearchName(self)->Self:
+        file_name = self.name \
+            .replace(' ', '_') \
+            .replace('-','_') \
+            .replace('.','_') \
+            .replace('/','_') \
+            .lower() + ".yml"
+        
+        #allow different capitalization FOR NOW in playbook file names
+        if (self.file_path is not None and file_name != self.file_path.name.lower()):
+            raise ValueError(f"The file name MUST be based off the content 'name' field:\n"\
+                            f"\t- Expected File Name: {file_name}\n"\
+                            f"\t- Actual File Name  : {self.file_path.name}")
+
+        return self
+
+    

contentctl/objects/playbook_tags.py

@@ -1,13 +1,50 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Optional, List
+from pydantic import BaseModel, Field
+import enum
+from contentctl.objects.detection import Detection
 
-from pydantic import BaseModel, validator, ValidationError
 
+class PlaybookProduct(str,enum.Enum):
+    SPLUNK_SOAR = "Splunk SOAR"
 
+class PlaybookUseCase(str,enum.Enum):
+    PHISHING = "Phishing"
+    ENDPOINT = "Endpoint"
+    ENRICHMENT = "Enrichment"
+    
+class PlaybookType(str,enum.Enum):
+    INPUT = "Input"
+    AUTOMATION = "Automation"
+
+class VpeType(str,enum.Enum):
+    MODERN = "Modern"
+    CLASSIC = "Classic"
+class DefendTechnique(str,enum.Enum):
+    D3_AL = "D3-AL"
+    D3_DNSDL = "D3-DNSDL"
+    D3_DA = "D3-DA"
+    D3_IAA = "D3-IAA"
+    D3_IRA = "D3-IRA"
+    D3_OTF = "D3-OTF"
+    D3_ER = "D3-ER"
+    D3_RE = "D3-RE"
+    D3_URA = "D3-URA"
+    D3_DNRA = "D3-DNRA"
+    D3_IPRA = "D3-IPRA"
+    D3_FHRA = "D3-FHRA"
+    D3_SRA = "D3-SRA"
+    D3_RUAA = "D3-RUAA"
 class PlaybookTag(BaseModel):
-    analytic_story: list = None
-    detections: list = None
-    platform_tags: list = None
-    playbook_fields: list = None
-    product: list = None
-    playbook_fields: list = None
-    detection_objects: list = None  
+    analytic_story: Optional[list] = None
+    detections: Optional[list] = None
+    platform_tags: list[str] = Field(...,min_length=0)
+    playbook_type: PlaybookType = Field(...)
+    vpe_type: VpeType = Field(...)
+    playbook_fields: list[str] = Field([], min_length=0)
+    product: list[PlaybookProduct] = Field([],min_length=0)
+    use_cases: list[PlaybookUseCase] = Field([],min_length=0)
+    defend_technique_id: Optional[List[DefendTechnique]] = None
+    
+    detection_objects: list[Detection] = []
     

contentctl/objects/security_content_object.py

@@ -1,8 +1,4 @@
-import abc
-import string
-import uuid
-from datetime import datetime
-from pydantic import BaseModel, validator, ValidationError
+from __future__ import annotations
 from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract
 
 class SecurityContentObject(SecurityContentObject_Abstract):

contentctl/objects/ssa_detection.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import uuid
 import string
 import requests
@@ -14,8 +15,8 @@ from contentctl.objects.enums import DataModel
 from contentctl.objects.enums import DetectionStatus
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.ssa_detection_tags import SSADetectionTags
-from contentctl.objects.config import ConfigDetectionConfiguration
-from contentctl.objects.unit_test import UnitTest
+from contentctl.objects.unit_test_ssa import UnitTestSSA
+from contentctl.objects.unit_test_old import UnitTestOld
 from contentctl.objects.macro import Macro
 from contentctl.objects.lookup import Lookup
 from contentctl.objects.baseline import Baseline
@@ -40,7 +41,7 @@ class SSADetection(BaseModel):
     known_false_positives: str
     references: list
     tags: SSADetectionTags
-    tests: list[UnitTest] = None
+    tests: list[UnitTestSSA] = None
 
     # enrichments
     annotations: dict = None
@@ -48,7 +49,7 @@ class SSADetection(BaseModel):
     mappings: dict = None
     file_path: str = None
     source: str = None
-    test: Union[UnitTest, dict] = None
+    test: Union[UnitTestSSA, dict, UnitTestOld] = None
     runtime: str = None
     internalVersion: int = None
 
@@ -61,6 +62,7 @@ class SSADetection(BaseModel):
     class Config:
         use_enum_values = True
 
+    '''
     @validator("name")
     def name_invalid_chars(cls, v):
         invalidChars = set(string.punctuation.replace("-", ""))
@@ -150,3 +152,5 @@ class SSADetection(BaseModel):
                 "At least one test is required for a production or validation detection: " + values["name"]
             )
         return v
+
+    '''

contentctl/objects/ssa_detection_tags.py

@@ -1,13 +1,15 @@
+from __future__ import annotations
 import re
+from typing import List
+from pydantic import BaseModel, validator, ValidationError, model_validator, Field
 
-from pydantic import BaseModel, validator, ValidationError, root_validator
 from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
 from contentctl.objects.constants import *
-
+from contentctl.objects.enums import SecurityContentProductName
 
 class SSADetectionTags(BaseModel):
     # detection spec
-    name: str
+    #name: str
     analytic_story: list
     asset_type: str
     automated_detection_testing: str = None
@@ -19,7 +21,7 @@ class SSADetectionTags(BaseModel):
     mitre_attack_id: list = None
     nist: list = None
     observable: list
-    product: list
+    product: List[SecurityContentProductName] = Field(...,min_length=1)
     required_fields: list
     risk_score: int
     security_domain: str
@@ -77,7 +79,7 @@ class SSADetectionTags(BaseModel):
     def tags_confidence(cls, v, values):
         v = int(v)
         if not (v > 0 and v <= 100):
-             raise ValueError('confidence score is out of range 1-100: ' + values["name"])
+             raise ValueError('confidence score is out of range 1-100.' )
         else:
             return v
 
@@ -85,7 +87,7 @@ class SSADetectionTags(BaseModel):
     @validator('impact')
     def tags_impact(cls, v, values):
         if not (v > 0 and v <= 100):
-             raise ValueError('impact score is out of range 1-100: ' + values["name"])
+             raise ValueError('impact score is out of range 1-100.')
         else:
             return v
 
@@ -94,7 +96,7 @@ class SSADetectionTags(BaseModel):
         valid_kill_chain_phases = SES_KILL_CHAIN_MAPPINGS.keys()
         for value in v:
             if value not in valid_kill_chain_phases:
-                raise ValueError('kill chain phase not valid for ' + values["name"] + '. valid options are ' + str(valid_kill_chain_phases))
+                raise ValueError('kill chain phase not valid. Valid options are ' + str(valid_kill_chain_phases))
         return v
 
     @validator('mitre_attack_id')
@@ -102,20 +104,10 @@ class SSADetectionTags(BaseModel):
         pattern = 'T[0-9]{4}'
         for value in v:
             if not re.match(pattern, value):
-                raise ValueError('Mitre Attack ID are not following the pattern Txxxx: ' + values["name"])
+                raise ValueError('Mitre Attack ID are not following the pattern Txxxx:' )
         return v
 
-    @validator('product')
-    def tags_product(cls, v, values):
-        valid_products = [
-            "Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud",
-            "Splunk Security Analytics for AWS", "Splunk Behavioral Analytics"
-        ]
 
-        for value in v:
-            if value not in valid_products:
-                raise ValueError('product is not valid for ' + values['name'] + '. valid products are ' + str(valid_products))
-        return v
 
     @validator('risk_score')
     def tags_calculate_risk_score(cls, v, values):
@@ -125,21 +117,22 @@ class SSADetectionTags(BaseModel):
                              f"\n  Expected risk_score={calculated_risk_score}, found risk_score={int(v)}: {values['name']}")
         return v
 
-    @root_validator
-    def tags_observable(cls, values):
+
+    @model_validator(mode="after")
+    def tags_observable(self):
         valid_roles = SES_OBSERVABLE_ROLE_MAPPING.keys()
         valid_types = SES_OBSERVABLE_TYPE_MAPPING.keys()
         
-        for value in values["observable"]:
+        for value in self.observable:
             if value['type'] in valid_types:
-                if 'Splunk Behavioral Analytics' in values["product"]:
+                if 'Splunk Behavioral Analytics' in self.product:
                     continue
 
                 if 'role' not in value:
-                    raise ValueError('Observable role is missing for ' + values["name"])
+                    raise ValueError('Observable role is missing')
                 for role in value['role']:
                     if role not in valid_roles:
-                        raise ValueError('Observable role ' + role + ' not valid for ' + values["name"] + '. valid options are ' + str(valid_roles))
+                        raise ValueError(f'Observable role ' + role + ' not valid. Valid options are {str(valid_roles)}')
             else:
-                raise ValueError('Observable type ' + value['type'] + ' not valid for ' + values["name"] + '. valid options are ' + str(valid_types))
-        return values
+                raise ValueError(f'Observable type ' + value['type'] + ' not valid. Valid options are {str(valid_types)}')
+        return self