contentctl 4.3.0__tar.gz → 4.3.2__tar.gz

{contentctl-4.3.0 → contentctl-4.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contentctl
-Version: 4.3.0
+Version: 4.3.2
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT
@@ -11,20 +11,20 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
-Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: attackcti (>=0.3.7,<0.5.0)
+Requires-Dist: PyYAML (>=6.0.2,<7.0.0)
+Requires-Dist: attackcti (>=0.4.0,<0.5.0)
 Requires-Dist: bottle (>=0.12.25,<0.13.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: gitpython (>=3.1.43,<4.0.0)
 Requires-Dist: pycvesearch (>=1.2,<2.0)
-Requires-Dist: pydantic (>=2.7.1,<3.0.0)
-Requires-Dist: pygit2 (>=1.14.1,<2.0.0)
+Requires-Dist: pydantic (>=2.8.2,<3.0.0)
+Requires-Dist: pygit2 (>=1.15.1,<2.0.0)
 Requires-Dist: questionary (>=2.0.1,<3.0.0)
-Requires-Dist: requests (>=2.32.2,<2.33.0)
+Requires-Dist: requests (>=2.32.3,<2.33.0)
 Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
-Requires-Dist: setuptools (>=69.5.1,<73.0.0)
-Requires-Dist: splunk-sdk (>=2.0.1,<3.0.0)
-Requires-Dist: tqdm (>=4.66.4,<5.0.0)
+Requires-Dist: setuptools (>=69.5.1,<74.0.0)
+Requires-Dist: splunk-sdk (>=2.0.2,<3.0.0)
+Requires-Dist: tqdm (>=4.66.5,<5.0.0)
 Requires-Dist: tyro (>=0.8.3,<0.9.0)
 Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
 Description-Content-Type: text/markdown

{contentctl-4.3.0 → contentctl-4.3.2}/contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -1060,7 +1060,9 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
             results = JSONResultsReader(job.results(output_mode="json"))
 
             # Consolidate a set of the distinct observable field names
-            observable_fields_set = set([o.name for o in detection.tags.observable])
+            observable_fields_set = set([o.name for o in detection.tags.observable]) # keeping this around for later
+            risk_object_fields_set = set([o.name for o in detection.tags.observable if "Victim" in o.role ]) # just the "Risk Objects"
+            threat_object_fields_set = set([o.name for o in detection.tags.observable if "Attacker" in o.role]) # just the "threat objects"
 
             # Ensure the search had at least one result
             if int(job.content.get("resultCount", "0")) > 0:
@@ -1068,7 +1070,10 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
                 test.result = UnitTestResult()
 
                 # Initialize the collection of fields that are empty that shouldn't be
+                present_threat_objects: set[str] = set()
                 empty_fields: set[str] = set()
+                
+                
 
                 # Filter out any messages in the results
                 for result in results:
@@ -1077,30 +1082,50 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
 
                     # If not a message, it is a dict and we will process it
                     results_fields_set = set(result.keys())
+                    # Guard against first events (relevant later)
 
-                    # Identify any observable fields that are not available in the results
-                    missing_fields = observable_fields_set - results_fields_set
-                    if len(missing_fields) > 0:
+                    # Identify any risk object fields that are not available in the results
+                    missing_risk_objects = risk_object_fields_set - results_fields_set
+                    if len(missing_risk_objects) > 0:
                         # Report a failure in such cases
-                        e = Exception(f"The observable field(s) {missing_fields} are missing in the detection results")
+                        e = Exception(f"The observable field(s) {missing_risk_objects} are missing in the detection results")
                         test.result.set_job_content(
                             job.content,
                             self.infrastructure,
-                            TestResultStatus.ERROR,
+                            TestResultStatus.FAIL,
                             exception=e,
                             duration=time.time() - search_start_time,
                         )
 
-                        return
+                        return                    
 
-                    # If we find one or more fields that contain the string "null" then they were
+                    # If we find one or more risk object fields that contain the string "null" then they were
                     # not populated and we should throw an error.  This can happen if there is a typo
                     # on a field.  In this case, the field will appear but will not contain any values
-                    current_empty_fields = set()
+                    current_empty_fields: set[str] = set()
+                    
                     for field in observable_fields_set:
                         if result.get(field, 'null') == 'null':
-                            current_empty_fields.add(field)
-
+                            if field in risk_object_fields_set:
+                                e = Exception(f"The risk object field {field} is missing in at least one result.")
+                                test.result.set_job_content(
+                                    job.content,
+                                    self.infrastructure,
+                                    TestResultStatus.FAIL,
+                                    exception=e,
+                                    duration=time.time() - search_start_time,
+                                )
+                                return
+                            else:
+                                if field in threat_object_fields_set:
+                                    current_empty_fields.add(field)
+                        else:
+                            if field in threat_object_fields_set:
+                                present_threat_objects.add(field)
+                                continue
+                                
+
+                    
                     # If everything succeeded up until now, and no empty fields are found in the
                     # current result, then the search was a success
                     if len(current_empty_fields) == 0:
@@ -1114,21 +1139,32 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
 
                     else:
                         empty_fields = empty_fields.union(current_empty_fields)
-
-                # Report a failure if there were empty fields in all results
-                e = Exception(
-                    f"One or more required observable fields {empty_fields} contained 'null' values.  Is the "
-                    "data being parsed correctly or is there an error in the naming of a field?"
+                        
+                        
+                missing_threat_objects = threat_object_fields_set - present_threat_objects
+                # Report a failure if there were empty fields in a threat object in all results
+                if len(missing_threat_objects) > 0:
+                    e = Exception(
+                        f"One or more required threat object fields {missing_threat_objects} contained 'null' values in all events. "
+                        "Is the data being parsed correctly or is there an error in the naming of a field?"
                     )
-                test.result.set_job_content(
-                    job.content,
-                    self.infrastructure,
-                    TestResultStatus.ERROR,
-                    exception=e,
-                    duration=time.time() - search_start_time,
-                )
+                    test.result.set_job_content(
+                        job.content,
+                        self.infrastructure,
+                        TestResultStatus.FAIL,
+                        exception=e,
+                        duration=time.time() - search_start_time,
+                    )
+                    return
+                
 
-                return
+                test.result.set_job_content(
+                            job.content,
+                            self.infrastructure,
+                            TestResultStatus.PASS,
+                            duration=time.time() - search_start_time,
+                        )
+                return               
 
             else:
                 # Report a failure if there were no results at all

{contentctl-4.3.0 → contentctl-4.3.2}/contentctl/enrichments/attack_enrichment.py

@@ -7,7 +7,7 @@ from attackcti import attack_client
 import logging
 from pydantic import BaseModel, Field
 from dataclasses import field
-from typing import Annotated
+from typing import Annotated,Any
 from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
 from contentctl.objects.config import validate
 logging.getLogger('taxii2client').setLevel(logging.CRITICAL)
@@ -33,21 +33,33 @@ class AttackEnrichment(BaseModel):
         else:
             raise Exception(f"Error, Unable to find Mitre Enrichment for MitreID {mitre_id}")
         
-
-    def addMitreID(self, technique:dict, tactics:list[str], groups:list[str])->None:
-        
+    def addMitreIDViaGroupNames(self, technique:dict, tactics:list[str], groupNames:list[str])->None:
         technique_id = technique['technique_id']
         technique_obj = technique['technique']
         tactics.sort()
-        groups.sort()
-
+        
         if technique_id in self.data:
             raise Exception(f"Error, trying to redefine MITRE ID '{technique_id}'")
+        self.data[technique_id] = MitreAttackEnrichment(mitre_attack_id=technique_id, 
+                                                        mitre_attack_technique=technique_obj, 
+                                                        mitre_attack_tactics=tactics, 
+                                                        mitre_attack_groups=groupNames,
+                                                        mitre_attack_group_objects=[]) 
+
+    def addMitreIDViaGroupObjects(self, technique:dict, tactics:list[str],  groupObjects:list[dict[str,Any]])->None:
+        technique_id = technique['technique_id']
+        technique_obj = technique['technique']
+        tactics.sort()
         
+        groupNames:list[str] = sorted([group['group'] for group in groupObjects])
+        
+        if technique_id in self.data:
+            raise Exception(f"Error, trying to redefine MITRE ID '{technique_id}'")
         self.data[technique_id] = MitreAttackEnrichment(mitre_attack_id=technique_id, 
                                                         mitre_attack_technique=technique_obj, 
                                                         mitre_attack_tactics=tactics, 
-                                                        mitre_attack_groups=groups)
+                                                        mitre_attack_groups=groupNames,
+                                                        mitre_attack_group_objects=groupObjects)
 
     
     def get_attack_lookup(self, input_path: str, store_csv: bool = False, force_cached_or_offline: bool = False, skip_enrichment:bool = False) -> dict:
@@ -86,19 +98,20 @@ class AttackEnrichment(BaseModel):
                 progress_percent = ((index+1)/len(all_enterprise_techniques)) * 100
                 if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()):
                     print(f"\r\t{'MITRE Technique Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
-                apt_groups = []
+                apt_groups:list[dict[str,Any]] = []
                 for relationship in enterprise_relationships:
                     if (relationship['target_object'] == technique['id']) and relationship['source_object'].startswith('intrusion-set'):
                         for group in enterprise_groups:
                             if relationship['source_object'] == group['id']:
-                                apt_groups.append(group['group'])
+                                apt_groups.append(group)
+                                #apt_groups.append(group['group'])
 
                 tactics = []
                 if ('tactic' in technique):
                     for tactic in technique['tactic']:
                         tactics.append(tactic.replace('-',' ').title())
 
-                self.addMitreID(technique, tactics, apt_groups)
+                self.addMitreIDViaGroupObjects(technique, tactics, apt_groups)
                 attack_lookup[technique['technique_id']] = {'technique': technique['technique'], 'tactics': tactics, 'groups': apt_groups}
 
             if store_csv:
@@ -131,7 +144,7 @@ class AttackEnrichment(BaseModel):
                 technique_input = {'technique_id': key , 'technique': attack_lookup[key]['technique'] }
                 tactics_input = attack_lookup[key]['tactics']
                 groups_input = attack_lookup[key]['groups']
-                self.addMitreID(technique=technique_input, tactics=tactics_input, groups=groups_input)
+                self.addMitreIDViaGroupNames(technique=technique_input, tactics=tactics_input, groups=groups_input)
             
                 
 

{contentctl-4.3.0 → contentctl-4.3.2}/contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -290,6 +290,11 @@ class Detection_Abstract(SecurityContentObject):
                 risk_object['threat_object_field'] = entity.name
                 risk_object['threat_object_type'] = "url"
                 risk_objects.append(risk_object)
+            
+            elif 'Attacker' in entity.role:
+                risk_object['threat_object_field'] = entity.name
+                risk_object['threat_object_type'] = entity.type.lower()
+                risk_objects.append(risk_object)
 
             else:
                 risk_object['risk_object_type'] = 'other'

{contentctl-4.3.0 → contentctl-4.3.2}/contentctl/objects/constants.py

@@ -132,3 +132,8 @@ SES_ATTACK_TACTICS_ID_MAPPING = {
     "Exfiltration": "TA0010",
     "Impact": "TA0040"
 }
+
+RBA_OBSERVABLE_ROLE_MAPPING = {
+    "Attacker": 0,
+    "Victim": 1
+}

contentctl-4.3.2/contentctl/objects/mitre_attack_enrichment.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+from pydantic import BaseModel, Field, ConfigDict, HttpUrl, field_validator
+from typing import List, Annotated
+from enum import StrEnum
+import datetime
+
+class MitreTactics(StrEnum):
+    RECONNAISSANCE = "Reconnaissance"
+    RESOURCE_DEVELOPMENT = "Resource Development"
+    INITIAL_ACCESS = "Initial Access"
+    EXECUTION = "Execution"
+    PERSISTENCE = "Persistence"
+    PRIVILEGE_ESCALATION = "Privilege Escalation"
+    DEFENSE_EVASION = "Defense Evasion"
+    CREDENTIAL_ACCESS = "Credential Access"
+    DISCOVERY = "Discovery"
+    LATERAL_MOVEMENT = "Lateral Movement"
+    COLLECTION = "Collection"
+    COMMAND_AND_CONTROL = "Command And Control"
+    EXFILTRATION = "Exfiltration"
+    IMPACT = "Impact"
+
+
+class AttackGroupMatrix(StrEnum):
+    enterprise_attack = "enterprise-attack"
+    ics_attack = "ics-attack"
+    mobile_attack = "mobile-attack"
+
+
+class AttackGroupType(StrEnum):
+    intrusion_set = "intrusion-set"
+
+class MitreExternalReference(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    source_name: str
+    external_id: None | str = None 
+    url: None | HttpUrl = None
+    description: None | str = None
+
+
+class MitreAttackGroup(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    contributors: list[str] = []
+    created: datetime.datetime
+    created_by_ref: str
+    external_references: list[MitreExternalReference]
+    group: str
+    group_aliases: list[str]
+    group_description: str
+    group_id: str
+    id: str
+    matrix: list[AttackGroupMatrix]
+    mitre_attack_spec_version: None | str
+    mitre_version: str
+    #assume that if the deprecated field is not present, then the group is not deprecated
+    mitre_deprecated: bool
+    modified: datetime.datetime
+    modified_by_ref: str
+    object_marking_refs: list[str]
+    type: AttackGroupType
+    url: HttpUrl
+    
+
+    @field_validator("mitre_deprecated", mode="before")
+    def standardize_mitre_deprecated(cls, mitre_deprecated:bool | None) -> bool:
+        '''
+        For some reason, the API will return either a bool for mitre_deprecated OR
+        None. We simplify our typing by converting None to False, and assuming that
+        if deprecated is None, then the group is not deprecated.
+        '''
+        if mitre_deprecated is None:
+            return False
+        return mitre_deprecated
+
+    @field_validator("contributors", mode="before")
+    def standardize_contributors(cls, contributors:list[str] | None) -> list[str]:
+        '''
+        For some reason, the API will return either a list of strings for contributors OR
+        None. We simplify our typing by converting None to an empty list.
+        '''
+        if contributors is None:
+            return []
+        return contributors
+
+class MitreAttackEnrichment(BaseModel):
+    ConfigDict(use_enum_values=True)
+    mitre_attack_id: Annotated[str, Field(pattern=r"^T\d{4}(.\d{3})?$")] = Field(...)
+    mitre_attack_technique: str = Field(...)
+    mitre_attack_tactics: List[MitreTactics] = Field(...)
+    mitre_attack_groups: List[str] = Field(...)
+    #Exclude this field from serialization - it is very large and not useful in JSON objects
+    mitre_attack_group_objects: list[MitreAttackGroup] = Field(..., exclude=True)
+    def __hash__(self) -> int:
+        return id(self)
+

{contentctl-4.3.0 → contentctl-4.3.2}/contentctl/objects/observable.py

@@ -1,5 +1,5 @@
 from pydantic import BaseModel, field_validator
-from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, SES_OBSERVABLE_ROLE_MAPPING
+from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, RBA_OBSERVABLE_ROLE_MAPPING
 
 
 class Observable(BaseModel):
@@ -26,10 +26,12 @@ class Observable(BaseModel):
     def check_roles(cls, v: list[str]):
         if len(v) == 0:
             raise ValueError("Error, at least 1 role must be listed for Observable.")
+        if len(v) > 1:
+            raise ValueError("Error, each Observable can only have one role.")
         for role in v:
-            if role not in SES_OBSERVABLE_ROLE_MAPPING.keys():
+            if role not in RBA_OBSERVABLE_ROLE_MAPPING.keys():
                 raise ValueError(
                     f"Invalid role '{role}' provided for observable.  Valid observable types are "
-                    f"{SES_OBSERVABLE_ROLE_MAPPING.keys()}"
+                    f"{RBA_OBSERVABLE_ROLE_MAPPING.keys()}"
                 )
         return v

{contentctl-4.3.0 → contentctl-4.3.2}/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml

@@ -53,11 +53,11 @@ tags:
   - name: parent_process_name
     type: Process
     role:
-    - Parent Process
+    - Attacker
   - name: process_name
     type: Process
     role:
-    - Child Process
+    - Attacker
   product:
   - Splunk Enterprise
   - Splunk Enterprise Security

{contentctl-4.3.0 → contentctl-4.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "contentctl"
-version = "4.3.0"
+version = "4.3.2"
 description = "Splunk Content Control Tool"
 authors = ["STRT <research@splunk.com>"]
 license = "Apache 2.0"
@@ -11,23 +11,23 @@ contentctl = 'contentctl.contentctl:main'
 
 [tool.poetry.dependencies]
 python = "^3.11"
-pydantic = "^2.7.1"
-PyYAML = "^6.0.1"
-requests = "~2.32.2"
+pydantic = "^2.8.2"
+PyYAML = "^6.0.2"
+requests = "~2.32.3"
 pycvesearch = "^1.2"
 xmltodict = "^0.13.0"
-attackcti = ">=0.3.7,<0.5.0"
+attackcti = "^0.4.0"
 Jinja2 = "^3.1.4"
 questionary = "^2.0.1"
 docker = "^7.1.0"
-splunk-sdk = "^2.0.1"
+splunk-sdk = "^2.0.2"
 semantic-version = "^2.10.0"
 bottle = "^0.12.25"
-tqdm = "^4.66.4"
-pygit2 = "^1.14.1"
+tqdm = "^4.66.5"
+pygit2 = "^1.15.1"
 tyro = "^0.8.3"
 gitpython = "^3.1.43"
-setuptools = ">=69.5.1,<73.0.0"
+setuptools = ">=69.5.1,<74.0.0"
 [tool.poetry.dev-dependencies]
 
 [build-system]

contentctl-4.3.0/contentctl/objects/mitre_attack_enrichment.py

@@ -1,32 +0,0 @@
-from __future__ import annotations
-from pydantic import BaseModel, Field, ConfigDict
-from typing import List, Annotated
-from enum import StrEnum
-
-
-class MitreTactics(StrEnum):
-    RECONNAISSANCE = "Reconnaissance"
-    RESOURCE_DEVELOPMENT = "Resource Development"
-    INITIAL_ACCESS = "Initial Access"
-    EXECUTION = "Execution"
-    PERSISTENCE = "Persistence"
-    PRIVILEGE_ESCALATION = "Privilege Escalation"
-    DEFENSE_EVASION = "Defense Evasion"
-    CREDENTIAL_ACCESS = "Credential Access"
-    DISCOVERY = "Discovery"
-    LATERAL_MOVEMENT = "Lateral Movement"
-    COLLECTION = "Collection"
-    COMMAND_AND_CONTROL = "Command And Control"
-    EXFILTRATION = "Exfiltration"
-    IMPACT = "Impact"
-
-
-class MitreAttackEnrichment(BaseModel):
-    ConfigDict(use_enum_values=True)
-    mitre_attack_id: Annotated[str, Field(pattern=r"^T\d{4}(.\d{3})?$")] = Field(...)
-    mitre_attack_technique: str = Field(...)
-    mitre_attack_tactics: List[MitreTactics] = Field(...)
-    mitre_attack_groups: List[str] = Field(...)
-
-    def __hash__(self) -> int:
-        return id(self)