contentctl 4.4.7__py3-none-any.whl → 5.0.0__py3-none-any.whl

contentctl/objects/risk_event.py

@@ -1,51 +1,18 @@
 import re
 from functools import cached_property
 
-from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field
-from contentctl.objects.errors import ValidationFailed
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    computed_field,
+    field_validator,
+)
+
 from contentctl.objects.detection import Detection
-from contentctl.objects.observable import Observable
-
-# TODO (#259): Map our observable types to more than user/system
-# TODO (#247): centralize this mapping w/ usage of SES_OBSERVABLE_TYPE_MAPPING (see
-#   observable.py) and the ad hoc mapping made in detection_abstract.py (see the risk property func)
-TYPE_MAP: dict[str, list[str]] = {
-    "system": [
-        "Hostname",
-        "IP Address",
-        "Endpoint"
-    ],
-    "user": [
-        "User",
-        "User Name",
-        "Email Address",
-        "Email"
-    ],
-    "hash_values": [],
-    "network_artifacts": [],
-    "host_artifacts": [],
-    "tools": [],
-    "other": [
-        "Process",
-        "URL String",
-        "Unknown",
-        "Process Name",
-        "MAC Address",
-        "File Name",
-        "File Hash",
-        "Resource UID",
-        "Uniform Resource Locator",
-        "File",
-        "Geo Location",
-        "Container",
-        "Registry Key",
-        "Registry Value",
-        "Other"
-    ]
-}
-
-# Roles that should not generate risks
-IGNORE_ROLES: list[str] = ["Attacker"]
+from contentctl.objects.errors import ValidationFailed
+from contentctl.objects.rba import RiskObject
 
 
 class RiskEvent(BaseModel):
@@ -55,10 +22,12 @@ class RiskEvent(BaseModel):
     search_name: str
 
     # The subject of the risk event (e.g. a username, process name, system name, account ID, etc.)
-    risk_object: int | str
+    # (not to be confused w/ the risk object from the detection)
+    es_risk_object: int | str = Field(alias="risk_object")
 
-    # The type of the risk object (e.g. user, system, or other)
-    risk_object_type: str
+    # The type of the risk object from ES (e.g. user, system, or other) (not to be confused w/
+    # the risk object from the detection)
+    es_risk_object_type: str = Field(alias="risk_object_type")
 
     # The level of risk associated w/ the risk event
     risk_score: int
@@ -74,22 +43,19 @@ class RiskEvent(BaseModel):
 
     # The MITRE ATT&CK IDs
     annotations_mitre_attack: list[str] = Field(
-        alias="annotations.mitre_attack",
-        default=[]
+        alias="annotations.mitre_attack", default=[]
     )
 
     # Contributing events search query (we use this to derive the corresponding field from the
-    # observables)
+    # detection's risk object definition)
     contributing_events_search: str
 
-    # Private attribute caching the observable this RiskEvent is mapped to
-    _matched_observable: Observable | None = PrivateAttr(default=None)
+    # Private attribute caching the risk object this RiskEvent is mapped to
+    _matched_risk_object: RiskObject | None = PrivateAttr(default=None)
 
     # Allowing fields that aren't explicitly defined to be passed since some of the risk event's
     # fields vary depending on the SPL which generated them
-    model_config = ConfigDict(
-        extra="allow"
-    )
+    model_config = ConfigDict(extra="allow")
 
     @field_validator("annotations_mitre_attack", "analyticstories", mode="before")
     @classmethod
@@ -108,10 +74,12 @@ class RiskEvent(BaseModel):
     def source_field_name(self) -> str:
         """
         A cached derivation of the source field name the risk event corresponds to in the relevant
-        event(s). Useful for mapping back to an observable in the detection.
+        event(s). Useful for mapping back to a risk object in the detection.
         """
         pattern = re.compile(
-            r"\| savedsearch \"" + self.search_name + r"\" \| search (?P<field>[^=]+)=.+"
+            r"\| savedsearch \""
+            + self.search_name
+            + r"\" \| search (?P<field>[^=]+)=.+"
         )
         match = pattern.search(self.contributing_events_search)
         if match is None:
@@ -128,13 +96,6 @@ class RiskEvent(BaseModel):
         :param detection: the detection associated w/ this risk event
         :raises: ValidationFailed
         """
-        # Check risk_score
-        if self.risk_score != detection.tags.risk_score:
-            raise ValidationFailed(
-                f"Risk score observed in risk event ({self.risk_score}) does not match risk score in "
-                f"detection ({detection.tags.risk_score})."
-            )
-
         # Check analyticstories
         self.validate_analyticstories(detection)
 
@@ -151,8 +112,15 @@ class RiskEvent(BaseModel):
         # Check risk_message
         self.validate_risk_message(detection)
 
-        # Check several conditions against the observables
-        self.validate_risk_against_observables(detection.tags.observable)
+        # Ensure the rba object is defined
+        if detection.rba is None:
+            raise ValidationFailed(
+                f"Unexpected error: Detection '{detection.name}' has no RBA objects associated "
+                "with it; cannot validate."
+            )
+
+        # Check several conditions against the detection's risk objects
+        self.validate_risk_against_risk_objects(detection.rba.risk_objects)
 
     def validate_mitre_ids(self, detection: Detection) -> None:
         """
@@ -160,7 +128,9 @@ class RiskEvent(BaseModel):
         :param detection: the detection associated w/ this risk event
         :raises: ValidationFailed
         """
-        if sorted(self.annotations_mitre_attack) != sorted(detection.tags.mitre_attack_id):
+        if sorted(self.annotations_mitre_attack) != sorted(
+            detection.tags.mitre_attack_id
+        ):
             raise ValidationFailed(
                 f"MITRE ATT&CK IDs in risk event ({self.annotations_mitre_attack}) do not match those"
                 f" in detection ({detection.tags.mitre_attack_id})."
@@ -173,11 +143,13 @@ class RiskEvent(BaseModel):
         :raises: ValidationFailed
         """
         # Render the detection analytic_story to a list of strings before comparing
-        detection_analytic_story = [story.name for story in detection.tags.analytic_story]
+        detection_analytic_story = [
+            story.name for story in detection.tags.analytic_story
+        ]
         if sorted(self.analyticstories) != sorted(detection_analytic_story):
             raise ValidationFailed(
                 f"Analytic stories in risk event ({self.analyticstories}) do not match those"
-                f" in detection ({detection.tags.analytic_story})."
+                f" in detection ({[x.name for x in detection.tags.analytic_story]})."
             )
 
     def validate_risk_message(self, detection: Detection) -> None:
@@ -186,10 +158,20 @@ class RiskEvent(BaseModel):
         :param detection: the detection associated w/ this risk event
         :raises: ValidationFailed
         """
+        # Ensure the rba object is defined
+        if detection.rba is None:
+            raise ValidationFailed(
+                f"Unexpected error: Detection '{detection.name}' has no RBA objects associated "
+                "with it; cannot validate."
+            )
+
         # Extract the field replacement tokens ("$...$")
         field_replacement_pattern = re.compile(r"\$\S+\$")
-        tokens = field_replacement_pattern.findall(detection.tags.message)
+        tokens = field_replacement_pattern.findall(detection.rba.message)
 
+        # TODO (#346): could expand this to get the field values from the raw events and check
+        #   to see that allexpected strings ARE in the risk message (as opposed to checking only
+        #   that unexpected strings aren't)
         # Check for the presence of each token in the message from the risk event
         for token in tokens:
             if token in self.risk_message:
@@ -203,16 +185,12 @@ class RiskEvent(BaseModel):
         # placeholder
         tmp_placeholder = "PLACEHOLDERPATTERNFORESCAPING"
         escaped_source_message_with_placeholder: str = re.escape(
-            field_replacement_pattern.sub(
-                tmp_placeholder,
-                detection.tags.message
-            )
+            field_replacement_pattern.sub(tmp_placeholder, detection.rba.message)
         )
         placeholder_replacement_pattern = re.compile(tmp_placeholder)
         final_risk_message_pattern = re.compile(
             placeholder_replacement_pattern.sub(
-                r"[\\s\\S]*\\S[\\s\\S]*",
-                escaped_source_message_with_placeholder
+                r"[\\s\\S]*\\S[\\s\\S]*", escaped_source_message_with_placeholder
             )
         )
 
@@ -220,115 +198,87 @@ class RiskEvent(BaseModel):
         if final_risk_message_pattern.match(self.risk_message) is None:
             raise ValidationFailed(
                 "Risk message in event does not match the pattern set by the detection. Message in "
-                f"risk event: \"{self.risk_message}\". Message in detection: "
-                f"\"{detection.tags.message}\"."
+                f'risk event: "{self.risk_message}". Message in detection: '
+                f'"{detection.rba.message}".'
             )
 
-    def validate_risk_against_observables(self, observables: list[Observable]) -> None:
+    def validate_risk_against_risk_objects(self, risk_objects: set[RiskObject]) -> None:
         """
-        Given the observables from the associated detection, validate the risk event against those
-        observables
-        :param observables: the Observable objects from the detection
+        Given the risk objects from the associated detection, validate the risk event against those
+        risk objects
+        :param risk_objects: the risk objects from the detection
         :raises: ValidationFailed
         """
-        # Get the matched observable; will raise validation errors if no match can be made or if
-        # risk is missing values associated w/ observables
-        matched_observable = self.get_matched_observable(observables)
+        # Get the matched risk object; will raise validation errors if no match can be made or if
+        # risk is missing values associated w/ risk objects
+        matched_risk_object = self.get_matched_risk_object(risk_objects)
 
-        # The risk object type should match our mapping of observable types to risk types
-        expected_type = RiskEvent.observable_type_to_risk_type(matched_observable.type)
-        if self.risk_object_type != expected_type:
+        # The risk object type from the risk event should match our mapping of internal risk object
+        # types
+        if self.es_risk_object_type != matched_risk_object.type.value:
             raise ValidationFailed(
-                f"The risk object type ({self.risk_object_type}) does not match the expected type "
-                f"based on the matched observable ({matched_observable.type}->{expected_type}): "
-                f"risk=(object={self.risk_object}, type={self.risk_object_type}, "
-                f"source_field_name={self.source_field_name}), "
-                f"observable=(name={matched_observable.name}, type={matched_observable.type}, "
-                f"role={matched_observable.role})"
+                f"The risk object type from the risk event ({self.es_risk_object_type}) does not match"
+                " the expected type based on the matched risk object "
+                f"({matched_risk_object.type.value}): risk event=(object={self.es_risk_object}, "
+                f"type={self.es_risk_object_type}, source_field_name={self.source_field_name}), "
+                f"risk object=(name={matched_risk_object.field}, "
+                f"type={matched_risk_object.type.value})"
             )
 
-    @staticmethod
-    def observable_type_to_risk_type(observable_type: str) -> str:
-        """
-        Given a string representing the observable type, use our mapping to convert it to the
-        expected type in the risk event
-        :param observable_type: the type of the observable
-        :returns: a string (the risk object type)
-        :raises ValueError: if the observable type has not yet been mapped to a risk object type
-        """
-        # Iterate over the map and search the lists for a match
-        for risk_type in TYPE_MAP:
-            if observable_type in TYPE_MAP[risk_type]:
-                return risk_type
-
-        raise ValueError(
-            f"Observable type {observable_type} does not have a mapping to a risk type in TYPE_MAP"
-        )
+        # Check risk_score
+        if self.risk_score != matched_risk_object.score:
+            raise ValidationFailed(
+                f"Risk score observed in risk event ({self.risk_score}) does not match risk score in "
+                f"matched risk object from detection ({matched_risk_object.score})."
+            )
 
-    @staticmethod
-    def ignore_observable(observable: Observable) -> bool:
-        """
-        Given an observable, determine based on its roles if it should be ignored in risk/observable
-        matching (e.g. Attacker role observables should not generate risk events)
-        :param observable: the Observable object we are checking the roles of
-        :returns: a bool indicating whether this observable should be ignored or not
-        """
-        ignore = False
-        for role in observable.role:
-            if role in IGNORE_ROLES:
-                ignore = True
-                break
-        return ignore
-
-    def get_matched_observable(self, observables: list[Observable]) -> Observable:
+    def get_matched_risk_object(self, risk_objects: set[RiskObject]) -> RiskObject:
         """
-        Given a list of observables, return the one this risk event matches
-        :param observables: the list of Observable objects we are checking against
-        :returns: the matched Observable object
+        Given a set of risk objects, return the one this risk event matches
+        :param risk_objects: the list of risk objects we are checking against
+        :returns: the matched risk object
         :raises ValidationFailed: if a match could not be made or if an expected field (based on
-            one of the observables) could not be found in the risk event
+            one of the risk objects) could not be found in the risk event
         """
         # Return the cached match if already found
-        if self._matched_observable is not None:
-            return self._matched_observable
+        if self._matched_risk_object is not None:
+            return self._matched_risk_object
 
-        matched_observable: Observable | None = None
+        matched_risk_object: RiskObject | None = None
 
         # Iterate over the obervables and check for a match
-        for observable in observables:
+        for risk_object in risk_objects:
             # TODO (#252): Refactor and re-enable per-field validation of risk events
-            # Each the field name used in each observable shoud be present in the risk event
-            # if not hasattr(self, observable.name):
+            # Each the field name used in each risk object shoud be present in the risk event
+            # if not hasattr(self, risk_object.field):
             #     raise ValidationFailed(
-            #         f"Observable field \"{observable.name}\" not found in risk event."
+            #         f"Risk object field \"{risk_object.field}\" not found in risk event."
             #     )
 
-            # Try to match the risk_object against a specific observable for the obervables with
-            # a valid role (some, like Attacker, shouldn't get converted to risk events)
-            if self.source_field_name == observable.name:
-                if matched_observable is not None:
+            # Try to match the risk_object against a specific risk object
+            if self.source_field_name == risk_object.field:
+                # TODO (#347): enforce that field names are not repeated across risk objects as
+                #   part of build/validate
+                if matched_risk_object is not None:
                     raise ValueError(
-                        "Unexpected conditon: we don't expect the source event field "
-                        "corresponding to an observables field name to be repeated."
+                        "Unexpected conditon: we don't expect multiple risk objects to use the "
+                        "same field name, so we should not be able match the risk event to "
+                        "multiple risk objects."
                     )
 
-                # Report any risk events we find that shouldn't be there
-                if RiskEvent.ignore_observable(observable):
-                    raise ValidationFailed(
-                        "Risk event matched an observable with an invalid role: "
-                        f"(name={observable.name}, type={observable.type}, role={observable.role})")
-                # NOTE: we explicitly do not break early as we want to check each observable
-                matched_observable = observable
+                # NOTE: we explicitly do not break early as we want to check each risk object
+                matched_risk_object = risk_object
 
-        # Ensure we were able to match the risk event to a specific observable
-        if matched_observable is None:
+        # Ensure we were able to match the risk event to a specific risk object
+        if matched_risk_object is None:
             raise ValidationFailed(
-                f"Unable to match risk event (object={self.risk_object}, type="
-                f"{self.risk_object_type}, source_field_name={self.source_field_name}) to an "
-                "observable; please check for errors in the observable roles/types for this "
-                "detection, as well as the risk event build process in contentctl."
+                f"Unable to match risk event (object={self.es_risk_object}, type="
+                f"{self.es_risk_object_type}, source_field_name={self.source_field_name}) to a "
+                "risk object in the detection; please check for errors in the risk object types for this "
+                "detection, as well as the risk event build process in contentctl (e.g. threat "
+                "objects aren't being converted to risk objects somehow)."
             )
 
-        # Cache and return the matched observable
-        self._matched_observable = matched_observable
-        return self._matched_observable
+        # Cache and return the matched risk object
+        self._matched_risk_object = matched_risk_object
+        return self._matched_risk_object

contentctl/objects/risk_object.py

@@ -13,6 +13,7 @@ class RiskObject(BaseModel):
     :param type_: the type of the risk object (e.g. "system")
     :param score: the risk score associated with the obersevable (e.g. 64)
     """
+
     field: str
     type: str
     score: int

contentctl/objects/savedsearches_conf.py

@@ -1,4 +1,3 @@
-
 from pathlib import Path
 from typing import Any, ClassVar
 import re
@@ -17,6 +16,7 @@ class SavedsearchesConf(BaseModel):
     NOTE: At present, this model only parses the detections themselves from the .conf; thing like
     baselines or response tasks are left alone currently
     """
+
     # The path to the conf file
     path: Path = Field(...)
 
@@ -112,8 +112,7 @@ class SavedsearchesConf(BaseModel):
 
         # Build the stanza model from the accumulated lines and adjust the state to end this section
         self.detection_stanzas[self._current_section_name] = DetectionStanza(
-            name=self._current_section_name,
-            lines=self._current_section_lines
+            name=self._current_section_name, lines=self._current_section_lines
         )
         self._in_section = False
 
@@ -170,7 +169,9 @@ class SavedsearchesConf(BaseModel):
                     self._in_detections = True
 
     @staticmethod
-    def init_from_package(package_path: Path, app_name: str, appid: str) -> "SavedsearchesConf":
+    def init_from_package(
+        package_path: Path, app_name: str, appid: str
+    ) -> "SavedsearchesConf":
         """
         Alternate constructor which can take an app package, and extract the savedsearches.conf from
         a temporary file.
@@ -188,9 +189,10 @@ class SavedsearchesConf(BaseModel):
             # Open the tar/gzip archive
             with tarfile.open(package_path) as package:
                 # Extract the savedsearches.conf and use it to init the model
-                package_conf_path = SavedsearchesConf.PACKAGE_CONF_PATH_FMT_STR.format(appid=appid)
+                package_conf_path = SavedsearchesConf.PACKAGE_CONF_PATH_FMT_STR.format(
+                    appid=appid
+                )
                 package.extract(package_conf_path, path=tmpdir)
                 return SavedsearchesConf(
-                    path=Path(tmpdir, package_conf_path),
-                    app_label=app_name
+                    path=Path(tmpdir, package_conf_path), app_label=app_name
                 )

contentctl/objects/security_content_object.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
-from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract
+from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
+    SecurityContentObject_Abstract,
+)
+
 
 class SecurityContentObject(SecurityContentObject_Abstract):
-    pass
+    pass

contentctl/objects/story.py

@@ -1,99 +1,106 @@
 from __future__ import annotations
-from typing import TYPE_CHECKING,List
-from contentctl.objects.story_tags import StoryTags
-from pydantic import Field, model_serializer,computed_field, model_validator
+
 import re
+from typing import TYPE_CHECKING, List, Literal
+
+from pydantic import Field, computed_field, model_serializer, model_validator
+
+from contentctl.objects.story_tags import StoryTags
+
 if TYPE_CHECKING:
-    from contentctl.objects.detection import Detection
-    from contentctl.objects.investigation import Investigation
     from contentctl.objects.baseline import Baseline
-    from contentctl.objects.data_source import DataSource
     from contentctl.objects.config import CustomApp
+    from contentctl.objects.data_source import DataSource
+    from contentctl.objects.detection import Detection
+    from contentctl.objects.investigation import Investigation
 
+from contentctl.objects.enums import DetectionStatus
 from contentctl.objects.security_content_object import SecurityContentObject
 
+
 class Story(SecurityContentObject):
     narrative: str = Field(...)
     tags: StoryTags = Field(...)
-
+    status: Literal[DetectionStatus.production, DetectionStatus.deprecated]
     # These are updated when detection and investigation objects are created.
     # Specifically in the model_post_init functions
-    detections:List[Detection] = []
+    detections: List[Detection] = []
     investigations: List[Investigation] = []
     baselines: List[Baseline] = []
-    
-    
+
     @computed_field
     @property
-    def data_sources(self)-> list[DataSource]:
+    def data_sources(self) -> list[DataSource]:
         # Only add a data_source if it does not already exist in the story
-        data_source_objects:set[DataSource] = set()
+        data_source_objects: set[DataSource] = set()
         for detection in self.detections:
             data_source_objects.update(set(detection.data_source_objects))
-        
+
         return sorted(list(data_source_objects))
 
+    def storyAndInvestigationNamesWithApp(self, app: CustomApp) -> List[str]:
+        return [
+            detection.get_conf_stanza_name(app) for detection in self.detections
+        ] + [
+            investigation.get_response_task_name(app)
+            for investigation in self.investigations
+        ]
 
-    def storyAndInvestigationNamesWithApp(self, app:CustomApp)->List[str]:
-        return [detection.get_conf_stanza_name(app) for detection in self.detections] + \
-               [investigation.get_response_task_name(app) for investigation in self.investigations]
-        
     @model_serializer
     def serialize_model(self):
-        #Call serializer for parent
+        # Call serializer for parent
         super_fields = super().serialize_model()
-        
-        #All fields custom to this model
-        model= {
+
+        # All fields custom to this model
+        model = {
             "narrative": self.narrative,
             "tags": self.tags.model_dump(),
             "detection_names": self.detection_names,
             "investigation_names": self.investigation_names,
             "baseline_names": self.baseline_names,
             "author_company": self.author_company,
-            "author_name":self.author_name
+            "author_name": self.author_name,
         }
         detections = []
         for detection in self.detections:
             new_detection = {
-                "name":detection.name,
-                "source":detection.source,
-                "type":detection.type
+                "name": detection.name,
+                "source": detection.source,
+                "type": detection.type,
             }
             if self.tags.mitre_attack_enrichments is not None:
-                new_detection['tags'] = {"mitre_attack_enrichments": [{"mitre_attack_technique":  enrichment.mitre_attack_technique} for enrichment in detection.tags.mitre_attack_enrichments]}
+                new_detection["tags"] = {
+                    "mitre_attack_enrichments": [
+                        {"mitre_attack_technique": enrichment.mitre_attack_technique}
+                        for enrichment in detection.tags.mitre_attack_enrichments
+                    ]
+                }
             detections.append(new_detection)
 
-        model['detections'] = detections
-        #Combine fields from this model with fields from parent
+        model["detections"] = detections
+        # Combine fields from this model with fields from parent
         super_fields.update(model)
-        
-        #return the model
+
+        # return the model
         return super_fields
 
     @model_validator(mode="after")
     def setTagsFields(self):
-        
         enrichments = []
         for detection in self.detections:
             enrichments.extend(detection.tags.mitre_attack_enrichments)
         self.tags.mitre_attack_enrichments = list(set(enrichments))
 
-    
         tactics = []
         for enrichment in self.tags.mitre_attack_enrichments:
             tactics.extend(enrichment.mitre_attack_tactics)
         self.tags.mitre_attack_tactics = set(tactics)
 
-    
-    
         datamodels = []
         for detection in self.detections:
             datamodels.extend(detection.datamodel)
         self.tags.datamodels = set(datamodels)
 
-    
-    
         kill_chain_phases = []
         for detection in self.detections:
             kill_chain_phases.extend(detection.tags.kill_chain_phases)
@@ -101,42 +108,40 @@ class Story(SecurityContentObject):
 
         return self
 
-
     @computed_field
     @property
-    def author_name(self)->str:
-        match_author = re.search(r'^([^,]+)', self.author)
+    def author_name(self) -> str:
+        match_author = re.search(r"^([^,]+)", self.author)
         if match_author is None:
-            return 'no'
+            return "no"
         else:
             return match_author.group(1)
 
     @computed_field
     @property
-    def author_company(self)->str:
-        match_company = re.search(r',\s?(.*)$', self.author)
+    def author_company(self) -> str:
+        match_company = re.search(r",\s?(.*)$", self.author)
         if match_company is None:
-            return 'no'
+            return "no"
         else:
             return match_company.group(1)
 
     @computed_field
     @property
-    def author_email(self)->str:
+    def author_email(self) -> str:
         return "-"
 
     @computed_field
     @property
-    def detection_names(self)->List[str]:
+    def detection_names(self) -> List[str]:
         return [detection.name for detection in self.detections]
-    
+
     @computed_field
     @property
-    def investigation_names(self)->List[str]:
+    def investigation_names(self) -> List[str]:
         return [investigation.name for investigation in self.investigations]
 
     @computed_field
     @property
-    def baseline_names(self)->List[str]:        
+    def baseline_names(self) -> List[str]:
         return [baseline.name for baseline in self.baselines]
-