contentctl 4.3.4__py3-none-any.whl → 4.4.0__py3-none-any.whl

contentctl/objects/errors.py

@@ -1,3 +1,7 @@
+from abc import ABC, abstractmethod
+from uuid import UUID
+
+
 class ValidationFailed(Exception):
     """Indicates not an error in execution, but a validation failure"""
     pass
@@ -16,3 +20,186 @@ class ServerError(IntegrationTestingError):
 class ClientError(IntegrationTestingError):
     """An error encounterd during integration testing, on the client's side (locally)"""
     pass
+
+
+class MetadataValidationError(Exception, ABC):
+    """
+    Base class for any errors arising from savedsearches.conf detection metadata validation
+    """
+    # The name of the rule the error relates to
+    rule_name: str
+
+    @property
+    @abstractmethod
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        raise NotImplementedError()
+
+    @property
+    @abstractmethod
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        raise NotImplementedError()
+
+
+class DetectionMissingError(MetadataValidationError):
+    """
+    An error indicating a detection in the prior build could not be found in the current build
+    """
+    def __init__(
+            self,
+            rule_name: str,
+            *args: object
+    ) -> None:
+        self.rule_name = rule_name
+        super().__init__(self.long_message, *args)
+
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' in previous build not found in current build; "
+            "detection may have been removed or renamed."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            "Detection from previous build not found in current build."
+        )
+
+
+class DetectionIDError(MetadataValidationError):
+    """
+    An error indicating the detection ID may have changed between builds
+    """
+    # The ID from the current build
+    current_id: UUID
+
+    # The ID from the previous build
+    previous_id: UUID
+
+    def __init__(
+            self,
+            rule_name: str,
+            current_id: UUID,
+            previous_id: UUID,
+            *args: object
+    ) -> None:
+        self.rule_name = rule_name
+        self.current_id = current_id
+        self.previous_id = previous_id
+        super().__init__(self.long_message, *args)
+
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' has ID {self.current_id} in current build "
+            f"and {self.previous_id} in previous build; detection IDs and "
+            "names should not change for the same detection between releases."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Detection ID {self.current_id} in current build does not match ID {self.previous_id} in previous build."
+        )
+
+
+class VersioningError(MetadataValidationError, ABC):
+    """
+    A base class for any metadata validation errors relating to detection versioning
+    """
+    # The version in the current build
+    current_version: int
+
+    # The version in the previous build
+    previous_version: int
+
+    def __init__(
+            self,
+            rule_name: str,
+            current_version: int,
+            previous_version: int,
+            *args: object
+    ) -> None:
+        self.rule_name = rule_name
+        self.current_version = current_version
+        self.previous_version = previous_version
+        super().__init__(self.long_message, *args)
+
+
+class VersionDecrementedError(VersioningError):
+    """
+    An error indicating the version number went down between builds
+    """
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' has version {self.current_version} in "
+            f"current build and {self.previous_version} in previous build; "
+            "detection versions cannot decrease in successive builds."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Detection version ({self.current_version}) in current build is less than version "
+            f"({self.previous_version}) in previous build."
+        )
+
+
+class VersionBumpingError(VersioningError):
+    """
+    An error indicating the detection changed but its version wasn't bumped appropriately
+    """
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' has changed in current build compared to previous "
+            "build (stanza hashes differ); the detection version should be bumped "
+            f"to at least {self.previous_version + 1}."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Detection version in current build should be bumped to at least {self.previous_version + 1}."
+        )

contentctl/objects/investigation.py

@@ -1,20 +1,23 @@
 from __future__ import annotations
 import re
-from typing import TYPE_CHECKING, Optional, List, Any
-from pydantic import field_validator, computed_field, Field, ValidationInfo, ConfigDict,model_serializer
-if TYPE_CHECKING:
-    from contentctl.input.director import DirectorOutputDto
+from typing import List, Any
+from pydantic import computed_field, Field, ConfigDict,model_serializer
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.enums import DataModel
 from contentctl.objects.investigation_tags import InvestigationTags
-
+from contentctl.objects.constants import (
+    CONTENTCTL_MAX_SEARCH_NAME_LENGTH,
+    CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE,
+    CONTENTCTL_MAX_STANZA_LENGTH
+)
+from contentctl.objects.config import CustomApp
 
 # TODO (#266): disable the use_enum_values configuration
 class Investigation(SecurityContentObject):
     model_config = ConfigDict(use_enum_values=True,validate_default=False)
     type: str = Field(...,pattern="^Investigation$")
     datamodel: list[DataModel] = Field(...)
-    
+    name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
     search: str = Field(...)
     how_to_implement: str = Field(...)
     known_false_positives: str = Field(...)
@@ -27,11 +30,11 @@ class Investigation(SecurityContentObject):
     @property
     def inputs(self)->List[str]:
         #Parse out and return all inputs from the searchj
-        inputs = []
+        inputs:List[str] = []
         pattern = r"\$([^\s.]*)\$"
 
         for input in re.findall(pattern, self.search):
-            inputs.append(input)
+            inputs.append(str(input))
 
         return inputs
 
@@ -40,6 +43,16 @@ class Investigation(SecurityContentObject):
     def lowercase_name(self)->str:
         return self.name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower().replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower()
 
+    
+    # This is a slightly modified version of the get_conf_stanza_name function from
+    # SecurityContentObject_Abstract
+    def get_response_task_name(self, app:CustomApp, max_stanza_length:int=CONTENTCTL_MAX_STANZA_LENGTH)->str:
+        stanza_name = CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name)
+        if len(stanza_name) > max_stanza_length:
+            raise ValueError(f"conf stanza may only be {max_stanza_length} characters, "
+                             f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' ")
+        return stanza_name
+    
 
     @model_serializer
     def serialize_model(self):
@@ -66,12 +79,7 @@ class Investigation(SecurityContentObject):
 
 
     def model_post_init(self, ctx:dict[str,Any]):
-        # director: Optional[DirectorOutputDto] = ctx.get("output_dto",None)
-        # if not isinstance(director,DirectorOutputDto):
-        #     raise ValueError("DirectorOutputDto was not passed in context of Detection model_post_init")
-        director: Optional[DirectorOutputDto] = ctx.get("output_dto",None)
+        # Ensure we link all stories this investigation references
+        # back to itself
         for story in self.tags.analytic_story:
             story.investigations.append(self)
-    
-
-    

contentctl/objects/investigation_tags.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
+from typing import List
 from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer
 from contentctl.objects.story import Story
 from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain
 
 class InvestigationTags(BaseModel):
-    analytic_story: list[Story] = Field([],min_length=1)
-    product: list[SecurityContentInvestigationProductName] = Field(...,min_length=1)
-    required_fields: list[str] = Field(min_length=1)
+    analytic_story: List[Story] = Field([],min_length=1)
+    product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1)
+    required_fields: List[str] = Field(min_length=1)
     security_domain: SecurityDomain = Field(...)
 
 

contentctl/objects/lookup.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
-from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer
+from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt
 from typing import TYPE_CHECKING, Optional, Any, Union
 import re
 import csv
+import uuid
+import datetime
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
     from contentctl.objects.config import validate
@@ -32,6 +34,11 @@ class Lookup(SecurityContentObject):
     match_type: Optional[str] = None
     min_matches: Optional[int] = None
     case_sensitive_match: Optional[bool] = None
+    # TODO: Add id field to all lookup ymls
+    id: uuid.UUID = Field(default_factory=uuid.uuid4) 
+    date: datetime.date = Field(datetime.date.today())
+    author: str = Field("NO AUTHOR DEFINED",max_length=255)
+    version: NonNegativeInt = 1
     
 
     @model_serializer

contentctl/objects/macro.py

@@ -3,12 +3,13 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING, List
 import re
-from pydantic import Field, model_serializer
+from pydantic import Field, model_serializer, NonNegativeInt
+import uuid
+import datetime
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
 from contentctl.objects.security_content_object import SecurityContentObject
 
-
 #The following macros are included in commonly-installed apps.
 #As such, we will ignore if they are missing from our app.
 #Included in 
@@ -22,7 +23,11 @@ MACROS_TO_IGNORE.add("cim_corporate_web_domain_search") #Part of CIM/Splunk_SA_C
 class Macro(SecurityContentObject):
     definition: str = Field(..., min_length=1)
     arguments: List[str] = Field([])
-    
+    # TODO: Add id field to all macro ymls
+    id: uuid.UUID = Field(default_factory=uuid.uuid4)
+    date: datetime.date = Field(datetime.date.today())
+    author: str = Field("NO AUTHOR DEFINED",max_length=255)
+    version: NonNegativeInt = 1
     
 
 
@@ -49,10 +54,15 @@ class Macro(SecurityContentObject):
         #If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here````
         #then there is a small edge case where the regex below does not work properly.  If that is 
         #the case, we edit the search slightly to insert a space
-        text_field = re.sub(r"\`\`\`\`", r"` ```", text_field)
-        text_field = re.sub(r"\`\`\`.*?\`\`\`", " ", text_field)
-        
+        if re.findall(r"\`\`\`\`", text_field):
+            raise ValueError("Search contained four or more '`' characters in a row which is invalid SPL"
+                            "This may have occurred when a macro was commented out.\n"
+                            "Please ammend your search to remove the substring '````'")
 
+        # replace all the macros with a space    
+        text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field) 
+        
+        
         macros_to_get = re.findall(r'`([^\s]+)`', text_field)
         #If macros take arguments, stop at the first argument.  We just want the name of the macro
         macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get])
@@ -62,4 +72,3 @@ class Macro(SecurityContentObject):
         macros_to_get -= macros_to_ignore
         return Macro.mapNamesToSecurityContentObjects(list(macros_to_get), director)
         
-    

contentctl/objects/notable_event.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from pydantic import ConfigDict, BaseModel
 
 from contentctl.objects.detection import Detection
 
@@ -11,10 +11,11 @@ class NotableEvent(BaseModel):
     # The search ID that found that generated this risk event
     orig_sid: str
 
-    class Config:
-        # Allowing fields that aren't explicitly defined to be passed since some of the risk event's
-        # fields vary depending on the SPL which generated them
-        extra = 'allow'
+    # Allowing fields that aren't explicitly defined to be passed since some of the risk event's
+    # fields vary depending on the SPL which generated them
+    model_config = ConfigDict(
+        extra='allow'
+    )
 
     def validate_against_detection(self, detection: Detection) -> None:
         raise NotImplementedError()

contentctl/objects/risk_analysis_action.py

@@ -1,7 +1,7 @@
 from typing import Any
 import json
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 
 from contentctl.objects.risk_object import RiskObject
 from contentctl.objects.threat_object import ThreatObject
@@ -21,11 +21,11 @@ class RiskAnalysisAction(BaseModel):
     risk_objects: list[RiskObject]
     message: str
 
-    @validator("message", always=True, pre=True)
+    @field_validator("message", mode="before")
     @classmethod
-    def _validate_message(cls, v, values) -> str:
+    def _validate_message(cls, v: Any) -> str:
         """
-        Validate splunk_path and derive if None
+        Validate message and derive if None
         """
         if v is None:
             raise ValueError(

contentctl/objects/risk_event.py

@@ -1,7 +1,7 @@
 import re
+from functools import cached_property
 
-from pydantic import BaseModel, Field, PrivateAttr, field_validator, computed_field
-
+from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field
 from contentctl.objects.errors import ValidationFailed
 from contentctl.objects.detection import Detection
 from contentctl.objects.observable import Observable
@@ -85,10 +85,11 @@ class RiskEvent(BaseModel):
     # Private attribute caching the observable this RiskEvent is mapped to
     _matched_observable: Observable | None = PrivateAttr(default=None)
 
-    class Config:
-        # Allowing fields that aren't explicitly defined to be passed since some of the risk event's
-        # fields vary depending on the SPL which generated them
-        extra = "allow"
+    # Allowing fields that aren't explicitly defined to be passed since some of the risk event's
+    # fields vary depending on the SPL which generated them
+    model_config = ConfigDict(
+        extra="allow"
+    )
 
     @field_validator("annotations_mitre_attack", "analyticstories", mode="before")
     @classmethod
@@ -103,7 +104,7 @@ class RiskEvent(BaseModel):
             return [v]
 
     @computed_field
-    @property
+    @cached_property
     def source_field_name(self) -> str:
         """
         A cached derivation of the source field name the risk event corresponds to in the relevant

contentctl/objects/savedsearches_conf.py

@@ -0,0 +1,196 @@
+
+from pathlib import Path
+from typing import Any, ClassVar
+import re
+import tempfile
+import tarfile
+
+from pydantic import BaseModel, Field, PrivateAttr
+
+from contentctl.objects.detection_stanza import DetectionStanza
+
+
+class SavedsearchesConf(BaseModel):
+    """
+    A model of the savedsearches.conf file, represented as a set of stanzas
+
+    NOTE: At present, this model only parses the detections themselves from the .conf; thing like
+    baselines or response tasks are left alone currently
+    """
+    # The path to the conf file
+    path: Path = Field(...)
+
+    # The app label (used for pattern matching in the conf) (e.g. ESCU)
+    app_label: str = Field(...)
+
+    # A dictionary mapping rule names to a model of the corresponding stanza in the conf
+    detection_stanzas: dict[str, DetectionStanza] = Field(default={}, init=False)
+
+    # A internal flag indicating whether we are currently in the detections portion of the conf
+    # during parsing
+    _in_detections: bool = PrivateAttr(default=False)
+
+    # A internal flag indicating whether we are currently in a specific section of the conf
+    # during parsing
+    _in_section: bool = PrivateAttr(default=False)
+
+    # A running list of the accumulated lines identified as part of the current section
+    _current_section_lines: list[str] = PrivateAttr(default=[])
+
+    # The name of the current section
+    _current_section_name: str | None = PrivateAttr(default=None)
+
+    # The current line number as we continue to parse the file
+    _current_line_no: int = PrivateAttr(default=0)
+
+    # A format string for the path to the savedsearches.conf in the app package
+    PACKAGE_CONF_PATH_FMT_STR: ClassVar[str] = "{appid}/default/savedsearches.conf"
+
+    def model_post_init(self, __context: Any) -> None:
+        super().model_post_init(__context)
+        self._parse_detection_stanzas()
+
+    def is_section_header(self, line: str) -> bool:
+        """
+        Given a line, determine if the line is a section header, indicating the start of a new
+        section
+
+        :param line: a line from the conf file
+        :type line: str
+
+        :returns: a bool indicating whether the current line is a section header or not
+        :rtype: bool
+        """
+        # Compile the pattern based on the app name
+        pattern = re.compile(r"\[" + self.app_label + r" - .+ - Rule\]")
+        if pattern.match(line):
+            return True
+        return False
+
+    def section_start(self, line: str) -> None:
+        """
+        Given a line, adjust the state to track a new section
+
+        :param line: a line from the conf file
+        :type line: str
+        """
+        # Determine the new section name:
+        new_section_name = line.strip().strip("[").strip("]")
+
+        # Raise if we are in a section already according to the state (we cannot statr a new section
+        # before ending the previous section)
+        if self._in_section:
+            raise Exception(
+                "Attempting to start a new section w/o ending the current one; check for "
+                f"parsing/serialization errors: (current section: '{self._current_section_name}', "
+                f"new section: '{new_section_name}') [see line {self._current_line_no} in "
+                f"{self.path}]"
+            )
+
+        # Capture the name of this section, reset the lines, and indicate that we are now in a
+        # section
+        self._current_section_name = new_section_name
+        self._current_section_lines = [line]
+        self._in_section = True
+
+    def section_end(self) -> None:
+        """
+        Adjust the state end the section we were enumerating; parse the lines as a DetectionStanza
+        """
+        # Name should have been set during section start
+        if self._current_section_name is None:
+            raise Exception(
+                "Name for the current section was never set; check for parsing/serialization "
+                f"errors [see line {self._current_line_no} in {self.path}]."
+            )
+        elif self._current_section_name in self.detection_stanzas:
+            # Each stanza should be unique, so the name should not already be in the dict
+            raise Exception(
+                f"Name '{self._current_section_name}' already in set of stanzas [see line "
+                f"{self._current_line_no} in {self.path}]."
+            )
+
+        # Build the stanza model from the accumulated lines and adjust the state to end this section
+        self.detection_stanzas[self._current_section_name] = DetectionStanza(
+            name=self._current_section_name,
+            lines=self._current_section_lines
+        )
+        self._in_section = False
+
+    def _parse_detection_stanzas(self) -> None:
+        """
+        Open the conf file, and parse out DetectionStanza objects from the raw conf stanzas
+        """
+        # We don't want to parse the stanzas twice (non-atomic operation)
+        if len(self.detection_stanzas) != 0:
+            raise Exception(
+                f"{len(self.detection_stanzas)} stanzas have already been parsed from this conf; we"
+                " do not need to parse them again"
+            )
+
+        # Open the conf file and iterate over the lines
+        with open(self.path, "r") as file:
+            for line in file:
+                self._current_line_no += 1
+
+                # Break when we get to the end of the app detections
+                if line.strip() == f"### END {self.app_label} DETECTIONS ###":
+                    break
+                elif self._in_detections:
+                    # Check if we are in the detections portion of the conf, and then if we are in a
+                    # section
+                    if self._in_section:
+                        # If we are w/in a section and have hit an empty line, close the section
+                        if line.strip() == "":
+                            self.section_end()
+                        elif self.is_section_header(line):
+                            # Raise if we encounter a section header w/in a section
+                            raise Exception(
+                                "Encountered section header while already in section (current "
+                                f"section: '{self._current_section_name}') [see line "
+                                f"{self._current_line_no} in {self.path}]."
+                            )
+                        else:
+                            # Otherwise, append the line
+                            self._current_section_lines.append(line)
+                    elif self.is_section_header(line):
+                        # If we encounter a section header while not already in a section, start a
+                        # new one
+                        self.section_start(line)
+                    elif line.strip() != "":
+                        # If we are not in a section and have encountered anything other than an
+                        # empty line, something is wrong
+                        raise Exception(
+                            "Found a non-empty line outside a stanza [see line "
+                            f"{self._current_line_no} in {self.path}]."
+                        )
+                elif line.strip() == f"### {self.app_label} DETECTIONS ###":
+                    # We have hit the detections portion of the conf and we adjust the state
+                    # accordingly
+                    self._in_detections = True
+
+    @staticmethod
+    def init_from_package(package_path: Path, app_name: str, appid: str) -> "SavedsearchesConf":
+        """
+        Alternate constructor which can take an app package, and extract the savedsearches.conf from
+        a temporary file.
+
+        :param package_path: Path to the app package
+        :type package_path: :class:`pathlib.Path`
+        :param app_name: the name of the app (e.g. ESCU)
+        :type app_name: str
+
+        :returns: a SavedsearchesConf object
+        :rtype: :class:`contentctl.objects.savedsearches_conf.SavedsearchesConf`
+        """
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Open the tar/gzip archive
+            with tarfile.open(package_path) as package:
+                # Extract the savedsearches.conf and use it to init the model
+                package_conf_path = SavedsearchesConf.PACKAGE_CONF_PATH_FMT_STR.format(appid=appid)
+                package.extract(package_conf_path, path=tmpdir)
+                return SavedsearchesConf(
+                    path=Path(tmpdir, package_conf_path),
+                    app_label=app_name
+                )

contentctl/objects/story.py

@@ -8,26 +8,14 @@ if TYPE_CHECKING:
     from contentctl.objects.investigation import Investigation
     from contentctl.objects.baseline import Baseline
     from contentctl.objects.data_source import DataSource
+    from contentctl.objects.config import CustomApp
 
 from contentctl.objects.security_content_object import SecurityContentObject
 
-
-
-
-
-#from contentctl.objects.investigation import Investigation
-
-
-
 class Story(SecurityContentObject):
     narrative: str = Field(...)
     tags: StoryTags = Field(...)
 
-    # enrichments
-    #detection_names: List[str] = []
-    #investigation_names: List[str] = []
-    #baseline_names: List[str] = []
-
     # These are updated when detection and investigation objects are created.
     # Specifically in the model_post_init functions
     detections:List[Detection] = []
@@ -46,9 +34,9 @@ class Story(SecurityContentObject):
         return sorted(list(data_source_objects))
 
 
-    def storyAndInvestigationNamesWithApp(self, app_name:str)->List[str]:
-        return [f"{app_name} - {name} - Rule" for name in self.detection_names] + \
-               [f"{app_name} - {name} - Response Task" for name in self.investigation_names]
+    def storyAndInvestigationNamesWithApp(self, app:CustomApp)->List[str]:
+        return [detection.get_conf_stanza_name(app) for detection in self.detections] + \
+               [investigation.get_response_task_name(app) for investigation in self.investigations]
         
     @model_serializer
     def serialize_model(self):