contentctl 4.3.5__py3-none-any.whl → 4.4.1__py3-none-any.whl

contentctl/input/new_content_questions.py

@@ -1,7 +1,11 @@
+from typing import Any
+from contentctl.objects.enums import DataSource
+
+
 class NewContentQuestions:
 
     @classmethod
-    def get_questions_detection(self) -> list:
+    def get_questions_detection(cls) -> list[dict[str,Any]]:
         questions = [
             {
                 "type": "text",
@@ -45,46 +49,9 @@ class NewContentQuestions:
                 'type': 'checkbox',
                 'message': 'Your data source',
                 'name': 'data_source',
-                'choices': [
-                    "OSQuery ES Process Events",
-                    "Powershell 4104",
-                    "Sysmon Event ID 1",
-                    "Sysmon Event ID 3",
-                    "Sysmon Event ID 5",
-                    "Sysmon Event ID 6",
-                    "Sysmon Event ID 7",
-                    "Sysmon Event ID 8",
-                    "Sysmon Event ID 9",
-                    "Sysmon Event ID 10",
-                    "Sysmon Event ID 11",
-                    "Sysmon Event ID 13",
-                    "Sysmon Event ID 15",
-                    "Sysmon Event ID 20",
-                    "Sysmon Event ID 21",
-                    "Sysmon Event ID 22",
-                    "Sysmon Event ID 23",
-                    "Windows Security 4624",
-                    "Windows Security 4625",
-                    "Windows Security 4648",
-                    "Windows Security 4663",
-                    "Windows Security 4688",
-                    "Windows Security 4698",
-                    "Windows Security 4703",
-                    "Windows Security 4720",
-                    "Windows Security 4732",
-                    "Windows Security 4738",
-                    "Windows Security 4741",
-                    "Windows Security 4742",
-                    "Windows Security 4768",
-                    "Windows Security 4769",
-                    "Windows Security 4771",
-                    "Windows Security 4776",
-                    "Windows Security 4781",
-                    "Windows Security 4798",
-                    "Windows Security 5136",
-                    "Windows Security 5145",
-                    "Windows System 7045"
-                ]
+                #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory
+                'choices': sorted(DataSource._value2member_map_ )
+                
             },
             {
                 "type": "text",
@@ -116,7 +83,7 @@ class NewContentQuestions:
         return questions
 
     @classmethod
-    def get_questions_story(self) -> list:
+    def get_questions_story(cls)-> list[dict[str,Any]]:
         questions = [
             {
                 "type": "text",

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -20,7 +20,8 @@ from contentctl.objects.lookup import Lookup
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
     from contentctl.objects.baseline import Baseline
-
+    from contentctl.objects.config import CustomApp
+    
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.enums import AnalyticsType
 from contentctl.objects.enums import DataModel
@@ -35,11 +36,17 @@ from contentctl.objects.test_group import TestGroup
 from contentctl.objects.integration_test import IntegrationTest
 from contentctl.objects.data_source import DataSource
 from contentctl.objects.base_test_result import TestResultStatus
-
-# from contentctl.objects.playbook import Playbook
+from contentctl.objects.drilldown import Drilldown, DRILLDOWN_SEARCH_PLACEHOLDER
 from contentctl.objects.enums import ProvidingTechnology
 from contentctl.enrichments.cve_enrichment import CveEnrichmentObj
 import datetime
+from contentctl.objects.constants import (
+    ES_MAX_STANZA_LENGTH,
+    ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE,
+    CONTENTCTL_MAX_SEARCH_NAME_LENGTH,
+    CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE
+)
+
 MISSING_SOURCES: set[str] = set()
 
 # Those AnalyticsTypes that we do not test via contentctl
@@ -51,8 +58,8 @@ SKIPPED_ANALYTICS_TYPES: set[str] = {
 # TODO (#266): disable the use_enum_values configuration
 class Detection_Abstract(SecurityContentObject):
     model_config = ConfigDict(use_enum_values=True)
-
-    # contentType: SecurityContentType = SecurityContentType.detections
+    name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
+    #contentType: SecurityContentType = SecurityContentType.detections
     type: AnalyticsType = Field(...)
     status: DetectionStatus = Field(...)
     data_source: list[str] = []
@@ -60,6 +67,16 @@ class Detection_Abstract(SecurityContentObject):
     search: str = Field(...)
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
+    explanation: None | str = Field(
+        default=None,
+        exclude=True, #Don't serialize this value when dumping the object
+        description="Provide an explanation to be included "
+        "in the 'Explanation' field of the Detection in "
+        "the Use Case Library. If this field is not "
+        "defined in the YML, it will default to the "
+        "value of the 'description' field when " 
+        "serialized in analyticstories_detections.j2",
+    )
 
     enabled_by_default: bool = False
     file_path: FilePath = Field(...)
@@ -70,9 +87,30 @@ class Detection_Abstract(SecurityContentObject):
     # https://github.com/pydantic/pydantic/issues/9101#issuecomment-2019032541
     tests: List[Annotated[Union[UnitTest, IntegrationTest, ManualTest], Field(union_mode='left_to_right')]] = []
     # A list of groups of tests, relying on the same data
-    test_groups: Union[list[TestGroup], None] = Field(None, validate_default=True)
+    test_groups: list[TestGroup] = []
 
     data_source_objects: list[DataSource] = []
+    drilldown_searches: list[Drilldown] = Field(default=[], description="A list of Drilldowns that should be included with this search")
+
+    def get_conf_stanza_name(self, app:CustomApp)->str:
+        stanza_name = CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name)
+        self.check_conf_stanza_max_length(stanza_name)
+        return stanza_name
+    
+
+    def get_action_dot_correlationsearch_dot_label(self, app:CustomApp, max_stanza_length:int=ES_MAX_STANZA_LENGTH)->str:
+        stanza_name = self.get_conf_stanza_name(app)
+        stanza_name_after_saving_in_es = ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format(
+            security_domain_value = self.tags.security_domain.value, 
+            search_name = stanza_name
+            )
+        
+    
+        if len(stanza_name_after_saving_in_es) > max_stanza_length:
+            raise ValueError(f"label may only be {max_stanza_length} characters to allow updating in-product, "
+                             f"but stanza was actually {len(stanza_name_after_saving_in_es)} characters: '{stanza_name_after_saving_in_es}' ")
+        
+        return stanza_name    
 
     @field_validator("search", mode="before")
     @classmethod
@@ -130,6 +168,7 @@ class Detection_Abstract(SecurityContentObject):
         the model from the list of unit tests. Also, preemptively skips all manual tests, as well as
         tests for experimental/deprecated detections and Correlation type detections.
         """
+        
         # Since ManualTest and UnitTest are not differentiable without looking at the manual_test
         # tag, Pydantic builds all tests as UnitTest objects. If we see the manual_test flag, we
         # convert these to ManualTest
@@ -248,6 +287,7 @@ class Detection_Abstract(SecurityContentObject):
             annotations_dict["cve"] = self.tags.cve
         annotations_dict["impact"] = self.tags.impact
         annotations_dict["type"] = self.type
+        annotations_dict["type_list"] = [self.type]
         # annotations_dict["version"] = self.version
 
         annotations_dict["data_source"] = self.data_source
@@ -518,13 +558,53 @@ class Detection_Abstract(SecurityContentObject):
         self.data_source_objects = matched_data_sources
 
         for story in self.tags.analytic_story:
-            story.detections.append(self)
+            story.detections.append(self)     
 
         self.cve_enrichment_func(__context)
 
         # Derive TestGroups and IntegrationTests, adjust for ManualTests, skip as needed
         self.adjust_tests_and_groups()
 
+        # Ensure that if there is at least 1 drilldown, at least
+        # 1 of the drilldowns contains the string Drilldown.SEARCH_PLACEHOLDER.
+        # This is presently a requirement when 1 or more drilldowns are added to a detection.
+        # Note that this is only required for production searches that are not hunting
+            
+        if self.type == AnalyticsType.Hunting.value or self.status != DetectionStatus.production.value:
+            #No additional check need to happen on the potential drilldowns.
+            pass
+        else:
+            found_placeholder = False
+            if len(self.drilldown_searches) < 2:
+                raise ValueError(f"This detection is required to have 2 drilldown_searches, but only has [{len(self.drilldown_searches)}]")
+            for drilldown in self.drilldown_searches:
+                if DRILLDOWN_SEARCH_PLACEHOLDER in drilldown.search:
+                    found_placeholder = True
+            if not found_placeholder:
+                raise ValueError("Detection has one or more drilldown_searches, but none of them "
+                                 f"contained '{DRILLDOWN_SEARCH_PLACEHOLDER}. This is a requirement "
+                                 "if drilldown_searches are defined.'")
+            
+        # Update the search fields with the original search, if required
+        for drilldown in self.drilldown_searches:
+            drilldown.perform_search_substitutions(self)
+
+        #For experimental purposes, add the default drilldowns
+        #self.drilldown_searches.extend(Drilldown.constructDrilldownsFromDetection(self))
+
+    @property
+    def drilldowns_in_JSON(self) -> list[dict[str,str]]:
+        """This function is required for proper JSON 
+        serializiation of drilldowns to occur in savedsearches.conf.
+        It returns the list[Drilldown] as a list[dict].
+        Without this function, the jinja template is unable
+        to convert list[Drilldown] to JSON
+
+        Returns:
+            list[dict[str,str]]: List of Drilldowns dumped to dict format
+        """        
+        return [drilldown.model_dump() for drilldown in self.drilldown_searches]
+
     @field_validator('lookups', mode="before")
     @classmethod
     def getDetectionLookups(cls, v:list[str], info:ValidationInfo) -> list[Lookup]:
@@ -653,6 +733,27 @@ class Detection_Abstract(SecurityContentObject):
         else:
             self.tags.nist = [NistCategory.DE_AE]
         return self
+        
+
+    @model_validator(mode="after")
+    def ensureThrottlingFieldsExist(self):
+        '''
+        For throttling to work properly, the fields to throttle on MUST
+        exist in the search itself.  If not, then we cannot apply the throttling
+        '''
+        if self.tags.throttling is None:
+            # No throttling configured for this detection
+            return self
+
+        missing_fields:list[str] = [field for field in self.tags.throttling.fields if field not in self.search]
+        if len(missing_fields) > 0:
+            raise ValueError(f"The following throttle fields were missing from the search: {missing_fields}")
+
+        else:
+            # All throttling fields present in search
+            return self
+            
+
 
     @model_validator(mode="after")
     def ensureProperObservablesExist(self):
@@ -730,6 +831,45 @@ class Detection_Abstract(SecurityContentObject):
         # Found everything
         return self
 
+    @field_validator("tests", mode="before")
+    def ensure_yml_test_is_unittest(cls, v:list[dict]):
+        """The typing for the tests field allows it to be one of
+        a number of different types of tests. However, ONLY
+        UnitTest should be allowed to be defined in the YML
+        file.  If part of the UnitTest defined in the YML
+        is incorrect, such as the attack_data file, then
+        it will FAIL to be instantiated as a UnitTest and
+        may instead be instantiated as a different type of
+        test, such as IntegrationTest (since that requires
+        less fields) which is incorrect. Ensure that any
+        raw data read from the YML can actually construct
+        a valid UnitTest and, if not, return errors right
+        away instead of letting Pydantic try to construct
+        it into a different type of test
+
+        Args:
+            v (list[dict]): list of dicts read from the yml. 
+            Each one SHOULD be a valid UnitTest. If we cannot
+            construct a valid unitTest from it, a ValueError should be raised
+
+        Returns:
+            _type_: The input of the function, assuming no 
+            ValueError is raised.
+        """        
+        valueErrors:list[ValueError] = []
+        for unitTest in v:
+            #This raises a ValueError on a failed UnitTest.
+            try:
+                UnitTest.model_validate(unitTest)
+            except ValueError as e:
+                valueErrors.append(e)
+        if len(valueErrors):
+            raise ValueError(valueErrors)
+        # All of these can be constructred as UnitTests with no
+        # Exceptions, so let the normal flow continue
+        return v
+        
+
     @field_validator("tests")
     def tests_validate(
         cls,

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -5,8 +5,10 @@ if TYPE_CHECKING:
     from contentctl.objects.deployment import Deployment
     from contentctl.objects.security_content_object import SecurityContentObject
     from contentctl.input.director import DirectorOutputDto
+    from contentctl.objects.config import CustomApp
 
 from contentctl.objects.enums import AnalyticsType
+from contentctl.objects.constants import CONTENTCTL_MAX_STANZA_LENGTH
 import abc
 import uuid
 import datetime
@@ -31,14 +33,14 @@ NO_FILE_NAME = "NO_FILE_NAME"
 
 # TODO (#266): disable the use_enum_values configuration
 class SecurityContentObject_Abstract(BaseModel, abc.ABC):
-    model_config = ConfigDict(use_enum_values=True, validate_default=True)
-
-    name: str = Field(...)
-    author: str = Field("Content Author", max_length=255)
-    date: datetime.date = Field(datetime.date.today())
-    version: NonNegativeInt = 1
-    id: uuid.UUID = Field(default_factory=uuid.uuid4)  # we set a default here until all content has a uuid
-    description: str = Field("Enter Description Here", max_length=10000)
+    model_config = ConfigDict(use_enum_values=True,validate_default=True)
+    
+    name: str = Field(...,max_length=99)
+    author: str = Field(...,max_length=255)
+    date: datetime.date = Field(...)
+    version: NonNegativeInt = Field(...)
+    id: uuid.UUID = Field(...) #we set a default here until all content has a uuid
+    description: str = Field(...,max_length=10000)
     file_path: Optional[FilePath] = None
     references: Optional[List[HttpUrl]] = None
 
@@ -56,7 +58,13 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
             "description": self.description,
             "references": [str(url) for url in self.references or []]
         }
-
+    
+    
+    def check_conf_stanza_max_length(self, stanza_name:str, max_stanza_length:int=CONTENTCTL_MAX_STANZA_LENGTH) -> None:
+        if len(stanza_name) > max_stanza_length:
+            raise ValueError(f"conf stanza may only be {max_stanza_length} characters, "
+                             f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' ")
+    
     @staticmethod
     def objectListToNameList(objects: list[SecurityContentObject]) -> list[str]:
         return [object.getName() for object in objects]

contentctl/objects/base_test_result.py

@@ -1,8 +1,8 @@
 from typing import Union, Any
 from enum import Enum
 
-from pydantic import BaseModel
-from splunklib.data import Record
+from pydantic import ConfigDict, BaseModel
+from splunklib.data import Record                                                                   # type: ignore
 
 from contentctl.helper.utils import Utils
 
@@ -53,11 +53,11 @@ class BaseTestResult(BaseModel):
     # The Splunk endpoint URL
     sid_link: Union[None, str] = None
 
-    class Config:
-        validate_assignment = True
-
-        # Needed to allow for embedding of Exceptions in the model
-        arbitrary_types_allowed = True
+    # Needed to allow for embedding of Exceptions in the model
+    model_config = ConfigDict(
+        validate_assignment=True,
+        arbitrary_types_allowed=True
+    )
 
     @property
     def passed(self) -> bool:

contentctl/objects/baseline.py

@@ -1,33 +1,21 @@
 
 from __future__ import annotations
-from typing import TYPE_CHECKING, Annotated, Optional, List,Any
+from typing import Annotated, Optional, List,Any
 from pydantic import field_validator, ValidationInfo, Field, model_serializer
-if TYPE_CHECKING:
-    from contentctl.input.director import DirectorOutputDto
-
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.enums import DataModel, AnalyticsType
+from contentctl.objects.enums import DataModel
 from contentctl.objects.baseline_tags import BaselineTags
-from contentctl.objects.enums import DeploymentType
-#from contentctl.objects.deployment import Deployment
 
-# from typing import TYPE_CHECKING
-# if TYPE_CHECKING:
-#     from contentctl.input.director import DirectorOutputDto
+from contentctl.objects.config import CustomApp
+
 
+from contentctl.objects.constants import CONTENTCTL_MAX_SEARCH_NAME_LENGTH,CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE
 
 class Baseline(SecurityContentObject):
-    # baseline spec
-    #name: str
-    #id: str
-    #version: int
-    #date: str
-    #author: str
-    #contentType: SecurityContentType = SecurityContentType.baselines
+    name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
     type: Annotated[str,Field(pattern="^Baseline$")] = Field(...)
     datamodel: Optional[List[DataModel]] = None
-    #description: str
     search: str = Field(..., min_length=4)
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
@@ -35,6 +23,12 @@ class Baseline(SecurityContentObject):
 
     # enrichment
     deployment: Deployment = Field({})
+    
+
+    def get_conf_stanza_name(self, app:CustomApp)->str:
+        stanza_name = CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name)
+        self.check_conf_stanza_max_length(stanza_name)
+        return stanza_name
 
     @field_validator("deployment", mode="before")
     def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment:

contentctl/objects/baseline_tags.py

@@ -1,15 +1,12 @@
 from __future__ import annotations
-from typing import TYPE_CHECKING
 from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer
 from typing import List, Any, Union
 
 from contentctl.objects.story import Story
-from contentctl.objects.deployment import Deployment
 from contentctl.objects.detection import Detection
 from contentctl.objects.enums import SecurityContentProductName
 from contentctl.objects.enums import SecurityDomain
-if TYPE_CHECKING:
-    from contentctl.input.director import DirectorOutputDto
+
 
 
 
@@ -19,7 +16,7 @@ class BaselineTags(BaseModel):
     #deployment: Deployment = Field('SET_IN_GET_DEPLOYMENT_FUNCTION')
     # TODO (#223): can we remove str from the possible types here?
     detections: List[Union[Detection,str]] = Field(...)
-    product: list[SecurityContentProductName] = Field(...,min_length=1)
+    product: List[SecurityContentProductName] = Field(...,min_length=1)
     required_fields: List[str] = Field(...,min_length=1)
     security_domain: SecurityDomain = Field(...)
 

contentctl/objects/config.py

@@ -158,8 +158,7 @@ class CustomApp(App_Base):
                                 str(destination), 
                                 verbose_print=True)
         return str(destination)
-
-
+    
 # TODO (#266): disable the use_enum_values configuration
 class Config_Base(BaseModel):
     model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
@@ -287,7 +286,6 @@ class build(validate):
 
     def getAppTemplatePath(self)->pathlib.Path:
         return self.path/"app_template"
-    
 
 
 class StackType(StrEnum):
@@ -310,6 +308,16 @@ class inspect(build):
             "should be enabled."
         )
     )
+    suppress_missing_content_exceptions: bool = Field(
+        default=False,
+        description=(
+            "Suppress exceptions during metadata validation if a detection that existed in "
+            "the previous build does not exist in this build. This is to ensure that content "
+            "is not accidentally removed. In order to support testing both public and private "
+            "content, this warning can be suppressed. If it is suppressed, it will still be "
+            "printed out as a warning."
+        )
+    )
     enrichments: bool = Field(
         default=True,
         description=(
@@ -951,15 +959,15 @@ class test_servers(test_common):
                 index+=1
 
 
-        
 class release_notes(Config_Base):
     old_tag:Optional[str] = Field(None, description="Name of the tag to diff against to find new content. "
                                           "If it is not supplied, then it will be inferred as the "
                                           "second newest tag at runtime.")
     new_tag:Optional[str] = Field(None, description="Name of the tag containing new content. If it is not supplied,"
                                           " then it will be inferred as the newest tag at runtime.")
-    latest_branch:Optional[str] = Field(None, description="Branch for which we are generating release notes")
-    
+    latest_branch:Optional[str] = Field(None, description="Branch name for which we are generating release notes for")
+    compare_against:Optional[str] = Field(default="develop", description="Branch name for which we are comparing the files changes against")
+
     def releaseNotesFilename(self, filename:str)->pathlib.Path:
         #Assume that notes are written to dist/. This does not respect build_dir since that is
         #only a member of build
@@ -1033,6 +1041,4 @@ class release_notes(Config_Base):
     #             raise ValueError("The latest_branch '{self.latest_branch}' was not found in the repository")
         
         
-    #     return self
-
-
+    #     return self

contentctl/objects/constants.py

@@ -1,3 +1,5 @@
+# Use for calculation of maximum length of name field
+from contentctl.objects.enums import SecurityDomain
 
 ATTACK_TACTICS_KILLCHAIN_MAPPING = {
     "Reconnaissance": "Reconnaissance",
@@ -140,3 +142,31 @@ RBA_OBSERVABLE_ROLE_MAPPING = {
 
 # The relative path to the directory where any apps/packages will be downloaded
 DOWNLOADS_DIRECTORY = "downloads"
+
+# Maximum length of the name field for a search.
+# This number is derived from a limitation that exists in 
+# ESCU where a search cannot be edited, due to validation
+# errors, if its name is longer than 99 characters.
+# When an saved search is cloned in Enterprise Security User Interface,
+# it is wrapped in the following: 
+# {Detection.tags.security_domain.value} - {SEARCH_STANZA_NAME} - Rule
+# Similarly, when we generate the search stanza name in contentctl, it
+# is app.label - detection.name - Rule
+# However, in product the search name is:
+# {CustomApp.label} - {detection.name} - Rule,
+# or in ESCU:
+# ESCU - {detection.name} - Rule,
+# this gives us a maximum length below.
+# When an ESCU search is cloned, it will 
+# have a full name like (the following is NOT a typo):
+# Endpoint - ESCU - Name of Search From YML File - Rule - Rule
+# The math below accounts for all these caveats
+ES_MAX_STANZA_LENGTH = 99
+CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name} - Rule"
+CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name}"
+CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name} - Response Task"
+
+ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE = "{security_domain_value} - {search_name} - Rule"
+SECURITY_DOMAIN_MAX_LENGTH = max([len(SecurityDomain[value]) for value in SecurityDomain._member_map_])
+CONTENTCTL_MAX_STANZA_LENGTH = ES_MAX_STANZA_LENGTH - len(ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format(security_domain_value="X"*SECURITY_DOMAIN_MAX_LENGTH,search_name=""))
+CONTENTCTL_MAX_SEARCH_NAME_LENGTH = CONTENTCTL_MAX_STANZA_LENGTH - len(CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(app_label="ESCU", detection_name=""))