contentctl 4.3.4__py3-none-any.whl → 4.4.0__py3-none-any.whl

contentctl/objects/config.py

@@ -1,26 +1,31 @@
 from __future__ import annotations
+
+from os import environ
+from datetime import datetime, UTC
+from typing import Optional, Any, List, Union, Self
+import random
+from enum import StrEnum, auto
+import pathlib
+from urllib.parse import urlparse
+from abc import ABC, abstractmethod
+from functools import partialmethod
+
+import tqdm
+import semantic_version
 from pydantic import (
     BaseModel, Field, field_validator, 
     field_serializer, ConfigDict, DirectoryPath,
     PositiveInt, FilePath, HttpUrl, AnyUrl, model_validator,
     ValidationInfo
 )
+
+from contentctl.objects.constants import DOWNLOADS_DIRECTORY
 from contentctl.output.yml_writer import YmlWriter
-from os import environ
-from datetime import datetime, UTC
-from typing import Optional,Any,Annotated,List,Union, Self
-import semantic_version
-import random
-from enum import StrEnum, auto
-import pathlib
 from contentctl.helper.utils import Utils
-from urllib.parse import urlparse
-from abc import ABC, abstractmethod
 from contentctl.objects.enums import PostTestBehavior, DetectionTestingMode
 from contentctl.objects.detection import Detection
 from contentctl.objects.annotated_types import APPID_TYPE
-import tqdm
-from functools import partialmethod
+from contentctl.helper.splunk_app import SplunkApp
 
 ENTERPRISE_SECURITY_UID = 263
 COMMON_INFORMATION_MODEL_UID = 1621
@@ -153,8 +158,7 @@ class CustomApp(App_Base):
                                 str(destination), 
                                 verbose_print=True)
         return str(destination)
-
-
+    
 # TODO (#266): disable the use_enum_values configuration
 class Config_Base(BaseModel):
     model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
@@ -171,7 +175,13 @@ class Config_Base(BaseModel):
         return str(path)
 
 class init(Config_Base):
-    pass
+    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
+    bare: bool = Field(default=False, description="contentctl normally provides some some example content "
+                       "(macros, stories, data_sources, and/or analytic stories).  This option disables "
+                       "initialization with that additional contnet.  Note that even if --bare is used, it "
+                       "init will still create the directory structure of the app, "
+                       "include the app_template directory with default content, and content in "
+                       "the deployment/ directory (since it is not yet easily customizable).")
 
 
 # TODO (#266): disable the use_enum_values configuration
@@ -185,8 +195,45 @@ class validate(Config_Base):
     build_api: bool = Field(default=False, description="Should api objects be built and output in the build_path?")
     data_source_TA_validation: bool = Field(default=False, description="Validate latest TA information from Splunkbase")
 
-    def getAtomicRedTeamRepoPath(self, atomic_red_team_repo_name:str = "atomic-red-team"):
-        return self.path/atomic_red_team_repo_name
+    @property
+    def external_repos_path(self)->pathlib.Path:
+        return self.path/"external_repos"
+
+    @property    
+    def mitre_cti_repo_path(self)->pathlib.Path:
+        return self.external_repos_path/"cti"
+
+    @property
+    def atomic_red_team_repo_path(self):
+        return self.external_repos_path/"atomic-red-team"
+
+    @model_validator(mode="after")
+    def ensureEnrichmentReposPresent(self)->Self:
+        '''
+        Ensures that the enrichments repos, the atomic red team repo and the 
+        mitre attack enrichment repo, are present at the inded path.
+        Raises a detailed exception if either of these are not present
+        when enrichments are enabled.
+        '''
+        if not self.enrichments:
+            return self
+        # If enrichments are enabled, ensure that all of the
+        # enrichment directories exist
+        missing_repos:list[str] = []
+        if not self.atomic_red_team_repo_path.is_dir():
+            missing_repos.append(f"https://github.com/redcanaryco/atomic-red-team {self.atomic_red_team_repo_path}")
+
+        if not self.mitre_cti_repo_path.is_dir():
+            missing_repos.append(f"https://github.com/mitre/cti {self.mitre_cti_repo_path}")
+
+        if len(missing_repos) > 0: 
+            msg_list = ["The following repositories, which are required for enrichment, have not "
+                        f"been checked out to the {self.external_repos_path} directory. "
+                        "Please check them out using the following commands:"]
+            msg_list.extend([f"git clone --single-branch {repo_string}" for repo_string in missing_repos])
+            msg = '\n\t'.join(msg_list)
+            raise FileNotFoundError(msg)
+        return self
 
 class report(validate):
     #reporting takes no extra args, but we define it here so that it can be a mode on the command line    
@@ -233,27 +280,111 @@ class build(validate):
             return self.getBuildDir() / f"{self.app.appid}-{self.app.version}.tar.gz"
         else:
             return self.getBuildDir() / f"{self.app.appid}-latest.tar.gz"
-    
-    def getSSAPath(self)->pathlib.Path:
-        return self.getBuildDir() / "ssa"
 
     def getAPIPath(self)->pathlib.Path:
         return self.getBuildDir() / "api"
 
     def getAppTemplatePath(self)->pathlib.Path:
         return self.path/"app_template"
-    
 
 
 class StackType(StrEnum):
     classic = auto()
     victoria = auto()
 
+
 class inspect(build):
-    splunk_api_username: str = Field(description="Splunk API username used for running appinspect.")
-    splunk_api_password: str = Field(exclude=True, description="Splunk API password used for running appinspect.")
+    splunk_api_username: str = Field(
+        description="Splunk API username used for appinspect and Splunkbase downloads."
+    )
+    splunk_api_password: str = Field(
+        exclude=True,
+        description="Splunk API password used for appinspect and Splunkbase downloads."
+    )
+    enable_metadata_validation: bool = Field(
+        default=False,
+        description=(
+            "Flag indicating whether detection metadata validation and versioning enforcement "
+            "should be enabled."
+        )
+    )
+    suppress_missing_content_exceptions: bool = Field(
+        default=False,
+        description=(
+            "Suppress exceptions during metadata validation if a detection that existed in "
+            "the previous build does not exist in this build. This is to ensure that content "
+            "is not accidentally removed. In order to support testing both public and private "
+            "content, this warning can be suppressed. If it is suppressed, it will still be "
+            "printed out as a warning."
+        )
+    )
+    enrichments: bool = Field(
+        default=True,
+        description=(
+            "[NOTE: enrichments must be ENABLED for inspect to run. Please adjust your config "
+            f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}"
+            )
+        )
+    # TODO (cmcginley): wording should change here if we want to be able to download any app from
+    #   Splunkbase
+    previous_build: str | None = Field(
+        default=None,
+        description=(
+            "Local path to the previous app build for metatdata validation and versioning "
+            "enforcement (defaults to the latest release of the app published on Splunkbase)."
+        )
+    )
     stack_type: StackType = Field(description="The type of your Splunk Cloud Stack")
 
+    @field_validator("enrichments", mode="after")
+    @classmethod
+    def validate_needed_flags_metadata_validation(cls, v: bool, info: ValidationInfo) -> bool:
+        """
+        Validates that `enrichments` is True for the inspect action
+
+        :param v: the field's value
+        :type v: bool
+        :param info: the ValidationInfo to be used
+        :type info: :class:`pydantic.ValidationInfo`
+
+        :returns: bool, for v
+        :rtype: bool
+        """
+        # Enforce that `enrichments` is True for the inspect action
+        if v is False:
+            raise ValueError("Field `enrichments` must be True for the `inspect` action")
+
+        return v
+
+    def get_previous_package_file_path(self) -> pathlib.Path:
+        """
+        Returns a Path object for the path to the prior package build. If no path was provided, the
+        latest version is downloaded from Splunkbase and it's filepath is returned, and saved to the
+        in-memory config (so download doesn't happen twice in the same run).
+
+        :returns: Path object to previous app build
+        :rtype: :class:`pathlib.Path`
+        """
+        previous_build_path = self.previous_build
+        # Download the previous build as the latest release on Splunkbase if no path was provided
+        if previous_build_path is None:
+            print(
+                f"Downloading latest {self.app.label} build from Splunkbase to serve as previous "
+                "build during validation..."
+            )
+            app = SplunkApp(app_uid=self.app.uid)
+            previous_build_path = app.download(
+                out=pathlib.Path(DOWNLOADS_DIRECTORY),
+                username=self.splunk_api_username,
+                password=self.splunk_api_password,
+                is_dir=True,
+                overwrite=True
+            )
+            print(f"Latest release downloaded from Splunkbase to: {previous_build_path}")
+            self.previous_build = str(previous_build_path)
+        return pathlib.Path(previous_build_path)
+
+
 class NewContentType(StrEnum):
     detection = auto()
     story = auto()
@@ -828,7 +959,6 @@ class test_servers(test_common):
                 index+=1
 
 
-        
 class release_notes(Config_Base):
     old_tag:Optional[str] = Field(None, description="Name of the tag to diff against to find new content. "
                                           "If it is not supplied, then it will be inferred as the "
@@ -910,6 +1040,4 @@ class release_notes(Config_Base):
     #             raise ValueError("The latest_branch '{self.latest_branch}' was not found in the repository")
         
         
-    #     return self
-
-
+    #     return self

contentctl/objects/constants.py

@@ -1,3 +1,5 @@
+# Use for calculation of maximum length of name field
+from contentctl.objects.enums import SecurityDomain
 
 ATTACK_TACTICS_KILLCHAIN_MAPPING = {
     "Reconnaissance": "Reconnaissance",
@@ -136,4 +138,35 @@ SES_ATTACK_TACTICS_ID_MAPPING = {
 RBA_OBSERVABLE_ROLE_MAPPING = {
     "Attacker": 0,
     "Victim": 1
-}
+}
+
+# The relative path to the directory where any apps/packages will be downloaded
+DOWNLOADS_DIRECTORY = "downloads"
+
+# Maximum length of the name field for a search.
+# This number is derived from a limitation that exists in 
+# ESCU where a search cannot be edited, due to validation
+# errors, if its name is longer than 99 characters.
+# When an saved search is cloned in Enterprise Security User Interface,
+# it is wrapped in the following: 
+# {Detection.tags.security_domain.value} - {SEARCH_STANZA_NAME} - Rule
+# Similarly, when we generate the search stanza name in contentctl, it
+# is app.label - detection.name - Rule
+# However, in product the search name is:
+# {CustomApp.label} - {detection.name} - Rule,
+# or in ESCU:
+# ESCU - {detection.name} - Rule,
+# this gives us a maximum length below.
+# When an ESCU search is cloned, it will 
+# have a full name like (the following is NOT a typo):
+# Endpoint - ESCU - Name of Search From YML File - Rule - Rule
+# The math below accounts for all these caveats
+ES_MAX_STANZA_LENGTH = 99
+CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name} - Rule"
+CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name}"
+CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE = "{app_label} - {detection_name} - Response Task"
+
+ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE = "{security_domain_value} - {search_name} - Rule"
+SECURITY_DOMAIN_MAX_LENGTH = max([len(SecurityDomain[value]) for value in SecurityDomain._member_map_])
+CONTENTCTL_MAX_STANZA_LENGTH = ES_MAX_STANZA_LENGTH - len(ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format(security_domain_value="X"*SECURITY_DOMAIN_MAX_LENGTH,search_name=""))
+CONTENTCTL_MAX_SEARCH_NAME_LENGTH = CONTENTCTL_MAX_STANZA_LENGTH - len(CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(app_label="ESCU", detection_name=""))

contentctl/objects/correlation_search.py

@@ -1,10 +1,11 @@
 import logging
 import time
 import json
-from typing import Union, Optional, Any
+from typing import Any
 from enum import Enum
+from functools import cached_property
 
-from pydantic import BaseModel, validator, Field, PrivateAttr
+from pydantic import ConfigDict, BaseModel, computed_field, Field, PrivateAttr
 from splunklib.results import JSONResultsReader, Message                                            # type: ignore
 from splunklib.binding import HTTPError, ResponseReader                                             # type: ignore
 import splunklib.client as splunklib                                                                # type: ignore
@@ -15,7 +16,7 @@ from contentctl.objects.notable_action import NotableAction
 from contentctl.objects.base_test_result import TestResultStatus
 from contentctl.objects.integration_test_result import IntegrationTestResult
 from contentctl.actions.detection_testing.progress_bar import (
-    format_pbar_string,
+    format_pbar_string,                                                                             # type: ignore
     TestReportingType,
     TestingStates
 )
@@ -178,13 +179,14 @@ class PbarData(BaseModel):
     :param fq_test_name: the fully qualifed (fq) test name ("<detection_name>:<test_name>") used for logging
     :param start_time: the start time used for logging
     """
-    pbar: tqdm
+    pbar: tqdm                                                                                      # type: ignore
     fq_test_name: str
     start_time: float
 
-    class Config:
-        # needed to support the tqdm type
-        arbitrary_types_allowed = True
+    # needed to support the tqdm type
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
 
 
 class CorrelationSearch(BaseModel):
@@ -197,143 +199,110 @@ class CorrelationSearch(BaseModel):
     :param pbar_data: the encapsulated info needed for logging w/ pbar
     :param test_index: the index attack data is forwarded to for testing (optionally used in cleanup)
     """
-    ## The following three fields are explicitly needed at instantiation                            # noqa: E266
-
     # the detection associated with the correlation search (e.g. "Windows Modify Registry EnableLinkedConnections")
-    detection: Detection
+    detection: Detection = Field(...)
 
     # a Service instance representing a connection to a Splunk instance
-    service: splunklib.Service
+    service: splunklib.Service = Field(...)
 
     # the encapsulated info needed for logging w/ pbar
-    pbar_data: PbarData
-
-    ## The following field is optional for instantiation                                            # noqa: E266
+    pbar_data: PbarData = Field(...)
 
     # The index attack data is sent to; can be None if we are relying on the caller to do our
     # cleanup of this index
-    test_index: Optional[str] = Field(default=None, min_length=1)
-
-    ## All remaining fields can be derived from other fields or have intentional defaults that      # noqa: E266
-    ## should not be changed (validators should prevent instantiating some of these fields directly # noqa: E266
-    ## to prevent undefined behavior)                                                               # noqa: E266
+    test_index: str | None = Field(default=None, min_length=1)
 
     # The logger to use (logs all go to a null pipe unless ENABLE_LOGGING is set to True, so as not
     # to conflict w/ tqdm)
-    logger: logging.Logger = Field(default_factory=get_logger)
-
-    # The search name (e.g. "ESCU - Windows Modify Registry EnableLinkedConnections - Rule")
-    name: Optional[str] = None
-
-    # The path to the saved search on the Splunk instance
-    splunk_path: Optional[str] = None
-
-    # A model of the saved search as provided by splunklib
-    saved_search: Optional[splunklib.SavedSearch] = None
+    logger: logging.Logger = Field(default_factory=get_logger, init=False)
 
     # The set of indexes to clear on cleanup
-    indexes_to_purge: set[str] = set()
+    indexes_to_purge: set[str] = Field(default=set(), init=False)
 
     # The risk analysis adaptive response action (if defined)
-    risk_analysis_action: Union[RiskAnalysisAction, None] = None
+    _risk_analysis_action: RiskAnalysisAction | None = PrivateAttr(default=None)
 
     # The notable adaptive response action (if defined)
-    notable_action: Union[NotableAction, None] = None
+    _notable_action: NotableAction | None = PrivateAttr(default=None)
 
     # The list of risk events found
-    _risk_events: Optional[list[RiskEvent]] = PrivateAttr(default=None)
+    _risk_events: list[RiskEvent] | None = PrivateAttr(default=None)
 
     # The list of notable events found
-    _notable_events: Optional[list[NotableEvent]] = PrivateAttr(default=None)
+    _notable_events: list[NotableEvent] | None = PrivateAttr(default=None)
 
-    class Config:
-        # needed to allow fields w/ types like SavedSearch
-        arbitrary_types_allowed = True
-        # We want to have more ridgid typing
-        extra = 'forbid'
+    # Need arbitrary types to allow fields w/ types like SavedSearch; we also want to forbid
+    # unexpected fields
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        extra='forbid'
+    )
 
-    @validator("name", always=True)
-    @classmethod
-    def _convert_detection_to_search_name(cls, v, values) -> str:
-        """
-        Validate name and derive if None
-        """
-        if "detection" not in values:
-            raise ValueError("detection missing; name is dependent on detection")
+    def model_post_init(self, __context: Any) -> None:
+        super().model_post_init(__context)
 
-        expected_name = f"ESCU - {values['detection'].name} - Rule"
-        if v is not None and v != expected_name:
-            raise ValueError(
-                "name must be derived from detection; leave as None and it will be derived automatically"
-            )
-        return expected_name
+        # Parse the initial values for the risk/notable actions
+        self._parse_risk_and_notable_actions()
 
-    @validator("splunk_path", always=True)
-    @classmethod
-    def _derive_splunk_path(cls, v, values) -> str:
+    @computed_field
+    @cached_property
+    def name(self) -> str:
         """
-        Validate splunk_path and derive if None
+        The search name (e.g. "ESCU - Windows Modify Registry EnableLinkedConnections - Rule")
+
+        :returns: the search name
+        :rtype: str
         """
-        if "name" not in values:
-            raise ValueError("name missing; splunk_path is dependent on name")
+        return f"ESCU - {self.detection.name} - Rule"
 
-        expected_path = f"saved/searches/{values['name']}"
-        if v is not None and v != expected_path:
-            raise ValueError(
-                "splunk_path must be derived from name; leave as None and it will be derived automatically"
-            )
-        return f"saved/searches/{values['name']}"
+    @computed_field
+    @cached_property
+    def splunk_path(self) -> str:
+        """
+        The path to the saved search on the Splunk instance
 
-    @validator("saved_search", always=True)
-    @classmethod
-    def _instantiate_saved_search(cls, v, values) -> str:
+        :returns: the search path
+        :rtype: str
         """
-        Ensure saved_search was initialized as None and derive
+        return f"/saved/searches/{self.name}"
+
+    @computed_field
+    @cached_property
+    def saved_search(self) -> splunklib.SavedSearch:
         """
-        if "splunk_path" not in values or "service" not in values:
-            raise ValueError("splunk_path or service missing; saved_search is dependent on both")
+        A model of the saved search as provided by splunklib
 
-        if v is not None:
-            raise ValueError(
-                "saved_search must be derived from the service and splunk_path; leave as None and it will be derived "
-                "automatically"
-            )
+        :returns: the SavedSearch object
+        :rtype: :class:`splunklib.client.SavedSearch`
+        """
         return splunklib.SavedSearch(
-            values['service'],
-            values['splunk_path'],
+            self.service,
+            self.splunk_path,
         )
 
-    @validator("risk_analysis_action", always=True)
-    @classmethod
-    def _init_risk_analysis_action(cls, v, values) -> Optional[RiskAnalysisAction]:
-        """
-        Initialize risk_analysis_action
+    # TODO (cmcginley): need to make this refreshable
+    @computed_field
+    @property
+    def risk_analysis_action(self) -> RiskAnalysisAction | None:
         """
-        if "saved_search" not in values:
-            raise ValueError("saved_search missing; risk_analysis_action is dependent on saved_search")
+        The risk analysis adaptive response action (if defined)
 
-        if v is not None:
-            raise ValueError(
-                "risk_analysis_action must be derived from the saved_search; leave as None and it will be derived "
-                "automatically"
-            )
-        return CorrelationSearch._get_risk_analysis_action(values['saved_search'].content)
-
-    @validator("notable_action", always=True)
-    @classmethod
-    def _init_notable_action(cls, v, values) -> Optional[NotableAction]:
+        :returns: the RiskAnalysisAction object, if it exists
+        :rtype: :class:`contentctl.objects.risk_analysis_action.RiskAnalysisAction` | None
         """
-        Initialize notable_action
+        return self._risk_analysis_action
+
+    # TODO (cmcginley): need to make this refreshable
+    @computed_field
+    @property
+    def notable_action(self) -> NotableAction | None:
         """
-        if "saved_search" not in values:
-            raise ValueError("saved_search missing; notable_action is dependent on saved_search")
+        The notable adaptive response action (if defined)
 
-        if v is not None:
-            raise ValueError(
-                "notable_action must be derived from the saved_search; leave as None and it will be derived "
-                "automatically"
-            )
-        return CorrelationSearch._get_notable_action(values['saved_search'].content)
+        :returns: the NotableAction object, if it exists
+        :rtype: :class:`contentctl.objects.notable_action.NotableAction` | None
+        """
+        return self._notable_action
 
     @property
     def earliest_time(self) -> str:
@@ -393,7 +362,7 @@ class CorrelationSearch(BaseModel):
         return self.notable_action is not None
 
     @staticmethod
-    def _get_risk_analysis_action(content: dict[str, Any]) -> Optional[RiskAnalysisAction]:
+    def _get_risk_analysis_action(content: dict[str, Any]) -> RiskAnalysisAction | None:
         """
         Given the saved search content, parse the risk analysis action
         :param content: a dict of strings to values
@@ -407,7 +376,7 @@ class CorrelationSearch(BaseModel):
         return None
 
     @staticmethod
-    def _get_notable_action(content: dict[str, Any]) -> Optional[NotableAction]:
+    def _get_notable_action(content: dict[str, Any]) -> NotableAction | None:
         """
         Given the saved search content, parse the notable action
         :param content: a dict of strings to values
@@ -431,10 +400,6 @@ class CorrelationSearch(BaseModel):
                 relevant.append(observable)
         return relevant
 
-    # TODO (PEX-484): ideally, we could handle this and the following init w/ a call to
-    #   model_post_init, so that all the logic is encapsulated w/in _parse_risk_and_notable_actions
-    #   but that is a pydantic v2 feature (see the init validators for risk/notable actions):
-    #   https://docs.pydantic.dev/latest/api/base_model/#pydantic.main.BaseModel.model_post_init
     def _parse_risk_and_notable_actions(self) -> None:
         """Parses the risk/notable metadata we care about from self.saved_search.content
 
@@ -445,12 +410,12 @@ class CorrelationSearch(BaseModel):
             unpacked to be anything other than a singleton
         """
         # grab risk details if present
-        self.risk_analysis_action = CorrelationSearch._get_risk_analysis_action(
+        self._risk_analysis_action = CorrelationSearch._get_risk_analysis_action(
             self.saved_search.content                                                               # type: ignore
         )
 
         # grab notable details if present
-        self.notable_action = CorrelationSearch._get_notable_action(self.saved_search.content)      # type: ignore
+        self._notable_action = CorrelationSearch._get_notable_action(self.saved_search.content)     # type: ignore
 
     def refresh(self) -> None:
         """Refreshes the metadata in the SavedSearch entity, and re-parses the fields we care about
@@ -738,7 +703,7 @@ class CorrelationSearch(BaseModel):
         # TODO (#250): Re-enable and refactor code that validates the specific risk counts
         # Validate risk events in aggregate; we should have an equal amount of risk events for each
         # relevant observable, and the total count should match the total number of events
-        # individual_count: Optional[int] = None
+        # individual_count: int | None = None
         # total_count = 0
         # for observable_str in observable_counts:
         #     self.logger.debug(
@@ -802,7 +767,7 @@ class CorrelationSearch(BaseModel):
             )
 
         # initialize result as None
-        result: Optional[IntegrationTestResult] = None
+        result: IntegrationTestResult | None = None
 
         # keep track of time slept and number of attempts for exponential backoff (base 2)
         elapsed_sleep_time = 0