contentctl 4.3.4__py3-none-any.whl → 4.3.5__py3-none-any.whl

contentctl/objects/config.py

@@ -1,26 +1,31 @@
 from __future__ import annotations
+
+from os import environ
+from datetime import datetime, UTC
+from typing import Optional, Any, List, Union, Self
+import random
+from enum import StrEnum, auto
+import pathlib
+from urllib.parse import urlparse
+from abc import ABC, abstractmethod
+from functools import partialmethod
+
+import tqdm
+import semantic_version
 from pydantic import (
     BaseModel, Field, field_validator, 
     field_serializer, ConfigDict, DirectoryPath,
     PositiveInt, FilePath, HttpUrl, AnyUrl, model_validator,
     ValidationInfo
 )
+
+from contentctl.objects.constants import DOWNLOADS_DIRECTORY
 from contentctl.output.yml_writer import YmlWriter
-from os import environ
-from datetime import datetime, UTC
-from typing import Optional,Any,Annotated,List,Union, Self
-import semantic_version
-import random
-from enum import StrEnum, auto
-import pathlib
 from contentctl.helper.utils import Utils
-from urllib.parse import urlparse
-from abc import ABC, abstractmethod
 from contentctl.objects.enums import PostTestBehavior, DetectionTestingMode
 from contentctl.objects.detection import Detection
 from contentctl.objects.annotated_types import APPID_TYPE
-import tqdm
-from functools import partialmethod
+from contentctl.helper.splunk_app import SplunkApp
 
 ENTERPRISE_SECURITY_UID = 263
 COMMON_INFORMATION_MODEL_UID = 1621
@@ -171,7 +176,13 @@ class Config_Base(BaseModel):
         return str(path)
 
 class init(Config_Base):
-    pass
+    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
+    bare: bool = Field(default=False, description="contentctl normally provides some some example content "
+                       "(macros, stories, data_sources, and/or analytic stories).  This option disables "
+                       "initialization with that additional contnet.  Note that even if --bare is used, it "
+                       "init will still create the directory structure of the app, "
+                       "include the app_template directory with default content, and content in "
+                       "the deployment/ directory (since it is not yet easily customizable).")
 
 
 # TODO (#266): disable the use_enum_values configuration
@@ -185,8 +196,45 @@ class validate(Config_Base):
     build_api: bool = Field(default=False, description="Should api objects be built and output in the build_path?")
     data_source_TA_validation: bool = Field(default=False, description="Validate latest TA information from Splunkbase")
 
-    def getAtomicRedTeamRepoPath(self, atomic_red_team_repo_name:str = "atomic-red-team"):
-        return self.path/atomic_red_team_repo_name
+    @property
+    def external_repos_path(self)->pathlib.Path:
+        return self.path/"external_repos"
+
+    @property    
+    def mitre_cti_repo_path(self)->pathlib.Path:
+        return self.external_repos_path/"cti"
+
+    @property
+    def atomic_red_team_repo_path(self):
+        return self.external_repos_path/"atomic-red-team"
+
+    @model_validator(mode="after")
+    def ensureEnrichmentReposPresent(self)->Self:
+        '''
+        Ensures that the enrichments repos, the atomic red team repo and the 
+        mitre attack enrichment repo, are present at the inded path.
+        Raises a detailed exception if either of these are not present
+        when enrichments are enabled.
+        '''
+        if not self.enrichments:
+            return self
+        # If enrichments are enabled, ensure that all of the
+        # enrichment directories exist
+        missing_repos:list[str] = []
+        if not self.atomic_red_team_repo_path.is_dir():
+            missing_repos.append(f"https://github.com/redcanaryco/atomic-red-team {self.atomic_red_team_repo_path}")
+
+        if not self.mitre_cti_repo_path.is_dir():
+            missing_repos.append(f"https://github.com/mitre/cti {self.mitre_cti_repo_path}")
+
+        if len(missing_repos) > 0: 
+            msg_list = ["The following repositories, which are required for enrichment, have not "
+                        f"been checked out to the {self.external_repos_path} directory. "
+                        "Please check them out using the following commands:"]
+            msg_list.extend([f"git clone --single-branch {repo_string}" for repo_string in missing_repos])
+            msg = '\n\t'.join(msg_list)
+            raise FileNotFoundError(msg)
+        return self
 
 class report(validate):
     #reporting takes no extra args, but we define it here so that it can be a mode on the command line    
@@ -233,9 +281,6 @@ class build(validate):
             return self.getBuildDir() / f"{self.app.appid}-{self.app.version}.tar.gz"
         else:
             return self.getBuildDir() / f"{self.app.appid}-latest.tar.gz"
-    
-    def getSSAPath(self)->pathlib.Path:
-        return self.getBuildDir() / "ssa"
 
     def getAPIPath(self)->pathlib.Path:
         return self.getBuildDir() / "api"
@@ -249,11 +294,89 @@ class StackType(StrEnum):
     classic = auto()
     victoria = auto()
 
+
 class inspect(build):
-    splunk_api_username: str = Field(description="Splunk API username used for running appinspect.")
-    splunk_api_password: str = Field(exclude=True, description="Splunk API password used for running appinspect.")
+    splunk_api_username: str = Field(
+        description="Splunk API username used for appinspect and Splunkbase downloads."
+    )
+    splunk_api_password: str = Field(
+        exclude=True,
+        description="Splunk API password used for appinspect and Splunkbase downloads."
+    )
+    enable_metadata_validation: bool = Field(
+        default=False,
+        description=(
+            "Flag indicating whether detection metadata validation and versioning enforcement "
+            "should be enabled."
+        )
+    )
+    enrichments: bool = Field(
+        default=True,
+        description=(
+            "[NOTE: enrichments must be ENABLED for inspect to run. Please adjust your config "
+            f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}"
+            )
+        )
+    # TODO (cmcginley): wording should change here if we want to be able to download any app from
+    #   Splunkbase
+    previous_build: str | None = Field(
+        default=None,
+        description=(
+            "Local path to the previous app build for metatdata validation and versioning "
+            "enforcement (defaults to the latest release of the app published on Splunkbase)."
+        )
+    )
     stack_type: StackType = Field(description="The type of your Splunk Cloud Stack")
 
+    @field_validator("enrichments", mode="after")
+    @classmethod
+    def validate_needed_flags_metadata_validation(cls, v: bool, info: ValidationInfo) -> bool:
+        """
+        Validates that `enrichments` is True for the inspect action
+
+        :param v: the field's value
+        :type v: bool
+        :param info: the ValidationInfo to be used
+        :type info: :class:`pydantic.ValidationInfo`
+
+        :returns: bool, for v
+        :rtype: bool
+        """
+        # Enforce that `enrichments` is True for the inspect action
+        if v is False:
+            raise ValueError("Field `enrichments` must be True for the `inspect` action")
+
+        return v
+
+    def get_previous_package_file_path(self) -> pathlib.Path:
+        """
+        Returns a Path object for the path to the prior package build. If no path was provided, the
+        latest version is downloaded from Splunkbase and it's filepath is returned, and saved to the
+        in-memory config (so download doesn't happen twice in the same run).
+
+        :returns: Path object to previous app build
+        :rtype: :class:`pathlib.Path`
+        """
+        previous_build_path = self.previous_build
+        # Download the previous build as the latest release on Splunkbase if no path was provided
+        if previous_build_path is None:
+            print(
+                f"Downloading latest {self.app.label} build from Splunkbase to serve as previous "
+                "build during validation..."
+            )
+            app = SplunkApp(app_uid=self.app.uid)
+            previous_build_path = app.download(
+                out=pathlib.Path(DOWNLOADS_DIRECTORY),
+                username=self.splunk_api_username,
+                password=self.splunk_api_password,
+                is_dir=True,
+                overwrite=True
+            )
+            print(f"Latest release downloaded from Splunkbase to: {previous_build_path}")
+            self.previous_build = str(previous_build_path)
+        return pathlib.Path(previous_build_path)
+
+
 class NewContentType(StrEnum):
     detection = auto()
     story = auto()

contentctl/objects/constants.py

@@ -136,4 +136,7 @@ SES_ATTACK_TACTICS_ID_MAPPING = {
 RBA_OBSERVABLE_ROLE_MAPPING = {
     "Attacker": 0,
     "Victim": 1
-}
+}
+
+# The relative path to the directory where any apps/packages will be downloaded
+DOWNLOADS_DIRECTORY = "downloads"

contentctl/objects/detection_metadata.py

@@ -0,0 +1,71 @@
+import uuid
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class DetectionMetadata(BaseModel):
+    """
+    A model of the metadata line in a detection stanza in savedsearches.conf
+    """
+    # A bool indicating whether the detection is deprecated (serialized as an int, 1 or 0)
+    deprecated: bool = Field(...)
+
+    # A UUID identifying the detection
+    detection_id: uuid.UUID = Field(...)
+
+    # The version of the detection
+    detection_version: int = Field(...)
+
+    # The time the detection was published. **NOTE** This field was added to the metadata in ESCU
+    # as of v4.39.0
+    publish_time: float = Field(...)
+
+    class Config:
+        # Allowing for future fields that may be added to the metadata JSON
+        extra = "allow"
+
+    @field_validator("deprecated", mode="before")
+    @classmethod
+    def validate_deprecated(cls, v: Any) -> Any:
+        """
+        Convert str to int, and then ints to bools for deprecated; raise if not 0 or 1 in the case
+        of an int, or if str cannot be converted to int.
+
+        :param v: the value passed
+        :type v: :class:`typing.Any`
+
+        :returns: the value
+        :rtype: :class:`typing.Any`
+        """
+        if isinstance(v, str):
+            try:
+                v = int(v)
+            except ValueError as e:
+                raise ValueError(f"Cannot convert str value ({v}) to int: {e}") from e
+        if isinstance(v, int):
+            if not (0 <= v <= 1):
+                raise ValueError(
+                    f"Value for field 'deprecated' ({v}) must be 0 or 1, if not a bool."
+                )
+            v = bool(v)
+        return v
+
+    @field_validator("detection_version", mode="before")
+    @classmethod
+    def validate_detection_version(cls, v: Any) -> Any:
+        """
+        Convert str to int; raise if str cannot be converted to int.
+
+        :param v: the value passed
+        :type v: :class:`typing.Any`
+
+        :returns: the value
+        :rtype: :class:`typing.Any`
+        """
+        if isinstance(v, str):
+            try:
+                v = int(v)
+            except ValueError as e:
+                raise ValueError(f"Cannot convert str value ({v}) to int: {e}") from e
+        return v

contentctl/objects/detection_stanza.py

@@ -0,0 +1,79 @@
+from typing import ClassVar
+import hashlib
+from functools import cached_property
+
+from pydantic import BaseModel, Field, computed_field
+
+from contentctl.objects.detection_metadata import DetectionMetadata
+
+
+class DetectionStanza(BaseModel):
+    """
+    A model representing a stanza for a detection in savedsearches.conf
+    """
+    # The lines that comprise this stanza, in the order they appear in the conf
+    lines: list[str] = Field(...)
+
+    # The full name of the detection (e.g. "ESCU - My Detection - Rule")
+    name: str = Field(...)
+
+    # The key prefix indicating the metadata attribute
+    METADATA_LINE_PREFIX: ClassVar[str] = "action.correlationsearch.metadata = "
+
+    @computed_field
+    @cached_property
+    def metadata(self) -> DetectionMetadata:
+        """
+        The metadata extracted from the stanza. Using the provided lines, parse out the metadata
+
+        :returns: the detection stanza's metadata
+        :rtype: :class:`contentctl.objects.detection_metadata.DetectionMetadata`
+        """
+        # Set a variable to store the metadata line in
+        meta_line: str | None = None
+
+        # Iterate over the lines to look for the metadata line
+        for line in self.lines:
+            if line.startswith(DetectionStanza.METADATA_LINE_PREFIX):
+                # If we find a matching line more than once, we've hit an error
+                if meta_line is not None:
+                    raise Exception(
+                        f"Metadata for detection '{self.name}' found twice in stanza."
+                    )
+                meta_line = line
+
+        # Report if we could not find the metadata line
+        if meta_line is None:
+            raise Exception(f"No metadata for detection '{self.name}' found in stanza.")
+
+        # Parse the metadata JSON into a model
+        return DetectionMetadata.model_validate_json(meta_line[len(DetectionStanza.METADATA_LINE_PREFIX):])
+
+    @computed_field
+    @cached_property
+    def hash(self) -> str:
+        """
+        The SHA256 hash of the lines of the stanza, excluding the metadata line
+
+        :returns: hexdigest
+        :rtype: str
+        """
+        hash = hashlib.sha256()
+        for line in self.lines:
+            if not line.startswith(DetectionStanza.METADATA_LINE_PREFIX):
+                hash.update(line.encode("utf-8"))
+        return hash.hexdigest()
+
+    def version_should_be_bumped(self, previous: "DetectionStanza") -> bool:
+        """
+        A helper method that compares this stanza against the same stanza from a previous build;
+        returns True if the version still needs to be bumped (e.g. the detection was changed but
+        the version was not), False otherwise.
+
+        :param previous: the previous build's DetectionStanza for comparison
+        :type previous: :class:`contentctl.objects.detection_stanza.DetectionStanza`
+
+        :returns: True if the version still needs to be bumped
+        :rtype: bool
+        """
+        return (self.hash != previous.hash) and (self.metadata.detection_version <= previous.metadata.detection_version)

contentctl/objects/detection_tags.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 import uuid
-from typing import TYPE_CHECKING, List, Optional, Annotated, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 from pydantic import (
     BaseModel,
     Field,
@@ -32,7 +32,7 @@ from contentctl.objects.enums import (
     RiskLevel,
     SecurityContentProductName
 )
-from contentctl.objects.atomic import AtomicTest
+from contentctl.objects.atomic import AtomicEnrichment, AtomicTest
 from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE, CVE_TYPE
 
 # TODO (#266): disable the use_enum_values configuration
@@ -240,7 +240,7 @@ class DetectionTags(BaseModel):
         if output_dto is None:
             raise ValueError("Context not provided to detection.detection_tags.atomic_guid validator")
 
-        all_tests: None | List[AtomicTest] = output_dto.atomic_tests
+        atomic_enrichment: AtomicEnrichment = output_dto.atomic_enrichment
 
         matched_tests: List[AtomicTest] = []
         missing_tests: List[UUID4] = []
@@ -254,7 +254,7 @@ class DetectionTags(BaseModel):
                 badly_formatted_guids.append(str(atomic_guid_str))
                 continue
             try:
-                matched_tests.append(AtomicTest.getAtomicByAtomicGuid(atomic_guid, all_tests))
+                matched_tests.append(atomic_enrichment.getAtomic(atomic_guid))
             except Exception:
                 missing_tests.append(atomic_guid)
 
@@ -265,7 +265,7 @@ class DetectionTags(BaseModel):
                 f"\n\tPlease review the output above for potential exception(s) when parsing the "
                 "Atomic Red Team Repo."
                 "\n\tVerify that these auto_generated_guid exist and try updating/pulling the "
-                f"repo again.: {[str(guid) for guid in missing_tests]}"
+                f"repo again: {[str(guid) for guid in missing_tests]}"
             )
         else:
             missing_tests_string = ""
@@ -278,6 +278,6 @@ class DetectionTags(BaseModel):
             raise ValueError(f"{bad_guids_string}{missing_tests_string}")
 
         elif len(missing_tests) > 0:
-            print(missing_tests_string)
+            raise ValueError(missing_tests_string)
 
         return matched_tests + [AtomicTest.AtomicTestWhenTestIsMissing(test) for test in missing_tests]

contentctl/objects/enums.py

@@ -54,7 +54,6 @@ class SecurityContentType(enum.Enum):
     deployments = 7
     investigations = 8
     unit_tests = 9
-    ssa_detections = 10
     data_sources = 11
 
 # Bringing these changes back in line will take some time after
@@ -69,7 +68,6 @@ class SecurityContentType(enum.Enum):
 
 class SecurityContentProduct(enum.Enum):
     SPLUNK_APP = 1
-    SSA = 2
     API = 3
     CUSTOM = 4
 

contentctl/objects/errors.py

@@ -1,3 +1,7 @@
+from abc import ABC, abstractmethod
+from uuid import UUID
+
+
 class ValidationFailed(Exception):
     """Indicates not an error in execution, but a validation failure"""
     pass
@@ -16,3 +20,186 @@ class ServerError(IntegrationTestingError):
 class ClientError(IntegrationTestingError):
     """An error encounterd during integration testing, on the client's side (locally)"""
     pass
+
+
+class MetadataValidationError(Exception, ABC):
+    """
+    Base class for any errors arising from savedsearches.conf detection metadata validation
+    """
+    # The name of the rule the error relates to
+    rule_name: str
+
+    @property
+    @abstractmethod
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        raise NotImplementedError()
+
+    @property
+    @abstractmethod
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        raise NotImplementedError()
+
+
+class DetectionMissingError(MetadataValidationError):
+    """
+    An error indicating a detection in the prior build could not be found in the current build
+    """
+    def __init__(
+            self,
+            rule_name: str,
+            *args: object
+    ) -> None:
+        self.rule_name = rule_name
+        super().__init__(self.long_message, *args)
+
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' in previous build not found in current build; "
+            "detection may have been removed or renamed."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            "Detection from previous build not found in current build."
+        )
+
+
+class DetectionIDError(MetadataValidationError):
+    """
+    An error indicating the detection ID may have changed between builds
+    """
+    # The ID from the current build
+    current_id: UUID
+
+    # The ID from the previous build
+    previous_id: UUID
+
+    def __init__(
+            self,
+            rule_name: str,
+            current_id: UUID,
+            previous_id: UUID,
+            *args: object
+    ) -> None:
+        self.rule_name = rule_name
+        self.current_id = current_id
+        self.previous_id = previous_id
+        super().__init__(self.long_message, *args)
+
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' has ID {self.current_id} in current build "
+            f"and {self.previous_id} in previous build; detection IDs and "
+            "names should not change for the same detection between releases."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Detection ID {self.current_id} in current build does not match ID {self.previous_id} in previous build."
+        )
+
+
+class VersioningError(MetadataValidationError, ABC):
+    """
+    A base class for any metadata validation errors relating to detection versioning
+    """
+    # The version in the current build
+    current_version: int
+
+    # The version in the previous build
+    previous_version: int
+
+    def __init__(
+            self,
+            rule_name: str,
+            current_version: int,
+            previous_version: int,
+            *args: object
+    ) -> None:
+        self.rule_name = rule_name
+        self.current_version = current_version
+        self.previous_version = previous_version
+        super().__init__(self.long_message, *args)
+
+
+class VersionDecrementedError(VersioningError):
+    """
+    An error indicating the version number went down between builds
+    """
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' has version {self.current_version} in "
+            f"current build and {self.previous_version} in previous build; "
+            "detection versions cannot decrease in successive builds."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Detection version ({self.current_version}) in current build is less than version "
+            f"({self.previous_version}) in previous build."
+        )
+
+
+class VersionBumpingError(VersioningError):
+    """
+    An error indicating the detection changed but its version wasn't bumped appropriately
+    """
+    @property
+    def long_message(self) -> str:
+        """
+        A long-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Rule '{self.rule_name}' has changed in current build compared to previous "
+            "build (stanza hashes differ); the detection version should be bumped "
+            f"to at least {self.previous_version + 1}."
+        )
+
+    @property
+    def short_message(self) -> str:
+        """
+        A short-form error message
+        :returns: a str, the message
+        """
+        return (
+            f"Detection version in current build should be bumped to at least {self.previous_version + 1}."
+        )