contentctl 5.0.1__tar.gz → 5.0.3__tar.gz

{contentctl-5.0.1 → contentctl-5.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: contentctl
-Version: 5.0.1
+Version: 5.0.3
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/actions/inspect.py

@@ -1,23 +1,45 @@
-import sys
-from dataclasses import dataclass
-import pathlib
-import json
 import datetime
-import timeit
+import json
+import pathlib
+import sys
 import time
+import timeit
+from dataclasses import dataclass
+from io import BufferedReader
 
-from requests import Session, post, get
+from requests import Session, get, post
 from requests.auth import HTTPBasicAuth
 
 from contentctl.objects.config import inspect
-from contentctl.objects.savedsearches_conf import SavedsearchesConf
 from contentctl.objects.errors import (
-    MetadataValidationError,
     DetectionIDError,
     DetectionMissingError,
-    VersionDecrementedError,
+    MetadataValidationError,
     VersionBumpingError,
+    VersionDecrementedError,
 )
+from contentctl.objects.savedsearches_conf import SavedsearchesConf
+
+"""
+The following list includes all appinspect tags available from:
+https://dev.splunk.com/enterprise/reference/appinspect/appinspecttagreference/
+
+This allows contentctl to be as forward-leaning as possible in catching
+any potential issues on the widest variety of stacks.
+"""
+INCLUDED_TAGS_LIST = [
+    "aarch64_compatibility",
+    "ast",
+    "cloud",
+    "future",
+    "manual",
+    "packaging_standards",
+    "private_app",
+    "private_classic",
+    "private_victoria",
+    "splunk_appinspect",
+]
+INCLUDED_TAGS_STRING = ",".join(INCLUDED_TAGS_LIST)
 
 
 @dataclass(frozen=True)
@@ -28,7 +50,6 @@ class InspectInputDto:
 class Inspect:
     def execute(self, config: inspect) -> str:
         if config.build_app or config.build_api:
-            self.inspectAppCLI(config)
             appinspect_token = self.inspectAppAPI(config)
 
             if config.enable_metadata_validation:
@@ -49,10 +70,6 @@ class Inspect:
         session.auth = HTTPBasicAuth(
             config.splunk_api_username, config.splunk_api_password
         )
-        if config.stack_type not in ["victoria", "classic"]:
-            raise Exception(
-                f"stack_type MUST be either 'classic' or 'victoria', NOT '{config.stack_type}'"
-            )
 
         APPINSPECT_API_LOGIN = "https://api.splunk.com/2.0/rest/login/splunk"
 
@@ -64,10 +81,6 @@ class Inspect:
         APPINSPECT_API_VALIDATION_REQUEST = (
             "https://appinspect.splunk.com/v1/app/validate"
         )
-        headers = {
-            "Authorization": f"bearer {authorization_bearer}",
-            "Cache-Control": "no-cache",
-        }
 
         package_path = config.getPackageFilePath(include_version=False)
         if not package_path.is_file():
@@ -77,18 +90,43 @@ class Inspect:
                 "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?"
             )
 
-        files = {
+        """
+        Some documentation on "files" argument for requests.post exists here:
+        https://docs.python-requests.org/en/latest/api/
+        The type (None, INCLUDED_TAGS_STRING) is intentional, and the None is important.
+        In curl syntax, the request we make below is equivalent to
+        curl -X POST \
+            -H "Authorization: bearer <TOKEN>" \
+            -H "Cache-Control: no-cache" \
+            -F "app_package=@<PATH/APP-PACKAGE>" \
+            -F "included_tags=cloud" \
+            --url "https://appinspect.splunk.com/v1/app/validate"
+        
+        This is confirmed by the great resource:
+        https://curlconverter.com/
+        """
+        data: dict[str, tuple[None, str] | BufferedReader] = {
             "app_package": open(package_path, "rb"),
-            "included_tags": (None, "cloud"),
+            "included_tags": (
+                None,
+                INCLUDED_TAGS_STRING,
+            ),  # tuple with None is intentional here
         }
 
-        res = post(APPINSPECT_API_VALIDATION_REQUEST, headers=headers, files=files)
+        headers = {
+            "Authorization": f"bearer {authorization_bearer}",
+            "Cache-Control": "no-cache",
+        }
+
+        res = post(APPINSPECT_API_VALIDATION_REQUEST, files=data, headers=headers)
 
         res.raise_for_status()
 
         request_id = res.json().get("request_id", None)
-        APPINSPECT_API_VALIDATION_STATUS = f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}?included_tags=private_{config.stack_type}"
-        headers = headers = {"Authorization": f"bearer {authorization_bearer}"}
+        APPINSPECT_API_VALIDATION_STATUS = (
+            f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}"
+        )
+
         startTime = timeit.default_timer()
         # the first time, wait for 40 seconds. subsequent times, wait for less.
         # this is because appinspect takes some time to return, so there is no sense
@@ -114,7 +152,9 @@ class Inspect:
                 raise Exception(f"Error - Unknown Appinspect API status '{status}'")
 
         # We have finished running appinspect, so get the report
-        APPINSPECT_API_REPORT = f"https://appinspect.splunk.com/v1/app/report/{request_id}?included_tags=private_{config.stack_type}"
+        APPINSPECT_API_REPORT = (
+            f"https://appinspect.splunk.com/v1/app/report/{request_id}"
+        )
         # Get human-readable HTML report
         headers = headers = {
             "Authorization": f"bearer {authorization_bearer}",
@@ -159,14 +199,14 @@ class Inspect:
                 "\t - https://dev.splunk.com/enterprise/docs/developapps/testvalidate/appinspect/useappinspectclitool/"
             )
             from splunk_appinspect.main import (
-                validate,
-                MODE_OPTION,
                 APP_PACKAGE_ARGUMENT,
-                OUTPUT_FILE_OPTION,
-                LOG_FILE_OPTION,
-                INCLUDED_TAGS_OPTION,
                 EXCLUDED_TAGS_OPTION,
+                INCLUDED_TAGS_OPTION,
+                LOG_FILE_OPTION,
+                MODE_OPTION,
+                OUTPUT_FILE_OPTION,
                 TEST_MODE,
+                validate,
             )
         except Exception as e:
             print(e)

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/contentctl.py

@@ -1,7 +1,9 @@
 import pathlib
+import random
 import sys
 import traceback
 import warnings
+from dataclasses import dataclass
 
 import tyro
 
@@ -155,6 +157,35 @@ YOU HAVE BEEN WARNED!
 """
 
 
+def get_random_compliment():
+    compliments = [
+        "Your detection rules are like a zero-day shield! 🛡️",
+        "You catch threats like it's child's play! 🎯",
+        "Your correlation rules are pure genius! 🧠",
+        "Threat actors fear your detection engineering! ⚔️",
+        "You're the SOC's secret weapon! 🦾",
+        "Your false positive rate is impressively low! 📊",
+        "Malware trembles at your detection logic! 🦠",
+        "You're the threat hunter extraordinaire! 🔍",
+        "Your MITRE mappings are a work of art! 🎨",
+        "APTs have nightmares about your detections! 👻",
+        "Your content testing is bulletproof! 🎯",
+        "You're the detection engineering MVP! 🏆",
+    ]
+    return random.choice(compliments)
+
+
+def recognize_func():
+    print(get_random_compliment())
+
+
+@dataclass
+class RecognizeCommand:
+    """Dummy subcommand for 'recognize' with no parameters."""
+
+    pass
+
+
 def main():
     print(CONTENTCTL_5_WARNING)
     try:
@@ -210,6 +241,7 @@ def main():
             "test_servers": test_servers.model_construct(**t.__dict__),
             "release_notes": release_notes.model_construct(**config_obj),
             "deploy_acs": deploy_acs.model_construct(**t.__dict__),
+            "recognize": RecognizeCommand(),
         }
     )
 
@@ -240,6 +272,8 @@ def main():
             deploy_acs_func(updated_config)
         elif type(config) is test or type(config) is test_servers:
             test_common_func(config)
+        elif type(config) is RecognizeCommand:
+            recognize_func()
         else:
             raise Exception(f"Unknown command line type '{type(config).__name__}'")
     except FileNotFoundError as e:

contentctl-5.0.3/contentctl/objects/annotated_types.py

@@ -0,0 +1,9 @@
+from typing import Annotated
+
+from pydantic import Field
+
+CVE_TYPE = Annotated[str, Field(pattern=r"^CVE-[1|2]\d{3}-\d+$")]
+MITRE_ATTACK_ID_TYPE_PARENT = Annotated[str, Field(pattern=r"^T\d{4}$")]
+MITRE_ATTACK_ID_TYPE_SUBTYPE = Annotated[str, Field(pattern=r"^T\d{4}(.\d{3})$")]
+MITRE_ATTACK_ID_TYPE = MITRE_ATTACK_ID_TYPE_PARENT | MITRE_ATTACK_ID_TYPE_SUBTYPE
+APPID_TYPE = Annotated[str, Field(pattern="^[a-zA-Z0-9_-]+$")]

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/objects/config.py

@@ -425,7 +425,6 @@ class inspect(build):
             "enforcement (defaults to the latest release of the app published on Splunkbase)."
         ),
     )
-    stack_type: StackType = Field(description="The type of your Splunk Cloud Stack")
 
     @field_validator("enrichments", mode="after")
     @classmethod
@@ -496,6 +495,9 @@ class new(Config_Base):
 
 class deploy_acs(inspect):
     model_config = ConfigDict(validate_default=False, arbitrary_types_allowed=True)
+
+    stack_type: StackType = Field(description="The type of your Splunk Cloud Stack")
+
     # ignore linter error
     splunk_cloud_jwt_token: str = Field(
         exclude=True,

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/objects/data_source.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
-from typing import Optional, Any
-from pydantic import Field, HttpUrl, model_serializer, BaseModel
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, HttpUrl, model_serializer
+
 from contentctl.objects.security_content_object import SecurityContentObject
 
 
@@ -20,6 +23,7 @@ class DataSource(SecurityContentObject):
     field_mappings: None | list = None
     convert_to_log_source: None | list = None
     example_log: None | str = None
+    output_fields: list[str] = []
 
     @model_serializer
     def serialize_model(self):

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/objects/detection_tags.py

@@ -33,7 +33,10 @@ from contentctl.objects.enums import (
     SecurityContentProductName,
     SecurityDomain,
 )
-from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
+from contentctl.objects.mitre_attack_enrichment import (
+    MitreAttackEnrichment,
+    MitreAttackGroup,
+)
 
 
 class DetectionTags(BaseModel):
@@ -44,7 +47,7 @@ class DetectionTags(BaseModel):
     asset_type: AssetType = Field(...)
     group: list[str] = []
 
-    mitre_attack_id: List[MITRE_ATTACK_ID_TYPE] = []
+    mitre_attack_id: list[MITRE_ATTACK_ID_TYPE] = []
     nist: list[NistCategory] = []
 
     product: list[SecurityContentProductName] = Field(..., min_length=1)
@@ -68,6 +71,15 @@ class DetectionTags(BaseModel):
                 phases.add(phase)
         return sorted(list(phases))
 
+    # We do not want this to be included in serialization. By default, @property
+    # objects are not included in dumps
+    @property
+    def unique_mitre_attack_groups(self) -> list[MitreAttackGroup]:
+        group_set: set[MitreAttackGroup] = set()
+        for enrichment in self.mitre_attack_enrichments:
+            group_set.update(set(enrichment.mitre_attack_group_objects))
+        return sorted(group_set, key=lambda k: k.group)
+
     # enum is intentionally Cis18 even though field is named cis20 for legacy reasons
     @computed_field
     @property
@@ -134,8 +146,8 @@ class DetectionTags(BaseModel):
 
         if len(missing_tactics) > 0:
             raise ValueError(f"Missing Mitre Attack IDs. {missing_tactics} not found.")
-        else:
-            self.mitre_attack_enrichments = mitre_enrichments
+
+        self.mitre_attack_enrichments = mitre_enrichments
 
         return self
 
@@ -159,6 +171,44 @@ class DetectionTags(BaseModel):
         return enrichments
     """
 
+    @field_validator("mitre_attack_id", mode="after")
+    @classmethod
+    def sameTypeAndSubtypeNotPresent(
+        cls, techniques_and_subtechniques: list[MITRE_ATTACK_ID_TYPE]
+    ) -> list[MITRE_ATTACK_ID_TYPE]:
+        techniques: list[str] = [
+            f"{unknown_technique}."
+            for unknown_technique in techniques_and_subtechniques
+            if "." not in unknown_technique
+        ]
+        subtechniques: list[MITRE_ATTACK_ID_TYPE] = [
+            unknown_technique
+            for unknown_technique in techniques_and_subtechniques
+            if "." in unknown_technique
+        ]
+        subtype_and_parent_exist_exceptions: list[ValueError] = []
+
+        for subtechnique in subtechniques:
+            for technique in techniques:
+                if subtechnique.startswith(technique):
+                    subtype_and_parent_exist_exceptions.append(
+                        ValueError(
+                            f"    Technique   : {technique.split('.')[0]}\n"
+                            f"    SubTechnique: {subtechnique}\n"
+                        )
+                    )
+
+        if len(subtype_and_parent_exist_exceptions):
+            error_string = "\n".join(
+                str(e) for e in subtype_and_parent_exist_exceptions
+            )
+            raise ValueError(
+                "Overlapping MITRE Attack ID Techniques and Subtechniques may not be defined. "
+                f"Remove the Technique and keep the Subtechnique:\n{error_string}"
+            )
+
+        return techniques_and_subtechniques
+
     @field_validator("analytic_story", mode="before")
     @classmethod
     def mapStoryNamesToStoryObjects(
@@ -238,3 +288,6 @@ class DetectionTags(BaseModel):
         return matched_tests + [
             AtomicTest.AtomicTestWhenTestIsMissing(test) for test in missing_tests
         ]
+        return matched_tests + [
+            AtomicTest.AtomicTestWhenTestIsMissing(test) for test in missing_tests
+        ]

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/objects/mitre_attack_enrichment.py

@@ -1,8 +1,11 @@
 from __future__ import annotations
-from pydantic import BaseModel, Field, ConfigDict, HttpUrl, field_validator
-from typing import List
-from enum import StrEnum
+
 import datetime
+from enum import StrEnum
+from typing import List
+
+from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
+
 from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE
 
 
@@ -84,6 +87,16 @@ class MitreAttackGroup(BaseModel):
             return []
         return contributors
 
+    def __lt__(self, other: MitreAttackGroup) -> bool:
+        if not isinstance(object, MitreAttackGroup):
+            raise Exception(
+                f"Cannot compare object of type MitreAttackGroup to object of type [{type(object).__name__}]"
+            )
+        return self.group < other.group
+
+    def __hash__(self) -> int:
+        return hash(self.group)
+
 
 class MitreAttackEnrichment(BaseModel):
     ConfigDict(extra="forbid")

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/objects/story_tags.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
-from pydantic import BaseModel, Field, model_serializer, ConfigDict
-from typing import List, Set, Optional
 
 from enum import Enum
+from typing import List, Optional, Set
 
-from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
+from pydantic import BaseModel, ConfigDict, Field, model_serializer
+
+from contentctl.objects.annotated_types import CVE_TYPE, MITRE_ATTACK_ID_TYPE
 from contentctl.objects.enums import (
-    StoryCategory,
     DataModel,
     KillChainPhase,
     SecurityContentProductName,
+    StoryCategory,
 )
-from contentctl.objects.annotated_types import CVE_TYPE, MITRE_ATTACK_ID_TYPE
+from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
 
 
 class StoryUseCase(str, Enum):

{contentctl-5.0.1 → contentctl-5.0.3}/contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml

@@ -60,7 +60,6 @@ tags:
   asset_type: Endpoint
   mitre_attack_id:
   - T1560.001
-  - T1560
   product:
   - Splunk Enterprise
   - Splunk Enterprise Security

{contentctl-5.0.1 → contentctl-5.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "contentctl"
-version = "5.0.1"
+version = "5.0.3"
 
 description = "Splunk Content Control Tool"
 authors = ["STRT <research@splunk.com>"]

contentctl-5.0.1/contentctl/objects/annotated_types.py

@@ -1,6 +0,0 @@
-from pydantic import Field
-from typing import Annotated
-
-CVE_TYPE = Annotated[str, Field(pattern=r"^CVE-[1|2]\d{3}-\d+$")]
-MITRE_ATTACK_ID_TYPE = Annotated[str, Field(pattern=r"^T\d{4}(.\d{3})?$")]
-APPID_TYPE = Annotated[str, Field(pattern="^[a-zA-Z0-9_-]+$")]