contentctl 4.3.4__py3-none-any.whl → 4.4.0__py3-none-any.whl

contentctl/enrichments/attack_enrichment.py

@@ -1,14 +1,13 @@
 
 from __future__ import annotations
-import csv
-import os
 import sys
 from attackcti import attack_client
 import logging
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from dataclasses import field
-from typing import Annotated,Any
-from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
+from typing import Any
+from pathlib import Path
+from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment, MitreTactics
 from contentctl.objects.config import validate
 from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE
 logging.getLogger('taxii2client').setLevel(logging.CRITICAL)
@@ -21,12 +20,12 @@ class AttackEnrichment(BaseModel):
     @staticmethod
     def getAttackEnrichment(config:validate)->AttackEnrichment:
         enrichment = AttackEnrichment(use_enrichment=config.enrichments)
-        _ = enrichment.get_attack_lookup(str(config.path))
+        _ = enrichment.get_attack_lookup(config.mitre_cti_repo_path, config.enrichments)
         return enrichment
     
     def getEnrichmentByMitreID(self, mitre_id:MITRE_ATTACK_ID_TYPE)->MitreAttackEnrichment:
         if not self.use_enrichment:
-            raise Exception(f"Error, trying to add Mitre Enrichment, but use_enrichment was set to False")
+            raise Exception("Error, trying to add Mitre Enrichment, but use_enrichment was set to False")
         
         enrichment = self.data.get(mitre_id, None)
         if enrichment is not None:
@@ -34,71 +33,69 @@ class AttackEnrichment(BaseModel):
         else:
             raise Exception(f"Error, Unable to find Mitre Enrichment for MitreID {mitre_id}")
         
-    def addMitreIDViaGroupNames(self, technique:dict, tactics:list[str], groupNames:list[str])->None:
+    def addMitreIDViaGroupNames(self, technique:dict[str,Any], tactics:list[str], groupNames:list[str])->None:
         technique_id = technique['technique_id']
         technique_obj = technique['technique']
         tactics.sort()
         
         if technique_id in self.data:
             raise Exception(f"Error, trying to redefine MITRE ID '{technique_id}'")
-        self.data[technique_id] = MitreAttackEnrichment(mitre_attack_id=technique_id, 
-                                                        mitre_attack_technique=technique_obj, 
-                                                        mitre_attack_tactics=tactics, 
-                                                        mitre_attack_groups=groupNames,
-                                                        mitre_attack_group_objects=[]) 
+        self.data[technique_id] = MitreAttackEnrichment.model_validate({'mitre_attack_id':technique_id, 
+                                                                        'mitre_attack_technique':technique_obj, 
+                                                                        'mitre_attack_tactics':tactics, 
+                                                                        'mitre_attack_groups':groupNames,
+                                                                        'mitre_attack_group_objects':[]}) 
 
-    def addMitreIDViaGroupObjects(self, technique:dict, tactics:list[str],  groupObjects:list[dict[str,Any]])->None:
+    def addMitreIDViaGroupObjects(self, technique:dict[str,Any], tactics:list[MitreTactics],  groupDicts:list[dict[str,Any]])->None:
         technique_id = technique['technique_id']
         technique_obj = technique['technique']
         tactics.sort()
         
-        groupNames:list[str] = sorted([group['group'] for group in groupObjects])
+        groupNames:list[str] = sorted([group['group'] for group in groupDicts])
         
         if technique_id in self.data:
             raise Exception(f"Error, trying to redefine MITRE ID '{technique_id}'")
-        self.data[technique_id] = MitreAttackEnrichment(mitre_attack_id=technique_id, 
-                                                        mitre_attack_technique=technique_obj, 
-                                                        mitre_attack_tactics=tactics, 
-                                                        mitre_attack_groups=groupNames,
-                                                        mitre_attack_group_objects=groupObjects)
+        
+        self.data[technique_id] = MitreAttackEnrichment.model_validate({'mitre_attack_id': technique_id, 
+                                                                        'mitre_attack_technique': technique_obj, 
+                                                                        'mitre_attack_tactics': tactics, 
+                                                                        'mitre_attack_groups': groupNames,
+                                                                        'mitre_attack_group_objects': groupDicts})
 
     
-    def get_attack_lookup(self, input_path: str, store_csv: bool = False, force_cached_or_offline: bool = False, skip_enrichment:bool = False) -> dict:
-        if not self.use_enrichment:
-            return {}
-        print("Getting MITRE Attack Enrichment Data. This may take some time...")
-        attack_lookup = dict()
-        file_path = os.path.join(input_path, "app_template", "lookups", "mitre_enrichment.csv")
-
-        if skip_enrichment is True:
-            print("Skipping enrichment")
+    def get_attack_lookup(self, input_path: Path, enrichments:bool = False) -> dict[str,MitreAttackEnrichment]:            
+        attack_lookup:dict[str,MitreAttackEnrichment] = {}
+        if not enrichments:
             return attack_lookup
+        
         try:
-
-            if force_cached_or_offline is True:
-                raise(Exception("WARNING - Using cached MITRE Attack Enrichment.  Attack Enrichment may be out of date. Only use this setting for offline environments and development purposes."))
-            print(f"\r{'Client'.rjust(23)}: [{0:3.0f}%]...", end="", flush=True)
-            lift = attack_client()
-            print(f"\r{'Client'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
+            print(f"Performing MITRE Enrichment using the repository at {input_path}...",end="", flush=True)
+            # The existence of the input_path is validated during cli argument validation, but it is 
+            # possible that the repo is in the wrong format. If the following directories do not
+            # exist, then attack_client will fall back to resolving via REST API. We do not 
+            # want this as it is slow and error prone, so we will force an exception to 
+            # be generated.
+            enterprise_path = input_path/"enterprise-attack"
+            mobile_path = input_path/"ics-attack"
+            ics_path = input_path/"mobile-attack"
+            if not (enterprise_path.is_dir() and mobile_path.is_dir() and ics_path.is_dir()):
+                raise FileNotFoundError("One or more of the following paths does not exist: "
+                                        f"{[str(enterprise_path),str(mobile_path),str(ics_path)]}. "
+                                        f"Please ensure that the {input_path} directory "
+                                        "has been git cloned correctly.") 
+            lift = attack_client(
+                local_paths= {
+                    "enterprise":str(enterprise_path),
+                    "mobile":str(mobile_path),
+                    "ics":str(ics_path)
+                }
+            )
             
-            print(f"\r{'Techniques'.rjust(23)}: [{0.0:3.0f}%]...", end="", flush=True)
             all_enterprise_techniques = lift.get_enterprise_techniques(stix_format=False)
-            
-            print(f"\r{'Techniques'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
-            
-            print(f"\r{'Relationships'.rjust(23)}: [{0.0:3.0f}%]...", end="", flush=True)
             enterprise_relationships = lift.get_enterprise_relationships(stix_format=False)
-            print(f"\r{'Relationships'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
-            
-            print(f"\r{'Groups'.rjust(23)}: [{0:3.0f}%]...", end="", flush=True)
             enterprise_groups = lift.get_enterprise_groups(stix_format=False)
-            print(f"\r{'Groups'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
-            
             
-            for index, technique in enumerate(all_enterprise_techniques):
-                progress_percent = ((index+1)/len(all_enterprise_techniques)) * 100
-                if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()):
-                    print(f"\r\t{'MITRE Technique Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
+            for technique in all_enterprise_techniques:
                 apt_groups:list[dict[str,Any]] = []
                 for relationship in enterprise_relationships:
                     if (relationship['target_object'] == technique['id']) and relationship['source_object'].startswith('intrusion-set'):
@@ -115,39 +112,10 @@ class AttackEnrichment(BaseModel):
                 self.addMitreIDViaGroupObjects(technique, tactics, apt_groups)
                 attack_lookup[technique['technique_id']] = {'technique': technique['technique'], 'tactics': tactics, 'groups': apt_groups}
 
-            if store_csv:
-                f = open(file_path, 'w')
-                writer = csv.writer(f)
-                writer.writerow(['mitre_id', 'technique', 'tactics' ,'groups'])
-                for key in attack_lookup.keys():
-                    if len(attack_lookup[key]['groups']) == 0:
-                        groups = 'no'
-                    else:
-                        groups = '|'.join(attack_lookup[key]['groups'])
-                    
-                    writer.writerow([
-                        key,
-                        attack_lookup[key]['technique'],
-                        '|'.join(attack_lookup[key]['tactics']),
-                        groups
-                    ])
-                
-                f.close()
 
+        
         except Exception as err:
-            print(f'\nError: {str(err)}')
-            print('Use local copy app_template/lookups/mitre_enrichment.csv')
-            with open(file_path, mode='r') as inp:
-                reader = csv.reader(inp)
-                attack_lookup = {rows[0]:{'technique': rows[1], 'tactics': rows[2].split('|'), 'groups': rows[3].split('|')} for rows in reader}
-            attack_lookup.pop('mitre_id')
-            for key in attack_lookup.keys():
-                technique_input = {'technique_id': key , 'technique': attack_lookup[key]['technique'] }
-                tactics_input = attack_lookup[key]['tactics']
-                groups_input = attack_lookup[key]['groups']
-                self.addMitreIDViaGroupNames(technique=technique_input, tactics=tactics_input, groups=groups_input)
-            
-                
-
+            raise Exception(f"Error getting MITRE Enrichment: {str(err)}")
+        
         print("Done!")
         return attack_lookup

contentctl/enrichments/cve_enrichment.py

@@ -5,7 +5,7 @@ import os
 import shelve
 import time
 from typing import Annotated, Any, Union, TYPE_CHECKING
-from pydantic import BaseModel,Field, computed_field
+from pydantic import ConfigDict, BaseModel,Field, computed_field
 from decimal import Decimal
 from requests.exceptions import ReadTimeout
 from contentctl.objects.annotated_types import CVE_TYPE
@@ -32,13 +32,12 @@ class CveEnrichmentObj(BaseModel):
 class CveEnrichment(BaseModel):
     use_enrichment: bool = True
     cve_api_obj: Union[CVESearch,None] = None
-    
 
-    class Config:
-        # Arbitrary_types are allowed to let us use the CVESearch Object
-        arbitrary_types_allowed = True
-        frozen = True
-        
+    # Arbitrary_types are allowed to let us use the CVESearch Object
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True
+    )
 
     @staticmethod
     def getCveEnrichment(config:validate, timeout_seconds:int=10, force_disable_enrichment:bool=True)->CveEnrichment:

contentctl/helper/splunk_app.py

@@ -1,20 +1,20 @@
-import os
-import time
 import json
+from typing import Optional, Collection
+from pathlib import Path
 import xml.etree.ElementTree as ET
-from typing import List, Tuple, Optional
 from urllib.parse import urlencode
 
 import requests
 import urllib3
 import xmltodict
 from requests.adapters import HTTPAdapter
-from requests.packages.urllib3.util.retry import Retry
+from urllib3.util.retry import Retry
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 MAX_RETRY = 3
 
+
 class APIEndPoint:
     """
     Class which contains Static Endpoint
@@ -27,6 +27,7 @@ class APIEndPoint:
     SPLUNK_BASE_GET_UID_REDIRECT = "https://apps.splunk.com/apps/id/{app_name_id}"
     SPLUNK_BASE_APP_INFO = "https://splunkbase.splunk.com/api/v1/app/{app_uid}"
 
+
 class RetryConstant:
     """
     Class which contains Retry Constant
@@ -53,11 +54,11 @@ class SplunkApp:
 
     @staticmethod
     def requests_retry_session(
-        retries=RetryConstant.RETRY_COUNT,
-        backoff_factor=1,
-        status_forcelist=(500, 502, 503, 504),
-        session=None,
-    ):
+        retries: int = RetryConstant.RETRY_COUNT,
+        backoff_factor: int = 1,
+        status_forcelist: Collection[int] = (500, 502, 503, 504),
+        session: requests.Session | None = None,
+    ) -> requests.Session:
         session = session or requests.Session()
         retry = Retry(
             total=retries,
@@ -260,4 +261,134 @@ class SplunkApp:
 
         # parse out the version number and fetch the download URL
         self.latest_version = info_url.split("/")[-1]
-        self.latest_version_download_url = self.__fetch_url_latest_version_download(info_url)
+        self.latest_version_download_url = self.__fetch_url_latest_version_download(info_url)
+
+    def __get_splunk_base_session_token(self, username: str, password: str) -> str:
+        """
+        This method will generate Splunk base session token
+
+        :param username: Splunkbase username
+        :type username: str
+        :param password: Splunkbase password
+        :type password: str
+
+        :return: Splunk base session token
+        :rtype: str
+        """
+        # Data payload for fetch splunk base session token
+        payload = urlencode(
+            {
+                "username": username,
+                "password": password,
+            }
+        )
+
+        headers = {
+            "content-type": "application/x-www-form-urlencoded",
+            "cache-control": "no-cache",
+        }
+
+        response = requests.request(
+            "POST",
+            APIEndPoint.SPLUNK_BASE_AUTH_URL,
+            data=payload,
+            headers=headers,
+        )
+
+        token_value = ""
+
+        if response.status_code != 200:
+            msg = (
+                f"Error occurred while executing the rest call for splunk base authentication api,"
+                f"{response.content}"
+            )
+            raise Exception(msg)
+        else:
+            root = ET.fromstring(response.content)
+            token_value = root.find("{http://www.w3.org/2005/Atom}id").text.strip()
+        return token_value
+
+    def download(
+            self,
+            out: Path,
+            username: str,
+            password: str,
+            is_dir: bool = False,
+            overwrite: bool = False
+    ) -> Path:
+        """
+        Given an output path, download the app to the specified location
+
+        :param out: the Path to download the app to
+        :type out: :class:`pathlib.Path`
+        :param username: Splunkbase username
+        :type username: str
+        :param password: Splunkbase password
+        :type password: str
+        :param is_dir: a flag indicating whether out is directory, otherwise a file (default: False)
+        :type is_dir: bool
+        :param overwrite: a flag indicating whether we can overwrite the file at out or not
+        :type overwrite: bool
+
+        :returns path: the Path the download was written to (needed when is_dir is True)
+        :rtype: :class:`pathlib.Path`
+        """
+        # Get the Splunkbase session token
+        token = self.__get_splunk_base_session_token(username, password)
+        response = requests.request(
+            "GET",
+            self.latest_version_download_url,
+            cookies={
+                "sessionid": token
+            }
+        )
+
+        # If the provided output path was a directory we need to try and pull the filename from the
+        # response headers
+        if is_dir:
+            try:
+                # Pull 'Content-Disposition' from the headers
+                content_disposition: str = response.headers['Content-Disposition']
+
+                # Attempt to parse the filename as a KV
+                key, value = content_disposition.strip().split("=")
+                if key != "attachment;filename":
+                    raise ValueError(f"Unexpected key in 'Content-Disposition' KV pair: {key}")
+
+                # Validate the filename is the expected .tgz file
+                filename = Path(value.strip().strip('"'))
+                if filename.suffixes != [".tgz"]:
+                    raise ValueError(f"Filename has unexpected extension(s): {filename.suffixes}")
+                out = Path(out, filename)
+            except KeyError as e:
+                raise KeyError(
+                    f"Unable to properly extract 'Content-Disposition' from response headers: {e}"
+                ) from e
+            except ValueError as e:
+                raise ValueError(
+                    f"Unable to parse filename from 'Content-Disposition' header: {e}"
+                ) from e
+
+        # Ensure the output path is not already occupied
+        if out.exists() and not overwrite:
+            msg = (
+                f"File already exists at {out}, cannot download the app."
+            )
+            raise Exception(msg)
+
+        # Make any parent directories as needed
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        # Check for HTTP errors
+        if response.status_code != 200:
+            msg = (
+                f"Error occurred while executing the rest call for splunk base authentication api,"
+                f"{response.content}"
+            )
+            raise Exception(msg)
+
+        # Write the app to disk
+        with open(out, "wb") as file:
+            file.write(response.content)
+
+        return out

contentctl/input/director.py

@@ -1,41 +1,29 @@
 import os
 import sys
-import pathlib
-from typing import Union
+from pathlib import Path
 from dataclasses import dataclass, field
 from pydantic import ValidationError
 from uuid import UUID
 from contentctl.input.yml_reader import YmlReader
 
-
 from contentctl.objects.detection import Detection
 from contentctl.objects.story import Story
 
-from contentctl.objects.enums import SecurityContentProduct
 from contentctl.objects.baseline import Baseline
 from contentctl.objects.investigation import Investigation
 from contentctl.objects.playbook import Playbook
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.macro import Macro
 from contentctl.objects.lookup import Lookup
-from contentctl.objects.ssa_detection import SSADetection
-from contentctl.objects.atomic import AtomicTest
+from contentctl.objects.atomic import AtomicEnrichment
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.data_source import DataSource
-from contentctl.objects.event_source import EventSource
-
+from contentctl.objects.dashboard import Dashboard
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 
 from contentctl.objects.config import validate
 from contentctl.objects.enums import SecurityContentType
-
-from contentctl.objects.enums import DetectionStatus
-from contentctl.helper.utils import Utils
-
-from contentctl.objects.enums import SecurityContentType
-
-from contentctl.objects.enums import DetectionStatus 
 from contentctl.helper.utils import Utils
 
 
@@ -43,7 +31,7 @@ from contentctl.helper.utils import Utils
 class DirectorOutputDto:
     # Atomic Tests are first because parsing them 
     # is far quicker than attack_enrichment
-    atomic_tests: None | list[AtomicTest]
+    atomic_enrichment: AtomicEnrichment
     attack_enrichment: AttackEnrichment
     cve_enrichment: CveEnrichment
     detections: list[Detection]
@@ -54,16 +42,15 @@ class DirectorOutputDto:
     macros: list[Macro]
     lookups: list[Lookup]
     deployments: list[Deployment]
+    dashboards: list[Dashboard]
+
     data_sources: list[DataSource]
     name_to_content_map: dict[str, SecurityContentObject] = field(default_factory=dict)
     uuid_to_content_map: dict[UUID, SecurityContentObject] = field(default_factory=dict)
 
     def addContentToDictMappings(self, content: SecurityContentObject):
         content_name = content.name
-        if isinstance(content, SSADetection):
-            # Since SSA detections may have the same name as ESCU detection,
-            # for this function we prepend 'SSA ' to the name.
-            content_name = f"SSA {content_name}"
+        
         
         if content_name in self.name_to_content_map:
             raise ValueError(
@@ -95,6 +82,9 @@ class DirectorOutputDto:
             self.stories.append(content)
         elif isinstance(content, Detection):
             self.detections.append(content)
+        elif isinstance(content, Dashboard):
+            self.dashboards.append(content)            
+
         elif isinstance(content, DataSource):
             self.data_sources.append(content)
         else:
@@ -122,7 +112,7 @@ class Director():
         self.createSecurityContent(SecurityContentType.data_sources)
         self.createSecurityContent(SecurityContentType.playbooks)
         self.createSecurityContent(SecurityContentType.detections)
-
+        self.createSecurityContent(SecurityContentType.dashboards)
         
         from contentctl.objects.abstract_security_content_objects.detection_abstract import MISSING_SOURCES
         if len(MISSING_SOURCES) > 0:
@@ -144,17 +134,18 @@ class Director():
             SecurityContentType.playbooks,
             SecurityContentType.detections,
             SecurityContentType.data_sources,
+            SecurityContentType.dashboards
         ]:
             files = Utils.get_all_yml_files_from_directory(
                 os.path.join(self.input_dto.path, str(contentType.name))
             )
             security_content_files = [
-                f for f in files if not f.name.startswith("ssa___")
+                f for f in files 
             ]
         else:
-            raise (Exception(f"Cannot createSecurityContent for unknown product."))
+            raise (Exception(f"Cannot createSecurityContent for unknown product {contentType}."))
 
-        validation_errors = []
+        validation_errors:list[tuple[Path,ValueError]] = []
 
         already_ran = False
         progress_percent = 0
@@ -196,6 +187,10 @@ class Director():
                 elif contentType == SecurityContentType.detections:
                     detection = Detection.model_validate(modelDict, context={"output_dto":self.output_dto, "app":self.input_dto.app})
                     self.output_dto.addContentToDictMappings(detection)
+                
+                elif contentType == SecurityContentType.dashboards:
+                        dashboard = Dashboard.model_validate(modelDict,context={"output_dto":self.output_dto})
+                        self.output_dto.addContentToDictMappings(dashboard)
 
                 elif contentType == SecurityContentType.data_sources:
                     data_source = DataSource.model_validate(

contentctl/input/new_content_questions.py

@@ -1,7 +1,11 @@
+from typing import Any
+from contentctl.objects.enums import DataSource
+
+
 class NewContentQuestions:
 
     @classmethod
-    def get_questions_detection(self) -> list:
+    def get_questions_detection(cls) -> list[dict[str,Any]]:
         questions = [
             {
                 "type": "text",
@@ -45,46 +49,9 @@ class NewContentQuestions:
                 'type': 'checkbox',
                 'message': 'Your data source',
                 'name': 'data_source',
-                'choices': [
-                    "OSQuery ES Process Events",
-                    "Powershell 4104",
-                    "Sysmon Event ID 1",
-                    "Sysmon Event ID 3",
-                    "Sysmon Event ID 5",
-                    "Sysmon Event ID 6",
-                    "Sysmon Event ID 7",
-                    "Sysmon Event ID 8",
-                    "Sysmon Event ID 9",
-                    "Sysmon Event ID 10",
-                    "Sysmon Event ID 11",
-                    "Sysmon Event ID 13",
-                    "Sysmon Event ID 15",
-                    "Sysmon Event ID 20",
-                    "Sysmon Event ID 21",
-                    "Sysmon Event ID 22",
-                    "Sysmon Event ID 23",
-                    "Windows Security 4624",
-                    "Windows Security 4625",
-                    "Windows Security 4648",
-                    "Windows Security 4663",
-                    "Windows Security 4688",
-                    "Windows Security 4698",
-                    "Windows Security 4703",
-                    "Windows Security 4720",
-                    "Windows Security 4732",
-                    "Windows Security 4738",
-                    "Windows Security 4741",
-                    "Windows Security 4742",
-                    "Windows Security 4768",
-                    "Windows Security 4769",
-                    "Windows Security 4771",
-                    "Windows Security 4776",
-                    "Windows Security 4781",
-                    "Windows Security 4798",
-                    "Windows Security 5136",
-                    "Windows Security 5145",
-                    "Windows System 7045"
-                ]
+                #In the future, we should dynamically populate this from the DataSource Objects we have parsed from the data_sources directory
+                'choices': sorted(DataSource._value2member_map_ )
+                
             },
             {
                 "type": "text",
@@ -116,7 +83,7 @@ class NewContentQuestions:
         return questions
 
     @classmethod
-    def get_questions_story(self) -> list:
+    def get_questions_story(cls)-> list[dict[str,Any]]:
         questions = [
             {
                 "type": "text",