contentctl 1.0.0__py3-none-any.whl

contentctl/Res.yml

@@ -0,0 +1,102 @@
+repo_path: /tmp/forDev/security_content 
+main_branch: "develop"
+test_branch: "develop"
+apps:
+  - uid: 6176
+    appid: "Splunk_TA_linux_sysmon"
+    title: "Add-on for Linux Sysmon"
+    release: "1.0.4"
+
+  - uid: 2757
+    appid: "Splunk_TA_paloalto"
+    title: "Palo Alto Networks Add-on for Splunk"
+    release: "7.1.0"
+
+  - uid: 2882
+    appid: "Splunk_SA_Scientific_Python_linux_x86_64"
+    title: "Python for Scientific Computing (for Linux 64-bit)"
+    release: "4.0.0"
+
+  - uid: 3719
+    appid: "Splunk_TA_aws-kinesis-firehose"
+    title: "Splunk Add-on for Amazon Kinesis Firehose"
+    release: "1.3.2"
+  
+  - uid: 4055
+    appid: "splunk_ta_o365"
+    title: "Splunk Add-on for Microsoft Office 365"
+    release: "4.1.0"
+
+  - uid: 5742
+    appid: "mitre_attck_heatmap"
+    title: "MITRE ATTCK Heatmap for Splunk"
+    release: "1.4.0"
+
+  - uid: 742
+    appid: "Splunk_TA_windows"
+    title: "Splunk Add-on for Microsoft Windows"
+    release: "8.5.0"
+
+  - uid: 3258
+    appid: "Splunk_TA_nginx"
+    title: "Splunk Add-on for NGINX"
+    release: "3.2.0"
+
+  - uid: 5238
+    appid: "Splunk_TA_stream"
+    title: "Splunk Add-on for Stream Forwarders"
+    release: "8.1.0"
+
+  - uid: 5234
+    appid: "Splunk_TA_stream_wire_data"
+    title: "Splunk Add-on for Stream Wire Data"
+    release: "8.1.0"
+
+  - uid: 5709
+    appid: "Splunk_TA_microsoft_sysmon"
+    title: "Splunk Add-on for Sysmon"
+    release: "3.0.0"
+
+  - uid: 833
+    appid: "Splunk_TA_nix"
+    title: "Splunk Add-on for Unix and Linux"
+    release: "8.7.0"
+
+  - uid: 1809
+    appid: "splunk_app_stream"
+    title: "Splunk App for Stream"
+    release: "8.1.0"
+
+  - uid: 1621
+    appid: "Splunk_SA_CIM"
+    title: "Splunk Common Information Model (CIM)"
+    release: "5.0.2"
+
+  - uid: 2890
+    appid: "Splunk_ML_Toolkit"
+    title: "Splunk Machine Learning Toolkit"
+    release: "5.3.3"
+
+  - uid: 5466
+    appid: "Splunk_TA_zeek"
+    title: "TA for Zeek"
+    release: "1.0.5"
+
+  - uid: 3110
+    appid: "Splunk_TA_microsoft-cloudservices"
+    title: "Splunk Add-on for Microsoft Cloud Services"
+    release: "4.5.0"
+
+  - uid: 2734
+    appid: "utbox"
+    title: "URL Toolbox"
+    release: "1.9.2"
+  
+  - uid: 3449
+    appid: "DA-ESS-ContentUpdate"
+    title: "Splunk ES Content Update"
+    release: "3.51.0"
+  
+
+splunkbase_username: notvalid
+splunkbase_password: notvalid

contentctl/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.0'

contentctl/actions/deploy.py

@@ -0,0 +1,147 @@
+import os
+import sys
+import json
+import requests
+from requests.auth import HTTPBasicAuth
+
+from dataclasses import dataclass
+from configparser import RawConfigParser
+import splunklib.client as client
+
+from contentctl.objects.config import Config
+
+
+@dataclass(frozen=True)
+class DeployInputDto:
+    path: str
+    config: Config
+
+
+class Deploy:
+    def fix_newlines_in_conf_files(self, conf_path: str) -> RawConfigParser:
+        parser = RawConfigParser()
+        with open(conf_path, "r") as conf_data_file:
+            conf_data = conf_data_file.read()
+
+        # ConfigParser cannot read multipleline strings that simply escape the newline character with \
+        # To include a newline, you need to include a space at the beginning of the newline.
+        # We will simply replace all \NEWLINE with NEWLINESPACE (removing the leading literal \).
+        # We will discuss whether we intend to make these changes to the underlying conf files
+        # or just apply the changes here
+        conf_data = conf_data.replace("\\\n", "\n ")
+
+        parser.read_string(conf_data)
+        return parser
+
+    def execute(self, input_dto: DeployInputDto) -> None:
+
+        splunk_args = {
+            "host": input_dto.config.deploy.server,
+            "port": 8089,
+            "username": input_dto.config.deploy.username,
+            "password": input_dto.config.deploy.password,
+            "owner": "nobody",
+            "app": input_dto.config.deploy.app,
+        }
+        service = client.connect(**splunk_args)
+
+        macros_parser = self.fix_newlines_in_conf_files(
+            os.path.join(
+                input_dto.path,
+                input_dto.config.build.splunk_app.path,
+                "default",
+                "macros.conf",
+            )
+        )
+        import tqdm
+
+        bar_format_macros = (
+            f"Deploying macros        "
+            + "{percentage:3.0f}%[{bar:20}]"
+            + "[{n_fmt}/{total_fmt} | ETA: {remaining}]"
+        )
+        bar_format_detections = (
+            f"Deploying saved searches"
+            + "{percentage:3.0f}%[{bar:20}]"
+            + "[{n_fmt}/{total_fmt} | ETA: {remaining}]"
+        )
+        for section in tqdm.tqdm(
+            macros_parser.sections(), bar_format=bar_format_macros
+        ):
+            try:
+                service.post("properties/macros", __stanza=section)
+                service.post("properties/macros/" + section, **macros_parser[section])
+                # print("Deployed macro: " + section)
+            except Exception as e:
+                tqdm.tqdm.write(f"Error deploying macro {section}: {str(e)}")
+
+        detection_parser = RawConfigParser()
+        detection_parser = self.fix_newlines_in_conf_files(
+            os.path.join(
+                input_dto.path,
+                input_dto.config.build.splunk_app.path,
+                "default",
+                "savedsearches.conf",
+            )
+        )
+        try:
+            service.delete("saved/searches/MSCA - Anomalous usage of 7zip - Rule")
+        except Exception as e:
+            pass
+
+        for section in tqdm.tqdm(
+            detection_parser.sections(), bar_format=bar_format_detections
+        ):
+            try:
+                if section.startswith(input_dto.config.build.splunk_app.prefix):
+                    params = detection_parser[section]
+                    params["name"] = section
+                    response_actions = []
+                    if (
+                        input_dto.config.detection_configuration.notable
+                        and input_dto.config.detection_configuration.notable.rule_description
+                    ):
+                        response_actions.append("notable")
+                    if (
+                        input_dto.config.detection_configuration.rba
+                        and input_dto.config.detection_configuration.rba.enabled
+                    ):
+                        response_actions.append("risk")
+                    params["actions"] = ",".join(response_actions)
+                    params["request.ui_dispatch_app"] = "ES Content Updates"
+                    params["request.ui_dispatch_view"] = "ES Content Updates"
+                    params["alert_type"] = params.pop("counttype")
+                    params["alert_comparator"] = params.pop("relation")
+                    params["alert_threshold"] = params.pop("quantity")
+                    params.pop("enablesched")
+
+                    service.post("saved/searches", **params)
+
+                    # print("Deployed detection: " + params["name"])
+            except Exception as e:
+                tqdm.tqdm.write(f"Error deploying saved search {section}: {str(e)}")
+
+        # story_parser = RawConfigParser()
+        # story_parser.read(os.path.join(input_dto.path, input_dto.config.build.splunk_app.path, "default", "analyticstories.conf"))
+
+        # for section in story_parser.sections():
+        #     if section.startswith("analytic_story"):
+        #         params = story_parser[section]
+        #         params = dict(params.items())
+        #         params["spec_version"] = 1
+        #         params["version"] = 1
+        #         name = section[17:]
+        #         #service.post('services/analyticstories/configs/analytic_story', name=name, content=json.dumps(params))
+
+        #         url = "https://3.72.220.157:8089/services/analyticstories/configs/analytic_story"
+        #         data = dict()
+        #         data["name"] = name
+        #         data["content"] = params
+        #         print(json.dumps(data))
+        #         response = requests.post(
+        #             url,
+        #             auth=HTTPBasicAuth('admin', 'fgWFshd0mm7eErMj9qX'),
+        #             data=json.dumps(data),
+        #             verify=False
+        #         )
+        #         print(response.text)

contentctl/actions/detection_testing/DataManipulation.py

@@ -0,0 +1,149 @@
+import json
+from datetime import datetime
+from datetime import timedelta
+#import fileinput
+import os
+import re
+import io
+
+class DataManipulation:
+
+    def manipulate_timestamp(self, file_path, sourcetype, source):
+
+
+        #print('Updating timestamps in attack_data before replaying')
+        if sourcetype == 'aws:cloudtrail':
+            self.manipulate_timestamp_cloudtrail(file_path)
+
+        if source == 'WinEventLog:System' or source == 'WinEventLog:Security':
+            self.manipulate_timestamp_windows_event_log_raw(file_path)
+
+        if source == 'exchange':
+            self.manipulate_timestamp_exchange_logs(file_path)
+
+
+    def manipulate_timestamp_exchange_logs(self, path):
+        f = io.open(path, "r", encoding="utf-8")
+
+        first_line = f.readline()
+        d = json.loads(first_line)
+        latest_event  = datetime.strptime(d["CreationTime"],"%Y-%m-%dT%H:%M:%S")
+
+        now = datetime.now()
+        now = now.strftime("%Y-%m-%dT%H:%M:%S")
+        now = datetime.strptime(now,"%Y-%m-%dT%H:%M:%S")
+
+        difference = now - latest_event
+        f.close()
+
+        #Mimic the behavior of fileinput but in a threadsafe way
+        #Rename the file, which fileinput does for inplace.
+        #Note that path will now be the new file
+        original_backup_file = f"{path}.bak"    
+        os.rename(path, original_backup_file)
+        
+        with open(original_backup_file, "r") as original_file:
+            with open(path, "w") as new_file:
+                for line in original_file:
+                    d = json.loads(line)
+                    original_time = datetime.strptime(d["CreationTime"],"%Y-%m-%dT%H:%M:%S")
+                    new_time = (difference + original_time)
+
+                    original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S")
+                    new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S")
+                    #There is no end character appended, no need for end=''
+                    new_file.write(line.replace(original_time, new_time))
+
+
+        os.remove(original_backup_file)
+
+    def manipulate_timestamp_windows_event_log_raw(self, path):
+
+        f = io.open(path, "r", encoding="utf-8")
+        self.now = datetime.now()
+        self.now = self.now.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        self.now = datetime.strptime(self.now,"%Y-%m-%dT%H:%M:%S.%fZ")
+
+        # read raw logs
+        regex = r'\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} [AP]M'
+        data = f.read()
+        lst_matches = re.findall(regex, data)
+        if len(lst_matches) > 0:
+            latest_event  = datetime.strptime(lst_matches[-1],"%m/%d/%Y %I:%M:%S %p")
+            self.difference = self.now - latest_event
+            f.close()
+
+            result = re.sub(regex, self.replacement_function, data)
+
+            with io.open(path, "w+", encoding='utf8') as f:
+                f.write(result)
+        else:
+            f.close()
+            return
+
+
+    def replacement_function(self, match):
+        try:
+            event_time = datetime.strptime(match.group(),"%m/%d/%Y %I:%M:%S %p")
+            new_time = self.difference + event_time
+            return new_time.strftime("%m/%d/%Y %I:%M:%S %p")
+        except Exception as e:
+            self.logger.error("Error in timestamp replacement occured: " + str(e))
+            return match.group()
+
+
+    def manipulate_timestamp_cloudtrail(self, path):
+        
+
+        f = io.open(path, "r", encoding="utf-8")
+
+        try:
+            first_line = f.readline()
+            d = json.loads(first_line)
+            latest_event  = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%S.%fZ")
+
+            now = datetime.now()
+            now = now.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+            now = datetime.strptime(now,"%Y-%m-%dT%H:%M:%S.%fZ")
+        except ValueError:
+            first_line = f.readline()
+            d = json.loads(first_line)
+            latest_event  = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%SZ")
+
+            now = datetime.now()
+            now = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+            now = datetime.strptime(now,"%Y-%m-%dT%H:%M:%SZ")
+
+        difference = now - latest_event
+        f.close()
+
+
+
+        #Mimic the behavior of fileinput but in a threadsafe way
+        #Rename the file, which fileinput does for inplace.
+        #Note that path will now be the new file
+        original_backup_file = f"{path}.bak"    
+        os.rename(path, original_backup_file)
+        
+        with open(original_backup_file, "r") as original_file:
+            with open(path, "w") as new_file:
+                for line in original_file:
+                    try:
+                        d = json.loads(line)
+                        original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%S.%fZ")
+                        new_time = (difference + original_time)
+
+                        original_time = original_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+                        new_time = new_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+                        new_file.write(line.replace(original_time, new_time))
+                    except ValueError:
+                        d = json.loads(line)
+                        original_time = datetime.strptime(d["eventTime"],"%Y-%m-%dT%H:%M:%SZ")
+                        new_time = (difference + original_time)
+
+                        original_time = original_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+                        new_time = new_time.strftime("%Y-%m-%dT%H:%M:%SZ")
+                        new_file.write(line.replace(original_time, new_time))
+
+
+        os.remove(original_backup_file)

contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -0,0 +1,189 @@
+from contentctl.objects.test_config import TestConfig
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
+    DetectionTestingInfrastructure,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
+    DetectionTestingInfrastructureContainer,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
+    DetectionTestingInfrastructureServer,
+)
+
+from contentctl.objects.app import App
+import pathlib
+import os
+from contentctl.helper.utils import Utils
+from urllib.parse import urlparse
+import time
+from copy import deepcopy
+from contentctl.objects.enums import DetectionTestingTargetInfrastructure
+import signal
+import datetime
+
+# from queue import Queue
+
+CONTAINER_APP_PATH = pathlib.Path("apps")
+
+from dataclasses import dataclass
+
+# import threading
+import ctypes
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
+    DetectionTestingInfrastructure,
+    DetectionTestingManagerOutputDto,
+)
+from contentctl.actions.detection_testing.views.DetectionTestingView import (
+    DetectionTestingView,
+)
+
+from contentctl.objects.enums import PostTestBehavior
+
+from pydantic import BaseModel, Field
+from contentctl.input.director import DirectorOutputDto
+from contentctl.objects.detection import Detection
+
+
+import concurrent.futures
+
+import tqdm
+
+
+@dataclass(frozen=False)
+class DetectionTestingManagerInputDto:
+    config: TestConfig
+    testContent: DirectorOutputDto
+    views: list[DetectionTestingView]
+
+
+class DetectionTestingManager(BaseModel):
+    input_dto: DetectionTestingManagerInputDto
+    output_dto: DetectionTestingManagerOutputDto
+    detectionTestingInfrastructureObjects: list[DetectionTestingInfrastructure] = []
+
+    def setup(self):
+        # Some views, such as the Web View, will require some initial setup.
+        # for view in self.input_dto.views:
+        #    view.setup()
+
+        # for content in self.input_dto.testContent.detections:
+        #    self.pending_queue.put(content)
+        self.output_dto.inputQueue = self.input_dto.testContent.detections
+        self.create_DetectionTestingInfrastructureObjects()
+
+    def execute(self) -> DetectionTestingManagerOutputDto:
+        def sigint_handler(signum, frame):
+            print("SIGINT (Ctrl-C Received.  Shutting down test...)")
+            self.output_dto.terminate = True
+            if self.input_dto.config.post_test_behavior in [
+                PostTestBehavior.always_pause,
+                PostTestBehavior.pause_on_failure,
+            ]:
+                # It is possible that we are stuck waiting at in input() prompt, so inject
+                # a newline '\r\n' which will cause that wait to stop
+                print("*******************************")
+                print(
+                    "If testing is paused and you are debugging a detection, you MUST hit CTRL-D at the prompt to complete shutdown."
+                )
+                print("*******************************")
+
+        signal.signal(signal.SIGINT, sigint_handler)
+
+        with concurrent.futures.ThreadPoolExecutor(
+            max_workers=self.input_dto.config.num_containers,
+        ) as instance_pool, concurrent.futures.ThreadPoolExecutor(
+            max_workers=len(self.input_dto.views)
+        ) as view_runner, concurrent.futures.ThreadPoolExecutor(
+            max_workers=self.input_dto.config.num_containers,
+        ) as view_shutdowner:
+
+            # Start all the views
+            future_views = {
+                view_runner.submit(view.setup): view for view in self.input_dto.views
+            }
+            # Configure all the instances
+            future_instances_setup = {
+                instance_pool.submit(instance.setup): instance
+                for instance in self.detectionTestingInfrastructureObjects
+            }
+
+            # Wait for all instances to be set up
+            for future in concurrent.futures.as_completed(future_instances_setup):
+                try:
+                    result = future.result()
+                except Exception as e:
+                    self.output_dto.terminate = True
+                    print(f"Error setting up container: {str(e)}")
+
+            # Start and wait for all tests to run
+            if not self.output_dto.terminate:
+                self.output_dto.start_time = datetime.datetime.now()
+                future_instances_execute = {
+                    instance_pool.submit(instance.execute): instance
+                    for instance in self.detectionTestingInfrastructureObjects
+                }
+                # What for execution to finish
+                for future in concurrent.futures.as_completed(future_instances_execute):
+                    try:
+                        result = future.result()
+                    except Exception as e:
+                        self.output_dto.terminate = True
+                        print(f"Error running in container: {str(e)}")
+
+            self.output_dto.terminate = True
+
+            future_views_shutdowner = {
+                view_shutdowner.submit(view.stop): view for view in self.input_dto.views
+            }
+            for future in concurrent.futures.as_completed(future_views_shutdowner):
+                try:
+                    result = future.result()
+                except Exception as e:
+                    print(f"Error stopping view: {str(e)}")
+
+            for future in concurrent.futures.as_completed(future_views):
+                try:
+                    result = future.result()
+                except Exception as e:
+                    print(f"Error running container: {str(e)}")
+
+        return self.output_dto
+
+    def create_DetectionTestingInfrastructureObjects(self):
+        import sys
+
+        for index in range(self.input_dto.config.num_containers):
+            instanceConfig = deepcopy(self.input_dto.config)
+            instanceConfig.api_port += index * 2
+            instanceConfig.hec_port += index * 2
+            instanceConfig.web_ui_port += index
+
+            instanceConfig.container_name = instanceConfig.container_name % (index,)
+
+            if (
+                self.input_dto.config.target_infrastructure
+                == DetectionTestingTargetInfrastructure.container
+            ):
+
+                self.detectionTestingInfrastructureObjects.append(
+                    DetectionTestingInfrastructureContainer(
+                        config=instanceConfig, sync_obj=self.output_dto
+                    )
+                )
+
+            elif (
+                self.input_dto.config.target_infrastructure
+                == DetectionTestingTargetInfrastructure.server
+            ):
+
+                self.detectionTestingInfrastructureObjects.append(
+                    DetectionTestingInfrastructureServer(
+                        config=instanceConfig, sync_obj=self.output_dto
+                    )
+                )
+
+            else:
+
+                print(
+                    f"Unsupported target infrastructure '{self.input_dto.config.target_infrastructure}'"
+                )
+                sys.exit(1)