contentctl 4.4.6__tar.gz → 5.0.0__tar.gz

{contentctl-4.4.6 → contentctl-5.0.0}/PKG-INFO

@@ -1,15 +1,16 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: contentctl
-Version: 4.4.6
+Version: 5.0.0
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT
 Author-email: research@splunk.com
-Requires-Python: >=3.11,<3.13
+Requires-Python: >=3.11,<3.14
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: PyYAML (>=6.0.2,<7.0.0)
 Requires-Dist: attackcti (>=0.4.0,<0.5.0)
@@ -17,7 +18,7 @@ Requires-Dist: bottle (>=0.12.25,<0.14.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: gitpython (>=3.1.43,<4.0.0)
 Requires-Dist: pycvesearch (>=1.2,<2.0)
-Requires-Dist: pydantic (>=2.8.2,<3.0.0)
+Requires-Dist: pydantic (>=2.9.2,<2.10.0)
 Requires-Dist: pygit2 (>=1.15.1,<2.0.0)
 Requires-Dist: questionary (>=2.0.1,<3.0.0)
 Requires-Dist: requests (>=2.32.3,<2.33.0)
@@ -25,7 +26,7 @@ Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
 Requires-Dist: setuptools (>=69.5.1,<76.0.0)
 Requires-Dist: splunk-sdk (>=2.0.2,<3.0.0)
 Requires-Dist: tqdm (>=4.66.5,<5.0.0)
-Requires-Dist: tyro (>=0.8.3,<0.9.0)
+Requires-Dist: tyro (>=0.9.2,<0.10.0)
 Requires-Dist: xmltodict (>=0.13,<0.15)
 Description-Content-Type: text/markdown
 

contentctl-5.0.0/contentctl/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

contentctl-5.0.0/contentctl/actions/build.py

@@ -0,0 +1,136 @@
+import shutil
+
+from dataclasses import dataclass
+
+from contentctl.input.director import DirectorOutputDto
+from contentctl.output.conf_output import ConfOutput
+from contentctl.output.conf_writer import ConfWriter
+from contentctl.output.api_json_output import ApiJsonOutput
+from contentctl.output.data_source_writer import DataSourceWriter
+from contentctl.objects.lookup import CSVLookup, Lookup_Type
+import pathlib
+import json
+import datetime
+import uuid
+
+from contentctl.objects.config import build
+
+
+@dataclass(frozen=True)
+class BuildInputDto:
+    director_output_dto: DirectorOutputDto
+    config: build
+
+
+class Build:
+    def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
+        if input_dto.config.build_app:
+            updated_conf_files: set[pathlib.Path] = set()
+            conf_output = ConfOutput(input_dto.config)
+
+            # Construct a path to a YML that does not actually exist.
+            # We mock this "fake" path since the YML does not exist.
+            # This ensures the checking for the existence of the CSV is correct
+            data_sources_fake_yml_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "data_sources.yml"
+            )
+
+            # Construct a special lookup whose CSV is created at runtime and
+            # written directly into the lookups folder. We will delete this after a build,
+            # assuming that it is successful.
+            data_sources_lookup_csv_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "data_sources.csv"
+            )
+
+            DataSourceWriter.writeDataSourceCsv(
+                input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
+            )
+            input_dto.director_output_dto.addContentToDictMappings(
+                CSVLookup.model_construct(
+                    name="data_sources",
+                    id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
+                    version=1,
+                    author=input_dto.config.app.author_name,
+                    date=datetime.date.today(),
+                    description="A lookup file that will contain the data source objects for detections.",
+                    lookup_type=Lookup_Type.csv,
+                    file_path=data_sources_fake_yml_path,
+                )
+            )
+            updated_conf_files.update(conf_output.writeHeaders())
+            updated_conf_files.update(
+                conf_output.writeLookups(input_dto.director_output_dto.lookups)
+            )
+            updated_conf_files.update(
+                conf_output.writeDetections(input_dto.director_output_dto.detections)
+            )
+            updated_conf_files.update(
+                conf_output.writeStories(input_dto.director_output_dto.stories)
+            )
+            updated_conf_files.update(
+                conf_output.writeBaselines(input_dto.director_output_dto.baselines)
+            )
+            updated_conf_files.update(
+                conf_output.writeInvestigations(
+                    input_dto.director_output_dto.investigations
+                )
+            )
+            updated_conf_files.update(
+                conf_output.writeMacros(input_dto.director_output_dto.macros)
+            )
+            updated_conf_files.update(
+                conf_output.writeDashboards(input_dto.director_output_dto.dashboards)
+            )
+            updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
+
+            # Ensure that the conf file we just generated/update is syntactically valid
+            for conf_file in updated_conf_files:
+                ConfWriter.validateConfFile(conf_file)
+
+            conf_output.packageApp()
+
+            print(
+                f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}"
+            )
+
+        if input_dto.config.build_api:
+            shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True)
+            input_dto.config.getAPIPath().mkdir(parents=True)
+            api_json_output = ApiJsonOutput(
+                input_dto.config.getAPIPath(), input_dto.config.app.label
+            )
+            api_json_output.writeDetections(input_dto.director_output_dto.detections)
+            api_json_output.writeStories(input_dto.director_output_dto.stories)
+            api_json_output.writeBaselines(input_dto.director_output_dto.baselines)
+            api_json_output.writeInvestigations(
+                input_dto.director_output_dto.investigations
+            )
+            api_json_output.writeLookups(input_dto.director_output_dto.lookups)
+            api_json_output.writeMacros(input_dto.director_output_dto.macros)
+            api_json_output.writeDeployments(input_dto.director_output_dto.deployments)
+
+            # create version file for sse api
+            version_file = input_dto.config.getAPIPath() / "version.json"
+            utc_time = (
+                datetime.datetime.now(datetime.timezone.utc)
+                .replace(microsecond=0, tzinfo=None)
+                .isoformat()
+            )
+            version_dict = {
+                "version": {
+                    "name": f"v{input_dto.config.app.version}",
+                    "published_at": f"{utc_time}Z",
+                }
+            }
+            with open(version_file, "w") as version_f:
+                json.dump(version_dict, version_f)
+
+            print(
+                f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}"
+            )
+
+        return input_dto.director_output_dto

{contentctl-4.4.6 → contentctl-5.0.0}/contentctl/actions/deploy_acs.py

@@ -4,52 +4,57 @@ import pprint
 
 
 class Deploy:
-    def execute(self, config: deploy_acs, appinspect_token:str) -> None:
-        
-        #The following common headers are used by both Clasic and Victoria
+    def execute(self, config: deploy_acs, appinspect_token: str) -> None:
+        # The following common headers are used by both Clasic and Victoria
         headers = {
-            'Authorization': f'Bearer {config.splunk_cloud_jwt_token}',
-            'ACS-Legal-Ack': 'Y'
+            "Authorization": f"Bearer {config.splunk_cloud_jwt_token}",
+            "ACS-Legal-Ack": "Y",
         }
         try:
-            
-            with open(config.getPackageFilePath(include_version=False),'rb') as app_data:
-                #request_data = app_data.read()
+            with open(
+                config.getPackageFilePath(include_version=False), "rb"
+            ) as app_data:
+                # request_data = app_data.read()
                 if config.stack_type == StackType.classic:
                     # Classic instead uses a form to store token and package
                     # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
                     address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
-                    
-                    form_data = {
-                        'token': (None, appinspect_token),
-                        'package': app_data
-                    }
-                    res = post(address, headers=headers, files = form_data)
+
+                    form_data = {"token": (None, appinspect_token), "package": app_data}
+                    res = post(address, headers=headers, files=form_data)
                 elif config.stack_type == StackType.victoria:
                     # Victoria uses the X-Splunk-Authorization Header
                     # It also uses --data-binary for the app content
                     # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
-                    headers.update({'X-Splunk-Authorization':  appinspect_token})
+                    headers.update({"X-Splunk-Authorization": appinspect_token})
                     address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
                     res = post(address, headers=headers, data=app_data.read())
                 else:
                     raise Exception(f"Unsupported stack type: '{config.stack_type}'")
         except Exception as e:
-            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}")
-        
+            raise Exception(
+                f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}"
+            )
+
         try:
             # Request went through and completed, but may have returned a non-successful error code.
             # This likely includes a more verbose response describing the error
             res.raise_for_status()
             print(res.json())
-        except Exception as e:
+        except Exception:
             try:
                 error_text = res.json()
-            except Exception as e:
+            except Exception:
                 error_text = "No error text - request failed"
             formatted_error_text = pprint.pformat(error_text)
-            print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
-                  "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.")
-            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
-        
-        print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")
+            print(
+                "While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
+                "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase."
+            )
+            raise Exception(
+                f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}"
+            )
+
+        print(
+            f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!"
+        )

{contentctl-4.4.6 → contentctl-5.0.0}/contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -1,26 +1,29 @@
-from typing import List,Union
-from contentctl.objects.config import test, test_servers, Container,Infrastructure
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer
-from urllib.parse import urlparse
-from copy import deepcopy
-from contentctl.objects.enums import DetectionTestingTargetInfrastructure
+from typing import List, Union
+from contentctl.objects.config import test, test_servers, Container, Infrastructure
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
+    DetectionTestingInfrastructure,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
+    DetectionTestingInfrastructureContainer,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
+    DetectionTestingInfrastructureServer,
+)
 import signal
 import datetime
+
 # from queue import Queue
 from dataclasses import dataclass
+
 # import threading
-import ctypes
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
-    DetectionTestingInfrastructure,
     DetectionTestingManagerOutputDto,
 )
 from contentctl.actions.detection_testing.views.DetectionTestingView import (
     DetectionTestingView,
 )
 from contentctl.objects.enums import PostTestBehavior
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from contentctl.objects.detection import Detection
 import concurrent.futures
 import docker
@@ -28,7 +31,7 @@ import docker
 
 @dataclass(frozen=False)
 class DetectionTestingManagerInputDto:
-    config: Union[test,test_servers]
+    config: Union[test, test_servers]
     detections: List[Detection]
     views: list[DetectionTestingView]
 
@@ -65,15 +68,18 @@ class DetectionTestingManager(BaseModel):
                 print("*******************************")
 
         signal.signal(signal.SIGINT, sigint_handler)
-        
-        with concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.config.test_instances),
-        ) as instance_pool, concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.views)
-        ) as view_runner, concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.config.test_instances),
-        ) as view_shutdowner:
 
+        with (
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.config.test_instances),
+            ) as instance_pool,
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.views)
+            ) as view_runner,
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.config.test_instances),
+            ) as view_shutdowner,
+        ):
             # Start all the views
             future_views = {
                 view_runner.submit(view.setup): view for view in self.input_dto.views
@@ -87,7 +93,7 @@ class DetectionTestingManager(BaseModel):
             # Wait for all instances to be set up
             for future in concurrent.futures.as_completed(future_instances_setup):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     self.output_dto.terminate = True
                     print(f"Error setting up container: {str(e)}")
@@ -102,7 +108,7 @@ class DetectionTestingManager(BaseModel):
                 # Wait for execution to finish
                 for future in concurrent.futures.as_completed(future_instances_execute):
                     try:
-                        result = future.result()
+                        future.result()
                     except Exception as e:
                         self.output_dto.terminate = True
                         print(f"Error running in container: {str(e)}")
@@ -115,34 +121,43 @@ class DetectionTestingManager(BaseModel):
             }
             for future in concurrent.futures.as_completed(future_views_shutdowner):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     print(f"Error stopping view: {str(e)}")
 
             # Wait for original view-related threads to complete
             for future in concurrent.futures.as_completed(future_views):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     print(f"Error running container: {str(e)}")
 
         return self.output_dto
 
     def create_DetectionTestingInfrastructureObjects(self):
-        #Make sure that, if we need to, we pull the appropriate container
+        # Make sure that, if we need to, we pull the appropriate container
         for infrastructure in self.input_dto.config.test_instances:
-            if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+            if isinstance(self.input_dto.config, test) and isinstance(
+                infrastructure, Container
+            ):
                 try:
                     client = docker.from_env()
-                except Exception as e:
-                    raise Exception("Unable to connect to docker.  Are you sure that docker is running on this host?")
+                except Exception:
+                    raise Exception(
+                        "Unable to connect to docker.  Are you sure that docker is running on this host?"
+                    )
                 try:
-                    
-                    parts = self.input_dto.config.container_settings.full_image_path.split(':')
+                    parts = (
+                        self.input_dto.config.container_settings.full_image_path.split(
+                            ":"
+                        )
+                    )
                     if len(parts) != 2:
-                        raise Exception(f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
-                                        f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'")
-                    
+                        raise Exception(
+                            f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
+                            f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'"
+                        )
+
                     print(
                         f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...",
                         end="",
@@ -152,12 +167,15 @@ class DetectionTestingManager(BaseModel):
                     print("done!")
                     break
                 except Exception as e:
-                    raise Exception(f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}")
+                    raise Exception(
+                        f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
+                    )
 
         already_staged_container_files = False
         for infrastructure in self.input_dto.config.test_instances:
-
-            if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+            if isinstance(self.input_dto.config, test) and isinstance(
+                infrastructure, Container
+            ):
                 # Stage the files in the apps dir so that they can be passed directly to
                 # subsequent containers. Do this here, instead of inside each container, to
                 # avoid duplicate downloads/moves/copies
@@ -167,18 +185,24 @@ class DetectionTestingManager(BaseModel):
 
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureContainer(
-                        global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
+                        global_config=self.input_dto.config,
+                        infrastructure=infrastructure,
+                        sync_obj=self.output_dto,
                     )
                 )
 
-            elif (isinstance(self.input_dto.config, test_servers) and isinstance(infrastructure, Infrastructure)):
+            elif isinstance(self.input_dto.config, test_servers) and isinstance(
+                infrastructure, Infrastructure
+            ):
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureServer(
-                        global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
+                        global_config=self.input_dto.config,
+                        infrastructure=infrastructure,
+                        sync_obj=self.output_dto,
                     )
                 )
 
             else:
-
-                raise Exception(f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}")
-                
+                raise Exception(
+                    f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}"
+                )