contentctl 5.0.0a2__tar.gz → 5.0.0a3__tar.gz

{contentctl-5.0.0a2 → contentctl-5.0.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: contentctl
-Version: 5.0.0a2
+Version: 5.0.0a3
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

contentctl-5.0.0a3/contentctl/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

contentctl-5.0.0a3/contentctl/actions/build.py

@@ -0,0 +1,136 @@
+import shutil
+
+from dataclasses import dataclass
+
+from contentctl.input.director import DirectorOutputDto
+from contentctl.output.conf_output import ConfOutput
+from contentctl.output.conf_writer import ConfWriter
+from contentctl.output.api_json_output import ApiJsonOutput
+from contentctl.output.data_source_writer import DataSourceWriter
+from contentctl.objects.lookup import CSVLookup, Lookup_Type
+import pathlib
+import json
+import datetime
+import uuid
+
+from contentctl.objects.config import build
+
+
+@dataclass(frozen=True)
+class BuildInputDto:
+    director_output_dto: DirectorOutputDto
+    config: build
+
+
+class Build:
+    def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
+        if input_dto.config.build_app:
+            updated_conf_files: set[pathlib.Path] = set()
+            conf_output = ConfOutput(input_dto.config)
+
+            # Construct a path to a YML that does not actually exist.
+            # We mock this "fake" path since the YML does not exist.
+            # This ensures the checking for the existence of the CSV is correct
+            data_sources_fake_yml_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "data_sources.yml"
+            )
+
+            # Construct a special lookup whose CSV is created at runtime and
+            # written directly into the lookups folder. We will delete this after a build,
+            # assuming that it is successful.
+            data_sources_lookup_csv_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "data_sources.csv"
+            )
+
+            DataSourceWriter.writeDataSourceCsv(
+                input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
+            )
+            input_dto.director_output_dto.addContentToDictMappings(
+                CSVLookup.model_construct(
+                    name="data_sources",
+                    id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
+                    version=1,
+                    author=input_dto.config.app.author_name,
+                    date=datetime.date.today(),
+                    description="A lookup file that will contain the data source objects for detections.",
+                    lookup_type=Lookup_Type.csv,
+                    file_path=data_sources_fake_yml_path,
+                )
+            )
+            updated_conf_files.update(conf_output.writeHeaders())
+            updated_conf_files.update(
+                conf_output.writeLookups(input_dto.director_output_dto.lookups)
+            )
+            updated_conf_files.update(
+                conf_output.writeDetections(input_dto.director_output_dto.detections)
+            )
+            updated_conf_files.update(
+                conf_output.writeStories(input_dto.director_output_dto.stories)
+            )
+            updated_conf_files.update(
+                conf_output.writeBaselines(input_dto.director_output_dto.baselines)
+            )
+            updated_conf_files.update(
+                conf_output.writeInvestigations(
+                    input_dto.director_output_dto.investigations
+                )
+            )
+            updated_conf_files.update(
+                conf_output.writeMacros(input_dto.director_output_dto.macros)
+            )
+            updated_conf_files.update(
+                conf_output.writeDashboards(input_dto.director_output_dto.dashboards)
+            )
+            updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
+
+            # Ensure that the conf file we just generated/update is syntactically valid
+            for conf_file in updated_conf_files:
+                ConfWriter.validateConfFile(conf_file)
+
+            conf_output.packageApp()
+
+            print(
+                f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}"
+            )
+
+        if input_dto.config.build_api:
+            shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True)
+            input_dto.config.getAPIPath().mkdir(parents=True)
+            api_json_output = ApiJsonOutput(
+                input_dto.config.getAPIPath(), input_dto.config.app.label
+            )
+            api_json_output.writeDetections(input_dto.director_output_dto.detections)
+            api_json_output.writeStories(input_dto.director_output_dto.stories)
+            api_json_output.writeBaselines(input_dto.director_output_dto.baselines)
+            api_json_output.writeInvestigations(
+                input_dto.director_output_dto.investigations
+            )
+            api_json_output.writeLookups(input_dto.director_output_dto.lookups)
+            api_json_output.writeMacros(input_dto.director_output_dto.macros)
+            api_json_output.writeDeployments(input_dto.director_output_dto.deployments)
+
+            # create version file for sse api
+            version_file = input_dto.config.getAPIPath() / "version.json"
+            utc_time = (
+                datetime.datetime.now(datetime.timezone.utc)
+                .replace(microsecond=0, tzinfo=None)
+                .isoformat()
+            )
+            version_dict = {
+                "version": {
+                    "name": f"v{input_dto.config.app.version}",
+                    "published_at": f"{utc_time}Z",
+                }
+            }
+            with open(version_file, "w") as version_f:
+                json.dump(version_dict, version_f)
+
+            print(
+                f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}"
+            )
+
+        return input_dto.director_output_dto

{contentctl-5.0.0a2 → contentctl-5.0.0a3}/contentctl/actions/deploy_acs.py

@@ -4,52 +4,57 @@ import pprint
 
 
 class Deploy:
-    def execute(self, config: deploy_acs, appinspect_token:str) -> None:
-        
-        #The following common headers are used by both Clasic and Victoria
+    def execute(self, config: deploy_acs, appinspect_token: str) -> None:
+        # The following common headers are used by both Clasic and Victoria
         headers = {
-            'Authorization': f'Bearer {config.splunk_cloud_jwt_token}',
-            'ACS-Legal-Ack': 'Y'
+            "Authorization": f"Bearer {config.splunk_cloud_jwt_token}",
+            "ACS-Legal-Ack": "Y",
         }
         try:
-            
-            with open(config.getPackageFilePath(include_version=False),'rb') as app_data:
-                #request_data = app_data.read()
+            with open(
+                config.getPackageFilePath(include_version=False), "rb"
+            ) as app_data:
+                # request_data = app_data.read()
                 if config.stack_type == StackType.classic:
                     # Classic instead uses a form to store token and package
                     # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
                     address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
-                    
-                    form_data = {
-                        'token': (None, appinspect_token),
-                        'package': app_data
-                    }
-                    res = post(address, headers=headers, files = form_data)
+
+                    form_data = {"token": (None, appinspect_token), "package": app_data}
+                    res = post(address, headers=headers, files=form_data)
                 elif config.stack_type == StackType.victoria:
                     # Victoria uses the X-Splunk-Authorization Header
                     # It also uses --data-binary for the app content
                     # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
-                    headers.update({'X-Splunk-Authorization':  appinspect_token})
+                    headers.update({"X-Splunk-Authorization": appinspect_token})
                     address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
                     res = post(address, headers=headers, data=app_data.read())
                 else:
                     raise Exception(f"Unsupported stack type: '{config.stack_type}'")
         except Exception as e:
-            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}")
-        
+            raise Exception(
+                f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}"
+            )
+
         try:
             # Request went through and completed, but may have returned a non-successful error code.
             # This likely includes a more verbose response describing the error
             res.raise_for_status()
             print(res.json())
-        except Exception as e:
+        except Exception:
             try:
                 error_text = res.json()
-            except Exception as e:
+            except Exception:
                 error_text = "No error text - request failed"
             formatted_error_text = pprint.pformat(error_text)
-            print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
-                  "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.")
-            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
-        
-        print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")
+            print(
+                "While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
+                "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase."
+            )
+            raise Exception(
+                f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}"
+            )
+
+        print(
+            f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!"
+        )

{contentctl-5.0.0a2 → contentctl-5.0.0a3}/contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -1,25 +1,29 @@
-from typing import List,Union
-from contentctl.objects.config import test, test_servers, Container,Infrastructure
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import DetectionTestingInfrastructure
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import DetectionTestingInfrastructureContainer
-from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer
-from urllib.parse import urlparse
-from copy import deepcopy
+from typing import List, Union
+from contentctl.objects.config import test, test_servers, Container, Infrastructure
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
+    DetectionTestingInfrastructure,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
+    DetectionTestingInfrastructureContainer,
+)
+from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
+    DetectionTestingInfrastructureServer,
+)
 import signal
 import datetime
+
 # from queue import Queue
 from dataclasses import dataclass
+
 # import threading
-import ctypes
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
-    DetectionTestingInfrastructure,
     DetectionTestingManagerOutputDto,
 )
 from contentctl.actions.detection_testing.views.DetectionTestingView import (
     DetectionTestingView,
 )
 from contentctl.objects.enums import PostTestBehavior
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from contentctl.objects.detection import Detection
 import concurrent.futures
 import docker
@@ -27,7 +31,7 @@ import docker
 
 @dataclass(frozen=False)
 class DetectionTestingManagerInputDto:
-    config: Union[test,test_servers]
+    config: Union[test, test_servers]
     detections: List[Detection]
     views: list[DetectionTestingView]
 
@@ -64,15 +68,18 @@ class DetectionTestingManager(BaseModel):
                 print("*******************************")
 
         signal.signal(signal.SIGINT, sigint_handler)
-        
-        with concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.config.test_instances),
-        ) as instance_pool, concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.views)
-        ) as view_runner, concurrent.futures.ThreadPoolExecutor(
-            max_workers=len(self.input_dto.config.test_instances),
-        ) as view_shutdowner:
 
+        with (
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.config.test_instances),
+            ) as instance_pool,
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.views)
+            ) as view_runner,
+            concurrent.futures.ThreadPoolExecutor(
+                max_workers=len(self.input_dto.config.test_instances),
+            ) as view_shutdowner,
+        ):
             # Start all the views
             future_views = {
                 view_runner.submit(view.setup): view for view in self.input_dto.views
@@ -86,7 +93,7 @@ class DetectionTestingManager(BaseModel):
             # Wait for all instances to be set up
             for future in concurrent.futures.as_completed(future_instances_setup):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     self.output_dto.terminate = True
                     print(f"Error setting up container: {str(e)}")
@@ -101,7 +108,7 @@ class DetectionTestingManager(BaseModel):
                 # Wait for execution to finish
                 for future in concurrent.futures.as_completed(future_instances_execute):
                     try:
-                        result = future.result()
+                        future.result()
                     except Exception as e:
                         self.output_dto.terminate = True
                         print(f"Error running in container: {str(e)}")
@@ -114,34 +121,43 @@ class DetectionTestingManager(BaseModel):
             }
             for future in concurrent.futures.as_completed(future_views_shutdowner):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     print(f"Error stopping view: {str(e)}")
 
             # Wait for original view-related threads to complete
             for future in concurrent.futures.as_completed(future_views):
                 try:
-                    result = future.result()
+                    future.result()
                 except Exception as e:
                     print(f"Error running container: {str(e)}")
 
         return self.output_dto
 
     def create_DetectionTestingInfrastructureObjects(self):
-        #Make sure that, if we need to, we pull the appropriate container
+        # Make sure that, if we need to, we pull the appropriate container
         for infrastructure in self.input_dto.config.test_instances:
-            if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+            if isinstance(self.input_dto.config, test) and isinstance(
+                infrastructure, Container
+            ):
                 try:
                     client = docker.from_env()
-                except Exception as e:
-                    raise Exception("Unable to connect to docker.  Are you sure that docker is running on this host?")
+                except Exception:
+                    raise Exception(
+                        "Unable to connect to docker.  Are you sure that docker is running on this host?"
+                    )
                 try:
-                    
-                    parts = self.input_dto.config.container_settings.full_image_path.split(':')
+                    parts = (
+                        self.input_dto.config.container_settings.full_image_path.split(
+                            ":"
+                        )
+                    )
                     if len(parts) != 2:
-                        raise Exception(f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
-                                        f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'")
-                    
+                        raise Exception(
+                            f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
+                            f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'"
+                        )
+
                     print(
                         f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...",
                         end="",
@@ -151,12 +167,15 @@ class DetectionTestingManager(BaseModel):
                     print("done!")
                     break
                 except Exception as e:
-                    raise Exception(f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}")
+                    raise Exception(
+                        f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
+                    )
 
         already_staged_container_files = False
         for infrastructure in self.input_dto.config.test_instances:
-
-            if (isinstance(self.input_dto.config, test) and isinstance(infrastructure, Container)):
+            if isinstance(self.input_dto.config, test) and isinstance(
+                infrastructure, Container
+            ):
                 # Stage the files in the apps dir so that they can be passed directly to
                 # subsequent containers. Do this here, instead of inside each container, to
                 # avoid duplicate downloads/moves/copies
@@ -166,18 +185,24 @@ class DetectionTestingManager(BaseModel):
 
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureContainer(
-                        global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
+                        global_config=self.input_dto.config,
+                        infrastructure=infrastructure,
+                        sync_obj=self.output_dto,
                     )
                 )
 
-            elif (isinstance(self.input_dto.config, test_servers) and isinstance(infrastructure, Infrastructure)):
+            elif isinstance(self.input_dto.config, test_servers) and isinstance(
+                infrastructure, Infrastructure
+            ):
                 self.detectionTestingInfrastructureObjects.append(
                     DetectionTestingInfrastructureServer(
-                        global_config=self.input_dto.config, infrastructure=infrastructure, sync_obj=self.output_dto
+                        global_config=self.input_dto.config,
+                        infrastructure=infrastructure,
+                        sync_obj=self.output_dto,
                     )
                 )
 
             else:
-
-                raise Exception(f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}")
-                
+                raise Exception(
+                    f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}"
+                )

{contentctl-5.0.0a2 → contentctl-5.0.0a3}/contentctl/actions/detection_testing/GitService.py

@@ -10,6 +10,7 @@ from pygit2.enums import DeltaStatus
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
 
+from contentctl.input.director import DirectorOutputDto
 from contentctl.objects.config import All, Changes, Selected, test_common
 from contentctl.objects.data_source import DataSource
 from contentctl.objects.detection import Detection
@@ -22,9 +23,6 @@ logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 LOGGER = logging.getLogger(__name__)
 
 
-from contentctl.input.director import DirectorOutputDto
-
-
 class GitService(BaseModel):
     director: DirectorOutputDto
     config: test_common
@@ -79,7 +77,7 @@ class GitService(BaseModel):
         updated_datasources: set[DataSource] = set()
 
         for diff in all_diffs:
-            if type(diff) == pygit2.Patch:
+            if type(diff) is pygit2.Patch:
                 if diff.delta.status in (
                     DeltaStatus.ADDED,
                     DeltaStatus.MODIFIED,

contentctl-5.0.0a3/contentctl/actions/detection_testing/generate_detection_coverage_badge.py

@@ -0,0 +1,83 @@
+import argparse
+import json
+import sys
+
+RAW_BADGE_SVG = """<?xml version="1.0"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="100" height="20">
+<linearGradient id="a" x2="0" y2="100%">
+  <stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
+  <stop offset="2" stop-opacity=".1"/>
+</linearGradient>
+
+<rect rx="3" width="60" height="20" fill="#555"/> <!-- Comment -->
+<rect rx="3" x="60" width="40" height="20" fill="#4c1"/>
+
+<path fill="#4c1" d="M58 0h4v20h-4z"/>
+
+<rect rx="3" width="100" height="20" fill="url(#a)"/>
+<g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11">
+    <text x="30" y="14">{}</text>
+    <text x="80" y="14">{}</text>
+</g>
+</svg>"""
+
+
+parser = argparse.ArgumentParser(
+    description="Use a summary.json file to generate a test coverage badge"
+)
+parser.add_argument(
+    "-i",
+    "--input_summary_file",
+    type=argparse.FileType("r"),
+    required=True,
+    help="Summary file to use to generate the pass percentage badge",
+)
+parser.add_argument(
+    "-o",
+    "--output_badge_file",
+    type=argparse.FileType("w"),
+    required=True,
+    help="Name of the badge to output",
+)
+parser.add_argument(
+    "-s", "--badge_string", type=str, required=True, help="Name of the badge to output"
+)
+
+
+try:
+    results = parser.parse_args()
+except Exception as e:
+    print(f"Error parsing arguments: {str(e)}")
+    exit(1)
+
+try:
+    summary_info = json.loads(results.input_summary_file.read())
+except Exception as e:
+    print(f"Error loading {results.input_summary_file.name} JSON file: {str(e)}")
+    sys.exit(1)
+
+if "summary" not in summary_info:
+    print("Missing 'summary' key in {results.input_summary_file.name}")
+    sys.exit(1)
+elif (
+    "PASS_RATE" not in summary_info["summary"]
+    or "TESTS_PASSED" not in summary_info["summary"]
+):
+    print(
+        f"Missing PASS_RATE in 'summary' section of {results.input_summary_file.name}"
+    )
+    sys.exit(1)
+pass_percent = 100 * summary_info["summary"]["PASS_RATE"]
+
+
+try:
+    results.output_badge_file.write(
+        RAW_BADGE_SVG.format(results.badge_string, "{:2.1f}%".format(pass_percent))
+    )
+except Exception as e:
+    print(f"Error generating badge: {str(e)}")
+    sys.exit(1)
+
+
+print(f"Badge {results.output_badge_file.name} successfully generated!")
+sys.exit(0)