contentctl 4.4.3__tar.gz → 4.4.5__tar.gz

{contentctl-4.4.3 → contentctl-4.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contentctl
-Version: 4.4.3
+Version: 4.4.5
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/actions/build.py

@@ -51,7 +51,9 @@ class Build:
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards))
-            updated_conf_files.update(conf_output.writeAppConf())
+            updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
+            
+
             
             #Ensure that the conf file we just generated/update is syntactically valid
             for conf_file in updated_conf_files:

contentctl-4.4.5/contentctl/actions/deploy_acs.py

@@ -0,0 +1,55 @@
+from contentctl.objects.config import deploy_acs, StackType
+from requests import post
+import pprint
+
+
+class Deploy:
+    def execute(self, config: deploy_acs, appinspect_token:str) -> None:
+        
+        #The following common headers are used by both Clasic and Victoria
+        headers = {
+            'Authorization': f'Bearer {config.splunk_cloud_jwt_token}',
+            'ACS-Legal-Ack': 'Y'
+        }
+        try:
+            
+            with open(config.getPackageFilePath(include_version=False),'rb') as app_data:
+                #request_data = app_data.read()
+                if config.stack_type == StackType.classic:
+                    # Classic instead uses a form to store token and package
+                    # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
+                    address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
+                    
+                    form_data = {
+                        'token': (None, appinspect_token),
+                        'package': app_data
+                    }
+                    res = post(address, headers=headers, files = form_data)
+                elif config.stack_type == StackType.victoria:
+                    # Victoria uses the X-Splunk-Authorization Header
+                    # It also uses --data-binary for the app content
+                    # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
+                    headers.update({'X-Splunk-Authorization':  appinspect_token})
+                    address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
+                    res = post(address, headers=headers, data=app_data.read())
+                else:
+                    raise Exception(f"Unsupported stack type: '{config.stack_type}'")
+        except Exception as e:
+            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}")
+        
+        try:
+            # Request went through and completed, but may have returned a non-successful error code.
+            # This likely includes a more verbose response describing the error
+            res.raise_for_status()
+            print(res.json())
+        except Exception as e:
+            try:
+                error_text = res.json()
+            except Exception as e:
+                error_text = "No error text - request failed"
+            formatted_error_text = pprint.pformat(error_text)
+            print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
+                  "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.")
+            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
+        
+        print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/actions/detection_testing/GitService.py

@@ -13,6 +13,7 @@ if TYPE_CHECKING:
 from contentctl.objects.macro import Macro
 from contentctl.objects.lookup import Lookup
 from contentctl.objects.detection import Detection
+from contentctl.objects.data_source import DataSource
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.config import test_common, All, Changes, Selected
 
@@ -67,9 +68,12 @@ class GitService(BaseModel):
 
         #Make a filename to content map
         filepath_to_content_map = { obj.file_path:obj for (_,obj) in self.director.name_to_content_map.items()} 
-        updated_detections:set[Detection] = set()
-        updated_macros:set[Macro] = set()
-        updated_lookups:set[Lookup] = set()
+
+        updated_detections: set[Detection] = set()
+        updated_macros: set[Macro] = set()
+        updated_lookups: set[Lookup] = set()
+        updated_datasources: set[DataSource] = set()
+
 
         for diff in all_diffs:
             if type(diff) == pygit2.Patch:
@@ -90,6 +94,13 @@ class GitService(BaseModel):
                             updated_macros.add(macroObject)
                         else:
                             raise Exception(f"Error getting macro object for file {str(decoded_path)}")
+                        
+                    elif decoded_path.is_relative_to(self.config.path/"data_sources") and decoded_path.suffix == ".yml":
+                        datasourceObject = filepath_to_content_map.get(decoded_path, None)
+                        if isinstance(datasourceObject, DataSource):
+                            updated_datasources.add(datasourceObject)
+                        else:
+                            raise Exception(f"Error getting data source object for file {str(decoded_path)}")
 
                     elif decoded_path.is_relative_to(self.config.path/"lookups"):
                         # We need to convert this to a yml. This means we will catch
@@ -115,7 +126,6 @@ class GitService(BaseModel):
                             # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
                             # this time, we will ignore this change.
                             updatedLookup = None
-                            
 
                         else:
                             raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
@@ -136,7 +146,8 @@ class GitService(BaseModel):
 
         # If a detection has at least one dependency on changed content,
         # then we must test it again
-        changed_macros_and_lookups:set[SecurityContentObject] = updated_macros.union(updated_lookups)
+
+        changed_macros_and_lookups_and_datasources:set[SecurityContentObject] = updated_macros.union(updated_lookups, updated_datasources)
         
         for detection in self.director.detections:
             if detection in updated_detections:
@@ -144,7 +155,7 @@ class GitService(BaseModel):
                 # to add it again
                 continue
 
-            for obj in changed_macros_and_lookups:
+            for obj in changed_macros_and_lookups_and_datasources:
                 if obj in detection.get_content_dependencies():
                    updated_detections.add(detection)
                    break

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/actions/new_content.py

@@ -29,8 +29,7 @@ class NewContent:
         answers['date'] = datetime.today().strftime('%Y-%m-%d')
         answers['author'] = answers['detection_author']
         del answers['detection_author']
-        answers['data_sources'] = answers['data_source']
-        del answers['data_source']
+        answers['data_source'] = answers['data_source']
         answers['type'] = answers['detection_type']
         del answers['detection_type']
         answers['status'] = "production" #start everything as production since that's what we INTEND the content to become   

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/contentctl.py

@@ -19,6 +19,7 @@ from contentctl.actions.test import TestInputDto
 from contentctl.actions.reporting import ReportingInputDto, Reporting
 from contentctl.actions.inspect import Inspect
 from contentctl.input.yml_reader import YmlReader
+from contentctl.actions.deploy_acs import Deploy
 from contentctl.actions.release_notes import ReleaseNotes
 
 # def print_ascii_art():
@@ -95,8 +96,11 @@ def new_func(config:new):
 
 
 def deploy_acs_func(config:deploy_acs):
-    #This is a bit challenging to get to work with the default values.
-    raise Exception("deploy acs not yet implemented") 
+    print("Building and inspecting app...")
+    token = inspect_func(config)
+    print("App successfully built and inspected.")
+    print("Deploying app...")
+    Deploy().execute(config, token)
 
 def test_common_func(config:test_common):
     if type(config) == test:

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -689,6 +689,7 @@ class Detection_Abstract(SecurityContentObject):
         objects: list[SecurityContentObject] = []
         objects += self.macros
         objects += self.lookups
+        objects += self.data_source_objects
         return objects
 
     @field_validator("deployment", mode="before")

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/objects/config.py

@@ -294,6 +294,7 @@ class StackType(StrEnum):
 
 
 class inspect(build):
+
     splunk_api_username: str = Field(
         description="Splunk API username used for appinspect and Splunkbase downloads."
     )

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/output/conf_output.py

@@ -57,19 +57,26 @@ class ConfOutput:
             pass
             
 
-    def writeAppConf(self)->set[pathlib.Path]:
+    
+
+    def writeMiscellaneousAppFiles(self)->set[pathlib.Path]:
         written_files:set[pathlib.Path] = set()
-        for output_app_path, template_name in [ ("default/app.conf", "app.conf.j2"),
-                                                ("default/content-version.conf", "content-version.j2")]:
-            written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path),
-                                    template_name,
-                                    self.config,
-                                    [self.config.app]))
+        
+        written_files.add(ConfWriter.writeConfFile(pathlib.Path("default/content-version.conf"),
+                                "content-version.j2",
+                                self.config,
+                                [self.config.app]))
         
         written_files.add(ConfWriter.writeManifestFile(pathlib.Path("app.manifest"),
                                               "app.manifest.j2",
                                               self.config,
                                               [self.config.app]))
+        
+        written_files.add(ConfWriter.writeServerConf(self.config))
+
+        written_files.add(ConfWriter.writeAppConf(self.config))
+                                              
+
         return written_files
 
         

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/output/conf_writer.py

@@ -12,6 +12,76 @@ from contentctl.objects.dashboard import Dashboard
 from contentctl.objects.config import build
 import xml.etree.ElementTree as ET
 
+# This list is not exhaustive of all default conf files, but should be
+# sufficient for our purposes.
+DEFAULT_CONF_FILES = [
+    "alert_actions.conf",
+    "app.conf",
+    "audit.conf",
+    "authentication.conf",
+    "authorize.conf",
+    "bookmarks.conf",
+    "checklist.conf",
+    "collections.conf",
+    "commands.conf",
+    "conf.conf",
+    "datamodels.conf",
+    "datatypesbnf.conf",
+    "default-mode.conf",
+    "deploymentclient.conf",
+    "distsearch.conf",
+    "event_renderers.conf",
+    "eventdiscoverer.conf",
+    "eventtypes.conf",
+    "federated.conf",
+    "fields.conf",
+    "global-banner.conf",
+    "health.conf",
+    "indexes.conf",
+    "inputs.conf",
+    "limits.conf",
+    "literals.conf",
+    "livetail.conf",
+    "macros.conf",
+    "messages.conf",
+    "metric_alerts.conf",
+    "metric_rollups.conf",
+    "multikv.conf",
+    "outputs.conf",
+    "passwords.conf",
+    "procmon-filters.conf",
+    "props.conf",
+    "pubsub.conf",
+    "restmap.conf",
+    "rolling_upgrade.conf",
+    "savedsearches.conf",
+    "searchbnf.conf",
+    "segmenters.conf",
+    "server.conf",
+    "serverclass.conf",
+    "serverclass.seed.xml.conf",
+    "source-classifier.conf",
+    "sourcetypes.conf",
+    "tags.conf",
+    "telemetry.conf",
+    "times.conf",
+    "transactiontypes.conf",
+    "transforms.conf",
+    "ui-prefs.conf",
+    "ui-tour.conf",
+    "user-prefs.conf",
+    "user-seed.conf",
+    "viewstates.conf",
+    "visualizations.conf",
+    "web-features.conf",
+    "web.conf",
+    "wmi.conf",
+    "workflow_actions.conf",
+    "workload_policy.conf",
+    "workload_pools.conf",
+    "workload_rules.conf",
+]
+
 class ConfWriter():
 
     @staticmethod
@@ -57,6 +127,52 @@ class ConfWriter():
         ConfWriter.validateConfFile(output_path)        
         return output_path
 
+    @staticmethod
+    def getCustomConfFileStems(config:build)->list[str]:
+        # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if
+        # they are custom conf files
+        default_path = config.getPackageDirectoryPath()/"default"
+        conf_files = default_path.glob("*.conf")
+        
+        custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES]
+        return sorted(custom_conf_file_stems)
+
+    @staticmethod
+    def writeServerConf(config: build) -> pathlib.Path:
+        app_output_path = pathlib.Path("default/server.conf")
+        template_name = "server.conf.j2"
+
+        j2_env = ConfWriter.getJ2Environment()
+        template = j2_env.get_template(template_name)
+
+        output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config))
+        
+        output_path = config.getPackageDirectoryPath()/app_output_path
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(output_path, 'a') as f:
+            output = output.encode('utf-8', 'ignore').decode('utf-8')
+            f.write(output)
+        return output_path
+
+
+    @staticmethod
+    def writeAppConf(config: build) -> pathlib.Path:
+        app_output_path = pathlib.Path("default/app.conf")
+        template_name = "app.conf.j2"
+
+        j2_env = ConfWriter.getJ2Environment()
+        template = j2_env.get_template(template_name)
+
+        output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config), 
+                                 app=config.app)
+        
+        output_path = config.getPackageDirectoryPath()/app_output_path
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(output_path, 'a') as f:
+            output = output.encode('utf-8', 'ignore').decode('utf-8')
+            f.write(output)
+        return output_path
+
     @staticmethod
     def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path:
         j2_env = ConfWriter.getJ2Environment()
@@ -70,6 +186,7 @@ class ConfWriter():
             output = output.encode('utf-8', 'ignore').decode('utf-8')
             f.write(output)
         return output_path
+    
 
 
     @staticmethod
@@ -218,8 +335,3 @@ class ConfWriter():
                 _ = json.load(manifestFile)
         except Exception as e:
             raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}")
-            
-
-
-
-

contentctl-4.4.5/contentctl/output/templates/app.conf.j2

@@ -0,0 +1,37 @@
+## Splunk app configuration file
+
+[install]
+is_configured = false
+state = enabled
+state_change_requires_restart = false
+build = {{ app.build }}
+
+[triggers]
+{% for custom_conf_file in custom_conf_files%}
+reload.{{custom_conf_file}} = simple
+{% endfor %}
+
+[launcher]
+author = {{ app.author_company }}
+version = {{ app.version }} 
+description = {{ app.description | escapeNewlines() }}
+
+[ui]
+is_visible = true
+label = {{ app.title }}
+
+[package]
+id = {{ app.appid }}
+{% if app.uid == 3449 %}
+check_for_updates = true
+{% else %}
+check_for_updates = false
+{% endif %}
+
+[id]
+version = {{ app.version }}
+name = {{ app.appid }}
+
+
+
+

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/output/templates/app.manifest.j2

@@ -1,5 +1,6 @@
 {
-  "schemaVersion": "1.0.0", 
+  "schemaVersion": "1.0.0",
+  "targetWorkloads": ["_search_heads"], 
   "info": {
     "title": "{{ objects[0].title }}", 
     "id": {

contentctl-4.4.5/contentctl/output/templates/server.conf.j2

@@ -0,0 +1,4 @@
+[shclustering]
+{% for custom_conf_file in custom_conf_files%}
+conf_replication_include.{{custom_conf_file}} = true
+{% endfor %}

{contentctl-4.4.3 → contentctl-4.4.5}/contentctl/templates/app_template/metadata/default.meta

@@ -1,6 +1,6 @@
 ## shared Application-level permissions
 []
-access = read : [ * ], write : [ admin ]
+access = read : [ * ], write : [ admin, sc_admin ]
 export = system
 
 [savedsearches]

{contentctl-4.4.3 → contentctl-4.4.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "contentctl"
-version = "4.4.3"
+version = "4.4.5"
 
 description = "Splunk Content Control Tool"
 authors = ["STRT <research@splunk.com>"]

contentctl-4.4.3/contentctl/actions/deploy_acs.py

@@ -1,38 +0,0 @@
-from dataclasses import dataclass
-from contentctl.input.director import DirectorInputDto
-from contentctl.output.conf_output import ConfOutput
-
-
-from typing import Union
-
-@dataclass(frozen=True)
-class ACSDeployInputDto:
-    director_input_dto: DirectorInputDto
-    splunk_api_username: str
-    splunk_api_password: str
-    splunk_cloud_jwt_token: str
-    splunk_cloud_stack: str
-    stack_type: str
-
-
-class Deploy:
-    def execute(self, input_dto: ACSDeployInputDto) -> None:
-        
-        conf_output = ConfOutput(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config)
-        
-        appinspect_token = conf_output.inspectAppAPI(input_dto.splunk_api_username, input_dto.splunk_api_password, input_dto.stack_type)
-        
-
-        if input_dto.splunk_cloud_jwt_token is None or input_dto.splunk_cloud_stack is None:
-            if input_dto.splunk_cloud_jwt_token is None:
-                raise Exception("Cannot deploy app via ACS, --splunk_cloud_jwt_token was not defined on command line.")
-            else:
-                raise Exception("Cannot deploy app via ACS, --splunk_cloud_stack was not defined on command line.")
-        
-        conf_output.deploy_via_acs(input_dto.splunk_cloud_jwt_token,
-                                input_dto.splunk_cloud_stack, 
-                                appinspect_token,
-                                input_dto.stack_type)
-
-        
-        

contentctl-4.4.3/contentctl/output/templates/app.conf.j2

@@ -1,35 +0,0 @@
-## Splunk app configuration file
-
-[install]
-is_configured = false
-state = enabled
-state_change_requires_restart = false
-build = {{ objects[0].build }}
-
-[triggers]
-reload.analytic_stories = simple
-reload.usage_searches = simple
-reload.use_case_library = simple
-reload.correlationsearches = simple
-reload.analyticstories = simple
-reload.governance = simple
-reload.managed_configurations = simple
-reload.postprocess = simple
-reload.content-version = simple
-reload.es_investigations = simple
-
-[launcher]
-author = {{ objects[0].author_company }}
-version = {{ objects[0].version }} 
-description = {{ objects[0].description | escapeNewlines() }}
-
-[ui]
-is_visible = true
-label = {{ objects[0].title }}
-
-[package]
-id = {{ objects[0].appid }}
-
-
-
-

contentctl-4.4.3/contentctl/templates/app_template/default/app.conf

@@ -1,30 +0,0 @@
-## Splunk app configuration file
-
-[install]
-is_configured = false
-state = enabled
-state_change_requires_restart = false
-build = 16367
-
-[triggers]
-reload.analytic_stories = simple
-reload.use_case_library = simple
-reload.correlationsearches = simple
-reload.analyticstories = simple
-reload.governance = simple
-reload.managed_configurations = simple
-reload.postprocess = simple
-reload.content-version = simple
-reload.es_investigations = simple
-
-[launcher]
-author = Splunk
-version = 4.9.0
-description = Explore the Analytic Stories included with ES Content Updates.
-
-[ui]
-is_visible = true
-label      = ES Content Updates
-
-[package]
-id = DA-ESS-ContentUpdate

contentctl-4.4.3/contentctl/templates/app_template/default/content-version.conf

@@ -1,2 +0,0 @@
-[content-version]
-version = 4.9.0