contentctl 4.0.4__py3-none-any.whl → 4.1.0__py3-none-any.whl

contentctl/actions/inspect.py

@@ -61,7 +61,7 @@ class Inspect:
         if not package_path.is_file():
             raise Exception(f"Cannot run Appinspect API on App '{config.app.title}' - "
                             f"no package exists as expected path '{package_path}'.\nAre you "
-                            "trying to 'contentctl acs_deploy' the package BEFORE running 'contentctl build'?")
+                            "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?")
         
         files = {
             "app_package": open(package_path,"rb"),

contentctl/actions/new_content.py

@@ -19,16 +19,22 @@ class NewContent:
         answers = questionary.prompt(questions)
         answers.update(answers)
         answers['name'] = answers['detection_name']
+        del answers['detection_name']
         answers['id'] = str(uuid.uuid4())
         answers['version'] = 1
         answers['date'] = datetime.today().strftime('%Y-%m-%d')
         answers['author'] = answers['detection_author']
-        answers['data_source'] = answers['data_source']
+        del answers['detection_author']
+        answers['data_sources'] = answers['data_source']
+        del answers['data_source']
         answers['type'] = answers['detection_type']
+        del answers['detection_type']
         answers['status'] = "production" #start everything as production since that's what we INTEND the content to become   
         answers['description'] = 'UPDATE_DESCRIPTION'   
         file_name = answers['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower()
+        answers['kind'] = answers['detection_kind']
         answers['search'] = answers['detection_search'] + ' | `' + file_name + '_filter`'
+        del answers['detection_search']
         answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT'
         answers['known_false_positives'] = 'UPDATE_KNOWN_FALSE_POSITIVES'            
         answers['references'] = ['REFERENCE']
@@ -44,6 +50,7 @@ class NewContent:
         answers['tags']['required_fields'] = ['UPDATE']
         answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100'
         answers['tags']['security_domain'] = answers['security_domain']
+        del answers["security_domain"]
         answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
         
         #generate the tests section
@@ -52,31 +59,35 @@ class NewContent:
                 'name': "True Positive Test",
                 'attack_data': [ 
                     {
-                    'data': "Enter URL for Dataset Here.  This may also be a relative or absolute path on your local system for testing.",
+                    'data': "https://github.com/splunk/contentctl/wiki",
                     "sourcetype": "UPDATE SOURCETYPE",
                     "source": "UPDATE SOURCE"
                     }
                 ]
             }
         ]
+        del answers["mitre_attack_ids"]
         return answers
 
     def buildStory(self)->dict[str,Any]:
         questions = NewContentQuestions.get_questions_story()
         answers = questionary.prompt(questions)
         answers['name'] = answers['story_name']
+        del answers['story_name']
         answers['id'] = str(uuid.uuid4())
         answers['version'] = 1
         answers['date'] = datetime.today().strftime('%Y-%m-%d')
         answers['author'] = answers['story_author']
+        del answers['story_author']
         answers['description'] = 'UPDATE_DESCRIPTION'
         answers['narrative'] = 'UPDATE_NARRATIVE'
         answers['references'] = []
         answers['tags'] = dict()
-        answers['tags']['analytic_story'] = answers['name']
         answers['tags']['category'] = answers['category']
+        del answers['category']
         answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']
         answers['tags']['usecase'] = answers['usecase']
+        del answers['usecase']
         answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
         return answers
     
@@ -84,13 +95,13 @@ class NewContent:
     def execute(self, input_dto: new) -> None:
         if input_dto.type == NewContentType.detection:
             content_dict = self.buildDetection()
-            subdirectory = pathlib.Path('detections') / content_dict.get('type')
+            subdirectory = pathlib.Path('detections') / content_dict.pop('detection_kind')
         elif input_dto.type == NewContentType.story:
             content_dict = self.buildStory()
             subdirectory = pathlib.Path('stories')
         else:
             raise Exception(f"Unsupported new content type: [{input_dto.type}]")
-    
+
         full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name'))
         YmlWriter.writeYmlFile(str(full_output_path), content_dict)
 
@@ -103,12 +114,12 @@ class NewContent:
             #make sure the output folder exists for this detection
             output_folder.mkdir(exist_ok=True)
 
-            YmlWriter.writeYmlFile(file_path, object)
+            YmlWriter.writeDetection(file_path, object)
             print("Successfully created detection " + file_path)
         
         elif type == NewContentType.story:
             file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product']))
-            YmlWriter.writeYmlFile(file_path, object)
+            YmlWriter.writeStory(file_path, object)
             print("Successfully created story " + file_path)        
         
         else:

contentctl/actions/validate.py

@@ -23,6 +23,7 @@ class Validate:
         director_output_dto = DirectorOutputDto(AtomicTest.getAtomicTestsFromArtRepo(repo_path=input_dto.getAtomicRedTeamRepoPath(), 
                                                                                      enabled=input_dto.enrichments),
                                                 AttackEnrichment.getAttackEnrichment(input_dto),
+                                                CveEnrichment.getCveEnrichment(input_dto),
                                                 [],[],[],[],[],[],[],[],[])
 
         

contentctl/api.py

@@ -0,0 +1,137 @@
+from pathlib import Path
+from typing import Any, Union, Type
+from contentctl.input.yml_reader import YmlReader
+from contentctl.objects.config import test_common, test, test_servers
+from contentctl.objects.security_content_object import SecurityContentObject
+from contentctl.input.director import DirectorOutputDto
+
+def config_from_file(path:Path=Path("contentctl.yml"), config: dict[str,Any]={}, 
+                   configType:Type[Union[test,test_servers]]=test)->test_common:
+    
+    """
+    Fetch a configuration object that can be used for a number of different contentctl
+    operations including validate, build, inspect, test, and test_servers. A file will 
+    be used as the basis for constructing the configuration.
+
+    Args:
+        path (Path, optional): Relative or absolute path to a contentctl config file. 
+        Defaults to Path("contentctl.yml"), which is the default name and location (in the current directory)
+        of the configuration files which are automatically generated for contentctl.
+        config (dict[], optional): Dictionary of values to override values read from the YML
+        path passed as the first argument. Defaults to {}, an empty dict meaning that nothing
+        will be overwritten 
+        configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. 
+        This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test.
+    Returns:
+        test_common: Returns a complete contentctl test_common configuration. Note that this configuration
+        will have all applicable field for validate and build as well, but can also be used for easily
+        construction a test or test_servers object.  
+    """    
+
+    try:
+        yml_dict = YmlReader.load_file(path, add_fields=False)
+        
+        
+    except Exception as e:
+        raise Exception(f"Failed to load contentctl configuration from file '{path}': {str(e)}")
+    
+    # Apply settings that have been overridden from the ones in the file
+    try:
+        yml_dict.update(config)
+    except Exception as e:
+        raise Exception(f"Failed updating dictionary of values read from file '{path}'"
+                        f" with the dictionary of arguments passed: {str(e)}")
+
+    # The function below will throw its own descriptive exception if it fails
+    configObject = config_from_dict(yml_dict, configType=configType)
+
+    return configObject
+
+
+
+
+def config_from_dict(config: dict[str,Any]={}, 
+                   configType:Type[Union[test,test_servers]]=test)->test_common:
+    """
+    Fetch a configuration object that can be used for a number of different contentctl
+    operations including validate, build, inspect, test, and test_servers. A dict will 
+    be used as the basis for constructing the configuration.
+
+    Args:
+        config (dict[str,Any],Optional): If a dictionary is not explicitly passed, then
+        an empty dict will be used to create a configuration, if possible, from default
+        values.  Note that based on default values in the contentctl/objects/config.py
+        file, this may raise an exception.  If so, please set appropriate default values
+        in the file above or supply those values via this argument.
+        configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate. 
+        This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test.
+    Returns:
+        test_common: Returns a complete contentctl test_common configuration. Note that this configuration
+        will have all applicable field for validate and build as well, but can also be used for easily
+        construction a test or test_servers object.  
+    """    
+    try:
+        test_object = configType.model_validate(config)
+    except Exception as e:
+        raise Exception(f"Failed to load contentctl configuration from dict:\n{str(e)}")
+    
+    return test_object
+
+
+def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,Any])->test_common:
+    
+    """Update any relevant keys in a config file with the specified values.
+    Full validation will be performed after this update and descriptive errors
+    will be produced
+
+    Args:
+        config (test_common): A previously-constructed test_common object.  This can be 
+        build using the configFromDict or configFromFile functions.
+        key_value_updates (kwargs, optional): Additional keyword/argument pairs to update
+        arbitrary fields in the configuration.
+
+    Returns:
+        test_common: A validated object which has had the relevant fields updated.
+        Note that descriptive Exceptions will be generated if updated values are either
+        invalid (have the wrong type, or disallowed values) or you attempt to update
+        fields that do not exist
+    """
+    # Create a copy so we don't change the underlying model
+    config_copy = config.model_copy(deep=True)
+
+    # Force validation of assignment since doing so via arbitrary dict can be error prone
+    # Also, ensure that we do not try to add fields that are not part of the model
+    config_copy.model_config.update({'validate_assignment': True, 'extra': 'forbid'})
+
+    
+    
+    # Collect any errors that may occur
+    errors:list[Exception] = []
+    
+    # We need to do this one by one because the extra:forbid argument does not appear to 
+    # be respected at this time.
+    for key, value in key_value_updates.items():
+        try:
+            setattr(config_copy,key,value)
+        except Exception as e:
+            errors.append(e)
+    if len(errors) > 0:
+        errors_string = '\n'.join([str(e) for e in errors])
+        raise Exception(f"Error(s) updaitng configuration:\n{errors_string}")
+    
+    return config_copy
+    
+
+
+def content_to_dict(director:DirectorOutputDto)->dict[str,list[dict[str,Any]]]:
+    output_dict:dict[str,list[dict[str,Any]]] = {}
+    for contentType in ['detections','stories','baselines','investigations',
+                        'playbooks','macros','lookups','deployments','ssa_detections']:
+        
+        output_dict[contentType] = []
+        t:list[SecurityContentObject] = getattr(director,contentType)
+        
+        for item in t:
+            output_dict[contentType].append(item.model_dump())
+    return output_dict
+            

contentctl/contentctl.py

@@ -1,6 +1,11 @@
-from contentctl.actions.initialize import Initialize
+import traceback
+import sys
+import warnings
+import pathlib
 import tyro
-from contentctl.objects.config import init, validate, build,  new, deploy_acs, deploy_rest, test, test_servers, inspect, report, test_common, release_notes
+
+from contentctl.actions.initialize import Initialize
+from contentctl.objects.config import init, validate, build,  new, deploy_acs, test, test_servers, inspect, report, test_common, release_notes
 from contentctl.actions.validate import Validate
 from contentctl.actions.new_content import NewContent
 from contentctl.actions.detection_testing.GitService import GitService
@@ -9,14 +14,10 @@ from contentctl.actions.build import (
      DirectorOutputDto,
      Build,
 )
-
 from contentctl.actions.test import Test
 from contentctl.actions.test import TestInputDto
 from contentctl.actions.reporting import ReportingInputDto, Reporting
 from contentctl.actions.inspect import Inspect
-import sys
-import warnings
-import pathlib
 from contentctl.input.yml_reader import YmlReader
 from contentctl.actions.release_notes import ReleaseNotes
 
@@ -95,13 +96,14 @@ def new_func(config:new):
 
 def deploy_acs_func(config:deploy_acs):
     #This is a bit challenging to get to work with the default values.
-    raise Exception("deploy acs not yet implemented")
-
-def deploy_rest_func(config:deploy_rest):
-    raise Exception("deploy rest not yet implemented")
-    
+    raise Exception("deploy acs not yet implemented") 
 
 def test_common_func(config:test_common):
+    if type(config) == test:
+        #construct the container Infrastructure objects
+        config.getContainerInfrastructureObjects()
+        #otherwise, they have already been passed as servers
+
     director_output_dto = build_func(config)
     gitServer = GitService(director=director_output_dto,config=config)
     detections_to_test = gitServer.getContent()
@@ -175,15 +177,14 @@ def main():
             "test":test.model_validate(config_obj),
             "test_servers":test_servers.model_construct(**t.__dict__),
             "release_notes": release_notes.model_construct(**config_obj),
-            "deploy_acs": deploy_acs.model_construct(**t.__dict__),
-            #"deploy_rest":deploy_rest()
+            "deploy_acs": deploy_acs.model_construct(**t.__dict__)
         }
     )
     
 
 
    
-    
+    config = None
     try:
         # Since some model(s) were constructed and not model_validated, we have to catch
         # warnings again when creating the cli
@@ -209,20 +210,23 @@ def main():
         elif type(config) == deploy_acs:
             updated_config = deploy_acs.model_validate(config)
             deploy_acs_func(updated_config)
-        elif type(config) == deploy_rest:
-            deploy_rest_func(config)
         elif type(config) == test or type(config) == test_servers:
-            if type(config) == test:
-                #construct the container Infrastructure objects
-                config.getContainerInfrastructureObjects()
-                #otherwise, they have already been passed as servers
             test_common_func(config)
         else:
             raise Exception(f"Unknown command line type '{type(config).__name__}'")
     except Exception as e:
-        import traceback
-        traceback.print_exc()
-        traceback.print_stack()
-        #print(e)
+        if config is None:
+            print("There was a serious issue where the config file could not be created.\n"
+                  "The entire stack trace is provided below (please include it if filing a bug report).\n")
+            traceback.print_exc()
+        elif config.verbose:
+            print("Verbose error logging is ENABLED.\n"
+                  "The entire stack trace has been provided below (please include it if filing a bug report):\n")
+            traceback.print_exc()
+        else:
+            print("Verbose error logging is DISABLED.\n"
+                  "Please use the --verbose command line argument if you need more context for your error or file a bug report.")
+            print(e)
+            
         sys.exit(1)
     

contentctl/enrichments/cve_enrichment.py

@@ -4,97 +4,62 @@ import functools
 import os
 import shelve
 import time
-from typing import Annotated
-from pydantic import BaseModel,Field,ConfigDict
-
+from typing import Annotated, Any, Union, TYPE_CHECKING
+from pydantic import BaseModel,Field, computed_field
 from decimal import Decimal
-CVESSEARCH_API_URL = 'https://cve.circl.lu'
-
-CVE_CACHE_FILENAME = "lookups/CVE_CACHE.db"
-
-NON_PERSISTENT_CACHE = {}
-
-
-''''''
-@functools.cache
-def cvesearch_helper(url:str, cve_id:str, force_cached_or_offline:bool=False, max_api_attempts:int=3, retry_sleep_seconds:int=5):
-    if max_api_attempts < 1:
-            raise(Exception(f"The minimum number of CVESearch API attempts is 1.  You have passed {max_api_attempts}"))
-
-    if force_cached_or_offline:
-        if not os.path.exists(CVE_CACHE_FILENAME):
-            print(f"Cache at {CVE_CACHE_FILENAME} not found - Creating it.")
-        cache = shelve.open(CVE_CACHE_FILENAME, flag='c', writeback=True)
-    else:
-        cache = NON_PERSISTENT_CACHE
-    if cve_id in cache:
-        result = cache[cve_id]
-        #print(f"hit cve_enrichment:  {time.time() - start:.2f}")
-    else:
-        api_attempts_remaining = max_api_attempts
-        result = None
-        while api_attempts_remaining > 0:
-            api_attempts_remaining -= 1
-            try:
-                cve = cvesearch_id_helper(url)
-                result = cve.id(cve_id)
-                break
-            except Exception as e:
-                if api_attempts_remaining > 0:
-                    print(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}: {str(e)}")
-                    print(f"Retrying the CVESearch API up to {api_attempts_remaining} more times after a sleep of {retry_sleep_seconds} seconds...")
-                    time.sleep(retry_sleep_seconds)
-                else:
-                    raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL} after {max_api_attempts} attempts: {str(e)}"))
-            
-        if result is None:
-            raise(Exception(f'CveEnrichment for [ {cve_id} ] failed - CVE does not exist'))
-        cache[cve_id] = result
-        
-    if isinstance(cache, shelve.Shelf):
-        #close the cache if it was a shelf
-        cache.close()
+from requests.exceptions import ReadTimeout
 
-    return result
+if TYPE_CHECKING:
+    from contentctl.objects.config import validate
 
-@functools.cache
-def cvesearch_id_helper(url:str):
-    #The initial CVESearch call takes some time.
-    #We cache it to avoid making this call each time we need to do a lookup
-    cve = CVESearch(CVESSEARCH_API_URL)
-    return cve
 
 
+CVESSEARCH_API_URL = 'https://cve.circl.lu'
 
 
 class CveEnrichmentObj(BaseModel):
     id:Annotated[str, "^CVE-[1|2][0-9]{3}-[0-9]+$"]
     cvss:Annotated[Decimal, Field(ge=.1, le=10, decimal_places=1)]
     summary:str
+    
+    @computed_field
+    @property
+    def url(self)->str:
+        BASE_NVD_URL = "https://nvd.nist.gov/vuln/detail/"
+        return f"{BASE_NVD_URL}{self.id}"
 
 
-    @staticmethod
-    def buildEnrichmentOnFailure(id:Annotated[str, "^CVE-[1|2][0-9]{3}-[0-9]+$"], errorMessage:str)->CveEnrichmentObj:
-        message = f"{errorMessage}. Default CVSS of 5.0 used"
-        print(message)
-        return CveEnrichmentObj(id=id, cvss=Decimal(5.0), summary=message)
+class CveEnrichment(BaseModel):
+    use_enrichment: bool = True
+    cve_api_obj: Union[CVESearch,None] = None
+    
 
-class CveEnrichment():
-    @classmethod
-    def enrich_cve(cls, cve_id: str, force_cached_or_offline: bool = False, treat_failures_as_warnings:bool=True) -> CveEnrichmentObj:
-        cve_enriched = dict()
-        try:
-            
-            result = cvesearch_helper(CVESSEARCH_API_URL, cve_id, force_cached_or_offline)
-            cve_enriched['id'] = cve_id
-            cve_enriched['cvss'] = result['cvss']
-            cve_enriched['summary'] = result['summary']
-        except Exception as e:
-            message = f"issue enriching {cve_id}, with error: {str(e)}"
-            if treat_failures_as_warnings: 
-                return CveEnrichmentObj.buildEnrichmentOnFailure(id = cve_id, errorMessage=f"WARNING, {message}")
-            else:
-                raise ValueError(f"ERROR, {message}")
+    class Config:
+        # Arbitrary_types are allowed to let us use the CVESearch Object
+        arbitrary_types_allowed = True
+        frozen = True
+        
+
+    @staticmethod
+    def getCveEnrichment(config:validate, timeout_seconds:int=10, force_disable_enrichment:bool=True)->CveEnrichment:
+        if force_disable_enrichment:
+            return CveEnrichment(use_enrichment=False, cve_api_obj=None)    
         
-        return CveEnrichmentObj.model_validate(cve_enriched)
+        if config.enrichments:
+            try:
+                cve_api_obj = CVESearch(CVESSEARCH_API_URL, timeout=timeout_seconds)
+                return CveEnrichment(use_enrichment=True, cve_api_obj=cve_api_obj)
+            except Exception as e:
+                raise Exception(f"Error setting CVE_SEARCH API to: {CVESSEARCH_API_URL}: {str(e)}")
         
+        return CveEnrichment(use_enrichment=False, cve_api_obj=None)
+
+
+    def enrich_cve(self, cve_id:str, raise_exception_on_failure:bool=True)->CveEnrichmentObj:
+
+        if not self.use_enrichment:
+            return CveEnrichmentObj(id=cve_id,cvss=Decimal(5.0),summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME")
+        else:
+            print("WARNING - Dynamic enrichment not supported at this time.")
+            return CveEnrichmentObj(id=cve_id,cvss=Decimal(5.0),summary="SUMMARY NOT AVAILABLE! ONLY THE LINK WILL BE USED AT THIS TIME")
+        # Depending on needs, we may add dynamic enrichment functionality back to the tool