contentctl 4.1.4__py3-none-any.whl → 4.1.5__py3-none-any.whl

contentctl/actions/detection_testing/GitService.py

@@ -111,11 +111,19 @@ class GitService(BaseModel):
                                 raise Exception(f"More than 1 Lookup reference the modified CSV file '{decoded_path}': {[l.file_path for l in matched ]}")
                             else:
                                 updatedLookup = matched[0]
+                        elif decoded_path.suffix == ".mlmodel":
+                            # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
+                            # this time, we will ignore this change.
+                            updatedLookup = None
+                            
+
                         else:
-                            raise Exception(f"Error getting lookup object for file {str(decoded_path)}")
+                            raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
+                                            "Only files ending in .csv, .yml, or .mlmodel are supported in this "
+                                            "directory. This file must be removed from the lookups/ directory.")
                         
-                        if updatedLookup not in updated_lookups:
-                            # It is possible that both th CSV and YML have been modified for the same lookup,
+                        if updatedLookup is not None and updatedLookup not in updated_lookups:
+                            # It is possible that both the CSV and YML have been modified for the same lookup,
                             # and we do not want to add it twice. 
                             updated_lookups.append(updatedLookup)
 

contentctl/actions/validate.py

@@ -1,20 +1,11 @@
-import sys
 
-from dataclasses import dataclass
-
-from pydantic import ValidationError
-from typing import Union
-
-from contentctl.objects.enums import SecurityContentProduct
-from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
-    SecurityContentObject_Abstract,
-)
+import pathlib
 from contentctl.input.director import Director, DirectorOutputDto
-
 from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.objects.atomic import AtomicTest
+from contentctl.helper.utils import Utils
 
 
 class Validate:
@@ -42,38 +33,44 @@ class Validate:
 
         director = Director(director_output_dto)
         director.execute(input_dto)
+        self.ensure_no_orphaned_files_in_lookups(input_dto.path, director_output_dto)
         return director_output_dto
 
-    def validate_duplicate_uuids(
-        self, security_content_objects: list[SecurityContentObject_Abstract]
-    ):
-        all_uuids = set()
-        duplicate_uuids = set()
-        for elem in security_content_objects:
-            if elem.id in all_uuids:
-                # The uuid has been found more than once
-                duplicate_uuids.add(elem.id)
-            else:
-                # This is the first time the uuid has been found
-                all_uuids.add(elem.id)
+    
+    def ensure_no_orphaned_files_in_lookups(self, repo_path:pathlib.Path, director_output_dto:DirectorOutputDto):
+        """
+        This function ensures that only files which are relevant to lookups are included in the lookups folder.
+        This means that a file must be either:
+        1. A lookup YML (.yml)
+        2. A lookup CSV (.csv) which is referenced by a YML
+        3. A lookup MLMODEL (.mlmodel) which is referenced by a YML.
+        
+        All other files, includes CSV and MLMODEL files which are NOT
+        referenced by a YML, will generate an exception from this function.
+        
+        Args:
+            repo_path (pathlib.Path): path to the root of the app
+            director_output_dto (DirectorOutputDto): director object with all constructed content
 
-        if len(duplicate_uuids) == 0:
-            return
+        Raises:
+            Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel 
+            files in this directory. Additionally, an exception will be raised if there 
+            exists one or more .csv or .mlmodel files that are not referenced by at least 1 
+            detection .yml file in this directory. 
+            This avoids having additional, unused files in this directory that may be copied into
+            the app when it is built (which can cause appinspect errors or larger app size.)
+        """        
+        lookupsDirectory = repo_path/"lookups"
+        
+        # Get all of the files referneced by Lookups
+        usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if lookup.filename is not None] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None]
 
-        # At least once duplicate uuid has been found. Enumerate all
-        # the pieces of content that use duplicate uuids
-        duplicate_messages = []
-        for uuid in duplicate_uuids:
-            duplicate_uuid_content = [
-                str(content.file_path)
-                for content in security_content_objects
-                if content.id in duplicate_uuids
-            ]
-            duplicate_messages.append(
-                f"Duplicate UUID [{uuid}] in {duplicate_uuid_content}"
-            )
-
-        raise ValueError(
-            "ERROR: Duplicate ID(s) found in objects:\n"
-            + "\n - ".join(duplicate_messages)
-        )
+        # Get all of the mlmodel and csv files in the lookups directory
+        csvAndMlmodelFiles  = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"])
+        
+        # Generate an exception of any csv or mlmodel files exist but are not used
+        unusedLookupFiles:list[pathlib.Path] = [testFile for testFile in csvAndMlmodelFiles if testFile not in usedLookupFiles]
+        if len(unusedLookupFiles) > 0:
+            raise Exception(f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}")
+        return
+    

contentctl/helper/utils.py

@@ -34,6 +34,49 @@ class Utils:
                     listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
     
         return sorted(listOfFiles)
+    
+    @staticmethod
+    def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExtensions:list[str]=[".yml"], fileExtensionsToReturn:list[str]=[".yml"]) -> list[pathlib.Path]:
+    
+        """
+        Get all of the Security Content Object Files rooted in a given directory.  These will almost
+        certain be YML files, but could be other file types as specified by the user
+
+        Args:
+            path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed. 
+            allowedFileExtensions (set[str], optional): File extensions which are allowed to be  present in this directory.  In most cases, we do not want to allow the presence of non-YML files. Defaults to [".yml"].
+            fileExtensionsToReturn (set[str], optional): Filenames with extensions that should be returned from this function. For example, the lookups/ directory contains YML, CSV, and MLMODEL directories, but only the YMLs are Security Content Objects for constructing Lookyps. Defaults to[".yml"].
+
+        Raises:
+            Exception: Will raise an exception if allowedFileExtensions is not a subset of fileExtensionsToReturn.
+            Exception: Will raise an exception if the path passed to the function does not exist or is not a directory
+            Exception: Will raise an exception if there are any files rooted in the directory which are not in allowedFileExtensions
+
+        Returns:
+            list[pathlib.Path]: list of files with an extension in fileExtensionsToReturn found in path
+        """
+        if not set(fileExtensionsToReturn).issubset(set(allowedFileExtensions)):
+            raise Exception(f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not")
+        
+        if not path.exists() or not path.is_dir():
+            raise Exception(f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory")
+        
+        allowedFiles:list[pathlib.Path] = []
+        erroneousFiles:list[pathlib.Path] = []
+        #Get every single file extension 
+        for filePath in path.glob("**/*.*"):
+            if filePath.suffix in allowedFileExtensions:
+                # Yes these are allowed
+                allowedFiles.append(filePath)
+            else:
+                # No these have not been allowed
+                erroneousFiles.append(filePath)
+
+        if len(erroneousFiles):
+            raise Exception(f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}")
+        
+        # There were no errorneous files, so return the requested files
+        return sorted([filePath for filePath in allowedFiles if filePath.suffix in fileExtensionsToReturn])
 
     @staticmethod
     def get_all_yml_files_from_directory_one_layer_deep(path: str) -> list[pathlib.Path]:

contentctl/objects/lookup.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer
 from typing import TYPE_CHECKING, Optional, Any, Union
 import re
+import csv
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
     from contentctl.objects.config import validate
@@ -61,15 +62,53 @@ class Lookup(SecurityContentObject):
                 raise ValueError("config required for constructing lookup filename, but it was not")
         return data
 
-    @field_validator('filename')
-    @classmethod
-    def lookup_file_valid(cls, v: Union[FilePath,None], info: ValidationInfo):
-        if not v:
-            return v
-        if not (v.name.endswith(".csv") or v.name.endswith(".mlmodel")):
-            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{v}'")
 
-        return v
+    def model_post_init(self, ctx:dict[str,Any]):
+        if not self.filename:
+            return
+        import pathlib
+        filenamePath = pathlib.Path(self.filename)
+        
+        if filenamePath.suffix not in [".csv", ".mlmodel"]:
+            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{filenamePath}'")
+        
+        
+
+        if filenamePath.suffix == ".mlmodel":
+            # Do not need any additional checks for an mlmodel file
+            return
+
+        # https://docs.python.org/3/library/csv.html#csv.DictReader
+        # Column Names (fieldnames) determine by the number of columns in the first row.
+        # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception
+        # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception.    
+        csv_errors:list[str] = []
+        with open(filenamePath, "r") as csv_fp:
+            RESTKEY = "extra_fields_in_a_row"
+            csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY)            
+            if csv_dict.fieldnames is None:
+                raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t"
+                                 "Unable to read fieldnames from CSV. Is the CSV empty?\n"
+                                 "  Please try opening the file with a CSV Editor to ensure that it is correct.")
+            # Remember that row 1 has the headers and we do not iterate over it in the loop below
+            # CSVs are typically indexed starting a row 1 for the header.
+            for row_index, data_row in enumerate(csv_dict):
+                row_index+=2
+                if len(data_row.get(RESTKEY,[])) > 0:
+                    csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns,"
+                                      f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY,[]))}].")
+                
+                for column_index, column_name in enumerate(data_row):
+                    if data_row[column_name] is None:
+                        csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, "
+                                          f"but instead had [{column_index}].")
+        if len(csv_errors) > 0:
+            err_string = '\n\t'.join(csv_errors)
+            raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t{err_string}\n"
+                             f"  Please try opening the file with a CSV Editor to ensure that it is correct.")
+    
+        return
+    
         
     @field_validator('match_type')
     @classmethod

{contentctl-4.1.4.dist-info → contentctl-4.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contentctl
-Version: 4.1.4
+Version: 4.1.5
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

{contentctl-4.1.4.dist-info → contentctl-4.1.5.dist-info}/RECORD

@@ -3,7 +3,7 @@ contentctl/actions/build.py,sha256=BVc-1E63zeUQ9wWAHTC_fLNvfEK5YT3Z6_QLiE72TQs,4
 contentctl/actions/convert.py,sha256=0KBWLxvP1hSPXpExePqpOQPRvlQLamvPLyQqeTIWNbk,704
 contentctl/actions/deploy_acs.py,sha256=mf3uk495H1EU_LNN-TiOsYCo18HMGoEBMb6ojeTr0zw,1418
 contentctl/actions/detection_testing/DetectionTestingManager.py,sha256=zg8JasDjCpSC-yhseEyUwO8qbDJIUJbhlus9Li9ZAnA,8818
-contentctl/actions/detection_testing/GitService.py,sha256=FY_JtMi3qL-uC31Yyf7CTWZ5kLQhAMAvcj-8QXtkfPM,8386
+contentctl/actions/detection_testing/GitService.py,sha256=xNhuvK8oUoTxFlC0XBhlew9V0DO7l2hqaBMffEk5ohM,9000
 contentctl/actions/detection_testing/generate_detection_coverage_badge.py,sha256=N5mznaeErVak3mOBwsd0RDBFJO3bku0EZvpayCyU-uk,2259
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=VFhSHdw_0N6ol668hDkaj7yFjPsZqBoFNC8FKzWKICc,53141
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py,sha256=HVGWCXy0GQeBqu2cVJn5H-I8GY8rwgkkc53ilO1TfZA,6846
@@ -21,7 +21,7 @@ contentctl/actions/new_content.py,sha256=o5ZYBQ216RN6TnW_wRxVGJybx2SsJ7ht4PAi1dw
 contentctl/actions/release_notes.py,sha256=akkFfLhsJuaPUyjsb6dLlKt9cUM-JApAjTFQMbYoXeM,13115
 contentctl/actions/reporting.py,sha256=MJEmvmoA1WnSFZEU9QM6daL_W94oOX0WXAcX1qAM2As,1583
 contentctl/actions/test.py,sha256=dx7f750_MrlvysxOmOdIro1bH0iVKF4K54TSwhvU2MU,5146
-contentctl/actions/validate.py,sha256=HnmB_qsluYr0BFHQzg0HvXGLHM4M1taBtsWt774esf8,2537
+contentctl/actions/validate.py,sha256=E6bQ0lZkFiFyC4hyRuypKiMybZSE4EXvzd94B0fQUFo,3590
 contentctl/api.py,sha256=FBOpRhbBCBdjORmwe_8MPQ3PRZ6T0KrrFcfKovVFkug,6343
 contentctl/contentctl.py,sha256=Vr2cuvaPjpJpYvD9kVoYq7iD6rhLQEpTKmcGoq4emhA,10470
 contentctl/enrichments/attack_enrichment.py,sha256=EkEloG3hMmPTloPyYiVkhq3iT_BieXaJmprJ5stfyRw,6732
@@ -29,7 +29,7 @@ contentctl/enrichments/cve_enrichment.py,sha256=IzkKSdnQi3JrAUUyLpcGA_Y2g_B7latq
 contentctl/enrichments/splunk_app_enrichment.py,sha256=zDNHFLZTi2dJ1gdnh0sHkD6F1VtkblqFnhacFcCMBfc,3418
 contentctl/helper/link_validator.py,sha256=-XorhxfGtjLynEL1X4hcpRMiyemogf2JEnvLwhHq80c,7139
 contentctl/helper/logger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-contentctl/helper/utils.py,sha256=THV4ZuaeEHG6PK5JeZUkTLBmvN4fUV0Ar6opH1zXxKs,16545
+contentctl/helper/utils.py,sha256=8ICRvE7DUiNL9BK4Hw71hCLFbd3R2u86OwKeDOdaBTY,19454
 contentctl/input/backend_splunk_ba.py,sha256=Y70tJqgaUM0nzfm2SiGMof4HkhY84feqf-xnRx1xPb4,5861
 contentctl/input/director.py,sha256=BR1RvBD0U_JtHtrM3jM_FpcvaaNlME7nc-gNO4RJLM8,13323
 contentctl/input/new_content_questions.py,sha256=o4prlBoUhEMxqpZukquI9WKbzfFJfYhEF7a8m2q_BEE,5565
@@ -63,7 +63,7 @@ contentctl/objects/integration_test.py,sha256=W_VksBN_cRo7DTXdr1aLujjS9mgkEp0uvo
 contentctl/objects/integration_test_result.py,sha256=DrIZRRlILSHGcsK_Rlm3KJLnbKPtIen8uEPFi4ZdJ8s,370
 contentctl/objects/investigation.py,sha256=JRoZxc_qi1fu_VFTRaxOc3B7zzSzCfEURsNzWPUCrtY,2620
 contentctl/objects/investigation_tags.py,sha256=nFpMRKBVBsW21YW_vy2G1lXaSARX-kfFyrPoCyE77Q8,1280
-contentctl/objects/lookup.py,sha256=P8YbzdDAj_MsTBJTEsym35zhQjiN9Eq0MlfON-qvuTM,4556
+contentctl/objects/lookup.py,sha256=TwNQqeMPeE8sfAjChxS2yDnejI2Xf3ils3_Xdgthr5c,6924
 contentctl/objects/macro.py,sha256=9nE-bxkFhtaltHOUCr0luU8jCCthmglHjhKs6Q2YzLU,2684
 contentctl/objects/mitre_attack_enrichment.py,sha256=bWrMG-Xj3knmULR5q2YZk7mloJBdQUzU1moZfEw9lQM,1073
 contentctl/objects/notable_action.py,sha256=ValkblBaG-60TF19y_vSnNzoNZ3eg48wIfr0qZxyKTA,1605
@@ -163,8 +163,8 @@ contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
 contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
 contentctl/templates/stories/cobalt_strike.yml,sha256=rlaXxMN-5k8LnKBLPafBoksyMtlmsPMHPJOjTiMiZ-M,3063
-contentctl-4.1.4.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
-contentctl-4.1.4.dist-info/METADATA,sha256=gyBEu2sSknwen-oTm0WdQtCvaPgbWuJBd4jvPuel6iQ,19706
-contentctl-4.1.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-contentctl-4.1.4.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
-contentctl-4.1.4.dist-info/RECORD,,
+contentctl-4.1.5.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
+contentctl-4.1.5.dist-info/METADATA,sha256=dzhmOQMe0mFoHlZ12p7FsQzHGF-jSKvzbAosP5fTsC8,19706
+contentctl-4.1.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+contentctl-4.1.5.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
+contentctl-4.1.5.dist-info/RECORD,,