contentctl 4.1.3__py3-none-any.whl → 4.1.5__py3-none-any.whl

contentctl/actions/detection_testing/GitService.py

@@ -111,11 +111,19 @@ class GitService(BaseModel):
                                 raise Exception(f"More than 1 Lookup reference the modified CSV file '{decoded_path}': {[l.file_path for l in matched ]}")
                             else:
                                 updatedLookup = matched[0]
+                        elif decoded_path.suffix == ".mlmodel":
+                            # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
+                            # this time, we will ignore this change.
+                            updatedLookup = None
+                            
+
                         else:
-                            raise Exception(f"Error getting lookup object for file {str(decoded_path)}")
+                            raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
+                                            "Only files ending in .csv, .yml, or .mlmodel are supported in this "
+                                            "directory. This file must be removed from the lookups/ directory.")
                         
-                        if updatedLookup not in updated_lookups:
-                            # It is possible that both th CSV and YML have been modified for the same lookup,
+                        if updatedLookup is not None and updatedLookup not in updated_lookups:
+                            # It is possible that both the CSV and YML have been modified for the same lookup,
                             # and we do not want to add it twice. 
                             updated_lookups.append(updatedLookup)
 

contentctl/actions/new_content.py

@@ -32,7 +32,6 @@ class NewContent:
         answers['status'] = "production" #start everything as production since that's what we INTEND the content to become   
         answers['description'] = 'UPDATE_DESCRIPTION'   
         file_name = answers['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower()
-        answers['kind'] = answers['detection_kind']
         answers['search'] = answers['detection_search'] + ' | `' + file_name + '_filter`'
         del answers['detection_search']
         answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT'

contentctl/actions/validate.py

@@ -1,20 +1,11 @@
-import sys
 
-from dataclasses import dataclass
-
-from pydantic import ValidationError
-from typing import Union
-
-from contentctl.objects.enums import SecurityContentProduct
-from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
-    SecurityContentObject_Abstract,
-)
+import pathlib
 from contentctl.input.director import Director, DirectorOutputDto
-
 from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
 from contentctl.objects.atomic import AtomicTest
+from contentctl.helper.utils import Utils
 
 
 class Validate:
@@ -42,38 +33,44 @@ class Validate:
 
         director = Director(director_output_dto)
         director.execute(input_dto)
+        self.ensure_no_orphaned_files_in_lookups(input_dto.path, director_output_dto)
         return director_output_dto
 
-    def validate_duplicate_uuids(
-        self, security_content_objects: list[SecurityContentObject_Abstract]
-    ):
-        all_uuids = set()
-        duplicate_uuids = set()
-        for elem in security_content_objects:
-            if elem.id in all_uuids:
-                # The uuid has been found more than once
-                duplicate_uuids.add(elem.id)
-            else:
-                # This is the first time the uuid has been found
-                all_uuids.add(elem.id)
+    
+    def ensure_no_orphaned_files_in_lookups(self, repo_path:pathlib.Path, director_output_dto:DirectorOutputDto):
+        """
+        This function ensures that only files which are relevant to lookups are included in the lookups folder.
+        This means that a file must be either:
+        1. A lookup YML (.yml)
+        2. A lookup CSV (.csv) which is referenced by a YML
+        3. A lookup MLMODEL (.mlmodel) which is referenced by a YML.
+        
+        All other files, includes CSV and MLMODEL files which are NOT
+        referenced by a YML, will generate an exception from this function.
+        
+        Args:
+            repo_path (pathlib.Path): path to the root of the app
+            director_output_dto (DirectorOutputDto): director object with all constructed content
 
-        if len(duplicate_uuids) == 0:
-            return
+        Raises:
+            Exception: An Exception will be raised if there are any non .yml, .csv, or .mlmodel 
+            files in this directory. Additionally, an exception will be raised if there 
+            exists one or more .csv or .mlmodel files that are not referenced by at least 1 
+            detection .yml file in this directory. 
+            This avoids having additional, unused files in this directory that may be copied into
+            the app when it is built (which can cause appinspect errors or larger app size.)
+        """        
+        lookupsDirectory = repo_path/"lookups"
+        
+        # Get all of the files referneced by Lookups
+        usedLookupFiles:list[pathlib.Path] = [lookup.filename for lookup in director_output_dto.lookups if lookup.filename is not None] + [lookup.file_path for lookup in director_output_dto.lookups if lookup.file_path is not None]
 
-        # At least once duplicate uuid has been found. Enumerate all
-        # the pieces of content that use duplicate uuids
-        duplicate_messages = []
-        for uuid in duplicate_uuids:
-            duplicate_uuid_content = [
-                str(content.file_path)
-                for content in security_content_objects
-                if content.id in duplicate_uuids
-            ]
-            duplicate_messages.append(
-                f"Duplicate UUID [{uuid}] in {duplicate_uuid_content}"
-            )
-
-        raise ValueError(
-            "ERROR: Duplicate ID(s) found in objects:\n"
-            + "\n - ".join(duplicate_messages)
-        )
+        # Get all of the mlmodel and csv files in the lookups directory
+        csvAndMlmodelFiles  = Utils.get_security_content_files_from_directory(lookupsDirectory, allowedFileExtensions=[".yml",".csv",".mlmodel"], fileExtensionsToReturn=[".csv",".mlmodel"])
+        
+        # Generate an exception of any csv or mlmodel files exist but are not used
+        unusedLookupFiles:list[pathlib.Path] = [testFile for testFile in csvAndMlmodelFiles if testFile not in usedLookupFiles]
+        if len(unusedLookupFiles) > 0:
+            raise Exception(f"The following .csv or .mlmodel files exist in '{lookupsDirectory}', but are not referenced by a lookup file: {[str(path) for path in unusedLookupFiles]}")
+        return
+    

contentctl/helper/utils.py

@@ -34,6 +34,49 @@ class Utils:
                     listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
     
         return sorted(listOfFiles)
+    
+    @staticmethod
+    def get_security_content_files_from_directory(path: pathlib.Path, allowedFileExtensions:list[str]=[".yml"], fileExtensionsToReturn:list[str]=[".yml"]) -> list[pathlib.Path]:
+    
+        """
+        Get all of the Security Content Object Files rooted in a given directory.  These will almost
+        certain be YML files, but could be other file types as specified by the user
+
+        Args:
+            path (pathlib.Path): The root path at which to enumerate all Security Content Files. All directories will be traversed. 
+            allowedFileExtensions (set[str], optional): File extensions which are allowed to be  present in this directory.  In most cases, we do not want to allow the presence of non-YML files. Defaults to [".yml"].
+            fileExtensionsToReturn (set[str], optional): Filenames with extensions that should be returned from this function. For example, the lookups/ directory contains YML, CSV, and MLMODEL directories, but only the YMLs are Security Content Objects for constructing Lookyps. Defaults to[".yml"].
+
+        Raises:
+            Exception: Will raise an exception if allowedFileExtensions is not a subset of fileExtensionsToReturn.
+            Exception: Will raise an exception if the path passed to the function does not exist or is not a directory
+            Exception: Will raise an exception if there are any files rooted in the directory which are not in allowedFileExtensions
+
+        Returns:
+            list[pathlib.Path]: list of files with an extension in fileExtensionsToReturn found in path
+        """
+        if not set(fileExtensionsToReturn).issubset(set(allowedFileExtensions)):
+            raise Exception(f"allowedFileExtensions {allowedFileExtensions} MUST be a subset of fileExtensionsToReturn {fileExtensionsToReturn}, but it is not")
+        
+        if not path.exists() or not path.is_dir():
+            raise Exception(f"Unable to get security_content files, required directory '{str(path)}' does not exist or is not a directory")
+        
+        allowedFiles:list[pathlib.Path] = []
+        erroneousFiles:list[pathlib.Path] = []
+        #Get every single file extension 
+        for filePath in path.glob("**/*.*"):
+            if filePath.suffix in allowedFileExtensions:
+                # Yes these are allowed
+                allowedFiles.append(filePath)
+            else:
+                # No these have not been allowed
+                erroneousFiles.append(filePath)
+
+        if len(erroneousFiles):
+            raise Exception(f"The following files are not allowed in the directory '{path}'. Only files with the extensions {allowedFileExtensions} are allowed:{[str(filePath) for filePath in erroneousFiles]}")
+        
+        # There were no errorneous files, so return the requested files
+        return sorted([filePath for filePath in allowedFiles if filePath.suffix in fileExtensionsToReturn])
 
     @staticmethod
     def get_all_yml_files_from_directory_one_layer_deep(path: str) -> list[pathlib.Path]:

contentctl/objects/lookup.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer
 from typing import TYPE_CHECKING, Optional, Any, Union
 import re
+import csv
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
     from contentctl.objects.config import validate
@@ -61,15 +62,53 @@ class Lookup(SecurityContentObject):
                 raise ValueError("config required for constructing lookup filename, but it was not")
         return data
 
-    @field_validator('filename')
-    @classmethod
-    def lookup_file_valid(cls, v: Union[FilePath,None], info: ValidationInfo):
-        if not v:
-            return v
-        if not (v.name.endswith(".csv") or v.name.endswith(".mlmodel")):
-            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{v}'")
 
-        return v
+    def model_post_init(self, ctx:dict[str,Any]):
+        if not self.filename:
+            return
+        import pathlib
+        filenamePath = pathlib.Path(self.filename)
+        
+        if filenamePath.suffix not in [".csv", ".mlmodel"]:
+            raise ValueError(f"All Lookup files must be CSV files and end in .csv.  The following file does not: '{filenamePath}'")
+        
+        
+
+        if filenamePath.suffix == ".mlmodel":
+            # Do not need any additional checks for an mlmodel file
+            return
+
+        # https://docs.python.org/3/library/csv.html#csv.DictReader
+        # Column Names (fieldnames) determine by the number of columns in the first row.
+        # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception
+        # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception.    
+        csv_errors:list[str] = []
+        with open(filenamePath, "r") as csv_fp:
+            RESTKEY = "extra_fields_in_a_row"
+            csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY)            
+            if csv_dict.fieldnames is None:
+                raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t"
+                                 "Unable to read fieldnames from CSV. Is the CSV empty?\n"
+                                 "  Please try opening the file with a CSV Editor to ensure that it is correct.")
+            # Remember that row 1 has the headers and we do not iterate over it in the loop below
+            # CSVs are typically indexed starting a row 1 for the header.
+            for row_index, data_row in enumerate(csv_dict):
+                row_index+=2
+                if len(data_row.get(RESTKEY,[])) > 0:
+                    csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns,"
+                                      f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY,[]))}].")
+                
+                for column_index, column_name in enumerate(data_row):
+                    if data_row[column_name] is None:
+                        csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, "
+                                          f"but instead had [{column_index}].")
+        if len(csv_errors) > 0:
+            err_string = '\n\t'.join(csv_errors)
+            raise ValueError(f"Error validating the CSV referenced by the lookup: {filenamePath}:\n\t{err_string}\n"
+                             f"  Please try opening the file with a CSV Editor to ensure that it is correct.")
+    
+        return
+    
         
     @field_validator('match_type')
     @classmethod

contentctl/templates/app_template/default/app.conf

@@ -8,7 +8,6 @@ build = 16367
 
 [triggers]
 reload.analytic_stories = simple
-reload.usage_searches = simple
 reload.use_case_library = simple
 reload.correlationsearches = simple
 reload.analyticstories = simple

{contentctl-4.1.3.dist-info → contentctl-4.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contentctl
-Version: 4.1.3
+Version: 4.1.5
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

{contentctl-4.1.3.dist-info → contentctl-4.1.5.dist-info}/RECORD

@@ -3,7 +3,7 @@ contentctl/actions/build.py,sha256=BVc-1E63zeUQ9wWAHTC_fLNvfEK5YT3Z6_QLiE72TQs,4
 contentctl/actions/convert.py,sha256=0KBWLxvP1hSPXpExePqpOQPRvlQLamvPLyQqeTIWNbk,704
 contentctl/actions/deploy_acs.py,sha256=mf3uk495H1EU_LNN-TiOsYCo18HMGoEBMb6ojeTr0zw,1418
 contentctl/actions/detection_testing/DetectionTestingManager.py,sha256=zg8JasDjCpSC-yhseEyUwO8qbDJIUJbhlus9Li9ZAnA,8818
-contentctl/actions/detection_testing/GitService.py,sha256=FY_JtMi3qL-uC31Yyf7CTWZ5kLQhAMAvcj-8QXtkfPM,8386
+contentctl/actions/detection_testing/GitService.py,sha256=xNhuvK8oUoTxFlC0XBhlew9V0DO7l2hqaBMffEk5ohM,9000
 contentctl/actions/detection_testing/generate_detection_coverage_badge.py,sha256=N5mznaeErVak3mOBwsd0RDBFJO3bku0EZvpayCyU-uk,2259
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=VFhSHdw_0N6ol668hDkaj7yFjPsZqBoFNC8FKzWKICc,53141
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py,sha256=HVGWCXy0GQeBqu2cVJn5H-I8GY8rwgkkc53ilO1TfZA,6846
@@ -17,11 +17,11 @@ contentctl/actions/doc_gen.py,sha256=YNc1VYA0ikL1hWDHYjfEOmUkfhy8PEIdvTyC4ZLxQRY
 contentctl/actions/initialize.py,sha256=2h3_A68mNWcyZjbrKF-OeQXBi5p4Zu3z74K7QxEtII4,1749
 contentctl/actions/initialize_old.py,sha256=0qXbW_fNDvkcnEeL6Zpte8d-hpTu1REyzHsXOCY-YB8,9333
 contentctl/actions/inspect.py,sha256=6gVVKmV5CUUYOkNNVTMPKj9bM1uXVthgGCoFKZGDeS8,12628
-contentctl/actions/new_content.py,sha256=4gTlxV0fmdsSETPX4T9uQPpIAm--Jf2vc6Vm3w-RkfI,6128
+contentctl/actions/new_content.py,sha256=o5ZYBQ216RN6TnW_wRxVGJybx2SsJ7ht4PAi1dw45Yg,6076
 contentctl/actions/release_notes.py,sha256=akkFfLhsJuaPUyjsb6dLlKt9cUM-JApAjTFQMbYoXeM,13115
 contentctl/actions/reporting.py,sha256=MJEmvmoA1WnSFZEU9QM6daL_W94oOX0WXAcX1qAM2As,1583
 contentctl/actions/test.py,sha256=dx7f750_MrlvysxOmOdIro1bH0iVKF4K54TSwhvU2MU,5146
-contentctl/actions/validate.py,sha256=HnmB_qsluYr0BFHQzg0HvXGLHM4M1taBtsWt774esf8,2537
+contentctl/actions/validate.py,sha256=E6bQ0lZkFiFyC4hyRuypKiMybZSE4EXvzd94B0fQUFo,3590
 contentctl/api.py,sha256=FBOpRhbBCBdjORmwe_8MPQ3PRZ6T0KrrFcfKovVFkug,6343
 contentctl/contentctl.py,sha256=Vr2cuvaPjpJpYvD9kVoYq7iD6rhLQEpTKmcGoq4emhA,10470
 contentctl/enrichments/attack_enrichment.py,sha256=EkEloG3hMmPTloPyYiVkhq3iT_BieXaJmprJ5stfyRw,6732
@@ -29,7 +29,7 @@ contentctl/enrichments/cve_enrichment.py,sha256=IzkKSdnQi3JrAUUyLpcGA_Y2g_B7latq
 contentctl/enrichments/splunk_app_enrichment.py,sha256=zDNHFLZTi2dJ1gdnh0sHkD6F1VtkblqFnhacFcCMBfc,3418
 contentctl/helper/link_validator.py,sha256=-XorhxfGtjLynEL1X4hcpRMiyemogf2JEnvLwhHq80c,7139
 contentctl/helper/logger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-contentctl/helper/utils.py,sha256=THV4ZuaeEHG6PK5JeZUkTLBmvN4fUV0Ar6opH1zXxKs,16545
+contentctl/helper/utils.py,sha256=8ICRvE7DUiNL9BK4Hw71hCLFbd3R2u86OwKeDOdaBTY,19454
 contentctl/input/backend_splunk_ba.py,sha256=Y70tJqgaUM0nzfm2SiGMof4HkhY84feqf-xnRx1xPb4,5861
 contentctl/input/director.py,sha256=BR1RvBD0U_JtHtrM3jM_FpcvaaNlME7nc-gNO4RJLM8,13323
 contentctl/input/new_content_questions.py,sha256=o4prlBoUhEMxqpZukquI9WKbzfFJfYhEF7a8m2q_BEE,5565
@@ -63,7 +63,7 @@ contentctl/objects/integration_test.py,sha256=W_VksBN_cRo7DTXdr1aLujjS9mgkEp0uvo
 contentctl/objects/integration_test_result.py,sha256=DrIZRRlILSHGcsK_Rlm3KJLnbKPtIen8uEPFi4ZdJ8s,370
 contentctl/objects/investigation.py,sha256=JRoZxc_qi1fu_VFTRaxOc3B7zzSzCfEURsNzWPUCrtY,2620
 contentctl/objects/investigation_tags.py,sha256=nFpMRKBVBsW21YW_vy2G1lXaSARX-kfFyrPoCyE77Q8,1280
-contentctl/objects/lookup.py,sha256=P8YbzdDAj_MsTBJTEsym35zhQjiN9Eq0MlfON-qvuTM,4556
+contentctl/objects/lookup.py,sha256=TwNQqeMPeE8sfAjChxS2yDnejI2Xf3ils3_Xdgthr5c,6924
 contentctl/objects/macro.py,sha256=9nE-bxkFhtaltHOUCr0luU8jCCthmglHjhKs6Q2YzLU,2684
 contentctl/objects/mitre_attack_enrichment.py,sha256=bWrMG-Xj3knmULR5q2YZk7mloJBdQUzU1moZfEw9lQM,1073
 contentctl/objects/notable_action.py,sha256=ValkblBaG-60TF19y_vSnNzoNZ3eg48wIfr0qZxyKTA,1605
@@ -135,13 +135,12 @@ contentctl/templates/app_template/README/essoc_summary.txt,sha256=u6wYNYBqmmm7Kn
 contentctl/templates/app_template/README/essoc_usage_dashboard.txt,sha256=xYUKKVtdgzPyT3mqdTccaBZuwWnC63lbc9zyYpmHN4o,2432
 contentctl/templates/app_template/README.md,sha256=RT-J9bgRSFsEFgNr9qV6yc2LkfUH_uiMJ2RV4NM9Ymo,366
 contentctl/templates/app_template/default/analytic_stories.conf,sha256=zWuCOOl8SiP7Kit2s-de4KRu3HySLtBSXcp1QnJx0ec,168
-contentctl/templates/app_template/default/app.conf,sha256=eTSq1QI4-BgylZJgnNVg5jQCZFXJVNyEJA33lQAgYoc,685
+contentctl/templates/app_template/default/app.conf,sha256=PrW8TosZ5oVBfpB0SoLxa5vk2ewEAbVKQ6rG8g5WDSQ,654
 contentctl/templates/app_template/default/commands.conf,sha256=U2ccwUeGXKKKt5jo14QY5swi-p9_TSJtaNquOkeF3Yk,319
 contentctl/templates/app_template/default/content-version.conf,sha256=TGzX6qLdzRK7x6b0y5AE8ZF59PLU-DrRfS43fVWITqo,34
 contentctl/templates/app_template/default/data/ui/nav/default.xml,sha256=fKN53HZCtNJbQqq_5pP8e5-5m30DRrJittr6q5s6V_0,236
 contentctl/templates/app_template/default/data/ui/views/escu_summary.xml,sha256=jQhkIthPgEEptCJ2wUCj2lWGHBvUl6JGsKkDfONloxI,8635
 contentctl/templates/app_template/default/data/ui/views/feedback.xml,sha256=uM71EMK2uFz8h68nOTNKGnYxob3HhE_caSL6yA-3H-k,696
-contentctl/templates/app_template/default/usage_searches.conf,sha256=mFnhAHGhFHIzl8xxA626thnAjyxs5ZQQfur1PP_Xmbg,4257
 contentctl/templates/app_template/default/use_case_library.conf,sha256=zWuCOOl8SiP7Kit2s-de4KRu3HySLtBSXcp1QnJx0ec,168
 contentctl/templates/app_template/lookups/mitre_enrichment.csv,sha256=tifPQjFoQHtvpb78hxSP2fKHnHeehNbZDwUjdvc0aEM,66072
 contentctl/templates/app_template/metadata/default.meta,sha256=tcYHZkDF44ApDoDQ_rp8MCA8cuT3DVd5atHgulR1Tvc,423
@@ -164,8 +163,8 @@ contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
 contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
 contentctl/templates/stories/cobalt_strike.yml,sha256=rlaXxMN-5k8LnKBLPafBoksyMtlmsPMHPJOjTiMiZ-M,3063
-contentctl-4.1.3.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
-contentctl-4.1.3.dist-info/METADATA,sha256=VxqSwKj59aGzVSuESf7w0PefidDnsaFwre-MlIh6nLI,19706
-contentctl-4.1.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-contentctl-4.1.3.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
-contentctl-4.1.3.dist-info/RECORD,,
+contentctl-4.1.5.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
+contentctl-4.1.5.dist-info/METADATA,sha256=dzhmOQMe0mFoHlZ12p7FsQzHGF-jSKvzbAosP5fTsC8,19706
+contentctl-4.1.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+contentctl-4.1.5.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
+contentctl-4.1.5.dist-info/RECORD,,

contentctl/templates/app_template/default/usage_searches.conf

@@ -1,73 +0,0 @@
-[escu-metrics-usage]
-action.email.useNSSubject = 1
-alert.digest_mode = True
-alert.suppress = 0
-alert.track = 0
-auto_summarize.dispatch.earliest_time = -1d@h
-dispatchAs = user
-search = index=_audit sourcetype="audittrail" \
-"ESCU - "\
-`comment("Find all the search names in the audittrail.")`\
-| stats count(search) by search savedsearch_name user\
-| eval usage=(if(savedsearch_name=="","Adhoc","Scheduled")) \
-`comment("If the savedsearch_name field in the audittrail is empty, the search was run adhoc. Otherwise it was run as a scheduled search")`\
-| rex field=search "\"(?<savedsearch_name>.*)\""\
-`comment("Extract the name of the search from the search string")`\
-| table savedsearch_name count(search) usage user | join savedsearch_name max=0 type=left [search sourcetype="manifests" | spath searches{} | mvexpand searches{} | spath input=searches{} | table category search_name | rename search_name as savedsearch_name | dedup savedsearch_name] | search category=*
-
-[escu-metrics-search]
-action.email.useNSSubject = 1
-alert.suppress = 0
-alert.track = 0
-auto_summarize.dispatch.earliest_time = -1d@h
-enableSched = 1
-cron_schedule = 0 0 * * *
-dispatch.earliest_time = -4h@h
-dispatch.latest_time = -1h@h
-search = index=_audit action=search | transaction search_id maxspan=3m | search ESCU | stats sum(total_run_time) avg(total_run_time) max(total_run_time) sum(result_count)
-
-[escu-metrics-search-events]
-action.email.useNSSubject = 1
-alert.digest_mode = True
-alert.suppress = 0
-alert.track = 0
-auto_summarize.dispatch.earliest_time = -1d@h
-cron_schedule = 0 0 * * *
-enableSched = 1
-dispatch.earliest_time = -4h@h
-dispatch.latest_time = -1h@h
-search = [search index=_audit sourcetype="audittrail" \"ESCU NOT "index=_audit" | where search !="" | dedup search_id | rex field=search "\"(?<search_name>.*)\"" | rex field=_raw "user=(?<user>[a-zA-Z0-9_\-]+)" | eval usage=if(savedsearch_name!="", "scheduled", "adhoc") | eval savedsearch_name=if(savedsearch_name != "", savedsearch_name, search_name) | table savedsearch_name search_id user _time usage | outputlookup escu_search_id.csv | table search_id] index=_audit total_run_time event_count result_count NOT "index=_audit" | lookup escu_search_id.csv search_id | stats count(savedsearch_name) AS search_count avg(total_run_time) AS search_avg_run_time sum(total_run_time) AS search_total_run_time sum(result_count) AS search_total_results earliest(_time) AS firsts latest(_time) AS lasts by savedsearch_name user usage| eval first_run=strftime(firsts, "%B %d %Y") | eval last_run=strftime(lasts, "%B %d %Y")
-
-[escu-metrics-search-longest-runtime]
-action.email.useNSSubject = 1
-alert.digest_mode = True
-alert.suppress = 0
-alert.track = 0
-auto_summarize.dispatch.earliest_time = -1d@h
-enableSched = 1
-cron_schedule = 0 0 * * *
-disabled = 1
-dispatch.earliest_time = -4h@h
-dispatch.latest_time = -1h@h
-search = index=_* ESCU [search index=_* action=search latest=-2h earliest=-1d| transaction search_id maxspan=3m | search ESCU | stats values(total_run_time) AS run by search_id | sort -run | head 1| table search_id] | table search search_id
-
-[escu-metrics-usage-search]
-action.email.useNSSubject = 1
-alert.digest_mode = True
-alert.suppress = 0
-alert.track = 0
-auto_summarize.dispatch.earliest_time = -1d@h
-cron_schedule = 0 0 * * *
-dispatch.earliest_time = -4h@h
-dispatch.latest_time = -1h@h
-enableSched = 1
-dispatchAs = user
-search = index=_audit sourcetype="audittrail" \
-"ESCU - "\
-`comment("Find all the search names in the audittrail. Ignore the last few minutes so we can exclude this search's text from the result.")`\
-| stats count(search) by search savedsearch_name user\
-| eval usage=(if(savedsearch_name=="","Adhoc","Scheduled")) \
-`comment("If the savedsearch_name field in the audittrail is empty, the search was run adhoc. Otherwise it was run as a scheduled search")`\
-| rex field=search "\"(?<savedsearch_name>.*)\""\
-`comment("Extract the name of the search from the search string")`\
-| table savedsearch_name count(search) usage user | join savedsearch_name max=0 type=left [search sourcetype="manifests" | spath searches{} | mvexpand searches{} | spath input=searches{} | table category search_name | rename search_name as savedsearch_name | dedup savedsearch_name] | search category=*