contentctl 5.0.0a0__py3-none-any.whl → 5.0.0a3__py3-none-any.whl

contentctl/objects/investigation_tags.py

@@ -1,33 +1,45 @@
 from __future__ import annotations
 from typing import List
-from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer,ConfigDict
+from pydantic import (
+    BaseModel,
+    Field,
+    field_validator,
+    ValidationInfo,
+    model_serializer,
+    ConfigDict,
+)
 from contentctl.objects.story import Story
-from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain
+from contentctl.objects.enums import (
+    SecurityContentInvestigationProductName,
+    SecurityDomain,
+)
+
 
 class InvestigationTags(BaseModel):
     model_config = ConfigDict(extra="forbid")
-    analytic_story: List[Story] = Field([],min_length=1)
-    product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1)
+    analytic_story: List[Story] = Field([], min_length=1)
+    product: List[SecurityContentInvestigationProductName] = Field(..., min_length=1)
     security_domain: SecurityDomain = Field(...)
 
-
-    @field_validator('analytic_story',mode="before")
+    @field_validator("analytic_story", mode="before")
     @classmethod
-    def mapStoryNamesToStoryObjects(cls, v:list[str], info:ValidationInfo)->list[Story]:
-        return Story.mapNamesToSecurityContentObjects(v, info.context.get("output_dto",None))
-    
+    def mapStoryNamesToStoryObjects(
+        cls, v: list[str], info: ValidationInfo
+    ) -> list[Story]:
+        return Story.mapNamesToSecurityContentObjects(
+            v, info.context.get("output_dto", None)
+        )
 
     @model_serializer
     def serialize_model(self):
-        #All fields custom to this model
-        model= {
+        # All fields custom to this model
+        model = {
             "analytic_story": [story.name for story in self.analytic_story],
             "product": self.product,
             "security_domain": self.security_domain,
         }
-        
-        #Combine fields from this model with fields from parent
-        
-        
-        #return the model
-        return model
+
+        # Combine fields from this model with fields from parent
+
+        # return the model
+        return model

contentctl/objects/lookup.py

@@ -1,35 +1,64 @@
 from __future__ import annotations
 
-from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt, computed_field, TypeAdapter
-from enum import StrEnum, auto
-from typing import TYPE_CHECKING, Optional, Any, Union, Literal, Annotated, Self
-import re
-import csv
 import abc
-from functools import cached_property
+import csv
 import pathlib
+import re
+from enum import StrEnum, auto
+from functools import cached_property
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Self
+
+from pydantic import (
+    Field,
+    FilePath,
+    NonNegativeInt,
+    TypeAdapter,
+    ValidationInfo,
+    computed_field,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
+
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
     from contentctl.objects.config import validate
+
 from contentctl.objects.security_content_object import SecurityContentObject
 
 # This section is used to ignore lookups that are NOT  shipped with ESCU app but are used in the detections. Adding exclusions here will so that contentctl builds will not fail.
 LOOKUPS_TO_IGNORE = set(["outputlookup"])
-LOOKUPS_TO_IGNORE.add("ut_shannon_lookup") #In the URL toolbox app which is recommended for ESCU
-LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Asset and Identity Framework
-LOOKUPS_TO_IGNORE.add("cim_corporate_web_domain_lookup") #Shipped with the Asset and Identity Framework
-LOOKUPS_TO_IGNORE.add("cim_corporate_email_domain_lookup") #Shipped with the Enterprise Security
-LOOKUPS_TO_IGNORE.add("cim_cloud_domain_lookup") #Shipped with the Enterprise Security
-
-LOOKUPS_TO_IGNORE.add("alexa_lookup_by_str") #Shipped with the Asset and Identity Framework
-LOOKUPS_TO_IGNORE.add("interesting_ports_lookup") #Shipped with the Asset and Identity Framework
-LOOKUPS_TO_IGNORE.add("asset_lookup_by_str") #Shipped with the Asset and Identity Framework
-LOOKUPS_TO_IGNORE.add("admon_groups_def") #Shipped with the SA-admon addon
-LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Enterprise Security
-
-#Special case for the Detection "Exploit Public Facing Application via Apache Commons Text"
-LOOKUPS_TO_IGNORE.add("=") 
-LOOKUPS_TO_IGNORE.add("other_lookups") 
+LOOKUPS_TO_IGNORE.add(
+    "ut_shannon_lookup"
+)  # In the URL toolbox app which is recommended for ESCU
+LOOKUPS_TO_IGNORE.add(
+    "identity_lookup_expanded"
+)  # Shipped with the Asset and Identity Framework
+LOOKUPS_TO_IGNORE.add(
+    "cim_corporate_web_domain_lookup"
+)  # Shipped with the Asset and Identity Framework
+LOOKUPS_TO_IGNORE.add(
+    "cim_corporate_email_domain_lookup"
+)  # Shipped with the Enterprise Security
+LOOKUPS_TO_IGNORE.add("cim_cloud_domain_lookup")  # Shipped with the Enterprise Security
+
+LOOKUPS_TO_IGNORE.add(
+    "alexa_lookup_by_str"
+)  # Shipped with the Asset and Identity Framework
+LOOKUPS_TO_IGNORE.add(
+    "interesting_ports_lookup"
+)  # Shipped with the Asset and Identity Framework
+LOOKUPS_TO_IGNORE.add(
+    "asset_lookup_by_str"
+)  # Shipped with the Asset and Identity Framework
+LOOKUPS_TO_IGNORE.add("admon_groups_def")  # Shipped with the SA-admon addon
+LOOKUPS_TO_IGNORE.add(
+    "identity_lookup_expanded"
+)  # Shipped with the Enterprise Security
+
+# Special case for the Detection "Exploit Public Facing Application via Apache Commons Text"
+LOOKUPS_TO_IGNORE.add("=")
+LOOKUPS_TO_IGNORE.add("other_lookups")
 
 
 class Lookup_Type(StrEnum):
@@ -38,164 +67,224 @@ class Lookup_Type(StrEnum):
     mlmodel = auto()
 
 
-
 # TODO (#220): Split Lookup into 2 classes
-class Lookup(SecurityContentObject, abc.ABC):    
+class Lookup(SecurityContentObject, abc.ABC):
     default_match: Optional[bool] = None
     # Per the documentation for transforms.conf, EXACT should not be specified in this list,
     # so we include only WILDCARD and CIDR
-    match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field(default=[])
+    match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field(
+        default=[]
+    )
     min_matches: None | NonNegativeInt = Field(default=None)
-    max_matches: None | Annotated[NonNegativeInt, Field(ge=1, le=1000)] = Field(default=None)    
+    max_matches: None | Annotated[NonNegativeInt, Field(ge=1, le=1000)] = Field(
+        default=None
+    )
     case_sensitive_match: None | bool = Field(default=None)
-    
-
-    
-
 
     @model_serializer
     def serialize_model(self):
-        #Call parent serializer
+        # Call parent serializer
         super_fields = super().serialize_model()
 
-        #All fields custom to this model
-        model= {
-            
+        # All fields custom to this model
+        model = {
             "default_match": "true" if self.default_match is True else "false",
             "match_type": self.match_type_to_conf_format,
             "min_matches": self.min_matches,
             "max_matches": self.max_matches,
-            "case_sensitive_match": "true" if self.case_sensitive_match is True else "false",
+            "case_sensitive_match": "true"
+            if self.case_sensitive_match is True
+            else "false",
         }
-        
-        #return the model
+
+        # return the model
         model.update(super_fields)
         return model
 
     @model_validator(mode="before")
-    def fix_lookup_path(cls, data:Any, info: ValidationInfo)->Any:
+    def fix_lookup_path(cls, data: Any, info: ValidationInfo) -> Any:
         if data.get("filename"):
-            config:validate = info.context.get("config",None)
+            config: validate = info.context.get("config", None)
             if config is not None:
                 data["filename"] = config.path / "lookups/" / data["filename"]
             else:
-                raise ValueError("config required for constructing lookup filename, but it was not")
+                raise ValueError(
+                    "config required for constructing lookup filename, but it was not"
+                )
         return data
 
-
     @computed_field
     @cached_property
-    def match_type_to_conf_format(self)->str:
-        return ', '.join(self.match_type)
-    
-        
+    def match_type_to_conf_format(self) -> str:
+        return ", ".join(self.match_type)
+
     @staticmethod
-    def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]:
+    def get_lookups(
+        text_field: str,
+        director: DirectorOutputDto,
+        ignore_lookups: set[str] = LOOKUPS_TO_IGNORE,
+    ) -> list[Lookup]:
         # Comprehensively match all kinds of lookups, including inputlookup and outputlookup
-        inputLookupsToGet = set(re.findall(r'[^\w]inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([\w]+)', text_field, re.IGNORECASE))
-        outputLookupsToGet = set(re.findall(r'[^\w]outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([\w]+)',text_field,re.IGNORECASE))
-        lookupsToGet = set(re.findall(r'[^\w](?:(?<!output)(?<!input))lookup(?:\s*(?:(?:local|update)\s*=\s*(?:true|t|false|f))){0,2}\s+([\w]+)', text_field, re.IGNORECASE))
-
-        
-        input_lookups = Lookup.mapNamesToSecurityContentObjects(list(inputLookupsToGet-LOOKUPS_TO_IGNORE), director)
-        output_lookups = Lookup.mapNamesToSecurityContentObjects(list(outputLookupsToGet-LOOKUPS_TO_IGNORE), director)
-        lookups = Lookup.mapNamesToSecurityContentObjects(list(lookupsToGet-LOOKUPS_TO_IGNORE), director)
+        inputLookupsToGet = set(
+            re.findall(
+                r"[^\w]inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([\w]+)",
+                text_field,
+                re.IGNORECASE,
+            )
+        )
+        outputLookupsToGet = set(
+            re.findall(
+                r"[^\w]outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([\w]+)",
+                text_field,
+                re.IGNORECASE,
+            )
+        )
+        lookupsToGet = set(
+            re.findall(
+                r"[^\w](?:(?<!output)(?<!input))lookup(?:\s*(?:(?:local|update)\s*=\s*(?:true|t|false|f))){0,2}\s+([\w]+)",
+                text_field,
+                re.IGNORECASE,
+            )
+        )
+
+        input_lookups = Lookup.mapNamesToSecurityContentObjects(
+            list(inputLookupsToGet - LOOKUPS_TO_IGNORE), director
+        )
+        output_lookups = Lookup.mapNamesToSecurityContentObjects(
+            list(outputLookupsToGet - LOOKUPS_TO_IGNORE), director
+        )
+        lookups = Lookup.mapNamesToSecurityContentObjects(
+            list(lookupsToGet - LOOKUPS_TO_IGNORE), director
+        )
 
         all_lookups = set(input_lookups + output_lookups + lookups)
 
         return list(all_lookups)
 
 
-
-
 class FileBackedLookup(Lookup, abc.ABC):
     # For purposes of the disciminated union, the child classes which
     # inherit from this class must declare the typing of lookup_type
     # themselves, hence it is not defined in the Lookup class
 
     @model_validator(mode="after")
-    def ensure_lookup_file_exists(self)->Self:
+    def ensure_lookup_file_exists(self) -> Self:
         if not self.filename.exists():
             raise ValueError(f"Expected lookup filename {self.filename} does not exist")
         return self
 
     @computed_field
     @cached_property
-    def filename(self)->FilePath:
-        if self.file_path is None:
-            raise ValueError(f"Cannot get the filename of the lookup {self.lookup_type} because the YML file_path attribute is None") #type: ignore
-        
-        csv_file = self.file_path.parent / f"{self.file_path.stem}.{self.lookup_type}" #type: ignore
-        return csv_file
-    
+    @abc.abstractmethod
+    def filename(self) -> FilePath:
+        """
+        This function computes the backing file for the lookup. It is abstract because different types of lookups
+        (CSV for MlModel) backing files have different name format.
+        """
+        pass
+
     @computed_field
     @cached_property
-    def app_filename(self)->FilePath:
-        '''
-        We may consider two options:
-        1. Always apply the datetime stamp to the end of the file. This makes the code easier
-        2. Only apply the datetime stamp if it is version > 1.  This makes the code a small fraction
-        more complicated, but preserves longstanding CSV that have not been modified in a long time
-        '''
-        return pathlib.Path(f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}") #type: ignore
+    @abc.abstractmethod
+    def app_filename(self) -> FilePath:
+        """
+        This function computes the filenames to write into the app itself.  This is abstract because
+        CSV and MLmodel requirements are different.
+        """
+        pass
+
 
 class CSVLookup(FileBackedLookup):
-    lookup_type:Literal[Lookup_Type.csv]
-    
+    lookup_type: Literal[Lookup_Type.csv]
+
     @model_serializer
     def serialize_model(self):
-        #Call parent serializer
+        # Call parent serializer
         super_fields = super().serialize_model()
 
-        #All fields custom to this model
-        model= {
-            "filename": self.app_filename.name
-        }
-        
-        #return the model
+        # All fields custom to this model
+        model = {"filename": self.app_filename.name}
+
+        # return the model
         model.update(super_fields)
         return model
-    
+
+    @computed_field
+    @cached_property
+    def filename(self) -> FilePath:
+        """
+        This function computes the backing file for the lookup. The names of CSV files must EXACTLY match the
+        names of their lookup definitions except with the CSV file extension rather than the YML file extension.
+        """
+        if self.file_path is None:
+            raise ValueError(
+                f"Cannot get the filename of the lookup {self.lookup_type} because the YML file_path attribute is None"
+            )  # type: ignore
+
+        csv_file = self.file_path.parent / f"{self.file_path.stem}.{self.lookup_type}"  # type: ignore
+
+        return csv_file
+
+    @computed_field
+    @cached_property
+    def app_filename(self) -> FilePath:
+        """
+        This function computes the filenames to write into the app itself.  This is abstract because
+        CSV and MLmodel requirements are different.
+        """
+        return pathlib.Path(
+            f"{self.filename.stem}_{self.date.year}{self.date.month:02}{self.date.day:02}.{self.lookup_type}"
+        )
+
     @model_validator(mode="after")
-    def ensure_correct_csv_structure(self)->Self:
+    def ensure_correct_csv_structure(self) -> Self:
         # https://docs.python.org/3/library/csv.html#csv.DictReader
         # Column Names (fieldnames) determine by the number of columns in the first row.
         # If a row has MORE fields than fieldnames, they will be dumped in a list under the key 'restkey' - this should throw an Exception
-        # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception.    
-        csv_errors:list[str] = []
+        # If a row has LESS fields than fieldnames, then the field should contain None by default. This should also throw an exception.
+        csv_errors: list[str] = []
         with open(self.filename, "r") as csv_fp:
             RESTKEY = "extra_fields_in_a_row"
-            csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY)            
+            csv_dict = csv.DictReader(csv_fp, restkey=RESTKEY)
             if csv_dict.fieldnames is None:
-                raise ValueError(f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t"
-                                 "Unable to read fieldnames from CSV. Is the CSV empty?\n"
-                                 "  Please try opening the file with a CSV Editor to ensure that it is correct.")
+                raise ValueError(
+                    f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t"
+                    "Unable to read fieldnames from CSV. Is the CSV empty?\n"
+                    "  Please try opening the file with a CSV Editor to ensure that it is correct."
+                )
             # Remember that row 1 has the headers and we do not iterate over it in the loop below
             # CSVs are typically indexed starting a row 1 for the header.
             for row_index, data_row in enumerate(csv_dict):
-                row_index+=2
-                if len(data_row.get(RESTKEY,[])) > 0:
-                    csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns,"
-                                      f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY,[]))}].")
-                
+                row_index += 2
+                if len(data_row.get(RESTKEY, [])) > 0:
+                    csv_errors.append(
+                        f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns,"
+                        f" but instead had [{len(csv_dict.fieldnames) + len(data_row.get(RESTKEY, []))}]."
+                    )
+
                 for column_index, column_name in enumerate(data_row):
                     if data_row[column_name] is None:
-                        csv_errors.append(f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, "
-                                          f"but instead had [{column_index}].")
+                        csv_errors.append(
+                            f"row [{row_index}] should have [{len(csv_dict.fieldnames)}] columns, "
+                            f"but instead had [{column_index}]."
+                        )
         if len(csv_errors) > 0:
-            err_string = '\n\t'.join(csv_errors)
-            raise ValueError(f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t{err_string}\n"
-                             f"  Please try opening the file with a CSV Editor to ensure that it is correct.")
-    
-        return self
+            err_string = "\n\t".join(csv_errors)
+            raise ValueError(
+                f"Error validating the CSV referenced by the lookup: {self.filename}:\n\t{err_string}\n"
+                f"  Please try opening the file with a CSV Editor to ensure that it is correct."
+            )
 
+        return self
 
 
 class KVStoreLookup(Lookup):
     lookup_type: Literal[Lookup_Type.kvstore]
-    fields: list[str] = Field(description="The names of the fields/headings for the KVStore.", min_length=1)
+    fields: list[str] = Field(
+        description="The names of the fields/headings for the KVStore.", min_length=1
+    )
 
-    @field_validator("fields", mode='after')
+    @field_validator("fields", mode="after")
     @classmethod
     def ensure_key(cls, values: list[str]):
         if values[0] != "_key":
@@ -204,32 +293,64 @@ class KVStoreLookup(Lookup):
 
     @computed_field
     @cached_property
-    def collection(self)->str:
+    def collection(self) -> str:
         return self.name
 
     @computed_field
     @cached_property
-    def fields_to_fields_list_conf_format(self)->str:
-        return ', '.join(self.fields)
+    def fields_to_fields_list_conf_format(self) -> str:
+        return ", ".join(self.fields)
 
     @model_serializer
     def serialize_model(self):
-        #Call parent serializer
+        # Call parent serializer
         super_fields = super().serialize_model()
 
-        #All fields custom to this model
-        model= {
+        # All fields custom to this model
+        model = {
             "collection": self.collection,
-            "fields_list": self.fields_to_fields_list_conf_format
+            "fields_list": self.fields_to_fields_list_conf_format,
         }
-        
-        #return the model
+
+        # return the model
         model.update(super_fields)
         return model
 
+
 class MlModel(FileBackedLookup):
     lookup_type: Literal[Lookup_Type.mlmodel]
-    
 
-LookupAdapter = TypeAdapter(Annotated[CSVLookup | KVStoreLookup | MlModel, Field(discriminator="lookup_type")])
+    @computed_field
+    @cached_property
+    def filename(self) -> FilePath:
+        """
+        This function computes the backing file for the lookup. The names of mlmodel files must EXACTLY match the
+        names of their lookup definitions except with:
+        - __mlspl_ prefix
+        - .mlmodel file extension rather than the YML file extension.
+        """
+        if self.file_path is None:
+            raise ValueError(
+                f"Cannot get the filename of the lookup {self.lookup_type} because the YML file_path attribute is None"
+            )  # type: ignore
+
+        if not self.file_path.stem.startswith("__mlspl_"):
+            raise ValueError(
+                f"The file_path for ML Model {self.name} MUST start with '__mlspl_', but it does not."
+            )
+
+        return self.file_path.parent / f"{self.file_path.stem}.{self.lookup_type}"
+
+    @computed_field
+    @cached_property
+    def app_filename(self) -> FilePath:
+        """
+        This function computes the filenames to write into the app itself.  This is abstract because
+        CSV and MLmodel requirements are different.
+        """
+        return pathlib.Path(f"{self.filename.stem}.{self.lookup_type}")
+
 
+LookupAdapter = TypeAdapter(
+    Annotated[CSVLookup | KVStoreLookup | MlModel, Field(discriminator="lookup_type")]
+)

contentctl/objects/macro.py

@@ -1,4 +1,4 @@
-# Used so that we can have a staticmethod that takes the class 
+# Used so that we can have a staticmethod that takes the class
 # type Macro as an argument
 from __future__ import annotations
 from typing import TYPE_CHECKING, List
@@ -6,18 +6,21 @@ import re
 from pydantic import Field, model_serializer, NonNegativeInt
 import uuid
 import datetime
+
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
 from contentctl.objects.security_content_object import SecurityContentObject
 
-#The following macros are included in commonly-installed apps.
-#As such, we will ignore if they are missing from our app.
-#Included in 
-MACROS_TO_IGNORE = set(["drop_dm_object_name"]) # Part of CIM/Splunk_SA_CIM
-MACROS_TO_IGNORE.add("get_asset") #SA-IdentityManagement, part of Enterprise Security
-MACROS_TO_IGNORE.add("get_risk_severity") #SA-ThreatIntelligence, part of Enterprise Security
-MACROS_TO_IGNORE.add("cim_corporate_web_domain_search") #Part of CIM/Splunk_SA_CIM
-#MACROS_TO_IGNORE.add("prohibited_processes")
+# The following macros are included in commonly-installed apps.
+# As such, we will ignore if they are missing from our app.
+# Included in
+MACROS_TO_IGNORE = set(["drop_dm_object_name"])  # Part of CIM/Splunk_SA_CIM
+MACROS_TO_IGNORE.add("get_asset")  # SA-IdentityManagement, part of Enterprise Security
+MACROS_TO_IGNORE.add(
+    "get_risk_severity"
+)  # SA-ThreatIntelligence, part of Enterprise Security
+MACROS_TO_IGNORE.add("cim_corporate_web_domain_search")  # Part of CIM/Splunk_SA_CIM
+# MACROS_TO_IGNORE.add("prohibited_processes")
 
 
 class Macro(SecurityContentObject):
@@ -26,48 +29,62 @@ class Macro(SecurityContentObject):
     # TODO: Add id field to all macro ymls
     id: uuid.UUID = Field(default_factory=uuid.uuid4)
     date: datetime.date = Field(datetime.date.today())
-    author: str = Field("NO AUTHOR DEFINED",max_length=255)
+    author: str = Field("NO AUTHOR DEFINED", max_length=255)
     version: NonNegativeInt = 1
-    
-
 
     @model_serializer
     def serialize_model(self):
-        #Call serializer for parent
+        # Call serializer for parent
         super_fields = super().serialize_model()
 
-        #All fields custom to this model
-        model= {
+        # All fields custom to this model
+        model = {
             "definition": self.definition,
             "description": self.description,
         }
-        
-        #return the model
+
+        # return the model
         model.update(super_fields)
-        
+
         return model
-    
+
     @staticmethod
-    def get_macros(text_field:str, director:DirectorOutputDto , ignore_macros:set[str]=MACROS_TO_IGNORE)->list[Macro]:
-        #Remove any comments, allowing there to be macros (which have a single backtick) inside those comments
-        #If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here````
-        #then there is a small edge case where the regex below does not work properly.  If that is 
-        #the case, we edit the search slightly to insert a space
+    def get_macros(
+        text_field: str,
+        director: DirectorOutputDto,
+        ignore_macros: set[str] = MACROS_TO_IGNORE,
+    ) -> list[Macro]:
+        # Remove any comments, allowing there to be macros (which have a single backtick) inside those comments
+        # If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here````
+        # then there is a small edge case where the regex below does not work properly.  If that is
+        # the case, we edit the search slightly to insert a space
         if re.findall(r"\`\`\`\`", text_field):
-            raise ValueError("Search contained four or more '`' characters in a row which is invalid SPL"
-                            "This may have occurred when a macro was commented out.\n"
-                            "Please ammend your search to remove the substring '````'")
+            raise ValueError(
+                "Search contained four or more '`' characters in a row which is invalid SPL"
+                "This may have occurred when a macro was commented out.\n"
+                "Please ammend your search to remove the substring '````'"
+            )
+
+        # Replace all the comments with a space. This prevents a comment from looking like a macro to the parser below
+        text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field)
+
+        # Find all the macros, which start and end with a '`' character
+        macros_to_get = re.findall(r"`([^\s]+)`", text_field)
+        # If macros take arguments, stop at the first argument.  We just want the name of the macro
+        macros_to_get = set(
+            [
+                macro[: macro.find("(")] if macro.find("(") != -1 else macro
+                for macro in macros_to_get
+            ]
+        )
 
-        # Replace all the comments with a space. This prevents a comment from looking like a macro to the parser below    
-        text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field) 
-        
-        # Find all the macros, which start and end with a '`' character  
-        macros_to_get = re.findall(r'`([^\s]+)`', text_field)
-        #If macros take arguments, stop at the first argument.  We just want the name of the macro
-        macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get])
-        
-        macros_to_ignore = set([macro for macro in macros_to_get if any(to_ignore in macro for to_ignore in ignore_macros)])
-        #remove the ones that we will ignore
+        macros_to_ignore = set(
+            [
+                macro
+                for macro in macros_to_get
+                if any(to_ignore in macro for to_ignore in ignore_macros)
+            ]
+        )
+        # remove the ones that we will ignore
         macros_to_get -= macros_to_ignore
         return Macro.mapNamesToSecurityContentObjects(list(macros_to_get), director)
-