contentctl 4.4.7__py3-none-any.whl → 5.0.0a2__py3-none-any.whl

contentctl/actions/build.py

@@ -4,17 +4,17 @@ import os
 
 from dataclasses import dataclass
 
-from contentctl.objects.enums import SecurityContentProduct, SecurityContentType
+from contentctl.objects.enums import SecurityContentType
 from contentctl.input.director import Director, DirectorOutputDto
 from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
 from contentctl.output.api_json_output import ApiJsonOutput
 from contentctl.output.data_source_writer import DataSourceWriter
-from contentctl.objects.lookup import Lookup
+from contentctl.objects.lookup import CSVLookup, Lookup_Type
 import pathlib
 import json
 import datetime
-from typing import Union
+import uuid
 
 from contentctl.objects.config import build
 
@@ -34,27 +34,41 @@ class Build:
             updated_conf_files:set[pathlib.Path] = set()
             conf_output = ConfOutput(input_dto.config)
 
+
+            # Construct a path to a YML that does not actually exist.
+            # We mock this "fake" path since the YML does not exist.
+            # This ensures the checking for the existence of the CSV is correct
+            data_sources_fake_yml_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.yml"
+
             # Construct a special lookup whose CSV is created at runtime and
-            # written directly into the output folder. It is created with model_construct,
-            # not model_validate, because the CSV does not exist yet.
+            # written directly into the lookups folder. We will delete this after a build, 
+            # assuming that it is successful.
             data_sources_lookup_csv_path = input_dto.config.getPackageDirectoryPath() / "lookups" / "data_sources.csv"
-            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path)
-            input_dto.director_output_dto.addContentToDictMappings(Lookup.model_construct(description= "A lookup file that will contain the data source objects for detections.", 
-                                                                        filename=data_sources_lookup_csv_path, 
-                                                                        name="data_sources"))
 
+
+            
+            DataSourceWriter.writeDataSourceCsv(input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path)
+            input_dto.director_output_dto.addContentToDictMappings(CSVLookup.model_construct(name="data_sources",
+                                                                            id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
+                                                                            version=1,
+                                                                            author=input_dto.config.app.author_name,
+                                                                            date = datetime.date.today(), 
+                                                                            description= "A lookup file that will contain the data source objects for detections.",
+                                                                            lookup_type=Lookup_Type.csv,
+                                                                            file_path=data_sources_fake_yml_path))
             updated_conf_files.update(conf_output.writeHeaders())
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.detections, SecurityContentType.detections))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.stories, SecurityContentType.stories))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.baselines, SecurityContentType.baselines))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.investigations, SecurityContentType.investigations))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
-            updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards))
+            updated_conf_files.update(conf_output.writeLookups(input_dto.director_output_dto.lookups))
+            updated_conf_files.update(conf_output.writeDetections(input_dto.director_output_dto.detections))
+            updated_conf_files.update(conf_output.writeStories(input_dto.director_output_dto.stories))
+            updated_conf_files.update(conf_output.writeBaselines(input_dto.director_output_dto.baselines))
+            updated_conf_files.update(conf_output.writeInvestigations(input_dto.director_output_dto.investigations))
+            updated_conf_files.update(conf_output.writeMacros(input_dto.director_output_dto.macros))
+            updated_conf_files.update(conf_output.writeDashboards(input_dto.director_output_dto.dashboards))
             updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
             
 
             
+            
             #Ensure that the conf file we just generated/update is syntactically valid
             for conf_file in updated_conf_files:
                 ConfWriter.validateConfFile(conf_file) 
@@ -67,17 +81,15 @@ class Build:
         if input_dto.config.build_api:    
             shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True)
             input_dto.config.getAPIPath().mkdir(parents=True)
-            api_json_output = ApiJsonOutput()
-            for output_objects, output_type in [(input_dto.director_output_dto.detections, SecurityContentType.detections),
-                                                (input_dto.director_output_dto.stories, SecurityContentType.stories),
-                                                (input_dto.director_output_dto.baselines, SecurityContentType.baselines),
-                                                (input_dto.director_output_dto.investigations, SecurityContentType.investigations),
-                                                (input_dto.director_output_dto.lookups, SecurityContentType.lookups),
-                                                (input_dto.director_output_dto.macros, SecurityContentType.macros),
-                                                (input_dto.director_output_dto.deployments, SecurityContentType.deployments)]:
-                api_json_output.writeObjects(output_objects, input_dto.config.getAPIPath(), input_dto.config.app.label, output_type )
-            
-           
+            api_json_output = ApiJsonOutput(input_dto.config.getAPIPath(), input_dto.config.app.label)
+            api_json_output.writeDetections(input_dto.director_output_dto.detections)
+            api_json_output.writeStories(input_dto.director_output_dto.stories)
+            api_json_output.writeBaselines(input_dto.director_output_dto.baselines)
+            api_json_output.writeInvestigations(input_dto.director_output_dto.investigations)
+            api_json_output.writeLookups(input_dto.director_output_dto.lookups)
+            api_json_output.writeMacros(input_dto.director_output_dto.macros)
+            api_json_output.writeDeployments(input_dto.director_output_dto.deployments)
+
             
             #create version file for sse api
             version_file = input_dto.config.getAPIPath()/"version.json"

contentctl/actions/detection_testing/DetectionTestingManager.py

@@ -5,7 +5,6 @@ from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfras
 from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import DetectionTestingInfrastructureServer
 from urllib.parse import urlparse
 from copy import deepcopy
-from contentctl.objects.enums import DetectionTestingTargetInfrastructure
 import signal
 import datetime
 # from queue import Queue

contentctl/actions/detection_testing/GitService.py

@@ -1,44 +1,41 @@
 import logging
 import os
 import pathlib
+from typing import TYPE_CHECKING, List, Optional
+
 import pygit2
-from pygit2.enums import DeltaStatus
-from typing import List, Optional
 from pydantic import BaseModel, FilePath
-from typing import TYPE_CHECKING
+from pygit2.enums import DeltaStatus
+
 if TYPE_CHECKING:
     from contentctl.input.director import DirectorOutputDto
-    
 
-from contentctl.objects.macro import Macro
-from contentctl.objects.lookup import Lookup
-from contentctl.objects.detection import Detection
+from contentctl.objects.config import All, Changes, Selected, test_common
 from contentctl.objects.data_source import DataSource
+from contentctl.objects.detection import Detection
+from contentctl.objects.lookup import CSVLookup, Lookup
+from contentctl.objects.macro import Macro
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.config import test_common, All, Changes, Selected
 
 # Logger
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 LOGGER = logging.getLogger(__name__)
 
 
-
 from contentctl.input.director import DirectorOutputDto
 
 
-
 class GitService(BaseModel):
     director: DirectorOutputDto
     config: test_common
     gitHash: Optional[str] = None
-    
-    def getHash(self)->str:
+
+    def getHash(self) -> str:
         if self.gitHash is None:
             raise Exception("Cannot get hash of repo, it was not set")
         return self.gitHash
 
-
-    def getContent(self)->List[Detection]:
+    def getContent(self) -> List[Detection]:
         if isinstance(self.config.mode, Selected):
             return self.getSelected(self.config.mode.files)
         elif isinstance(self.config.mode, Changes):
@@ -46,142 +43,205 @@ class GitService(BaseModel):
         if isinstance(self.config.mode, All):
             return self.getAll()
         else:
-            raise Exception(f"Could not get content to test. Unsupported test mode '{self.config.mode}'")
-    def getAll(self)->List[Detection]:
+            raise Exception(
+                f"Could not get content to test. Unsupported test mode '{self.config.mode}'"
+            )
+
+    def getAll(self) -> List[Detection]:
         return self.director.detections
-    
-    def getChanges(self, target_branch:str)->List[Detection]:
+
+    def getChanges(self, target_branch: str) -> List[Detection]:
         repo = pygit2.Repository(path=str(self.config.path))
 
         try:
             target_tree = repo.revparse_single(target_branch).tree
             self.gitHash = target_tree.id
             diffs = repo.index.diff_to_tree(target_tree)
-        except Exception as e:
-            raise Exception(f"Error parsing diff target_branch '{target_branch}'. Are you certain that it exists?")
-        
-        #Get the uncommitted changes in the current directory
+        except Exception:
+            raise Exception(
+                f"Error parsing diff target_branch '{target_branch}'. Are you certain that it exists?"
+            )
+
+        # Get the uncommitted changes in the current directory
         diffs2 = repo.index.diff_to_workdir()
-        
-        #Combine the uncommitted changes with the committed changes
+
+        # Combine the uncommitted changes with the committed changes
         all_diffs = list(diffs) + list(diffs2)
 
-        #Make a filename to content map
-        filepath_to_content_map = { obj.file_path:obj for (_,obj) in self.director.name_to_content_map.items()} 
+        # Make a filename to content map
+        filepath_to_content_map = {
+            obj.file_path: obj for (_, obj) in self.director.name_to_content_map.items()
+        }
 
         updated_detections: set[Detection] = set()
         updated_macros: set[Macro] = set()
         updated_lookups: set[Lookup] = set()
         updated_datasources: set[DataSource] = set()
 
-
         for diff in all_diffs:
             if type(diff) == pygit2.Patch:
-                if diff.delta.status in (DeltaStatus.ADDED, DeltaStatus.MODIFIED, DeltaStatus.RENAMED):
-                    #print(f"{DeltaStatus(diff.delta.status).name:<8}:{diff.delta.new_file.raw_path}")
-                    decoded_path = pathlib.Path(diff.delta.new_file.raw_path.decode('utf-8'))
+                if diff.delta.status in (
+                    DeltaStatus.ADDED,
+                    DeltaStatus.MODIFIED,
+                    DeltaStatus.RENAMED,
+                ):
+                    # print(f"{DeltaStatus(diff.delta.status).name:<8}:{diff.delta.new_file.raw_path}")
+                    decoded_path = pathlib.Path(
+                        diff.delta.new_file.raw_path.decode("utf-8")
+                    )
                     # Note that we only handle updates to detections, lookups, and macros at this time. All other changes are ignored.
-                    if decoded_path.is_relative_to(self.config.path/"detections") and decoded_path.suffix == ".yml":
-                        detectionObject = filepath_to_content_map.get(decoded_path, None)
+                    if (
+                        decoded_path.is_relative_to(self.config.path / "detections")
+                        and decoded_path.suffix == ".yml"
+                    ):
+                        detectionObject = filepath_to_content_map.get(
+                            decoded_path, None
+                        )
                         if isinstance(detectionObject, Detection):
                             updated_detections.add(detectionObject)
                         else:
-                            raise Exception(f"Error getting detection object for file {str(decoded_path)}")
-                        
-                    elif decoded_path.is_relative_to(self.config.path/"macros") and decoded_path.suffix == ".yml":
+                            raise Exception(
+                                f"Error getting detection object for file {str(decoded_path)}"
+                            )
+
+                    elif (
+                        decoded_path.is_relative_to(self.config.path / "macros")
+                        and decoded_path.suffix == ".yml"
+                    ):
                         macroObject = filepath_to_content_map.get(decoded_path, None)
                         if isinstance(macroObject, Macro):
                             updated_macros.add(macroObject)
                         else:
-                            raise Exception(f"Error getting macro object for file {str(decoded_path)}")
-                        
-                    elif decoded_path.is_relative_to(self.config.path/"data_sources") and decoded_path.suffix == ".yml":
-                        datasourceObject = filepath_to_content_map.get(decoded_path, None)
+                            raise Exception(
+                                f"Error getting macro object for file {str(decoded_path)}"
+                            )
+
+                    elif (
+                        decoded_path.is_relative_to(self.config.path / "data_sources")
+                        and decoded_path.suffix == ".yml"
+                    ):
+                        datasourceObject = filepath_to_content_map.get(
+                            decoded_path, None
+                        )
                         if isinstance(datasourceObject, DataSource):
                             updated_datasources.add(datasourceObject)
                         else:
-                            raise Exception(f"Error getting data source object for file {str(decoded_path)}")
+                            raise Exception(
+                                f"Error getting data source object for file {str(decoded_path)}"
+                            )
 
-                    elif decoded_path.is_relative_to(self.config.path/"lookups"):
+                    elif decoded_path.is_relative_to(self.config.path / "lookups"):
                         # We need to convert this to a yml. This means we will catch
                         # both changes to a csv AND changes to the YML that uses it
                         if decoded_path.suffix == ".yml":
-                            updatedLookup = filepath_to_content_map.get(decoded_path, None)
-                            if not isinstance(updatedLookup,Lookup):
-                                raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(updatedLookup))}")
+                            updatedLookup = filepath_to_content_map.get(
+                                decoded_path, None
+                            )
+                            if not isinstance(updatedLookup, Lookup):
+                                raise Exception(
+                                    f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(updatedLookup))}"
+                                )
                             updated_lookups.add(updatedLookup)
 
                         elif decoded_path.suffix == ".csv":
-                            # If the CSV was updated, we want to make sure that we 
+                            # If the CSV was updated, we want to make sure that we
                             # add the correct corresponding Lookup object.
-                            #Filter to find the Lookup Object the references this CSV
-                            matched = list(filter(lambda x: x.filename is not None and x.filename == decoded_path, self.director.lookups))
+                            # Filter to find the Lookup Object the references this CSV
+                            matched = list(
+                                filter(
+                                    lambda x: isinstance(x, CSVLookup)
+                                    and x.filename == decoded_path,
+                                    self.director.lookups,
+                                )
+                            )
                             if len(matched) == 0:
-                                raise Exception(f"Failed to find any lookups that reference the modified CSV file  '{decoded_path}'")
+                                raise Exception(
+                                    f"Failed to find any lookups that reference the modified CSV file  '{decoded_path}'"
+                                )
                             elif len(matched) > 1:
-                                raise Exception(f"More than 1 Lookup reference the modified CSV file '{decoded_path}': {[l.file_path for l in matched ]}")
+                                raise Exception(
+                                    f"More than 1 Lookup reference the modified CSV file '{decoded_path}': {[match.file_path for match in matched]}"
+                                )
                             else:
                                 updatedLookup = matched[0]
                         elif decoded_path.suffix == ".mlmodel":
-                            # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
+                            # Detected a changed .mlmodel file. However, since we do not have testing for these detections at
                             # this time, we will ignore this change.
                             updatedLookup = None
 
                         else:
-                            raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
-                                            "Only files ending in .csv, .yml, or .mlmodel are supported in this "
-                                            "directory. This file must be removed from the lookups/ directory.")
-                        
-                        if updatedLookup is not None and updatedLookup not in updated_lookups:
+                            raise Exception(
+                                f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
+                                "Only files ending in .csv, .yml, or .mlmodel are supported in this "
+                                "directory. This file must be removed from the lookups/ directory."
+                            )
+
+                        if (
+                            updatedLookup is not None
+                            and updatedLookup not in updated_lookups
+                        ):
                             # It is possible that both the CSV and YML have been modified for the same lookup,
-                            # and we do not want to add it twice. 
+                            # and we do not want to add it twice.
                             updated_lookups.add(updatedLookup)
 
                     else:
                         pass
-                        #print(f"Ignore changes to file {decoded_path} since it is not a detection, macro, or lookup.")
+                        # print(f"Ignore changes to file {decoded_path} since it is not a detection, macro, or lookup.")
             else:
                 raise Exception(f"Unrecognized diff type {type(diff)}")
 
-
         # If a detection has at least one dependency on changed content,
         # then we must test it again
 
-        changed_macros_and_lookups_and_datasources:set[SecurityContentObject] = updated_macros.union(updated_lookups, updated_datasources)
-        
+        changed_macros_and_lookups_and_datasources: set[Macro | Lookup | DataSource] = (
+            updated_macros.union(updated_lookups, updated_datasources)
+        )
+
         for detection in self.director.detections:
             if detection in updated_detections:
-                # we are already planning to test it, don't need 
+                # we are already planning to test it, don't need
                 # to add it again
                 continue
 
             for obj in changed_macros_and_lookups_and_datasources:
                 if obj in detection.get_content_dependencies():
-                   updated_detections.add(detection)
-                   break
+                    updated_detections.add(detection)
+                    break
 
-        #Print out the names of all modified/new content
-        modifiedAndNewContentString = "\n - ".join(sorted([d.name for d in updated_detections]))
+        # Print out the names of all modified/new content
+        modifiedAndNewContentString = "\n - ".join(
+            sorted([d.name for d in updated_detections])
+        )
 
-        print(f"[{len(updated_detections)}] Pieces of modifed and new content (this may include experimental/deprecated/manual_test content):\n - {modifiedAndNewContentString}")
+        print(
+            f"[{len(updated_detections)}] Pieces of modifed and new content (this may include experimental/deprecated/manual_test content):\n - {modifiedAndNewContentString}"
+        )
         return sorted(list(updated_detections))
 
     def getSelected(self, detectionFilenames: List[FilePath]) -> List[Detection]:
         filepath_to_content_map: dict[FilePath, SecurityContentObject] = {
-        obj.file_path: obj for (_, obj) in self.director.name_to_content_map.items() if obj.file_path is not None
-    }
+            obj.file_path: obj
+            for (_, obj) in self.director.name_to_content_map.items()
+            if obj.file_path is not None
+        }
         errors = []
         detections: List[Detection] = []
         for name in detectionFilenames:
             obj = filepath_to_content_map.get(name, None)
             if obj is None:
-                errors.append(f"There is no detection file or security_content_object at '{name}'")
+                errors.append(
+                    f"There is no detection file or security_content_object at '{name}'"
+                )
             elif not isinstance(obj, Detection):
-                errors.append(f"The security_content_object at '{name}' is of type '{type(obj).__name__}', NOT '{Detection.__name__}'")
+                errors.append(
+                    f"The security_content_object at '{name}' is of type '{type(obj).__name__}', NOT '{Detection.__name__}'"
+                )
             else:
                 detections.append(obj)
 
         if errors:
             errorsString = "\n - ".join(errors)
-            raise Exception(f"The following errors were encountered while getting selected detections to test:\n - {errorsString}")
-        return detections
+            raise Exception(
+                f"The following errors were encountered while getting selected detections to test:\n - {errorsString}"
+            )
+        return detections