contentctl 4.4.7__py3-none-any.whl → 5.0.0a0__py3-none-any.whl

contentctl/output/api_json_output.py

@@ -1,246 +1,259 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from contentctl.objects.detection import Detection
+    from contentctl.objects.lookup import Lookup
+    from contentctl.objects.macro import Macro
+    from contentctl.objects.story import Story
+    from contentctl.objects.baseline import Baseline
+    from contentctl.objects.investigation import Investigation
+    from contentctl.objects.deployment import Deployment
+
 import os
-import json
 import pathlib
 
 from contentctl.output.json_writer import JsonWriter
-from contentctl.objects.enums import SecurityContentType
-from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
-    SecurityContentObject_Abstract,
-)
 
 
 
 class ApiJsonOutput:
+    output_path: pathlib.Path
+    app_label: str
+
+    def __init__(self, output_path:pathlib.Path, app_label: str):
+        self.output_path = output_path
+        self.app_label = app_label
 
-    def writeObjects(
+    def writeDetections(
         self,
-        objects: list[SecurityContentObject_Abstract],
-        output_path: pathlib.Path,
-        app_label:str = "ESCU",
-        contentType: SecurityContentType = None
+        objects: list[Detection],
     ) -> None:
-        """#Serialize all objects
-        try:
-            for obj in objects:
-
-                serialized_objects.append(obj.model_dump())
-        except Exception as e:
-            raise Exception(f"Error serializing object with name '{obj.name}' and type '{type(obj).__name__}': '{str(e)}'")
-        """
-
-        if contentType == SecurityContentType.detections:
-            detections = [
-                detection.model_dump(
-                    include=set(
-                        [
-                            "name",
-                            "author",
-                            "date",
-                            "version",
-                            "id",
-                            "description",
-                            "tags",
-                            "search",
-                            "how_to_implement",
-                            "known_false_positives",
-                            "references",
-                            "datamodel",
-                            "macros",
-                            "lookups",
-                            "source",
-                            "nes_fields",
-                        ]
-                    )
+        detections = [
+            detection.model_dump(
+                include=set(
+                    [
+                        "name",
+                        "author",
+                        "date",
+                        "version",
+                        "id",
+                        "description",
+                        "tags",
+                        "search",
+                        "how_to_implement",
+                        "known_false_positives",
+                        "references",
+                        "datamodel",
+                        "macros",
+                        "lookups",
+                        "source",
+                        "nes_fields",
+                    ]
                 )
-                for detection in objects
-            ]
-            #Only a subset of macro fields are required:
-            # for detection in detections:
-            #     new_macros = []
-            #     for macro in detection.get("macros",[]):
-            #         new_macro_fields = {}
-            #         new_macro_fields["name"] = macro.get("name")
-            #         new_macro_fields["definition"] = macro.get("definition")
-            #         new_macro_fields["description"] = macro.get("description")
-            #         if len(macro.get("arguments", [])) > 0:
-            #             new_macro_fields["arguments"] = macro.get("arguments") 
-            #         new_macros.append(new_macro_fields)
-            #     detection["macros"] = new_macros
-            #     del()
-                    
-            
-            JsonWriter.writeJsonObject(
-                os.path.join(output_path, "detections.json"), "detections", detections
-            )
-
-        elif contentType == SecurityContentType.macros:
-            macros = [
-                macro.model_dump(include=set(["definition", "description", "name"]))
-                for macro in objects
-            ]
-            for macro in macros:
-                for k in ["author", "date","version","id","references"]:
-                    if k in macro:
-                        del(macro[k])
-            JsonWriter.writeJsonObject(
-                os.path.join(output_path, "macros.json"), "macros", macros
             )
-
-        elif contentType == SecurityContentType.stories:
-            stories = [
-                story.model_dump(
-                    include=set(
-                        [
-                            "name",
-                            "author",
-                            "date",
-                            "version",
-                            "id",
-                            "description",
-                            "narrative",
-                            "references",
-                            "tags",
-                            "detections_names",
-                            "investigation_names",
-                            "baseline_names",
-                            "detections",
-                        ]
-                    )
-                )
-                for story in objects
-            ]
-            # Only get certain fields from detections
-            for story in stories:
-                # Only use a small subset of fields from the detection
-                story["detections"] = [
-                    {
-                        "name": detection["name"],
-                        "source": detection["source"],
-                        "type": detection["type"],
-                        "tags": detection["tags"].get("mitre_attack_enrichments", []),
-                    }
-                    for detection in story["detections"]
-                ]
-                story["detection_names"] = [f"{app_label} - {name} - Rule" for name in story["detection_names"]]
+            for detection in objects
+        ]
+        #Only a subset of macro fields are required:
+        # for detection in detections:
+        #     new_macros = []
+        #     for macro in detection.get("macros",[]):
+        #         new_macro_fields = {}
+        #         new_macro_fields["name"] = macro.get("name")
+        #         new_macro_fields["definition"] = macro.get("definition")
+        #         new_macro_fields["description"] = macro.get("description")
+        #         if len(macro.get("arguments", [])) > 0:
+        #             new_macro_fields["arguments"] = macro.get("arguments") 
+        #         new_macros.append(new_macro_fields)
+        #     detection["macros"] = new_macros
+        #     del()
                 
-
-            JsonWriter.writeJsonObject(
-                os.path.join(output_path, "stories.json"), "stories", stories
+        
+        JsonWriter.writeJsonObject(
+            os.path.join(self.output_path, "detections.json"), "detections", detections
+        )
+    
+    def writeMacros(
+        self,
+        objects: list[Macro],
+    ) -> None:
+        macros = [
+            macro.model_dump(include=set(["definition", "description", "name"]))
+            for macro in objects
+        ]
+        for macro in macros:
+            for k in ["author", "date","version","id","references"]:
+                if k in macro:
+                    del(macro[k])
+        JsonWriter.writeJsonObject(
+            os.path.join(self.output_path, "macros.json"), "macros", macros
+        )
+    
+    def writeStories(
+        self,
+        objects: list[Story],
+    ) -> None:
+        stories = [
+            story.model_dump(
+                include=set(
+                    [
+                        "name",
+                        "author",
+                        "date",
+                        "version",
+                        "id",
+                        "description",
+                        "narrative",
+                        "references",
+                        "tags",
+                        "detections_names",
+                        "investigation_names",
+                        "baseline_names",
+                        "detections",
+                    ]
+                )
             )
+            for story in objects
+        ]
+        # Only get certain fields from detections
+        for story in stories:
+            # Only use a small subset of fields from the detection
+            story["detections"] = [
+                {
+                    "name": detection["name"],
+                    "source": detection["source"],
+                    "type": detection["type"],
+                    "tags": detection["tags"].get("mitre_attack_enrichments", []),
+                }
+                for detection in story["detections"]
+            ]
+            story["detection_names"] = [f"{self.app_label} - {name} - Rule" for name in story["detection_names"]]
+            
 
-        elif contentType == SecurityContentType.baselines:
-            try:
-                baselines = [
-                    baseline.model_dump(
-                        include=set(
-                            [
-                                "name",
-                                "author",
-                                "date",
-                                "version",
-                                "id",
-                                "description",
-                                "type",
-                                "datamodel",
-                                "search",
-                                "how_to_implement",
-                                "known_false_positives",
-                                "references",
-                                "tags",
-                            ]
-                        )
-                    )
-                    for baseline in objects
-                ]
-            except Exception as e:
-                print(e)
-                print('wait')
+        JsonWriter.writeJsonObject(
+            os.path.join(self.output_path, "stories.json"), "stories", stories
+        )
 
-            JsonWriter.writeJsonObject(
-                    os.path.join(output_path, "baselines.json"), "baselines", baselines
+    def writeBaselines(
+        self,
+        objects: list[Baseline],
+    ) -> None:
+        baselines = [
+            baseline.model_dump(
+                include=set(
+                    [
+                        "name",
+                        "author",
+                        "date",
+                        "version",
+                        "id",
+                        "description",
+                        "type",
+                        "datamodel",
+                        "search",
+                        "how_to_implement",
+                        "known_false_positives",
+                        "references",
+                        "tags",
+                    ]
                 )
+            )
+            for baseline in objects
+        ]
+        
+        JsonWriter.writeJsonObject(
+                os.path.join(self.output_path, "baselines.json"), "baselines", baselines
+            )
 
-        elif contentType == SecurityContentType.investigations:
-            investigations = [
-                investigation.model_dump(
-                    include=set(
-                        [
-                            "name",
-                            "author",
-                            "date",
-                            "version",
-                            "id",
-                            "description",
-                            "type",
-                            "datamodel",
-                            "search",
-                            "how_to_implemnet",
-                            "known_false_positives",
-                            "references",
-                            "inputs",
-                            "tags",
-                            "lowercase_name",
-                        ]
-                    )
+    def writeInvestigations(
+        self,
+        objects: list[Investigation],
+    ) -> None:
+        investigations = [
+            investigation.model_dump(
+                include=set(
+                    [
+                        "name",
+                        "author",
+                        "date",
+                        "version",
+                        "id",
+                        "description",
+                        "type",
+                        "datamodel",
+                        "search",
+                        "how_to_implemnet",
+                        "known_false_positives",
+                        "references",
+                        "inputs",
+                        "tags",
+                        "lowercase_name",
+                    ]
                 )
-                for investigation in objects
-            ]
-            JsonWriter.writeJsonObject(
-                os.path.join(output_path, "response_tasks.json"),
-                "response_tasks",
-                investigations,
             )
+            for investigation in objects
+        ]
+        JsonWriter.writeJsonObject(
+            os.path.join(self.output_path, "response_tasks.json"),
+            "response_tasks",
+            investigations,
+        )
 
-        elif contentType == SecurityContentType.lookups:
-            lookups = [
-                lookup.model_dump(
-                    include=set(
-                        [
-                            "name",
-                            "description",
-                            "collection",
-                            "fields_list",
-                            "filename",
-                            "default_match",
-                            "match_type",
-                            "min_matches",
-                            "case_sensitive_match",
-                        ]
-                    )
+    def writeLookups(
+        self,
+        objects: list[Lookup],
+    ) -> None:
+        lookups = [
+            lookup.model_dump(
+                include=set(
+                    [
+                        "name",
+                        "description",
+                        "collection",
+                        "fields_list",
+                        "filename",
+                        "default_match",
+                        "match_type",
+                        "min_matches",
+                        "case_sensitive_match",
+                    ]
                 )
-                for lookup in objects
-            ]
-            for lookup in lookups:
-                for k in ["author","date","version","id","references"]:
-                    if k in lookup:
-                        del(lookup[k]) 
-            JsonWriter.writeJsonObject(
-                os.path.join(output_path, "lookups.json"), "lookups", lookups
             )
+            for lookup in objects
+        ]
+        for lookup in lookups:
+            for k in ["author","date","version","id","references"]:
+                if k in lookup:
+                    del(lookup[k]) 
+        JsonWriter.writeJsonObject(
+            os.path.join(self.output_path, "lookups.json"), "lookups", lookups
+        )
 
-        elif contentType == SecurityContentType.deployments:
-            deployments = [
-                deployment.model_dump(
-                    include=set(
-                        [
-                            "name",
-                            "author",
-                            "date",
-                            "version",
-                            "id",
-                            "description",
-                            "scheduling",
-                            "rba",
-                            "tags"
-                        ] 
-                    )
+    def writeDeployments(
+        self,
+        objects: list[Deployment],
+    ) -> None:
+        deployments = [
+            deployment.model_dump(
+                include=set(
+                    [
+                        "name",
+                        "author",
+                        "date",
+                        "version",
+                        "id",
+                        "description",
+                        "scheduling",
+                        "rba",
+                        "tags"
+                    ] 
                 )
-                for deployment in objects
-            ]
-            #references are not to be included, but have been deleted in the
-            #model_serialization logic
-            JsonWriter.writeJsonObject(
-                os.path.join(output_path, "deployments.json"),
-                "deployments",
-                deployments,
-            )
+            )
+            for deployment in objects
+        ]
+        #references are not to be included, but have been deleted in the
+        #model_serialization logic
+        JsonWriter.writeJsonObject(
+            os.path.join(self.output_path, "deployments.json"),
+            "deployments",
+            deployments,
+        )

contentctl/output/conf_output.py

@@ -1,22 +1,22 @@
-from dataclasses import dataclass
-import os
-import glob
+from __future__ import annotations
+from typing import TYPE_CHECKING, Callable
+if TYPE_CHECKING:
+    from contentctl.objects.detection import Detection
+    from contentctl.objects.lookup import Lookup
+    from contentctl.objects.macro import Macro
+    from contentctl.objects.dashboard import Dashboard
+    from contentctl.objects.story import Story
+    from contentctl.objects.baseline import Baseline
+    from contentctl.objects.investigation import Investigation
+
+from contentctl.objects.lookup import FileBackedLookup
 import shutil
-import sys
 import tarfile
-from typing import Union
-from pathlib import Path
 import pathlib
-import time
 import timeit
 import datetime
-import shutil
-import json
 from contentctl.output.conf_writer import ConfWriter
-from contentctl.objects.enums import SecurityContentType
 from contentctl.objects.config import build
-from requests import Session, post, get
-from requests.auth import HTTPBasicAuth
 
 class ConfOutput:
     config: build    
@@ -80,25 +80,33 @@ class ConfOutput:
         return written_files
 
         
-    def writeObjects(self, objects: list, type: SecurityContentType = None) -> set[pathlib.Path]:
+    def writeDetections(self, objects:list[Detection]) -> set[pathlib.Path]:
         written_files:set[pathlib.Path] = set()
-        if type == SecurityContentType.detections:
-            for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_detections.j2'),
-                                                    ('default/analyticstories.conf', 'analyticstories_detections.j2')]:
-                written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path),
-                                                           template_name, self.config, objects))
-        
-        elif type == SecurityContentType.stories:
+        for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_detections.j2'),
+                                                ('default/analyticstories.conf', 'analyticstories_detections.j2')]:
+            written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path),
+                                                        template_name, self.config, objects))
+        return written_files
+
+
+    def writeStories(self, objects:list[Story]) -> set[pathlib.Path]:
+            written_files:set[pathlib.Path] = set()
             written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/analyticstories.conf'), 
                                     'analyticstories_stories.j2',
                                     self.config, objects))
+            return written_files
+    
 
-        elif type == SecurityContentType.baselines:
+    def writeBaselines(self, objects:list[Baseline]) -> set[pathlib.Path]:
+            written_files:set[pathlib.Path] = set()
             written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/savedsearches.conf'),
                                                       'savedsearches_baselines.j2', 
                                                        self.config, objects))
+            return written_files
+    
 
-        elif type == SecurityContentType.investigations:
+    def writeInvestigations(self, objects:list[Investigation]) -> set[pathlib.Path]:
+            written_files:set[pathlib.Path] = set()
             for output_app_path, template_name in [ ('default/savedsearches.conf', 'savedsearches_investigations.j2'),
                                                     ('default/analyticstories.conf', 'analyticstories_investigations.j2')]:
                 ConfWriter.writeConfFile(pathlib.Path(output_app_path),
@@ -106,7 +114,7 @@ class ConfOutput:
                                          self.config,
                                          objects)
                 
-            workbench_panels = []
+            workbench_panels:list[Investigation] = []
             for investigation in objects:
                 if investigation.inputs:
                     response_file_name_xml = investigation.lowercase_name + "___response_task.xml"
@@ -128,8 +136,11 @@ class ConfOutput:
                                                          template_name,
                                                         self.config,
                                                         workbench_panels))
+            return written_files
+    
 
-        elif type == SecurityContentType.lookups:
+    def writeLookups(self, objects:list[Lookup]) -> set[pathlib.Path]:
+            written_files:set[pathlib.Path] = set()
             for output_app_path, template_name in [ ('default/collections.conf', 'collections.j2'),
                                                     ('default/transforms.conf', 'transforms.j2')]:
                 written_files.add(ConfWriter.writeConfFile(pathlib.Path(output_app_path),
@@ -137,9 +148,7 @@ class ConfOutput:
                                             self.config,
                                             objects))
             
-        
-            #we want to copy all *.mlmodel files as well, not just csvs
-            files = list(glob.iglob(str(self.config.path/ 'lookups/*.csv'))) + list(glob.iglob(str(self.config.path / 'lookups/*.mlmodel')))
+            #Get the path to the lookups folder
             lookup_folder = self.config.getPackageDirectoryPath()/"lookups"
             
             # Make the new folder for the lookups 
@@ -147,26 +156,24 @@ class ConfOutput:
             lookup_folder.mkdir(exist_ok=True)
 
             #Copy each lookup into the folder
-            for lookup_name in files:
-                lookup_path = pathlib.Path(lookup_name)
-                if lookup_path.is_file():
-                    shutil.copy(lookup_path, lookup_folder/lookup_path.name)
-                else:
-                    raise(Exception(f"Error copying lookup/mlmodel file.  Path {lookup_path} does not exist or is not a file."))
-
-        elif type == SecurityContentType.macros:
+            for lookup in objects:
+                if isinstance(lookup, FileBackedLookup):
+                    shutil.copy(lookup.filename, lookup_folder/lookup.app_filename.name)
+            return written_files
+    
+
+    def writeMacros(self, objects:list[Macro]) -> set[pathlib.Path]:
+            written_files:set[pathlib.Path] = set()
             written_files.add(ConfWriter.writeConfFile(pathlib.Path('default/macros.conf'),
                                     'macros.j2',
                                     self.config, objects))
-        
-        elif type == SecurityContentType.dashboards:
-            written_files.update(ConfWriter.writeDashboardFiles(self.config, objects))
-
-
-        return written_files
-            
-
+            return written_files
+    
 
+    def writeDashboards(self, objects:list[Dashboard]) -> set[pathlib.Path]:
+            written_files:set[pathlib.Path] = set()
+            written_files.update(ConfWriter.writeDashboardFiles(self.config, objects))
+            return written_files
 
     
     def packageAppTar(self) -> None:
@@ -202,7 +209,7 @@ class ConfOutput:
             
         
         
-    def packageApp(self, method=packageAppTar)->None:
+    def packageApp(self, method: Callable[[ConfOutput],None]=packageAppTar)->None:
         return method(self)