contentctl 3.6.0__py3-none-any.whl → 4.0.2__py3-none-any.whl

contentctl/output/api_json_output.py

@@ -1,174 +1,246 @@
 import os
 import json
-
+import pathlib
 
 from contentctl.output.json_writer import JsonWriter
 from contentctl.objects.enums import SecurityContentType
+from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
+    SecurityContentObject_Abstract,
+)
 
-# Maximum Lambda Request Response Limit is 6MB
-# https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html
-# Note that if you are not using AWS Lambda, this file size may be increased.
-AWS_LAMBDA_LIMIT = 1024 * 1024 * 6 - 1
 
 
 class ApiJsonOutput:
+
     def writeObjects(
-        self, objects: list, output_path: str, type: SecurityContentType = None
+        self,
+        objects: list[SecurityContentObject_Abstract],
+        output_path: pathlib.Path,
+        app_label:str = "ESCU",
+        contentType: SecurityContentType = None
     ) -> None:
-        if type == SecurityContentType.detections:
-            obj_array = []
-            for detection in objects:
-                detection.id = str(detection.id)
-                obj_array.append(
-                    detection.dict(
-                        exclude_none=True,
-                        exclude={
-                            "deprecated": True,
-                            "experimental": True,
-                            "annotations": True,
-                            "risk": True,
-                            "playbooks": True,
-                            "baselines": True,
-                            "mappings": True,
-                            "test": True,
-                            "deployment": True,
-                            "type": True,
-                            "status": True,
-                            "data_source": True,
-                            "tests": True,
-                            "cve_enrichment": True,
-                            "file_path": True,
-                            "tags": {
-                                "file_path": True,
-                                "required_fields": True,
-                                "confidence": True,
-                                "impact": True,
-                                "product": True,
-                                "cve": True,
-                            },
-                        },
+        """#Serialize all objects
+        try:
+            for obj in objects:
+
+                serialized_objects.append(obj.model_dump())
+        except Exception as e:
+            raise Exception(f"Error serializing object with name '{obj.name}' and type '{type(obj).__name__}': '{str(e)}'")
+        """
+
+        if contentType == SecurityContentType.detections:
+            detections = [
+                detection.model_dump(
+                    include=set(
+                        [
+                            "name",
+                            "author",
+                            "date",
+                            "version",
+                            "id",
+                            "description",
+                            "tags",
+                            "search",
+                            "how_to_implement",
+                            "known_false_positives",
+                            "references",
+                            "datamodel",
+                            "macros",
+                            "lookups",
+                            "source",
+                            "nes_fields",
+                        ]
                     )
                 )
-
-            for detection in obj_array:
-                # Loop through each macro in the detection
-                for macro in detection["macros"]:
-                    # Remove the 'file_path' key if it exists
-                    macro.pop("file_path", None)
-
+                for detection in objects
+            ]
+            #Only a subset of macro fields are required:
+            # for detection in detections:
+            #     new_macros = []
+            #     for macro in detection.get("macros",[]):
+            #         new_macro_fields = {}
+            #         new_macro_fields["name"] = macro.get("name")
+            #         new_macro_fields["definition"] = macro.get("definition")
+            #         new_macro_fields["description"] = macro.get("description")
+            #         if len(macro.get("arguments", [])) > 0:
+            #             new_macro_fields["arguments"] = macro.get("arguments") 
+            #         new_macros.append(new_macro_fields)
+            #     detection["macros"] = new_macros
+            #     del()
+                    
+            
             JsonWriter.writeJsonObject(
-                os.path.join(output_path, "detections.json"), {"detections": obj_array}
+                os.path.join(output_path, "detections.json"), "detections", detections
             )
 
-            ### Code to be added to contentctl to ship filter macros to macros.json
-
-            obj_array = []
-            for detection in objects:
-                detection_dict = detection.dict()
-                if "macros" in detection_dict:
-                    for macro in detection_dict["macros"]:
-                        obj_array.append(macro)
-
-            uniques: set[str] = set()
-            for obj in obj_array:
-                if obj.get("arguments", None) != None:
-                    uniques.add(json.dumps(obj, sort_keys=True))
-                else:
-                    obj.pop("arguments")
-                    uniques.add(json.dumps(obj, sort_keys=True))
-
-            obj_array = []
-            for item in uniques:
-                obj_array.append(json.loads(item))
-
-            for obj in obj_array:
-                if "file_path" in obj:
-                    del obj["file_path"]
-
+        elif contentType == SecurityContentType.macros:
+            macros = [
+                macro.model_dump(include=set(["definition", "description", "name"]))
+                for macro in objects
+            ]
+            for macro in macros:
+                for k in ["author", "date","version","id","references"]:
+                    if k in macro:
+                        del(macro[k])
             JsonWriter.writeJsonObject(
-                os.path.join(output_path, "macros.json"), {"macros": obj_array}
+                os.path.join(output_path, "macros.json"), "macros", macros
             )
 
-        elif type == SecurityContentType.stories:
-            obj_array = []
-            for story in objects:
-                story.id = str(story.id)
-                obj_array.append(
-                    story.dict(
-                        exclude_none=True,
-                        exclude={"investigations": True, "file_path": True},
+        elif contentType == SecurityContentType.stories:
+            stories = [
+                story.model_dump(
+                    include=set(
+                        [
+                            "name",
+                            "author",
+                            "date",
+                            "version",
+                            "id",
+                            "description",
+                            "narrative",
+                            "references",
+                            "tags",
+                            "detections_names",
+                            "investigation_names",
+                            "baseline_names",
+                            "detections",
+                        ]
                     )
                 )
+                for story in objects
+            ]
+            # Only get certain fields from detections
+            for story in stories:
+                # Only use a small subset of fields from the detection
+                story["detections"] = [
+                    {
+                        "name": detection["name"],
+                        "source": detection["source"],
+                        "type": detection["type"],
+                        "tags": detection["tags"].get("mitre_attack_enrichments", []),
+                    }
+                    for detection in story["detections"]
+                ]
+                story["detection_names"] = [f"{app_label} - {name} - Rule" for name in story["detection_names"]]
+                
 
             JsonWriter.writeJsonObject(
-                os.path.join(output_path, "stories.json"), {"stories": obj_array}
+                os.path.join(output_path, "stories.json"), "stories", stories
             )
 
-        elif type == SecurityContentType.baselines:
-            obj_array = []
-            for baseline in objects:
-                baseline.id = str(baseline.id)
-                obj_array.append(
-                    baseline.dict(
-                        exclude={
-                            "deployment": True,
-                            "check_references": True,
-                            "file_path": True,
-                        }
+        elif contentType == SecurityContentType.baselines:
+            try:
+                baselines = [
+                    baseline.model_dump(
+                        include=set(
+                            [
+                                "name",
+                                "author",
+                                "date",
+                                "version",
+                                "id",
+                                "description",
+                                "type",
+                                "datamodel",
+                                "search",
+                                "how_to_implement",
+                                "known_false_positives",
+                                "references",
+                                "tags",
+                            ]
+                        )
                     )
-                )
+                    for baseline in objects
+                ]
+            except Exception as e:
+                print(e)
+                print('wait')
 
             JsonWriter.writeJsonObject(
-                os.path.join(output_path, "baselines.json"), {"baselines": obj_array}
-            )
+                    os.path.join(output_path, "baselines.json"), "baselines", baselines
+                )
 
-        elif type == SecurityContentType.investigations:
-            obj_array = []
-            for investigation in objects:
-                investigation.id = str(investigation.id)
-                obj_array.append(
-                    investigation.dict(
-                        exclude={
-                            "file_path": True,
-                        }
+        elif contentType == SecurityContentType.investigations:
+            investigations = [
+                investigation.model_dump(
+                    include=set(
+                        [
+                            "name",
+                            "author",
+                            "date",
+                            "version",
+                            "id",
+                            "description",
+                            "type",
+                            "datamodel",
+                            "search",
+                            "how_to_implemnet",
+                            "known_false_positives",
+                            "references",
+                            "inputs",
+                            "tags",
+                            "lowercase_name",
+                        ]
                     )
                 )
-
+                for investigation in objects
+            ]
             JsonWriter.writeJsonObject(
                 os.path.join(output_path, "response_tasks.json"),
-                {"response_tasks": obj_array},
+                "response_tasks",
+                investigations,
             )
 
-        elif type == SecurityContentType.lookups:
-            obj_array = []
-            for lookup in objects:
-
-                obj_array.append(
-                    lookup.dict(
-                        exclude={
-                            "file_path": True,
-                        }
+        elif contentType == SecurityContentType.lookups:
+            lookups = [
+                lookup.model_dump(
+                    include=set(
+                        [
+                            "name",
+                            "description",
+                            "collection",
+                            "fields_list",
+                            "filename",
+                            "default_match",
+                            "match_type",
+                            "min_matches",
+                            "case_sensitive_match",
+                        ]
                     )
                 )
-
+                for lookup in objects
+            ]
+            for lookup in lookups:
+                for k in ["author","date","version","id","references"]:
+                    if k in lookup:
+                        del(lookup[k]) 
             JsonWriter.writeJsonObject(
-                os.path.join(output_path, "lookups.json"), {"lookups": obj_array}
+                os.path.join(output_path, "lookups.json"), "lookups", lookups
             )
 
-        elif type == SecurityContentType.deployments:
-            obj_array = []
-            for deployment in objects:
-                deployment.id = str(deployment.id)
-                obj_array.append(
-                    deployment.dict(
-                        exclude_none=True,
-                        exclude={
-                            "file_path": True,
-                        },
+        elif contentType == SecurityContentType.deployments:
+            deployments = [
+                deployment.model_dump(
+                    include=set(
+                        [
+                            "name",
+                            "author",
+                            "date",
+                            "version",
+                            "id",
+                            "description",
+                            "scheduling",
+                            "rba",
+                            "tags"
+                        ] 
                     )
                 )
-
+                for deployment in objects
+            ]
+            #references are not to be included, but have been deleted in the
+            #model_serialization logic
             JsonWriter.writeJsonObject(
                 os.path.join(output_path, "deployments.json"),
-                {"deployments": obj_array},
-            )
+                "deployments",
+                deployments,
+            )

contentctl/output/attack_nav_output.py

@@ -1,28 +1,39 @@
 import os
+from typing import List,Union
+import pathlib
 
-
-from contentctl.objects.enums import SecurityContentType
+from contentctl.objects.detection import Detection
 from contentctl.output.attack_nav_writer import AttackNavWriter
 
 
 class AttackNavOutput():
 
-    def writeObjects(self, objects: list, output_path: str, type: SecurityContentType = None) -> None:
-        techniques = dict()
+    def writeObjects(self, detections: List[Detection], output_path: pathlib.Path) -> None:
+        techniques:dict[str,dict[str,Union[List[str],int]]] = {}
+        for detection in detections:
+            for tactic in detection.tags.mitre_attack_id:
+                if tactic not in techniques:
+                    techniques[tactic] = {'score':0,'file_paths':[]}
+                
+                detection_url = f"https://github.com/splunk/security_content/blob/develop/detections/{detection.source}/{detection.file_path.name}"
+                techniques[tactic]['score'] += 1
+                techniques[tactic]['file_paths'].append(detection_url)
+                
+        '''
         for detection in objects:
             if detection.tags.mitre_attack_enrichments:
                 for mitre_attack_enrichment in detection.tags.mitre_attack_enrichments:
                     if not mitre_attack_enrichment.mitre_attack_id in techniques:
                         techniques[mitre_attack_enrichment.mitre_attack_id] = {
                                 'score': 1,
-                                'file_paths': ['https://github.com/splunk/security_content/blob/develop/detections/' + detection.source + '/' + self.convertNameToFileName(detection.name)]
+                                'file_paths': ['https://github.com/splunk/security_content/blob/develop/detections/' + detection.getSource() + '/' + self.convertNameToFileName(detection.name)]
                             }
                     else:
                         techniques[mitre_attack_enrichment.mitre_attack_id]['score'] = techniques[mitre_attack_enrichment.mitre_attack_id]['score'] + 1
-                        techniques[mitre_attack_enrichment.mitre_attack_id]['file_paths'].append('https://github.com/splunk/security_content/blob/develop/detections/' + detection.source + '/' + self.convertNameToFileName(detection.name))
-
-        AttackNavWriter.writeAttackNavFile(techniques, os.path.join(output_path, 'coverage.json'))
-
+                        techniques[mitre_attack_enrichment.mitre_attack_id]['file_paths'].append('https://github.com/splunk/security_content/blob/develop/detections/' + detection.getSource() + '/' + self.convertNameToFileName(detection.name))
+        '''
+        AttackNavWriter.writeAttackNavFile(techniques, output_path / 'coverage.json')
+        
 
     def convertNameToFileName(self, name: str):
         file_name = name \

contentctl/output/attack_nav_writer.py

@@ -1,7 +1,7 @@
 
 import json
-
-
+from typing import Union, List
+import pathlib
 VERSION = "4.3"
 NAME = "Detection Coverage"
 DESCRIPTION = "security_content detection coverage"
@@ -11,7 +11,7 @@ DOMAIN = "mitre-enterprise"
 class AttackNavWriter():
 
     @staticmethod
-    def writeAttackNavFile(mitre_techniques : dict, output_path : str) -> None:
+    def writeAttackNavFile(mitre_techniques : dict[str,dict[str,Union[List[str],int]]], output_path : pathlib.Path) -> None:
         max_count = 0
         for technique_id in mitre_techniques.keys():
             if mitre_techniques[technique_id]['score'] > max_count:

contentctl/output/ba_yml_output.py

@@ -21,7 +21,7 @@ class BAYmlOutput():
             YmlWriter.writeYmlFile(file_path, object)
 
 
-    def writeObjects(self, objects: list, output_path: str, type: SecurityContentType = None) -> None:
+    def writeObjects(self, objects: list, output_path: str, contentType: SecurityContentType = None) -> None:
         for obj in objects: 
             file_name = "ssa___" + self.convertNameToFileName(obj.name, obj.tags)
             if self.isComplexBARule(obj.search):
@@ -46,8 +46,9 @@ class BAYmlOutput():
                 }
                 test_dict["tests"][0]["name"] = obj.name
                 for count in range(len(test_dict["tests"][0]["attack_data"])):
-                    a = urlparse(test_dict["tests"][0]["attack_data"][count]["data"])
+                    a = urlparse(str(test_dict["tests"][0]["attack_data"][count]["data"]))
                     test_dict["tests"][0]["attack_data"][count]["file_name"] = os.path.basename(a.path)
+
                 test = UnitTestOld.parse_obj(test_dict)
 
                 obj.test = test
@@ -150,4 +151,3 @@ class BAYmlOutput():
     def isComplexBARule(self, search):
         return re.findall("stats|first_time_event|adaptive_threshold", search)
 
-