contentctl 5.1.0__py3-none-any.whl → 5.3.0__py3-none-any.whl

contentctl/objects/playbook.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
+
+import pathlib
 from typing import Self
-from pydantic import model_validator, Field, FilePath
 
+from pydantic import Field, FilePath, field_validator, model_validator
 
+from contentctl.objects.enums import ContentStatus, PlaybookType
 from contentctl.objects.playbook_tags import PlaybookTag
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.enums import PlaybookType
 
 
 class Playbook(SecurityContentObject):
@@ -19,6 +21,16 @@ class Playbook(SecurityContentObject):
     playbook: str = Field(min_length=4)
     app_list: list[str] = Field(..., min_length=0)
     tags: PlaybookTag = Field(...)
+    status: ContentStatus = ContentStatus.production
+
+    @field_validator("status", mode="after")
+    @classmethod
+    def NarrowStatus(cls, status: ContentStatus) -> ContentStatus:
+        return cls.NarrowStatusTemplate(status, [ContentStatus.production])
+
+    @classmethod
+    def containing_folder(cls) -> pathlib.Path:
+        return pathlib.Path("playbooks")
 
     @model_validator(mode="after")
     def ensureJsonAndPyFilesExist(self) -> Self:
@@ -66,3 +78,5 @@ class Playbook(SecurityContentObject):
             )
 
         return self
+        return self
+        return self

contentctl/objects/rba.py

@@ -4,9 +4,7 @@ from abc import ABC
 from enum import Enum
 from typing import Annotated, Set
 
-from pydantic import BaseModel, Field, computed_field, model_serializer
-
-from contentctl.objects.enums import RiskSeverity
+from pydantic import BaseModel, Field, model_serializer
 
 RiskScoreValue_Type = Annotated[int, Field(ge=1, le=100)]
 
@@ -108,36 +106,6 @@ class RBAObject(BaseModel, ABC):
     risk_objects: Annotated[Set[RiskObject], Field(min_length=1)]
     threat_objects: Set[ThreatObject]
 
-    @computed_field
-    @property
-    def risk_score(self) -> RiskScoreValue_Type:
-        # First get the maximum score associated with
-        # a risk object. If there are no objects, then
-        # we should throw an exception.
-        if len(self.risk_objects) == 0:
-            raise Exception(
-                "There must be at least one Risk Object present to get Severity."
-            )
-        return max([risk_object.score for risk_object in self.risk_objects])
-
-    @computed_field
-    @property
-    def severity(self) -> RiskSeverity:
-        if 0 <= self.risk_score <= 20:
-            return RiskSeverity.INFORMATIONAL
-        elif 20 < self.risk_score <= 40:
-            return RiskSeverity.LOW
-        elif 40 < self.risk_score <= 60:
-            return RiskSeverity.MEDIUM
-        elif 60 < self.risk_score <= 80:
-            return RiskSeverity.HIGH
-        elif 80 < self.risk_score <= 100:
-            return RiskSeverity.CRITICAL
-        else:
-            raise Exception(
-                f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}"
-            )
-
     @model_serializer
     def serialize_rba(self) -> dict[str, str | list[dict[str, str | int]]]:
         return {

contentctl/objects/removed_security_content_object.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import pathlib
+from functools import cached_property
+
+from pydantic import ConfigDict, HttpUrl, computed_field, field_validator
+
+from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
+    SecurityContentObject_Abstract,
+)
+from contentctl.objects.enums import ContentStatus
+
+
+class SecurityContentObject(SecurityContentObject_Abstract):
+    pass
+
+
+class RemovedSecurityContentObject(SecurityContentObject):
+    # We MUST allow extra fields here because the python definitions of the underlying
+    # objects can change. We do not want to throw pasing errors on any of these, but we will
+    # only expose fields that are defined in the SecurityContentObject definiton directly
+    model_config = ConfigDict(validate_default=True, extra="ignore")
+    status: ContentStatus
+
+    @field_validator("status", mode="after")
+    @classmethod
+    def NarrowStatus(cls, status: ContentStatus) -> ContentStatus:
+        return cls.NarrowStatusTemplate(status, [ContentStatus.removed])
+
+    @computed_field
+    @cached_property
+    def migration_guide(self) -> HttpUrl:
+        """
+        A link to the research site containing a migration guide for the content
+
+        :returns: URL to the research site
+        :rtype: HTTPUrl
+        """
+        # this is split up so that we can explicilty ignore the warning on constructing
+        # the HttpUrl but catch other type issues
+        # link = f"https://research.splunk.com/migration_guide/{self.id}"
+        # This can likely be dynamically generated per detection, but for now we
+        # just make it static
+        link = "https://research.splunk.com/migration_guide/"
+
+        return HttpUrl(url=link)  # type: ignore
+
+    @classmethod
+    def containing_folder(cls) -> pathlib.Path:
+        return pathlib.Path("removed")

contentctl/objects/security_content_object.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+
 from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
     SecurityContentObject_Abstract,
 )

contentctl/objects/story.py

@@ -1,9 +1,17 @@
 from __future__ import annotations
 
 import re
-from typing import TYPE_CHECKING, List, Literal
-
-from pydantic import Field, computed_field, model_serializer, model_validator
+from functools import cached_property
+from typing import TYPE_CHECKING, List
+
+from pydantic import (
+    Field,
+    HttpUrl,
+    computed_field,
+    field_validator,
+    model_serializer,
+    model_validator,
+)
 
 from contentctl.objects.story_tags import StoryTags
 
@@ -14,20 +22,40 @@ if TYPE_CHECKING:
     from contentctl.objects.detection import Detection
     from contentctl.objects.investigation import Investigation
 
-from contentctl.objects.enums import DetectionStatus
+import pathlib
+
+from contentctl.objects.enums import ContentStatus
 from contentctl.objects.security_content_object import SecurityContentObject
 
 
 class Story(SecurityContentObject):
     narrative: str = Field(...)
     tags: StoryTags = Field(...)
-    status: Literal[DetectionStatus.production, DetectionStatus.deprecated]
+    status: ContentStatus
     # These are updated when detection and investigation objects are created.
     # Specifically in the model_post_init functions
     detections: List[Detection] = []
     investigations: List[Investigation] = []
     baselines: List[Baseline] = []
 
+    @field_validator("status", mode="after")
+    @classmethod
+    def NarrowStatus(cls, status: ContentStatus) -> ContentStatus:
+        return cls.NarrowStatusTemplate(
+            status, [ContentStatus.production, ContentStatus.deprecated]
+        )
+
+    @classmethod
+    def containing_folder(cls) -> pathlib.Path:
+        return pathlib.Path("stories")
+
+    @computed_field
+    @cached_property
+    def researchSiteLink(self) -> HttpUrl:
+        return HttpUrl(
+            url=f"https://research.splunk.com/stories/{self.name.lower().replace(' ', '_')}"
+        )  # type:ignore
+
     @computed_field
     @property
     def data_sources(self) -> list[DataSource]:
@@ -145,3 +173,7 @@ class Story(SecurityContentObject):
     @property
     def baseline_names(self) -> List[str]:
         return [baseline.name for baseline in self.baselines]
+
+    @classmethod
+    def static_get_conf_stanza_name(cls, name: str, app: CustomApp) -> str:
+        return name

contentctl/output/api_json_output.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
+
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
+    from contentctl.objects.baseline import Baseline
+    from contentctl.objects.deployment import Deployment
     from contentctl.objects.detection import Detection
+    from contentctl.objects.investigation import Investigation
     from contentctl.objects.lookup import Lookup
     from contentctl.objects.macro import Macro
     from contentctl.objects.story import Story
-    from contentctl.objects.baseline import Baseline
-    from contentctl.objects.investigation import Investigation
-    from contentctl.objects.deployment import Deployment
 
 import os
 import pathlib
@@ -42,6 +43,7 @@ class ApiJsonOutput:
                         "search",
                         "how_to_implement",
                         "known_false_positives",
+                        "rba",
                         "references",
                         "datamodel",
                         "macros",

contentctl/output/attack_nav_output.py

@@ -1,5 +1,5 @@
-from typing import List, Union
 import pathlib
+from typing import List, Union
 
 from contentctl.objects.detection import Detection
 from contentctl.output.attack_nav_writer import AttackNavWriter
@@ -10,14 +10,21 @@ class AttackNavOutput:
         self, detections: List[Detection], output_path: pathlib.Path
     ) -> None:
         techniques: dict[str, dict[str, Union[List[str], int]]] = {}
+
         for detection in detections:
             for tactic in detection.tags.mitre_attack_id:
                 if tactic not in techniques:
                     techniques[tactic] = {"score": 0, "file_paths": []}
 
-                detection_url = f"https://github.com/splunk/security_content/blob/develop/detections/{detection.source}/{detection.file_path.name}"
-                techniques[tactic]["score"] += 1
-                techniques[tactic]["file_paths"].append(detection_url)
+                detection_type = detection.source
+                detection_id = detection.id
+
+                # Store all three pieces of information separately
+                detection_info = f"{detection_type}|{detection_id}|{detection.name}"
+
+                techniques[tactic]["score"] = techniques[tactic].get("score", 0) + 1
+                if isinstance(techniques[tactic]["file_paths"], list):
+                    techniques[tactic]["file_paths"].append(detection_info)
 
         """
         for detection in objects:

contentctl/output/attack_nav_writer.py

@@ -1,11 +1,11 @@
 import json
-from typing import Union, List
 import pathlib
+from typing import List, Union
 
-VERSION = "4.3"
+VERSION = "4.5"
 NAME = "Detection Coverage"
-DESCRIPTION = "security_content detection coverage"
-DOMAIN = "mitre-enterprise"
+DESCRIPTION = "Security Content Detection Coverage"
+DOMAIN = "enterprise-attack"
 
 
 class AttackNavWriter:
@@ -14,52 +14,68 @@ class AttackNavWriter:
         mitre_techniques: dict[str, dict[str, Union[List[str], int]]],
         output_path: pathlib.Path,
     ) -> None:
-        max_count = 0
-        for technique_id in mitre_techniques.keys():
-            if mitre_techniques[technique_id]["score"] > max_count:
-                max_count = mitre_techniques[technique_id]["score"]
+        max_count = max(
+            (technique["score"] for technique in mitre_techniques.values()), default=0
+        )
 
         layer_json = {
-            "version": VERSION,
+            "versions": {"attack": "16", "navigator": "5.1.0", "layer": VERSION},
             "name": NAME,
             "description": DESCRIPTION,
             "domain": DOMAIN,
             "techniques": [],
+            "gradient": {
+                "colors": ["#ffffff", "#66b1ff", "#096ed7"],
+                "minValue": 0,
+                "maxValue": max_count,
+            },
+            "filters": {
+                "platforms": [
+                    "Windows",
+                    "Linux",
+                    "macOS",
+                    "Network",
+                    "AWS",
+                    "GCP",
+                    "Azure",
+                    "Azure AD",
+                    "Office 365",
+                    "SaaS",
+                ]
+            },
+            "layout": {
+                "layout": "side",
+                "showName": True,
+                "showID": True,
+                "showAggregateScores": False,
+            },
+            "legendItems": [
+                {"label": "No detections", "color": "#ffffff"},
+                {"label": "Has detections", "color": "#66b1ff"},
+            ],
+            "showTacticRowBackground": True,
+            "tacticRowBackground": "#dddddd",
+            "selectTechniquesAcrossTactics": True,
         }
 
-        layer_json["gradient"] = {
-            "colors": ["#ffffff", "#66b1ff", "#096ed7"],
-            "minValue": 0,
-            "maxValue": max_count,
-        }
-
-        layer_json["filters"] = {
-            "platforms": [
-                "Windows",
-                "Linux",
-                "macOS",
-                "AWS",
-                "GCP",
-                "Azure",
-                "Office 365",
-                "SaaS",
-            ]
-        }
+        for technique_id, data in mitre_techniques.items():
+            links = []
+            for detection_info in data["file_paths"]:
+                # Split the detection info into its components
+                detection_type, detection_id, detection_name = detection_info.split("|")
 
-        layer_json["legendItems"] = [
-            {"label": "NO available detections", "color": "#ffffff"},
-            {"label": "Some detections available", "color": "#66b1ff"},
-        ]
+                # Construct research website URL (without the name)
+                research_url = (
+                    f"https://research.splunk.com/{detection_type}/{detection_id}/"
+                )
 
-        layer_json["showTacticRowBackground"] = True
-        layer_json["tacticRowBackground"] = "#dddddd"
-        layer_json["sorting"] = 3
+                links.append({"label": detection_name, "url": research_url})
 
-        for technique_id in mitre_techniques.keys():
             layer_technique = {
                 "techniqueID": technique_id,
-                "score": mitre_techniques[technique_id]["score"],
-                "comment": "\n\n".join(mitre_techniques[technique_id]["file_paths"]),
+                "score": data["score"],
+                "enabled": True,
+                "links": links,
             }
             layer_json["techniques"].append(layer_technique)
 

contentctl/output/conf_output.py

@@ -93,6 +93,10 @@ class ConfOutput:
 
         return written_files
 
+    # TODO (#339): we could have a discrepancy between detections tested and those delivered
+    #   based on the jinja2 template
+    #   {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or
+    #       detection.type == 'Hunting' or detection.type == 'Correlation') %}
     def writeDetections(self, objects: list[Detection]) -> set[pathlib.Path]:
         written_files: set[pathlib.Path] = set()
         for output_app_path, template_name in [
@@ -211,7 +215,11 @@ class ConfOutput:
             # even though the MLModel info was intentionally not written to the
             # transforms.conf file as noted above.
             if isinstance(lookup, FileBackedLookup):
-                shutil.copy(lookup.filename, lookup_folder / lookup.app_filename.name)
+                with (
+                    open(lookup_folder / lookup.app_filename.name, "w") as output_file,
+                    lookup.content_file_handle as output,
+                ):
+                    output_file.write(output.read())
         return written_files
 
     def writeMacros(self, objects: list[Macro]) -> set[pathlib.Path]:

contentctl/output/runtime_csv_writer.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import csv
+from io import StringIO
+from typing import TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from contentctl.input.director import DirectorOutputDto
+
+from contentctl.objects.config import CustomApp
+from contentctl.objects.data_source import DataSource
+
+
+class RuntimeCsvWriter:
+    @staticmethod
+    def generateDeprecationCSVContent(
+        director: DirectorOutputDto, app: CustomApp
+    ) -> str:
+        with StringIO() as output_buffer:
+            fieldNames = [
+                "Name",
+                "Content Type",
+                "Removed in Version",
+                "Reason",
+                "Replacement Content",
+                "Replacement Content Link",
+            ]
+
+            writer = csv.DictWriter(output_buffer, fieldnames=fieldNames)
+            writer.writeheader()
+
+            for content in director.name_to_content_map.values():
+                if content.deprecation_info is not None:
+                    try:
+                        writer.writerow(
+                            {
+                                "Name": content.deprecation_info.contentType.static_get_conf_stanza_name(
+                                    content.name, app
+                                ),
+                                "Content Type": content.deprecation_info.contentType.__name__,
+                                "Removed in Version": content.deprecation_info.removed_in_version,
+                                "Reason": content.deprecation_info.reason,
+                                "Replacement Content": "\n".join(
+                                    [
+                                        c.name
+                                        for c in content.deprecation_info.replacement_content
+                                    ]
+                                )
+                                or "No Replacement Content Available",
+                                "Replacement Content Link": "\n".join(
+                                    [
+                                        str(c.researchSiteLink)
+                                        for c in content.deprecation_info.replacement_content
+                                    ]
+                                )
+                                or "No Content Link Available",
+                            }
+                        )
+                    except Exception as e:
+                        print(e)
+                        import code
+
+                        code.interact(local=locals())
+            return output_buffer.getvalue()
+
+    @staticmethod
+    def generateDatasourceCSVContent(
+        data_source_objects: List[DataSource],
+    ) -> str:
+        with StringIO() as output_buffer:
+            writer = csv.writer(output_buffer)
+            # Write the header
+            writer.writerow(
+                [
+                    "name",
+                    "id",
+                    "author",
+                    "source",
+                    "sourcetype",
+                    "separator",
+                    "supported_TA_name",
+                    "supported_TA_version",
+                    "supported_TA_url",
+                    "description",
+                ]
+            )
+            # Write the data
+            for data_source in data_source_objects:
+                if len(data_source.supported_TA) > 0:
+                    supported_TA_name = data_source.supported_TA[0].name
+                    supported_TA_version = data_source.supported_TA[0].version
+                    supported_TA_url = data_source.supported_TA[0].url or ""
+                else:
+                    supported_TA_name = ""
+                    supported_TA_version = ""
+                    supported_TA_url = ""
+                writer.writerow(
+                    [
+                        data_source.name,
+                        data_source.id,
+                        data_source.author,
+                        data_source.source,
+                        data_source.sourcetype,
+                        data_source.separator,
+                        supported_TA_name,
+                        supported_TA_version,
+                        supported_TA_url,
+                        data_source.description,
+                    ]
+                )
+            return output_buffer.getvalue()

contentctl/output/svg_output.py

@@ -1,10 +1,9 @@
 import pathlib
-from typing import List, Any
+from typing import Any, List
 
-from contentctl.objects.enums import SecurityContentType
-from contentctl.output.jinja_writer import JinjaWriter
-from contentctl.objects.enums import DetectionStatus
 from contentctl.objects.detection import Detection
+from contentctl.objects.enums import ContentStatus, SecurityContentType
+from contentctl.output.jinja_writer import JinjaWriter
 
 
 class SvgOutput:
@@ -49,7 +48,7 @@ class SvgOutput:
             [
                 detection
                 for detection in detections
-                if detection.status == DetectionStatus.production
+                if detection.status == ContentStatus.production
             ],
         )
         # deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated])

contentctl/output/templates/savedsearches_detections.j2

@@ -65,14 +65,10 @@ action.notable.param.nes_fields = {{ detection.nes_fields }}
 action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}}
 action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%}
 action.notable.param.security_domain = {{ detection.tags.security_domain }}
-{% if detection.rba %}
-action.notable.param.severity = {{ detection.rba.severity }}
-{% else %}
-{# Correlations do not have detection.rba defined, but should get a default severity #}
-action.notable.param.severity = high 
-{% endif %}
+action.notable.param.severity = {{ detection.severity }}
 {% endif %}
 {% if detection.deployment.alert_action.email %}
+action.email = 1
 action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }}
 action.email.to = {{ detection.deployment.alert_action.email.to }}
 action.email.message.alert = {{ detection.deployment.alert_action.email.message | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }}

contentctl/output/templates/transforms.j2

@@ -7,8 +7,8 @@ filename = {{ lookup.app_filename.name  }}
 collection = {{ lookup.collection }}
 external_type = kvstore
 {% endif %}
-{% if lookup.default_match is defined and lookup.default_match != None  %}
-default_match = {{ lookup.default_match | lower }}
+{% if lookup.default_match != '' %}
+default_match = {{ lookup.default_match }}
 {% endif %}
 {% if lookup.case_sensitive_match is defined and lookup.case_sensitive_match != None %}
 case_sensitive_match = {{ lookup.case_sensitive_match | lower }}

{contentctl-5.1.0.dist-info → contentctl-5.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: contentctl
-Version: 5.1.0
+Version: 5.3.0
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
 Requires-Dist: PyYAML (>=6.0.2,<7.0.0)
-Requires-Dist: attackcti (>=0.4.0,<0.5.0)
+Requires-Dist: attackcti (>=0.5.4,<0.6)
 Requires-Dist: bottle (>=0.12.25,<0.14.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: gitpython (>=3.1.43,<4.0.0)
@@ -22,8 +22,9 @@ Requires-Dist: pydantic (>=2.9.2,<2.10.0)
 Requires-Dist: pygit2 (>=1.15.1,<2.0.0)
 Requires-Dist: questionary (>=2.0.1,<3.0.0)
 Requires-Dist: requests (>=2.32.3,<2.33.0)
+Requires-Dist: rich (>=14.0.0,<15.0.0)
 Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
-Requires-Dist: setuptools (>=69.5.1,<76.0.0)
+Requires-Dist: setuptools (>=69.5.1,<79.0.0)
 Requires-Dist: splunk-sdk (>=2.0.2,<3.0.0)
 Requires-Dist: tqdm (>=4.66.5,<5.0.0)
 Requires-Dist: tyro (>=0.9.2,<0.10.0)