contentctl 5.1.0__py3-none-any.whl → 5.2.0__py3-none-any.whl

contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py

@@ -89,7 +89,7 @@ class DetectionTestingManagerOutputDto:
     start_time: Union[datetime.datetime, None] = None
     replay_index: str = "contentctl_testing_index"
     replay_host: str = "CONTENTCTL_HOST"
-    timeout_seconds: int = 60
+    timeout_seconds: int = 120
     terminate: bool = False
 
 

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -474,7 +474,7 @@ class Detection_Abstract(SecurityContentObject):
                         "name": lookup.name,
                         "description": lookup.description,
                         "filename": lookup.filename.name,
-                        "default_match": "true" if lookup.default_match else "false",
+                        "default_match": lookup.default_match,
                         "case_sensitive_match": "true"
                         if lookup.case_sensitive_match
                         else "false",
@@ -1055,3 +1055,30 @@ class Detection_Abstract(SecurityContentObject):
         # Return the summary
 
         return summary_dict
+
+    @model_validator(mode="after")
+    def validate_data_source_output_fields(self):
+        # Skip validation for Hunting and Correlation types, or non-production detections
+        if self.status != DetectionStatus.production or self.type in {
+            AnalyticsType.Hunting,
+            AnalyticsType.Correlation,
+        }:
+            return self
+
+        # Validate that all required output fields are present in the search
+        for data_source in self.data_source_objects:
+            if not data_source.output_fields:
+                continue
+
+            missing_fields = [
+                field for field in data_source.output_fields if field not in self.search
+            ]
+
+            if missing_fields:
+                raise ValueError(
+                    f"Data source '{data_source.name}' has output fields "
+                    f"{missing_fields} that are not present in the search "
+                    f"for detection '{self.name}'"
+                )
+
+        return self

contentctl/objects/data_source.py

@@ -17,10 +17,12 @@ class DataSource(SecurityContentObject):
     source: str = Field(...)
     sourcetype: str = Field(...)
     separator: Optional[str] = None
+    separator_value: None | str = None
     configuration: Optional[str] = None
     supported_TA: list[TA] = []
     fields: None | list = None
     field_mappings: None | list = None
+    mitre_components: list[str] = []
     convert_to_log_source: None | list = None
     example_log: None | str = None
     output_fields: list[str] = []

contentctl/objects/lookup.py

@@ -6,9 +6,10 @@ import pathlib
 import re
 from enum import StrEnum, auto
 from functools import cached_property
-from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Self
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Self
 
 from pydantic import (
+    BeforeValidator,
     Field,
     FilePath,
     NonNegativeInt,
@@ -69,7 +70,19 @@ class Lookup_Type(StrEnum):
 
 # TODO (#220): Split Lookup into 2 classes
 class Lookup(SecurityContentObject, abc.ABC):
-    default_match: Optional[bool] = None
+    # We need to make sure that this is converted to a string because we widely
+    # use the string "False" in our lookup content.  However, PyYAML reads this
+    # as a BOOL and this causes parsing to fail. As such, we will always
+    # convert this to a string if it is passed as a bool
+    default_match: Annotated[
+        str, BeforeValidator(lambda dm: str(dm).lower() if isinstance(dm, bool) else dm)
+    ] = Field(
+        default="",
+        description="This field is given a default value of ''"
+        "because it is the default value specified in the transforms.conf "
+        "docs. Giving it a type of str rather than str | None simplifies "
+        "the typing for the field.",
+    )
     # Per the documentation for transforms.conf, EXACT should not be specified in this list,
     # so we include only WILDCARD and CIDR
     match_type: list[Annotated[str, Field(pattern=r"(^WILDCARD|CIDR)\(.+\)$")]] = Field(
@@ -88,7 +101,7 @@ class Lookup(SecurityContentObject, abc.ABC):
 
         # All fields custom to this model
         model = {
-            "default_match": "true" if self.default_match is True else "false",
+            "default_match": self.default_match,
             "match_type": self.match_type_to_conf_format,
             "min_matches": self.min_matches,
             "max_matches": self.max_matches,

contentctl/output/attack_nav_output.py

@@ -1,5 +1,5 @@
-from typing import List, Union
 import pathlib
+from typing import List, Union
 
 from contentctl.objects.detection import Detection
 from contentctl.output.attack_nav_writer import AttackNavWriter
@@ -10,14 +10,21 @@ class AttackNavOutput:
         self, detections: List[Detection], output_path: pathlib.Path
     ) -> None:
         techniques: dict[str, dict[str, Union[List[str], int]]] = {}
+
         for detection in detections:
             for tactic in detection.tags.mitre_attack_id:
                 if tactic not in techniques:
                     techniques[tactic] = {"score": 0, "file_paths": []}
 
-                detection_url = f"https://github.com/splunk/security_content/blob/develop/detections/{detection.source}/{detection.file_path.name}"
-                techniques[tactic]["score"] += 1
-                techniques[tactic]["file_paths"].append(detection_url)
+                detection_type = detection.source
+                detection_id = detection.id
+
+                # Store all three pieces of information separately
+                detection_info = f"{detection_type}|{detection_id}|{detection.name}"
+
+                techniques[tactic]["score"] = techniques[tactic].get("score", 0) + 1
+                if isinstance(techniques[tactic]["file_paths"], list):
+                    techniques[tactic]["file_paths"].append(detection_info)
 
         """
         for detection in objects:

contentctl/output/attack_nav_writer.py

@@ -1,11 +1,11 @@
 import json
-from typing import Union, List
 import pathlib
+from typing import List, Union
 
-VERSION = "4.3"
+VERSION = "4.5"
 NAME = "Detection Coverage"
-DESCRIPTION = "security_content detection coverage"
-DOMAIN = "mitre-enterprise"
+DESCRIPTION = "Security Content Detection Coverage"
+DOMAIN = "enterprise-attack"
 
 
 class AttackNavWriter:
@@ -14,52 +14,68 @@ class AttackNavWriter:
         mitre_techniques: dict[str, dict[str, Union[List[str], int]]],
         output_path: pathlib.Path,
     ) -> None:
-        max_count = 0
-        for technique_id in mitre_techniques.keys():
-            if mitre_techniques[technique_id]["score"] > max_count:
-                max_count = mitre_techniques[technique_id]["score"]
+        max_count = max(
+            (technique["score"] for technique in mitre_techniques.values()), default=0
+        )
 
         layer_json = {
-            "version": VERSION,
+            "versions": {"attack": "16", "navigator": "5.1.0", "layer": VERSION},
             "name": NAME,
             "description": DESCRIPTION,
             "domain": DOMAIN,
             "techniques": [],
+            "gradient": {
+                "colors": ["#ffffff", "#66b1ff", "#096ed7"],
+                "minValue": 0,
+                "maxValue": max_count,
+            },
+            "filters": {
+                "platforms": [
+                    "Windows",
+                    "Linux",
+                    "macOS",
+                    "Network",
+                    "AWS",
+                    "GCP",
+                    "Azure",
+                    "Azure AD",
+                    "Office 365",
+                    "SaaS",
+                ]
+            },
+            "layout": {
+                "layout": "side",
+                "showName": True,
+                "showID": True,
+                "showAggregateScores": False,
+            },
+            "legendItems": [
+                {"label": "No detections", "color": "#ffffff"},
+                {"label": "Has detections", "color": "#66b1ff"},
+            ],
+            "showTacticRowBackground": True,
+            "tacticRowBackground": "#dddddd",
+            "selectTechniquesAcrossTactics": True,
         }
 
-        layer_json["gradient"] = {
-            "colors": ["#ffffff", "#66b1ff", "#096ed7"],
-            "minValue": 0,
-            "maxValue": max_count,
-        }
-
-        layer_json["filters"] = {
-            "platforms": [
-                "Windows",
-                "Linux",
-                "macOS",
-                "AWS",
-                "GCP",
-                "Azure",
-                "Office 365",
-                "SaaS",
-            ]
-        }
+        for technique_id, data in mitre_techniques.items():
+            links = []
+            for detection_info in data["file_paths"]:
+                # Split the detection info into its components
+                detection_type, detection_id, detection_name = detection_info.split("|")
 
-        layer_json["legendItems"] = [
-            {"label": "NO available detections", "color": "#ffffff"},
-            {"label": "Some detections available", "color": "#66b1ff"},
-        ]
+                # Construct research website URL (without the name)
+                research_url = (
+                    f"https://research.splunk.com/{detection_type}/{detection_id}/"
+                )
 
-        layer_json["showTacticRowBackground"] = True
-        layer_json["tacticRowBackground"] = "#dddddd"
-        layer_json["sorting"] = 3
+                links.append({"label": detection_name, "url": research_url})
 
-        for technique_id in mitre_techniques.keys():
             layer_technique = {
                 "techniqueID": technique_id,
-                "score": mitre_techniques[technique_id]["score"],
-                "comment": "\n\n".join(mitre_techniques[technique_id]["file_paths"]),
+                "score": data["score"],
+                "enabled": True,
+                "links": links,
             }
             layer_json["techniques"].append(layer_technique)
 

contentctl/output/templates/transforms.j2

@@ -7,8 +7,8 @@ filename = {{ lookup.app_filename.name  }}
 collection = {{ lookup.collection }}
 external_type = kvstore
 {% endif %}
-{% if lookup.default_match is defined and lookup.default_match != None  %}
-default_match = {{ lookup.default_match | lower }}
+{% if lookup.default_match != '' %}
+default_match = {{ lookup.default_match }}
 {% endif %}
 {% if lookup.case_sensitive_match is defined and lookup.case_sensitive_match != None %}
 case_sensitive_match = {{ lookup.case_sensitive_match | lower }}

{contentctl-5.1.0.dist-info → contentctl-5.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: contentctl
-Version: 5.1.0
+Version: 5.2.0
 Summary: Splunk Content Control Tool
 License: Apache 2.0
 Author: STRT

{contentctl-5.1.0.dist-info → contentctl-5.2.0.dist-info}/RECORD

@@ -4,7 +4,7 @@ contentctl/actions/deploy_acs.py,sha256=w3OqO8GXzB_5zHrE8lDYbadAy4Etw7F2o84Gze74
 contentctl/actions/detection_testing/DetectionTestingManager.py,sha256=TWZpmDjMqWRWyzsLyiYol_jAovAr6ok9J_GzE9-kNN0,9079
 contentctl/actions/detection_testing/GitService.py,sha256=a6y7lqCgSL1KdSVEgJDxawea8ZgEkGNfOKEf9v_BgLo,11135
 contentctl/actions/detection_testing/generate_detection_coverage_badge.py,sha256=bGUVKjKv96lTw1GZ4Kw1JX-Yicu4aOJWm-IL524e9HI,2302
-contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=52Xsbyq4M913kXuQ8JcjYfP2BvwRJo3chK1p2hK76o0,57281
+contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=dus-8ANS0VgKq1HOdis4wVF5DKtdot6csbcpayxqXDo,57282
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py,sha256=qYWgRW7uc-15jzwv5xSUF2xyLDmtyGyMfuXkQK9j-aM,7221
 contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureServer.py,sha256=Q1ZfCYOp54O39bgTScZMInkmZiU-bGAM9Hiwr2mq5ms,370
 contentctl/actions/detection_testing/progress_bar.py,sha256=UrpNCqxTmQ4hfoRZgxPJ1xvDVwMrTq0UnotdryHN0gM,3232
@@ -32,7 +32,7 @@ contentctl/helper/utils.py,sha256=rigwZzCwWzn11sKTVWDkYEtLmRSf0yBbJ671OSRQnOM,19
 contentctl/input/director.py,sha256=7mx93-k_8DUc4pwYjxwq39n8-BlDYZOvRBkdzdGA8Qc,10919
 contentctl/input/new_content_questions.py,sha256=z2C4Mg7-EyxtiF2z9m4SnSbi6QO4CUPB3wg__JeMXIQ,4067
 contentctl/input/yml_reader.py,sha256=ymmAqsWsf9Oj56waDOhCh_E4SomkSCmu4dAx7iURFt8,2050
-contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=T5SAFOwEeCDCe79IyVryoLO9k6VaTiwbQPWSNo1epk0,43001
+contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=VMt_0eA8eFa64DD2j96kIqfhREVNMPBeJAclsv0eFUw,43978
 contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py,sha256=P1v5n8hM4XzJOjNZbJ5m3y4Bu-cQYaddLPdbE6K-4Xw,12164
 contentctl/objects/alert_action.py,sha256=iEvdEOT4TrTXT0z4rQ_W5v79hPJpPhFPSzo7TuHDxwA,1376
 contentctl/objects/annotated_types.py,sha256=xR4EKvdOpNDEt0doGs8XjxCzKK99J2NHZgHFAmt7p2c,424
@@ -45,7 +45,7 @@ contentctl/objects/config.py,sha256=3l8tFVwrBDpAnS7aBgj6to0Kc8_s4bxuZY5Bm5vel8k,
 contentctl/objects/constants.py,sha256=VwwQtJBGC_zb3ukjb3A7P0CwAlyhacWiXczwAW5Jiog,5466
 contentctl/objects/correlation_search.py,sha256=ab6v-0nbzujhTMpwaXynQiInWpRO1zB5KR4eZLCav_M,45234
 contentctl/objects/dashboard.py,sha256=9DiHP_Nx2XBQv4-zUaz6v9XH5yeTJhxaGDlaQqCsbIU,4468
-contentctl/objects/data_source.py,sha256=qt4W14DEwKGO69oLGdJeuYqbWvGkZ6j5Nz0R1RhDQEQ,1491
+contentctl/objects/data_source.py,sha256=AtB6lAe43mx0GicphWTkmYupp8Fb1hmWgm_GpbX73wo,1567
 contentctl/objects/deployment.py,sha256=FRsgsX2T1gvA_0A44_sFPr22rsedxXVIhtO7o9F7eZM,2902
 contentctl/objects/deployment_email.py,sha256=_Sdr_BNjvXECiFonRHLkiOrIQp3slnUaERbptqRbD0Q,206
 contentctl/objects/deployment_notable.py,sha256=j5AniTRDcw32El5H91qKOXDVZvUYxnIuM4Zzlhrm9cM,258
@@ -64,7 +64,7 @@ contentctl/objects/integration_test.py,sha256=TYjKyH4YinUnYXOse5BQGCa4-ez_5mtoMw
 contentctl/objects/integration_test_result.py,sha256=_uUSgqgjFhEZM8UwOJI6Q9K-ekIrbKU6OPdqHZycl-s,279
 contentctl/objects/investigation.py,sha256=kmvQ0grCd1YVEW5wVF4-_Rx87PnOxPiNoN_ufTLc3ZM,3659
 contentctl/objects/investigation_tags.py,sha256=qDGNusrWDvCX_GcBEzag2MydSV0LIhGxoXZGgxDXfHA,1317
-contentctl/objects/lookup.py,sha256=mzOPhMDyoNZKLAj8zf6Wg6i9FJKMu3qHWinATtH75I8,13015
+contentctl/objects/lookup.py,sha256=uP8N2Munu9vDGOzkC3mBlM93xkICyVBi62fAXAloI_I,13656
 contentctl/objects/macro.py,sha256=usbxyOPIRIJoDmvawfP2DxtFNf71GaDwffxiZsRkP5A,3594
 contentctl/objects/manual_test.py,sha256=cx_XAtQ8VG8Ui_F553Xnut75vFEOtRwm1dDIIWNpOaM,952
 contentctl/objects/manual_test_result.py,sha256=FyCVVf-f1DKs-qBkM4tbKfY6mkrW25NcIEBqyaDC2rE,156
@@ -89,8 +89,8 @@ contentctl/objects/unit_test.py,sha256=-rtSmZ8N2UZ4NkDsfzNXzXiF6dTDwt_jsQ_14xp0h
 contentctl/objects/unit_test_baseline.py,sha256=ezg8Ctih_3che2ln2tuVCAtRPHaf5tDMR3dGb34MqaA,287
 contentctl/objects/unit_test_result.py,sha256=gqHqYN5XGBKdV-mdKhAdwfOw4_PpN3i9z_b6ciByDSc,2928
 contentctl/output/api_json_output.py,sha256=QWe_KWlHHxE4Mhd3BHRfJbUJ4z2mLHZn_eMWfMVInik,8237
-contentctl/output/attack_nav_output.py,sha256=2_JISJ3sL4dVAwrIfZ7c426CGz5gjUBVkWh0uFO2MXU,2276
-contentctl/output/attack_nav_writer.py,sha256=FEua57vv347PjFiu1skOEGAbxIqPWMN8Iyp8nDrIvAA,2044
+contentctl/output/attack_nav_output.py,sha256=cbQNZkcNCKaQm7Ht70_tcmTvixtsuVDjQB4BpZ8s-Ts,2489
+contentctl/output/attack_nav_writer.py,sha256=AiQU3q8hzz_lJECI-sjyqOsWx64HUugg3aAHEeZl-qM,2750
 contentctl/output/conf_output.py,sha256=2_ofRqMro4xmFzf6ZmPRDd93pCG-LQhOiB_kE4owADc,10609
 contentctl/output/conf_writer.py,sha256=9eqt2tm1xjs397pwWLz5oPJcMHbs62ejRG7KghGQQCI,15137
 contentctl/output/data_source_writer.py,sha256=hjr0b5zfJ2UHcDLbCkmTrqma1ngu8F5vWFPJEwOZwU8,1860
@@ -124,7 +124,7 @@ contentctl/output/templates/savedsearches_baselines.j2,sha256=WHZB4e0vmeym8832Vx
 contentctl/output/templates/savedsearches_detections.j2,sha256=B7_b8aBjEKAV7mJm0DxUS_HyCt63bQZtpacCQfDgDqc,6033
 contentctl/output/templates/savedsearches_investigations.j2,sha256=KH2r8SgyAMiettSHypSbA2-1XmQ_8_8xzk3BkbZ1Re4,1196
 contentctl/output/templates/server.conf.j2,sha256=sPZUkiuJNGm9R8rpjfRKyuAvmmQb0C4w9Q6hpmvmPeU,127
-contentctl/output/templates/transforms.j2,sha256=mM9vIIGqxlhLk1W8sHoUgppfK1FO3ESQD1WCVAewhBw,1386
+contentctl/output/templates/transforms.j2,sha256=TEKZi8DWpcCysRTNvuLEgAwx-g1SZ2E0CkLiu6v6AlU,1339
 contentctl/output/templates/workflow_actions.j2,sha256=DFoZVnCa8dMRHjW2AdpoydBC0THgiH_W-Nx7WI4-uR4,925
 contentctl/output/yml_writer.py,sha256=gGgbamHWunHKjj47TcqB04k0xliX6w3H7iajZtUZRSU,2124
 contentctl/templates/README.md,sha256=GoRmywUqwnjaehY_GLmGqxsFXCLP9plpDYwB6W6nVPs,428
@@ -161,8 +161,8 @@ contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
 contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
 contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
 contentctl/templates/stories/cobalt_strike.yml,sha256=uj8idtDNOAIqpZ9p8usQg6mop1CQkJ5TlB4Q7CJdTIE,3082
-contentctl-5.1.0.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
-contentctl-5.1.0.dist-info/METADATA,sha256=F4a_TCVzB6ayCKCjSNSaX0cPIaZl12-DoLClq1xliNc,5097
-contentctl-5.1.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-contentctl-5.1.0.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
-contentctl-5.1.0.dist-info/RECORD,,
+contentctl-5.2.0.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
+contentctl-5.2.0.dist-info/METADATA,sha256=SxtNJ4XnhZzf7fFzZR3NWsl7hPwh3gX1lqwRTzQFaGc,5097
+contentctl-5.2.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+contentctl-5.2.0.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
+contentctl-5.2.0.dist-info/RECORD,,