contentctl 4.3.5__py3-none-any.whl → 4.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/build.py +1 -0
- contentctl/actions/detection_testing/GitService.py +10 -10
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +68 -38
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +5 -1
- contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py +10 -8
- contentctl/actions/inspect.py +6 -4
- contentctl/actions/new_content.py +10 -2
- contentctl/actions/validate.py +2 -1
- contentctl/enrichments/cve_enrichment.py +6 -7
- contentctl/input/director.py +14 -12
- contentctl/input/new_content_questions.py +9 -42
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +147 -7
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +17 -9
- contentctl/objects/base_test_result.py +7 -7
- contentctl/objects/baseline.py +12 -18
- contentctl/objects/baseline_tags.py +2 -5
- contentctl/objects/config.py +12 -7
- contentctl/objects/constants.py +30 -0
- contentctl/objects/correlation_search.py +79 -114
- contentctl/objects/dashboard.py +100 -0
- contentctl/objects/deployment.py +20 -5
- contentctl/objects/detection_tags.py +22 -20
- contentctl/objects/drilldown.py +70 -0
- contentctl/objects/enums.py +26 -22
- contentctl/objects/investigation.py +23 -15
- contentctl/objects/investigation_tags.py +4 -3
- contentctl/objects/lookup.py +8 -1
- contentctl/objects/macro.py +16 -7
- contentctl/objects/notable_event.py +6 -5
- contentctl/objects/risk_analysis_action.py +4 -4
- contentctl/objects/risk_event.py +8 -7
- contentctl/objects/story.py +4 -16
- contentctl/objects/throttling.py +46 -0
- contentctl/output/conf_output.py +4 -0
- contentctl/output/conf_writer.py +20 -3
- contentctl/output/templates/analyticstories_detections.j2 +2 -2
- contentctl/output/templates/analyticstories_investigations.j2 +5 -5
- contentctl/output/templates/analyticstories_stories.j2 +1 -1
- contentctl/output/templates/savedsearches_baselines.j2 +2 -3
- contentctl/output/templates/savedsearches_detections.j2 +12 -7
- contentctl/output/templates/savedsearches_investigations.j2 +3 -4
- contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +10 -1
- {contentctl-4.3.5.dist-info → contentctl-4.4.0.dist-info}/METADATA +3 -2
- {contentctl-4.3.5.dist-info → contentctl-4.4.0.dist-info}/RECORD +47 -45
- {contentctl-4.3.5.dist-info → contentctl-4.4.0.dist-info}/WHEEL +1 -1
- contentctl/output/templates/finding_report.j2 +0 -30
- {contentctl-4.3.5.dist-info → contentctl-4.4.0.dist-info}/LICENSE.md +0 -0
- {contentctl-4.3.5.dist-info → contentctl-4.4.0.dist-info}/entry_points.txt +0 -0
contentctl/objects/enums.py
CHANGED
|
@@ -55,6 +55,8 @@ class SecurityContentType(enum.Enum):
|
|
|
55
55
|
investigations = 8
|
|
56
56
|
unit_tests = 9
|
|
57
57
|
data_sources = 11
|
|
58
|
+
dashboards = 12
|
|
59
|
+
|
|
58
60
|
|
|
59
61
|
# Bringing these changes back in line will take some time after
|
|
60
62
|
# the initial merge is complete
|
|
@@ -195,21 +197,21 @@ class KillChainPhase(str, enum.Enum):
|
|
|
195
197
|
class DataSource(str,enum.Enum):
|
|
196
198
|
OSQUERY_ES_PROCESS_EVENTS = "OSQuery ES Process Events"
|
|
197
199
|
POWERSHELL_4104 = "Powershell 4104"
|
|
198
|
-
SYSMON_EVENT_ID_1 = "Sysmon
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
200
|
+
SYSMON_EVENT_ID_1 = "Sysmon EventID 1"
|
|
201
|
+
SYSMON_EVENT_ID_3 = "Sysmon EventID 3"
|
|
202
|
+
SYSMON_EVENT_ID_5 = "Sysmon EventID 5"
|
|
203
|
+
SYSMON_EVENT_ID_6 = "Sysmon EventID 6"
|
|
204
|
+
SYSMON_EVENT_ID_7 = "Sysmon EventID 7"
|
|
205
|
+
SYSMON_EVENT_ID_8 = "Sysmon EventID 8"
|
|
206
|
+
SYSMON_EVENT_ID_9 = "Sysmon EventID 9"
|
|
207
|
+
SYSMON_EVENT_ID_10 = "Sysmon EventID 10"
|
|
208
|
+
SYSMON_EVENT_ID_11 = "Sysmon EventID 11"
|
|
209
|
+
SYSMON_EVENT_ID_13 = "Sysmon EventID 13"
|
|
210
|
+
SYSMON_EVENT_ID_15 = "Sysmon EventID 15"
|
|
211
|
+
SYSMON_EVENT_ID_20 = "Sysmon EventID 20"
|
|
212
|
+
SYSMON_EVENT_ID_21 = "Sysmon EventID 21"
|
|
213
|
+
SYSMON_EVENT_ID_22 = "Sysmon EventID 22"
|
|
214
|
+
SYSMON_EVENT_ID_23 = "Sysmon EventID 23"
|
|
213
215
|
WINDOWS_SECURITY_4624 = "Windows Security 4624"
|
|
214
216
|
WINDOWS_SECURITY_4625 = "Windows Security 4625"
|
|
215
217
|
WINDOWS_SECURITY_4648 = "Windows Security 4648"
|
|
@@ -405,14 +407,16 @@ class NistCategory(str, enum.Enum):
|
|
|
405
407
|
RC_IM = "RC.IM"
|
|
406
408
|
RC_CO = "RC.CO"
|
|
407
409
|
|
|
408
|
-
class RiskLevel(str,enum.Enum):
|
|
409
|
-
INFO = "Info"
|
|
410
|
-
LOW = "Low"
|
|
411
|
-
MEDIUM = "Medium"
|
|
412
|
-
HIGH = "High"
|
|
413
|
-
CRITICAL = "Critical"
|
|
414
|
-
|
|
415
410
|
class RiskSeverity(str,enum.Enum):
|
|
411
|
+
# Levels taken from the following documentation link
|
|
412
|
+
# https://docs.splunk.com/Documentation/ES/7.3.2/User/RiskScoring
|
|
413
|
+
# 20 - info (0-20 for us)
|
|
414
|
+
# 40 - low (21-40 for us)
|
|
415
|
+
# 60 - medium (41-60 for us)
|
|
416
|
+
# 80 - high (61-80 for us)
|
|
417
|
+
# 100 - critical (81 - 100 for us)
|
|
418
|
+
INFORMATIONAL = "informational"
|
|
416
419
|
LOW = "low"
|
|
417
420
|
MEDIUM = "medium"
|
|
418
421
|
HIGH = "high"
|
|
422
|
+
CRITICAL = "critical"
|
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
import re
|
|
3
|
-
from typing import
|
|
4
|
-
from pydantic import
|
|
5
|
-
if TYPE_CHECKING:
|
|
6
|
-
from contentctl.input.director import DirectorOutputDto
|
|
3
|
+
from typing import List, Any
|
|
4
|
+
from pydantic import computed_field, Field, ConfigDict,model_serializer
|
|
7
5
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
8
6
|
from contentctl.objects.enums import DataModel
|
|
9
7
|
from contentctl.objects.investigation_tags import InvestigationTags
|
|
10
|
-
|
|
8
|
+
from contentctl.objects.constants import (
|
|
9
|
+
CONTENTCTL_MAX_SEARCH_NAME_LENGTH,
|
|
10
|
+
CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE,
|
|
11
|
+
CONTENTCTL_MAX_STANZA_LENGTH
|
|
12
|
+
)
|
|
13
|
+
from contentctl.objects.config import CustomApp
|
|
11
14
|
|
|
12
15
|
# TODO (#266): disable the use_enum_values configuration
|
|
13
16
|
class Investigation(SecurityContentObject):
|
|
14
17
|
model_config = ConfigDict(use_enum_values=True,validate_default=False)
|
|
15
18
|
type: str = Field(...,pattern="^Investigation$")
|
|
16
19
|
datamodel: list[DataModel] = Field(...)
|
|
17
|
-
|
|
20
|
+
name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
|
|
18
21
|
search: str = Field(...)
|
|
19
22
|
how_to_implement: str = Field(...)
|
|
20
23
|
known_false_positives: str = Field(...)
|
|
@@ -27,11 +30,11 @@ class Investigation(SecurityContentObject):
|
|
|
27
30
|
@property
|
|
28
31
|
def inputs(self)->List[str]:
|
|
29
32
|
#Parse out and return all inputs from the searchj
|
|
30
|
-
inputs = []
|
|
33
|
+
inputs:List[str] = []
|
|
31
34
|
pattern = r"\$([^\s.]*)\$"
|
|
32
35
|
|
|
33
36
|
for input in re.findall(pattern, self.search):
|
|
34
|
-
inputs.append(input)
|
|
37
|
+
inputs.append(str(input))
|
|
35
38
|
|
|
36
39
|
return inputs
|
|
37
40
|
|
|
@@ -40,6 +43,16 @@ class Investigation(SecurityContentObject):
|
|
|
40
43
|
def lowercase_name(self)->str:
|
|
41
44
|
return self.name.replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower().replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower()
|
|
42
45
|
|
|
46
|
+
|
|
47
|
+
# This is a slightly modified version of the get_conf_stanza_name function from
|
|
48
|
+
# SecurityContentObject_Abstract
|
|
49
|
+
def get_response_task_name(self, app:CustomApp, max_stanza_length:int=CONTENTCTL_MAX_STANZA_LENGTH)->str:
|
|
50
|
+
stanza_name = CONTENTCTL_RESPONSE_TASK_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name)
|
|
51
|
+
if len(stanza_name) > max_stanza_length:
|
|
52
|
+
raise ValueError(f"conf stanza may only be {max_stanza_length} characters, "
|
|
53
|
+
f"but stanza was actually {len(stanza_name)} characters: '{stanza_name}' ")
|
|
54
|
+
return stanza_name
|
|
55
|
+
|
|
43
56
|
|
|
44
57
|
@model_serializer
|
|
45
58
|
def serialize_model(self):
|
|
@@ -66,12 +79,7 @@ class Investigation(SecurityContentObject):
|
|
|
66
79
|
|
|
67
80
|
|
|
68
81
|
def model_post_init(self, ctx:dict[str,Any]):
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
# raise ValueError("DirectorOutputDto was not passed in context of Detection model_post_init")
|
|
72
|
-
director: Optional[DirectorOutputDto] = ctx.get("output_dto",None)
|
|
82
|
+
# Ensure we link all stories this investigation references
|
|
83
|
+
# back to itself
|
|
73
84
|
for story in self.tags.analytic_story:
|
|
74
85
|
story.investigations.append(self)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
from typing import List
|
|
2
3
|
from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer
|
|
3
4
|
from contentctl.objects.story import Story
|
|
4
5
|
from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain
|
|
5
6
|
|
|
6
7
|
class InvestigationTags(BaseModel):
|
|
7
|
-
analytic_story:
|
|
8
|
-
product:
|
|
9
|
-
required_fields:
|
|
8
|
+
analytic_story: List[Story] = Field([],min_length=1)
|
|
9
|
+
product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1)
|
|
10
|
+
required_fields: List[str] = Field(min_length=1)
|
|
10
11
|
security_domain: SecurityDomain = Field(...)
|
|
11
12
|
|
|
12
13
|
|
contentctl/objects/lookup.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer
|
|
2
|
+
from pydantic import field_validator, ValidationInfo, model_validator, FilePath, model_serializer, Field, NonNegativeInt
|
|
3
3
|
from typing import TYPE_CHECKING, Optional, Any, Union
|
|
4
4
|
import re
|
|
5
5
|
import csv
|
|
6
|
+
import uuid
|
|
7
|
+
import datetime
|
|
6
8
|
if TYPE_CHECKING:
|
|
7
9
|
from contentctl.input.director import DirectorOutputDto
|
|
8
10
|
from contentctl.objects.config import validate
|
|
@@ -32,6 +34,11 @@ class Lookup(SecurityContentObject):
|
|
|
32
34
|
match_type: Optional[str] = None
|
|
33
35
|
min_matches: Optional[int] = None
|
|
34
36
|
case_sensitive_match: Optional[bool] = None
|
|
37
|
+
# TODO: Add id field to all lookup ymls
|
|
38
|
+
id: uuid.UUID = Field(default_factory=uuid.uuid4)
|
|
39
|
+
date: datetime.date = Field(datetime.date.today())
|
|
40
|
+
author: str = Field("NO AUTHOR DEFINED",max_length=255)
|
|
41
|
+
version: NonNegativeInt = 1
|
|
35
42
|
|
|
36
43
|
|
|
37
44
|
@model_serializer
|
contentctl/objects/macro.py
CHANGED
|
@@ -3,12 +3,13 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
from typing import TYPE_CHECKING, List
|
|
5
5
|
import re
|
|
6
|
-
from pydantic import Field, model_serializer
|
|
6
|
+
from pydantic import Field, model_serializer, NonNegativeInt
|
|
7
|
+
import uuid
|
|
8
|
+
import datetime
|
|
7
9
|
if TYPE_CHECKING:
|
|
8
10
|
from contentctl.input.director import DirectorOutputDto
|
|
9
11
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
10
12
|
|
|
11
|
-
|
|
12
13
|
#The following macros are included in commonly-installed apps.
|
|
13
14
|
#As such, we will ignore if they are missing from our app.
|
|
14
15
|
#Included in
|
|
@@ -22,7 +23,11 @@ MACROS_TO_IGNORE.add("cim_corporate_web_domain_search") #Part of CIM/Splunk_SA_C
|
|
|
22
23
|
class Macro(SecurityContentObject):
|
|
23
24
|
definition: str = Field(..., min_length=1)
|
|
24
25
|
arguments: List[str] = Field([])
|
|
25
|
-
|
|
26
|
+
# TODO: Add id field to all macro ymls
|
|
27
|
+
id: uuid.UUID = Field(default_factory=uuid.uuid4)
|
|
28
|
+
date: datetime.date = Field(datetime.date.today())
|
|
29
|
+
author: str = Field("NO AUTHOR DEFINED",max_length=255)
|
|
30
|
+
version: NonNegativeInt = 1
|
|
26
31
|
|
|
27
32
|
|
|
28
33
|
|
|
@@ -49,10 +54,15 @@ class Macro(SecurityContentObject):
|
|
|
49
54
|
#If a comment ENDS in a macro, for example ```this is a comment with a macro `macro_here````
|
|
50
55
|
#then there is a small edge case where the regex below does not work properly. If that is
|
|
51
56
|
#the case, we edit the search slightly to insert a space
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
57
|
+
if re.findall(r"\`\`\`\`", text_field):
|
|
58
|
+
raise ValueError("Search contained four or more '`' characters in a row which is invalid SPL"
|
|
59
|
+
"This may have occurred when a macro was commented out.\n"
|
|
60
|
+
"Please ammend your search to remove the substring '````'")
|
|
55
61
|
|
|
62
|
+
# replace all the macros with a space
|
|
63
|
+
text_field = re.sub(r"\`\`\`[\s\S]*?\`\`\`", " ", text_field)
|
|
64
|
+
|
|
65
|
+
|
|
56
66
|
macros_to_get = re.findall(r'`([^\s]+)`', text_field)
|
|
57
67
|
#If macros take arguments, stop at the first argument. We just want the name of the macro
|
|
58
68
|
macros_to_get = set([macro[:macro.find('(')] if macro.find('(') != -1 else macro for macro in macros_to_get])
|
|
@@ -62,4 +72,3 @@ class Macro(SecurityContentObject):
|
|
|
62
72
|
macros_to_get -= macros_to_ignore
|
|
63
73
|
return Macro.mapNamesToSecurityContentObjects(list(macros_to_get), director)
|
|
64
74
|
|
|
65
|
-
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from pydantic import BaseModel
|
|
1
|
+
from pydantic import ConfigDict, BaseModel
|
|
2
2
|
|
|
3
3
|
from contentctl.objects.detection import Detection
|
|
4
4
|
|
|
@@ -11,10 +11,11 @@ class NotableEvent(BaseModel):
|
|
|
11
11
|
# The search ID that found that generated this risk event
|
|
12
12
|
orig_sid: str
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
extra
|
|
14
|
+
# Allowing fields that aren't explicitly defined to be passed since some of the risk event's
|
|
15
|
+
# fields vary depending on the SPL which generated them
|
|
16
|
+
model_config = ConfigDict(
|
|
17
|
+
extra='allow'
|
|
18
|
+
)
|
|
18
19
|
|
|
19
20
|
def validate_against_detection(self, detection: Detection) -> None:
|
|
20
21
|
raise NotImplementedError()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
import json
|
|
3
3
|
|
|
4
|
-
from pydantic import BaseModel,
|
|
4
|
+
from pydantic import BaseModel, field_validator
|
|
5
5
|
|
|
6
6
|
from contentctl.objects.risk_object import RiskObject
|
|
7
7
|
from contentctl.objects.threat_object import ThreatObject
|
|
@@ -21,11 +21,11 @@ class RiskAnalysisAction(BaseModel):
|
|
|
21
21
|
risk_objects: list[RiskObject]
|
|
22
22
|
message: str
|
|
23
23
|
|
|
24
|
-
@
|
|
24
|
+
@field_validator("message", mode="before")
|
|
25
25
|
@classmethod
|
|
26
|
-
def _validate_message(cls, v
|
|
26
|
+
def _validate_message(cls, v: Any) -> str:
|
|
27
27
|
"""
|
|
28
|
-
Validate
|
|
28
|
+
Validate message and derive if None
|
|
29
29
|
"""
|
|
30
30
|
if v is None:
|
|
31
31
|
raise ValueError(
|
contentctl/objects/risk_event.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import re
|
|
2
|
+
from functools import cached_property
|
|
2
3
|
|
|
3
|
-
from pydantic import BaseModel, Field, PrivateAttr, field_validator, computed_field
|
|
4
|
-
|
|
4
|
+
from pydantic import ConfigDict, BaseModel, Field, PrivateAttr, field_validator, computed_field
|
|
5
5
|
from contentctl.objects.errors import ValidationFailed
|
|
6
6
|
from contentctl.objects.detection import Detection
|
|
7
7
|
from contentctl.objects.observable import Observable
|
|
@@ -85,10 +85,11 @@ class RiskEvent(BaseModel):
|
|
|
85
85
|
# Private attribute caching the observable this RiskEvent is mapped to
|
|
86
86
|
_matched_observable: Observable | None = PrivateAttr(default=None)
|
|
87
87
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
extra
|
|
88
|
+
# Allowing fields that aren't explicitly defined to be passed since some of the risk event's
|
|
89
|
+
# fields vary depending on the SPL which generated them
|
|
90
|
+
model_config = ConfigDict(
|
|
91
|
+
extra="allow"
|
|
92
|
+
)
|
|
92
93
|
|
|
93
94
|
@field_validator("annotations_mitre_attack", "analyticstories", mode="before")
|
|
94
95
|
@classmethod
|
|
@@ -103,7 +104,7 @@ class RiskEvent(BaseModel):
|
|
|
103
104
|
return [v]
|
|
104
105
|
|
|
105
106
|
@computed_field
|
|
106
|
-
@
|
|
107
|
+
@cached_property
|
|
107
108
|
def source_field_name(self) -> str:
|
|
108
109
|
"""
|
|
109
110
|
A cached derivation of the source field name the risk event corresponds to in the relevant
|
contentctl/objects/story.py
CHANGED
|
@@ -8,26 +8,14 @@ if TYPE_CHECKING:
|
|
|
8
8
|
from contentctl.objects.investigation import Investigation
|
|
9
9
|
from contentctl.objects.baseline import Baseline
|
|
10
10
|
from contentctl.objects.data_source import DataSource
|
|
11
|
+
from contentctl.objects.config import CustomApp
|
|
11
12
|
|
|
12
13
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
#from contentctl.objects.investigation import Investigation
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
15
|
class Story(SecurityContentObject):
|
|
23
16
|
narrative: str = Field(...)
|
|
24
17
|
tags: StoryTags = Field(...)
|
|
25
18
|
|
|
26
|
-
# enrichments
|
|
27
|
-
#detection_names: List[str] = []
|
|
28
|
-
#investigation_names: List[str] = []
|
|
29
|
-
#baseline_names: List[str] = []
|
|
30
|
-
|
|
31
19
|
# These are updated when detection and investigation objects are created.
|
|
32
20
|
# Specifically in the model_post_init functions
|
|
33
21
|
detections:List[Detection] = []
|
|
@@ -46,9 +34,9 @@ class Story(SecurityContentObject):
|
|
|
46
34
|
return sorted(list(data_source_objects))
|
|
47
35
|
|
|
48
36
|
|
|
49
|
-
def storyAndInvestigationNamesWithApp(self,
|
|
50
|
-
return [
|
|
51
|
-
[
|
|
37
|
+
def storyAndInvestigationNamesWithApp(self, app:CustomApp)->List[str]:
|
|
38
|
+
return [detection.get_conf_stanza_name(app) for detection in self.detections] + \
|
|
39
|
+
[investigation.get_response_task_name(app) for investigation in self.investigations]
|
|
52
40
|
|
|
53
41
|
@model_serializer
|
|
54
42
|
def serialize_model(self):
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from pydantic import BaseModel, Field, field_validator
|
|
2
|
+
from typing import Annotated
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
# Alert Suppression/Throttling settings have been taken from
|
|
6
|
+
# https://docs.splunk.com/Documentation/Splunk/9.2.2/Admin/Savedsearchesconf
|
|
7
|
+
class Throttling(BaseModel):
|
|
8
|
+
fields: list[str] = Field(..., description="The list of fields to throttle on. These fields MUST occur in the search.", min_length=1)
|
|
9
|
+
period: Annotated[str,Field(pattern="^[0-9]+[smh]$")] = Field(..., description="How often the alert should be triggered. "
|
|
10
|
+
"This may be specified in seconds, minutes, or hours. "
|
|
11
|
+
"For example, if an alert should be triggered once a day,"
|
|
12
|
+
" it may be specified in seconds (86400s), minutes (1440m), or hours import (24h).")
|
|
13
|
+
|
|
14
|
+
@field_validator("fields")
|
|
15
|
+
def no_spaces_in_fields(cls, v:list[str])->list[str]:
|
|
16
|
+
for field in v:
|
|
17
|
+
if ' ' in field:
|
|
18
|
+
raise ValueError("Spaces are not presently supported in 'alert.suppress.fields' / throttling fields in conf files. "
|
|
19
|
+
"The field '{field}' has a space in it. If this is a blocker, please raise this as an issue on the Project.")
|
|
20
|
+
return v
|
|
21
|
+
|
|
22
|
+
def conf_formatted_fields(self)->str:
|
|
23
|
+
'''
|
|
24
|
+
TODO:
|
|
25
|
+
The field alert.suppress.fields is defined as follows:
|
|
26
|
+
alert.suppress.fields = <comma-delimited-field-list>
|
|
27
|
+
* List of fields to use when suppressing per-result alerts. This field *must*
|
|
28
|
+
be specified if the digest mode is disabled and suppression is enabled.
|
|
29
|
+
|
|
30
|
+
In order to support fields with spaces in them, we may need to wrap each
|
|
31
|
+
field in "".
|
|
32
|
+
This function returns a properly formatted value, where each field
|
|
33
|
+
is wrapped in "" and separated with a comma. For example, the fields
|
|
34
|
+
["field1", "field 2", "field3"] would be returned as the string
|
|
35
|
+
|
|
36
|
+
"field1","field 2","field3
|
|
37
|
+
|
|
38
|
+
However, for now, we will error on fields with spaces and simply
|
|
39
|
+
separate with commas
|
|
40
|
+
'''
|
|
41
|
+
|
|
42
|
+
return ",".join(self.fields)
|
|
43
|
+
|
|
44
|
+
# The following may be used once we determine proper support
|
|
45
|
+
# for fields with spaces
|
|
46
|
+
#return ",".join([f'"{field}"' for field in self.fields])
|
contentctl/output/conf_output.py
CHANGED
contentctl/output/conf_writer.py
CHANGED
|
@@ -8,6 +8,7 @@ from xmlrpc.client import APPLICATION_ERROR
|
|
|
8
8
|
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
|
9
9
|
import pathlib
|
|
10
10
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
11
|
+
from contentctl.objects.dashboard import Dashboard
|
|
11
12
|
from contentctl.objects.config import build
|
|
12
13
|
import xml.etree.ElementTree as ET
|
|
13
14
|
|
|
@@ -61,7 +62,7 @@ class ConfWriter():
|
|
|
61
62
|
j2_env = ConfWriter.getJ2Environment()
|
|
62
63
|
template = j2_env.get_template(template_name)
|
|
63
64
|
|
|
64
|
-
output = template.render(objects=objects,
|
|
65
|
+
output = template.render(objects=objects, app=config.app, currentDate=datetime.datetime.now(datetime.UTC).date().isoformat())
|
|
65
66
|
|
|
66
67
|
output_path = config.getPackageDirectoryPath()/app_output_path
|
|
67
68
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -94,7 +95,7 @@ class ConfWriter():
|
|
|
94
95
|
j2_env = ConfWriter.getJ2Environment()
|
|
95
96
|
template = j2_env.get_template(template_name)
|
|
96
97
|
|
|
97
|
-
output = template.render(objects=objects,
|
|
98
|
+
output = template.render(objects=objects, app=config.app)
|
|
98
99
|
|
|
99
100
|
output_path = config.getPackageDirectoryPath()/app_output_path
|
|
100
101
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -107,6 +108,22 @@ class ConfWriter():
|
|
|
107
108
|
|
|
108
109
|
|
|
109
110
|
|
|
111
|
+
@staticmethod
|
|
112
|
+
def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.Path]:
|
|
113
|
+
written_files:set[pathlib.Path] = set()
|
|
114
|
+
for dashboard in dashboards:
|
|
115
|
+
output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config)
|
|
116
|
+
# Check that the full output path does not exist so that we are not having an
|
|
117
|
+
# name collision with a file in app_template
|
|
118
|
+
if (config.getPackageDirectoryPath()/output_file_path).exists():
|
|
119
|
+
raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?")
|
|
120
|
+
|
|
121
|
+
ConfWriter.writeXmlFileHeader(output_file_path, config)
|
|
122
|
+
dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config)
|
|
123
|
+
ConfWriter.validateXmlFile(config.getPackageDirectoryPath()/output_file_path)
|
|
124
|
+
written_files.add(output_file_path)
|
|
125
|
+
return written_files
|
|
126
|
+
|
|
110
127
|
|
|
111
128
|
@staticmethod
|
|
112
129
|
def writeXmlFileHeader(app_output_path:pathlib.Path, config: build) -> None:
|
|
@@ -142,7 +159,7 @@ class ConfWriter():
|
|
|
142
159
|
j2_env = ConfWriter.getJ2Environment()
|
|
143
160
|
|
|
144
161
|
template = j2_env.get_template(template_name)
|
|
145
|
-
output = template.render(objects=objects,
|
|
162
|
+
output = template.render(objects=objects, app=config.app)
|
|
146
163
|
|
|
147
164
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
148
165
|
with open(output_path, 'a') as f:
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
|
|
4
4
|
{% for detection in objects %}
|
|
5
5
|
{% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %}
|
|
6
|
-
[savedsearch://{{
|
|
6
|
+
[savedsearch://{{ detection.get_conf_stanza_name(app) }}]
|
|
7
7
|
type = detection
|
|
8
8
|
asset_type = {{ detection.tags.asset_type.value }}
|
|
9
9
|
confidence = medium
|
|
10
|
-
explanation = {{ detection.description | escapeNewlines() }}
|
|
10
|
+
explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }}
|
|
11
11
|
{% if detection.how_to_implement is defined %}
|
|
12
12
|
how_to_implement = {{ detection.how_to_implement | escapeNewlines() }}
|
|
13
13
|
{% else %}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
|
|
2
2
|
### RESPONSE TASKS ###
|
|
3
3
|
|
|
4
|
-
{% for
|
|
5
|
-
{% if (
|
|
6
|
-
[savedsearch://{{
|
|
4
|
+
{% for investigation in objects %}
|
|
5
|
+
{% if (investigation.type == 'Investigation') %}
|
|
6
|
+
[savedsearch://{{ investigation.get_response_task_name(app) }}]
|
|
7
7
|
type = investigation
|
|
8
8
|
explanation = none
|
|
9
|
-
{% if
|
|
10
|
-
how_to_implement = {{
|
|
9
|
+
{% if investigation.how_to_implement is defined %}
|
|
10
|
+
how_to_implement = {{ investigation.how_to_implement | escapeNewlines() }}
|
|
11
11
|
{% else %}
|
|
12
12
|
how_to_implement = none
|
|
13
13
|
{% endif %}
|
|
@@ -10,7 +10,7 @@ version = {{ story.version }}
|
|
|
10
10
|
references = {{ story.getReferencesListForJson() | tojson }}
|
|
11
11
|
maintainers = [{"company": "{{ story.author_company }}", "email": "{{ story.author_email }}", "name": "{{ story.author_name }}"}]
|
|
12
12
|
spec_version = 3
|
|
13
|
-
searches = {{ story.storyAndInvestigationNamesWithApp(
|
|
13
|
+
searches = {{ story.storyAndInvestigationNamesWithApp(app) | tojson }}
|
|
14
14
|
description = {{ story.description | escapeNewlines() }}
|
|
15
15
|
{% if story.narrative is defined %}
|
|
16
16
|
narrative = {{ story.narrative | escapeNewlines() }}
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
### {{
|
|
3
|
+
### {{app.label}} BASELINES ###
|
|
4
4
|
|
|
5
5
|
{% for detection in objects %}
|
|
6
6
|
{% if (detection.type == 'Baseline') %}
|
|
7
|
-
[{{
|
|
7
|
+
[{{ detection.get_conf_stanza_name(app) }}]
|
|
8
8
|
action.escu = 0
|
|
9
9
|
action.escu.enabled = 1
|
|
10
10
|
action.escu.search_type = support
|
|
11
|
-
action.escu.full_search_name = {{APP_NAME}} - {{ detection.name }}
|
|
12
11
|
description = {{ detection.description | escapeNewlines() }}
|
|
13
12
|
action.escu.creation_date = {{ detection.date }}
|
|
14
13
|
action.escu.modification_date = {{ detection.date }}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
### {{
|
|
1
|
+
### {{app.label}} DETECTIONS ###
|
|
2
2
|
|
|
3
3
|
{% for detection in objects %}
|
|
4
4
|
{% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %}
|
|
5
|
-
[{{
|
|
5
|
+
[{{ detection.get_conf_stanza_name(app) }}]
|
|
6
6
|
action.escu = 0
|
|
7
7
|
action.escu.enabled = 1
|
|
8
8
|
{% if detection.status == "deprecated" %}
|
|
@@ -28,7 +28,6 @@ action.escu.known_false_positives = None
|
|
|
28
28
|
action.escu.creation_date = {{ detection.date }}
|
|
29
29
|
action.escu.modification_date = {{ detection.date }}
|
|
30
30
|
action.escu.confidence = high
|
|
31
|
-
action.escu.full_search_name = {{APP_NAME}} - {{ detection.name }} - Rule
|
|
32
31
|
action.escu.search_type = detection
|
|
33
32
|
{% if detection.tags.product is defined %}
|
|
34
33
|
action.escu.product = {{ detection.tags.product | tojson }}
|
|
@@ -57,7 +56,7 @@ cron_schedule = {{ detection.deployment.scheduling.cron_schedule }}
|
|
|
57
56
|
dispatch.earliest_time = {{ detection.deployment.scheduling.earliest_time }}
|
|
58
57
|
dispatch.latest_time = {{ detection.deployment.scheduling.latest_time }}
|
|
59
58
|
action.correlationsearch.enabled = 1
|
|
60
|
-
action.correlationsearch.label = {{
|
|
59
|
+
action.correlationsearch.label = {{ detection.get_action_dot_correlationsearch_dot_label(app) }}
|
|
61
60
|
action.correlationsearch.annotations = {{ detection.annotations | tojson }}
|
|
62
61
|
action.correlationsearch.metadata = {{ detection.metadata | tojson }}
|
|
63
62
|
{% if detection.deployment.scheduling.schedule_window is defined %}
|
|
@@ -72,7 +71,7 @@ action.notable.param.nes_fields = {{ detection.nes_fields }}
|
|
|
72
71
|
action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}}
|
|
73
72
|
action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%}
|
|
74
73
|
action.notable.param.security_domain = {{ detection.tags.security_domain.value }}
|
|
75
|
-
action.notable.param.severity =
|
|
74
|
+
action.notable.param.severity = {{ detection.tags.severity.value }}
|
|
76
75
|
{% endif %}
|
|
77
76
|
{% if detection.deployment.alert_action.email %}
|
|
78
77
|
action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }}
|
|
@@ -107,8 +106,14 @@ relation = greater than
|
|
|
107
106
|
quantity = 0
|
|
108
107
|
realtime_schedule = 0
|
|
109
108
|
is_visible = false
|
|
109
|
+
{% if detection.tags.throttling %}
|
|
110
|
+
alert.suppress = true
|
|
111
|
+
alert.suppress.fields = {{ detection.tags.throttling.conf_formatted_fields() }}
|
|
112
|
+
alert.suppress.period = {{ detection.tags.throttling.period }}
|
|
113
|
+
{% endif %}
|
|
110
114
|
search = {{ detection.search | escapeNewlines() }}
|
|
111
|
-
|
|
115
|
+
action.notable.param.drilldown_searches = {{ detection.drilldowns_in_JSON | tojson | escapeNewlines() }}
|
|
112
116
|
{% endif %}
|
|
117
|
+
|
|
113
118
|
{% endfor %}
|
|
114
|
-
### END {{
|
|
119
|
+
### END {{ app.label }} DETECTIONS ###
|
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
### {{
|
|
3
|
+
### {{app.label}} RESPONSE TASKS ###
|
|
4
4
|
|
|
5
5
|
{% for detection in objects %}
|
|
6
6
|
{% if (detection.type == 'Investigation') %}
|
|
7
7
|
{% if detection.search is defined %}
|
|
8
|
-
[{{
|
|
8
|
+
[{{ detection.get_response_task_name(app) }}]
|
|
9
9
|
action.escu = 0
|
|
10
10
|
action.escu.enabled = 1
|
|
11
11
|
action.escu.search_type = investigative
|
|
12
|
-
action.escu.full_search_name = {{APP_NAME}} - {{ detection.name }} - Response Task
|
|
13
12
|
description = {{ detection.description | escapeNewlines() }}
|
|
14
13
|
action.escu.creation_date = {{ detection.date }}
|
|
15
14
|
action.escu.modification_date = {{ detection.date }}
|
|
@@ -35,4 +34,4 @@ search = {{ detection.search | escapeNewlines() }}
|
|
|
35
34
|
{% endfor %}
|
|
36
35
|
|
|
37
36
|
|
|
38
|
-
### END {{
|
|
37
|
+
### END {{ app.label }} RESPONSE TASKS ###
|
|
@@ -29,6 +29,15 @@ references:
|
|
|
29
29
|
- https://attack.mitre.org/techniques/T1560/001/
|
|
30
30
|
- https://www.microsoft.com/security/blog/2021/01/20/deep-dive-into-the-solorigate-second-stage-activation-from-sunburst-to-teardrop-and-raindrop/
|
|
31
31
|
- https://thedfirreport.com/2021/01/31/bazar-no-ryuk/
|
|
32
|
+
drilldown_searches:
|
|
33
|
+
- name: View the detection results for $user$ and $dest$
|
|
34
|
+
search: '%original_detection_search% | search user = $user$ dest = $dest$'
|
|
35
|
+
earliest_offset: $info_min_time$
|
|
36
|
+
latest_offset: $info_max_time$
|
|
37
|
+
- name: View risk events for the last 7 days for $user$ and $dest$
|
|
38
|
+
search: '| from datamodel Risk.All_Risk | search normalized_risk_object IN ($user$, $dest$) starthoursago=168 endhoursago=1 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`'
|
|
39
|
+
earliest_offset: $info_min_time$
|
|
40
|
+
latest_offset: $info_max_time$
|
|
32
41
|
tags:
|
|
33
42
|
analytic_story:
|
|
34
43
|
- Cobalt Strike
|
|
@@ -80,4 +89,4 @@ tests:
|
|
|
80
89
|
attack_data:
|
|
81
90
|
- data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1560.001/archive_utility/windows-sysmon.log
|
|
82
91
|
source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational
|
|
83
|
-
sourcetype: xmlwineventlog
|
|
92
|
+
sourcetype: xmlwineventlog
|