contentctl 5.2.0__py3-none-any.whl → 5.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/build.py +5 -43
- contentctl/actions/detection_testing/DetectionTestingManager.py +64 -24
- contentctl/actions/detection_testing/GitService.py +4 -1
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +146 -42
- contentctl/actions/detection_testing/views/DetectionTestingView.py +5 -6
- contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +2 -0
- contentctl/actions/initialize.py +35 -9
- contentctl/actions/release_notes.py +14 -12
- contentctl/actions/test.py +16 -20
- contentctl/actions/validate.py +9 -16
- contentctl/helper/utils.py +69 -20
- contentctl/input/director.py +147 -119
- contentctl/input/yml_reader.py +39 -27
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +95 -21
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +548 -8
- contentctl/objects/baseline.py +24 -6
- contentctl/objects/config.py +32 -8
- contentctl/objects/content_versioning_service.py +508 -0
- contentctl/objects/correlation_search.py +53 -63
- contentctl/objects/dashboard.py +15 -1
- contentctl/objects/data_source.py +13 -1
- contentctl/objects/deployment.py +23 -9
- contentctl/objects/detection.py +2 -0
- contentctl/objects/enums.py +28 -18
- contentctl/objects/investigation.py +40 -20
- contentctl/objects/lookup.py +62 -6
- contentctl/objects/macro.py +19 -4
- contentctl/objects/playbook.py +16 -2
- contentctl/objects/rba.py +1 -33
- contentctl/objects/removed_security_content_object.py +50 -0
- contentctl/objects/security_content_object.py +1 -0
- contentctl/objects/story.py +37 -5
- contentctl/output/api_json_output.py +5 -3
- contentctl/output/conf_output.py +9 -1
- contentctl/output/runtime_csv_writer.py +111 -0
- contentctl/output/svg_output.py +4 -5
- contentctl/output/templates/savedsearches_detections.j2 +2 -6
- {contentctl-5.2.0.dist-info → contentctl-5.3.1.dist-info}/METADATA +4 -3
- {contentctl-5.2.0.dist-info → contentctl-5.3.1.dist-info}/RECORD +42 -40
- {contentctl-5.2.0.dist-info → contentctl-5.3.1.dist-info}/WHEEL +1 -1
- contentctl/output/data_source_writer.py +0 -52
- {contentctl-5.2.0.dist-info → contentctl-5.3.1.dist-info}/LICENSE.md +0 -0
- {contentctl-5.2.0.dist-info → contentctl-5.3.1.dist-info}/entry_points.txt +0 -0
|
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Annotated, Any, List, Optional, Union
|
|
|
8
8
|
from pydantic import (
|
|
9
9
|
Field,
|
|
10
10
|
FilePath,
|
|
11
|
+
HttpUrl,
|
|
11
12
|
ValidationInfo,
|
|
12
13
|
computed_field,
|
|
13
14
|
field_validator,
|
|
@@ -24,6 +25,7 @@ if TYPE_CHECKING:
|
|
|
24
25
|
from contentctl.objects.config import CustomApp
|
|
25
26
|
|
|
26
27
|
import datetime
|
|
28
|
+
from functools import cached_property
|
|
27
29
|
|
|
28
30
|
from contentctl.enrichments.cve_enrichment import CveEnrichmentObj
|
|
29
31
|
from contentctl.objects.base_test_result import TestResultStatus
|
|
@@ -39,14 +41,15 @@ from contentctl.objects.detection_tags import DetectionTags
|
|
|
39
41
|
from contentctl.objects.drilldown import DRILLDOWN_SEARCH_PLACEHOLDER, Drilldown
|
|
40
42
|
from contentctl.objects.enums import (
|
|
41
43
|
AnalyticsType,
|
|
44
|
+
ContentStatus,
|
|
42
45
|
DataModel,
|
|
43
|
-
DetectionStatus,
|
|
44
46
|
NistCategory,
|
|
45
47
|
ProvidingTechnology,
|
|
48
|
+
RiskSeverity,
|
|
46
49
|
)
|
|
47
50
|
from contentctl.objects.integration_test import IntegrationTest
|
|
48
51
|
from contentctl.objects.manual_test import ManualTest
|
|
49
|
-
from contentctl.objects.rba import RBAObject
|
|
52
|
+
from contentctl.objects.rba import RBAObject, RiskScoreValue_Type
|
|
50
53
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
51
54
|
from contentctl.objects.test_group import TestGroup
|
|
52
55
|
from contentctl.objects.unit_test import UnitTest
|
|
@@ -59,13 +62,61 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
59
62
|
name: str = Field(..., max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
|
|
60
63
|
# contentType: SecurityContentType = SecurityContentType.detections
|
|
61
64
|
type: AnalyticsType = Field(...)
|
|
62
|
-
status:
|
|
65
|
+
status: ContentStatus
|
|
63
66
|
data_source: list[str] = []
|
|
64
67
|
tags: DetectionTags = Field(...)
|
|
65
68
|
search: str = Field(...)
|
|
66
69
|
how_to_implement: str = Field(..., min_length=4)
|
|
67
70
|
known_false_positives: str = Field(..., min_length=4)
|
|
68
71
|
rba: Optional[RBAObject] = Field(default=None)
|
|
72
|
+
|
|
73
|
+
@computed_field
|
|
74
|
+
@property
|
|
75
|
+
def risk_score(self) -> RiskScoreValue_Type:
|
|
76
|
+
# First get the maximum score associated with
|
|
77
|
+
# a risk object. If there are no objects, then
|
|
78
|
+
# we should throw an exception.
|
|
79
|
+
if self.rba is None or len(self.rba.risk_objects) == 0:
|
|
80
|
+
raise Exception(
|
|
81
|
+
"There must be at least one Risk Object present to get Severity."
|
|
82
|
+
)
|
|
83
|
+
return max([risk_object.score for risk_object in self.rba.risk_objects])
|
|
84
|
+
|
|
85
|
+
@computed_field
|
|
86
|
+
@property
|
|
87
|
+
def severity(self) -> RiskSeverity:
|
|
88
|
+
"""
|
|
89
|
+
Severity is required for notables (but not risk objects).
|
|
90
|
+
In the contentctl codebase, instead of requiring an additional
|
|
91
|
+
field to be added to the YMLs, we derive the severity from the
|
|
92
|
+
HIGHEST risk score of any risk object that is part of this detection.
|
|
93
|
+
However, if a detection does not have a risk object but still has a notable,
|
|
94
|
+
we will use a default value of high. This only impact Correlation searches. As
|
|
95
|
+
TTP searches, which also generate notables, must also have risk object(s)
|
|
96
|
+
"""
|
|
97
|
+
try:
|
|
98
|
+
risk_score = self.risk_score
|
|
99
|
+
except Exception:
|
|
100
|
+
# This object does not have any RBA objects,
|
|
101
|
+
# hence no disk score is returned. So we will
|
|
102
|
+
# return the defualt value of high
|
|
103
|
+
return RiskSeverity.HIGH
|
|
104
|
+
|
|
105
|
+
if 0 <= risk_score <= 20:
|
|
106
|
+
return RiskSeverity.INFORMATIONAL
|
|
107
|
+
elif 20 < risk_score <= 40:
|
|
108
|
+
return RiskSeverity.LOW
|
|
109
|
+
elif 40 < risk_score <= 60:
|
|
110
|
+
return RiskSeverity.MEDIUM
|
|
111
|
+
elif 60 < risk_score <= 80:
|
|
112
|
+
return RiskSeverity.HIGH
|
|
113
|
+
elif 80 < risk_score <= 100:
|
|
114
|
+
return RiskSeverity.CRITICAL
|
|
115
|
+
else:
|
|
116
|
+
raise Exception(
|
|
117
|
+
f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}"
|
|
118
|
+
)
|
|
119
|
+
|
|
69
120
|
explanation: None | str = Field(
|
|
70
121
|
default=None,
|
|
71
122
|
exclude=True, # Don't serialize this value when dumping the object
|
|
@@ -99,11 +150,36 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
99
150
|
description="A list of Drilldowns that should be included with this search",
|
|
100
151
|
)
|
|
101
152
|
|
|
102
|
-
|
|
153
|
+
@field_validator("status", mode="after")
|
|
154
|
+
@classmethod
|
|
155
|
+
def NarrowStatus(cls, status: ContentStatus) -> ContentStatus:
|
|
156
|
+
return cls.NarrowStatusTemplate(
|
|
157
|
+
status,
|
|
158
|
+
[
|
|
159
|
+
ContentStatus.experimental,
|
|
160
|
+
ContentStatus.production,
|
|
161
|
+
ContentStatus.deprecated,
|
|
162
|
+
],
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
@classmethod
|
|
166
|
+
def containing_folder(cls) -> pathlib.Path:
|
|
167
|
+
return pathlib.Path("detections")
|
|
168
|
+
|
|
169
|
+
@computed_field
|
|
170
|
+
@cached_property
|
|
171
|
+
def researchSiteLink(self) -> HttpUrl:
|
|
172
|
+
return HttpUrl(url=f"https://research.splunk.com/{self.source}/{self.id}") # type: ignore
|
|
173
|
+
|
|
174
|
+
@classmethod
|
|
175
|
+
def static_get_conf_stanza_name(cls, name: str, app: CustomApp) -> str:
|
|
176
|
+
"""
|
|
177
|
+
This is exposed as a static method since it may need to be used for SecurityContentObject which does not
|
|
178
|
+
pass all currenty validations - most notable Deprecated content.
|
|
179
|
+
"""
|
|
103
180
|
stanza_name = CONTENTCTL_DETECTION_STANZA_NAME_FORMAT_TEMPLATE.format(
|
|
104
|
-
app_label=app.label, detection_name=
|
|
181
|
+
app_label=app.label, detection_name=name
|
|
105
182
|
)
|
|
106
|
-
self.check_conf_stanza_max_length(stanza_name)
|
|
107
183
|
return stanza_name
|
|
108
184
|
|
|
109
185
|
def get_action_dot_correlationsearch_dot_label(
|
|
@@ -219,7 +295,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
219
295
|
# https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
|
|
220
296
|
|
|
221
297
|
# Skip tests for non-production detections
|
|
222
|
-
if self.status !=
|
|
298
|
+
if self.status != ContentStatus.production:
|
|
223
299
|
self.skip_all_tests(
|
|
224
300
|
f"TEST SKIPPED: Detection is non-production ({self.status})"
|
|
225
301
|
)
|
|
@@ -406,7 +482,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
406
482
|
# break the `inspect` action.
|
|
407
483
|
return {
|
|
408
484
|
"detection_id": str(self.id),
|
|
409
|
-
"deprecated": "1" if self.status ==
|
|
485
|
+
"deprecated": "1" if self.status == ContentStatus.deprecated else "0", # type: ignore
|
|
410
486
|
"detection_version": str(self.version),
|
|
411
487
|
"publish_time": datetime.datetime(
|
|
412
488
|
self.date.year,
|
|
@@ -435,12 +511,10 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
435
511
|
"datamodel": self.datamodel,
|
|
436
512
|
"source": self.source,
|
|
437
513
|
"nes_fields": self.nes_fields,
|
|
514
|
+
"rba": self.rba or {},
|
|
438
515
|
}
|
|
439
|
-
if self.
|
|
440
|
-
model["risk_severity"] = self.
|
|
441
|
-
model["tags"]["risk_score"] = self.rba.risk_score
|
|
442
|
-
else:
|
|
443
|
-
model["tags"]["risk_score"] = 0
|
|
516
|
+
if self.deployment.alert_action.notable:
|
|
517
|
+
model["risk_severity"] = self.severity
|
|
444
518
|
|
|
445
519
|
# Only a subset of macro fields are required:
|
|
446
520
|
all_macros: list[dict[str, str | list[str]]] = []
|
|
@@ -473,7 +547,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
473
547
|
{
|
|
474
548
|
"name": lookup.name,
|
|
475
549
|
"description": lookup.description,
|
|
476
|
-
"filename": lookup.filename.name,
|
|
550
|
+
"filename": lookup.filename.name, # This does not cause an issue for RuntimeCSV type because they are not used by any detections
|
|
477
551
|
"default_match": lookup.default_match,
|
|
478
552
|
"case_sensitive_match": "true"
|
|
479
553
|
if lookup.case_sensitive_match
|
|
@@ -542,7 +616,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
542
616
|
|
|
543
617
|
if (
|
|
544
618
|
self.type == AnalyticsType.Hunting
|
|
545
|
-
or self.status !=
|
|
619
|
+
or self.status != ContentStatus.production
|
|
546
620
|
):
|
|
547
621
|
# No additional check need to happen on the potential drilldowns.
|
|
548
622
|
pass
|
|
@@ -694,13 +768,13 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
694
768
|
if v is False:
|
|
695
769
|
return v
|
|
696
770
|
|
|
697
|
-
status =
|
|
771
|
+
status = ContentStatus(info.data.get("status"))
|
|
698
772
|
searchType = AnalyticsType(info.data.get("type"))
|
|
699
773
|
errors: list[str] = []
|
|
700
|
-
if status !=
|
|
774
|
+
if status != ContentStatus.production:
|
|
701
775
|
errors.append(
|
|
702
776
|
f"status is '{status.name}'. Detections that are enabled by default MUST be "
|
|
703
|
-
f"'{
|
|
777
|
+
f"'{ContentStatus.production}'"
|
|
704
778
|
)
|
|
705
779
|
|
|
706
780
|
if searchType not in [
|
|
@@ -830,7 +904,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
830
904
|
f"the search: {missing_fields}"
|
|
831
905
|
)
|
|
832
906
|
|
|
833
|
-
if len(error_messages) > 0 and self.status ==
|
|
907
|
+
if len(error_messages) > 0 and self.status == ContentStatus.production:
|
|
834
908
|
msg = (
|
|
835
909
|
"Use of fields in rba/messages that do not appear in search:\n\t- "
|
|
836
910
|
"\n\t- ".join(error_messages)
|
|
@@ -881,7 +955,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
881
955
|
cls, v: list[UnitTest | IntegrationTest | ManualTest], info: ValidationInfo
|
|
882
956
|
) -> list[UnitTest | IntegrationTest | ManualTest]:
|
|
883
957
|
# Only production analytics require tests
|
|
884
|
-
if info.data.get("status", "") !=
|
|
958
|
+
if info.data.get("status", "") != ContentStatus.production:
|
|
885
959
|
return v
|
|
886
960
|
|
|
887
961
|
# All types EXCEPT Correlation MUST have test(s). Any other type, including newly defined
|
|
@@ -1059,7 +1133,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
1059
1133
|
@model_validator(mode="after")
|
|
1060
1134
|
def validate_data_source_output_fields(self):
|
|
1061
1135
|
# Skip validation for Hunting and Correlation types, or non-production detections
|
|
1062
|
-
if self.status !=
|
|
1136
|
+
if self.status != ContentStatus.production or self.type in {
|
|
1063
1137
|
AnalyticsType.Hunting,
|
|
1064
1138
|
AnalyticsType.Correlation,
|
|
1065
1139
|
}:
|