contentctl 4.1.0__py3-none-any.whl → 4.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/detection_testing/GitService.py +7 -16
- contentctl/actions/test.py +4 -1
- contentctl/actions/validate.py +38 -18
- contentctl/helper/utils.py +21 -0
- contentctl/input/director.py +119 -55
- contentctl/input/ssa_detection_builder.py +2 -5
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +18 -5
- contentctl/objects/data_source.py +18 -11
- contentctl/objects/enums.py +2 -0
- contentctl/objects/event_source.py +10 -0
- contentctl/objects/story.py +2 -2
- {contentctl-4.1.0.dist-info → contentctl-4.1.2.dist-info}/METADATA +1 -1
- {contentctl-4.1.0.dist-info → contentctl-4.1.2.dist-info}/RECORD +16 -15
- {contentctl-4.1.0.dist-info → contentctl-4.1.2.dist-info}/LICENSE.md +0 -0
- {contentctl-4.1.0.dist-info → contentctl-4.1.2.dist-info}/WHEEL +0 -0
- {contentctl-4.1.0.dist-info → contentctl-4.1.2.dist-info}/entry_points.txt +0 -0
|
@@ -76,33 +76,28 @@ class GitService(BaseModel):
|
|
|
76
76
|
if diff.delta.status in (DeltaStatus.ADDED, DeltaStatus.MODIFIED, DeltaStatus.RENAMED):
|
|
77
77
|
#print(f"{DeltaStatus(diff.delta.status).name:<8}:{diff.delta.new_file.raw_path}")
|
|
78
78
|
decoded_path = pathlib.Path(diff.delta.new_file.raw_path.decode('utf-8'))
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
#Also ignore ssa detections
|
|
82
|
-
pass
|
|
83
|
-
elif 'detections/' in str(decoded_path) and decoded_path.suffix == ".yml":
|
|
79
|
+
# Note that we only handle updates to detections, lookups, and macros at this time. All other changes are ignored.
|
|
80
|
+
if decoded_path.is_relative_to(self.config.path/"detections") and decoded_path.suffix == ".yml":
|
|
84
81
|
detectionObject = filepath_to_content_map.get(decoded_path, None)
|
|
85
82
|
if isinstance(detectionObject, Detection):
|
|
86
83
|
updated_detections.append(detectionObject)
|
|
87
84
|
else:
|
|
88
85
|
raise Exception(f"Error getting detection object for file {str(decoded_path)}")
|
|
89
86
|
|
|
90
|
-
elif
|
|
87
|
+
elif decoded_path.is_relative_to(self.config.path/"macros") and decoded_path.suffix == ".yml":
|
|
91
88
|
macroObject = filepath_to_content_map.get(decoded_path, None)
|
|
92
89
|
if isinstance(macroObject, Macro):
|
|
93
90
|
updated_macros.append(macroObject)
|
|
94
91
|
else:
|
|
95
92
|
raise Exception(f"Error getting macro object for file {str(decoded_path)}")
|
|
96
93
|
|
|
97
|
-
elif
|
|
94
|
+
elif decoded_path.is_relative_to(self.config.path/"lookups"):
|
|
98
95
|
# We need to convert this to a yml. This means we will catch
|
|
99
96
|
# both changes to a csv AND changes to the YML that uses it
|
|
100
|
-
|
|
101
|
-
|
|
102
97
|
if decoded_path.suffix == ".yml":
|
|
103
98
|
updatedLookup = filepath_to_content_map.get(decoded_path, None)
|
|
104
99
|
if not isinstance(updatedLookup,Lookup):
|
|
105
|
-
raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(
|
|
100
|
+
raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(updatedLookup))}")
|
|
106
101
|
updated_lookups.append(updatedLookup)
|
|
107
102
|
|
|
108
103
|
elif decoded_path.suffix == ".csv":
|
|
@@ -127,12 +122,8 @@ class GitService(BaseModel):
|
|
|
127
122
|
else:
|
|
128
123
|
pass
|
|
129
124
|
#print(f"Ignore changes to file {decoded_path} since it is not a detection, macro, or lookup.")
|
|
130
|
-
|
|
131
|
-
# else:
|
|
132
|
-
# print(f"{diff.delta.new_file.raw_path}:{DeltaStatus(diff.delta.status).name} (IGNORED)")
|
|
133
|
-
# pass
|
|
134
125
|
else:
|
|
135
|
-
raise Exception(f"Unrecognized type {type(diff)}")
|
|
126
|
+
raise Exception(f"Unrecognized diff type {type(diff)}")
|
|
136
127
|
|
|
137
128
|
|
|
138
129
|
# If a detection has at least one dependency on changed content,
|
|
@@ -153,7 +144,7 @@ class GitService(BaseModel):
|
|
|
153
144
|
#Print out the names of all modified/new content
|
|
154
145
|
modifiedAndNewContentString = "\n - ".join(sorted([d.name for d in updated_detections]))
|
|
155
146
|
|
|
156
|
-
print(f"[{len(updated_detections)}] Pieces of modifed and new content
|
|
147
|
+
print(f"[{len(updated_detections)}] Pieces of modifed and new content (this may include experimental/deprecated/manual_test content):\n - {modifiedAndNewContentString}")
|
|
157
148
|
return updated_detections
|
|
158
149
|
|
|
159
150
|
def getSelected(self, detectionFilenames:List[FilePath])->List[Detection]:
|
contentctl/actions/test.py
CHANGED
|
@@ -90,6 +90,9 @@ class Test:
|
|
|
90
90
|
|
|
91
91
|
if len(input_dto.detections) == 0:
|
|
92
92
|
print(f"With Detection Testing Mode '{input_dto.config.getModeName()}', there were [0] detections found to test.\nAs such, we will quit immediately.")
|
|
93
|
+
# Directly call stop so that the summary.yml will be generated. Of course it will not have any test results, but we still want it to contain
|
|
94
|
+
# a summary showing that now detections were tested.
|
|
95
|
+
file.stop()
|
|
93
96
|
else:
|
|
94
97
|
print(f"MODE: [{input_dto.config.getModeName()}] - Test [{len(input_dto.detections)}] detections")
|
|
95
98
|
if input_dto.config.mode in [DetectionTestingMode.changes, DetectionTestingMode.selected]:
|
|
@@ -98,7 +101,7 @@ class Test:
|
|
|
98
101
|
|
|
99
102
|
manager.setup()
|
|
100
103
|
manager.execute()
|
|
101
|
-
|
|
104
|
+
|
|
102
105
|
try:
|
|
103
106
|
summary_results = file.getSummaryObject()
|
|
104
107
|
summary = summary_results.get("summary", {})
|
contentctl/actions/validate.py
CHANGED
|
@@ -6,33 +6,47 @@ from pydantic import ValidationError
|
|
|
6
6
|
from typing import Union
|
|
7
7
|
|
|
8
8
|
from contentctl.objects.enums import SecurityContentProduct
|
|
9
|
-
from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import
|
|
10
|
-
|
|
11
|
-
Director,
|
|
12
|
-
DirectorOutputDto
|
|
9
|
+
from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
|
|
10
|
+
SecurityContentObject_Abstract,
|
|
13
11
|
)
|
|
12
|
+
from contentctl.input.director import Director, DirectorOutputDto
|
|
14
13
|
|
|
15
14
|
from contentctl.objects.config import validate
|
|
16
15
|
from contentctl.enrichments.attack_enrichment import AttackEnrichment
|
|
17
16
|
from contentctl.enrichments.cve_enrichment import CveEnrichment
|
|
18
17
|
from contentctl.objects.atomic import AtomicTest
|
|
19
18
|
|
|
19
|
+
|
|
20
20
|
class Validate:
|
|
21
21
|
def execute(self, input_dto: validate) -> DirectorOutputDto:
|
|
22
|
-
|
|
23
|
-
director_output_dto = DirectorOutputDto(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
22
|
+
|
|
23
|
+
director_output_dto = DirectorOutputDto(
|
|
24
|
+
AtomicTest.getAtomicTestsFromArtRepo(
|
|
25
|
+
repo_path=input_dto.getAtomicRedTeamRepoPath(),
|
|
26
|
+
enabled=input_dto.enrichments,
|
|
27
|
+
),
|
|
28
|
+
AttackEnrichment.getAttackEnrichment(input_dto),
|
|
29
|
+
CveEnrichment.getCveEnrichment(input_dto),
|
|
30
|
+
[],
|
|
31
|
+
[],
|
|
32
|
+
[],
|
|
33
|
+
[],
|
|
34
|
+
[],
|
|
35
|
+
[],
|
|
36
|
+
[],
|
|
37
|
+
[],
|
|
38
|
+
[],
|
|
39
|
+
[],
|
|
40
|
+
[],
|
|
41
|
+
)
|
|
42
|
+
|
|
30
43
|
director = Director(director_output_dto)
|
|
31
44
|
director.execute(input_dto)
|
|
32
|
-
|
|
33
45
|
return director_output_dto
|
|
34
46
|
|
|
35
|
-
def validate_duplicate_uuids(
|
|
47
|
+
def validate_duplicate_uuids(
|
|
48
|
+
self, security_content_objects: list[SecurityContentObject_Abstract]
|
|
49
|
+
):
|
|
36
50
|
all_uuids = set()
|
|
37
51
|
duplicate_uuids = set()
|
|
38
52
|
for elem in security_content_objects:
|
|
@@ -45,14 +59,20 @@ class Validate:
|
|
|
45
59
|
|
|
46
60
|
if len(duplicate_uuids) == 0:
|
|
47
61
|
return
|
|
48
|
-
|
|
62
|
+
|
|
49
63
|
# At least once duplicate uuid has been found. Enumerate all
|
|
50
64
|
# the pieces of content that use duplicate uuids
|
|
51
65
|
duplicate_messages = []
|
|
52
66
|
for uuid in duplicate_uuids:
|
|
53
|
-
duplicate_uuid_content = [
|
|
54
|
-
|
|
55
|
-
|
|
67
|
+
duplicate_uuid_content = [
|
|
68
|
+
str(content.file_path)
|
|
69
|
+
for content in security_content_objects
|
|
70
|
+
if content.id in duplicate_uuids
|
|
71
|
+
]
|
|
72
|
+
duplicate_messages.append(
|
|
73
|
+
f"Duplicate UUID [{uuid}] in {duplicate_uuid_content}"
|
|
74
|
+
)
|
|
75
|
+
|
|
56
76
|
raise ValueError(
|
|
57
77
|
"ERROR: Duplicate ID(s) found in objects:\n"
|
|
58
78
|
+ "\n - ".join(duplicate_messages)
|
contentctl/helper/utils.py
CHANGED
|
@@ -25,6 +25,9 @@ class Utils:
|
|
|
25
25
|
@staticmethod
|
|
26
26
|
def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]:
|
|
27
27
|
listOfFiles:list[pathlib.Path] = []
|
|
28
|
+
base_path = pathlib.Path(path)
|
|
29
|
+
if not base_path.exists():
|
|
30
|
+
return listOfFiles
|
|
28
31
|
for (dirpath, dirnames, filenames) in os.walk(path):
|
|
29
32
|
for file in filenames:
|
|
30
33
|
if file.endswith(".yml"):
|
|
@@ -32,6 +35,24 @@ class Utils:
|
|
|
32
35
|
|
|
33
36
|
return sorted(listOfFiles)
|
|
34
37
|
|
|
38
|
+
@staticmethod
|
|
39
|
+
def get_all_yml_files_from_directory_one_layer_deep(path: str) -> list[pathlib.Path]:
|
|
40
|
+
listOfFiles: list[pathlib.Path] = []
|
|
41
|
+
base_path = pathlib.Path(path)
|
|
42
|
+
if not base_path.exists():
|
|
43
|
+
return listOfFiles
|
|
44
|
+
# Check the base directory
|
|
45
|
+
for item in base_path.iterdir():
|
|
46
|
+
if item.is_file() and item.suffix == '.yml':
|
|
47
|
+
listOfFiles.append(item)
|
|
48
|
+
# Check one subfolder level deep
|
|
49
|
+
for subfolder in base_path.iterdir():
|
|
50
|
+
if subfolder.is_dir() and subfolder.name != "cim":
|
|
51
|
+
for item in subfolder.iterdir():
|
|
52
|
+
if item.is_file() and item.suffix == '.yml':
|
|
53
|
+
listOfFiles.append(item)
|
|
54
|
+
return sorted(listOfFiles)
|
|
55
|
+
|
|
35
56
|
|
|
36
57
|
@staticmethod
|
|
37
58
|
def add_id(id_dict:dict[str, list[pathlib.Path]], obj:SecurityContentObject, path:pathlib.Path) -> None:
|
contentctl/input/director.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import sys
|
|
3
|
+
import pathlib
|
|
3
4
|
from typing import Union
|
|
4
5
|
from dataclasses import dataclass, field
|
|
5
6
|
from pydantic import ValidationError
|
|
@@ -20,11 +21,24 @@ from contentctl.objects.lookup import Lookup
|
|
|
20
21
|
from contentctl.objects.ssa_detection import SSADetection
|
|
21
22
|
from contentctl.objects.atomic import AtomicTest
|
|
22
23
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
24
|
+
from contentctl.objects.data_source import DataSource
|
|
25
|
+
from contentctl.objects.event_source import EventSource
|
|
23
26
|
|
|
24
27
|
from contentctl.enrichments.attack_enrichment import AttackEnrichment
|
|
25
28
|
from contentctl.enrichments.cve_enrichment import CveEnrichment
|
|
26
29
|
|
|
27
30
|
from contentctl.objects.config import validate
|
|
31
|
+
from contentctl.input.ssa_detection_builder import SSADetectionBuilder
|
|
32
|
+
from contentctl.objects.enums import SecurityContentType
|
|
33
|
+
|
|
34
|
+
from contentctl.objects.enums import DetectionStatus
|
|
35
|
+
from contentctl.helper.utils import Utils
|
|
36
|
+
|
|
37
|
+
from contentctl.input.ssa_detection_builder import SSADetectionBuilder
|
|
38
|
+
from contentctl.objects.enums import SecurityContentType
|
|
39
|
+
|
|
40
|
+
from contentctl.objects.enums import DetectionStatus
|
|
41
|
+
from contentctl.helper.utils import Utils
|
|
28
42
|
|
|
29
43
|
|
|
30
44
|
@dataclass
|
|
@@ -43,7 +57,8 @@ class DirectorOutputDto:
|
|
|
43
57
|
lookups: list[Lookup]
|
|
44
58
|
deployments: list[Deployment]
|
|
45
59
|
ssa_detections: list[SSADetection]
|
|
46
|
-
|
|
60
|
+
data_sources: list[DataSource]
|
|
61
|
+
event_sources: list[EventSource]
|
|
47
62
|
name_to_content_map: dict[str, SecurityContentObject] = field(default_factory=dict)
|
|
48
63
|
uuid_to_content_map: dict[UUID, SecurityContentObject] = field(default_factory=dict)
|
|
49
64
|
|
|
@@ -92,66 +107,84 @@ class DirectorOutputDto:
|
|
|
92
107
|
self.uuid_to_content_map[content.id] = content
|
|
93
108
|
|
|
94
109
|
|
|
95
|
-
from contentctl.input.ssa_detection_builder import SSADetectionBuilder
|
|
96
|
-
from contentctl.objects.enums import SecurityContentType
|
|
97
|
-
|
|
98
|
-
from contentctl.objects.enums import DetectionStatus
|
|
99
|
-
from contentctl.helper.utils import Utils
|
|
100
|
-
|
|
101
|
-
|
|
102
110
|
class Director():
|
|
103
111
|
input_dto: validate
|
|
104
112
|
output_dto: DirectorOutputDto
|
|
105
113
|
ssa_detection_builder: SSADetectionBuilder
|
|
106
|
-
|
|
107
|
-
|
|
108
114
|
|
|
109
115
|
def __init__(self, output_dto: DirectorOutputDto) -> None:
|
|
110
116
|
self.output_dto = output_dto
|
|
111
117
|
self.ssa_detection_builder = SSADetectionBuilder()
|
|
112
|
-
|
|
118
|
+
|
|
113
119
|
def execute(self, input_dto: validate) -> None:
|
|
114
120
|
self.input_dto = input_dto
|
|
115
|
-
|
|
116
|
-
|
|
117
121
|
self.createSecurityContent(SecurityContentType.deployments)
|
|
118
122
|
self.createSecurityContent(SecurityContentType.lookups)
|
|
119
123
|
self.createSecurityContent(SecurityContentType.macros)
|
|
120
124
|
self.createSecurityContent(SecurityContentType.stories)
|
|
121
125
|
self.createSecurityContent(SecurityContentType.baselines)
|
|
122
126
|
self.createSecurityContent(SecurityContentType.investigations)
|
|
127
|
+
self.createSecurityContent(SecurityContentType.event_sources)
|
|
128
|
+
self.createSecurityContent(SecurityContentType.data_sources)
|
|
123
129
|
self.createSecurityContent(SecurityContentType.playbooks)
|
|
124
130
|
self.createSecurityContent(SecurityContentType.detections)
|
|
125
|
-
|
|
126
|
-
|
|
127
131
|
self.createSecurityContent(SecurityContentType.ssa_detections)
|
|
128
|
-
|
|
129
132
|
|
|
130
133
|
def createSecurityContent(self, contentType: SecurityContentType) -> None:
|
|
131
134
|
if contentType == SecurityContentType.ssa_detections:
|
|
132
|
-
files = Utils.get_all_yml_files_from_directory(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
135
|
+
files = Utils.get_all_yml_files_from_directory(
|
|
136
|
+
os.path.join(self.input_dto.path, "ssa_detections")
|
|
137
|
+
)
|
|
138
|
+
security_content_files = [f for f in files if f.name.startswith("ssa___")]
|
|
139
|
+
|
|
140
|
+
elif contentType == SecurityContentType.data_sources:
|
|
141
|
+
security_content_files = (
|
|
142
|
+
Utils.get_all_yml_files_from_directory_one_layer_deep(
|
|
143
|
+
os.path.join(self.input_dto.path, "data_sources")
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
elif contentType == SecurityContentType.event_sources:
|
|
148
|
+
security_content_files = Utils.get_all_yml_files_from_directory(
|
|
149
|
+
os.path.join(self.input_dto.path, "data_sources", "cloud", "event_sources")
|
|
150
|
+
)
|
|
151
|
+
security_content_files.extend(
|
|
152
|
+
Utils.get_all_yml_files_from_directory(
|
|
153
|
+
os.path.join(self.input_dto.path, "data_sources", "endpoint", "event_sources")
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
security_content_files.extend(
|
|
157
|
+
Utils.get_all_yml_files_from_directory(
|
|
158
|
+
os.path.join(self.input_dto.path, "data_sources", "network", "event_sources")
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
elif contentType in [
|
|
163
|
+
SecurityContentType.deployments,
|
|
164
|
+
SecurityContentType.lookups,
|
|
165
|
+
SecurityContentType.macros,
|
|
166
|
+
SecurityContentType.stories,
|
|
167
|
+
SecurityContentType.baselines,
|
|
168
|
+
SecurityContentType.investigations,
|
|
169
|
+
SecurityContentType.playbooks,
|
|
170
|
+
SecurityContentType.detections,
|
|
171
|
+
]:
|
|
172
|
+
files = Utils.get_all_yml_files_from_directory(
|
|
173
|
+
os.path.join(self.input_dto.path, str(contentType.name))
|
|
174
|
+
)
|
|
175
|
+
security_content_files = [
|
|
176
|
+
f for f in files if not f.name.startswith("ssa___")
|
|
177
|
+
]
|
|
145
178
|
else:
|
|
146
|
-
|
|
179
|
+
raise (Exception(f"Cannot createSecurityContent for unknown product."))
|
|
147
180
|
|
|
148
181
|
validation_errors = []
|
|
149
|
-
|
|
182
|
+
|
|
150
183
|
already_ran = False
|
|
151
184
|
progress_percent = 0
|
|
152
|
-
|
|
153
|
-
for index,file in enumerate(security_content_files):
|
|
154
|
-
progress_percent = ((index+1)/len(security_content_files)) * 100
|
|
185
|
+
|
|
186
|
+
for index, file in enumerate(security_content_files):
|
|
187
|
+
progress_percent = ((index + 1) / len(security_content_files)) * 100
|
|
155
188
|
try:
|
|
156
189
|
type_string = contentType.name.upper()
|
|
157
190
|
modelDict = YmlReader.load_file(file)
|
|
@@ -167,7 +200,7 @@ class Director():
|
|
|
167
200
|
elif contentType == SecurityContentType.deployments:
|
|
168
201
|
deployment = Deployment.model_validate(modelDict,context={"output_dto":self.output_dto})
|
|
169
202
|
self.output_dto.addContentToDictMappings(deployment)
|
|
170
|
-
|
|
203
|
+
|
|
171
204
|
elif contentType == SecurityContentType.playbooks:
|
|
172
205
|
playbook = Playbook.model_validate(modelDict,context={"output_dto":self.output_dto})
|
|
173
206
|
self.output_dto.addContentToDictMappings(playbook)
|
|
@@ -193,36 +226,67 @@ class Director():
|
|
|
193
226
|
ssa_detection = self.ssa_detection_builder.getObject()
|
|
194
227
|
if ssa_detection.status in [DetectionStatus.production.value, DetectionStatus.validation.value]:
|
|
195
228
|
self.output_dto.addContentToDictMappings(ssa_detection)
|
|
229
|
+
|
|
230
|
+
elif contentType == SecurityContentType.data_sources:
|
|
231
|
+
data_source = DataSource.model_validate(
|
|
232
|
+
modelDict, context={"output_dto": self.output_dto}
|
|
233
|
+
)
|
|
234
|
+
self.output_dto.data_sources.append(data_source)
|
|
235
|
+
|
|
236
|
+
elif contentType == SecurityContentType.event_sources:
|
|
237
|
+
event_source = EventSource.model_validate(
|
|
238
|
+
modelDict, context={"output_dto": self.output_dto}
|
|
239
|
+
)
|
|
240
|
+
self.output_dto.event_sources.append(event_source)
|
|
196
241
|
|
|
197
242
|
else:
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
if (
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
243
|
+
raise Exception(f"Unsupported type: [{contentType}]")
|
|
244
|
+
|
|
245
|
+
if (
|
|
246
|
+
sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()
|
|
247
|
+
) or not already_ran:
|
|
248
|
+
already_ran = True
|
|
249
|
+
print(
|
|
250
|
+
f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...",
|
|
251
|
+
end="",
|
|
252
|
+
flush=True,
|
|
253
|
+
)
|
|
208
254
|
|
|
209
|
-
|
|
255
|
+
except (ValidationError, ValueError) as e:
|
|
256
|
+
relative_path = file.absolute().relative_to(
|
|
257
|
+
self.input_dto.path.absolute()
|
|
258
|
+
)
|
|
259
|
+
validation_errors.append((relative_path, e))
|
|
260
|
+
|
|
261
|
+
print(
|
|
262
|
+
f"\r{f'{contentType.name.upper()} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...",
|
|
263
|
+
end="",
|
|
264
|
+
flush=True,
|
|
265
|
+
)
|
|
210
266
|
print("Done!")
|
|
211
267
|
|
|
212
268
|
if len(validation_errors) > 0:
|
|
213
|
-
errors_string =
|
|
214
|
-
|
|
269
|
+
errors_string = "\n\n".join(
|
|
270
|
+
[
|
|
271
|
+
f"File: {e_tuple[0]}\nError: {str(e_tuple[1])}"
|
|
272
|
+
for e_tuple in validation_errors
|
|
273
|
+
]
|
|
274
|
+
)
|
|
275
|
+
# print(f"The following {len(validation_errors)} error(s) were found during validation:\n\n{errors_string}\n\nVALIDATION FAILED")
|
|
215
276
|
# We quit after validation a single type/group of content because it can cause significant cascading errors in subsequent
|
|
216
277
|
# types of content (since they may import or otherwise use it)
|
|
217
|
-
raise Exception(
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
278
|
+
raise Exception(
|
|
279
|
+
f"The following {len(validation_errors)} error(s) were found during validation:\n\n{errors_string}\n\nVALIDATION FAILED"
|
|
280
|
+
)
|
|
222
281
|
|
|
223
|
-
def constructSSADetection(
|
|
282
|
+
def constructSSADetection(
|
|
283
|
+
self,
|
|
284
|
+
builder: SSADetectionBuilder,
|
|
285
|
+
directorOutput: DirectorOutputDto,
|
|
286
|
+
file_path: str,
|
|
287
|
+
) -> None:
|
|
224
288
|
builder.reset()
|
|
225
|
-
builder.setObject(file_path
|
|
289
|
+
builder.setObject(file_path)
|
|
226
290
|
builder.addMitreAttackEnrichmentNew(directorOutput.attack_enrichment)
|
|
227
291
|
builder.addKillChainPhase()
|
|
228
292
|
builder.addCIS()
|
|
@@ -13,17 +13,14 @@ from contentctl.enrichments.cve_enrichment import CveEnrichment
|
|
|
13
13
|
from contentctl.enrichments.splunk_app_enrichment import SplunkAppEnrichment
|
|
14
14
|
from contentctl.objects.ssa_detection import SSADetection
|
|
15
15
|
from contentctl.objects.constants import *
|
|
16
|
-
from contentctl.input.director import DirectorOutputDto
|
|
17
16
|
from contentctl.enrichments.attack_enrichment import AttackEnrichment
|
|
18
17
|
|
|
19
18
|
class SSADetectionBuilder():
|
|
20
19
|
security_content_obj : SSADetection
|
|
21
20
|
|
|
22
21
|
|
|
23
|
-
def setObject(self, path: str
|
|
24
|
-
|
|
25
|
-
yml_dict = YmlReader.load_file(path)
|
|
26
|
-
#self.security_content_obj = SSADetection.model_validate(yml_dict, context={"output_dto":output_dto})
|
|
22
|
+
def setObject(self, path: str) -> None:
|
|
23
|
+
yml_dict = YmlReader.load_file(path)
|
|
27
24
|
self.security_content_obj = SSADetection.parse_obj(yml_dict)
|
|
28
25
|
self.security_content_obj.source = os.path.split(os.path.dirname(self.security_content_obj.file_path))[-1]
|
|
29
26
|
|
|
@@ -40,7 +40,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
40
40
|
search: Union[str, dict[str,Any]] = Field(...)
|
|
41
41
|
how_to_implement: str = Field(..., min_length=4)
|
|
42
42
|
known_false_positives: str = Field(..., min_length=4)
|
|
43
|
-
|
|
43
|
+
data_source_objects: Optional[List[DataSource]] = None
|
|
44
44
|
|
|
45
45
|
enabled_by_default: bool = False
|
|
46
46
|
file_path: FilePath = Field(...)
|
|
@@ -369,8 +369,6 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
369
369
|
# if not isinstance(director,DirectorOutputDto):
|
|
370
370
|
# raise ValueError("DirectorOutputDto was not passed in context of Detection model_post_init")
|
|
371
371
|
director: Optional[DirectorOutputDto] = ctx.get("output_dto",None)
|
|
372
|
-
for story in self.tags.analytic_story:
|
|
373
|
-
story.detections.append(self)
|
|
374
372
|
|
|
375
373
|
#Ensure that all baselines link to this detection
|
|
376
374
|
for baseline in self.baselines:
|
|
@@ -385,10 +383,25 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
385
383
|
if replaced is False:
|
|
386
384
|
raise ValueError(f"Error, failed to replace detection reference in Baseline '{baseline.name}' to detection '{self.name}'")
|
|
387
385
|
baseline.tags.detections = new_detections
|
|
388
|
-
|
|
389
|
-
return self
|
|
390
386
|
|
|
387
|
+
self.data_source_objects = []
|
|
388
|
+
for data_source_obj in director.data_sources:
|
|
389
|
+
for detection_data_source in self.data_source:
|
|
390
|
+
if data_source_obj.name in detection_data_source:
|
|
391
|
+
self.data_source_objects.append(data_source_obj)
|
|
392
|
+
|
|
393
|
+
# Remove duplicate data source objects based on their 'name' property
|
|
394
|
+
unique_data_sources = {}
|
|
395
|
+
for data_source_obj in self.data_source_objects:
|
|
396
|
+
if data_source_obj.name not in unique_data_sources:
|
|
397
|
+
unique_data_sources[data_source_obj.name] = data_source_obj
|
|
398
|
+
self.data_source_objects = list(unique_data_sources.values())
|
|
391
399
|
|
|
400
|
+
for story in self.tags.analytic_story:
|
|
401
|
+
story.detections.append(self)
|
|
402
|
+
story.data_sources.extend(self.data_source_objects)
|
|
403
|
+
|
|
404
|
+
return self
|
|
392
405
|
|
|
393
406
|
|
|
394
407
|
@field_validator('lookups',mode="before")
|
|
@@ -2,20 +2,27 @@ from __future__ import annotations
|
|
|
2
2
|
from pydantic import BaseModel
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
|
|
6
5
|
class DataSource(BaseModel):
|
|
7
6
|
name: str
|
|
8
7
|
id: str
|
|
9
|
-
date: str
|
|
10
8
|
author: str
|
|
11
|
-
type: str
|
|
12
9
|
source: str
|
|
13
10
|
sourcetype: str
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
11
|
+
separator: str = None
|
|
12
|
+
configuration: str = None
|
|
13
|
+
supported_TA: dict
|
|
14
|
+
event_names: list = None
|
|
15
|
+
event_sources: list = None
|
|
16
|
+
fields: list = None
|
|
17
|
+
example_log: str = None
|
|
18
|
+
|
|
19
|
+
def model_post_init(self, ctx:dict[str,Any]):
|
|
20
|
+
context = ctx.get("output_dto")
|
|
21
|
+
|
|
22
|
+
if self.event_names:
|
|
23
|
+
self.event_sources = []
|
|
24
|
+
for event_source in context.event_sources:
|
|
25
|
+
if any(event['event_name'] == event_source.event_name for event in self.event_names):
|
|
26
|
+
self.event_sources.append(event_source)
|
|
27
|
+
|
|
28
|
+
return self
|
contentctl/objects/enums.py
CHANGED
contentctl/objects/story.py
CHANGED
|
@@ -7,7 +7,7 @@ if TYPE_CHECKING:
|
|
|
7
7
|
from contentctl.objects.detection import Detection
|
|
8
8
|
from contentctl.objects.investigation import Investigation
|
|
9
9
|
from contentctl.objects.baseline import Baseline
|
|
10
|
-
|
|
10
|
+
from contentctl.objects.data_source import DataSource
|
|
11
11
|
|
|
12
12
|
from contentctl.objects.security_content_object import SecurityContentObject
|
|
13
13
|
|
|
@@ -33,7 +33,7 @@ class Story(SecurityContentObject):
|
|
|
33
33
|
detections:List[Detection] = []
|
|
34
34
|
investigations: List[Investigation] = []
|
|
35
35
|
baselines: List[Baseline] = []
|
|
36
|
-
|
|
36
|
+
data_sources: List[DataSource] = []
|
|
37
37
|
|
|
38
38
|
def storyAndInvestigationNamesWithApp(self, app_name:str)->List[str]:
|
|
39
39
|
return [f"{app_name} - {name} - Rule" for name in self.detection_names] + \
|
|
@@ -3,7 +3,7 @@ contentctl/actions/build.py,sha256=BVc-1E63zeUQ9wWAHTC_fLNvfEK5YT3Z6_QLiE72TQs,4
|
|
|
3
3
|
contentctl/actions/convert.py,sha256=0KBWLxvP1hSPXpExePqpOQPRvlQLamvPLyQqeTIWNbk,704
|
|
4
4
|
contentctl/actions/deploy_acs.py,sha256=mf3uk495H1EU_LNN-TiOsYCo18HMGoEBMb6ojeTr0zw,1418
|
|
5
5
|
contentctl/actions/detection_testing/DetectionTestingManager.py,sha256=zg8JasDjCpSC-yhseEyUwO8qbDJIUJbhlus9Li9ZAnA,8818
|
|
6
|
-
contentctl/actions/detection_testing/GitService.py,sha256=
|
|
6
|
+
contentctl/actions/detection_testing/GitService.py,sha256=FY_JtMi3qL-uC31Yyf7CTWZ5kLQhAMAvcj-8QXtkfPM,8386
|
|
7
7
|
contentctl/actions/detection_testing/generate_detection_coverage_badge.py,sha256=N5mznaeErVak3mOBwsd0RDBFJO3bku0EZvpayCyU-uk,2259
|
|
8
8
|
contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=VFhSHdw_0N6ol668hDkaj7yFjPsZqBoFNC8FKzWKICc,53141
|
|
9
9
|
contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py,sha256=HVGWCXy0GQeBqu2cVJn5H-I8GY8rwgkkc53ilO1TfZA,6846
|
|
@@ -20,8 +20,8 @@ contentctl/actions/inspect.py,sha256=6gVVKmV5CUUYOkNNVTMPKj9bM1uXVthgGCoFKZGDeS8
|
|
|
20
20
|
contentctl/actions/new_content.py,sha256=4gTlxV0fmdsSETPX4T9uQPpIAm--Jf2vc6Vm3w-RkfI,6128
|
|
21
21
|
contentctl/actions/release_notes.py,sha256=akkFfLhsJuaPUyjsb6dLlKt9cUM-JApAjTFQMbYoXeM,13115
|
|
22
22
|
contentctl/actions/reporting.py,sha256=MJEmvmoA1WnSFZEU9QM6daL_W94oOX0WXAcX1qAM2As,1583
|
|
23
|
-
contentctl/actions/test.py,sha256=
|
|
24
|
-
contentctl/actions/validate.py,sha256=
|
|
23
|
+
contentctl/actions/test.py,sha256=dx7f750_MrlvysxOmOdIro1bH0iVKF4K54TSwhvU2MU,5146
|
|
24
|
+
contentctl/actions/validate.py,sha256=HnmB_qsluYr0BFHQzg0HvXGLHM4M1taBtsWt774esf8,2537
|
|
25
25
|
contentctl/api.py,sha256=FBOpRhbBCBdjORmwe_8MPQ3PRZ6T0KrrFcfKovVFkug,6343
|
|
26
26
|
contentctl/contentctl.py,sha256=Vr2cuvaPjpJpYvD9kVoYq7iD6rhLQEpTKmcGoq4emhA,10470
|
|
27
27
|
contentctl/enrichments/attack_enrichment.py,sha256=EkEloG3hMmPTloPyYiVkhq3iT_BieXaJmprJ5stfyRw,6732
|
|
@@ -29,14 +29,14 @@ contentctl/enrichments/cve_enrichment.py,sha256=IzkKSdnQi3JrAUUyLpcGA_Y2g_B7latq
|
|
|
29
29
|
contentctl/enrichments/splunk_app_enrichment.py,sha256=zDNHFLZTi2dJ1gdnh0sHkD6F1VtkblqFnhacFcCMBfc,3418
|
|
30
30
|
contentctl/helper/link_validator.py,sha256=-XorhxfGtjLynEL1X4hcpRMiyemogf2JEnvLwhHq80c,7139
|
|
31
31
|
contentctl/helper/logger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
-
contentctl/helper/utils.py,sha256=
|
|
32
|
+
contentctl/helper/utils.py,sha256=THV4ZuaeEHG6PK5JeZUkTLBmvN4fUV0Ar6opH1zXxKs,16545
|
|
33
33
|
contentctl/input/backend_splunk_ba.py,sha256=Y70tJqgaUM0nzfm2SiGMof4HkhY84feqf-xnRx1xPb4,5861
|
|
34
|
-
contentctl/input/director.py,sha256=
|
|
34
|
+
contentctl/input/director.py,sha256=BR1RvBD0U_JtHtrM3jM_FpcvaaNlME7nc-gNO4RJLM8,13323
|
|
35
35
|
contentctl/input/new_content_questions.py,sha256=o4prlBoUhEMxqpZukquI9WKbzfFJfYhEF7a8m2q_BEE,5565
|
|
36
36
|
contentctl/input/sigma_converter.py,sha256=ATFNW7boNngp5dmWM7Gr4rMZrUKjvKW2_qu28--FdiU,19391
|
|
37
|
-
contentctl/input/ssa_detection_builder.py,sha256=
|
|
37
|
+
contentctl/input/ssa_detection_builder.py,sha256=dke9mPn2VQVSpYiaGWjZn3PkqVJTe58gcT2Vifv9_yc,8159
|
|
38
38
|
contentctl/input/yml_reader.py,sha256=oaal24UP8rDXkCmN5I3GnIheZrsgkhbKOlzXtyhB474,1475
|
|
39
|
-
contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=
|
|
39
|
+
contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=XhyMpghvizU37sOHKE009la1wo__EZqGdemXt9En5wc,34039
|
|
40
40
|
contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py,sha256=IVr26xFrIlTvsqQoqwYl4cmcfaP9BeYt9I0QTriKmwE,8451
|
|
41
41
|
contentctl/objects/alert_action.py,sha256=E9gjCn5C31h0sN7k90KNe4agRxFFSnMW_Z-Ri_3YQss,1335
|
|
42
42
|
contentctl/objects/atomic.py,sha256=a_G_iliAm86BunpAAG86aAL3LAEGpd9Crp7t7-PxYvI,8979
|
|
@@ -47,7 +47,7 @@ contentctl/objects/baseline_tags.py,sha256=JLdlCUc_DEccMQD6f-sa2qD8pcxYiwMUT_sRZ
|
|
|
47
47
|
contentctl/objects/config.py,sha256=tK0BY4A9Go5jp8tpOSwgczuOAyu9dMPvC0nyOHeO-74,43642
|
|
48
48
|
contentctl/objects/constants.py,sha256=1LjiK9A7t0aHHkJz2mrW-DImdW1P98GPssTwmwNNI_M,3468
|
|
49
49
|
contentctl/objects/correlation_search.py,sha256=B97vCt2Ew7PGgqd5Y9l6RD3DJdy51Eh7Gzkxxs2xqZ0,36891
|
|
50
|
-
contentctl/objects/data_source.py,sha256=
|
|
50
|
+
contentctl/objects/data_source.py,sha256=UoI1zLyrwwTtKqvXf_K-TifJEY5HaVBItR3vR4J43iw,768
|
|
51
51
|
contentctl/objects/deployment.py,sha256=Qc6M4yeOvxjqFKR8sfjd4CG06AbVheTOqP1mwqo4t8s,2651
|
|
52
52
|
contentctl/objects/deployment_email.py,sha256=Zu9cXZdfOP6noa_mZpiK1GrYCTgi3Mim94iLGjE674c,147
|
|
53
53
|
contentctl/objects/deployment_notable.py,sha256=QhOI7HEkUuuqk0fum9SD8IpYBlbwIsJUff8s3kCKKj4,198
|
|
@@ -57,7 +57,8 @@ contentctl/objects/deployment_scheduling.py,sha256=bQjbJHNaUGdU1VAGV8-nFOHzHutbI
|
|
|
57
57
|
contentctl/objects/deployment_slack.py,sha256=P6z8OLHDKcDWx7nbKWasqBc3dFRatGcpO2GtmxzVV8I,135
|
|
58
58
|
contentctl/objects/detection.py,sha256=3W41cXf3ECjWuPqWrseqSLC3PAA7O5_nENWWM6MPK0Y,620
|
|
59
59
|
contentctl/objects/detection_tags.py,sha256=QR906JN8cf5et5aPf-AluEEyP3IvdUQ_KzxKffMSjrc,10261
|
|
60
|
-
contentctl/objects/enums.py,sha256=
|
|
60
|
+
contentctl/objects/enums.py,sha256=cW-orYfVBgMdZKVS8ANAkSZ-zygbrhJZX6FP4TxNGgg,14075
|
|
61
|
+
contentctl/objects/event_source.py,sha256=oOCCSQpfpSbYw6_v103I4VxwqjpXP4gsTbds06qiEa0,251
|
|
61
62
|
contentctl/objects/integration_test.py,sha256=W_VksBN_cRo7DTXdr1aLujjS9mgkEp0uvoNpmL0dVnQ,1273
|
|
62
63
|
contentctl/objects/integration_test_result.py,sha256=DrIZRRlILSHGcsK_Rlm3KJLnbKPtIen8uEPFi4ZdJ8s,370
|
|
63
64
|
contentctl/objects/investigation.py,sha256=JRoZxc_qi1fu_VFTRaxOc3B7zzSzCfEURsNzWPUCrtY,2620
|
|
@@ -74,7 +75,7 @@ contentctl/objects/risk_object.py,sha256=yY4NmEwEKaRl4sLzCRZb1n8kdpV3HzYbQVQ1ClQ
|
|
|
74
75
|
contentctl/objects/security_content_object.py,sha256=j8KNDwSMfZsSIzJucC3NuZo0SlFVpqHfDc6y3-YHjHI,234
|
|
75
76
|
contentctl/objects/ssa_detection.py,sha256=-G6tXfVVlZgPWS64hIIy3M-aMePANAuQvdpXPlgUyUs,5873
|
|
76
77
|
contentctl/objects/ssa_detection_tags.py,sha256=u8annjzo3MYZ-16wyFnuR8qJJzRa4LEhdprMIrQ47G0,5224
|
|
77
|
-
contentctl/objects/story.py,sha256=
|
|
78
|
+
contentctl/objects/story.py,sha256=GMcMAaDcX16B2mbSAzrzpVD8IMyOILAyIo2rR8jySto,4618
|
|
78
79
|
contentctl/objects/story_tags.py,sha256=0oF1OePLBxa-RQPb438tXrrfosa939CP8UbNV0_S8XY,2225
|
|
79
80
|
contentctl/objects/test_group.py,sha256=Yb1sqGom6SkVL8B3czPndz8w3CK8WdwZ39V_cn0_JZQ,2600
|
|
80
81
|
contentctl/objects/threat_object.py,sha256=S8B7RQFfLxN_g7yKPrDTuYhIy9JvQH3YwJ_T5LUZIa4,711
|
|
@@ -163,8 +164,8 @@ contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
|
|
|
163
164
|
contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
|
|
164
165
|
contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
|
|
165
166
|
contentctl/templates/stories/cobalt_strike.yml,sha256=rlaXxMN-5k8LnKBLPafBoksyMtlmsPMHPJOjTiMiZ-M,3063
|
|
166
|
-
contentctl-4.1.
|
|
167
|
-
contentctl-4.1.
|
|
168
|
-
contentctl-4.1.
|
|
169
|
-
contentctl-4.1.
|
|
170
|
-
contentctl-4.1.
|
|
167
|
+
contentctl-4.1.2.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
|
|
168
|
+
contentctl-4.1.2.dist-info/METADATA,sha256=FJsw6lsgRLBq0oHaLRtL_BoJdhe4W2xwzcMu1PZei5A,19706
|
|
169
|
+
contentctl-4.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
170
|
+
contentctl-4.1.2.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
|
|
171
|
+
contentctl-4.1.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|