contentctl 5.0.0a2__py3-none-any.whl → 5.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/__init__.py +1 -1
- contentctl/actions/build.py +88 -55
- contentctl/actions/deploy_acs.py +29 -24
- contentctl/actions/detection_testing/DetectionTestingManager.py +66 -41
- contentctl/actions/detection_testing/GitService.py +2 -4
- contentctl/actions/detection_testing/generate_detection_coverage_badge.py +48 -30
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +163 -124
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +45 -32
- contentctl/actions/detection_testing/progress_bar.py +3 -0
- contentctl/actions/detection_testing/views/DetectionTestingView.py +15 -18
- contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +1 -5
- contentctl/actions/detection_testing/views/DetectionTestingViewFile.py +2 -2
- contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py +1 -4
- contentctl/actions/doc_gen.py +9 -5
- contentctl/actions/initialize.py +45 -33
- contentctl/actions/inspect.py +118 -61
- contentctl/actions/new_content.py +83 -53
- contentctl/actions/release_notes.py +276 -146
- contentctl/actions/reporting.py +23 -19
- contentctl/actions/test.py +31 -25
- contentctl/actions/validate.py +54 -34
- contentctl/api.py +54 -45
- contentctl/contentctl.py +10 -10
- contentctl/enrichments/attack_enrichment.py +112 -72
- contentctl/enrichments/cve_enrichment.py +34 -28
- contentctl/enrichments/splunk_app_enrichment.py +38 -36
- contentctl/helper/link_validator.py +101 -78
- contentctl/helper/splunk_app.py +69 -41
- contentctl/helper/utils.py +58 -39
- contentctl/input/director.py +69 -37
- contentctl/input/new_content_questions.py +26 -34
- contentctl/input/yml_reader.py +22 -17
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +255 -323
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +111 -46
- contentctl/objects/alert_action.py +8 -8
- contentctl/objects/annotated_types.py +1 -1
- contentctl/objects/atomic.py +64 -54
- contentctl/objects/base_test.py +2 -1
- contentctl/objects/base_test_result.py +16 -8
- contentctl/objects/baseline.py +47 -35
- contentctl/objects/baseline_tags.py +29 -22
- contentctl/objects/config.py +1 -1
- contentctl/objects/constants.py +32 -58
- contentctl/objects/correlation_search.py +75 -55
- contentctl/objects/dashboard.py +55 -41
- contentctl/objects/data_source.py +13 -13
- contentctl/objects/deployment.py +44 -37
- contentctl/objects/deployment_email.py +1 -1
- contentctl/objects/deployment_notable.py +2 -1
- contentctl/objects/deployment_phantom.py +5 -5
- contentctl/objects/deployment_rba.py +1 -1
- contentctl/objects/deployment_scheduling.py +1 -1
- contentctl/objects/deployment_slack.py +1 -1
- contentctl/objects/detection.py +5 -2
- contentctl/objects/detection_metadata.py +1 -0
- contentctl/objects/detection_stanza.py +7 -2
- contentctl/objects/detection_tags.py +54 -64
- contentctl/objects/drilldown.py +66 -35
- contentctl/objects/enums.py +61 -43
- contentctl/objects/errors.py +16 -24
- contentctl/objects/integration_test.py +3 -3
- contentctl/objects/integration_test_result.py +1 -0
- contentctl/objects/investigation.py +53 -31
- contentctl/objects/investigation_tags.py +29 -17
- contentctl/objects/lookup.py +234 -113
- contentctl/objects/macro.py +55 -38
- contentctl/objects/manual_test.py +3 -3
- contentctl/objects/manual_test_result.py +1 -0
- contentctl/objects/mitre_attack_enrichment.py +17 -16
- contentctl/objects/notable_action.py +2 -1
- contentctl/objects/notable_event.py +1 -3
- contentctl/objects/playbook.py +37 -35
- contentctl/objects/playbook_tags.py +22 -16
- contentctl/objects/rba.py +68 -11
- contentctl/objects/risk_analysis_action.py +15 -11
- contentctl/objects/risk_event.py +27 -20
- contentctl/objects/risk_object.py +1 -0
- contentctl/objects/savedsearches_conf.py +9 -7
- contentctl/objects/security_content_object.py +5 -2
- contentctl/objects/story.py +54 -49
- contentctl/objects/story_tags.py +56 -44
- contentctl/objects/test_group.py +5 -2
- contentctl/objects/threat_object.py +1 -0
- contentctl/objects/throttling.py +27 -18
- contentctl/objects/unit_test.py +3 -4
- contentctl/objects/unit_test_baseline.py +4 -5
- contentctl/objects/unit_test_result.py +6 -6
- contentctl/output/api_json_output.py +22 -22
- contentctl/output/attack_nav_output.py +21 -21
- contentctl/output/attack_nav_writer.py +29 -37
- contentctl/output/conf_output.py +230 -174
- contentctl/output/data_source_writer.py +38 -25
- contentctl/output/doc_md_output.py +53 -27
- contentctl/output/jinja_writer.py +19 -15
- contentctl/output/json_writer.py +20 -8
- contentctl/output/svg_output.py +56 -38
- contentctl/output/templates/analyticstories_detections.j2 +1 -1
- contentctl/output/templates/analyticstories_stories.j2 +1 -1
- contentctl/output/templates/es_investigations_investigations.j2 +1 -1
- contentctl/output/templates/es_investigations_stories.j2 +1 -1
- contentctl/output/templates/savedsearches_baselines.j2 +2 -2
- contentctl/output/templates/savedsearches_detections.j2 +2 -8
- contentctl/output/templates/savedsearches_investigations.j2 +2 -2
- contentctl/output/templates/transforms.j2 +2 -4
- contentctl/output/yml_writer.py +18 -24
- contentctl/templates/stories/cobalt_strike.yml +1 -0
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.1.dist-info}/METADATA +1 -1
- contentctl-5.0.1.dist-info/RECORD +168 -0
- contentctl/actions/initialize_old.py +0 -245
- contentctl/objects/observable.py +0 -39
- contentctl-5.0.0a2.dist-info/RECORD +0 -170
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.1.dist-info}/LICENSE.md +0 -0
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.1.dist-info}/WHEEL +0 -0
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.1.dist-info}/entry_points.txt +0 -0
contentctl/actions/inspect.py
CHANGED
|
@@ -16,7 +16,7 @@ from contentctl.objects.errors import (
|
|
|
16
16
|
DetectionIDError,
|
|
17
17
|
DetectionMissingError,
|
|
18
18
|
VersionDecrementedError,
|
|
19
|
-
VersionBumpingError
|
|
19
|
+
VersionBumpingError,
|
|
20
20
|
)
|
|
21
21
|
|
|
22
22
|
|
|
@@ -26,10 +26,8 @@ class InspectInputDto:
|
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class Inspect:
|
|
29
|
-
|
|
30
29
|
def execute(self, config: inspect) -> str:
|
|
31
30
|
if config.build_app or config.build_api:
|
|
32
|
-
|
|
33
31
|
self.inspectAppCLI(config)
|
|
34
32
|
appinspect_token = self.inspectAppAPI(config)
|
|
35
33
|
|
|
@@ -48,9 +46,13 @@ class Inspect:
|
|
|
48
46
|
|
|
49
47
|
def inspectAppAPI(self, config: inspect) -> str:
|
|
50
48
|
session = Session()
|
|
51
|
-
session.auth = HTTPBasicAuth(
|
|
52
|
-
|
|
53
|
-
|
|
49
|
+
session.auth = HTTPBasicAuth(
|
|
50
|
+
config.splunk_api_username, config.splunk_api_password
|
|
51
|
+
)
|
|
52
|
+
if config.stack_type not in ["victoria", "classic"]:
|
|
53
|
+
raise Exception(
|
|
54
|
+
f"stack_type MUST be either 'classic' or 'victoria', NOT '{config.stack_type}'"
|
|
55
|
+
)
|
|
54
56
|
|
|
55
57
|
APPINSPECT_API_LOGIN = "https://api.splunk.com/2.0/rest/login/splunk"
|
|
56
58
|
|
|
@@ -59,21 +61,25 @@ class Inspect:
|
|
|
59
61
|
res.raise_for_status()
|
|
60
62
|
|
|
61
63
|
authorization_bearer = res.json().get("data", {}).get("token", None)
|
|
62
|
-
APPINSPECT_API_VALIDATION_REQUEST =
|
|
64
|
+
APPINSPECT_API_VALIDATION_REQUEST = (
|
|
65
|
+
"https://appinspect.splunk.com/v1/app/validate"
|
|
66
|
+
)
|
|
63
67
|
headers = {
|
|
64
68
|
"Authorization": f"bearer {authorization_bearer}",
|
|
65
|
-
"Cache-Control": "no-cache"
|
|
69
|
+
"Cache-Control": "no-cache",
|
|
66
70
|
}
|
|
67
71
|
|
|
68
72
|
package_path = config.getPackageFilePath(include_version=False)
|
|
69
73
|
if not package_path.is_file():
|
|
70
|
-
raise Exception(
|
|
71
|
-
|
|
72
|
-
|
|
74
|
+
raise Exception(
|
|
75
|
+
f"Cannot run Appinspect API on App '{config.app.title}' - "
|
|
76
|
+
f"no package exists as expected path '{package_path}'.\nAre you "
|
|
77
|
+
"trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?"
|
|
78
|
+
)
|
|
73
79
|
|
|
74
80
|
files = {
|
|
75
81
|
"app_package": open(package_path, "rb"),
|
|
76
|
-
"included_tags": (None, "cloud")
|
|
82
|
+
"included_tags": (None, "cloud"),
|
|
77
83
|
}
|
|
78
84
|
|
|
79
85
|
res = post(APPINSPECT_API_VALIDATION_REQUEST, headers=headers, files=files)
|
|
@@ -82,26 +88,27 @@ class Inspect:
|
|
|
82
88
|
|
|
83
89
|
request_id = res.json().get("request_id", None)
|
|
84
90
|
APPINSPECT_API_VALIDATION_STATUS = f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}?included_tags=private_{config.stack_type}"
|
|
85
|
-
headers = headers = {
|
|
86
|
-
"Authorization": f"bearer {authorization_bearer}"
|
|
87
|
-
}
|
|
91
|
+
headers = headers = {"Authorization": f"bearer {authorization_bearer}"}
|
|
88
92
|
startTime = timeit.default_timer()
|
|
89
93
|
# the first time, wait for 40 seconds. subsequent times, wait for less.
|
|
90
94
|
# this is because appinspect takes some time to return, so there is no sense
|
|
91
95
|
# checking many times when we know it will take at least 40 seconds to run.
|
|
92
96
|
iteration_wait_time = 40
|
|
93
97
|
while True:
|
|
94
|
-
|
|
95
98
|
res = get(APPINSPECT_API_VALIDATION_STATUS, headers=headers)
|
|
96
99
|
res.raise_for_status()
|
|
97
100
|
status = res.json().get("status", None)
|
|
98
101
|
if status in ["PROCESSING", "PREPARING"]:
|
|
99
|
-
print(
|
|
102
|
+
print(
|
|
103
|
+
f"[{self.getElapsedTime(startTime)}] Appinspect API is {status}..."
|
|
104
|
+
)
|
|
100
105
|
time.sleep(iteration_wait_time)
|
|
101
106
|
iteration_wait_time = 1
|
|
102
107
|
continue
|
|
103
108
|
elif status == "SUCCESS":
|
|
104
|
-
print(
|
|
109
|
+
print(
|
|
110
|
+
f"[{self.getElapsedTime(startTime)}] Appinspect API has finished!"
|
|
111
|
+
)
|
|
105
112
|
break
|
|
106
113
|
else:
|
|
107
114
|
raise Exception(f"Error - Unknown Appinspect API status '{status}'")
|
|
@@ -111,7 +118,7 @@ class Inspect:
|
|
|
111
118
|
# Get human-readable HTML report
|
|
112
119
|
headers = headers = {
|
|
113
120
|
"Authorization": f"bearer {authorization_bearer}",
|
|
114
|
-
"Content-Type": "text/html"
|
|
121
|
+
"Content-Type": "text/html",
|
|
115
122
|
}
|
|
116
123
|
res = get(APPINSPECT_API_REPORT, headers=headers)
|
|
117
124
|
res.raise_for_status()
|
|
@@ -120,7 +127,7 @@ class Inspect:
|
|
|
120
127
|
# Get JSON report for processing
|
|
121
128
|
headers = headers = {
|
|
122
129
|
"Authorization": f"bearer {authorization_bearer}",
|
|
123
|
-
"Content-Type": "application/json"
|
|
130
|
+
"Content-Type": "application/json",
|
|
124
131
|
}
|
|
125
132
|
res = get(APPINSPECT_API_REPORT, headers=headers)
|
|
126
133
|
res.raise_for_status()
|
|
@@ -128,8 +135,12 @@ class Inspect:
|
|
|
128
135
|
|
|
129
136
|
# Just get app path here to avoid long function calls in the open() calls below
|
|
130
137
|
appPath = config.getPackageFilePath(include_version=True)
|
|
131
|
-
appinpect_html_path = appPath.with_suffix(
|
|
132
|
-
|
|
138
|
+
appinpect_html_path = appPath.with_suffix(
|
|
139
|
+
appPath.suffix + ".appinspect_api_results.html"
|
|
140
|
+
)
|
|
141
|
+
appinspect_json_path = appPath.with_suffix(
|
|
142
|
+
appPath.suffix + ".appinspect_api_results.json"
|
|
143
|
+
)
|
|
133
144
|
# Use the full path of the app, but update the suffix to include info about appinspect
|
|
134
145
|
with open(appinpect_html_path, "wb") as report:
|
|
135
146
|
report.write(report_html)
|
|
@@ -148,9 +159,15 @@ class Inspect:
|
|
|
148
159
|
"\t - https://dev.splunk.com/enterprise/docs/developapps/testvalidate/appinspect/useappinspectclitool/"
|
|
149
160
|
)
|
|
150
161
|
from splunk_appinspect.main import (
|
|
151
|
-
validate,
|
|
152
|
-
|
|
153
|
-
|
|
162
|
+
validate,
|
|
163
|
+
MODE_OPTION,
|
|
164
|
+
APP_PACKAGE_ARGUMENT,
|
|
165
|
+
OUTPUT_FILE_OPTION,
|
|
166
|
+
LOG_FILE_OPTION,
|
|
167
|
+
INCLUDED_TAGS_OPTION,
|
|
168
|
+
EXCLUDED_TAGS_OPTION,
|
|
169
|
+
TEST_MODE,
|
|
170
|
+
)
|
|
154
171
|
except Exception as e:
|
|
155
172
|
print(e)
|
|
156
173
|
# print("******WARNING******")
|
|
@@ -175,10 +192,18 @@ class Inspect:
|
|
|
175
192
|
included_tags = []
|
|
176
193
|
excluded_tags = []
|
|
177
194
|
|
|
178
|
-
appinspect_output =
|
|
179
|
-
|
|
195
|
+
appinspect_output = (
|
|
196
|
+
self.dist
|
|
197
|
+
/ f"{self.config.build.title}-{self.config.build.version}.appinspect_cli_results.json"
|
|
198
|
+
)
|
|
199
|
+
appinspect_logging = (
|
|
200
|
+
self.dist
|
|
201
|
+
/ f"{self.config.build.title}-{self.config.build.version}.appinspect_cli_logging.log"
|
|
202
|
+
)
|
|
180
203
|
try:
|
|
181
|
-
arguments_list = [
|
|
204
|
+
arguments_list = [
|
|
205
|
+
(APP_PACKAGE_ARGUMENT, str(self.getPackagePath(include_version=False)))
|
|
206
|
+
]
|
|
182
207
|
options_list = []
|
|
183
208
|
options_list += [MODE_OPTION, TEST_MODE]
|
|
184
209
|
options_list += [OUTPUT_FILE_OPTION, str(appinspect_output)]
|
|
@@ -198,16 +223,22 @@ class Inspect:
|
|
|
198
223
|
# The sys.exit called inside of appinspect validate closes stdin. We need to
|
|
199
224
|
# reopen it.
|
|
200
225
|
sys.stdin = open("/dev/stdin", "r")
|
|
201
|
-
print(
|
|
226
|
+
print(
|
|
227
|
+
f"AppInspect passed! Please check [ {appinspect_output} , {appinspect_logging} ] for verbose information."
|
|
228
|
+
)
|
|
202
229
|
else:
|
|
203
|
-
if sys.version.startswith(
|
|
204
|
-
raise Exception(
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
230
|
+
if sys.version.startswith("3.11") or sys.version.startswith("3.12"):
|
|
231
|
+
raise Exception(
|
|
232
|
+
"At this time, AppInspect may fail on valid apps under Python>=3.11 with "
|
|
233
|
+
"the error 'global flags not at the start of the expression at position 1'. "
|
|
234
|
+
"If you encounter this error, please run AppInspect on a version of Python "
|
|
235
|
+
"<3.11. This issue is currently tracked. Please review the appinspect "
|
|
236
|
+
"report output above for errors."
|
|
237
|
+
)
|
|
209
238
|
else:
|
|
210
|
-
raise Exception(
|
|
239
|
+
raise Exception(
|
|
240
|
+
"AppInspect Failure - Please review the appinspect report output above for errors."
|
|
241
|
+
)
|
|
211
242
|
finally:
|
|
212
243
|
# appinspect outputs the log in json format, but does not format it to be easier
|
|
213
244
|
# to read (it is all in one line). Read back that file and write it so it
|
|
@@ -217,20 +248,26 @@ class Inspect:
|
|
|
217
248
|
self.parseAppinspectJsonLogFile(appinspect_output)
|
|
218
249
|
|
|
219
250
|
def parseAppinspectJsonLogFile(
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
251
|
+
self,
|
|
252
|
+
logfile_path: pathlib.Path,
|
|
253
|
+
status_types: list[str] = ["error", "failure", "manual_check", "warning"],
|
|
254
|
+
exception_types: list[str] = ["error", "failure", "manual_check"],
|
|
224
255
|
) -> None:
|
|
225
256
|
if not set(exception_types).issubset(set(status_types)):
|
|
226
|
-
raise Exception(
|
|
257
|
+
raise Exception(
|
|
258
|
+
f"Error - exception_types {exception_types} MUST be a subset of status_types {status_types}, but it is not"
|
|
259
|
+
)
|
|
227
260
|
with open(logfile_path, "r+") as logfile:
|
|
228
261
|
j = json.load(logfile)
|
|
229
262
|
# Seek back to the beginning of the file. We don't need to clear
|
|
230
263
|
# it sice we will always write AT LEAST the same number of characters
|
|
231
264
|
# back as we read (due to the addition of whitespace)
|
|
232
265
|
logfile.seek(0)
|
|
233
|
-
json.dump(
|
|
266
|
+
json.dump(
|
|
267
|
+
j,
|
|
268
|
+
logfile,
|
|
269
|
+
indent=3,
|
|
270
|
+
)
|
|
234
271
|
|
|
235
272
|
reports = j.get("reports", [])
|
|
236
273
|
if len(reports) != 1:
|
|
@@ -240,7 +277,9 @@ class Inspect:
|
|
|
240
277
|
for group in reports[0].get("groups", []):
|
|
241
278
|
for check in group.get("checks", []):
|
|
242
279
|
if check.get("result", "") in status_types:
|
|
243
|
-
verbose_errors.append(
|
|
280
|
+
verbose_errors.append(
|
|
281
|
+
f" - {check.get('result', '')} [{group.get('name', 'NONAME')}: {check.get('name', 'NONAME')}]"
|
|
282
|
+
)
|
|
244
283
|
verbose_errors.sort()
|
|
245
284
|
|
|
246
285
|
summary = j.get("summary", None)
|
|
@@ -250,17 +289,21 @@ class Inspect:
|
|
|
250
289
|
generated_exception = False
|
|
251
290
|
for key in status_types:
|
|
252
291
|
if summary.get(key, 0) > 0:
|
|
253
|
-
msgs.append(f" - {summary.get(key,0)} {key}s")
|
|
292
|
+
msgs.append(f" - {summary.get(key, 0)} {key}s")
|
|
254
293
|
if key in exception_types:
|
|
255
294
|
generated_exception = True
|
|
256
295
|
if len(msgs) > 0 or len(verbose_errors):
|
|
257
|
-
summary =
|
|
258
|
-
details =
|
|
296
|
+
summary = "\n".join(msgs)
|
|
297
|
+
details = "\n".join(verbose_errors)
|
|
259
298
|
summary = f"{summary}\nDetails:\n{details}"
|
|
260
299
|
if generated_exception:
|
|
261
|
-
raise Exception(
|
|
300
|
+
raise Exception(
|
|
301
|
+
f"AppInspect found [{','.join(exception_types)}] that MUST be addressed to pass AppInspect API:\n{summary}"
|
|
302
|
+
)
|
|
262
303
|
else:
|
|
263
|
-
print(
|
|
304
|
+
print(
|
|
305
|
+
f"AppInspect found [{','.join(status_types)}] that MAY cause a failure during AppInspect API:\n{summary}"
|
|
306
|
+
)
|
|
264
307
|
else:
|
|
265
308
|
print("AppInspect was successful!")
|
|
266
309
|
|
|
@@ -283,12 +326,12 @@ class Inspect:
|
|
|
283
326
|
current_build_conf = SavedsearchesConf.init_from_package(
|
|
284
327
|
package_path=config.getPackageFilePath(include_version=False),
|
|
285
328
|
app_name=config.app.label,
|
|
286
|
-
appid=config.app.appid
|
|
329
|
+
appid=config.app.appid,
|
|
287
330
|
)
|
|
288
331
|
previous_build_conf = SavedsearchesConf.init_from_package(
|
|
289
332
|
package_path=config.get_previous_package_file_path(),
|
|
290
333
|
app_name=config.app.label,
|
|
291
|
-
appid=config.app.appid
|
|
334
|
+
appid=config.app.appid,
|
|
292
335
|
)
|
|
293
336
|
|
|
294
337
|
# Compare the conf files
|
|
@@ -298,31 +341,41 @@ class Inspect:
|
|
|
298
341
|
# No detections should be removed from build to build
|
|
299
342
|
if rule_name not in current_build_conf.detection_stanzas:
|
|
300
343
|
if config.suppress_missing_content_exceptions:
|
|
301
|
-
print(
|
|
344
|
+
print(
|
|
345
|
+
f"[SUPPRESSED] {DetectionMissingError(rule_name=rule_name).long_message}"
|
|
346
|
+
)
|
|
302
347
|
else:
|
|
303
|
-
validation_errors[rule_name].append(
|
|
348
|
+
validation_errors[rule_name].append(
|
|
349
|
+
DetectionMissingError(rule_name=rule_name)
|
|
350
|
+
)
|
|
304
351
|
continue
|
|
305
352
|
# Pull out the individual stanza for readability
|
|
306
353
|
previous_stanza = previous_build_conf.detection_stanzas[rule_name]
|
|
307
354
|
current_stanza = current_build_conf.detection_stanzas[rule_name]
|
|
308
355
|
|
|
309
356
|
# Detection IDs should not change
|
|
310
|
-
if
|
|
357
|
+
if (
|
|
358
|
+
current_stanza.metadata.detection_id
|
|
359
|
+
!= previous_stanza.metadata.detection_id
|
|
360
|
+
):
|
|
311
361
|
validation_errors[rule_name].append(
|
|
312
362
|
DetectionIDError(
|
|
313
363
|
rule_name=rule_name,
|
|
314
364
|
current_id=current_stanza.metadata.detection_id,
|
|
315
|
-
previous_id=previous_stanza.metadata.detection_id
|
|
365
|
+
previous_id=previous_stanza.metadata.detection_id,
|
|
316
366
|
)
|
|
317
367
|
)
|
|
318
368
|
|
|
319
369
|
# Versions should never decrement in successive builds
|
|
320
|
-
if
|
|
370
|
+
if (
|
|
371
|
+
current_stanza.metadata.detection_version
|
|
372
|
+
< previous_stanza.metadata.detection_version
|
|
373
|
+
):
|
|
321
374
|
validation_errors[rule_name].append(
|
|
322
375
|
VersionDecrementedError(
|
|
323
376
|
rule_name=rule_name,
|
|
324
377
|
current_version=current_stanza.metadata.detection_version,
|
|
325
|
-
previous_version=previous_stanza.metadata.detection_version
|
|
378
|
+
previous_version=previous_stanza.metadata.detection_version,
|
|
326
379
|
)
|
|
327
380
|
)
|
|
328
381
|
|
|
@@ -332,12 +385,14 @@ class Inspect:
|
|
|
332
385
|
VersionBumpingError(
|
|
333
386
|
rule_name=rule_name,
|
|
334
387
|
current_version=current_stanza.metadata.detection_version,
|
|
335
|
-
previous_version=previous_stanza.metadata.detection_version
|
|
388
|
+
previous_version=previous_stanza.metadata.detection_version,
|
|
336
389
|
)
|
|
337
390
|
)
|
|
338
391
|
|
|
339
392
|
# Convert our dict mapping to a flat list of errors for use in reporting
|
|
340
|
-
validation_error_list = [
|
|
393
|
+
validation_error_list = [
|
|
394
|
+
x for inner_list in validation_errors.values() for x in inner_list
|
|
395
|
+
]
|
|
341
396
|
|
|
342
397
|
# Report failure/success
|
|
343
398
|
print("\nDetection Metadata Validation:")
|
|
@@ -350,11 +405,13 @@ class Inspect:
|
|
|
350
405
|
print(f"\t\t🔸 {error.short_message}")
|
|
351
406
|
else:
|
|
352
407
|
# If no errors in the list, report success
|
|
353
|
-
print(
|
|
408
|
+
print(
|
|
409
|
+
"\t✅ Detection metadata looks good and all versions were bumped appropriately :)"
|
|
410
|
+
)
|
|
354
411
|
|
|
355
412
|
# Raise an ExceptionGroup for all validation issues
|
|
356
413
|
if len(validation_error_list) > 0:
|
|
357
414
|
raise ExceptionGroup(
|
|
358
415
|
"Validation errors when comparing detection stanzas in current and previous build:",
|
|
359
|
-
validation_error_list
|
|
360
|
-
)
|
|
416
|
+
validation_error_list,
|
|
417
|
+
)
|
|
@@ -1,33 +1,44 @@
|
|
|
1
|
-
|
|
2
|
-
import questionary
|
|
3
|
-
from typing import Any
|
|
4
|
-
from contentctl.input.new_content_questions import NewContentQuestions
|
|
5
|
-
from contentctl.objects.config import new, NewContentType
|
|
1
|
+
import pathlib
|
|
6
2
|
import uuid
|
|
7
3
|
from datetime import datetime
|
|
8
|
-
import
|
|
9
|
-
|
|
10
|
-
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import questionary
|
|
7
|
+
|
|
8
|
+
from contentctl.input.new_content_questions import NewContentQuestions
|
|
9
|
+
from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import (
|
|
10
|
+
SecurityContentObject_Abstract,
|
|
11
|
+
)
|
|
12
|
+
from contentctl.objects.config import NewContentType, new
|
|
11
13
|
from contentctl.objects.enums import AssetType
|
|
12
|
-
from contentctl.
|
|
14
|
+
from contentctl.output.yml_writer import YmlWriter
|
|
15
|
+
|
|
16
|
+
|
|
13
17
|
class NewContent:
|
|
14
18
|
UPDATE_PREFIX = "__UPDATE__"
|
|
15
|
-
|
|
19
|
+
|
|
16
20
|
DEFAULT_DRILLDOWN_DEF = [
|
|
17
21
|
{
|
|
18
22
|
"name": f'View the detection results for - "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" and "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"',
|
|
19
23
|
"search": f'%original_detection_search% | search "${UPDATE_PREFIX}FIRST_RISK_OBJECT = "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" second_observable_type_here = "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"',
|
|
20
|
-
"earliest_offset":
|
|
21
|
-
"latest_offset":
|
|
24
|
+
"earliest_offset": "$info_min_time$",
|
|
25
|
+
"latest_offset": "$info_max_time$",
|
|
22
26
|
},
|
|
23
27
|
{
|
|
24
28
|
"name": f'View risk events for the last 7 days for - "${UPDATE_PREFIX}FIRST_RISK_OBJECT$" and "${UPDATE_PREFIX}SECOND_RISK_OBJECT$"',
|
|
25
29
|
"search": f'| from datamodel Risk.All_Risk | search normalized_risk_object IN ("${UPDATE_PREFIX}FIRST_RISK_OBJECT$", "${UPDATE_PREFIX}SECOND_RISK_OBJECT$") starthoursago=168 | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as "Search Name" values(risk_message) as "Risk Message" values(analyticstories) as "Analytic Stories" values(annotations._all) as "Annotations" values(annotations.mitre_attack.mitre_tactic) as "ATT&CK Tactics" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`',
|
|
26
|
-
"earliest_offset":
|
|
27
|
-
"latest_offset":
|
|
28
|
-
}
|
|
30
|
+
"earliest_offset": "$info_min_time$",
|
|
31
|
+
"latest_offset": "$info_max_time$",
|
|
32
|
+
},
|
|
29
33
|
]
|
|
30
|
-
|
|
34
|
+
|
|
35
|
+
DEFAULT_RBA = {
|
|
36
|
+
"message": "Risk Message goes here",
|
|
37
|
+
"risk_objects": [{"field": "dest", "type": "system", "score": 10}],
|
|
38
|
+
"threat_objects": [
|
|
39
|
+
{"field": "parent_process_name", "type": "parent_process_name"}
|
|
40
|
+
],
|
|
41
|
+
}
|
|
31
42
|
|
|
32
43
|
def buildDetection(self) -> tuple[dict[str, Any], str]:
|
|
33
44
|
questions = NewContentQuestions.get_questions_detection()
|
|
@@ -39,7 +50,9 @@ class NewContent:
|
|
|
39
50
|
raise ValueError("User didn't answer one or more questions!")
|
|
40
51
|
|
|
41
52
|
data_source_field = (
|
|
42
|
-
answers["
|
|
53
|
+
answers["data_sources"]
|
|
54
|
+
if len(answers["data_sources"]) > 0
|
|
55
|
+
else [f"{NewContent.UPDATE_PREFIX} zero or more data_sources"]
|
|
43
56
|
)
|
|
44
57
|
file_name = (
|
|
45
58
|
answers["detection_name"]
|
|
@@ -50,12 +63,16 @@ class NewContent:
|
|
|
50
63
|
.lower()
|
|
51
64
|
)
|
|
52
65
|
|
|
53
|
-
#Minimum lenght for a mitre tactic is 5 characters: T1000
|
|
66
|
+
# Minimum lenght for a mitre tactic is 5 characters: T1000
|
|
54
67
|
if len(answers["mitre_attack_ids"]) >= 5:
|
|
55
|
-
mitre_attack_ids = [
|
|
68
|
+
mitre_attack_ids = [
|
|
69
|
+
x.strip() for x in answers["mitre_attack_ids"].split(",")
|
|
70
|
+
]
|
|
56
71
|
else:
|
|
57
|
-
#string was too short, so just put a placeholder
|
|
58
|
-
mitre_attack_ids = [
|
|
72
|
+
# string was too short, so just put a placeholder
|
|
73
|
+
mitre_attack_ids = [
|
|
74
|
+
f"{NewContent.UPDATE_PREFIX} zero or more mitre_attack_ids"
|
|
75
|
+
]
|
|
59
76
|
|
|
60
77
|
output_file_answers: dict[str, Any] = {
|
|
61
78
|
"name": answers["detection_name"],
|
|
@@ -70,18 +87,17 @@ class NewContent:
|
|
|
70
87
|
"search": f"{answers['detection_search']} | `{file_name}_filter`",
|
|
71
88
|
"how_to_implement": f"{NewContent.UPDATE_PREFIX} how to implement your search",
|
|
72
89
|
"known_false_positives": f"{NewContent.UPDATE_PREFIX} known false positives for your search",
|
|
73
|
-
"references": [
|
|
90
|
+
"references": [
|
|
91
|
+
f"{NewContent.UPDATE_PREFIX} zero or more http references to provide more information about your search"
|
|
92
|
+
],
|
|
74
93
|
"drilldown_searches": NewContent.DEFAULT_DRILLDOWN_DEF,
|
|
94
|
+
"rba": NewContent.DEFAULT_RBA,
|
|
75
95
|
"tags": {
|
|
76
|
-
"analytic_story": [
|
|
96
|
+
"analytic_story": [
|
|
97
|
+
f"{NewContent.UPDATE_PREFIX} by providing zero or more analytic stories"
|
|
98
|
+
],
|
|
77
99
|
"asset_type": f"{NewContent.UPDATE_PREFIX} by providing and asset type from {list(AssetType._value2member_map_)}",
|
|
78
|
-
"confidence": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100",
|
|
79
|
-
"impact": f"{NewContent.UPDATE_PREFIX} by providing a value between 1-100",
|
|
80
|
-
"message": f"{NewContent.UPDATE_PREFIX} by providing a risk message. Fields in your search results can be referenced using $fieldName$",
|
|
81
100
|
"mitre_attack_id": mitre_attack_ids,
|
|
82
|
-
"observable": [
|
|
83
|
-
{"name": f"{NewContent.UPDATE_PREFIX} the field name of the observable. This is a field that exists in your search results.", "type": f"{NewContent.UPDATE_PREFIX} the type of your observable from the list {list(SES_OBSERVABLE_TYPE_MAPPING.keys())}.", "role": [f"{NewContent.UPDATE_PREFIX} the role from the list {list(SES_OBSERVABLE_ROLE_MAPPING.keys())}"]}
|
|
84
|
-
],
|
|
85
101
|
"product": [
|
|
86
102
|
"Splunk Enterprise",
|
|
87
103
|
"Splunk Enterprise Security",
|
|
@@ -107,44 +123,58 @@ class NewContent:
|
|
|
107
123
|
if answers["detection_type"] not in ["TTP", "Anomaly", "Correlation"]:
|
|
108
124
|
del output_file_answers["drilldown_searches"]
|
|
109
125
|
|
|
110
|
-
|
|
126
|
+
if answers["detection_type"] not in ["TTP", "Anomaly"]:
|
|
127
|
+
del output_file_answers["rba"]
|
|
128
|
+
|
|
129
|
+
return output_file_answers, answers["detection_kind"]
|
|
111
130
|
|
|
112
131
|
def buildStory(self) -> dict[str, Any]:
|
|
113
132
|
questions = NewContentQuestions.get_questions_story()
|
|
114
133
|
answers = questionary.prompt(
|
|
115
|
-
questions,
|
|
116
|
-
kbi_msg="User did not answer all of the prompt questions. Exiting..."
|
|
134
|
+
questions,
|
|
135
|
+
kbi_msg="User did not answer all of the prompt questions. Exiting...",
|
|
136
|
+
)
|
|
117
137
|
if not answers:
|
|
118
138
|
raise ValueError("User didn't answer one or more questions!")
|
|
119
|
-
answers[
|
|
120
|
-
del answers[
|
|
121
|
-
answers[
|
|
122
|
-
answers[
|
|
123
|
-
answers[
|
|
124
|
-
answers[
|
|
125
|
-
|
|
126
|
-
answers[
|
|
127
|
-
answers[
|
|
128
|
-
answers[
|
|
129
|
-
answers[
|
|
130
|
-
answers[
|
|
131
|
-
|
|
132
|
-
answers[
|
|
133
|
-
answers[
|
|
134
|
-
|
|
135
|
-
|
|
139
|
+
answers["name"] = answers["story_name"]
|
|
140
|
+
del answers["story_name"]
|
|
141
|
+
answers["id"] = str(uuid.uuid4())
|
|
142
|
+
answers["version"] = 1
|
|
143
|
+
answers["status"] = "production"
|
|
144
|
+
answers["date"] = datetime.today().strftime("%Y-%m-%d")
|
|
145
|
+
answers["author"] = answers["story_author"]
|
|
146
|
+
del answers["story_author"]
|
|
147
|
+
answers["description"] = "UPDATE_DESCRIPTION"
|
|
148
|
+
answers["narrative"] = "UPDATE_NARRATIVE"
|
|
149
|
+
answers["references"] = []
|
|
150
|
+
answers["tags"] = dict()
|
|
151
|
+
answers["tags"]["category"] = answers["category"]
|
|
152
|
+
del answers["category"]
|
|
153
|
+
answers["tags"]["product"] = [
|
|
154
|
+
"Splunk Enterprise",
|
|
155
|
+
"Splunk Enterprise Security",
|
|
156
|
+
"Splunk Cloud",
|
|
157
|
+
]
|
|
158
|
+
answers["tags"]["usecase"] = answers["usecase"]
|
|
159
|
+
del answers["usecase"]
|
|
160
|
+
answers["tags"]["cve"] = ["UPDATE WITH CVE(S) IF APPLICABLE"]
|
|
136
161
|
return answers
|
|
137
162
|
|
|
138
163
|
def execute(self, input_dto: new) -> None:
|
|
139
164
|
if input_dto.type == NewContentType.detection:
|
|
140
165
|
content_dict, detection_kind = self.buildDetection()
|
|
141
|
-
subdirectory = pathlib.Path(
|
|
166
|
+
subdirectory = pathlib.Path("detections") / detection_kind
|
|
142
167
|
elif input_dto.type == NewContentType.story:
|
|
143
168
|
content_dict = self.buildStory()
|
|
144
|
-
subdirectory = pathlib.Path(
|
|
169
|
+
subdirectory = pathlib.Path("stories")
|
|
145
170
|
else:
|
|
146
171
|
raise Exception(f"Unsupported new content type: [{input_dto.type}]")
|
|
147
172
|
|
|
148
|
-
full_output_path =
|
|
173
|
+
full_output_path = (
|
|
174
|
+
input_dto.path
|
|
175
|
+
/ subdirectory
|
|
176
|
+
/ SecurityContentObject_Abstract.contentNameToFileName(
|
|
177
|
+
content_dict.get("name")
|
|
178
|
+
)
|
|
179
|
+
)
|
|
149
180
|
YmlWriter.writeYmlFile(str(full_output_path), content_dict)
|
|
150
|
-
|