contentctl 4.4.7__py3-none-any.whl → 5.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/build.py +39 -27
- contentctl/actions/detection_testing/DetectionTestingManager.py +0 -1
- contentctl/actions/detection_testing/GitService.py +132 -72
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +32 -26
- contentctl/actions/detection_testing/progress_bar.py +6 -6
- contentctl/actions/detection_testing/views/DetectionTestingView.py +4 -4
- contentctl/actions/new_content.py +98 -81
- contentctl/actions/test.py +4 -5
- contentctl/actions/validate.py +2 -1
- contentctl/contentctl.py +114 -80
- contentctl/helper/utils.py +0 -14
- contentctl/input/director.py +5 -5
- contentctl/input/new_content_questions.py +2 -2
- contentctl/input/yml_reader.py +11 -6
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +228 -120
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +5 -7
- contentctl/objects/alert_action.py +2 -1
- contentctl/objects/atomic.py +1 -0
- contentctl/objects/base_test.py +4 -3
- contentctl/objects/base_test_result.py +3 -3
- contentctl/objects/baseline.py +26 -6
- contentctl/objects/baseline_tags.py +2 -3
- contentctl/objects/config.py +789 -596
- contentctl/objects/constants.py +4 -1
- contentctl/objects/correlation_search.py +89 -95
- contentctl/objects/data_source.py +5 -6
- contentctl/objects/deployment.py +2 -10
- contentctl/objects/deployment_email.py +2 -1
- contentctl/objects/deployment_notable.py +2 -1
- contentctl/objects/deployment_phantom.py +2 -1
- contentctl/objects/deployment_rba.py +2 -1
- contentctl/objects/deployment_scheduling.py +2 -1
- contentctl/objects/deployment_slack.py +2 -1
- contentctl/objects/detection_tags.py +7 -42
- contentctl/objects/drilldown.py +1 -0
- contentctl/objects/enums.py +21 -58
- contentctl/objects/investigation.py +6 -5
- contentctl/objects/investigation_tags.py +2 -3
- contentctl/objects/lookup.py +145 -63
- contentctl/objects/macro.py +2 -3
- contentctl/objects/mitre_attack_enrichment.py +2 -2
- contentctl/objects/observable.py +3 -1
- contentctl/objects/playbook_tags.py +5 -1
- contentctl/objects/rba.py +90 -0
- contentctl/objects/risk_event.py +87 -144
- contentctl/objects/story_tags.py +1 -2
- contentctl/objects/test_attack_data.py +2 -1
- contentctl/objects/unit_test_baseline.py +2 -1
- contentctl/output/api_json_output.py +233 -220
- contentctl/output/conf_output.py +51 -44
- contentctl/output/conf_writer.py +201 -125
- contentctl/output/data_source_writer.py +0 -1
- contentctl/output/json_writer.py +2 -4
- contentctl/output/svg_output.py +1 -1
- contentctl/output/templates/analyticstories_detections.j2 +1 -1
- contentctl/output/templates/collections.j2 +1 -1
- contentctl/output/templates/doc_detections.j2 +0 -5
- contentctl/output/templates/savedsearches_detections.j2 +8 -3
- contentctl/output/templates/transforms.j2 +4 -4
- contentctl/output/yml_writer.py +15 -0
- contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +16 -34
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/METADATA +5 -4
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/RECORD +66 -69
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/WHEEL +1 -1
- contentctl/objects/event_source.py +0 -11
- contentctl/output/detection_writer.py +0 -28
- contentctl/output/new_content_yml_output.py +0 -56
- contentctl/output/yml_output.py +0 -66
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/LICENSE.md +0 -0
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/entry_points.txt +0 -0
contentctl/output/conf_writer.py
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
|
-
|
|
1
|
+
import configparser
|
|
2
2
|
import datetime
|
|
3
|
-
import re
|
|
4
|
-
import os
|
|
5
3
|
import json
|
|
6
|
-
import
|
|
7
|
-
from xmlrpc.client import APPLICATION_ERROR
|
|
8
|
-
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
|
4
|
+
import os
|
|
9
5
|
import pathlib
|
|
10
|
-
|
|
11
|
-
from contentctl.objects.dashboard import Dashboard
|
|
12
|
-
from contentctl.objects.config import build
|
|
6
|
+
import re
|
|
13
7
|
import xml.etree.ElementTree as ET
|
|
8
|
+
from typing import Any, Sequence
|
|
9
|
+
|
|
10
|
+
from jinja2 import Environment, FileSystemLoader, StrictUndefined
|
|
11
|
+
|
|
12
|
+
from contentctl.objects.config import CustomApp, build
|
|
13
|
+
from contentctl.objects.dashboard import Dashboard
|
|
14
|
+
from contentctl.objects.security_content_object import SecurityContentObject
|
|
14
15
|
|
|
15
16
|
# This list is not exhaustive of all default conf files, but should be
|
|
16
17
|
# sufficient for our purposes.
|
|
@@ -82,59 +83,68 @@ DEFAULT_CONF_FILES = [
|
|
|
82
83
|
"workload_rules.conf",
|
|
83
84
|
]
|
|
84
85
|
|
|
85
|
-
class ConfWriter():
|
|
86
86
|
|
|
87
|
+
class ConfWriter:
|
|
87
88
|
@staticmethod
|
|
88
|
-
def custom_jinja2_enrichment_filter(string:str, object:SecurityContentObject):
|
|
89
|
+
def custom_jinja2_enrichment_filter(string: str, object: SecurityContentObject):
|
|
89
90
|
substitutions = re.findall(r"%[^%]*%", string)
|
|
90
91
|
updated_string = string
|
|
91
92
|
for sub in substitutions:
|
|
92
|
-
sub_without_percents = sub.replace("%","")
|
|
93
|
+
sub_without_percents = sub.replace("%", "")
|
|
93
94
|
if hasattr(object, sub_without_percents):
|
|
94
|
-
updated_string = updated_string.replace(
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
updated_string = updated_string.replace(
|
|
96
|
+
sub, str(getattr(object, sub_without_percents))
|
|
97
|
+
)
|
|
98
|
+
elif hasattr(object, "tags") and hasattr(object.tags, sub_without_percents):
|
|
99
|
+
updated_string = updated_string.replace(
|
|
100
|
+
sub, str(getattr(object.tags, sub_without_percents))
|
|
101
|
+
)
|
|
97
102
|
else:
|
|
98
103
|
raise Exception(f"Unable to find field {sub} in object {object.name}")
|
|
99
|
-
|
|
104
|
+
|
|
100
105
|
return updated_string
|
|
101
|
-
|
|
106
|
+
|
|
102
107
|
@staticmethod
|
|
103
|
-
def escapeNewlines(obj:Any):
|
|
108
|
+
def escapeNewlines(obj: Any):
|
|
104
109
|
# Ensure that any newlines that occur in a string are escaped with a \.
|
|
105
110
|
# Failing to do so will result in an improperly formatted conf files that
|
|
106
111
|
# cannot be parsed
|
|
107
|
-
if isinstance(obj,str):
|
|
108
|
-
# Remove leading and trailing characters. Conf parsers may erroneously
|
|
109
|
-
# Parse fields if they have leading or trailing newlines/whitespace and we
|
|
112
|
+
if isinstance(obj, str):
|
|
113
|
+
# Remove leading and trailing characters. Conf parsers may erroneously
|
|
114
|
+
# Parse fields if they have leading or trailing newlines/whitespace and we
|
|
110
115
|
# probably don't want that anyway as it doesn't look good in output
|
|
111
|
-
return obj.strip().replace(
|
|
116
|
+
return obj.strip().replace("\n", " \\\n")
|
|
112
117
|
else:
|
|
113
118
|
return obj
|
|
114
119
|
|
|
115
|
-
|
|
116
120
|
@staticmethod
|
|
117
|
-
def writeConfFileHeader(
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
+
def writeConfFileHeader(
|
|
122
|
+
app_output_path: pathlib.Path, config: build
|
|
123
|
+
) -> pathlib.Path:
|
|
124
|
+
output = ConfWriter.writeFileHeader(app_output_path, config)
|
|
125
|
+
|
|
126
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
121
127
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
122
|
-
with open(output_path,
|
|
123
|
-
output = output.encode(
|
|
128
|
+
with open(output_path, "w") as f:
|
|
129
|
+
output = output.encode("utf-8", "ignore").decode("utf-8")
|
|
124
130
|
f.write(output)
|
|
125
131
|
|
|
126
|
-
#Ensure that the conf file we just generated/update is syntactically valid
|
|
127
|
-
ConfWriter.validateConfFile(output_path)
|
|
132
|
+
# Ensure that the conf file we just generated/update is syntactically valid
|
|
133
|
+
ConfWriter.validateConfFile(output_path)
|
|
128
134
|
return output_path
|
|
129
135
|
|
|
130
136
|
@staticmethod
|
|
131
|
-
def getCustomConfFileStems(config:build)->list[str]:
|
|
137
|
+
def getCustomConfFileStems(config: build) -> list[str]:
|
|
132
138
|
# Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if
|
|
133
139
|
# they are custom conf files
|
|
134
|
-
default_path = config.getPackageDirectoryPath()/"default"
|
|
140
|
+
default_path = config.getPackageDirectoryPath() / "default"
|
|
135
141
|
conf_files = default_path.glob("*.conf")
|
|
136
|
-
|
|
137
|
-
custom_conf_file_stems = [
|
|
142
|
+
|
|
143
|
+
custom_conf_file_stems = [
|
|
144
|
+
conf_file.stem
|
|
145
|
+
for conf_file in conf_files
|
|
146
|
+
if conf_file.name not in DEFAULT_CONF_FILES
|
|
147
|
+
]
|
|
138
148
|
return sorted(custom_conf_file_stems)
|
|
139
149
|
|
|
140
150
|
@staticmethod
|
|
@@ -145,16 +155,17 @@ class ConfWriter():
|
|
|
145
155
|
j2_env = ConfWriter.getJ2Environment()
|
|
146
156
|
template = j2_env.get_template(template_name)
|
|
147
157
|
|
|
148
|
-
output = template.render(
|
|
149
|
-
|
|
150
|
-
|
|
158
|
+
output = template.render(
|
|
159
|
+
custom_conf_files=ConfWriter.getCustomConfFileStems(config)
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
151
163
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
152
|
-
with open(output_path,
|
|
153
|
-
output = output.encode(
|
|
164
|
+
with open(output_path, "a") as f:
|
|
165
|
+
output = output.encode("utf-8", "ignore").decode("utf-8")
|
|
154
166
|
f.write(output)
|
|
155
167
|
return output_path
|
|
156
168
|
|
|
157
|
-
|
|
158
169
|
@staticmethod
|
|
159
170
|
def writeAppConf(config: build) -> pathlib.Path:
|
|
160
171
|
app_output_path = pathlib.Path("default/app.conf")
|
|
@@ -163,135 +174,195 @@ class ConfWriter():
|
|
|
163
174
|
j2_env = ConfWriter.getJ2Environment()
|
|
164
175
|
template = j2_env.get_template(template_name)
|
|
165
176
|
|
|
166
|
-
output = template.render(
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
177
|
+
output = template.render(
|
|
178
|
+
custom_conf_files=ConfWriter.getCustomConfFileStems(config), app=config.app
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
170
182
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
171
|
-
with open(output_path,
|
|
172
|
-
output = output.encode(
|
|
183
|
+
with open(output_path, "a") as f:
|
|
184
|
+
output = output.encode("utf-8", "ignore").decode("utf-8")
|
|
173
185
|
f.write(output)
|
|
174
186
|
return output_path
|
|
175
187
|
|
|
176
188
|
@staticmethod
|
|
177
|
-
def writeManifestFile(
|
|
189
|
+
def writeManifestFile(
|
|
190
|
+
app_output_path: pathlib.Path,
|
|
191
|
+
template_name: str,
|
|
192
|
+
config: build,
|
|
193
|
+
objects: list[CustomApp],
|
|
194
|
+
) -> pathlib.Path:
|
|
178
195
|
j2_env = ConfWriter.getJ2Environment()
|
|
179
196
|
template = j2_env.get_template(template_name)
|
|
180
|
-
|
|
181
|
-
output = template.render(
|
|
182
|
-
|
|
183
|
-
|
|
197
|
+
|
|
198
|
+
output = template.render(
|
|
199
|
+
objects=objects,
|
|
200
|
+
app=config.app,
|
|
201
|
+
currentDate=datetime.datetime.now(datetime.UTC).date().isoformat(),
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
184
205
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
185
|
-
with open(output_path,
|
|
186
|
-
output = output.encode(
|
|
206
|
+
with open(output_path, "w") as f:
|
|
207
|
+
output = output.encode("utf-8", "ignore").decode("utf-8")
|
|
187
208
|
f.write(output)
|
|
188
209
|
return output_path
|
|
189
|
-
|
|
190
|
-
|
|
191
210
|
|
|
192
211
|
@staticmethod
|
|
193
|
-
def writeFileHeader(app_output_path:pathlib.Path, config: build) -> str:
|
|
194
|
-
#Do not output microseconds or +00:000 at the end of the datetime string
|
|
195
|
-
utc_time =
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
212
|
+
def writeFileHeader(app_output_path: pathlib.Path, config: build) -> str:
|
|
213
|
+
# Do not output microseconds or +00:000 at the end of the datetime string
|
|
214
|
+
utc_time = (
|
|
215
|
+
datetime.datetime.now(datetime.UTC)
|
|
216
|
+
.replace(microsecond=0, tzinfo=None)
|
|
217
|
+
.isoformat()
|
|
218
|
+
)
|
|
200
219
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
220
|
+
j2_env = Environment(
|
|
221
|
+
loader=FileSystemLoader(
|
|
222
|
+
os.path.join(os.path.dirname(__file__), "templates")
|
|
223
|
+
),
|
|
224
|
+
trim_blocks=True,
|
|
225
|
+
)
|
|
205
226
|
|
|
227
|
+
template = j2_env.get_template("header.j2")
|
|
228
|
+
output = template.render(
|
|
229
|
+
time=utc_time,
|
|
230
|
+
author=" - ".join([config.app.author_name, config.app.author_company]),
|
|
231
|
+
author_email=config.app.author_email,
|
|
232
|
+
)
|
|
206
233
|
|
|
234
|
+
return output
|
|
207
235
|
|
|
208
236
|
@staticmethod
|
|
209
|
-
def writeXmlFile(
|
|
210
|
-
|
|
211
|
-
|
|
237
|
+
def writeXmlFile(
|
|
238
|
+
app_output_path: pathlib.Path,
|
|
239
|
+
template_name: str,
|
|
240
|
+
config: build,
|
|
241
|
+
objects: list[str],
|
|
242
|
+
) -> None:
|
|
212
243
|
j2_env = ConfWriter.getJ2Environment()
|
|
213
244
|
template = j2_env.get_template(template_name)
|
|
214
|
-
|
|
245
|
+
|
|
215
246
|
output = template.render(objects=objects, app=config.app)
|
|
216
|
-
|
|
217
|
-
output_path = config.getPackageDirectoryPath()/app_output_path
|
|
247
|
+
|
|
248
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
218
249
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
219
|
-
with open(output_path,
|
|
220
|
-
output = output.encode(
|
|
250
|
+
with open(output_path, "a") as f:
|
|
251
|
+
output = output.encode("utf-8", "ignore").decode("utf-8")
|
|
221
252
|
f.write(output)
|
|
222
|
-
|
|
223
|
-
#Ensure that the conf file we just generated/update is syntactically valid
|
|
224
|
-
ConfWriter.validateXmlFile(output_path)
|
|
225
253
|
|
|
226
|
-
|
|
254
|
+
# Ensure that the conf file we just generated/update is syntactically valid
|
|
255
|
+
ConfWriter.validateXmlFile(output_path)
|
|
227
256
|
|
|
228
257
|
@staticmethod
|
|
229
|
-
def writeDashboardFiles(
|
|
230
|
-
|
|
258
|
+
def writeDashboardFiles(
|
|
259
|
+
config: build, dashboards: list[Dashboard]
|
|
260
|
+
) -> set[pathlib.Path]:
|
|
261
|
+
written_files: set[pathlib.Path] = set()
|
|
231
262
|
for dashboard in dashboards:
|
|
232
263
|
output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config)
|
|
233
264
|
# Check that the full output path does not exist so that we are not having an
|
|
234
265
|
# name collision with a file in app_template
|
|
235
|
-
if (config.getPackageDirectoryPath()/output_file_path).exists():
|
|
236
|
-
raise FileExistsError(
|
|
237
|
-
|
|
266
|
+
if (config.getPackageDirectoryPath() / output_file_path).exists():
|
|
267
|
+
raise FileExistsError(
|
|
268
|
+
f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path / 'dashboards'}?"
|
|
269
|
+
)
|
|
270
|
+
|
|
238
271
|
ConfWriter.writeXmlFileHeader(output_file_path, config)
|
|
239
272
|
dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config)
|
|
240
|
-
ConfWriter.validateXmlFile(
|
|
273
|
+
ConfWriter.validateXmlFile(
|
|
274
|
+
config.getPackageDirectoryPath() / output_file_path
|
|
275
|
+
)
|
|
241
276
|
written_files.add(output_file_path)
|
|
242
277
|
return written_files
|
|
243
278
|
|
|
244
|
-
|
|
245
279
|
@staticmethod
|
|
246
|
-
def writeXmlFileHeader(app_output_path:pathlib.Path, config: build) -> None:
|
|
247
|
-
output = ConfWriter.writeFileHeader(app_output_path, config)
|
|
280
|
+
def writeXmlFileHeader(app_output_path: pathlib.Path, config: build) -> None:
|
|
281
|
+
output = ConfWriter.writeFileHeader(app_output_path, config)
|
|
248
282
|
output_with_xml_comment = f"<!--\n{output}-->\n"
|
|
249
283
|
|
|
250
|
-
output_path = config.getPackageDirectoryPath()/app_output_path
|
|
284
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
251
285
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
252
|
-
with open(output_path,
|
|
253
|
-
output_with_xml_comment = output_with_xml_comment.encode(
|
|
286
|
+
with open(output_path, "w") as f:
|
|
287
|
+
output_with_xml_comment = output_with_xml_comment.encode(
|
|
288
|
+
"utf-8", "ignore"
|
|
289
|
+
).decode("utf-8")
|
|
254
290
|
f.write(output_with_xml_comment)
|
|
255
|
-
|
|
256
|
-
# We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now,
|
|
257
|
-
# the file is an empty XML document (besides the commented header). This means that it will FAIL validation
|
|
258
291
|
|
|
292
|
+
# We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now,
|
|
293
|
+
# the file is an empty XML document (besides the commented header). This means that it will FAIL validation
|
|
259
294
|
|
|
260
295
|
@staticmethod
|
|
261
|
-
def getJ2Environment()->Environment:
|
|
296
|
+
def getJ2Environment() -> Environment:
|
|
262
297
|
j2_env = Environment(
|
|
263
|
-
loader=FileSystemLoader(
|
|
298
|
+
loader=FileSystemLoader(
|
|
299
|
+
os.path.join(os.path.dirname(__file__), "templates")
|
|
300
|
+
),
|
|
264
301
|
trim_blocks=True,
|
|
265
|
-
undefined=StrictUndefined
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
302
|
+
undefined=StrictUndefined,
|
|
303
|
+
)
|
|
304
|
+
j2_env.globals.update(
|
|
305
|
+
objectListToNameList=SecurityContentObject.objectListToNameList
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
j2_env.filters["custom_jinja2_enrichment_filter"] = (
|
|
309
|
+
ConfWriter.custom_jinja2_enrichment_filter
|
|
310
|
+
)
|
|
311
|
+
j2_env.filters["escapeNewlines"] = ConfWriter.escapeNewlines
|
|
271
312
|
return j2_env
|
|
272
313
|
|
|
273
314
|
@staticmethod
|
|
274
|
-
def writeConfFile(
|
|
275
|
-
|
|
315
|
+
def writeConfFile(
|
|
316
|
+
app_output_path: pathlib.Path,
|
|
317
|
+
template_name: str,
|
|
318
|
+
config: build,
|
|
319
|
+
objects: Sequence[SecurityContentObject] | list[CustomApp],
|
|
320
|
+
) -> pathlib.Path:
|
|
321
|
+
output_path = config.getPackageDirectoryPath() / app_output_path
|
|
276
322
|
j2_env = ConfWriter.getJ2Environment()
|
|
277
|
-
|
|
323
|
+
|
|
278
324
|
template = j2_env.get_template(template_name)
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
325
|
+
|
|
326
|
+
# The following code, which is commented out, serializes one object at a time.
|
|
327
|
+
# This is extremely useful from a debugging perspective, because sometimes when
|
|
328
|
+
# serializing a large number of objects, exceptions throw in Jinja2 templates can
|
|
329
|
+
# be quite hard to diagnose. We leave this code in for use in debugging workflows:
|
|
330
|
+
SERIALIZE_ONE_AT_A_TIME = False
|
|
331
|
+
if SERIALIZE_ONE_AT_A_TIME:
|
|
332
|
+
outputs: list[str] = []
|
|
333
|
+
for obj in objects:
|
|
334
|
+
try:
|
|
335
|
+
outputs.append(template.render(objects=[obj], app=config.app))
|
|
336
|
+
except Exception as e:
|
|
337
|
+
raise Exception(
|
|
338
|
+
f"Failed writing the following object to file:\n"
|
|
339
|
+
f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n"
|
|
340
|
+
f"Type {type(obj)}: \n"
|
|
341
|
+
f"Output File: {app_output_path}\n"
|
|
342
|
+
f"Error: {str(e)}\n"
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
346
|
+
with open(output_path, "a") as f:
|
|
347
|
+
output = "".join(outputs).encode("utf-8", "ignore").decode("utf-8")
|
|
348
|
+
f.write(output)
|
|
349
|
+
else:
|
|
350
|
+
output = template.render(objects=objects, app=config.app)
|
|
351
|
+
|
|
352
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
353
|
+
with open(output_path, "a") as f:
|
|
354
|
+
output = output.encode("utf-8", "ignore").decode("utf-8")
|
|
355
|
+
f.write(output)
|
|
356
|
+
|
|
285
357
|
return output_path
|
|
286
|
-
|
|
287
|
-
|
|
358
|
+
|
|
288
359
|
@staticmethod
|
|
289
|
-
def validateConfFile(path:pathlib.Path):
|
|
360
|
+
def validateConfFile(path: pathlib.Path):
|
|
290
361
|
"""Ensure that the conf file is valid. We will do this by reading back
|
|
291
362
|
the conf using RawConfigParser to ensure that it does not throw any parsing errors.
|
|
292
363
|
This is particularly relevant because newlines contained in string fields may
|
|
293
364
|
break the formatting of the conf file if they have been incorrectly escaped with
|
|
294
|
-
the 'ConfWriter.escapeNewlines()' function.
|
|
365
|
+
the 'ConfWriter.escapeNewlines()' function.
|
|
295
366
|
|
|
296
367
|
If a conf file failes validation, we will throw an exception
|
|
297
368
|
|
|
@@ -300,7 +371,7 @@ class ConfWriter():
|
|
|
300
371
|
"""
|
|
301
372
|
return
|
|
302
373
|
if path.suffix != ".conf":
|
|
303
|
-
#there may be some other files built, so just ignore them
|
|
374
|
+
# there may be some other files built, so just ignore them
|
|
304
375
|
return
|
|
305
376
|
try:
|
|
306
377
|
_ = configparser.RawConfigParser().read(path)
|
|
@@ -308,30 +379,35 @@ class ConfWriter():
|
|
|
308
379
|
raise Exception(f"Failed to validate .conf file {str(path)}: {str(e)}")
|
|
309
380
|
|
|
310
381
|
@staticmethod
|
|
311
|
-
def validateXmlFile(path:pathlib.Path):
|
|
382
|
+
def validateXmlFile(path: pathlib.Path):
|
|
312
383
|
"""Ensure that the XML file is valid XML.
|
|
313
384
|
|
|
314
385
|
Args:
|
|
315
386
|
path (pathlib.Path): path to the xml file to validate
|
|
316
|
-
"""
|
|
317
|
-
|
|
387
|
+
"""
|
|
388
|
+
|
|
318
389
|
try:
|
|
319
|
-
with open(path,
|
|
390
|
+
with open(path, "r") as xmlFile:
|
|
320
391
|
_ = ET.fromstring(xmlFile.read())
|
|
321
392
|
except Exception as e:
|
|
322
393
|
raise Exception(f"Failed to validate .xml file {str(path)}: {str(e)}")
|
|
323
|
-
|
|
324
394
|
|
|
325
395
|
@staticmethod
|
|
326
|
-
def validateManifestFile(path:pathlib.Path):
|
|
396
|
+
def validateManifestFile(path: pathlib.Path):
|
|
327
397
|
"""Ensure that the Manifest file is valid JSON.
|
|
328
398
|
|
|
329
399
|
Args:
|
|
330
400
|
path (pathlib.Path): path to the manifest JSON file to validate
|
|
331
|
-
"""
|
|
401
|
+
"""
|
|
332
402
|
return
|
|
333
403
|
try:
|
|
334
|
-
with open(path,
|
|
404
|
+
with open(path, "r") as manifestFile:
|
|
335
405
|
_ = json.load(manifestFile)
|
|
336
406
|
except Exception as e:
|
|
337
|
-
raise Exception(
|
|
407
|
+
raise Exception(
|
|
408
|
+
f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}"
|
|
409
|
+
)
|
|
410
|
+
except Exception as e:
|
|
411
|
+
raise Exception(
|
|
412
|
+
f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}"
|
|
413
|
+
)
|
contentctl/output/json_writer.py
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import json
|
|
2
|
-
from
|
|
3
|
-
from typing import List
|
|
4
|
-
from io import TextIOWrapper
|
|
2
|
+
from typing import Any
|
|
5
3
|
class JsonWriter():
|
|
6
4
|
|
|
7
5
|
@staticmethod
|
|
8
|
-
def writeJsonObject(file_path : str, object_name: str, objs:
|
|
6
|
+
def writeJsonObject(file_path : str, object_name: str, objs: list[dict[str,Any]],readable_output:bool=True) -> None:
|
|
9
7
|
try:
|
|
10
8
|
with open(file_path, 'w') as outfile:
|
|
11
9
|
if readable_output:
|
contentctl/output/svg_output.py
CHANGED
|
@@ -35,7 +35,7 @@ class SvgOutput():
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
total_dict:dict[str,Any] = self.get_badge_dict("Detections", detections, detections)
|
|
38
|
-
production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production
|
|
38
|
+
production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production])
|
|
39
39
|
#deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated])
|
|
40
40
|
#experimental_dict = self.get_badge_dict("Experimental", detections, [detection for detection in detections if detection.status == DetectionStatus.experimental])
|
|
41
41
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
{% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %}
|
|
6
6
|
[savedsearch://{{ detection.get_conf_stanza_name(app) }}]
|
|
7
7
|
type = detection
|
|
8
|
-
asset_type = {{ detection.tags.asset_type
|
|
8
|
+
asset_type = {{ detection.tags.asset_type }}
|
|
9
9
|
confidence = medium
|
|
10
10
|
explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }}
|
|
11
11
|
{% if detection.how_to_implement is defined %}
|
|
@@ -162,11 +162,6 @@ The SPL above uses the following Lookups:
|
|
|
162
162
|
{% endfor %}
|
|
163
163
|
{% endif -%}
|
|
164
164
|
|
|
165
|
-
#### Required field
|
|
166
|
-
{% for field in object.tags.required_fields -%}
|
|
167
|
-
* {{ field }}
|
|
168
|
-
{% endfor %}
|
|
169
|
-
|
|
170
165
|
#### How To Implement
|
|
171
166
|
{{ object.how_to_implement}}
|
|
172
167
|
|
|
@@ -44,7 +44,7 @@ action.escu.providing_technologies = null
|
|
|
44
44
|
action.escu.analytic_story = {{ objectListToNameList(detection.tags.analytic_story) | tojson }}
|
|
45
45
|
{% if detection.deployment.alert_action.rba.enabled%}
|
|
46
46
|
action.risk = 1
|
|
47
|
-
action.risk.param._risk_message = {{ detection.
|
|
47
|
+
action.risk.param._risk_message = {{ detection.rba.message | escapeNewlines() }}
|
|
48
48
|
action.risk.param._risk = {{ detection.risk | tojson }}
|
|
49
49
|
action.risk.param._risk_score = 0
|
|
50
50
|
action.risk.param.verbose = 0
|
|
@@ -70,8 +70,13 @@ action.notable.param.nes_fields = {{ detection.nes_fields }}
|
|
|
70
70
|
{% endif %}
|
|
71
71
|
action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}}
|
|
72
72
|
action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%}
|
|
73
|
-
action.notable.param.security_domain = {{ detection.tags.security_domain
|
|
74
|
-
|
|
73
|
+
action.notable.param.security_domain = {{ detection.tags.security_domain }}
|
|
74
|
+
{% if detection.rba %}
|
|
75
|
+
action.notable.param.severity = {{ detection.rba.severity }}
|
|
76
|
+
{% else %}
|
|
77
|
+
{# Correlations do not have detection.rba defined, but should get a default severity #}
|
|
78
|
+
action.notable.param.severity = high
|
|
79
|
+
{% endif %}
|
|
75
80
|
{% endif %}
|
|
76
81
|
{% if detection.deployment.alert_action.email %}
|
|
77
82
|
action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
|
|
2
2
|
{% for lookup in objects %}
|
|
3
3
|
[{{ lookup.name }}]
|
|
4
|
-
{% if lookup.
|
|
5
|
-
filename = {{ lookup.
|
|
4
|
+
{% if lookup.app_filename is defined and lookup.app_filename != None %}
|
|
5
|
+
filename = {{ lookup.app_filename.name }}
|
|
6
6
|
{% else %}
|
|
7
7
|
collection = {{ lookup.collection }}
|
|
8
8
|
external_type = kvstore
|
|
@@ -25,8 +25,8 @@ max_matches = {{ lookup.max_matches }}
|
|
|
25
25
|
{% if lookup.min_matches is defined and lookup.min_matches != None %}
|
|
26
26
|
min_matches = {{ lookup.min_matches }}
|
|
27
27
|
{% endif %}
|
|
28
|
-
{% if lookup.
|
|
29
|
-
fields_list = {{ lookup.
|
|
28
|
+
{% if lookup.fields_to_fields_list_conf_format is defined %}
|
|
29
|
+
fields_list = {{ lookup.fields_to_fields_list_conf_format }}
|
|
30
30
|
{% endif %}
|
|
31
31
|
{% if lookup.filter is defined and lookup.filter != None %}
|
|
32
32
|
filter = {{ lookup.filter }}
|
contentctl/output/yml_writer.py
CHANGED
|
@@ -1,6 +1,21 @@
|
|
|
1
1
|
|
|
2
2
|
import yaml
|
|
3
3
|
from typing import Any
|
|
4
|
+
from enum import StrEnum, IntEnum
|
|
5
|
+
|
|
6
|
+
# Set the following so that we can write StrEnum and IntEnum
|
|
7
|
+
# to files. Otherwise, we will get the following errors when trying
|
|
8
|
+
# to write to files:
|
|
9
|
+
# yaml.representer.RepresenterError: ('cannot represent an object',.....
|
|
10
|
+
yaml.SafeDumper.add_multi_representer(
|
|
11
|
+
StrEnum,
|
|
12
|
+
yaml.representer.SafeRepresenter.represent_str
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
yaml.SafeDumper.add_multi_representer(
|
|
16
|
+
IntEnum,
|
|
17
|
+
yaml.representer.SafeRepresenter.represent_int
|
|
18
|
+
)
|
|
4
19
|
|
|
5
20
|
class YmlWriter:
|
|
6
21
|
|