contentctl 4.4.7__py3-none-any.whl → 5.0.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. contentctl/actions/build.py +39 -27
  2. contentctl/actions/detection_testing/DetectionTestingManager.py +0 -1
  3. contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +32 -26
  4. contentctl/actions/detection_testing/progress_bar.py +6 -6
  5. contentctl/actions/detection_testing/views/DetectionTestingView.py +4 -4
  6. contentctl/actions/new_content.py +98 -81
  7. contentctl/actions/test.py +4 -5
  8. contentctl/actions/validate.py +2 -1
  9. contentctl/contentctl.py +114 -79
  10. contentctl/helper/utils.py +0 -14
  11. contentctl/input/director.py +5 -5
  12. contentctl/input/new_content_questions.py +2 -2
  13. contentctl/input/yml_reader.py +11 -6
  14. contentctl/objects/abstract_security_content_objects/detection_abstract.py +228 -120
  15. contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +5 -7
  16. contentctl/objects/alert_action.py +2 -1
  17. contentctl/objects/atomic.py +1 -0
  18. contentctl/objects/base_test.py +4 -3
  19. contentctl/objects/base_test_result.py +3 -3
  20. contentctl/objects/baseline.py +26 -6
  21. contentctl/objects/baseline_tags.py +2 -3
  22. contentctl/objects/config.py +26 -45
  23. contentctl/objects/constants.py +4 -1
  24. contentctl/objects/correlation_search.py +89 -95
  25. contentctl/objects/data_source.py +5 -6
  26. contentctl/objects/deployment.py +2 -10
  27. contentctl/objects/deployment_email.py +2 -1
  28. contentctl/objects/deployment_notable.py +2 -1
  29. contentctl/objects/deployment_phantom.py +2 -1
  30. contentctl/objects/deployment_rba.py +2 -1
  31. contentctl/objects/deployment_scheduling.py +2 -1
  32. contentctl/objects/deployment_slack.py +2 -1
  33. contentctl/objects/detection_tags.py +7 -42
  34. contentctl/objects/drilldown.py +1 -0
  35. contentctl/objects/enums.py +21 -58
  36. contentctl/objects/investigation.py +6 -5
  37. contentctl/objects/investigation_tags.py +2 -3
  38. contentctl/objects/lookup.py +145 -63
  39. contentctl/objects/macro.py +2 -3
  40. contentctl/objects/mitre_attack_enrichment.py +2 -2
  41. contentctl/objects/observable.py +3 -1
  42. contentctl/objects/playbook_tags.py +5 -1
  43. contentctl/objects/rba.py +90 -0
  44. contentctl/objects/risk_event.py +87 -144
  45. contentctl/objects/story_tags.py +1 -2
  46. contentctl/objects/test_attack_data.py +2 -1
  47. contentctl/objects/unit_test_baseline.py +2 -1
  48. contentctl/output/api_json_output.py +233 -220
  49. contentctl/output/conf_output.py +51 -44
  50. contentctl/output/conf_writer.py +201 -125
  51. contentctl/output/data_source_writer.py +0 -1
  52. contentctl/output/json_writer.py +2 -4
  53. contentctl/output/svg_output.py +1 -1
  54. contentctl/output/templates/analyticstories_detections.j2 +1 -1
  55. contentctl/output/templates/collections.j2 +1 -1
  56. contentctl/output/templates/doc_detections.j2 +0 -5
  57. contentctl/output/templates/savedsearches_detections.j2 +8 -3
  58. contentctl/output/templates/transforms.j2 +4 -4
  59. contentctl/output/yml_writer.py +15 -0
  60. contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +16 -34
  61. {contentctl-4.4.7.dist-info → contentctl-5.0.0a0.dist-info}/METADATA +5 -4
  62. {contentctl-4.4.7.dist-info → contentctl-5.0.0a0.dist-info}/RECORD +65 -68
  63. {contentctl-4.4.7.dist-info → contentctl-5.0.0a0.dist-info}/WHEEL +1 -1
  64. contentctl/objects/event_source.py +0 -11
  65. contentctl/output/detection_writer.py +0 -28
  66. contentctl/output/new_content_yml_output.py +0 -56
  67. contentctl/output/yml_output.py +0 -66
  68. {contentctl-4.4.7.dist-info → contentctl-5.0.0a0.dist-info}/LICENSE.md +0 -0
  69. {contentctl-4.4.7.dist-info → contentctl-5.0.0a0.dist-info}/entry_points.txt +0 -0
@@ -1,16 +1,17 @@
1
- from typing import Any
1
+ import configparser
2
2
  import datetime
3
- import re
4
- import os
5
3
  import json
6
- import configparser
7
- from xmlrpc.client import APPLICATION_ERROR
8
- from jinja2 import Environment, FileSystemLoader, StrictUndefined
4
+ import os
9
5
  import pathlib
10
- from contentctl.objects.security_content_object import SecurityContentObject
11
- from contentctl.objects.dashboard import Dashboard
12
- from contentctl.objects.config import build
6
+ import re
13
7
  import xml.etree.ElementTree as ET
8
+ from typing import Any, Sequence
9
+
10
+ from jinja2 import Environment, FileSystemLoader, StrictUndefined
11
+
12
+ from contentctl.objects.config import CustomApp, build
13
+ from contentctl.objects.dashboard import Dashboard
14
+ from contentctl.objects.security_content_object import SecurityContentObject
14
15
 
15
16
  # This list is not exhaustive of all default conf files, but should be
16
17
  # sufficient for our purposes.
@@ -82,59 +83,68 @@ DEFAULT_CONF_FILES = [
82
83
  "workload_rules.conf",
83
84
  ]
84
85
 
85
- class ConfWriter():
86
86
 
87
+ class ConfWriter:
87
88
  @staticmethod
88
- def custom_jinja2_enrichment_filter(string:str, object:SecurityContentObject):
89
+ def custom_jinja2_enrichment_filter(string: str, object: SecurityContentObject):
89
90
  substitutions = re.findall(r"%[^%]*%", string)
90
91
  updated_string = string
91
92
  for sub in substitutions:
92
- sub_without_percents = sub.replace("%","")
93
+ sub_without_percents = sub.replace("%", "")
93
94
  if hasattr(object, sub_without_percents):
94
- updated_string = updated_string.replace(sub, str(getattr(object, sub_without_percents)))
95
- elif hasattr(object,'tags') and hasattr(object.tags, sub_without_percents):
96
- updated_string = updated_string.replace(sub, str(getattr(object.tags, sub_without_percents)))
95
+ updated_string = updated_string.replace(
96
+ sub, str(getattr(object, sub_without_percents))
97
+ )
98
+ elif hasattr(object, "tags") and hasattr(object.tags, sub_without_percents):
99
+ updated_string = updated_string.replace(
100
+ sub, str(getattr(object.tags, sub_without_percents))
101
+ )
97
102
  else:
98
103
  raise Exception(f"Unable to find field {sub} in object {object.name}")
99
-
104
+
100
105
  return updated_string
101
-
106
+
102
107
  @staticmethod
103
- def escapeNewlines(obj:Any):
108
+ def escapeNewlines(obj: Any):
104
109
  # Ensure that any newlines that occur in a string are escaped with a \.
105
110
  # Failing to do so will result in an improperly formatted conf files that
106
111
  # cannot be parsed
107
- if isinstance(obj,str):
108
- # Remove leading and trailing characters. Conf parsers may erroneously
109
- # Parse fields if they have leading or trailing newlines/whitespace and we
112
+ if isinstance(obj, str):
113
+ # Remove leading and trailing characters. Conf parsers may erroneously
114
+ # Parse fields if they have leading or trailing newlines/whitespace and we
110
115
  # probably don't want that anyway as it doesn't look good in output
111
- return obj.strip().replace(f"\n"," \\\n")
116
+ return obj.strip().replace("\n", " \\\n")
112
117
  else:
113
118
  return obj
114
119
 
115
-
116
120
  @staticmethod
117
- def writeConfFileHeader(app_output_path:pathlib.Path, config: build) -> pathlib.Path:
118
- output = ConfWriter.writeFileHeader(app_output_path, config)
119
-
120
- output_path = config.getPackageDirectoryPath()/app_output_path
121
+ def writeConfFileHeader(
122
+ app_output_path: pathlib.Path, config: build
123
+ ) -> pathlib.Path:
124
+ output = ConfWriter.writeFileHeader(app_output_path, config)
125
+
126
+ output_path = config.getPackageDirectoryPath() / app_output_path
121
127
  output_path.parent.mkdir(parents=True, exist_ok=True)
122
- with open(output_path, 'w') as f:
123
- output = output.encode('utf-8', 'ignore').decode('utf-8')
128
+ with open(output_path, "w") as f:
129
+ output = output.encode("utf-8", "ignore").decode("utf-8")
124
130
  f.write(output)
125
131
 
126
- #Ensure that the conf file we just generated/update is syntactically valid
127
- ConfWriter.validateConfFile(output_path)
132
+ # Ensure that the conf file we just generated/update is syntactically valid
133
+ ConfWriter.validateConfFile(output_path)
128
134
  return output_path
129
135
 
130
136
  @staticmethod
131
- def getCustomConfFileStems(config:build)->list[str]:
137
+ def getCustomConfFileStems(config: build) -> list[str]:
132
138
  # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if
133
139
  # they are custom conf files
134
- default_path = config.getPackageDirectoryPath()/"default"
140
+ default_path = config.getPackageDirectoryPath() / "default"
135
141
  conf_files = default_path.glob("*.conf")
136
-
137
- custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES]
142
+
143
+ custom_conf_file_stems = [
144
+ conf_file.stem
145
+ for conf_file in conf_files
146
+ if conf_file.name not in DEFAULT_CONF_FILES
147
+ ]
138
148
  return sorted(custom_conf_file_stems)
139
149
 
140
150
  @staticmethod
@@ -145,16 +155,17 @@ class ConfWriter():
145
155
  j2_env = ConfWriter.getJ2Environment()
146
156
  template = j2_env.get_template(template_name)
147
157
 
148
- output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config))
149
-
150
- output_path = config.getPackageDirectoryPath()/app_output_path
158
+ output = template.render(
159
+ custom_conf_files=ConfWriter.getCustomConfFileStems(config)
160
+ )
161
+
162
+ output_path = config.getPackageDirectoryPath() / app_output_path
151
163
  output_path.parent.mkdir(parents=True, exist_ok=True)
152
- with open(output_path, 'a') as f:
153
- output = output.encode('utf-8', 'ignore').decode('utf-8')
164
+ with open(output_path, "a") as f:
165
+ output = output.encode("utf-8", "ignore").decode("utf-8")
154
166
  f.write(output)
155
167
  return output_path
156
168
 
157
-
158
169
  @staticmethod
159
170
  def writeAppConf(config: build) -> pathlib.Path:
160
171
  app_output_path = pathlib.Path("default/app.conf")
@@ -163,135 +174,195 @@ class ConfWriter():
163
174
  j2_env = ConfWriter.getJ2Environment()
164
175
  template = j2_env.get_template(template_name)
165
176
 
166
- output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config),
167
- app=config.app)
168
-
169
- output_path = config.getPackageDirectoryPath()/app_output_path
177
+ output = template.render(
178
+ custom_conf_files=ConfWriter.getCustomConfFileStems(config), app=config.app
179
+ )
180
+
181
+ output_path = config.getPackageDirectoryPath() / app_output_path
170
182
  output_path.parent.mkdir(parents=True, exist_ok=True)
171
- with open(output_path, 'a') as f:
172
- output = output.encode('utf-8', 'ignore').decode('utf-8')
183
+ with open(output_path, "a") as f:
184
+ output = output.encode("utf-8", "ignore").decode("utf-8")
173
185
  f.write(output)
174
186
  return output_path
175
187
 
176
188
  @staticmethod
177
- def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path:
189
+ def writeManifestFile(
190
+ app_output_path: pathlib.Path,
191
+ template_name: str,
192
+ config: build,
193
+ objects: list[CustomApp],
194
+ ) -> pathlib.Path:
178
195
  j2_env = ConfWriter.getJ2Environment()
179
196
  template = j2_env.get_template(template_name)
180
-
181
- output = template.render(objects=objects, app=config.app, currentDate=datetime.datetime.now(datetime.UTC).date().isoformat())
182
-
183
- output_path = config.getPackageDirectoryPath()/app_output_path
197
+
198
+ output = template.render(
199
+ objects=objects,
200
+ app=config.app,
201
+ currentDate=datetime.datetime.now(datetime.UTC).date().isoformat(),
202
+ )
203
+
204
+ output_path = config.getPackageDirectoryPath() / app_output_path
184
205
  output_path.parent.mkdir(parents=True, exist_ok=True)
185
- with open(output_path, 'w') as f:
186
- output = output.encode('utf-8', 'ignore').decode('utf-8')
206
+ with open(output_path, "w") as f:
207
+ output = output.encode("utf-8", "ignore").decode("utf-8")
187
208
  f.write(output)
188
209
  return output_path
189
-
190
-
191
210
 
192
211
  @staticmethod
193
- def writeFileHeader(app_output_path:pathlib.Path, config: build) -> str:
194
- #Do not output microseconds or +00:000 at the end of the datetime string
195
- utc_time = datetime.datetime.now(datetime.UTC).replace(microsecond=0,tzinfo=None).isoformat()
196
-
197
- j2_env = Environment(
198
- loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
199
- trim_blocks=True)
212
+ def writeFileHeader(app_output_path: pathlib.Path, config: build) -> str:
213
+ # Do not output microseconds or +00:000 at the end of the datetime string
214
+ utc_time = (
215
+ datetime.datetime.now(datetime.UTC)
216
+ .replace(microsecond=0, tzinfo=None)
217
+ .isoformat()
218
+ )
200
219
 
201
- template = j2_env.get_template('header.j2')
202
- output = template.render(time=utc_time, author=' - '.join([config.app.author_name,config.app.author_company]), author_email=config.app.author_email)
203
-
204
- return output
220
+ j2_env = Environment(
221
+ loader=FileSystemLoader(
222
+ os.path.join(os.path.dirname(__file__), "templates")
223
+ ),
224
+ trim_blocks=True,
225
+ )
205
226
 
227
+ template = j2_env.get_template("header.j2")
228
+ output = template.render(
229
+ time=utc_time,
230
+ author=" - ".join([config.app.author_name, config.app.author_company]),
231
+ author_email=config.app.author_email,
232
+ )
206
233
 
234
+ return output
207
235
 
208
236
  @staticmethod
209
- def writeXmlFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> None:
210
-
211
-
237
+ def writeXmlFile(
238
+ app_output_path: pathlib.Path,
239
+ template_name: str,
240
+ config: build,
241
+ objects: list[str],
242
+ ) -> None:
212
243
  j2_env = ConfWriter.getJ2Environment()
213
244
  template = j2_env.get_template(template_name)
214
-
245
+
215
246
  output = template.render(objects=objects, app=config.app)
216
-
217
- output_path = config.getPackageDirectoryPath()/app_output_path
247
+
248
+ output_path = config.getPackageDirectoryPath() / app_output_path
218
249
  output_path.parent.mkdir(parents=True, exist_ok=True)
219
- with open(output_path, 'a') as f:
220
- output = output.encode('utf-8', 'ignore').decode('utf-8')
250
+ with open(output_path, "a") as f:
251
+ output = output.encode("utf-8", "ignore").decode("utf-8")
221
252
  f.write(output)
222
-
223
- #Ensure that the conf file we just generated/update is syntactically valid
224
- ConfWriter.validateXmlFile(output_path)
225
253
 
226
-
254
+ # Ensure that the conf file we just generated/update is syntactically valid
255
+ ConfWriter.validateXmlFile(output_path)
227
256
 
228
257
  @staticmethod
229
- def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.Path]:
230
- written_files:set[pathlib.Path] = set()
258
+ def writeDashboardFiles(
259
+ config: build, dashboards: list[Dashboard]
260
+ ) -> set[pathlib.Path]:
261
+ written_files: set[pathlib.Path] = set()
231
262
  for dashboard in dashboards:
232
263
  output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config)
233
264
  # Check that the full output path does not exist so that we are not having an
234
265
  # name collision with a file in app_template
235
- if (config.getPackageDirectoryPath()/output_file_path).exists():
236
- raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?")
237
-
266
+ if (config.getPackageDirectoryPath() / output_file_path).exists():
267
+ raise FileExistsError(
268
+ f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path / 'dashboards'}?"
269
+ )
270
+
238
271
  ConfWriter.writeXmlFileHeader(output_file_path, config)
239
272
  dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config)
240
- ConfWriter.validateXmlFile(config.getPackageDirectoryPath()/output_file_path)
273
+ ConfWriter.validateXmlFile(
274
+ config.getPackageDirectoryPath() / output_file_path
275
+ )
241
276
  written_files.add(output_file_path)
242
277
  return written_files
243
278
 
244
-
245
279
  @staticmethod
246
- def writeXmlFileHeader(app_output_path:pathlib.Path, config: build) -> None:
247
- output = ConfWriter.writeFileHeader(app_output_path, config)
280
+ def writeXmlFileHeader(app_output_path: pathlib.Path, config: build) -> None:
281
+ output = ConfWriter.writeFileHeader(app_output_path, config)
248
282
  output_with_xml_comment = f"<!--\n{output}-->\n"
249
283
 
250
- output_path = config.getPackageDirectoryPath()/app_output_path
284
+ output_path = config.getPackageDirectoryPath() / app_output_path
251
285
  output_path.parent.mkdir(parents=True, exist_ok=True)
252
- with open(output_path, 'w') as f:
253
- output_with_xml_comment = output_with_xml_comment.encode('utf-8', 'ignore').decode('utf-8')
286
+ with open(output_path, "w") as f:
287
+ output_with_xml_comment = output_with_xml_comment.encode(
288
+ "utf-8", "ignore"
289
+ ).decode("utf-8")
254
290
  f.write(output_with_xml_comment)
255
-
256
- # We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now,
257
- # the file is an empty XML document (besides the commented header). This means that it will FAIL validation
258
291
 
292
+ # We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now,
293
+ # the file is an empty XML document (besides the commented header). This means that it will FAIL validation
259
294
 
260
295
  @staticmethod
261
- def getJ2Environment()->Environment:
296
+ def getJ2Environment() -> Environment:
262
297
  j2_env = Environment(
263
- loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
298
+ loader=FileSystemLoader(
299
+ os.path.join(os.path.dirname(__file__), "templates")
300
+ ),
264
301
  trim_blocks=True,
265
- undefined=StrictUndefined)
266
- j2_env.globals.update(objectListToNameList=SecurityContentObject.objectListToNameList)
267
-
268
-
269
- j2_env.filters['custom_jinja2_enrichment_filter'] = ConfWriter.custom_jinja2_enrichment_filter
270
- j2_env.filters['escapeNewlines'] = ConfWriter.escapeNewlines
302
+ undefined=StrictUndefined,
303
+ )
304
+ j2_env.globals.update(
305
+ objectListToNameList=SecurityContentObject.objectListToNameList
306
+ )
307
+
308
+ j2_env.filters["custom_jinja2_enrichment_filter"] = (
309
+ ConfWriter.custom_jinja2_enrichment_filter
310
+ )
311
+ j2_env.filters["escapeNewlines"] = ConfWriter.escapeNewlines
271
312
  return j2_env
272
313
 
273
314
  @staticmethod
274
- def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list) -> pathlib.Path:
275
- output_path = config.getPackageDirectoryPath()/app_output_path
315
+ def writeConfFile(
316
+ app_output_path: pathlib.Path,
317
+ template_name: str,
318
+ config: build,
319
+ objects: Sequence[SecurityContentObject] | list[CustomApp],
320
+ ) -> pathlib.Path:
321
+ output_path = config.getPackageDirectoryPath() / app_output_path
276
322
  j2_env = ConfWriter.getJ2Environment()
277
-
323
+
278
324
  template = j2_env.get_template(template_name)
279
- output = template.render(objects=objects, app=config.app)
280
-
281
- output_path.parent.mkdir(parents=True, exist_ok=True)
282
- with open(output_path, 'a') as f:
283
- output = output.encode('utf-8', 'ignore').decode('utf-8')
284
- f.write(output)
325
+
326
+ # The following code, which is commented out, serializes one object at a time.
327
+ # This is extremely useful from a debugging perspective, because sometimes when
328
+ # serializing a large number of objects, exceptions throw in Jinja2 templates can
329
+ # be quite hard to diagnose. We leave this code in for use in debugging workflows:
330
+ SERIALIZE_ONE_AT_A_TIME = False
331
+ if SERIALIZE_ONE_AT_A_TIME:
332
+ outputs: list[str] = []
333
+ for obj in objects:
334
+ try:
335
+ outputs.append(template.render(objects=[obj], app=config.app))
336
+ except Exception as e:
337
+ raise Exception(
338
+ f"Failed writing the following object to file:\n"
339
+ f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n"
340
+ f"Type {type(obj)}: \n"
341
+ f"Output File: {app_output_path}\n"
342
+ f"Error: {str(e)}\n"
343
+ )
344
+
345
+ output_path.parent.mkdir(parents=True, exist_ok=True)
346
+ with open(output_path, "a") as f:
347
+ output = "".join(outputs).encode("utf-8", "ignore").decode("utf-8")
348
+ f.write(output)
349
+ else:
350
+ output = template.render(objects=objects, app=config.app)
351
+
352
+ output_path.parent.mkdir(parents=True, exist_ok=True)
353
+ with open(output_path, "a") as f:
354
+ output = output.encode("utf-8", "ignore").decode("utf-8")
355
+ f.write(output)
356
+
285
357
  return output_path
286
-
287
-
358
+
288
359
  @staticmethod
289
- def validateConfFile(path:pathlib.Path):
360
+ def validateConfFile(path: pathlib.Path):
290
361
  """Ensure that the conf file is valid. We will do this by reading back
291
362
  the conf using RawConfigParser to ensure that it does not throw any parsing errors.
292
363
  This is particularly relevant because newlines contained in string fields may
293
364
  break the formatting of the conf file if they have been incorrectly escaped with
294
- the 'ConfWriter.escapeNewlines()' function.
365
+ the 'ConfWriter.escapeNewlines()' function.
295
366
 
296
367
  If a conf file failes validation, we will throw an exception
297
368
 
@@ -300,7 +371,7 @@ class ConfWriter():
300
371
  """
301
372
  return
302
373
  if path.suffix != ".conf":
303
- #there may be some other files built, so just ignore them
374
+ # there may be some other files built, so just ignore them
304
375
  return
305
376
  try:
306
377
  _ = configparser.RawConfigParser().read(path)
@@ -308,30 +379,35 @@ class ConfWriter():
308
379
  raise Exception(f"Failed to validate .conf file {str(path)}: {str(e)}")
309
380
 
310
381
  @staticmethod
311
- def validateXmlFile(path:pathlib.Path):
382
+ def validateXmlFile(path: pathlib.Path):
312
383
  """Ensure that the XML file is valid XML.
313
384
 
314
385
  Args:
315
386
  path (pathlib.Path): path to the xml file to validate
316
- """
317
-
387
+ """
388
+
318
389
  try:
319
- with open(path, 'r') as xmlFile:
390
+ with open(path, "r") as xmlFile:
320
391
  _ = ET.fromstring(xmlFile.read())
321
392
  except Exception as e:
322
393
  raise Exception(f"Failed to validate .xml file {str(path)}: {str(e)}")
323
-
324
394
 
325
395
  @staticmethod
326
- def validateManifestFile(path:pathlib.Path):
396
+ def validateManifestFile(path: pathlib.Path):
327
397
  """Ensure that the Manifest file is valid JSON.
328
398
 
329
399
  Args:
330
400
  path (pathlib.Path): path to the manifest JSON file to validate
331
- """
401
+ """
332
402
  return
333
403
  try:
334
- with open(path, 'r') as manifestFile:
404
+ with open(path, "r") as manifestFile:
335
405
  _ = json.load(manifestFile)
336
406
  except Exception as e:
337
- raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}")
407
+ raise Exception(
408
+ f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}"
409
+ )
410
+ except Exception as e:
411
+ raise Exception(
412
+ f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}"
413
+ )
@@ -1,6 +1,5 @@
1
1
  import csv
2
2
  from contentctl.objects.data_source import DataSource
3
- from contentctl.objects.event_source import EventSource
4
3
  from typing import List
5
4
  import pathlib
6
5
 
@@ -1,11 +1,9 @@
1
1
  import json
2
- from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract
3
- from typing import List
4
- from io import TextIOWrapper
2
+ from typing import Any
5
3
  class JsonWriter():
6
4
 
7
5
  @staticmethod
8
- def writeJsonObject(file_path : str, object_name: str, objs: List[dict],readable_output=False) -> None:
6
+ def writeJsonObject(file_path : str, object_name: str, objs: list[dict[str,Any]],readable_output:bool=True) -> None:
9
7
  try:
10
8
  with open(file_path, 'w') as outfile:
11
9
  if readable_output:
@@ -35,7 +35,7 @@ class SvgOutput():
35
35
 
36
36
 
37
37
  total_dict:dict[str,Any] = self.get_badge_dict("Detections", detections, detections)
38
- production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production.value])
38
+ production_dict:dict[str,Any] = self.get_badge_dict("% Production", detections, [detection for detection in detections if detection.status == DetectionStatus.production])
39
39
  #deprecated_dict = self.get_badge_dict("Deprecated", detections, [detection for detection in detections if detection.status == DetectionStatus.deprecated])
40
40
  #experimental_dict = self.get_badge_dict("Experimental", detections, [detection for detection in detections if detection.status == DetectionStatus.experimental])
41
41
 
@@ -5,7 +5,7 @@
5
5
  {% if (detection.type == 'TTP' or detection.type == 'Anomaly' or detection.type == 'Hunting' or detection.type == 'Correlation') %}
6
6
  [savedsearch://{{ detection.get_conf_stanza_name(app) }}]
7
7
  type = detection
8
- asset_type = {{ detection.tags.asset_type.value }}
8
+ asset_type = {{ detection.tags.asset_type }}
9
9
  confidence = medium
10
10
  explanation = {{ (detection.explanation if detection.explanation else detection.description) | escapeNewlines() }}
11
11
  {% if detection.how_to_implement is defined %}
@@ -1,6 +1,6 @@
1
1
 
2
2
  {% for lookup in objects %}
3
- {% if lookup.collection is defined and lookup.collection != None %}
3
+ {% if lookup.collection is defined %}
4
4
  [{{ lookup.name }}]
5
5
  enforceTypes = false
6
6
  replicate = false
@@ -162,11 +162,6 @@ The SPL above uses the following Lookups:
162
162
  {% endfor %}
163
163
  {% endif -%}
164
164
 
165
- #### Required field
166
- {% for field in object.tags.required_fields -%}
167
- * {{ field }}
168
- {% endfor %}
169
-
170
165
  #### How To Implement
171
166
  {{ object.how_to_implement}}
172
167
 
@@ -44,7 +44,7 @@ action.escu.providing_technologies = null
44
44
  action.escu.analytic_story = {{ objectListToNameList(detection.tags.analytic_story) | tojson }}
45
45
  {% if detection.deployment.alert_action.rba.enabled%}
46
46
  action.risk = 1
47
- action.risk.param._risk_message = {{ detection.tags.message | escapeNewlines() }}
47
+ action.risk.param._risk_message = {{ detection.rba.message | escapeNewlines() }}
48
48
  action.risk.param._risk = {{ detection.risk | tojson }}
49
49
  action.risk.param._risk_score = 0
50
50
  action.risk.param.verbose = 0
@@ -70,8 +70,13 @@ action.notable.param.nes_fields = {{ detection.nes_fields }}
70
70
  {% endif %}
71
71
  action.notable.param.rule_description = {{ detection.deployment.alert_action.notable.rule_description | custom_jinja2_enrichment_filter(detection) | escapeNewlines()}}
72
72
  action.notable.param.rule_title = {% if detection.type | lower == "correlation" %}RBA: {{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% else %}{{ detection.deployment.alert_action.notable.rule_title | custom_jinja2_enrichment_filter(detection) }}{% endif +%}
73
- action.notable.param.security_domain = {{ detection.tags.security_domain.value }}
74
- action.notable.param.severity = {{ detection.tags.severity.value }}
73
+ action.notable.param.security_domain = {{ detection.tags.security_domain }}
74
+ {% if detection.rba %}
75
+ action.notable.param.severity = {{ detection.rba.severity }}
76
+ {% else %}
77
+ {# Correlations do not have detection.rba defined, but should get a default severity #}
78
+ action.notable.param.severity = high
79
+ {% endif %}
75
80
  {% endif %}
76
81
  {% if detection.deployment.alert_action.email %}
77
82
  action.email.subject.alert = {{ detection.deployment.alert_action.email.subject | custom_jinja2_enrichment_filter(detection) | escapeNewlines() }}
@@ -1,8 +1,8 @@
1
1
 
2
2
  {% for lookup in objects %}
3
3
  [{{ lookup.name }}]
4
- {% if lookup.filename is defined and lookup.filename != None %}
5
- filename = {{ lookup.filename.name }}
4
+ {% if lookup.app_filename is defined and lookup.app_filename != None %}
5
+ filename = {{ lookup.app_filename.name }}
6
6
  {% else %}
7
7
  collection = {{ lookup.collection }}
8
8
  external_type = kvstore
@@ -25,8 +25,8 @@ max_matches = {{ lookup.max_matches }}
25
25
  {% if lookup.min_matches is defined and lookup.min_matches != None %}
26
26
  min_matches = {{ lookup.min_matches }}
27
27
  {% endif %}
28
- {% if lookup.fields_list is defined and lookup.fields_list != None %}
29
- fields_list = {{ lookup.fields_list }}
28
+ {% if lookup.fields_to_fields_list_conf_format is defined %}
29
+ fields_list = {{ lookup.fields_to_fields_list_conf_format }}
30
30
  {% endif %}
31
31
  {% if lookup.filter is defined and lookup.filter != None %}
32
32
  filter = {{ lookup.filter }}
@@ -1,6 +1,21 @@
1
1
 
2
2
  import yaml
3
3
  from typing import Any
4
+ from enum import StrEnum, IntEnum
5
+
6
+ # Set the following so that we can write StrEnum and IntEnum
7
+ # to files. Otherwise, we will get the following errors when trying
8
+ # to write to files:
9
+ # yaml.representer.RepresenterError: ('cannot represent an object',.....
10
+ yaml.SafeDumper.add_multi_representer(
11
+ StrEnum,
12
+ yaml.representer.SafeRepresenter.represent_str
13
+ )
14
+
15
+ yaml.SafeDumper.add_multi_representer(
16
+ IntEnum,
17
+ yaml.representer.SafeRepresenter.represent_int
18
+ )
4
19
 
5
20
  class YmlWriter:
6
21