contentctl 4.4.7__py3-none-any.whl → 5.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/build.py +39 -27
- contentctl/actions/detection_testing/DetectionTestingManager.py +0 -1
- contentctl/actions/detection_testing/GitService.py +132 -72
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +32 -26
- contentctl/actions/detection_testing/progress_bar.py +6 -6
- contentctl/actions/detection_testing/views/DetectionTestingView.py +4 -4
- contentctl/actions/new_content.py +98 -81
- contentctl/actions/test.py +4 -5
- contentctl/actions/validate.py +2 -1
- contentctl/contentctl.py +114 -80
- contentctl/helper/utils.py +0 -14
- contentctl/input/director.py +5 -5
- contentctl/input/new_content_questions.py +2 -2
- contentctl/input/yml_reader.py +11 -6
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +228 -120
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +5 -7
- contentctl/objects/alert_action.py +2 -1
- contentctl/objects/atomic.py +1 -0
- contentctl/objects/base_test.py +4 -3
- contentctl/objects/base_test_result.py +3 -3
- contentctl/objects/baseline.py +26 -6
- contentctl/objects/baseline_tags.py +2 -3
- contentctl/objects/config.py +789 -596
- contentctl/objects/constants.py +4 -1
- contentctl/objects/correlation_search.py +89 -95
- contentctl/objects/data_source.py +5 -6
- contentctl/objects/deployment.py +2 -10
- contentctl/objects/deployment_email.py +2 -1
- contentctl/objects/deployment_notable.py +2 -1
- contentctl/objects/deployment_phantom.py +2 -1
- contentctl/objects/deployment_rba.py +2 -1
- contentctl/objects/deployment_scheduling.py +2 -1
- contentctl/objects/deployment_slack.py +2 -1
- contentctl/objects/detection_tags.py +7 -42
- contentctl/objects/drilldown.py +1 -0
- contentctl/objects/enums.py +21 -58
- contentctl/objects/investigation.py +6 -5
- contentctl/objects/investigation_tags.py +2 -3
- contentctl/objects/lookup.py +145 -63
- contentctl/objects/macro.py +2 -3
- contentctl/objects/mitre_attack_enrichment.py +2 -2
- contentctl/objects/observable.py +3 -1
- contentctl/objects/playbook_tags.py +5 -1
- contentctl/objects/rba.py +90 -0
- contentctl/objects/risk_event.py +87 -144
- contentctl/objects/story_tags.py +1 -2
- contentctl/objects/test_attack_data.py +2 -1
- contentctl/objects/unit_test_baseline.py +2 -1
- contentctl/output/api_json_output.py +233 -220
- contentctl/output/conf_output.py +51 -44
- contentctl/output/conf_writer.py +201 -125
- contentctl/output/data_source_writer.py +0 -1
- contentctl/output/json_writer.py +2 -4
- contentctl/output/svg_output.py +1 -1
- contentctl/output/templates/analyticstories_detections.j2 +1 -1
- contentctl/output/templates/collections.j2 +1 -1
- contentctl/output/templates/doc_detections.j2 +0 -5
- contentctl/output/templates/savedsearches_detections.j2 +8 -3
- contentctl/output/templates/transforms.j2 +4 -4
- contentctl/output/yml_writer.py +15 -0
- contentctl/templates/detections/endpoint/anomalous_usage_of_7zip.yml +16 -34
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/METADATA +5 -4
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/RECORD +66 -69
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/WHEEL +1 -1
- contentctl/objects/event_source.py +0 -11
- contentctl/output/detection_writer.py +0 -28
- contentctl/output/new_content_yml_output.py +0 -56
- contentctl/output/yml_output.py +0 -66
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/LICENSE.md +0 -0
- {contentctl-4.4.7.dist-info → contentctl-5.0.0a2.dist-info}/entry_points.txt +0 -0
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
from typing import TYPE_CHECKING, Union, Optional, List, Any, Annotated
|
|
3
3
|
import re
|
|
4
4
|
import pathlib
|
|
5
|
-
from enum import
|
|
5
|
+
from enum import StrEnum
|
|
6
6
|
|
|
7
7
|
from pydantic import (
|
|
8
8
|
field_validator,
|
|
@@ -16,7 +16,7 @@ from pydantic import (
|
|
|
16
16
|
)
|
|
17
17
|
|
|
18
18
|
from contentctl.objects.macro import Macro
|
|
19
|
-
from contentctl.objects.lookup import Lookup
|
|
19
|
+
from contentctl.objects.lookup import Lookup, FileBackedLookup, KVStoreLookup
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from contentctl.input.director import DirectorOutputDto
|
|
22
22
|
from contentctl.objects.baseline import Baseline
|
|
@@ -35,6 +35,9 @@ from contentctl.objects.manual_test import ManualTest
|
|
|
35
35
|
from contentctl.objects.test_group import TestGroup
|
|
36
36
|
from contentctl.objects.integration_test import IntegrationTest
|
|
37
37
|
from contentctl.objects.data_source import DataSource
|
|
38
|
+
|
|
39
|
+
from contentctl.objects.rba import RBAObject
|
|
40
|
+
|
|
38
41
|
from contentctl.objects.base_test_result import TestResultStatus
|
|
39
42
|
from contentctl.objects.drilldown import Drilldown, DRILLDOWN_SEARCH_PLACEHOLDER
|
|
40
43
|
from contentctl.objects.enums import ProvidingTechnology
|
|
@@ -51,13 +54,11 @@ MISSING_SOURCES: set[str] = set()
|
|
|
51
54
|
|
|
52
55
|
# Those AnalyticsTypes that we do not test via contentctl
|
|
53
56
|
SKIPPED_ANALYTICS_TYPES: set[str] = {
|
|
54
|
-
AnalyticsType.Correlation
|
|
57
|
+
AnalyticsType.Correlation
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
|
|
58
|
-
# TODO (#266): disable the use_enum_values configuration
|
|
59
61
|
class Detection_Abstract(SecurityContentObject):
|
|
60
|
-
model_config = ConfigDict(use_enum_values=True)
|
|
61
62
|
name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
|
|
62
63
|
#contentType: SecurityContentType = SecurityContentType.detections
|
|
63
64
|
type: AnalyticsType = Field(...)
|
|
@@ -67,6 +68,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
67
68
|
search: str = Field(...)
|
|
68
69
|
how_to_implement: str = Field(..., min_length=4)
|
|
69
70
|
known_false_positives: str = Field(..., min_length=4)
|
|
71
|
+
rba: Optional[RBAObject] = Field(default=None)
|
|
70
72
|
explanation: None | str = Field(
|
|
71
73
|
default=None,
|
|
72
74
|
exclude=True, #Don't serialize this value when dumping the object
|
|
@@ -78,6 +80,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
78
80
|
"serialized in analyticstories_detections.j2",
|
|
79
81
|
)
|
|
80
82
|
|
|
83
|
+
|
|
81
84
|
enabled_by_default: bool = False
|
|
82
85
|
file_path: FilePath = Field(...)
|
|
83
86
|
# For model construction to first attempt construction of the leftmost object.
|
|
@@ -101,7 +104,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
101
104
|
def get_action_dot_correlationsearch_dot_label(self, app:CustomApp, max_stanza_length:int=ES_MAX_STANZA_LENGTH)->str:
|
|
102
105
|
stanza_name = self.get_conf_stanza_name(app)
|
|
103
106
|
stanza_name_after_saving_in_es = ES_SEARCH_STANZA_NAME_FORMAT_AFTER_CLONING_IN_PRODUCT_TEMPLATE.format(
|
|
104
|
-
security_domain_value = self.tags.security_domain
|
|
107
|
+
security_domain_value = self.tags.security_domain,
|
|
105
108
|
search_name = stanza_name
|
|
106
109
|
)
|
|
107
110
|
|
|
@@ -210,7 +213,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
210
213
|
# https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
|
|
211
214
|
|
|
212
215
|
# Skip tests for non-production detections
|
|
213
|
-
if self.status != DetectionStatus.production
|
|
216
|
+
if self.status != DetectionStatus.production:
|
|
214
217
|
self.skip_all_tests(f"TEST SKIPPED: Detection is non-production ({self.status})")
|
|
215
218
|
|
|
216
219
|
# Skip tests for detecton types like Correlation which are not supported via contentctl
|
|
@@ -263,7 +266,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
263
266
|
@computed_field
|
|
264
267
|
@property
|
|
265
268
|
def datamodel(self) -> List[DataModel]:
|
|
266
|
-
return [dm for dm in DataModel if dm
|
|
269
|
+
return [dm for dm in DataModel if dm in self.search]
|
|
267
270
|
|
|
268
271
|
|
|
269
272
|
|
|
@@ -282,10 +285,8 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
282
285
|
|
|
283
286
|
annotations_dict: dict[str, str | list[str] | int] = {}
|
|
284
287
|
annotations_dict["analytic_story"] = [story.name for story in self.tags.analytic_story]
|
|
285
|
-
annotations_dict["confidence"] = self.tags.confidence
|
|
286
288
|
if len(self.tags.cve or []) > 0:
|
|
287
289
|
annotations_dict["cve"] = self.tags.cve
|
|
288
|
-
annotations_dict["impact"] = self.tags.impact
|
|
289
290
|
annotations_dict["type"] = self.type
|
|
290
291
|
annotations_dict["type_list"] = [self.type]
|
|
291
292
|
# annotations_dict["version"] = self.version
|
|
@@ -308,13 +309,13 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
308
309
|
def mappings(self) -> dict[str, List[str]]:
|
|
309
310
|
mappings: dict[str, Any] = {}
|
|
310
311
|
if len(self.tags.cis20) > 0:
|
|
311
|
-
mappings["cis20"] = [tag
|
|
312
|
+
mappings["cis20"] = [tag for tag in self.tags.cis20]
|
|
312
313
|
if len(self.tags.kill_chain_phases) > 0:
|
|
313
|
-
mappings['kill_chain_phases'] = [phase
|
|
314
|
+
mappings['kill_chain_phases'] = [phase for phase in self.tags.kill_chain_phases]
|
|
314
315
|
if len(self.tags.mitre_attack_id) > 0:
|
|
315
316
|
mappings['mitre_attack'] = self.tags.mitre_attack_id
|
|
316
317
|
if len(self.tags.nist) > 0:
|
|
317
|
-
mappings['nist'] = [category
|
|
318
|
+
mappings['nist'] = [category for category in self.tags.nist]
|
|
318
319
|
|
|
319
320
|
# No need to sort the dict! It has been constructed in-order.
|
|
320
321
|
# However, if this logic is changed, then consider reordering or
|
|
@@ -361,66 +362,87 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
361
362
|
def providing_technologies(self) -> List[ProvidingTechnology]:
|
|
362
363
|
return ProvidingTechnology.getProvidingTechFromSearch(self.search)
|
|
363
364
|
|
|
364
|
-
|
|
365
|
+
|
|
365
366
|
@computed_field
|
|
366
367
|
@property
|
|
367
368
|
def risk(self) -> list[dict[str, Any]]:
|
|
368
369
|
risk_objects: list[dict[str, str | int]] = []
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
risk_object_system_types = {'device', 'endpoint', 'hostname', 'ip address'}
|
|
372
|
-
process_threat_object_types = {'process name', 'process'}
|
|
373
|
-
file_threat_object_types = {'file name', 'file', 'file hash'}
|
|
374
|
-
url_threat_object_types = {'url string', 'url'}
|
|
375
|
-
ip_threat_object_types = {'ip address'}
|
|
376
|
-
|
|
377
|
-
for entity in self.tags.observable:
|
|
370
|
+
|
|
371
|
+
for entity in self.rba.risk_objects:
|
|
378
372
|
risk_object: dict[str, str | int] = dict()
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
373
|
+
risk_object['risk_object_type'] = entity.type
|
|
374
|
+
risk_object['risk_object_field'] = entity.field
|
|
375
|
+
risk_object['risk_score'] = entity.score
|
|
376
|
+
risk_objects.append(risk_object)
|
|
377
|
+
|
|
378
|
+
for entity in self.rba.threat_objects:
|
|
379
|
+
threat_object: dict[str, str] = dict()
|
|
380
|
+
threat_object['threat_object_field'] = entity.field
|
|
381
|
+
threat_object['threat_object_type'] = entity.type
|
|
382
|
+
risk_objects.append(threat_object)
|
|
383
|
+
return risk_objects
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
# TODO Remove observable code
|
|
387
|
+
# @computed_field
|
|
388
|
+
# @property
|
|
389
|
+
# def risk(self) -> list[dict[str, Any]]:
|
|
390
|
+
# risk_objects: list[dict[str, str | int]] = []
|
|
391
|
+
# # TODO (#246): "User Name" type should map to a "user" risk object and not "other"
|
|
392
|
+
# risk_object_user_types = {'user', 'username', 'email address'}
|
|
393
|
+
# risk_object_system_types = {'device', 'endpoint', 'hostname', 'ip address'}
|
|
394
|
+
# process_threat_object_types = {'process name', 'process'}
|
|
395
|
+
# file_threat_object_types = {'file name', 'file', 'file hash'}
|
|
396
|
+
# url_threat_object_types = {'url string', 'url'}
|
|
397
|
+
# ip_threat_object_types = {'ip address'}
|
|
398
|
+
|
|
399
|
+
# for entity in self.tags.observable:
|
|
400
|
+
# risk_object: dict[str, str | int] = dict()
|
|
401
|
+
# if 'Victim' in entity.role and entity.type.lower() in risk_object_user_types:
|
|
402
|
+
# risk_object['risk_object_type'] = 'user'
|
|
403
|
+
# risk_object['risk_object_field'] = entity.name
|
|
404
|
+
# risk_object['risk_score'] = self.tags.risk_score
|
|
405
|
+
# risk_objects.append(risk_object)
|
|
406
|
+
|
|
407
|
+
# elif 'Victim' in entity.role and entity.type.lower() in risk_object_system_types:
|
|
408
|
+
# risk_object['risk_object_type'] = 'system'
|
|
409
|
+
# risk_object['risk_object_field'] = entity.name
|
|
410
|
+
# risk_object['risk_score'] = self.tags.risk_score
|
|
411
|
+
# risk_objects.append(risk_object)
|
|
412
|
+
|
|
413
|
+
# elif 'Attacker' in entity.role and entity.type.lower() in process_threat_object_types:
|
|
414
|
+
# risk_object['threat_object_field'] = entity.name
|
|
415
|
+
# risk_object['threat_object_type'] = "process"
|
|
416
|
+
# risk_objects.append(risk_object)
|
|
417
|
+
|
|
418
|
+
# elif 'Attacker' in entity.role and entity.type.lower() in file_threat_object_types:
|
|
419
|
+
# risk_object['threat_object_field'] = entity.name
|
|
420
|
+
# risk_object['threat_object_type'] = "file_name"
|
|
421
|
+
# risk_objects.append(risk_object)
|
|
422
|
+
|
|
423
|
+
# elif 'Attacker' in entity.role and entity.type.lower() in ip_threat_object_types:
|
|
424
|
+
# risk_object['threat_object_field'] = entity.name
|
|
425
|
+
# risk_object['threat_object_type'] = "ip_address"
|
|
426
|
+
# risk_objects.append(risk_object)
|
|
427
|
+
|
|
428
|
+
# elif 'Attacker' in entity.role and entity.type.lower() in url_threat_object_types:
|
|
429
|
+
# risk_object['threat_object_field'] = entity.name
|
|
430
|
+
# risk_object['threat_object_type'] = "url"
|
|
431
|
+
# risk_objects.append(risk_object)
|
|
410
432
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
433
|
+
# elif 'Attacker' in entity.role:
|
|
434
|
+
# risk_object['threat_object_field'] = entity.name
|
|
435
|
+
# risk_object['threat_object_type'] = entity.type.lower()
|
|
436
|
+
# risk_objects.append(risk_object)
|
|
415
437
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
438
|
+
# else:
|
|
439
|
+
# risk_object['risk_object_type'] = 'other'
|
|
440
|
+
# risk_object['risk_object_field'] = entity.name
|
|
441
|
+
# risk_object['risk_score'] = self.tags.risk_score
|
|
442
|
+
# risk_objects.append(risk_object)
|
|
443
|
+
# continue
|
|
422
444
|
|
|
423
|
-
|
|
445
|
+
# return risk_objects
|
|
424
446
|
|
|
425
447
|
@computed_field
|
|
426
448
|
@property
|
|
@@ -435,7 +457,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
435
457
|
# break the `inspect` action.
|
|
436
458
|
return {
|
|
437
459
|
'detection_id': str(self.id),
|
|
438
|
-
'deprecated': '1' if self.status == DetectionStatus.deprecated
|
|
460
|
+
'deprecated': '1' if self.status == DetectionStatus.deprecated else '0', # type: ignore
|
|
439
461
|
'detection_version': str(self.version),
|
|
440
462
|
'publish_time': datetime.datetime(self.date.year,self.date.month,self.date.day,0,0,0,0,tzinfo=datetime.timezone.utc).timestamp()
|
|
441
463
|
}
|
|
@@ -456,6 +478,11 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
456
478
|
"source": self.source,
|
|
457
479
|
"nes_fields": self.nes_fields,
|
|
458
480
|
}
|
|
481
|
+
if self.rba is not None:
|
|
482
|
+
model["risk_severity"] = self.rba.severity
|
|
483
|
+
model['tags']['risk_score'] = self.rba.risk_score
|
|
484
|
+
else:
|
|
485
|
+
model['tags']['risk_score'] = 0
|
|
459
486
|
|
|
460
487
|
# Only a subset of macro fields are required:
|
|
461
488
|
all_macros: list[dict[str, str | list[str]]] = []
|
|
@@ -473,17 +500,17 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
473
500
|
|
|
474
501
|
all_lookups: list[dict[str, str | int | None]] = []
|
|
475
502
|
for lookup in self.lookups:
|
|
476
|
-
if lookup
|
|
503
|
+
if isinstance(lookup, KVStoreLookup):
|
|
477
504
|
all_lookups.append(
|
|
478
505
|
{
|
|
479
506
|
"name": lookup.name,
|
|
480
507
|
"description": lookup.description,
|
|
481
508
|
"collection": lookup.collection,
|
|
482
509
|
"case_sensitive_match": None,
|
|
483
|
-
"fields_list": lookup.
|
|
510
|
+
"fields_list": lookup.fields_to_fields_list_conf_format
|
|
484
511
|
}
|
|
485
512
|
)
|
|
486
|
-
elif lookup
|
|
513
|
+
elif isinstance(lookup, FileBackedLookup):
|
|
487
514
|
all_lookups.append(
|
|
488
515
|
{
|
|
489
516
|
"name": lookup.name,
|
|
@@ -491,9 +518,8 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
491
518
|
"filename": lookup.filename.name,
|
|
492
519
|
"default_match": "true" if lookup.default_match else "false",
|
|
493
520
|
"case_sensitive_match": "true" if lookup.case_sensitive_match else "false",
|
|
494
|
-
"match_type": lookup.
|
|
495
|
-
"min_matches": lookup.min_matches
|
|
496
|
-
"fields_list": lookup.fields_list
|
|
521
|
+
"match_type": lookup.match_type_to_conf_format,
|
|
522
|
+
"min_matches": lookup.min_matches
|
|
497
523
|
}
|
|
498
524
|
)
|
|
499
525
|
model['lookups'] = all_lookups # type: ignore
|
|
@@ -570,7 +596,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
570
596
|
# This is presently a requirement when 1 or more drilldowns are added to a detection.
|
|
571
597
|
# Note that this is only required for production searches that are not hunting
|
|
572
598
|
|
|
573
|
-
if self.type == AnalyticsType.Hunting
|
|
599
|
+
if self.type == AnalyticsType.Hunting or self.status != DetectionStatus.production:
|
|
574
600
|
#No additional check need to happen on the potential drilldowns.
|
|
575
601
|
pass
|
|
576
602
|
else:
|
|
@@ -713,14 +739,14 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
713
739
|
if status != DetectionStatus.production:
|
|
714
740
|
errors.append(
|
|
715
741
|
f"status is '{status.name}'. Detections that are enabled by default MUST be "
|
|
716
|
-
f"'{DetectionStatus.production
|
|
742
|
+
f"'{DetectionStatus.production}'"
|
|
717
743
|
)
|
|
718
744
|
|
|
719
745
|
if searchType not in [AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]:
|
|
720
746
|
errors.append(
|
|
721
|
-
f"type is '{searchType
|
|
747
|
+
f"type is '{searchType}'. Detections that are enabled by default MUST be one"
|
|
722
748
|
" of the following types: "
|
|
723
|
-
f"{[AnalyticsType.Anomaly
|
|
749
|
+
f"{[AnalyticsType.Anomaly, AnalyticsType.Correlation, AnalyticsType.TTP]}")
|
|
724
750
|
if len(errors) > 0:
|
|
725
751
|
error_message = "\n - ".join(errors)
|
|
726
752
|
raise ValueError(f"Detection is 'enabled_by_default: true' however \n - {error_message}")
|
|
@@ -729,7 +755,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
729
755
|
|
|
730
756
|
@model_validator(mode="after")
|
|
731
757
|
def addTags_nist(self):
|
|
732
|
-
if self.type == AnalyticsType.TTP
|
|
758
|
+
if self.type == AnalyticsType.TTP:
|
|
733
759
|
self.tags.nist = [NistCategory.DE_CM]
|
|
734
760
|
else:
|
|
735
761
|
self.tags.nist = [NistCategory.DE_AE]
|
|
@@ -757,50 +783,93 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
757
783
|
|
|
758
784
|
|
|
759
785
|
@model_validator(mode="after")
|
|
760
|
-
def
|
|
786
|
+
def ensureProperRBAConfig(self):
|
|
761
787
|
"""
|
|
762
|
-
If a
|
|
763
|
-
|
|
788
|
+
If a detection has an RBA deployment and is PRODUCTION, then it must have an RBA config, with at least one risk object
|
|
789
|
+
|
|
764
790
|
Returns:
|
|
765
|
-
self: Returns itself if the
|
|
791
|
+
self: Returns itself if the validation passes
|
|
766
792
|
"""
|
|
767
|
-
# NOTE: we ignore the type error around self.status because we are using Pydantic's
|
|
768
|
-
# use_enum_values configuration
|
|
769
|
-
# https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
|
|
770
|
-
if self.status not in [DetectionStatus.production.value]: # type: ignore
|
|
771
|
-
# Only perform this validation on production detections
|
|
772
|
-
return self
|
|
773
793
|
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
794
|
+
|
|
795
|
+
if self.deployment.alert_action.rba is None or self.deployment.alert_action.rba.enabled is False:
|
|
796
|
+
# confirm we don't have an RBA config
|
|
797
|
+
if self.rba is None:
|
|
798
|
+
return self
|
|
799
|
+
else:
|
|
800
|
+
raise ValueError(
|
|
801
|
+
"Detection does not have a matching RBA deployment config, the RBA portion should be omitted."
|
|
802
|
+
)
|
|
803
|
+
else:
|
|
804
|
+
if self.rba is None:
|
|
805
|
+
raise ValueError(
|
|
806
|
+
"Detection is expected to have an RBA object based on its deployment config"
|
|
807
|
+
)
|
|
808
|
+
else:
|
|
809
|
+
if len(self.rba.risk_objects) > 0: # type: ignore
|
|
810
|
+
return self
|
|
811
|
+
else:
|
|
812
|
+
raise ValueError(
|
|
813
|
+
"Detection expects an RBA config with at least one risk object."
|
|
814
|
+
)
|
|
782
815
|
|
|
783
|
-
if roles.count("Victim") == 0:
|
|
784
|
-
raise ValueError(
|
|
785
|
-
"Error, there must be AT LEAST 1 Observable with the role 'Victim' declared in "
|
|
786
|
-
"Detection.tags.observables. However, none were found."
|
|
787
|
-
)
|
|
788
816
|
|
|
789
|
-
|
|
790
|
-
|
|
817
|
+
# TODO - Remove old observable code
|
|
818
|
+
# @model_validator(mode="after")
|
|
819
|
+
# def ensureProperObservablesExist(self):
|
|
820
|
+
# """
|
|
821
|
+
# If a detections is PRODUCTION and either TTP or ANOMALY, then it MUST have an Observable with the VICTIM role.
|
|
822
|
+
|
|
823
|
+
# Returns:
|
|
824
|
+
# self: Returns itself if the valdiation passes
|
|
825
|
+
# """
|
|
826
|
+
# # NOTE: we ignore the type error around self.status because we are using Pydantic's
|
|
827
|
+
# # use_enum_values configuration
|
|
828
|
+
# # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
|
|
829
|
+
# if self.status not in [DetectionStatus.production.value]: # type: ignore
|
|
830
|
+
# # Only perform this validation on production detections
|
|
831
|
+
# return self
|
|
832
|
+
|
|
833
|
+
# if self.type not in [AnalyticsType.TTP.value, AnalyticsType.Anomaly.value]:
|
|
834
|
+
# # Only perform this validation on TTP and Anomaly detections
|
|
835
|
+
# return self
|
|
836
|
+
|
|
837
|
+
# # Detection is required to have a victim
|
|
838
|
+
# roles: list[str] = []
|
|
839
|
+
# for observable in self.tags.observable:
|
|
840
|
+
# roles.extend(observable.role)
|
|
841
|
+
|
|
842
|
+
# if roles.count("Victim") == 0:
|
|
843
|
+
# raise ValueError(
|
|
844
|
+
# "Error, there must be AT LEAST 1 Observable with the role 'Victim' declared in "
|
|
845
|
+
# "Detection.tags.observables. However, none were found."
|
|
846
|
+
# )
|
|
847
|
+
|
|
848
|
+
# # Exactly one victim was found
|
|
849
|
+
# return self
|
|
791
850
|
|
|
792
851
|
@model_validator(mode="after")
|
|
793
|
-
def
|
|
794
|
-
|
|
852
|
+
def search_rba_fields_exist_validate(self):
|
|
853
|
+
# Return immediately if RBA isn't required
|
|
854
|
+
if (self.deployment.alert_action.rba.enabled is False or self.deployment.alert_action.rba is None) and self.rba is None: #type: ignore
|
|
855
|
+
return self
|
|
856
|
+
|
|
857
|
+
# Raise error if RBA isn't present
|
|
858
|
+
if self.rba is None:
|
|
859
|
+
raise ValueError(
|
|
860
|
+
"RBA is required for this detection based on its deployment config"
|
|
861
|
+
)
|
|
862
|
+
risk_fields = [ob.field.lower() for ob in self.rba.risk_objects]
|
|
863
|
+
threat_fields = [ob.field.lower() for ob in self.rba.threat_objects]
|
|
864
|
+
rba_fields = risk_fields + threat_fields
|
|
795
865
|
|
|
796
|
-
# All $field$ fields from the message must appear in the search
|
|
797
866
|
field_match_regex = r"\$([^\s.]*)\$"
|
|
798
867
|
|
|
799
868
|
missing_fields: set[str]
|
|
800
|
-
if self.
|
|
801
|
-
matches = re.findall(field_match_regex, self.
|
|
869
|
+
if self.rba.message:
|
|
870
|
+
matches = re.findall(field_match_regex, self.rba.message.lower())
|
|
802
871
|
message_fields = [match.replace("$", "").lower() for match in matches]
|
|
803
|
-
missing_fields = set([field for field in
|
|
872
|
+
missing_fields = set([field for field in rba_fields if field not in self.search.lower()])
|
|
804
873
|
else:
|
|
805
874
|
message_fields = []
|
|
806
875
|
missing_fields = set()
|
|
@@ -808,10 +877,9 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
808
877
|
error_messages: list[str] = []
|
|
809
878
|
if len(missing_fields) > 0:
|
|
810
879
|
error_messages.append(
|
|
811
|
-
"The following fields are declared
|
|
880
|
+
"The following fields are declared in the rba config, but do not exist in the "
|
|
812
881
|
f"search: {missing_fields}"
|
|
813
882
|
)
|
|
814
|
-
|
|
815
883
|
missing_fields = set([field for field in message_fields if field not in self.search.lower()])
|
|
816
884
|
if len(missing_fields) > 0:
|
|
817
885
|
error_messages.append(
|
|
@@ -819,19 +887,59 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
819
887
|
f"the search: {missing_fields}"
|
|
820
888
|
)
|
|
821
889
|
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
# https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
|
|
825
|
-
if len(error_messages) > 0 and self.status == DetectionStatus.production.value: # type: ignore
|
|
890
|
+
if len(error_messages) > 0 and self.status == DetectionStatus.production:
|
|
891
|
+
|
|
826
892
|
msg = (
|
|
827
|
-
"Use of fields in
|
|
893
|
+
"Use of fields in rba/messages that do not appear in search:\n\t- "
|
|
828
894
|
"\n\t- ".join(error_messages)
|
|
829
895
|
)
|
|
830
896
|
raise ValueError(msg)
|
|
831
|
-
|
|
832
|
-
# Found everything
|
|
833
897
|
return self
|
|
834
898
|
|
|
899
|
+
# TODO: Remove old observable code
|
|
900
|
+
# @model_validator(mode="after")
|
|
901
|
+
# def search_observables_exist_validate(self):
|
|
902
|
+
# observable_fields = [ob.name.lower() for ob in self.tags.observable]
|
|
903
|
+
|
|
904
|
+
# # All $field$ fields from the message must appear in the search
|
|
905
|
+
# field_match_regex = r"\$([^\s.]*)\$"
|
|
906
|
+
|
|
907
|
+
# missing_fields: set[str]
|
|
908
|
+
# if self.tags.message:
|
|
909
|
+
# matches = re.findall(field_match_regex, self.tags.message.lower())
|
|
910
|
+
# message_fields = [match.replace("$", "").lower() for match in matches]
|
|
911
|
+
# missing_fields = set([field for field in observable_fields if field not in self.search.lower()])
|
|
912
|
+
# else:
|
|
913
|
+
# message_fields = []
|
|
914
|
+
# missing_fields = set()
|
|
915
|
+
|
|
916
|
+
# error_messages: list[str] = []
|
|
917
|
+
# if len(missing_fields) > 0:
|
|
918
|
+
# error_messages.append(
|
|
919
|
+
# "The following fields are declared as observables, but do not exist in the "
|
|
920
|
+
# f"search: {missing_fields}"
|
|
921
|
+
# )
|
|
922
|
+
|
|
923
|
+
# missing_fields = set([field for field in message_fields if field not in self.search.lower()])
|
|
924
|
+
# if len(missing_fields) > 0:
|
|
925
|
+
# error_messages.append(
|
|
926
|
+
# "The following fields are used as fields in the message, but do not exist in "
|
|
927
|
+
# f"the search: {missing_fields}"
|
|
928
|
+
# )
|
|
929
|
+
|
|
930
|
+
# # NOTE: we ignore the type error around self.status because we are using Pydantic's
|
|
931
|
+
# # use_enum_values configuration
|
|
932
|
+
# # https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
|
|
933
|
+
# if len(error_messages) > 0 and self.status == DetectionStatus.production.value: # type: ignore
|
|
934
|
+
# msg = (
|
|
935
|
+
# "Use of fields in observables/messages that do not appear in search:\n\t- "
|
|
936
|
+
# "\n\t- ".join(error_messages)
|
|
937
|
+
# )
|
|
938
|
+
# raise ValueError(msg)
|
|
939
|
+
|
|
940
|
+
# # Found everything
|
|
941
|
+
# return self
|
|
942
|
+
|
|
835
943
|
@field_validator("tests", mode="before")
|
|
836
944
|
def ensure_yml_test_is_unittest(cls, v:list[dict]):
|
|
837
945
|
"""The typing for the tests field allows it to be one of
|
|
@@ -878,7 +986,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
878
986
|
info: ValidationInfo
|
|
879
987
|
) -> list[UnitTest | IntegrationTest | ManualTest]:
|
|
880
988
|
# Only production analytics require tests
|
|
881
|
-
if info.data.get("status", "") != DetectionStatus.production
|
|
989
|
+
if info.data.get("status", "") != DetectionStatus.production:
|
|
882
990
|
return v
|
|
883
991
|
|
|
884
992
|
# All types EXCEPT Correlation MUST have test(s). Any other type, including newly defined
|
|
@@ -991,7 +1099,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
991
1099
|
value = getattr(self, field)
|
|
992
1100
|
|
|
993
1101
|
# Enums and Path objects cannot be serialized directly, so we convert it to a string
|
|
994
|
-
if isinstance(value,
|
|
1102
|
+
if isinstance(value, StrEnum) or isinstance(value, pathlib.Path):
|
|
995
1103
|
value = str(value)
|
|
996
1104
|
|
|
997
1105
|
# Alias any fields as needed
|
|
@@ -1013,7 +1121,7 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
1013
1121
|
# Initialize the dict as a mapping of strings to str/bool
|
|
1014
1122
|
result: dict[str, Union[str, bool]] = {
|
|
1015
1123
|
"name": test.name,
|
|
1016
|
-
"test_type": test.test_type
|
|
1124
|
+
"test_type": test.test_type
|
|
1017
1125
|
}
|
|
1018
1126
|
|
|
1019
1127
|
# If result is not None, get a summary of the test result w/ the requested fields
|
|
@@ -31,10 +31,8 @@ import pathlib
|
|
|
31
31
|
NO_FILE_NAME = "NO_FILE_NAME"
|
|
32
32
|
|
|
33
33
|
|
|
34
|
-
# TODO (#266): disable the use_enum_values configuration
|
|
35
34
|
class SecurityContentObject_Abstract(BaseModel, abc.ABC):
|
|
36
|
-
model_config = ConfigDict(
|
|
37
|
-
|
|
35
|
+
model_config = ConfigDict(validate_default=True,extra="forbid")
|
|
38
36
|
name: str = Field(...,max_length=99)
|
|
39
37
|
author: str = Field(...,max_length=255)
|
|
40
38
|
date: datetime.date = Field(...)
|
|
@@ -162,10 +160,10 @@ class SecurityContentObject_Abstract(BaseModel, abc.ABC):
|
|
|
162
160
|
raise ValueError("Cannot set deployment - DirectorOutputDto not passed to Detection Constructor in context")
|
|
163
161
|
|
|
164
162
|
type_to_deployment_name_map = {
|
|
165
|
-
AnalyticsType.TTP
|
|
166
|
-
AnalyticsType.Hunting
|
|
167
|
-
AnalyticsType.Correlation
|
|
168
|
-
AnalyticsType.Anomaly
|
|
163
|
+
AnalyticsType.TTP: "ESCU Default Configuration TTP",
|
|
164
|
+
AnalyticsType.Hunting: "ESCU Default Configuration Hunting",
|
|
165
|
+
AnalyticsType.Correlation: "ESCU Default Configuration Correlation",
|
|
166
|
+
AnalyticsType.Anomaly: "ESCU Default Configuration Anomaly",
|
|
169
167
|
"Baseline": "ESCU Default Configuration Baseline"
|
|
170
168
|
}
|
|
171
169
|
converted_type_field = type_to_deployment_name_map[typeField]
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
from pydantic import BaseModel, model_serializer
|
|
2
|
+
from pydantic import BaseModel, model_serializer, ConfigDict
|
|
3
3
|
from typing import Optional
|
|
4
4
|
|
|
5
5
|
from contentctl.objects.deployment_email import DeploymentEmail
|
|
@@ -9,6 +9,7 @@ from contentctl.objects.deployment_slack import DeploymentSlack
|
|
|
9
9
|
from contentctl.objects.deployment_phantom import DeploymentPhantom
|
|
10
10
|
|
|
11
11
|
class AlertAction(BaseModel):
|
|
12
|
+
model_config = ConfigDict(extra="forbid")
|
|
12
13
|
email: Optional[DeploymentEmail] = None
|
|
13
14
|
notable: Optional[DeploymentNotable] = None
|
|
14
15
|
rba: Optional[DeploymentRBA] = DeploymentRBA()
|
contentctl/objects/atomic.py
CHANGED
contentctl/objects/base_test.py
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
from enum import
|
|
1
|
+
from enum import StrEnum
|
|
2
2
|
from typing import Union
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
4
|
|
|
5
|
-
from pydantic import BaseModel
|
|
5
|
+
from pydantic import BaseModel,ConfigDict
|
|
6
6
|
|
|
7
7
|
from contentctl.objects.base_test_result import BaseTestResult
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
class TestType(
|
|
10
|
+
class TestType(StrEnum):
|
|
11
11
|
"""
|
|
12
12
|
Types of tests
|
|
13
13
|
"""
|
|
@@ -21,6 +21,7 @@ class TestType(str, Enum):
|
|
|
21
21
|
|
|
22
22
|
# TODO (#224): enforce distinct test names w/in detections
|
|
23
23
|
class BaseTest(BaseModel, ABC):
|
|
24
|
+
model_config = ConfigDict(extra="forbid")
|
|
24
25
|
"""
|
|
25
26
|
A test case for a detection
|
|
26
27
|
"""
|