contentctl 4.3.3__py3-none-any.whl → 4.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +0 -6
- contentctl/enrichments/attack_enrichment.py +2 -1
- contentctl/enrichments/cve_enrichment.py +2 -2
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +3 -2
- contentctl/objects/annotated_types.py +6 -0
- contentctl/objects/config.py +3 -3
- contentctl/objects/correlation_search.py +35 -28
- contentctl/objects/detection_tags.py +5 -3
- contentctl/objects/mitre_attack_enrichment.py +2 -1
- contentctl/objects/risk_event.py +94 -76
- contentctl/objects/story_tags.py +3 -3
- {contentctl-4.3.3.dist-info → contentctl-4.3.4.dist-info}/METADATA +2 -2
- {contentctl-4.3.3.dist-info → contentctl-4.3.4.dist-info}/RECORD +16 -15
- {contentctl-4.3.3.dist-info → contentctl-4.3.4.dist-info}/LICENSE.md +0 -0
- {contentctl-4.3.3.dist-info → contentctl-4.3.4.dist-info}/WHEEL +0 -0
- {contentctl-4.3.3.dist-info → contentctl-4.3.4.dist-info}/entry_points.txt +0 -0
|
@@ -374,12 +374,6 @@ class DetectionTestingInfrastructure(BaseModel, abc.ABC):
|
|
|
374
374
|
return
|
|
375
375
|
|
|
376
376
|
try:
|
|
377
|
-
# NOTE: (THIS CODE HAS MOVED) we handle skipping entire detections differently than
|
|
378
|
-
# we do skipping individual test cases; we skip entire detections by excluding
|
|
379
|
-
# them to an entirely separate queue, while we skip individual test cases via the
|
|
380
|
-
# BaseTest.skip() method, such as when we are skipping all integration tests (see
|
|
381
|
-
# DetectionBuilder.skipIntegrationTests)
|
|
382
|
-
# TODO: are we skipping by production status elsewhere?
|
|
383
377
|
detection = self.sync_obj.inputQueue.pop()
|
|
384
378
|
self.sync_obj.currentTestingQueue[self.get_name()] = detection
|
|
385
379
|
except IndexError:
|
|
@@ -10,6 +10,7 @@ from dataclasses import field
|
|
|
10
10
|
from typing import Annotated,Any
|
|
11
11
|
from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
|
|
12
12
|
from contentctl.objects.config import validate
|
|
13
|
+
from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE
|
|
13
14
|
logging.getLogger('taxii2client').setLevel(logging.CRITICAL)
|
|
14
15
|
|
|
15
16
|
|
|
@@ -23,7 +24,7 @@ class AttackEnrichment(BaseModel):
|
|
|
23
24
|
_ = enrichment.get_attack_lookup(str(config.path))
|
|
24
25
|
return enrichment
|
|
25
26
|
|
|
26
|
-
def getEnrichmentByMitreID(self, mitre_id:
|
|
27
|
+
def getEnrichmentByMitreID(self, mitre_id:MITRE_ATTACK_ID_TYPE)->MitreAttackEnrichment:
|
|
27
28
|
if not self.use_enrichment:
|
|
28
29
|
raise Exception(f"Error, trying to add Mitre Enrichment, but use_enrichment was set to False")
|
|
29
30
|
|
|
@@ -8,7 +8,7 @@ from typing import Annotated, Any, Union, TYPE_CHECKING
|
|
|
8
8
|
from pydantic import BaseModel,Field, computed_field
|
|
9
9
|
from decimal import Decimal
|
|
10
10
|
from requests.exceptions import ReadTimeout
|
|
11
|
-
|
|
11
|
+
from contentctl.objects.annotated_types import CVE_TYPE
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
13
|
from contentctl.objects.config import validate
|
|
14
14
|
|
|
@@ -18,7 +18,7 @@ CVESSEARCH_API_URL = 'https://cve.circl.lu'
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class CveEnrichmentObj(BaseModel):
|
|
21
|
-
id:
|
|
21
|
+
id: CVE_TYPE
|
|
22
22
|
cvss: Annotated[Decimal, Field(ge=.1, le=10, decimal_places=1)]
|
|
23
23
|
summary: str
|
|
24
24
|
|
|
@@ -322,12 +322,13 @@ class Detection_Abstract(SecurityContentObject):
|
|
|
322
322
|
@property
|
|
323
323
|
def providing_technologies(self) -> List[ProvidingTechnology]:
|
|
324
324
|
return ProvidingTechnology.getProvidingTechFromSearch(self.search)
|
|
325
|
-
|
|
326
|
-
|
|
325
|
+
|
|
326
|
+
# TODO (#247): Refactor the risk property of detection_abstract
|
|
327
327
|
@computed_field
|
|
328
328
|
@property
|
|
329
329
|
def risk(self) -> list[dict[str, Any]]:
|
|
330
330
|
risk_objects: list[dict[str, str | int]] = []
|
|
331
|
+
# TODO (#246): "User Name" type should map to a "user" risk object and not "other"
|
|
331
332
|
risk_object_user_types = {'user', 'username', 'email address'}
|
|
332
333
|
risk_object_system_types = {'device', 'endpoint', 'hostname', 'ip address'}
|
|
333
334
|
process_threat_object_types = {'process name', 'process'}
|
contentctl/objects/config.py
CHANGED
|
@@ -18,7 +18,7 @@ from urllib.parse import urlparse
|
|
|
18
18
|
from abc import ABC, abstractmethod
|
|
19
19
|
from contentctl.objects.enums import PostTestBehavior, DetectionTestingMode
|
|
20
20
|
from contentctl.objects.detection import Detection
|
|
21
|
-
|
|
21
|
+
from contentctl.objects.annotated_types import APPID_TYPE
|
|
22
22
|
import tqdm
|
|
23
23
|
from functools import partialmethod
|
|
24
24
|
|
|
@@ -33,7 +33,7 @@ class App_Base(BaseModel,ABC):
|
|
|
33
33
|
model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
|
|
34
34
|
uid: Optional[int] = Field(default=None)
|
|
35
35
|
title: str = Field(description="Human-readable name used by the app. This can have special characters.")
|
|
36
|
-
appid: Optional[
|
|
36
|
+
appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. "
|
|
37
37
|
"It may ONLY have characters, numbers, and underscores. No other characters are allowed.")
|
|
38
38
|
version: str = Field(description="The version of your Content Pack. This must follow semantic versioning guidelines.")
|
|
39
39
|
description: Optional[str] = Field(default="description of app",description="Free text description of the Content Pack.")
|
|
@@ -101,7 +101,7 @@ class CustomApp(App_Base):
|
|
|
101
101
|
# https://docs.splunk.com/Documentation/Splunk/9.0.4/Admin/Appconf
|
|
102
102
|
uid: int = Field(ge=2, lt=100000, default_factory=lambda:random.randint(20000,100000))
|
|
103
103
|
title: str = Field(default="Content Pack",description="Human-readable name used by the app. This can have special characters.")
|
|
104
|
-
appid:
|
|
104
|
+
appid: APPID_TYPE = Field(default="ContentPack",description="Internal name used by your app. "
|
|
105
105
|
"It may ONLY have characters, numbers, and underscores. No other characters are allowed.")
|
|
106
106
|
version: str = Field(default="0.0.1",description="The version of your Content Pack. This must follow semantic versioning guidelines.", validate_default=True)
|
|
107
107
|
|
|
@@ -575,10 +575,11 @@ class CorrelationSearch(BaseModel):
|
|
|
575
575
|
self.logger.debug(f"Using cached risk events ({len(self._risk_events)} total).")
|
|
576
576
|
return self._risk_events
|
|
577
577
|
|
|
578
|
+
# TODO (#248): Refactor risk/notable querying to pin to a single savedsearch ID
|
|
578
579
|
# Search for all risk events from a single scheduled search (indicated by orig_sid)
|
|
579
580
|
query = (
|
|
580
581
|
f'search index=risk search_name="{self.name}" [search index=risk search '
|
|
581
|
-
f'search_name="{self.name}" |
|
|
582
|
+
f'search_name="{self.name}" | tail 1 | fields orig_sid] | tojson'
|
|
582
583
|
)
|
|
583
584
|
result_iterator = self._search(query)
|
|
584
585
|
|
|
@@ -643,7 +644,7 @@ class CorrelationSearch(BaseModel):
|
|
|
643
644
|
# Search for all notable events from a single scheduled search (indicated by orig_sid)
|
|
644
645
|
query = (
|
|
645
646
|
f'search index=notable search_name="{self.name}" [search index=notable search '
|
|
646
|
-
f'search_name="{self.name}" |
|
|
647
|
+
f'search_name="{self.name}" | tail 1 | fields orig_sid] | tojson'
|
|
647
648
|
)
|
|
648
649
|
result_iterator = self._search(query)
|
|
649
650
|
|
|
@@ -686,15 +687,17 @@ class CorrelationSearch(BaseModel):
|
|
|
686
687
|
check the risks/notables
|
|
687
688
|
:returns: an IntegrationTestResult on failure; None on success
|
|
688
689
|
"""
|
|
689
|
-
# TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the false
|
|
690
|
-
# positive rate in risk/obseravble matching
|
|
691
690
|
# Create a mapping of the relevant observables to counters
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
#
|
|
696
|
-
#
|
|
697
|
-
|
|
691
|
+
observables = CorrelationSearch._get_relevant_observables(self.detection.tags.observable)
|
|
692
|
+
observable_counts: dict[str, int] = {str(x): 0 for x in observables}
|
|
693
|
+
|
|
694
|
+
# NOTE: we intentionally want this to be an error state and not a failure state, as
|
|
695
|
+
# ultimately this validation should be handled during the build process
|
|
696
|
+
if len(observables) != len(observable_counts):
|
|
697
|
+
raise ClientError(
|
|
698
|
+
f"At least two observables in '{self.detection.name}' have the same name; "
|
|
699
|
+
"each observable for a detection should be unique."
|
|
700
|
+
)
|
|
698
701
|
|
|
699
702
|
# Get the risk events; note that we use the cached risk events, expecting they were
|
|
700
703
|
# saved by a prior call to risk_event_exists
|
|
@@ -710,25 +713,29 @@ class CorrelationSearch(BaseModel):
|
|
|
710
713
|
)
|
|
711
714
|
event.validate_against_detection(self.detection)
|
|
712
715
|
|
|
713
|
-
# TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the
|
|
714
|
-
# false positive rate in risk/obseravble matching
|
|
715
716
|
# Update observable count based on match
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
#
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
717
|
+
matched_observable = event.get_matched_observable(self.detection.tags.observable)
|
|
718
|
+
self.logger.debug(
|
|
719
|
+
f"Matched risk event (object={event.risk_object}, type={event.risk_object_type}) "
|
|
720
|
+
f"to observable (name={matched_observable.name}, type={matched_observable.type}, "
|
|
721
|
+
f"role={matched_observable.role}) using the source field "
|
|
722
|
+
f"'{event.source_field_name}'"
|
|
723
|
+
)
|
|
724
|
+
observable_counts[str(matched_observable)] += 1
|
|
725
|
+
|
|
726
|
+
# Report any observables which did not have at least one match to a risk event
|
|
727
|
+
for observable in observables:
|
|
728
|
+
self.logger.debug(
|
|
729
|
+
f"Matched observable (name={observable.name}, type={observable.type}, "
|
|
730
|
+
f"role={observable.role}) to {observable_counts[str(observable)]} risk events."
|
|
731
|
+
)
|
|
732
|
+
if observable_counts[str(observable)] == 0:
|
|
733
|
+
raise ValidationFailed(
|
|
734
|
+
f"Observable (name={observable.name}, type={observable.type}, "
|
|
735
|
+
f"role={observable.role}) was not matched to any risk events."
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
# TODO (#250): Re-enable and refactor code that validates the specific risk counts
|
|
732
739
|
# Validate risk events in aggregate; we should have an equal amount of risk events for each
|
|
733
740
|
# relevant observable, and the total count should match the total number of events
|
|
734
741
|
# individual_count: Optional[int] = None
|
|
@@ -33,7 +33,7 @@ from contentctl.objects.enums import (
|
|
|
33
33
|
SecurityContentProductName
|
|
34
34
|
)
|
|
35
35
|
from contentctl.objects.atomic import AtomicTest
|
|
36
|
-
|
|
36
|
+
from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE, CVE_TYPE
|
|
37
37
|
|
|
38
38
|
# TODO (#266): disable the use_enum_values configuration
|
|
39
39
|
class DetectionTags(BaseModel):
|
|
@@ -50,8 +50,10 @@ class DetectionTags(BaseModel):
|
|
|
50
50
|
def risk_score(self) -> int:
|
|
51
51
|
return round((self.confidence * self.impact)/100)
|
|
52
52
|
|
|
53
|
-
mitre_attack_id: List[
|
|
53
|
+
mitre_attack_id: List[MITRE_ATTACK_ID_TYPE] = []
|
|
54
54
|
nist: list[NistCategory] = []
|
|
55
|
+
|
|
56
|
+
# TODO (#249): Add pydantic validator to ensure observables are unique within a detection
|
|
55
57
|
observable: List[Observable] = []
|
|
56
58
|
message: str = Field(...)
|
|
57
59
|
product: list[SecurityContentProductName] = Field(..., min_length=1)
|
|
@@ -69,7 +71,7 @@ class DetectionTags(BaseModel):
|
|
|
69
71
|
else:
|
|
70
72
|
return RiskSeverity('low')
|
|
71
73
|
|
|
72
|
-
cve: List[
|
|
74
|
+
cve: List[CVE_TYPE] = []
|
|
73
75
|
atomic_guid: List[AtomicTest] = []
|
|
74
76
|
drilldown_search: Optional[str] = None
|
|
75
77
|
|
|
@@ -3,6 +3,7 @@ from pydantic import BaseModel, Field, ConfigDict, HttpUrl, field_validator
|
|
|
3
3
|
from typing import List, Annotated
|
|
4
4
|
from enum import StrEnum
|
|
5
5
|
import datetime
|
|
6
|
+
from contentctl.objects.annotated_types import MITRE_ATTACK_ID_TYPE
|
|
6
7
|
|
|
7
8
|
class MitreTactics(StrEnum):
|
|
8
9
|
RECONNAISSANCE = "Reconnaissance"
|
|
@@ -85,7 +86,7 @@ class MitreAttackGroup(BaseModel):
|
|
|
85
86
|
# TODO (#266): disable the use_enum_values configuration
|
|
86
87
|
class MitreAttackEnrichment(BaseModel):
|
|
87
88
|
ConfigDict(use_enum_values=True)
|
|
88
|
-
mitre_attack_id:
|
|
89
|
+
mitre_attack_id: MITRE_ATTACK_ID_TYPE = Field(...)
|
|
89
90
|
mitre_attack_technique: str = Field(...)
|
|
90
91
|
mitre_attack_tactics: List[MitreTactics] = Field(...)
|
|
91
92
|
mitre_attack_groups: List[str] = Field(...)
|
contentctl/objects/risk_event.py
CHANGED
|
@@ -1,32 +1,51 @@
|
|
|
1
1
|
import re
|
|
2
|
-
from typing import Union, Optional
|
|
3
2
|
|
|
4
|
-
from pydantic import BaseModel, Field, PrivateAttr, field_validator
|
|
3
|
+
from pydantic import BaseModel, Field, PrivateAttr, field_validator, computed_field
|
|
5
4
|
|
|
6
5
|
from contentctl.objects.errors import ValidationFailed
|
|
7
6
|
from contentctl.objects.detection import Detection
|
|
8
7
|
from contentctl.objects.observable import Observable
|
|
9
8
|
|
|
10
|
-
# TODO (
|
|
9
|
+
# TODO (#259): Map our observable types to more than user/system
|
|
10
|
+
# TODO (#247): centralize this mapping w/ usage of SES_OBSERVABLE_TYPE_MAPPING (see
|
|
11
|
+
# observable.py) and the ad hoc mapping made in detection_abstract.py (see the risk property func)
|
|
11
12
|
TYPE_MAP: dict[str, list[str]] = {
|
|
12
|
-
"
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
"system": [
|
|
14
|
+
"Hostname",
|
|
15
|
+
"IP Address",
|
|
16
|
+
"Endpoint"
|
|
17
|
+
],
|
|
18
|
+
"user": [
|
|
19
|
+
"User",
|
|
20
|
+
"User Name",
|
|
21
|
+
"Email Address",
|
|
22
|
+
"Email"
|
|
23
|
+
],
|
|
24
|
+
"hash_values": [],
|
|
25
|
+
"network_artifacts": [],
|
|
26
|
+
"host_artifacts": [],
|
|
27
|
+
"tools": [],
|
|
28
|
+
"other": [
|
|
29
|
+
"Process",
|
|
30
|
+
"URL String",
|
|
31
|
+
"Unknown",
|
|
32
|
+
"Process Name",
|
|
33
|
+
"MAC Address",
|
|
34
|
+
"File Name",
|
|
35
|
+
"File Hash",
|
|
36
|
+
"Resource UID",
|
|
37
|
+
"Uniform Resource Locator",
|
|
38
|
+
"File",
|
|
39
|
+
"Geo Location",
|
|
40
|
+
"Container",
|
|
41
|
+
"Registry Key",
|
|
42
|
+
"Registry Value",
|
|
43
|
+
"Other"
|
|
44
|
+
]
|
|
15
45
|
}
|
|
16
|
-
# TODO (PEX-433): 'Email Address', 'File Name', 'File Hash', 'Other', 'User Name', 'File',
|
|
17
|
-
# 'Process Name'
|
|
18
46
|
|
|
19
|
-
#
|
|
47
|
+
# Roles that should not generate risks
|
|
20
48
|
IGNORE_ROLES: list[str] = ["Attacker"]
|
|
21
|
-
# Known valid roles: Victim, Parent Process, Child Process
|
|
22
|
-
# TODO (PEX-433): 'Other', 'Target', 'Unknown'
|
|
23
|
-
# TODO (PEX-433): is Other a valid role
|
|
24
|
-
|
|
25
|
-
# TODO (PEX-433): do we need User Name in conjunction w/ User? User Name doesn't get mapped to
|
|
26
|
-
# "user" in risk events
|
|
27
|
-
# TODO (PEX-433): similarly, do we need Process and Process Name?
|
|
28
|
-
|
|
29
|
-
RESERVED_FIELDS = ["host"]
|
|
30
49
|
|
|
31
50
|
|
|
32
51
|
class RiskEvent(BaseModel):
|
|
@@ -36,7 +55,7 @@ class RiskEvent(BaseModel):
|
|
|
36
55
|
search_name: str
|
|
37
56
|
|
|
38
57
|
# The subject of the risk event (e.g. a username, process name, system name, account ID, etc.)
|
|
39
|
-
risk_object:
|
|
58
|
+
risk_object: int | str
|
|
40
59
|
|
|
41
60
|
# The type of the risk object (e.g. user, system, or other)
|
|
42
61
|
risk_object_type: str
|
|
@@ -59,8 +78,12 @@ class RiskEvent(BaseModel):
|
|
|
59
78
|
default=[]
|
|
60
79
|
)
|
|
61
80
|
|
|
81
|
+
# Contributing events search query (we use this to derive the corresponding field from the
|
|
82
|
+
# observables)
|
|
83
|
+
contributing_events_search: str
|
|
84
|
+
|
|
62
85
|
# Private attribute caching the observable this RiskEvent is mapped to
|
|
63
|
-
_matched_observable:
|
|
86
|
+
_matched_observable: Observable | None = PrivateAttr(default=None)
|
|
64
87
|
|
|
65
88
|
class Config:
|
|
66
89
|
# Allowing fields that aren't explicitly defined to be passed since some of the risk event's
|
|
@@ -69,7 +92,7 @@ class RiskEvent(BaseModel):
|
|
|
69
92
|
|
|
70
93
|
@field_validator("annotations_mitre_attack", "analyticstories", mode="before")
|
|
71
94
|
@classmethod
|
|
72
|
-
def _convert_str_value_to_singleton(cls, v:
|
|
95
|
+
def _convert_str_value_to_singleton(cls, v: str | list[str]) -> list[str]:
|
|
73
96
|
"""
|
|
74
97
|
Given a value, determine if its a list or a single str value; if a single value, return as a
|
|
75
98
|
singleton. Do nothing if anything else.
|
|
@@ -79,6 +102,25 @@ class RiskEvent(BaseModel):
|
|
|
79
102
|
else:
|
|
80
103
|
return [v]
|
|
81
104
|
|
|
105
|
+
@computed_field
|
|
106
|
+
@property
|
|
107
|
+
def source_field_name(self) -> str:
|
|
108
|
+
"""
|
|
109
|
+
A cached derivation of the source field name the risk event corresponds to in the relevant
|
|
110
|
+
event(s). Useful for mapping back to an observable in the detection.
|
|
111
|
+
"""
|
|
112
|
+
pattern = re.compile(
|
|
113
|
+
r"\| savedsearch \"" + self.search_name + r"\" \| search (?P<field>[^=]+)=.+"
|
|
114
|
+
)
|
|
115
|
+
match = pattern.search(self.contributing_events_search)
|
|
116
|
+
if match is None:
|
|
117
|
+
raise ValueError(
|
|
118
|
+
"Unable to parse source field name from risk event using "
|
|
119
|
+
f"'contributing_events_search' ('{self.contributing_events_search}') using "
|
|
120
|
+
f"pattern: {pattern}"
|
|
121
|
+
)
|
|
122
|
+
return match.group("field")
|
|
123
|
+
|
|
82
124
|
def validate_against_detection(self, detection: Detection) -> None:
|
|
83
125
|
"""
|
|
84
126
|
Given the associated detection, validate the risk event against its fields
|
|
@@ -108,10 +150,8 @@ class RiskEvent(BaseModel):
|
|
|
108
150
|
# Check risk_message
|
|
109
151
|
self.validate_risk_message(detection)
|
|
110
152
|
|
|
111
|
-
# TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the false
|
|
112
|
-
# positive rate in risk/obseravble matching
|
|
113
153
|
# Check several conditions against the observables
|
|
114
|
-
|
|
154
|
+
self.validate_risk_against_observables(detection.tags.observable)
|
|
115
155
|
|
|
116
156
|
def validate_mitre_ids(self, detection: Detection) -> None:
|
|
117
157
|
"""
|
|
@@ -199,7 +239,11 @@ class RiskEvent(BaseModel):
|
|
|
199
239
|
if self.risk_object_type != expected_type:
|
|
200
240
|
raise ValidationFailed(
|
|
201
241
|
f"The risk object type ({self.risk_object_type}) does not match the expected type "
|
|
202
|
-
f"based on the matched observable ({matched_observable.type}
|
|
242
|
+
f"based on the matched observable ({matched_observable.type}->{expected_type}): "
|
|
243
|
+
f"risk=(object={self.risk_object}, type={self.risk_object_type}, "
|
|
244
|
+
f"source_field_name={self.source_field_name}), "
|
|
245
|
+
f"observable=(name={matched_observable.name}, type={matched_observable.type}, "
|
|
246
|
+
f"role={matched_observable.role})"
|
|
203
247
|
)
|
|
204
248
|
|
|
205
249
|
@staticmethod
|
|
@@ -220,8 +264,6 @@ class RiskEvent(BaseModel):
|
|
|
220
264
|
f"Observable type {observable_type} does not have a mapping to a risk type in TYPE_MAP"
|
|
221
265
|
)
|
|
222
266
|
|
|
223
|
-
# TODO (PEX-433): should this be an observable instance method? It feels less relevant to
|
|
224
|
-
# observables themselves, as it's really only relevant to the handling of risk events
|
|
225
267
|
@staticmethod
|
|
226
268
|
def ignore_observable(observable: Observable) -> bool:
|
|
227
269
|
"""
|
|
@@ -230,8 +272,6 @@ class RiskEvent(BaseModel):
|
|
|
230
272
|
:param observable: the Observable object we are checking the roles of
|
|
231
273
|
:returns: a bool indicating whether this observable should be ignored or not
|
|
232
274
|
"""
|
|
233
|
-
# TODO (PEX-433): could there be a case where an observable has both an Attacker and Victim
|
|
234
|
-
# (or equivalent) role? If so, how should we handle ignoring it?
|
|
235
275
|
ignore = False
|
|
236
276
|
for role in observable.role:
|
|
237
277
|
if role in IGNORE_ROLES:
|
|
@@ -239,29 +279,6 @@ class RiskEvent(BaseModel):
|
|
|
239
279
|
break
|
|
240
280
|
return ignore
|
|
241
281
|
|
|
242
|
-
# TODO (PEX-433): two possibilities: alway check for the field itself and the field prefixed
|
|
243
|
-
# w/ "orig_" OR more explicitly maintain a list of known "reserved fields", like "host". I
|
|
244
|
-
# think I like option 2 better as it can have fewer unknown side effects
|
|
245
|
-
def matches_observable(self, observable: Observable) -> bool:
|
|
246
|
-
"""
|
|
247
|
-
Given an observable, check if the risk event matches is
|
|
248
|
-
:param observable: the Observable object we are comparing the risk event against
|
|
249
|
-
:returns: bool indicating a match or not
|
|
250
|
-
"""
|
|
251
|
-
# When field names collide w/ reserved fields in Splunk events (e.g. sourcetype or host)
|
|
252
|
-
# they get prefixed w/ "orig_"
|
|
253
|
-
attribute_name = observable.name
|
|
254
|
-
if attribute_name in RESERVED_FIELDS:
|
|
255
|
-
attribute_name = f"orig_{attribute_name}"
|
|
256
|
-
|
|
257
|
-
# Retrieve the value of this attribute and see if it matches the risk_object
|
|
258
|
-
value: Union[str, list[str]] = getattr(self, attribute_name)
|
|
259
|
-
if isinstance(value, str):
|
|
260
|
-
value = [value]
|
|
261
|
-
|
|
262
|
-
# The value of the attribute may be a list of values, so check for any matches
|
|
263
|
-
return self.risk_object in value
|
|
264
|
-
|
|
265
282
|
def get_matched_observable(self, observables: list[Observable]) -> Observable:
|
|
266
283
|
"""
|
|
267
284
|
Given a list of observables, return the one this risk event matches
|
|
@@ -274,40 +291,41 @@ class RiskEvent(BaseModel):
|
|
|
274
291
|
if self._matched_observable is not None:
|
|
275
292
|
return self._matched_observable
|
|
276
293
|
|
|
277
|
-
matched_observable:
|
|
294
|
+
matched_observable: Observable | None = None
|
|
278
295
|
|
|
279
296
|
# Iterate over the obervables and check for a match
|
|
280
297
|
for observable in observables:
|
|
298
|
+
# TODO (#252): Refactor and re-enable per-field validation of risk events
|
|
281
299
|
# Each the field name used in each observable shoud be present in the risk event
|
|
282
|
-
#
|
|
283
|
-
#
|
|
284
|
-
#
|
|
285
|
-
#
|
|
286
|
-
if not hasattr(self, observable.name):
|
|
287
|
-
raise ValidationFailed(
|
|
288
|
-
f"Observable field \"{observable.name}\" not found in risk event."
|
|
289
|
-
)
|
|
300
|
+
# if not hasattr(self, observable.name):
|
|
301
|
+
# raise ValidationFailed(
|
|
302
|
+
# f"Observable field \"{observable.name}\" not found in risk event."
|
|
303
|
+
# )
|
|
290
304
|
|
|
291
305
|
# Try to match the risk_object against a specific observable for the obervables with
|
|
292
|
-
# a valid role (some, like Attacker,
|
|
293
|
-
if
|
|
294
|
-
if
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
306
|
+
# a valid role (some, like Attacker, shouldn't get converted to risk events)
|
|
307
|
+
if self.source_field_name == observable.name:
|
|
308
|
+
if matched_observable is not None:
|
|
309
|
+
raise ValueError(
|
|
310
|
+
"Unexpected conditon: we don't expect the source event field "
|
|
311
|
+
"corresponding to an observables field name to be repeated."
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# Report any risk events we find that shouldn't be there
|
|
315
|
+
if RiskEvent.ignore_observable(observable):
|
|
316
|
+
raise ValidationFailed(
|
|
317
|
+
"Risk event matched an observable with an invalid role: "
|
|
318
|
+
f"(name={observable.name}, type={observable.type}, role={observable.role})")
|
|
319
|
+
# NOTE: we explicitly do not break early as we want to check each observable
|
|
320
|
+
matched_observable = observable
|
|
305
321
|
|
|
306
322
|
# Ensure we were able to match the risk event to a specific observable
|
|
307
323
|
if matched_observable is None:
|
|
308
324
|
raise ValidationFailed(
|
|
309
|
-
f"Unable to match risk event ({self.risk_object},
|
|
310
|
-
"
|
|
325
|
+
f"Unable to match risk event (object={self.risk_object}, type="
|
|
326
|
+
f"{self.risk_object_type}, source_field_name={self.source_field_name}) to an "
|
|
327
|
+
"observable; please check for errors in the observable roles/types for this "
|
|
328
|
+
"detection, as well as the risk event build process in contentctl."
|
|
311
329
|
)
|
|
312
330
|
|
|
313
331
|
# Cache and return the matched observable
|
contentctl/objects/story_tags.py
CHANGED
|
@@ -6,7 +6,7 @@ from enum import Enum
|
|
|
6
6
|
|
|
7
7
|
from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
|
|
8
8
|
from contentctl.objects.enums import StoryCategory, DataModel, KillChainPhase, SecurityContentProductName
|
|
9
|
-
|
|
9
|
+
from contentctl.objects.annotated_types import CVE_TYPE,MITRE_ATTACK_ID_TYPE
|
|
10
10
|
|
|
11
11
|
class StoryUseCase(str,Enum):
|
|
12
12
|
FRAUD_DETECTION = "Fraud Detection"
|
|
@@ -27,10 +27,10 @@ class StoryTags(BaseModel):
|
|
|
27
27
|
|
|
28
28
|
# enrichment
|
|
29
29
|
mitre_attack_enrichments: Optional[List[MitreAttackEnrichment]] = None
|
|
30
|
-
mitre_attack_tactics: Optional[Set[
|
|
30
|
+
mitre_attack_tactics: Optional[Set[MITRE_ATTACK_ID_TYPE]] = None
|
|
31
31
|
datamodels: Optional[Set[DataModel]] = None
|
|
32
32
|
kill_chain_phases: Optional[Set[KillChainPhase]] = None
|
|
33
|
-
cve: List[
|
|
33
|
+
cve: List[CVE_TYPE] = []
|
|
34
34
|
group: List[str] = Field([], description="A list of groups who leverage the techniques list in this Analytic Story.")
|
|
35
35
|
|
|
36
36
|
def getCategory_conf(self) -> str:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: contentctl
|
|
3
|
-
Version: 4.3.
|
|
3
|
+
Version: 4.3.4
|
|
4
4
|
Summary: Splunk Content Control Tool
|
|
5
5
|
License: Apache 2.0
|
|
6
6
|
Author: STRT
|
|
@@ -22,7 +22,7 @@ Requires-Dist: pygit2 (>=1.15.1,<2.0.0)
|
|
|
22
22
|
Requires-Dist: questionary (>=2.0.1,<3.0.0)
|
|
23
23
|
Requires-Dist: requests (>=2.32.3,<2.33.0)
|
|
24
24
|
Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
|
|
25
|
-
Requires-Dist: setuptools (>=69.5.1,<
|
|
25
|
+
Requires-Dist: setuptools (>=69.5.1,<75.0.0)
|
|
26
26
|
Requires-Dist: splunk-sdk (>=2.0.2,<3.0.0)
|
|
27
27
|
Requires-Dist: tqdm (>=4.66.5,<5.0.0)
|
|
28
28
|
Requires-Dist: tyro (>=0.8.3,<0.9.0)
|
|
@@ -4,7 +4,7 @@ contentctl/actions/deploy_acs.py,sha256=mf3uk495H1EU_LNN-TiOsYCo18HMGoEBMb6ojeTr
|
|
|
4
4
|
contentctl/actions/detection_testing/DetectionTestingManager.py,sha256=zg8JasDjCpSC-yhseEyUwO8qbDJIUJbhlus9Li9ZAnA,8818
|
|
5
5
|
contentctl/actions/detection_testing/GitService.py,sha256=W1vnDDt8JvIL7Z1Lve3D3RS7h8qwMxrW0BMXVGuDZDM,9007
|
|
6
6
|
contentctl/actions/detection_testing/generate_detection_coverage_badge.py,sha256=N5mznaeErVak3mOBwsd0RDBFJO3bku0EZvpayCyU-uk,2259
|
|
7
|
-
contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=
|
|
7
|
+
contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py,sha256=00ymK5PyAn_FREi8Cj0HqpUt-U6XMpSHrN0QNqIrbDA,55190
|
|
8
8
|
contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py,sha256=REM3WB-DQAczeknGAKMzJhnvHgnt-u9yDG2UKGVj2vM,6854
|
|
9
9
|
contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureServer.py,sha256=Q1ZfCYOp54O39bgTScZMInkmZiU-bGAM9Hiwr2mq5ms,370
|
|
10
10
|
contentctl/actions/detection_testing/progress_bar.py,sha256=OK9oRnPlzPAswt9KZNYID-YLHxqaYPY821kIE4-rCeA,3244
|
|
@@ -23,8 +23,8 @@ contentctl/actions/test.py,sha256=jv12UO_PTjZwvo4G-Dr8fE2gsuWvuvAmO2QQM4q7TL0,59
|
|
|
23
23
|
contentctl/actions/validate.py,sha256=2MQ8yumCKj7zD8iUuA5gfFEMcE-GPRzYqkvuOexn0JA,5633
|
|
24
24
|
contentctl/api.py,sha256=FBOpRhbBCBdjORmwe_8MPQ3PRZ6T0KrrFcfKovVFkug,6343
|
|
25
25
|
contentctl/contentctl.py,sha256=JXbUD5l1PziRRJxUc1UHrveM33CHryZPmc0RxudDpIs,10328
|
|
26
|
-
contentctl/enrichments/attack_enrichment.py,sha256=
|
|
27
|
-
contentctl/enrichments/cve_enrichment.py,sha256=
|
|
26
|
+
contentctl/enrichments/attack_enrichment.py,sha256=XEcLRnXKfJeChax5gfHDGea5D5MCFjP4bWp8hRWn3d8,7871
|
|
27
|
+
contentctl/enrichments/cve_enrichment.py,sha256=rRdf62sKkBzCBLCNwzAmEhxNiPV2px1VS6MzDiS-uBw,2337
|
|
28
28
|
contentctl/enrichments/splunk_app_enrichment.py,sha256=zDNHFLZTi2dJ1gdnh0sHkD6F1VtkblqFnhacFcCMBfc,3418
|
|
29
29
|
contentctl/helper/link_validator.py,sha256=-XorhxfGtjLynEL1X4hcpRMiyemogf2JEnvLwhHq80c,7139
|
|
30
30
|
contentctl/helper/logger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -33,17 +33,18 @@ contentctl/helper/utils.py,sha256=8ICRvE7DUiNL9BK4Hw71hCLFbd3R2u86OwKeDOdaBTY,19
|
|
|
33
33
|
contentctl/input/director.py,sha256=kTqdN_rCzRMn4dR32hPaVyx2llhAxyhJgoGjowhsHzs,10887
|
|
34
34
|
contentctl/input/new_content_questions.py,sha256=o4prlBoUhEMxqpZukquI9WKbzfFJfYhEF7a8m2q_BEE,5565
|
|
35
35
|
contentctl/input/yml_reader.py,sha256=hyVUYhx4Ka8C618kP2D_E3sDUKEQGC6ty_QZQArHKd4,1489
|
|
36
|
-
contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=
|
|
36
|
+
contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=U3IvEQO3D5ab7YPUz8JnAnUCNtN--INOs2AP-ew5qn8,38867
|
|
37
37
|
contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py,sha256=vdZvybF34Zlxf6XOjw400gYbpkPUkOtlu-JiWlAof40,9877
|
|
38
38
|
contentctl/objects/alert_action.py,sha256=E9gjCn5C31h0sN7k90KNe4agRxFFSnMW_Z-Ri_3YQss,1335
|
|
39
|
+
contentctl/objects/annotated_types.py,sha256=jnX02BQT4dHbd_DCIjik0PNN3kgsvb7sxAz_1Jy8TOY,259
|
|
39
40
|
contentctl/objects/atomic.py,sha256=BP27gP8KHeODp6UazhVFxwDQ64wuJCARGsLfIH34h7U,8768
|
|
40
41
|
contentctl/objects/base_test.py,sha256=qUtKQJrqCto_fwCBdiH68_tXqokhcv9ceu2fQlBxsjA,1045
|
|
41
42
|
contentctl/objects/base_test_result.py,sha256=jVroyGLb9GD6Wm2QzvgIEA3SWCZqxPsHp9PzxSvpyIs,5101
|
|
42
43
|
contentctl/objects/baseline.py,sha256=Lb1vJKtDdlDrzWgrdkC9oQao_TnRrOxSwOWHf4trtaU,2150
|
|
43
44
|
contentctl/objects/baseline_tags.py,sha256=fVhLF-NmisavybB_idu3N0Con0Ymj8clKfRMkWzBB-k,1762
|
|
44
|
-
contentctl/objects/config.py,sha256=
|
|
45
|
+
contentctl/objects/config.py,sha256=_DRRMdtDKxjg2u-7iEbBrvKwtABxtlrmAEC8XYBQGk8,44487
|
|
45
46
|
contentctl/objects/constants.py,sha256=lfCcr1DsTZvANHj4Ee1_sEV-SebHwAn41-5EvmoEX2E,3537
|
|
46
|
-
contentctl/objects/correlation_search.py,sha256=
|
|
47
|
+
contentctl/objects/correlation_search.py,sha256=ZZVoO3M594qCy_aAMhQiOPWn8FiSFbRShUCCLx6zhNc,48434
|
|
47
48
|
contentctl/objects/data_source.py,sha256=aRr6lHu-EtGmi6J2nXKD7i2ozUPtp7X-vDkQiutvD3I,1545
|
|
48
49
|
contentctl/objects/deployment.py,sha256=Qc6M4yeOvxjqFKR8sfjd4CG06AbVheTOqP1mwqo4t8s,2651
|
|
49
50
|
contentctl/objects/deployment_email.py,sha256=Zu9cXZdfOP6noa_mZpiK1GrYCTgi3Mim94iLGjE674c,147
|
|
@@ -53,7 +54,7 @@ contentctl/objects/deployment_rba.py,sha256=YFLSKzLU7s8Bt1cJkSBWlfCsc_2MfgiwyaDi
|
|
|
53
54
|
contentctl/objects/deployment_scheduling.py,sha256=bQjbJHNaUGdU1VAGV8-nFOHzHutbIlt7FZpUvR1CV4Y,198
|
|
54
55
|
contentctl/objects/deployment_slack.py,sha256=P6z8OLHDKcDWx7nbKWasqBc3dFRatGcpO2GtmxzVV8I,135
|
|
55
56
|
contentctl/objects/detection.py,sha256=3W41cXf3ECjWuPqWrseqSLC3PAA7O5_nENWWM6MPK0Y,620
|
|
56
|
-
contentctl/objects/detection_tags.py,sha256=
|
|
57
|
+
contentctl/objects/detection_tags.py,sha256=r7nIYMMspPk68aQx5q04jQaFGO4zTYG1P1UAUrX9qtU,11023
|
|
57
58
|
contentctl/objects/enums.py,sha256=37v7w8xCg5j5hxP3kod0S3HQ9BY-CqZulPiwhnTtEvs,14052
|
|
58
59
|
contentctl/objects/errors.py,sha256=gnD99z4O00EBbMerUjt4368q8mohm3Zb9HByG3CP_A0,525
|
|
59
60
|
contentctl/objects/event_source.py,sha256=G9P7rtcN5hcBNQx6DG37mR3QyQufx--T6kgQGNqQuKk,415
|
|
@@ -65,20 +66,20 @@ contentctl/objects/lookup.py,sha256=oZwBiHfRRrv2ZXdGyWIJWSWZMpuUbsXydaDDfpenk-4,
|
|
|
65
66
|
contentctl/objects/macro.py,sha256=9nE-bxkFhtaltHOUCr0luU8jCCthmglHjhKs6Q2YzLU,2684
|
|
66
67
|
contentctl/objects/manual_test.py,sha256=YNquEQ0UCzZGJ0uvHBgJ3Efho-F80ZG885ABLtqB7TI,1022
|
|
67
68
|
contentctl/objects/manual_test_result.py,sha256=C4AYW3jlMsxVzCPzCA5dpAcbKgCpmDO43JmptFm--Q4,155
|
|
68
|
-
contentctl/objects/mitre_attack_enrichment.py,sha256=
|
|
69
|
+
contentctl/objects/mitre_attack_enrichment.py,sha256=4_9hvrxCXnGfyWqoj7C-0pCfGXEBJXfhrcSfb1cmPjs,3387
|
|
69
70
|
contentctl/objects/notable_action.py,sha256=ValkblBaG-60TF19y_vSnNzoNZ3eg48wIfr0qZxyKTA,1605
|
|
70
71
|
contentctl/objects/notable_event.py,sha256=ITcwLzeatSGpe8267PYN-EhgqOSoWTfciCBVu8zjOXE,682
|
|
71
72
|
contentctl/objects/observable.py,sha256=pw0Ehi_KMb7nXzw2kuw1FnCknpD8zDkCAqBTa-M_F28,1313
|
|
72
73
|
contentctl/objects/playbook.py,sha256=hSYYpdMhctgpp7uwaPciFqu1yuFI4M1NHy1WBBLyvzM,2469
|
|
73
74
|
contentctl/objects/playbook_tags.py,sha256=NrhTGcgoYSGEZggrfebko0GBOXN9x05IadRUUL_CVfQ,1436
|
|
74
75
|
contentctl/objects/risk_analysis_action.py,sha256=Glzcq99DAqqOJ2eZYCkUI3R5hA5cZGU0ZuCSinFf2R8,4278
|
|
75
|
-
contentctl/objects/risk_event.py,sha256=
|
|
76
|
+
contentctl/objects/risk_event.py,sha256=b5Smh3w5Hecmi7E-Ub5DvO8iOPwnVg2ux47u7oemxX4,14041
|
|
76
77
|
contentctl/objects/risk_object.py,sha256=yY4NmEwEKaRl4sLzCRZb1n8kdpV3HzYbQVQ1ClQWYHw,904
|
|
77
78
|
contentctl/objects/security_content_object.py,sha256=j8KNDwSMfZsSIzJucC3NuZo0SlFVpqHfDc6y3-YHjHI,234
|
|
78
79
|
contentctl/objects/ssa_detection.py,sha256=ud0T6lq-5XUlmeK8Jzw_aNLe6podVcA1o7THDYvWbik,5934
|
|
79
80
|
contentctl/objects/ssa_detection_tags.py,sha256=9aRwbpQHi79NIS9rofjgxDJpw7cWXqG534_kSbvHJh8,5220
|
|
80
81
|
contentctl/objects/story.py,sha256=FXe11LV19xJTtCgx7DKdvV9cL0gKeryUnE3yjpnDmrU,4957
|
|
81
|
-
contentctl/objects/story_tags.py,sha256=
|
|
82
|
+
contentctl/objects/story_tags.py,sha256=cOL8PUzdlFdLPQHc54_-9sdI8nCE1D04oKY7KriOssI,2293
|
|
82
83
|
contentctl/objects/test_attack_data.py,sha256=9OgErjdPR4S-SJpQePt0uwBLPYHYPtqKDd-auhjz7Uc,430
|
|
83
84
|
contentctl/objects/test_group.py,sha256=DCtm4ChGYksOwZQVHsioaweOvI37CSlTZJzKvBX-jbY,2586
|
|
84
85
|
contentctl/objects/threat_object.py,sha256=S8B7RQFfLxN_g7yKPrDTuYhIy9JvQH3YwJ_T5LUZIa4,711
|
|
@@ -165,8 +166,8 @@ contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
|
|
|
165
166
|
contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
|
|
166
167
|
contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
|
|
167
168
|
contentctl/templates/stories/cobalt_strike.yml,sha256=rlaXxMN-5k8LnKBLPafBoksyMtlmsPMHPJOjTiMiZ-M,3063
|
|
168
|
-
contentctl-4.3.
|
|
169
|
-
contentctl-4.3.
|
|
170
|
-
contentctl-4.3.
|
|
171
|
-
contentctl-4.3.
|
|
172
|
-
contentctl-4.3.
|
|
169
|
+
contentctl-4.3.4.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
|
|
170
|
+
contentctl-4.3.4.dist-info/METADATA,sha256=YgRlkSBe1UQmgQfU3wIVwH0lufqLvfhjnnhY2qBNxiU,20925
|
|
171
|
+
contentctl-4.3.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
172
|
+
contentctl-4.3.4.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
|
|
173
|
+
contentctl-4.3.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|