contentctl 4.3.3__py3-none-any.whl → 4.3.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +0 -6
- contentctl/actions/initialize.py +28 -12
- contentctl/actions/inspect.py +189 -91
- contentctl/actions/validate.py +3 -7
- contentctl/api.py +1 -1
- contentctl/contentctl.py +3 -0
- contentctl/enrichments/attack_enrichment.py +51 -82
- contentctl/enrichments/cve_enrichment.py +2 -2
- contentctl/helper/splunk_app.py +141 -10
- contentctl/input/director.py +5 -12
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +11 -8
- contentctl/objects/annotated_types.py +6 -0
- contentctl/objects/atomic.py +51 -77
- contentctl/objects/config.py +145 -22
- contentctl/objects/constants.py +4 -1
- contentctl/objects/correlation_search.py +35 -28
- contentctl/objects/detection_metadata.py +71 -0
- contentctl/objects/detection_stanza.py +79 -0
- contentctl/objects/detection_tags.py +11 -9
- contentctl/objects/enums.py +0 -2
- contentctl/objects/errors.py +187 -0
- contentctl/objects/mitre_attack_enrichment.py +2 -1
- contentctl/objects/risk_event.py +94 -76
- contentctl/objects/savedsearches_conf.py +196 -0
- contentctl/objects/story_tags.py +3 -3
- contentctl/output/conf_writer.py +4 -1
- contentctl/output/new_content_yml_output.py +4 -9
- {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/METADATA +4 -4
- {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/RECORD +32 -32
- contentctl/objects/ssa_detection.py +0 -157
- contentctl/objects/ssa_detection_tags.py +0 -138
- contentctl/objects/unit_test_old.py +0 -10
- contentctl/objects/unit_test_ssa.py +0 -31
- {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/LICENSE.md +0 -0
- {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/WHEEL +0 -0
- {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/entry_points.txt +0 -0
contentctl/objects/atomic.py
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
if TYPE_CHECKING:
|
|
4
|
+
from contentctl.objects.config import validate
|
|
5
|
+
|
|
2
6
|
from contentctl.input.yml_reader import YmlReader
|
|
3
7
|
from pydantic import BaseModel, model_validator, ConfigDict, FilePath, UUID4
|
|
8
|
+
import dataclasses
|
|
4
9
|
from typing import List, Optional, Dict, Union, Self
|
|
5
10
|
import pathlib
|
|
6
|
-
|
|
7
|
-
|
|
8
11
|
from enum import StrEnum, auto
|
|
9
|
-
|
|
12
|
+
import uuid
|
|
10
13
|
|
|
11
14
|
class SupportedPlatform(StrEnum):
|
|
12
15
|
windows = auto()
|
|
@@ -84,15 +87,6 @@ class AtomicTest(BaseModel):
|
|
|
84
87
|
dependencies: Optional[List[AtomicDependency]] = None
|
|
85
88
|
dependency_executor_name: Optional[DependencyExecutorType] = None
|
|
86
89
|
|
|
87
|
-
@staticmethod
|
|
88
|
-
def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4) -> AtomicTest:
|
|
89
|
-
return AtomicTest(name="Placeholder Atomic Test (enrichment disabled)",
|
|
90
|
-
auto_generated_guid=auto_generated_guid,
|
|
91
|
-
description="This is a placeholder AtomicTest. Because enrichments were not enabled, it has not been validated against the real Atomic Red Team Repo.",
|
|
92
|
-
supported_platforms=[],
|
|
93
|
-
executor=AtomicExecutor(name="Placeholder Executor (enrichment disabled)",
|
|
94
|
-
command="Placeholder command (enrichment disabled)"))
|
|
95
|
-
|
|
96
90
|
@staticmethod
|
|
97
91
|
def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4) -> AtomicTest:
|
|
98
92
|
return AtomicTest(name="Missing Atomic",
|
|
@@ -100,31 +94,16 @@ class AtomicTest(BaseModel):
|
|
|
100
94
|
description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile.",
|
|
101
95
|
supported_platforms=[],
|
|
102
96
|
executor=AtomicExecutor(name="Placeholder Executor (failed to find auto_generated_guid)",
|
|
103
|
-
command="Placeholder command (failed to find auto_generated_guid)"))
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
@classmethod
|
|
107
|
-
def getAtomicByAtomicGuid(cls, guid: UUID4, all_atomics:list[AtomicTest] | None)->AtomicTest:
|
|
108
|
-
if all_atomics is None:
|
|
109
|
-
return AtomicTest.AtomicTestWhenEnrichmentIsDisabled(guid)
|
|
110
|
-
matching_atomics = [atomic for atomic in all_atomics if atomic.auto_generated_guid == guid]
|
|
111
|
-
if len(matching_atomics) == 0:
|
|
112
|
-
raise ValueError(f"Unable to find atomic_guid {guid} in {len(all_atomics)} atomic_tests from ART Repo")
|
|
113
|
-
elif len(matching_atomics) > 1:
|
|
114
|
-
raise ValueError(f"Found {len(matching_atomics)} matching tests for atomic_guid {guid} in {len(all_atomics)} atomic_tests from ART Repo")
|
|
115
|
-
|
|
116
|
-
return matching_atomics[0]
|
|
97
|
+
command="Placeholder command (failed to find auto_generated_guid)"))
|
|
117
98
|
|
|
118
99
|
@classmethod
|
|
119
|
-
def parseArtRepo(cls, repo_path:pathlib.Path)->
|
|
120
|
-
|
|
121
|
-
print(f"WARNING: Atomic Red Team repo does NOT exist at {repo_path.absolute()}. You can check it out with:\n * git clone --single-branch https://github.com/redcanaryco/atomic-red-team. This will ONLY throw a validation error if you reference atomid_guids in your detection(s).")
|
|
122
|
-
return []
|
|
100
|
+
def parseArtRepo(cls, repo_path:pathlib.Path)->dict[uuid.UUID, AtomicTest]:
|
|
101
|
+
test_mapping: dict[uuid.UUID, AtomicTest] = {}
|
|
123
102
|
atomics_path = repo_path/"atomics"
|
|
124
103
|
if not atomics_path.is_dir():
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
104
|
+
raise FileNotFoundError(f"WARNING: Atomic Red Team repo exists at {repo_path}, "
|
|
105
|
+
f"but atomics directory does NOT exist at {atomics_path}. "
|
|
106
|
+
"Was it deleted or renamed?")
|
|
128
107
|
|
|
129
108
|
atomic_files:List[AtomicFile] = []
|
|
130
109
|
error_messages:List[str] = []
|
|
@@ -133,6 +112,7 @@ class AtomicTest(BaseModel):
|
|
|
133
112
|
atomic_files.append(cls.constructAtomicFile(obj_path))
|
|
134
113
|
except Exception as e:
|
|
135
114
|
error_messages.append(f"File [{obj_path}]\n{str(e)}")
|
|
115
|
+
|
|
136
116
|
if len(error_messages) > 0:
|
|
137
117
|
exceptions_string = '\n\n'.join(error_messages)
|
|
138
118
|
print(f"WARNING: The following [{len(error_messages)}] ERRORS were generated when parsing the Atomic Red Team Repo.\n"
|
|
@@ -140,38 +120,28 @@ class AtomicTest(BaseModel):
|
|
|
140
120
|
"Note that this is only a warning and contentctl will ignore Atomics contained in these files.\n"
|
|
141
121
|
f"However, if you have written a detection that references them, 'contentctl build --enrichments' will fail:\n\n{exceptions_string}")
|
|
142
122
|
|
|
143
|
-
return
|
|
123
|
+
# Now iterate over all the files, collect all the tests, and return the dict mapping
|
|
124
|
+
redefined_guids:set[uuid.UUID] = set()
|
|
125
|
+
for atomic_file in atomic_files:
|
|
126
|
+
for atomic_test in atomic_file.atomic_tests:
|
|
127
|
+
if atomic_test.auto_generated_guid in test_mapping:
|
|
128
|
+
redefined_guids.add(atomic_test.auto_generated_guid)
|
|
129
|
+
else:
|
|
130
|
+
test_mapping[atomic_test.auto_generated_guid] = atomic_test
|
|
131
|
+
if len(redefined_guids) > 0:
|
|
132
|
+
guids_string = '\n\t'.join([str(guid) for guid in redefined_guids])
|
|
133
|
+
raise Exception(f"The following [{len(redefined_guids)}] Atomic Test"
|
|
134
|
+
" auto_generated_guid(s) were defined more than once. "
|
|
135
|
+
f"auto_generated_guids MUST be unique:\n\t{guids_string}")
|
|
136
|
+
|
|
137
|
+
print(f"Successfully parsed [{len(test_mapping)}] Atomic Red Team Tests!")
|
|
138
|
+
return test_mapping
|
|
144
139
|
|
|
145
140
|
@classmethod
|
|
146
141
|
def constructAtomicFile(cls, file_path:pathlib.Path)->AtomicFile:
|
|
147
142
|
yml_dict = YmlReader.load_file(file_path)
|
|
148
143
|
atomic_file = AtomicFile.model_validate(yml_dict)
|
|
149
144
|
return atomic_file
|
|
150
|
-
|
|
151
|
-
@classmethod
|
|
152
|
-
def getAtomicTestsFromArtRepo(cls, repo_path:pathlib.Path, enabled:bool=True)->list[AtomicTest] | None:
|
|
153
|
-
# Get all the atomic files. Note that if the ART repo is not found, we will not throw an error,
|
|
154
|
-
# but will not have any atomics. This means that if atomic_guids are referenced during validation,
|
|
155
|
-
# validation for those detections will fail
|
|
156
|
-
if not enabled:
|
|
157
|
-
return None
|
|
158
|
-
|
|
159
|
-
atomic_files = cls.getAtomicFilesFromArtRepo(repo_path)
|
|
160
|
-
|
|
161
|
-
atomic_tests:List[AtomicTest] = []
|
|
162
|
-
for atomic_file in atomic_files:
|
|
163
|
-
atomic_tests.extend(atomic_file.atomic_tests)
|
|
164
|
-
print(f"Found [{len(atomic_tests)}] Atomic Simulations in the Atomic Red Team Repo!")
|
|
165
|
-
return atomic_tests
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
@classmethod
|
|
169
|
-
def getAtomicFilesFromArtRepo(cls, repo_path:pathlib.Path)->List[AtomicFile]:
|
|
170
|
-
return cls.parseArtRepo(repo_path)
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
145
|
|
|
176
146
|
|
|
177
147
|
class AtomicFile(BaseModel):
|
|
@@ -182,27 +152,31 @@ class AtomicFile(BaseModel):
|
|
|
182
152
|
atomic_tests: List[AtomicTest]
|
|
183
153
|
|
|
184
154
|
|
|
155
|
+
class AtomicEnrichment(BaseModel):
|
|
156
|
+
data: dict[uuid.UUID,AtomicTest] = dataclasses.field(default_factory = dict)
|
|
157
|
+
use_enrichment: bool = False
|
|
185
158
|
|
|
159
|
+
@classmethod
|
|
160
|
+
def getAtomicEnrichment(cls, config:validate)->AtomicEnrichment:
|
|
161
|
+
enrichment = AtomicEnrichment(use_enrichment=config.enrichments)
|
|
162
|
+
if config.enrichments:
|
|
163
|
+
enrichment.data = AtomicTest.parseArtRepo(config.atomic_red_team_repo_path)
|
|
164
|
+
|
|
165
|
+
return enrichment
|
|
166
|
+
|
|
167
|
+
def getAtomic(self, atomic_guid: uuid.UUID)->AtomicTest:
|
|
168
|
+
if self.use_enrichment:
|
|
169
|
+
if atomic_guid in self.data:
|
|
170
|
+
return self.data[atomic_guid]
|
|
171
|
+
else:
|
|
172
|
+
raise Exception(f"Atomic with GUID {atomic_guid} not found.")
|
|
173
|
+
else:
|
|
174
|
+
# If enrichment is not enabled, for the sake of compatability
|
|
175
|
+
# return a stub test with no useful or meaningful information.
|
|
176
|
+
return AtomicTest.AtomicTestWhenTestIsMissing(atomic_guid)
|
|
186
177
|
|
|
187
|
-
|
|
188
|
-
# atomic_objects = []
|
|
189
|
-
# atomic_simulations = []
|
|
190
|
-
# for obj_path in ATOMICS_PATH.glob("**/T*.yaml"):
|
|
191
|
-
# try:
|
|
192
|
-
# with open(obj_path, 'r', encoding="utf-8") as obj_handle:
|
|
193
|
-
# obj_data = yaml.load(obj_handle, Loader=yaml.CSafeLoader)
|
|
194
|
-
# atomic_obj = AtomicFile.model_validate(obj_data)
|
|
195
|
-
# except Exception as e:
|
|
196
|
-
# print(f"Error parsing object at path {obj_path}: {str(e)}")
|
|
197
|
-
# print(f"We have successfully parsed {len(atomic_objects)}, however!")
|
|
198
|
-
# sys.exit(1)
|
|
199
|
-
|
|
200
|
-
# print(f"Successfully parsed {obj_path}!")
|
|
201
|
-
# atomic_objects.append(atomic_obj)
|
|
202
|
-
# atomic_simulations += atomic_obj.atomic_tests
|
|
178
|
+
|
|
203
179
|
|
|
204
|
-
# print(f"Successfully parsed all {len(atomic_objects)} files!")
|
|
205
|
-
# print(f"Successfully parsed all {len(atomic_simulations)} simulations!")
|
|
206
180
|
|
|
207
181
|
|
|
208
182
|
|
contentctl/objects/config.py
CHANGED
|
@@ -1,26 +1,31 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from os import environ
|
|
4
|
+
from datetime import datetime, UTC
|
|
5
|
+
from typing import Optional, Any, List, Union, Self
|
|
6
|
+
import random
|
|
7
|
+
from enum import StrEnum, auto
|
|
8
|
+
import pathlib
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from functools import partialmethod
|
|
12
|
+
|
|
13
|
+
import tqdm
|
|
14
|
+
import semantic_version
|
|
2
15
|
from pydantic import (
|
|
3
16
|
BaseModel, Field, field_validator,
|
|
4
17
|
field_serializer, ConfigDict, DirectoryPath,
|
|
5
18
|
PositiveInt, FilePath, HttpUrl, AnyUrl, model_validator,
|
|
6
19
|
ValidationInfo
|
|
7
20
|
)
|
|
21
|
+
|
|
22
|
+
from contentctl.objects.constants import DOWNLOADS_DIRECTORY
|
|
8
23
|
from contentctl.output.yml_writer import YmlWriter
|
|
9
|
-
from os import environ
|
|
10
|
-
from datetime import datetime, UTC
|
|
11
|
-
from typing import Optional,Any,Annotated,List,Union, Self
|
|
12
|
-
import semantic_version
|
|
13
|
-
import random
|
|
14
|
-
from enum import StrEnum, auto
|
|
15
|
-
import pathlib
|
|
16
24
|
from contentctl.helper.utils import Utils
|
|
17
|
-
from urllib.parse import urlparse
|
|
18
|
-
from abc import ABC, abstractmethod
|
|
19
25
|
from contentctl.objects.enums import PostTestBehavior, DetectionTestingMode
|
|
20
26
|
from contentctl.objects.detection import Detection
|
|
21
|
-
|
|
22
|
-
import
|
|
23
|
-
from functools import partialmethod
|
|
27
|
+
from contentctl.objects.annotated_types import APPID_TYPE
|
|
28
|
+
from contentctl.helper.splunk_app import SplunkApp
|
|
24
29
|
|
|
25
30
|
ENTERPRISE_SECURITY_UID = 263
|
|
26
31
|
COMMON_INFORMATION_MODEL_UID = 1621
|
|
@@ -33,7 +38,7 @@ class App_Base(BaseModel,ABC):
|
|
|
33
38
|
model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
|
|
34
39
|
uid: Optional[int] = Field(default=None)
|
|
35
40
|
title: str = Field(description="Human-readable name used by the app. This can have special characters.")
|
|
36
|
-
appid: Optional[
|
|
41
|
+
appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. "
|
|
37
42
|
"It may ONLY have characters, numbers, and underscores. No other characters are allowed.")
|
|
38
43
|
version: str = Field(description="The version of your Content Pack. This must follow semantic versioning guidelines.")
|
|
39
44
|
description: Optional[str] = Field(default="description of app",description="Free text description of the Content Pack.")
|
|
@@ -101,7 +106,7 @@ class CustomApp(App_Base):
|
|
|
101
106
|
# https://docs.splunk.com/Documentation/Splunk/9.0.4/Admin/Appconf
|
|
102
107
|
uid: int = Field(ge=2, lt=100000, default_factory=lambda:random.randint(20000,100000))
|
|
103
108
|
title: str = Field(default="Content Pack",description="Human-readable name used by the app. This can have special characters.")
|
|
104
|
-
appid:
|
|
109
|
+
appid: APPID_TYPE = Field(default="ContentPack",description="Internal name used by your app. "
|
|
105
110
|
"It may ONLY have characters, numbers, and underscores. No other characters are allowed.")
|
|
106
111
|
version: str = Field(default="0.0.1",description="The version of your Content Pack. This must follow semantic versioning guidelines.", validate_default=True)
|
|
107
112
|
|
|
@@ -171,7 +176,13 @@ class Config_Base(BaseModel):
|
|
|
171
176
|
return str(path)
|
|
172
177
|
|
|
173
178
|
class init(Config_Base):
|
|
174
|
-
|
|
179
|
+
model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
|
|
180
|
+
bare: bool = Field(default=False, description="contentctl normally provides some some example content "
|
|
181
|
+
"(macros, stories, data_sources, and/or analytic stories). This option disables "
|
|
182
|
+
"initialization with that additional contnet. Note that even if --bare is used, it "
|
|
183
|
+
"init will still create the directory structure of the app, "
|
|
184
|
+
"include the app_template directory with default content, and content in "
|
|
185
|
+
"the deployment/ directory (since it is not yet easily customizable).")
|
|
175
186
|
|
|
176
187
|
|
|
177
188
|
# TODO (#266): disable the use_enum_values configuration
|
|
@@ -185,8 +196,45 @@ class validate(Config_Base):
|
|
|
185
196
|
build_api: bool = Field(default=False, description="Should api objects be built and output in the build_path?")
|
|
186
197
|
data_source_TA_validation: bool = Field(default=False, description="Validate latest TA information from Splunkbase")
|
|
187
198
|
|
|
188
|
-
|
|
189
|
-
|
|
199
|
+
@property
|
|
200
|
+
def external_repos_path(self)->pathlib.Path:
|
|
201
|
+
return self.path/"external_repos"
|
|
202
|
+
|
|
203
|
+
@property
|
|
204
|
+
def mitre_cti_repo_path(self)->pathlib.Path:
|
|
205
|
+
return self.external_repos_path/"cti"
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def atomic_red_team_repo_path(self):
|
|
209
|
+
return self.external_repos_path/"atomic-red-team"
|
|
210
|
+
|
|
211
|
+
@model_validator(mode="after")
|
|
212
|
+
def ensureEnrichmentReposPresent(self)->Self:
|
|
213
|
+
'''
|
|
214
|
+
Ensures that the enrichments repos, the atomic red team repo and the
|
|
215
|
+
mitre attack enrichment repo, are present at the inded path.
|
|
216
|
+
Raises a detailed exception if either of these are not present
|
|
217
|
+
when enrichments are enabled.
|
|
218
|
+
'''
|
|
219
|
+
if not self.enrichments:
|
|
220
|
+
return self
|
|
221
|
+
# If enrichments are enabled, ensure that all of the
|
|
222
|
+
# enrichment directories exist
|
|
223
|
+
missing_repos:list[str] = []
|
|
224
|
+
if not self.atomic_red_team_repo_path.is_dir():
|
|
225
|
+
missing_repos.append(f"https://github.com/redcanaryco/atomic-red-team {self.atomic_red_team_repo_path}")
|
|
226
|
+
|
|
227
|
+
if not self.mitre_cti_repo_path.is_dir():
|
|
228
|
+
missing_repos.append(f"https://github.com/mitre/cti {self.mitre_cti_repo_path}")
|
|
229
|
+
|
|
230
|
+
if len(missing_repos) > 0:
|
|
231
|
+
msg_list = ["The following repositories, which are required for enrichment, have not "
|
|
232
|
+
f"been checked out to the {self.external_repos_path} directory. "
|
|
233
|
+
"Please check them out using the following commands:"]
|
|
234
|
+
msg_list.extend([f"git clone --single-branch {repo_string}" for repo_string in missing_repos])
|
|
235
|
+
msg = '\n\t'.join(msg_list)
|
|
236
|
+
raise FileNotFoundError(msg)
|
|
237
|
+
return self
|
|
190
238
|
|
|
191
239
|
class report(validate):
|
|
192
240
|
#reporting takes no extra args, but we define it here so that it can be a mode on the command line
|
|
@@ -233,9 +281,6 @@ class build(validate):
|
|
|
233
281
|
return self.getBuildDir() / f"{self.app.appid}-{self.app.version}.tar.gz"
|
|
234
282
|
else:
|
|
235
283
|
return self.getBuildDir() / f"{self.app.appid}-latest.tar.gz"
|
|
236
|
-
|
|
237
|
-
def getSSAPath(self)->pathlib.Path:
|
|
238
|
-
return self.getBuildDir() / "ssa"
|
|
239
284
|
|
|
240
285
|
def getAPIPath(self)->pathlib.Path:
|
|
241
286
|
return self.getBuildDir() / "api"
|
|
@@ -249,11 +294,89 @@ class StackType(StrEnum):
|
|
|
249
294
|
classic = auto()
|
|
250
295
|
victoria = auto()
|
|
251
296
|
|
|
297
|
+
|
|
252
298
|
class inspect(build):
|
|
253
|
-
splunk_api_username: str = Field(
|
|
254
|
-
|
|
299
|
+
splunk_api_username: str = Field(
|
|
300
|
+
description="Splunk API username used for appinspect and Splunkbase downloads."
|
|
301
|
+
)
|
|
302
|
+
splunk_api_password: str = Field(
|
|
303
|
+
exclude=True,
|
|
304
|
+
description="Splunk API password used for appinspect and Splunkbase downloads."
|
|
305
|
+
)
|
|
306
|
+
enable_metadata_validation: bool = Field(
|
|
307
|
+
default=False,
|
|
308
|
+
description=(
|
|
309
|
+
"Flag indicating whether detection metadata validation and versioning enforcement "
|
|
310
|
+
"should be enabled."
|
|
311
|
+
)
|
|
312
|
+
)
|
|
313
|
+
enrichments: bool = Field(
|
|
314
|
+
default=True,
|
|
315
|
+
description=(
|
|
316
|
+
"[NOTE: enrichments must be ENABLED for inspect to run. Please adjust your config "
|
|
317
|
+
f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}"
|
|
318
|
+
)
|
|
319
|
+
)
|
|
320
|
+
# TODO (cmcginley): wording should change here if we want to be able to download any app from
|
|
321
|
+
# Splunkbase
|
|
322
|
+
previous_build: str | None = Field(
|
|
323
|
+
default=None,
|
|
324
|
+
description=(
|
|
325
|
+
"Local path to the previous app build for metatdata validation and versioning "
|
|
326
|
+
"enforcement (defaults to the latest release of the app published on Splunkbase)."
|
|
327
|
+
)
|
|
328
|
+
)
|
|
255
329
|
stack_type: StackType = Field(description="The type of your Splunk Cloud Stack")
|
|
256
330
|
|
|
331
|
+
@field_validator("enrichments", mode="after")
|
|
332
|
+
@classmethod
|
|
333
|
+
def validate_needed_flags_metadata_validation(cls, v: bool, info: ValidationInfo) -> bool:
|
|
334
|
+
"""
|
|
335
|
+
Validates that `enrichments` is True for the inspect action
|
|
336
|
+
|
|
337
|
+
:param v: the field's value
|
|
338
|
+
:type v: bool
|
|
339
|
+
:param info: the ValidationInfo to be used
|
|
340
|
+
:type info: :class:`pydantic.ValidationInfo`
|
|
341
|
+
|
|
342
|
+
:returns: bool, for v
|
|
343
|
+
:rtype: bool
|
|
344
|
+
"""
|
|
345
|
+
# Enforce that `enrichments` is True for the inspect action
|
|
346
|
+
if v is False:
|
|
347
|
+
raise ValueError("Field `enrichments` must be True for the `inspect` action")
|
|
348
|
+
|
|
349
|
+
return v
|
|
350
|
+
|
|
351
|
+
def get_previous_package_file_path(self) -> pathlib.Path:
|
|
352
|
+
"""
|
|
353
|
+
Returns a Path object for the path to the prior package build. If no path was provided, the
|
|
354
|
+
latest version is downloaded from Splunkbase and it's filepath is returned, and saved to the
|
|
355
|
+
in-memory config (so download doesn't happen twice in the same run).
|
|
356
|
+
|
|
357
|
+
:returns: Path object to previous app build
|
|
358
|
+
:rtype: :class:`pathlib.Path`
|
|
359
|
+
"""
|
|
360
|
+
previous_build_path = self.previous_build
|
|
361
|
+
# Download the previous build as the latest release on Splunkbase if no path was provided
|
|
362
|
+
if previous_build_path is None:
|
|
363
|
+
print(
|
|
364
|
+
f"Downloading latest {self.app.label} build from Splunkbase to serve as previous "
|
|
365
|
+
"build during validation..."
|
|
366
|
+
)
|
|
367
|
+
app = SplunkApp(app_uid=self.app.uid)
|
|
368
|
+
previous_build_path = app.download(
|
|
369
|
+
out=pathlib.Path(DOWNLOADS_DIRECTORY),
|
|
370
|
+
username=self.splunk_api_username,
|
|
371
|
+
password=self.splunk_api_password,
|
|
372
|
+
is_dir=True,
|
|
373
|
+
overwrite=True
|
|
374
|
+
)
|
|
375
|
+
print(f"Latest release downloaded from Splunkbase to: {previous_build_path}")
|
|
376
|
+
self.previous_build = str(previous_build_path)
|
|
377
|
+
return pathlib.Path(previous_build_path)
|
|
378
|
+
|
|
379
|
+
|
|
257
380
|
class NewContentType(StrEnum):
|
|
258
381
|
detection = auto()
|
|
259
382
|
story = auto()
|
contentctl/objects/constants.py
CHANGED
|
@@ -575,10 +575,11 @@ class CorrelationSearch(BaseModel):
|
|
|
575
575
|
self.logger.debug(f"Using cached risk events ({len(self._risk_events)} total).")
|
|
576
576
|
return self._risk_events
|
|
577
577
|
|
|
578
|
+
# TODO (#248): Refactor risk/notable querying to pin to a single savedsearch ID
|
|
578
579
|
# Search for all risk events from a single scheduled search (indicated by orig_sid)
|
|
579
580
|
query = (
|
|
580
581
|
f'search index=risk search_name="{self.name}" [search index=risk search '
|
|
581
|
-
f'search_name="{self.name}" |
|
|
582
|
+
f'search_name="{self.name}" | tail 1 | fields orig_sid] | tojson'
|
|
582
583
|
)
|
|
583
584
|
result_iterator = self._search(query)
|
|
584
585
|
|
|
@@ -643,7 +644,7 @@ class CorrelationSearch(BaseModel):
|
|
|
643
644
|
# Search for all notable events from a single scheduled search (indicated by orig_sid)
|
|
644
645
|
query = (
|
|
645
646
|
f'search index=notable search_name="{self.name}" [search index=notable search '
|
|
646
|
-
f'search_name="{self.name}" |
|
|
647
|
+
f'search_name="{self.name}" | tail 1 | fields orig_sid] | tojson'
|
|
647
648
|
)
|
|
648
649
|
result_iterator = self._search(query)
|
|
649
650
|
|
|
@@ -686,15 +687,17 @@ class CorrelationSearch(BaseModel):
|
|
|
686
687
|
check the risks/notables
|
|
687
688
|
:returns: an IntegrationTestResult on failure; None on success
|
|
688
689
|
"""
|
|
689
|
-
# TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the false
|
|
690
|
-
# positive rate in risk/obseravble matching
|
|
691
690
|
# Create a mapping of the relevant observables to counters
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
#
|
|
696
|
-
#
|
|
697
|
-
|
|
691
|
+
observables = CorrelationSearch._get_relevant_observables(self.detection.tags.observable)
|
|
692
|
+
observable_counts: dict[str, int] = {str(x): 0 for x in observables}
|
|
693
|
+
|
|
694
|
+
# NOTE: we intentionally want this to be an error state and not a failure state, as
|
|
695
|
+
# ultimately this validation should be handled during the build process
|
|
696
|
+
if len(observables) != len(observable_counts):
|
|
697
|
+
raise ClientError(
|
|
698
|
+
f"At least two observables in '{self.detection.name}' have the same name; "
|
|
699
|
+
"each observable for a detection should be unique."
|
|
700
|
+
)
|
|
698
701
|
|
|
699
702
|
# Get the risk events; note that we use the cached risk events, expecting they were
|
|
700
703
|
# saved by a prior call to risk_event_exists
|
|
@@ -710,25 +713,29 @@ class CorrelationSearch(BaseModel):
|
|
|
710
713
|
)
|
|
711
714
|
event.validate_against_detection(self.detection)
|
|
712
715
|
|
|
713
|
-
# TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the
|
|
714
|
-
# false positive rate in risk/obseravble matching
|
|
715
716
|
# Update observable count based on match
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
#
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
717
|
+
matched_observable = event.get_matched_observable(self.detection.tags.observable)
|
|
718
|
+
self.logger.debug(
|
|
719
|
+
f"Matched risk event (object={event.risk_object}, type={event.risk_object_type}) "
|
|
720
|
+
f"to observable (name={matched_observable.name}, type={matched_observable.type}, "
|
|
721
|
+
f"role={matched_observable.role}) using the source field "
|
|
722
|
+
f"'{event.source_field_name}'"
|
|
723
|
+
)
|
|
724
|
+
observable_counts[str(matched_observable)] += 1
|
|
725
|
+
|
|
726
|
+
# Report any observables which did not have at least one match to a risk event
|
|
727
|
+
for observable in observables:
|
|
728
|
+
self.logger.debug(
|
|
729
|
+
f"Matched observable (name={observable.name}, type={observable.type}, "
|
|
730
|
+
f"role={observable.role}) to {observable_counts[str(observable)]} risk events."
|
|
731
|
+
)
|
|
732
|
+
if observable_counts[str(observable)] == 0:
|
|
733
|
+
raise ValidationFailed(
|
|
734
|
+
f"Observable (name={observable.name}, type={observable.type}, "
|
|
735
|
+
f"role={observable.role}) was not matched to any risk events."
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
# TODO (#250): Re-enable and refactor code that validates the specific risk counts
|
|
732
739
|
# Validate risk events in aggregate; we should have an equal amount of risk events for each
|
|
733
740
|
# relevant observable, and the total count should match the total number of events
|
|
734
741
|
# individual_count: Optional[int] = None
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field, field_validator
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class DetectionMetadata(BaseModel):
|
|
8
|
+
"""
|
|
9
|
+
A model of the metadata line in a detection stanza in savedsearches.conf
|
|
10
|
+
"""
|
|
11
|
+
# A bool indicating whether the detection is deprecated (serialized as an int, 1 or 0)
|
|
12
|
+
deprecated: bool = Field(...)
|
|
13
|
+
|
|
14
|
+
# A UUID identifying the detection
|
|
15
|
+
detection_id: uuid.UUID = Field(...)
|
|
16
|
+
|
|
17
|
+
# The version of the detection
|
|
18
|
+
detection_version: int = Field(...)
|
|
19
|
+
|
|
20
|
+
# The time the detection was published. **NOTE** This field was added to the metadata in ESCU
|
|
21
|
+
# as of v4.39.0
|
|
22
|
+
publish_time: float = Field(...)
|
|
23
|
+
|
|
24
|
+
class Config:
|
|
25
|
+
# Allowing for future fields that may be added to the metadata JSON
|
|
26
|
+
extra = "allow"
|
|
27
|
+
|
|
28
|
+
@field_validator("deprecated", mode="before")
|
|
29
|
+
@classmethod
|
|
30
|
+
def validate_deprecated(cls, v: Any) -> Any:
|
|
31
|
+
"""
|
|
32
|
+
Convert str to int, and then ints to bools for deprecated; raise if not 0 or 1 in the case
|
|
33
|
+
of an int, or if str cannot be converted to int.
|
|
34
|
+
|
|
35
|
+
:param v: the value passed
|
|
36
|
+
:type v: :class:`typing.Any`
|
|
37
|
+
|
|
38
|
+
:returns: the value
|
|
39
|
+
:rtype: :class:`typing.Any`
|
|
40
|
+
"""
|
|
41
|
+
if isinstance(v, str):
|
|
42
|
+
try:
|
|
43
|
+
v = int(v)
|
|
44
|
+
except ValueError as e:
|
|
45
|
+
raise ValueError(f"Cannot convert str value ({v}) to int: {e}") from e
|
|
46
|
+
if isinstance(v, int):
|
|
47
|
+
if not (0 <= v <= 1):
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"Value for field 'deprecated' ({v}) must be 0 or 1, if not a bool."
|
|
50
|
+
)
|
|
51
|
+
v = bool(v)
|
|
52
|
+
return v
|
|
53
|
+
|
|
54
|
+
@field_validator("detection_version", mode="before")
|
|
55
|
+
@classmethod
|
|
56
|
+
def validate_detection_version(cls, v: Any) -> Any:
|
|
57
|
+
"""
|
|
58
|
+
Convert str to int; raise if str cannot be converted to int.
|
|
59
|
+
|
|
60
|
+
:param v: the value passed
|
|
61
|
+
:type v: :class:`typing.Any`
|
|
62
|
+
|
|
63
|
+
:returns: the value
|
|
64
|
+
:rtype: :class:`typing.Any`
|
|
65
|
+
"""
|
|
66
|
+
if isinstance(v, str):
|
|
67
|
+
try:
|
|
68
|
+
v = int(v)
|
|
69
|
+
except ValueError as e:
|
|
70
|
+
raise ValueError(f"Cannot convert str value ({v}) to int: {e}") from e
|
|
71
|
+
return v
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from typing import ClassVar
|
|
2
|
+
import hashlib
|
|
3
|
+
from functools import cached_property
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field, computed_field
|
|
6
|
+
|
|
7
|
+
from contentctl.objects.detection_metadata import DetectionMetadata
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DetectionStanza(BaseModel):
|
|
11
|
+
"""
|
|
12
|
+
A model representing a stanza for a detection in savedsearches.conf
|
|
13
|
+
"""
|
|
14
|
+
# The lines that comprise this stanza, in the order they appear in the conf
|
|
15
|
+
lines: list[str] = Field(...)
|
|
16
|
+
|
|
17
|
+
# The full name of the detection (e.g. "ESCU - My Detection - Rule")
|
|
18
|
+
name: str = Field(...)
|
|
19
|
+
|
|
20
|
+
# The key prefix indicating the metadata attribute
|
|
21
|
+
METADATA_LINE_PREFIX: ClassVar[str] = "action.correlationsearch.metadata = "
|
|
22
|
+
|
|
23
|
+
@computed_field
|
|
24
|
+
@cached_property
|
|
25
|
+
def metadata(self) -> DetectionMetadata:
|
|
26
|
+
"""
|
|
27
|
+
The metadata extracted from the stanza. Using the provided lines, parse out the metadata
|
|
28
|
+
|
|
29
|
+
:returns: the detection stanza's metadata
|
|
30
|
+
:rtype: :class:`contentctl.objects.detection_metadata.DetectionMetadata`
|
|
31
|
+
"""
|
|
32
|
+
# Set a variable to store the metadata line in
|
|
33
|
+
meta_line: str | None = None
|
|
34
|
+
|
|
35
|
+
# Iterate over the lines to look for the metadata line
|
|
36
|
+
for line in self.lines:
|
|
37
|
+
if line.startswith(DetectionStanza.METADATA_LINE_PREFIX):
|
|
38
|
+
# If we find a matching line more than once, we've hit an error
|
|
39
|
+
if meta_line is not None:
|
|
40
|
+
raise Exception(
|
|
41
|
+
f"Metadata for detection '{self.name}' found twice in stanza."
|
|
42
|
+
)
|
|
43
|
+
meta_line = line
|
|
44
|
+
|
|
45
|
+
# Report if we could not find the metadata line
|
|
46
|
+
if meta_line is None:
|
|
47
|
+
raise Exception(f"No metadata for detection '{self.name}' found in stanza.")
|
|
48
|
+
|
|
49
|
+
# Parse the metadata JSON into a model
|
|
50
|
+
return DetectionMetadata.model_validate_json(meta_line[len(DetectionStanza.METADATA_LINE_PREFIX):])
|
|
51
|
+
|
|
52
|
+
@computed_field
|
|
53
|
+
@cached_property
|
|
54
|
+
def hash(self) -> str:
|
|
55
|
+
"""
|
|
56
|
+
The SHA256 hash of the lines of the stanza, excluding the metadata line
|
|
57
|
+
|
|
58
|
+
:returns: hexdigest
|
|
59
|
+
:rtype: str
|
|
60
|
+
"""
|
|
61
|
+
hash = hashlib.sha256()
|
|
62
|
+
for line in self.lines:
|
|
63
|
+
if not line.startswith(DetectionStanza.METADATA_LINE_PREFIX):
|
|
64
|
+
hash.update(line.encode("utf-8"))
|
|
65
|
+
return hash.hexdigest()
|
|
66
|
+
|
|
67
|
+
def version_should_be_bumped(self, previous: "DetectionStanza") -> bool:
|
|
68
|
+
"""
|
|
69
|
+
A helper method that compares this stanza against the same stanza from a previous build;
|
|
70
|
+
returns True if the version still needs to be bumped (e.g. the detection was changed but
|
|
71
|
+
the version was not), False otherwise.
|
|
72
|
+
|
|
73
|
+
:param previous: the previous build's DetectionStanza for comparison
|
|
74
|
+
:type previous: :class:`contentctl.objects.detection_stanza.DetectionStanza`
|
|
75
|
+
|
|
76
|
+
:returns: True if the version still needs to be bumped
|
|
77
|
+
:rtype: bool
|
|
78
|
+
"""
|
|
79
|
+
return (self.hash != previous.hash) and (self.metadata.detection_version <= previous.metadata.detection_version)
|