contentctl 4.3.3__py3-none-any.whl → 4.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +0 -6
  2. contentctl/actions/initialize.py +28 -12
  3. contentctl/actions/inspect.py +189 -91
  4. contentctl/actions/validate.py +3 -7
  5. contentctl/api.py +1 -1
  6. contentctl/contentctl.py +3 -0
  7. contentctl/enrichments/attack_enrichment.py +51 -82
  8. contentctl/enrichments/cve_enrichment.py +2 -2
  9. contentctl/helper/splunk_app.py +141 -10
  10. contentctl/input/director.py +5 -12
  11. contentctl/objects/abstract_security_content_objects/detection_abstract.py +11 -8
  12. contentctl/objects/annotated_types.py +6 -0
  13. contentctl/objects/atomic.py +51 -77
  14. contentctl/objects/config.py +145 -22
  15. contentctl/objects/constants.py +4 -1
  16. contentctl/objects/correlation_search.py +35 -28
  17. contentctl/objects/detection_metadata.py +71 -0
  18. contentctl/objects/detection_stanza.py +79 -0
  19. contentctl/objects/detection_tags.py +11 -9
  20. contentctl/objects/enums.py +0 -2
  21. contentctl/objects/errors.py +187 -0
  22. contentctl/objects/mitre_attack_enrichment.py +2 -1
  23. contentctl/objects/risk_event.py +94 -76
  24. contentctl/objects/savedsearches_conf.py +196 -0
  25. contentctl/objects/story_tags.py +3 -3
  26. contentctl/output/conf_writer.py +4 -1
  27. contentctl/output/new_content_yml_output.py +4 -9
  28. {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/METADATA +4 -4
  29. {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/RECORD +32 -32
  30. contentctl/objects/ssa_detection.py +0 -157
  31. contentctl/objects/ssa_detection_tags.py +0 -138
  32. contentctl/objects/unit_test_old.py +0 -10
  33. contentctl/objects/unit_test_ssa.py +0 -31
  34. {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/LICENSE.md +0 -0
  35. {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/WHEEL +0 -0
  36. {contentctl-4.3.3.dist-info → contentctl-4.3.5.dist-info}/entry_points.txt +0 -0
@@ -1,12 +1,15 @@
1
1
  from __future__ import annotations
2
+ from typing import TYPE_CHECKING
3
+ if TYPE_CHECKING:
4
+ from contentctl.objects.config import validate
5
+
2
6
  from contentctl.input.yml_reader import YmlReader
3
7
  from pydantic import BaseModel, model_validator, ConfigDict, FilePath, UUID4
8
+ import dataclasses
4
9
  from typing import List, Optional, Dict, Union, Self
5
10
  import pathlib
6
-
7
-
8
11
  from enum import StrEnum, auto
9
-
12
+ import uuid
10
13
 
11
14
  class SupportedPlatform(StrEnum):
12
15
  windows = auto()
@@ -84,15 +87,6 @@ class AtomicTest(BaseModel):
84
87
  dependencies: Optional[List[AtomicDependency]] = None
85
88
  dependency_executor_name: Optional[DependencyExecutorType] = None
86
89
 
87
- @staticmethod
88
- def AtomicTestWhenEnrichmentIsDisabled(auto_generated_guid: UUID4) -> AtomicTest:
89
- return AtomicTest(name="Placeholder Atomic Test (enrichment disabled)",
90
- auto_generated_guid=auto_generated_guid,
91
- description="This is a placeholder AtomicTest. Because enrichments were not enabled, it has not been validated against the real Atomic Red Team Repo.",
92
- supported_platforms=[],
93
- executor=AtomicExecutor(name="Placeholder Executor (enrichment disabled)",
94
- command="Placeholder command (enrichment disabled)"))
95
-
96
90
  @staticmethod
97
91
  def AtomicTestWhenTestIsMissing(auto_generated_guid: UUID4) -> AtomicTest:
98
92
  return AtomicTest(name="Missing Atomic",
@@ -100,31 +94,16 @@ class AtomicTest(BaseModel):
100
94
  description="This is a placeholder AtomicTest. Either the auto_generated_guid is incorrect or it there was an exception while parsing its AtomicFile.",
101
95
  supported_platforms=[],
102
96
  executor=AtomicExecutor(name="Placeholder Executor (failed to find auto_generated_guid)",
103
- command="Placeholder command (failed to find auto_generated_guid)"))
104
-
105
-
106
- @classmethod
107
- def getAtomicByAtomicGuid(cls, guid: UUID4, all_atomics:list[AtomicTest] | None)->AtomicTest:
108
- if all_atomics is None:
109
- return AtomicTest.AtomicTestWhenEnrichmentIsDisabled(guid)
110
- matching_atomics = [atomic for atomic in all_atomics if atomic.auto_generated_guid == guid]
111
- if len(matching_atomics) == 0:
112
- raise ValueError(f"Unable to find atomic_guid {guid} in {len(all_atomics)} atomic_tests from ART Repo")
113
- elif len(matching_atomics) > 1:
114
- raise ValueError(f"Found {len(matching_atomics)} matching tests for atomic_guid {guid} in {len(all_atomics)} atomic_tests from ART Repo")
115
-
116
- return matching_atomics[0]
97
+ command="Placeholder command (failed to find auto_generated_guid)"))
117
98
 
118
99
  @classmethod
119
- def parseArtRepo(cls, repo_path:pathlib.Path)->List[AtomicFile]:
120
- if not repo_path.is_dir():
121
- print(f"WARNING: Atomic Red Team repo does NOT exist at {repo_path.absolute()}. You can check it out with:\n * git clone --single-branch https://github.com/redcanaryco/atomic-red-team. This will ONLY throw a validation error if you reference atomid_guids in your detection(s).")
122
- return []
100
+ def parseArtRepo(cls, repo_path:pathlib.Path)->dict[uuid.UUID, AtomicTest]:
101
+ test_mapping: dict[uuid.UUID, AtomicTest] = {}
123
102
  atomics_path = repo_path/"atomics"
124
103
  if not atomics_path.is_dir():
125
- print(f"WARNING: Atomic Red Team repo exists at {repo_path.absolute}, but atomics directory does NOT exist at {atomics_path.absolute()}. Was it deleted or renamed? This will ONLY throw a validation error if you reference atomid_guids in your detection(s).")
126
- return []
127
-
104
+ raise FileNotFoundError(f"WARNING: Atomic Red Team repo exists at {repo_path}, "
105
+ f"but atomics directory does NOT exist at {atomics_path}. "
106
+ "Was it deleted or renamed?")
128
107
 
129
108
  atomic_files:List[AtomicFile] = []
130
109
  error_messages:List[str] = []
@@ -133,6 +112,7 @@ class AtomicTest(BaseModel):
133
112
  atomic_files.append(cls.constructAtomicFile(obj_path))
134
113
  except Exception as e:
135
114
  error_messages.append(f"File [{obj_path}]\n{str(e)}")
115
+
136
116
  if len(error_messages) > 0:
137
117
  exceptions_string = '\n\n'.join(error_messages)
138
118
  print(f"WARNING: The following [{len(error_messages)}] ERRORS were generated when parsing the Atomic Red Team Repo.\n"
@@ -140,38 +120,28 @@ class AtomicTest(BaseModel):
140
120
  "Note that this is only a warning and contentctl will ignore Atomics contained in these files.\n"
141
121
  f"However, if you have written a detection that references them, 'contentctl build --enrichments' will fail:\n\n{exceptions_string}")
142
122
 
143
- return atomic_files
123
+ # Now iterate over all the files, collect all the tests, and return the dict mapping
124
+ redefined_guids:set[uuid.UUID] = set()
125
+ for atomic_file in atomic_files:
126
+ for atomic_test in atomic_file.atomic_tests:
127
+ if atomic_test.auto_generated_guid in test_mapping:
128
+ redefined_guids.add(atomic_test.auto_generated_guid)
129
+ else:
130
+ test_mapping[atomic_test.auto_generated_guid] = atomic_test
131
+ if len(redefined_guids) > 0:
132
+ guids_string = '\n\t'.join([str(guid) for guid in redefined_guids])
133
+ raise Exception(f"The following [{len(redefined_guids)}] Atomic Test"
134
+ " auto_generated_guid(s) were defined more than once. "
135
+ f"auto_generated_guids MUST be unique:\n\t{guids_string}")
136
+
137
+ print(f"Successfully parsed [{len(test_mapping)}] Atomic Red Team Tests!")
138
+ return test_mapping
144
139
 
145
140
  @classmethod
146
141
  def constructAtomicFile(cls, file_path:pathlib.Path)->AtomicFile:
147
142
  yml_dict = YmlReader.load_file(file_path)
148
143
  atomic_file = AtomicFile.model_validate(yml_dict)
149
144
  return atomic_file
150
-
151
- @classmethod
152
- def getAtomicTestsFromArtRepo(cls, repo_path:pathlib.Path, enabled:bool=True)->list[AtomicTest] | None:
153
- # Get all the atomic files. Note that if the ART repo is not found, we will not throw an error,
154
- # but will not have any atomics. This means that if atomic_guids are referenced during validation,
155
- # validation for those detections will fail
156
- if not enabled:
157
- return None
158
-
159
- atomic_files = cls.getAtomicFilesFromArtRepo(repo_path)
160
-
161
- atomic_tests:List[AtomicTest] = []
162
- for atomic_file in atomic_files:
163
- atomic_tests.extend(atomic_file.atomic_tests)
164
- print(f"Found [{len(atomic_tests)}] Atomic Simulations in the Atomic Red Team Repo!")
165
- return atomic_tests
166
-
167
-
168
- @classmethod
169
- def getAtomicFilesFromArtRepo(cls, repo_path:pathlib.Path)->List[AtomicFile]:
170
- return cls.parseArtRepo(repo_path)
171
-
172
-
173
-
174
-
175
145
 
176
146
 
177
147
  class AtomicFile(BaseModel):
@@ -182,27 +152,31 @@ class AtomicFile(BaseModel):
182
152
  atomic_tests: List[AtomicTest]
183
153
 
184
154
 
155
+ class AtomicEnrichment(BaseModel):
156
+ data: dict[uuid.UUID,AtomicTest] = dataclasses.field(default_factory = dict)
157
+ use_enrichment: bool = False
185
158
 
159
+ @classmethod
160
+ def getAtomicEnrichment(cls, config:validate)->AtomicEnrichment:
161
+ enrichment = AtomicEnrichment(use_enrichment=config.enrichments)
162
+ if config.enrichments:
163
+ enrichment.data = AtomicTest.parseArtRepo(config.atomic_red_team_repo_path)
164
+
165
+ return enrichment
166
+
167
+ def getAtomic(self, atomic_guid: uuid.UUID)->AtomicTest:
168
+ if self.use_enrichment:
169
+ if atomic_guid in self.data:
170
+ return self.data[atomic_guid]
171
+ else:
172
+ raise Exception(f"Atomic with GUID {atomic_guid} not found.")
173
+ else:
174
+ # If enrichment is not enabled, for the sake of compatability
175
+ # return a stub test with no useful or meaningful information.
176
+ return AtomicTest.AtomicTestWhenTestIsMissing(atomic_guid)
186
177
 
187
- # ATOMICS_PATH = pathlib.Path("./atomics")
188
- # atomic_objects = []
189
- # atomic_simulations = []
190
- # for obj_path in ATOMICS_PATH.glob("**/T*.yaml"):
191
- # try:
192
- # with open(obj_path, 'r', encoding="utf-8") as obj_handle:
193
- # obj_data = yaml.load(obj_handle, Loader=yaml.CSafeLoader)
194
- # atomic_obj = AtomicFile.model_validate(obj_data)
195
- # except Exception as e:
196
- # print(f"Error parsing object at path {obj_path}: {str(e)}")
197
- # print(f"We have successfully parsed {len(atomic_objects)}, however!")
198
- # sys.exit(1)
199
-
200
- # print(f"Successfully parsed {obj_path}!")
201
- # atomic_objects.append(atomic_obj)
202
- # atomic_simulations += atomic_obj.atomic_tests
178
+
203
179
 
204
- # print(f"Successfully parsed all {len(atomic_objects)} files!")
205
- # print(f"Successfully parsed all {len(atomic_simulations)} simulations!")
206
180
 
207
181
 
208
182
 
@@ -1,26 +1,31 @@
1
1
  from __future__ import annotations
2
+
3
+ from os import environ
4
+ from datetime import datetime, UTC
5
+ from typing import Optional, Any, List, Union, Self
6
+ import random
7
+ from enum import StrEnum, auto
8
+ import pathlib
9
+ from urllib.parse import urlparse
10
+ from abc import ABC, abstractmethod
11
+ from functools import partialmethod
12
+
13
+ import tqdm
14
+ import semantic_version
2
15
  from pydantic import (
3
16
  BaseModel, Field, field_validator,
4
17
  field_serializer, ConfigDict, DirectoryPath,
5
18
  PositiveInt, FilePath, HttpUrl, AnyUrl, model_validator,
6
19
  ValidationInfo
7
20
  )
21
+
22
+ from contentctl.objects.constants import DOWNLOADS_DIRECTORY
8
23
  from contentctl.output.yml_writer import YmlWriter
9
- from os import environ
10
- from datetime import datetime, UTC
11
- from typing import Optional,Any,Annotated,List,Union, Self
12
- import semantic_version
13
- import random
14
- from enum import StrEnum, auto
15
- import pathlib
16
24
  from contentctl.helper.utils import Utils
17
- from urllib.parse import urlparse
18
- from abc import ABC, abstractmethod
19
25
  from contentctl.objects.enums import PostTestBehavior, DetectionTestingMode
20
26
  from contentctl.objects.detection import Detection
21
-
22
- import tqdm
23
- from functools import partialmethod
27
+ from contentctl.objects.annotated_types import APPID_TYPE
28
+ from contentctl.helper.splunk_app import SplunkApp
24
29
 
25
30
  ENTERPRISE_SECURITY_UID = 263
26
31
  COMMON_INFORMATION_MODEL_UID = 1621
@@ -33,7 +38,7 @@ class App_Base(BaseModel,ABC):
33
38
  model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
34
39
  uid: Optional[int] = Field(default=None)
35
40
  title: str = Field(description="Human-readable name used by the app. This can have special characters.")
36
- appid: Optional[Annotated[str, Field(pattern="^[a-zA-Z0-9_-]+$")]]= Field(default=None,description="Internal name used by your app. "
41
+ appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. "
37
42
  "It may ONLY have characters, numbers, and underscores. No other characters are allowed.")
38
43
  version: str = Field(description="The version of your Content Pack. This must follow semantic versioning guidelines.")
39
44
  description: Optional[str] = Field(default="description of app",description="Free text description of the Content Pack.")
@@ -101,7 +106,7 @@ class CustomApp(App_Base):
101
106
  # https://docs.splunk.com/Documentation/Splunk/9.0.4/Admin/Appconf
102
107
  uid: int = Field(ge=2, lt=100000, default_factory=lambda:random.randint(20000,100000))
103
108
  title: str = Field(default="Content Pack",description="Human-readable name used by the app. This can have special characters.")
104
- appid: Annotated[str, Field(pattern="^[a-zA-Z0-9_-]+$")]= Field(default="ContentPack",description="Internal name used by your app. "
109
+ appid: APPID_TYPE = Field(default="ContentPack",description="Internal name used by your app. "
105
110
  "It may ONLY have characters, numbers, and underscores. No other characters are allowed.")
106
111
  version: str = Field(default="0.0.1",description="The version of your Content Pack. This must follow semantic versioning guidelines.", validate_default=True)
107
112
 
@@ -171,7 +176,13 @@ class Config_Base(BaseModel):
171
176
  return str(path)
172
177
 
173
178
  class init(Config_Base):
174
- pass
179
+ model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
180
+ bare: bool = Field(default=False, description="contentctl normally provides some some example content "
181
+ "(macros, stories, data_sources, and/or analytic stories). This option disables "
182
+ "initialization with that additional contnet. Note that even if --bare is used, it "
183
+ "init will still create the directory structure of the app, "
184
+ "include the app_template directory with default content, and content in "
185
+ "the deployment/ directory (since it is not yet easily customizable).")
175
186
 
176
187
 
177
188
  # TODO (#266): disable the use_enum_values configuration
@@ -185,8 +196,45 @@ class validate(Config_Base):
185
196
  build_api: bool = Field(default=False, description="Should api objects be built and output in the build_path?")
186
197
  data_source_TA_validation: bool = Field(default=False, description="Validate latest TA information from Splunkbase")
187
198
 
188
- def getAtomicRedTeamRepoPath(self, atomic_red_team_repo_name:str = "atomic-red-team"):
189
- return self.path/atomic_red_team_repo_name
199
+ @property
200
+ def external_repos_path(self)->pathlib.Path:
201
+ return self.path/"external_repos"
202
+
203
+ @property
204
+ def mitre_cti_repo_path(self)->pathlib.Path:
205
+ return self.external_repos_path/"cti"
206
+
207
+ @property
208
+ def atomic_red_team_repo_path(self):
209
+ return self.external_repos_path/"atomic-red-team"
210
+
211
+ @model_validator(mode="after")
212
+ def ensureEnrichmentReposPresent(self)->Self:
213
+ '''
214
+ Ensures that the enrichments repos, the atomic red team repo and the
215
+ mitre attack enrichment repo, are present at the inded path.
216
+ Raises a detailed exception if either of these are not present
217
+ when enrichments are enabled.
218
+ '''
219
+ if not self.enrichments:
220
+ return self
221
+ # If enrichments are enabled, ensure that all of the
222
+ # enrichment directories exist
223
+ missing_repos:list[str] = []
224
+ if not self.atomic_red_team_repo_path.is_dir():
225
+ missing_repos.append(f"https://github.com/redcanaryco/atomic-red-team {self.atomic_red_team_repo_path}")
226
+
227
+ if not self.mitre_cti_repo_path.is_dir():
228
+ missing_repos.append(f"https://github.com/mitre/cti {self.mitre_cti_repo_path}")
229
+
230
+ if len(missing_repos) > 0:
231
+ msg_list = ["The following repositories, which are required for enrichment, have not "
232
+ f"been checked out to the {self.external_repos_path} directory. "
233
+ "Please check them out using the following commands:"]
234
+ msg_list.extend([f"git clone --single-branch {repo_string}" for repo_string in missing_repos])
235
+ msg = '\n\t'.join(msg_list)
236
+ raise FileNotFoundError(msg)
237
+ return self
190
238
 
191
239
  class report(validate):
192
240
  #reporting takes no extra args, but we define it here so that it can be a mode on the command line
@@ -233,9 +281,6 @@ class build(validate):
233
281
  return self.getBuildDir() / f"{self.app.appid}-{self.app.version}.tar.gz"
234
282
  else:
235
283
  return self.getBuildDir() / f"{self.app.appid}-latest.tar.gz"
236
-
237
- def getSSAPath(self)->pathlib.Path:
238
- return self.getBuildDir() / "ssa"
239
284
 
240
285
  def getAPIPath(self)->pathlib.Path:
241
286
  return self.getBuildDir() / "api"
@@ -249,11 +294,89 @@ class StackType(StrEnum):
249
294
  classic = auto()
250
295
  victoria = auto()
251
296
 
297
+
252
298
  class inspect(build):
253
- splunk_api_username: str = Field(description="Splunk API username used for running appinspect.")
254
- splunk_api_password: str = Field(exclude=True, description="Splunk API password used for running appinspect.")
299
+ splunk_api_username: str = Field(
300
+ description="Splunk API username used for appinspect and Splunkbase downloads."
301
+ )
302
+ splunk_api_password: str = Field(
303
+ exclude=True,
304
+ description="Splunk API password used for appinspect and Splunkbase downloads."
305
+ )
306
+ enable_metadata_validation: bool = Field(
307
+ default=False,
308
+ description=(
309
+ "Flag indicating whether detection metadata validation and versioning enforcement "
310
+ "should be enabled."
311
+ )
312
+ )
313
+ enrichments: bool = Field(
314
+ default=True,
315
+ description=(
316
+ "[NOTE: enrichments must be ENABLED for inspect to run. Please adjust your config "
317
+ f"or CLI invocation appropriately] {validate.model_fields['enrichments'].description}"
318
+ )
319
+ )
320
+ # TODO (cmcginley): wording should change here if we want to be able to download any app from
321
+ # Splunkbase
322
+ previous_build: str | None = Field(
323
+ default=None,
324
+ description=(
325
+ "Local path to the previous app build for metatdata validation and versioning "
326
+ "enforcement (defaults to the latest release of the app published on Splunkbase)."
327
+ )
328
+ )
255
329
  stack_type: StackType = Field(description="The type of your Splunk Cloud Stack")
256
330
 
331
+ @field_validator("enrichments", mode="after")
332
+ @classmethod
333
+ def validate_needed_flags_metadata_validation(cls, v: bool, info: ValidationInfo) -> bool:
334
+ """
335
+ Validates that `enrichments` is True for the inspect action
336
+
337
+ :param v: the field's value
338
+ :type v: bool
339
+ :param info: the ValidationInfo to be used
340
+ :type info: :class:`pydantic.ValidationInfo`
341
+
342
+ :returns: bool, for v
343
+ :rtype: bool
344
+ """
345
+ # Enforce that `enrichments` is True for the inspect action
346
+ if v is False:
347
+ raise ValueError("Field `enrichments` must be True for the `inspect` action")
348
+
349
+ return v
350
+
351
+ def get_previous_package_file_path(self) -> pathlib.Path:
352
+ """
353
+ Returns a Path object for the path to the prior package build. If no path was provided, the
354
+ latest version is downloaded from Splunkbase and it's filepath is returned, and saved to the
355
+ in-memory config (so download doesn't happen twice in the same run).
356
+
357
+ :returns: Path object to previous app build
358
+ :rtype: :class:`pathlib.Path`
359
+ """
360
+ previous_build_path = self.previous_build
361
+ # Download the previous build as the latest release on Splunkbase if no path was provided
362
+ if previous_build_path is None:
363
+ print(
364
+ f"Downloading latest {self.app.label} build from Splunkbase to serve as previous "
365
+ "build during validation..."
366
+ )
367
+ app = SplunkApp(app_uid=self.app.uid)
368
+ previous_build_path = app.download(
369
+ out=pathlib.Path(DOWNLOADS_DIRECTORY),
370
+ username=self.splunk_api_username,
371
+ password=self.splunk_api_password,
372
+ is_dir=True,
373
+ overwrite=True
374
+ )
375
+ print(f"Latest release downloaded from Splunkbase to: {previous_build_path}")
376
+ self.previous_build = str(previous_build_path)
377
+ return pathlib.Path(previous_build_path)
378
+
379
+
257
380
  class NewContentType(StrEnum):
258
381
  detection = auto()
259
382
  story = auto()
@@ -136,4 +136,7 @@ SES_ATTACK_TACTICS_ID_MAPPING = {
136
136
  RBA_OBSERVABLE_ROLE_MAPPING = {
137
137
  "Attacker": 0,
138
138
  "Victim": 1
139
- }
139
+ }
140
+
141
+ # The relative path to the directory where any apps/packages will be downloaded
142
+ DOWNLOADS_DIRECTORY = "downloads"
@@ -575,10 +575,11 @@ class CorrelationSearch(BaseModel):
575
575
  self.logger.debug(f"Using cached risk events ({len(self._risk_events)} total).")
576
576
  return self._risk_events
577
577
 
578
+ # TODO (#248): Refactor risk/notable querying to pin to a single savedsearch ID
578
579
  # Search for all risk events from a single scheduled search (indicated by orig_sid)
579
580
  query = (
580
581
  f'search index=risk search_name="{self.name}" [search index=risk search '
581
- f'search_name="{self.name}" | head 1 | fields orig_sid] | tojson'
582
+ f'search_name="{self.name}" | tail 1 | fields orig_sid] | tojson'
582
583
  )
583
584
  result_iterator = self._search(query)
584
585
 
@@ -643,7 +644,7 @@ class CorrelationSearch(BaseModel):
643
644
  # Search for all notable events from a single scheduled search (indicated by orig_sid)
644
645
  query = (
645
646
  f'search index=notable search_name="{self.name}" [search index=notable search '
646
- f'search_name="{self.name}" | head 1 | fields orig_sid] | tojson'
647
+ f'search_name="{self.name}" | tail 1 | fields orig_sid] | tojson'
647
648
  )
648
649
  result_iterator = self._search(query)
649
650
 
@@ -686,15 +687,17 @@ class CorrelationSearch(BaseModel):
686
687
  check the risks/notables
687
688
  :returns: an IntegrationTestResult on failure; None on success
688
689
  """
689
- # TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the false
690
- # positive rate in risk/obseravble matching
691
690
  # Create a mapping of the relevant observables to counters
692
- # observables = CorrelationSearch._get_relevant_observables(self.detection.tags.observable)
693
- # observable_counts: dict[str, int] = {str(x): 0 for x in observables}
694
- # if len(observables) != len(observable_counts):
695
- # raise ClientError(
696
- # f"At least two observables in '{self.detection.name}' have the same name."
697
- # )
691
+ observables = CorrelationSearch._get_relevant_observables(self.detection.tags.observable)
692
+ observable_counts: dict[str, int] = {str(x): 0 for x in observables}
693
+
694
+ # NOTE: we intentionally want this to be an error state and not a failure state, as
695
+ # ultimately this validation should be handled during the build process
696
+ if len(observables) != len(observable_counts):
697
+ raise ClientError(
698
+ f"At least two observables in '{self.detection.name}' have the same name; "
699
+ "each observable for a detection should be unique."
700
+ )
698
701
 
699
702
  # Get the risk events; note that we use the cached risk events, expecting they were
700
703
  # saved by a prior call to risk_event_exists
@@ -710,25 +713,29 @@ class CorrelationSearch(BaseModel):
710
713
  )
711
714
  event.validate_against_detection(self.detection)
712
715
 
713
- # TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the
714
- # false positive rate in risk/obseravble matching
715
716
  # Update observable count based on match
716
- # matched_observable = event.get_matched_observable(self.detection.tags.observable)
717
- # self.logger.debug(
718
- # f"Matched risk event ({event.risk_object}, {event.risk_object_type}) to observable "
719
- # f"({matched_observable.name}, {matched_observable.type}, {matched_observable.role})"
720
- # )
721
- # observable_counts[str(matched_observable)] += 1
722
-
723
- # TODO (PEX-433): test my new contentctl logic against an old ESCU build; my logic should
724
- # detect the faulty attacker events -> this was the issue from the 4.28/4.27 release;
725
- # recreate by testing against one of those old builds w/ the bad config
726
- # TODO (PEX-433): Re-enable this check once we have refined the logic and reduced the false
727
- # positive
728
- # rate in risk/obseravble matching
729
- # TODO (PEX-433): I foresee issues here if for example a parent and child process share a
730
- # name (matched observable could be either) -> these issues are confirmed to exist, e.g.
731
- # `Windows Steal Authentication Certificates Export Certificate`
717
+ matched_observable = event.get_matched_observable(self.detection.tags.observable)
718
+ self.logger.debug(
719
+ f"Matched risk event (object={event.risk_object}, type={event.risk_object_type}) "
720
+ f"to observable (name={matched_observable.name}, type={matched_observable.type}, "
721
+ f"role={matched_observable.role}) using the source field "
722
+ f"'{event.source_field_name}'"
723
+ )
724
+ observable_counts[str(matched_observable)] += 1
725
+
726
+ # Report any observables which did not have at least one match to a risk event
727
+ for observable in observables:
728
+ self.logger.debug(
729
+ f"Matched observable (name={observable.name}, type={observable.type}, "
730
+ f"role={observable.role}) to {observable_counts[str(observable)]} risk events."
731
+ )
732
+ if observable_counts[str(observable)] == 0:
733
+ raise ValidationFailed(
734
+ f"Observable (name={observable.name}, type={observable.type}, "
735
+ f"role={observable.role}) was not matched to any risk events."
736
+ )
737
+
738
+ # TODO (#250): Re-enable and refactor code that validates the specific risk counts
732
739
  # Validate risk events in aggregate; we should have an equal amount of risk events for each
733
740
  # relevant observable, and the total count should match the total number of events
734
741
  # individual_count: Optional[int] = None
@@ -0,0 +1,71 @@
1
+ import uuid
2
+ from typing import Any
3
+
4
+ from pydantic import BaseModel, Field, field_validator
5
+
6
+
7
+ class DetectionMetadata(BaseModel):
8
+ """
9
+ A model of the metadata line in a detection stanza in savedsearches.conf
10
+ """
11
+ # A bool indicating whether the detection is deprecated (serialized as an int, 1 or 0)
12
+ deprecated: bool = Field(...)
13
+
14
+ # A UUID identifying the detection
15
+ detection_id: uuid.UUID = Field(...)
16
+
17
+ # The version of the detection
18
+ detection_version: int = Field(...)
19
+
20
+ # The time the detection was published. **NOTE** This field was added to the metadata in ESCU
21
+ # as of v4.39.0
22
+ publish_time: float = Field(...)
23
+
24
+ class Config:
25
+ # Allowing for future fields that may be added to the metadata JSON
26
+ extra = "allow"
27
+
28
+ @field_validator("deprecated", mode="before")
29
+ @classmethod
30
+ def validate_deprecated(cls, v: Any) -> Any:
31
+ """
32
+ Convert str to int, and then ints to bools for deprecated; raise if not 0 or 1 in the case
33
+ of an int, or if str cannot be converted to int.
34
+
35
+ :param v: the value passed
36
+ :type v: :class:`typing.Any`
37
+
38
+ :returns: the value
39
+ :rtype: :class:`typing.Any`
40
+ """
41
+ if isinstance(v, str):
42
+ try:
43
+ v = int(v)
44
+ except ValueError as e:
45
+ raise ValueError(f"Cannot convert str value ({v}) to int: {e}") from e
46
+ if isinstance(v, int):
47
+ if not (0 <= v <= 1):
48
+ raise ValueError(
49
+ f"Value for field 'deprecated' ({v}) must be 0 or 1, if not a bool."
50
+ )
51
+ v = bool(v)
52
+ return v
53
+
54
+ @field_validator("detection_version", mode="before")
55
+ @classmethod
56
+ def validate_detection_version(cls, v: Any) -> Any:
57
+ """
58
+ Convert str to int; raise if str cannot be converted to int.
59
+
60
+ :param v: the value passed
61
+ :type v: :class:`typing.Any`
62
+
63
+ :returns: the value
64
+ :rtype: :class:`typing.Any`
65
+ """
66
+ if isinstance(v, str):
67
+ try:
68
+ v = int(v)
69
+ except ValueError as e:
70
+ raise ValueError(f"Cannot convert str value ({v}) to int: {e}") from e
71
+ return v
@@ -0,0 +1,79 @@
1
+ from typing import ClassVar
2
+ import hashlib
3
+ from functools import cached_property
4
+
5
+ from pydantic import BaseModel, Field, computed_field
6
+
7
+ from contentctl.objects.detection_metadata import DetectionMetadata
8
+
9
+
10
+ class DetectionStanza(BaseModel):
11
+ """
12
+ A model representing a stanza for a detection in savedsearches.conf
13
+ """
14
+ # The lines that comprise this stanza, in the order they appear in the conf
15
+ lines: list[str] = Field(...)
16
+
17
+ # The full name of the detection (e.g. "ESCU - My Detection - Rule")
18
+ name: str = Field(...)
19
+
20
+ # The key prefix indicating the metadata attribute
21
+ METADATA_LINE_PREFIX: ClassVar[str] = "action.correlationsearch.metadata = "
22
+
23
+ @computed_field
24
+ @cached_property
25
+ def metadata(self) -> DetectionMetadata:
26
+ """
27
+ The metadata extracted from the stanza. Using the provided lines, parse out the metadata
28
+
29
+ :returns: the detection stanza's metadata
30
+ :rtype: :class:`contentctl.objects.detection_metadata.DetectionMetadata`
31
+ """
32
+ # Set a variable to store the metadata line in
33
+ meta_line: str | None = None
34
+
35
+ # Iterate over the lines to look for the metadata line
36
+ for line in self.lines:
37
+ if line.startswith(DetectionStanza.METADATA_LINE_PREFIX):
38
+ # If we find a matching line more than once, we've hit an error
39
+ if meta_line is not None:
40
+ raise Exception(
41
+ f"Metadata for detection '{self.name}' found twice in stanza."
42
+ )
43
+ meta_line = line
44
+
45
+ # Report if we could not find the metadata line
46
+ if meta_line is None:
47
+ raise Exception(f"No metadata for detection '{self.name}' found in stanza.")
48
+
49
+ # Parse the metadata JSON into a model
50
+ return DetectionMetadata.model_validate_json(meta_line[len(DetectionStanza.METADATA_LINE_PREFIX):])
51
+
52
+ @computed_field
53
+ @cached_property
54
+ def hash(self) -> str:
55
+ """
56
+ The SHA256 hash of the lines of the stanza, excluding the metadata line
57
+
58
+ :returns: hexdigest
59
+ :rtype: str
60
+ """
61
+ hash = hashlib.sha256()
62
+ for line in self.lines:
63
+ if not line.startswith(DetectionStanza.METADATA_LINE_PREFIX):
64
+ hash.update(line.encode("utf-8"))
65
+ return hash.hexdigest()
66
+
67
+ def version_should_be_bumped(self, previous: "DetectionStanza") -> bool:
68
+ """
69
+ A helper method that compares this stanza against the same stanza from a previous build;
70
+ returns True if the version still needs to be bumped (e.g. the detection was changed but
71
+ the version was not), False otherwise.
72
+
73
+ :param previous: the previous build's DetectionStanza for comparison
74
+ :type previous: :class:`contentctl.objects.detection_stanza.DetectionStanza`
75
+
76
+ :returns: True if the version still needs to be bumped
77
+ :rtype: bool
78
+ """
79
+ return (self.hash != previous.hash) and (self.metadata.detection_version <= previous.metadata.detection_version)