contentctl 5.0.5__py3-none-any.whl → 5.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
contentctl/contentctl.py CHANGED
@@ -143,20 +143,6 @@ def test_common_func(config: test_common):
143
143
  raise Exception("There was at least one unsuccessful test")
144
144
 
145
145
 
146
- CONTENTCTL_5_WARNING = """
147
- *****************************************************************************
148
- WARNING - THIS IS AN ALPHA BUILD OF CONTENTCTL 5.
149
- THERE HAVE BEEN NUMEROUS CHANGES IN CONTENTCTL (ESPECIALLY TO YML FORMATS).
150
- YOU ALMOST CERTAINLY DO NOT WANT TO USE THIS BUILD.
151
- IF YOU ENCOUNTER ERRORS, PLEASE USE THE LATEST CURRENTYLY SUPPORTED RELEASE:
152
-
153
- CONTENTCTL==4.4.7
154
-
155
- YOU HAVE BEEN WARNED!
156
- *****************************************************************************
157
- """
158
-
159
-
160
146
  def get_random_compliment():
161
147
  compliments = [
162
148
  "Your detection rules are like a zero-day shield! 🛡️",
@@ -187,7 +173,6 @@ class RecognizeCommand:
187
173
 
188
174
 
189
175
  def main():
190
- print(CONTENTCTL_5_WARNING)
191
176
  try:
192
177
  configFile = pathlib.Path("contentctl.yml")
193
178
 
@@ -299,7 +284,6 @@ def main():
299
284
  )
300
285
 
301
286
  print(e)
302
- print(CONTENTCTL_5_WARNING)
303
287
  sys.exit(1)
304
288
 
305
289
 
@@ -1,30 +1,29 @@
1
1
  import os
2
2
  import sys
3
- from pathlib import Path
4
3
  from dataclasses import dataclass, field
5
- from pydantic import ValidationError
4
+ from pathlib import Path
6
5
  from uuid import UUID
7
- from contentctl.input.yml_reader import YmlReader
8
6
 
9
- from contentctl.objects.detection import Detection
10
- from contentctl.objects.story import Story
7
+ from pydantic import ValidationError
11
8
 
12
- from contentctl.objects.baseline import Baseline
13
- from contentctl.objects.investigation import Investigation
14
- from contentctl.objects.playbook import Playbook
15
- from contentctl.objects.deployment import Deployment
16
- from contentctl.objects.macro import Macro
17
- from contentctl.objects.lookup import LookupAdapter, Lookup
18
- from contentctl.objects.atomic import AtomicEnrichment
19
- from contentctl.objects.security_content_object import SecurityContentObject
20
- from contentctl.objects.data_source import DataSource
21
- from contentctl.objects.dashboard import Dashboard
22
9
  from contentctl.enrichments.attack_enrichment import AttackEnrichment
23
10
  from contentctl.enrichments.cve_enrichment import CveEnrichment
24
-
11
+ from contentctl.helper.utils import Utils
12
+ from contentctl.input.yml_reader import YmlReader
13
+ from contentctl.objects.atomic import AtomicEnrichment
14
+ from contentctl.objects.baseline import Baseline
25
15
  from contentctl.objects.config import validate
16
+ from contentctl.objects.dashboard import Dashboard
17
+ from contentctl.objects.data_source import DataSource
18
+ from contentctl.objects.deployment import Deployment
19
+ from contentctl.objects.detection import Detection
26
20
  from contentctl.objects.enums import SecurityContentType
27
- from contentctl.helper.utils import Utils
21
+ from contentctl.objects.investigation import Investigation
22
+ from contentctl.objects.lookup import Lookup, LookupAdapter
23
+ from contentctl.objects.macro import Macro
24
+ from contentctl.objects.playbook import Playbook
25
+ from contentctl.objects.security_content_object import SecurityContentObject
26
+ from contentctl.objects.story import Story
28
27
 
29
28
 
30
29
  @dataclass
@@ -113,20 +112,6 @@ class Director:
113
112
  self.createSecurityContent(SecurityContentType.detections)
114
113
  self.createSecurityContent(SecurityContentType.dashboards)
115
114
 
116
- from contentctl.objects.abstract_security_content_objects.detection_abstract import (
117
- MISSING_SOURCES,
118
- )
119
-
120
- if len(MISSING_SOURCES) > 0:
121
- missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES)))
122
- print(
123
- "WARNING: The following data_sources have been used in detections, but are not yet defined.\n"
124
- "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 "
125
- f"{missing_sources_string}"
126
- )
127
- else:
128
- print("No missing data_sources!")
129
-
130
115
  def createSecurityContent(self, contentType: SecurityContentType) -> None:
131
116
  if contentType in [
132
117
  SecurityContentType.deployments,
@@ -51,8 +51,6 @@ from contentctl.objects.security_content_object import SecurityContentObject
51
51
  from contentctl.objects.test_group import TestGroup
52
52
  from contentctl.objects.unit_test import UnitTest
53
53
 
54
- MISSING_SOURCES: set[str] = set()
55
-
56
54
  # Those AnalyticsTypes that we do not test via contentctl
57
55
  SKIPPED_ANALYTICS_TYPES: set[str] = {AnalyticsType.Correlation}
58
56
 
@@ -514,7 +512,7 @@ class Detection_Abstract(SecurityContentObject):
514
512
  baseline.tags.detections = new_detections
515
513
 
516
514
  # Data source may be defined 1 on each line, OR they may be defined as
517
- # SOUCE_1 AND ANOTHERSOURCE AND A_THIRD_SOURCE
515
+ # SOURCE_1 AND ANOTHERSOURCE AND A_THIRD_SOURCE
518
516
  # if more than 1 data source is required for a detection (for example, because it includes a join)
519
517
  # Parse and update the list to resolve individual names and remove potential duplicates
520
518
  updated_data_source_names: set[str] = set()
@@ -524,27 +522,9 @@ class Detection_Abstract(SecurityContentObject):
524
522
  updated_data_source_names.update(split_data_sources)
525
523
 
526
524
  sources = sorted(list(updated_data_source_names))
527
-
528
- matched_data_sources: list[DataSource] = []
529
- missing_sources: list[str] = []
530
- for source in sources:
531
- try:
532
- matched_data_sources += DataSource.mapNamesToSecurityContentObjects(
533
- [source], director
534
- )
535
- except Exception:
536
- # We gobble this up and add it to a global set so that we
537
- # can print it ONCE at the end of the build of datasources.
538
- # This will be removed later as per the note below
539
- MISSING_SOURCES.add(source)
540
-
541
- if len(missing_sources) > 0:
542
- # This will be changed to ValueError when we have a complete list of data sources
543
- print(
544
- "WARNING: The following exception occurred when mapping the data_source field to "
545
- f"DataSource objects:{missing_sources}"
546
- )
547
-
525
+ matched_data_sources = DataSource.mapNamesToSecurityContentObjects(
526
+ sources, director
527
+ )
548
528
  self.data_source_objects = matched_data_sources
549
529
 
550
530
  for story in self.tags.analytic_story:
@@ -0,0 +1,74 @@
1
+ Metadata-Version: 2.3
2
+ Name: contentctl
3
+ Version: 5.1.0
4
+ Summary: Splunk Content Control Tool
5
+ License: Apache 2.0
6
+ Author: STRT
7
+ Author-email: research@splunk.com
8
+ Requires-Python: >=3.11,<3.14
9
+ Classifier: License :: Other/Proprietary License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
15
+ Requires-Dist: PyYAML (>=6.0.2,<7.0.0)
16
+ Requires-Dist: attackcti (>=0.4.0,<0.5.0)
17
+ Requires-Dist: bottle (>=0.12.25,<0.14.0)
18
+ Requires-Dist: docker (>=7.1.0,<8.0.0)
19
+ Requires-Dist: gitpython (>=3.1.43,<4.0.0)
20
+ Requires-Dist: pycvesearch (>=1.2,<2.0)
21
+ Requires-Dist: pydantic (>=2.9.2,<2.10.0)
22
+ Requires-Dist: pygit2 (>=1.15.1,<2.0.0)
23
+ Requires-Dist: questionary (>=2.0.1,<3.0.0)
24
+ Requires-Dist: requests (>=2.32.3,<2.33.0)
25
+ Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
26
+ Requires-Dist: setuptools (>=69.5.1,<76.0.0)
27
+ Requires-Dist: splunk-sdk (>=2.0.2,<3.0.0)
28
+ Requires-Dist: tqdm (>=4.66.5,<5.0.0)
29
+ Requires-Dist: tyro (>=0.9.2,<0.10.0)
30
+ Requires-Dist: xmltodict (>=0.13,<0.15)
31
+ Description-Content-Type: text/markdown
32
+
33
+ # contentctl
34
+ <p align="center">
35
+ <img src="https://raw.githubusercontent.com/splunk/contentctl/refs/heads/main/docs/contentctl_logo_white.png" title="In case you're wondering, it's a capybara" alt="the logo for the contentctl project, which depicts a doodled 4 legged animal that is supposed to represent a capybara, with the name of the project below it" width="250" height="250"></p>
36
+
37
+ > [!NOTE]
38
+ > Looking to migrate from an earlier release to the new contentctl v5+ ? Check out our migration guide [here](docs/contentctl_v5_migration_guide.md).
39
+
40
+ ## What is contentctl?
41
+ `contentctl` is a tool developed by the Splunk Threat Research Team to help with managing the content living in [splunk/security_content](https://github.com/splunk/security_content) and producing the Enterprise Security Content Update app for Splunk. While its development is largely driven by STRT's needs, it has been somewhat genericized and can be used by customers and partners to package their own content. Simply put, `contentctl` is the workhorse that packages detections, macros, lookups, dashboards into a Splunk app that you can use, and that understands the YAML structure and project layout we've selected to keep development clean.
42
+
43
+ ## Quick Start Guide
44
+ Check out our [User Guide](docs/UserGuide.md) to get started!
45
+
46
+ ## Content Testing
47
+ Read more about how `contentctl` can help test and validate your content in a real Splunk instance [here](docs/ContentTestingGuide.md).
48
+
49
+ ## Sample CICD Workflows
50
+ Already using `contentctl`, or looking to get started with it already configured in GitHub Actions? [Our guide](docs/Sample_CICD_Templates.md) includes workflows to help you build and test your app.
51
+
52
+ ## Contribution Guide
53
+ Read [the Contribution Guidelines](CONTRIBUTING.md) for this project before opening a Pull Request.
54
+
55
+ ## Ecosystem
56
+ | Project | Description |
57
+ | --------------------- | ------------------------------------------------------- |
58
+ | [Splunk Security Content](https://github.com/splunk/security_content) | Splunk Threat Research Team's Content included in the [Enterprise Security Content Update App (ESCU)](https://splunkbase.splunk.com/app/3449)|
59
+ | [Splunk Attack Range](https://github.com/splunk/attack_range) | Easily deploy a preconfigured Splunk Environment locally or on AWS containing a Splunk Instance, Windows and Linux Machines, and Attacker Tools like Kali Linux. Automatically simulate attacks or run your own|
60
+ | [Splunk Attack Data](https://github.com/splunk/attack_data) | Repository of Attack Simulation Data for writing and Testing Detections| |
61
+ | [Splunk contentctl](https://github.com/splunk/contentctl) | Generate, validate, build, test, and deploy custom Security Content|
62
+ | [SigmaHQ Sigma Rules](https://github.com/SigmaHQ/sigma) | Official Repository for Sigma Rules. These rules are an excellent starting point for new content. |
63
+ | [PurpleSharp Attack Simulation](https://github.com/mvelazc0/PurpleSharp) | Open source adversary simulation tool for Windows Active Directory environments (integrated into Attack Range)|
64
+ | [Red Canary Atomic Red Team](https://github.com/redcanaryco/atomic-red-team) | Library of attack simulations mapped to the MITRE ATT&CK® framework (integrated into Attack Range)|
65
+
66
+ ## License
67
+ Copyright 2023 Splunk Inc.
68
+
69
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
70
+
71
+ http://www.apache.org/licenses/LICENSE-2.0
72
+
73
+ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
74
+
@@ -21,7 +21,7 @@ contentctl/actions/reporting.py,sha256=GF32i7sHdc47bw-VWSW-nZ1QBaUl6Ni1JjV5_SOyi
21
21
  contentctl/actions/test.py,sha256=GTtvHi1yB5yDm1jPMyuc4WxczOq-O7f2N8LpTmMaWgU,6014
22
22
  contentctl/actions/validate.py,sha256=thnxanLj6mfw5g17C1FrzWlkpyIT_AjnDxv_wyNdspA,5837
23
23
  contentctl/api.py,sha256=6s17vNOW1E1EzQqOCXAa5uWuhwwShu-JkGSgrsOFEMs,6329
24
- contentctl/contentctl.py,sha256=pKtjRHsdTHb505Xjvrdx8v_EGJQ9ZAMfKK4P5N4ge-I,12312
24
+ contentctl/contentctl.py,sha256=nR8nHxXY0elvQogVHFqsyid7Ch5sMnIiNAOFbCa0yzI,11755
25
25
  contentctl/enrichments/attack_enrichment.py,sha256=68C9xQ8Q3YX-luRdK2hLnwWtRFpheFA2kE4v5GOLGEo,6358
26
26
  contentctl/enrichments/cve_enrichment.py,sha256=TsZ52ef2njt19lPf_VyclY_-5Z5iQ1boVOAxFbjGdSQ,2431
27
27
  contentctl/enrichments/splunk_app_enrichment.py,sha256=Xynxjjkqlw0_RtQ1thGSFwy1I3HdmPAJmNKZezyqniU,3419
@@ -29,10 +29,10 @@ contentctl/helper/link_validator.py,sha256=kzEi2GdncPWSi-UKNerXm2jtTJfFQ5goS9pqy
29
29
  contentctl/helper/logger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
30
  contentctl/helper/splunk_app.py,sha256=Zq_C9rjNVqCjBNgm-5CWdBpXyeX5jSpbE-QTGptEZlk,14571
31
31
  contentctl/helper/utils.py,sha256=rigwZzCwWzn11sKTVWDkYEtLmRSf0yBbJ671OSRQnOM,19094
32
- contentctl/input/director.py,sha256=asK4yUlSVdv0QDUzCrTEXQUm0j9hbzVu55o_-wD-eWc,11560
32
+ contentctl/input/director.py,sha256=7mx93-k_8DUc4pwYjxwq39n8-BlDYZOvRBkdzdGA8Qc,10919
33
33
  contentctl/input/new_content_questions.py,sha256=z2C4Mg7-EyxtiF2z9m4SnSbi6QO4CUPB3wg__JeMXIQ,4067
34
34
  contentctl/input/yml_reader.py,sha256=ymmAqsWsf9Oj56waDOhCh_E4SomkSCmu4dAx7iURFt8,2050
35
- contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=9uUb3BkH9d5vc9qn7RURhki9HZZJSGBTYfnqMImsiY4,43814
35
+ contentctl/objects/abstract_security_content_objects/detection_abstract.py,sha256=T5SAFOwEeCDCe79IyVryoLO9k6VaTiwbQPWSNo1epk0,43001
36
36
  contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py,sha256=P1v5n8hM4XzJOjNZbJ5m3y4Bu-cQYaddLPdbE6K-4Xw,12164
37
37
  contentctl/objects/alert_action.py,sha256=iEvdEOT4TrTXT0z4rQ_W5v79hPJpPhFPSzo7TuHDxwA,1376
38
38
  contentctl/objects/annotated_types.py,sha256=xR4EKvdOpNDEt0doGs8XjxCzKK99J2NHZgHFAmt7p2c,424
@@ -161,8 +161,8 @@ contentctl/templates/detections/web/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
161
161
  contentctl/templates/macros/security_content_ctime.yml,sha256=Gg1YNllHVsX_YB716H1SJLWzxXZEfuJlnsgB2fuyoHU,159
162
162
  contentctl/templates/macros/security_content_summariesonly.yml,sha256=9BYUxAl2E4Nwh8K19F3AJS8Ka7ceO6ZDBjFiO3l3LY0,162
163
163
  contentctl/templates/stories/cobalt_strike.yml,sha256=uj8idtDNOAIqpZ9p8usQg6mop1CQkJ5TlB4Q7CJdTIE,3082
164
- contentctl-5.0.5.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
165
- contentctl-5.0.5.dist-info/METADATA,sha256=lOd3L_c_TdnKj_63UvFyAFgNgf8cZ52BxiJ2Z1GB7PI,21539
166
- contentctl-5.0.5.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
167
- contentctl-5.0.5.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
168
- contentctl-5.0.5.dist-info/RECORD,,
164
+ contentctl-5.1.0.dist-info/LICENSE.md,sha256=hQWUayRk-pAiOZbZnuy8djmoZkjKBx8MrCFpW-JiOgo,11344
165
+ contentctl-5.1.0.dist-info/METADATA,sha256=F4a_TCVzB6ayCKCjSNSaX0cPIaZl12-DoLClq1xliNc,5097
166
+ contentctl-5.1.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
167
+ contentctl-5.1.0.dist-info/entry_points.txt,sha256=5bjZ2NkbQfSwK47uOnA77yCtjgXhvgxnmCQiynRF_-U,57
168
+ contentctl-5.1.0.dist-info/RECORD,,
@@ -1,263 +0,0 @@
1
- Metadata-Version: 2.3
2
- Name: contentctl
3
- Version: 5.0.5
4
- Summary: Splunk Content Control Tool
5
- License: Apache 2.0
6
- Author: STRT
7
- Author-email: research@splunk.com
8
- Requires-Python: >=3.11,<3.14
9
- Classifier: License :: Other/Proprietary License
10
- Classifier: Programming Language :: Python :: 3
11
- Classifier: Programming Language :: Python :: 3.11
12
- Classifier: Programming Language :: Python :: 3.12
13
- Classifier: Programming Language :: Python :: 3.13
14
- Requires-Dist: Jinja2 (>=3.1.4,<4.0.0)
15
- Requires-Dist: PyYAML (>=6.0.2,<7.0.0)
16
- Requires-Dist: attackcti (>=0.4.0,<0.5.0)
17
- Requires-Dist: bottle (>=0.12.25,<0.14.0)
18
- Requires-Dist: docker (>=7.1.0,<8.0.0)
19
- Requires-Dist: gitpython (>=3.1.43,<4.0.0)
20
- Requires-Dist: pycvesearch (>=1.2,<2.0)
21
- Requires-Dist: pydantic (>=2.9.2,<2.10.0)
22
- Requires-Dist: pygit2 (>=1.15.1,<2.0.0)
23
- Requires-Dist: questionary (>=2.0.1,<3.0.0)
24
- Requires-Dist: requests (>=2.32.3,<2.33.0)
25
- Requires-Dist: semantic-version (>=2.10.0,<3.0.0)
26
- Requires-Dist: setuptools (>=69.5.1,<76.0.0)
27
- Requires-Dist: splunk-sdk (>=2.0.2,<3.0.0)
28
- Requires-Dist: tqdm (>=4.66.5,<5.0.0)
29
- Requires-Dist: tyro (>=0.9.2,<0.10.0)
30
- Requires-Dist: xmltodict (>=0.13,<0.15)
31
- Description-Content-Type: text/markdown
32
-
33
-
34
- # contentctl
35
- <p align="center">
36
- <img src="docs/contentctl_logo_white.png" title="In case you're wondering, it's a capybara" alt="contentctl logo" width="250" height="250"></p>
37
-
38
- # contentctl Quick Start Guide
39
- If you are already familiar with contentctl, the following common commands may be very useful for basic operations
40
-
41
- | Operation | Command |
42
- |-----------|---------|
43
- | Create a repository | `contentctl init` |
44
- | Validate Your Content | `contentctl validate` |
45
- | Validate Your Content, performing MITRE Enrichments | `contentctl validate --enrichments`|
46
- | Build Your App | `contentctl build` |
47
- | Test All the content in your app, pausing so that you can debug a search if it fails | `contentctl test --post-test-behavior pause_on_failure mode:all` |
48
- | Test All the content in your app, pausing after every detection to allow debugging | `contentctl test --post-test-behavior always_pause mode:all` |
49
- | Test 1 or more specified detections. If you are testing more than one detection, the paths are space-separated. You may also use shell-expanded regexes | `contentctl test --post-test-behavior always_pause mode:selected --mode.files detections/endpoint/7zip_commandline_to_smb_share_path.yml detections/cloud/aws_multi_factor_authentication_disabled.yml detections/application/okta*` |
50
- | Diff your current branch with a target_branch and test detections that have been updated. Your current branch **must be DIFFERENT** than the target_branch | `contentctl test --post-test-behavior always_pause mode:changes --mode.target_branch develop` |
51
- | Perform Integration Testing of all content. Note that Enterprise Security MUST be listed as an app in your contentctl.yml folder, otherwise all tests will subsequently fail | `contentctl test --enable-integration-testing --post-test-behavior never_pause mode:all` |
52
-
53
- # Introduction
54
- #### Security Is Hard
55
- Anyone who has managed a [SOC](#acronyms) will tell you it's hard work. SOC Managers, Detection Engineers, and Cybersecurity Professionals must understand and manage countless tools and data sources while ensuring the reliability and security of their network and applications. At the same time, they must be able to detect and react to data breeches, vulnerabilities, and performance degradation in minutes - not seconds or hours.
56
- These responsibilities leave little time for writing new content - let alone documentation, maintenance, and testing of legacy content. Existing solutions like Wikis, JIRA tickets, Excel Spreadsheets, and "asking that one team member who knows everything" don't scale and spread information across various systems.
57
- #### contentctl Makes It ~~Easy~~ Less Hard
58
- contentctl is a single application that support the full cycle of security content development (each of the links below will redirect to the appropriate section of the README/Wiki):
59
-
60
- - [Create Content Packs](#contentctl-init) - Version-Controlled collections of searches, documentation, and test data
61
- - [Add New Content](#contentctl-new---type-type) - searches, macros, lookups, and other content
62
- - [Statically Validate](#contentctl-validate) Content Against a Well-Defined Baseline
63
- - [Building](#contentctl-build) a Content Pack for your target - Use a supported output format or design your own
64
- - [Test](#contentctl-test) your Content Pack on a running Splunk Instance
65
- - [Deploy](#contentctl-deploy) your Content Pack to Splunk Cloud Instance, via REST API, or create a Splunkbase Application for manual deployment
66
-
67
-
68
- # Ecosystem
69
- | Project | Description |
70
- | --------------------- | ------------------------------------------------------- |
71
- | [Splunk Attack Range](https://github.com/splunk/attack_range) | Easily deploy a preconfigured Splunk Environment locally or on AWS containing a Splunk Instance, Windows and Linux Machines, and Attacker Tools like Kali Linux. Automatically simulate attacks or run your own|
72
- | [PurpleSharp Attack Simulation](https://github.com/mvelazc0/PurpleSharp) | Open source adversary simulation tool for Windows Active Directory environments (integrated into Attack Range)|
73
- | [Red Canary Atomic Red Team](https://github.com/redcanaryco/atomic-red-team) | Library of attack simulations mapped to the MITRE ATT&CK® framework (integrated into Attack Range)|
74
- | [Splunk Attack Data](https://github.com/splunk/attack_data) | Repository of Attack Simulation Data for writing and Testing Detections| |
75
- | [Splunk Security Content](https://github.com/splunk/security_content) | Splunk Threat Research Team's Content included in the [Enterprise Security Content Update App (ESCU)](https://splunkbase.splunk.com/app/3449)|
76
- | [Splunk contentctl](https://github.com/splunk/contentctl) | Generate, validate, build, test, and deploy custom Security Content|
77
- | [SigmaHQ Sigma Rules](https://github.com/SigmaHQ/sigma) | Official Repository for Sigma Rules. These rules are an excellent starting point for new content. |
78
-
79
-
80
-
81
-
82
- ## Workflow
83
- ```mermaid
84
- graph TD
85
- starting_state --contentctl init--> contentPack
86
- newContent{new_content} --contentctl new sigma content--> contentPack
87
- newContent{new_content} --contentctl new spl content--> contentPack
88
- contentPack --contentctl validate--> ValidatedContent
89
- ValidatedContent --contentctl build--> SplunkContent
90
- ValidatedContent --contentctl docs--> WebsiteDocumentation
91
- ValidatedContent --contentctl docs--> MITRE_mapDocumentation
92
- SplunkContent --contentctl test via container-->TestResults
93
- TestResults --contentctl deploy-->SplunkRESTAPI
94
- TestResults --contentctl deploy-->SplunkbaseApp
95
- TestResults --contentctl deploy-->SplunkCloudInstance
96
- ```
97
-
98
- ## Installation
99
- ### Requirements
100
- Testing is run using [GitHub Hosted Runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). Ubuntu22.04 is the recommended configuration.
101
- | Operating System | Supported | Passing Intregration Tests |
102
- | ---------------- | --------- | ------ |
103
- | Windows 10 | Yes | Not tested yet |
104
- | Windows 11 | Yes | Not tested yet |
105
- | Ununtu 20.04 LTS | Yes | YES (locally + GitHub Actions) |
106
- | *Ubuntu 22.04 LTS| Yes | YES (locally + GitHub Actions) |
107
- | macOS Montery 12 | Yes | Not tested yet |
108
- | macOS Ventura 13 | Yes (Intel and Apple Silicon) | Yes (locally) |
109
-
110
- | Requirement | Supported | Description | Passing Integration Tests |
111
- | --------------------- | ----- | ---- | ------ |
112
- | Python 3.11+ | Yes | contentctl tool is written in Python | Yes (locally + GitHub Actions) |
113
- | Docker (local) | Yes | A running Splunk Server is required for Dynamic Testing. contentctl can automatically create, configure, and destroy this server as a Splunk container during the lifetime of a test. | (locally + GitHub Actions) |
114
- | Docker (remote) | Planned | A running Splunk Server is required for Dynamic Testing. contentctl can automatically create, configure, and destroy this server as a Splunk container during the lifetime of a test. | No |
115
-
116
-
117
-
118
- **Because contentctl uses a large number of dependencies, it's recommended to install contentctl in a virtual environment.
119
- The instructions below using "poetry" ALSO follow this recommendation. Poetry will automatically create a new virtual environment for you.
120
- It is typically recommended to install poetry to the Global Python Environment.**
121
-
122
- #### Install via pip (recommended):
123
- ```
124
- python3.11 -m venv .venv
125
- source .venv/bin/activate
126
- pip install contentctl
127
- ```
128
-
129
- #### From Source (advanced)
130
- ```
131
- git clone https://github.com/splunk/contentctl
132
- cd contentctl
133
- python3.11 -m pip install poetry
134
- poetry install
135
- poetry shell
136
- contentctl --help
137
- ```
138
-
139
-
140
- # Usage
141
-
142
- ### contentctl init
143
- Creates a new Content Pack in the current directory as well as a configuration file called [contentctl.yml](/contentctl/templates/contentctl_default.yml) which contains a number of important configuration options.
144
- The content pack contains a wide variety of content types:
145
- - [detections](/contentctl/objects/detection.py) - A piece of content that wraps and enriches a Splunk Search. [Example Detection](/contentctl/templates/detections/anomalous_usage_of_7zip.yml)
146
- - [baselines](/contentctl/objects/baseline.py) - This content is not currently supported.
147
- - [lookups](/contentctl//objects/lookup.py) - Static files, such as CSVs, that can be loaded into Splunk for use in lookup commands. [Example Lookup](https://github.com/splunk/security_content/blob/develop/lookups/attacker_tools.csv)
148
- - [macros](/contentctl/objects/macro.py) - Common code that is re-used across a wide set of detections. [Example Macro](/contentctl/templates/macros/security_content_summariesonly.yml)
149
- - [stories](/contentctl/objects/story.py) - Analytic Stories Allow a developer to tie together a group of detections, for example detections that all relate to Ransomware. [Example Analytic Story](/contentctl/templates/stories/cobalt_strike.yml)
150
-
151
- ### contentctl validate
152
- Performs static validation on all of the content in this Content Pack. Writing validation is extremely complex. Each piece of content can one or dozens of fields ranging from free text to numbers to references to other content. contentctl's build in validation ensures that a number of conditions are met:
153
- - Required fields in content are defined
154
- - Values in fields are appropriate. For example, if content references a Datamodel, then that Datamodel must be valid. Similarly, numeric values that must fall within a certain range must be validated.
155
- - If a piece of content references other content, then it exists. For example, if a detection makes use of a macro then the existence and correctness of the macro must be validated.
156
- - There are no conflicts between content. For example, two detections cannot have the same name.
157
- If any of these conditions are not met, then a descriptive error will be printed and contentctl will return a nonzero exit code. This makes it suitable for running in an automated workflow or other CICD context. Instead of exiting on the first error, _contentctl validate_ will continue validating all content so that it can present all relevant error before exiting.
158
-
159
- ### contentctl build
160
- Generates Content Packs in the output format defined in the [contentctl.yml](/contentctl/templates/contentctl_default.yml) configuration file. These outputs may include {splunk_app, api, ba}. When _contentctl build_ runs, it first performs a _contentctl validate_ in order to ensure that a valid app is generated. Note that it is *NOT* required or recommended to run _contentctl validate_ separately if the intention is to build a Content Pack.
161
-
162
- ### contentctl report
163
- This section is under active development. It will allow you to a [MITRE Map](https://mitremap.splunkresearch.com/) showing your coverage of MITRE Techniques. The link here shows the generation of that MITRE Map using the content in Splunk's [security_content repo](https://github.com/splunk/security_content).
164
-
165
- ### contentctl new [--type TYPE]
166
- Choose TYPE {detection, story} to create new content for the Content Pack. The tool will interactively ask a series of questions required for generating a basic piece of content and automatically add it to the Content Pack.
167
-
168
- ### contentctl inspect
169
- This section is under development. The inspect action performs a number of post-build validations. Primarily, it will enable the user to perform an appinspect of the content pack in preparation for deployment onto a Splunk Instance or via Splunk Cloud. It also compares detections in the new build against a prior build, confirming that any changed detections have had their versions incremented (this comparison happens at the savedsearch.conf level, which is why it must happen after the build). Please also note that new versions of contentctl may result in the generation of different savedsearches.conf files without any content changes in YML (new keys at the .conf level which will necessitate bumping of the version in the YML file).
170
-
171
- ### contentctl deploy
172
- The reason to build content is so that it can be deployed to your environment. However, deploying content to multiple servers and different types of infrastructure can be tricky and time-consuming. contentctl makes this easy by supporting a number of different deployment mechanisms. Deployment targets can be defined in [contentctl.yml](/contentctl/templates/contentctl_default.yml).
173
- - Deploy via API - Using the REST API, individual pieces of content are deployed to a running server. This is a great way to deploy all of the content in a content pack, but can also be used to deploy individual peices of content.
174
- - Deploy to Splunk Cloud via ACS - Using the Automated Private App Vetting (APAV) Feature of the Admin Config Service (ACS), Splunk Cloud customers can easily deploy custom apps to their environments.
175
-
176
- ### contentctl docs
177
- This section is under active development. It will allow you to easily generate verbose [web-based documentation](https://research.splunk.com) from all of your content. The link here shows the generation of that documentation using the content in Splunk's [security_content repo](https://github.com/splunk/security_content).
178
-
179
- ### contentctl test
180
- The static validation performed by *contentctl validate* can only take you so far. While it's powerful, and fast, it can only tell determine if the content is *syntactically* correct. *contentctl test* can test your content on real Splunk Infrastructure to ensure there are no errors in your SPL, raw data can be properly ingested/processed/accelerated on your server, your search finds the event(s) you're looking for in raw data, and even provides high-level runtime performance metrics about your searches. As part of this workflow, a large number of apps will be installed by default. You can see, and edit, the list of apps in the [app_default.yml](/contentctl/templates/app_default.yml) file. We plan on making this even easier to update in the future, so stay tuned for updates.
181
- The following diagram shows this workflow at a high level.
182
-
183
- ```mermaid
184
- graph TD
185
- SplunkContainer--start-->SplunkInstance
186
- SplunkServer-->SplunkInstance
187
- SplunkInstance--Install Apps-->ConfiguredInstance
188
- SplunkInstance--Configure Settings-->ConfiguredInstance
189
- ContentPack--Deploy-->ConfiguredInstance
190
- ConfiguredInstance--Start Test-->Testing
191
- Testing--Test Detection-->Detection
192
-
193
- Detection--Replay Raw Data-->Result
194
- subgraph Test Indidivual Detection
195
- Detection--Execute Search-->Result
196
- end
197
- Result--Cleanup-->Testing
198
- Testing--Cleanup-->TestResults
199
- ```
200
-
201
- #### Testing Modes
202
- There are a number of different testing modes that control which content will be tested. This can be controlled with the `--mode {}` option at the command line
203
- - all - This will test all of the content in the content pack. Please note that by default only detections marked production will be tested (detections marked as deprecated or experimental, for example, will be ignored).
204
- - selected - Detections whose relative paths are provided on the command line will be tested. This is particularly useful if you would like to troubleshoot or update just a handful of detections and can save a significant amount of time. For example, to test two detections use the following command `contentctl test --mode selected --detections_list detections/detection_one.yml detections/detection_two.yml`
205
- - **changes - (not currently supported) If you have a large number of detections and use a branching workflow to create new content or update content, then this is an easy way to automatically find and test only that content automatically. This prevents you from needing to explicitly list the subset of content to test using "selected"**
206
-
207
- #### Testing Behavior
208
- contentctl test's default mode allows it to quickly test all content with requiring user interaction. This makes it suitable for local tests as well as CI/CD workflows. However, users often want to troubleshoot a test if it fails. contentctl allows you to change the tool's behavior if and/or when a test fails:
209
- - --behavior never_pause - The default behavior. If a test does not pass, the tool begins the next test immediately
210
- - --behavior pause_on_failure - If a test fails, then additional information about that test, and the raw SPL of the test, is printed to the terminal. A user may then click (or CMD+Click) the "LINK" to begin interactively debugging the failed test on the Splunk Server. Note that the credentials for the server are printed out at the very beginning of the test. After you have finished debugging the failure, hit "Return" in the terminal to move on to the next test. The attack_data for this test remains loaded on the server for debugging until the user moves on to the next test.
211
- - --behavior always_pause - Similar to pause_on_failure, but this pauses after every test regardless of whether it passes or fails.
212
-
213
-
214
- 1. **init** - Initilialize a new repo from scratch so you can easily add your own content to a custom application.
215
- 2. **new** - Creates new content (detection, story)
216
- 3. **validate** - Validates written content
217
- 4. **build** - Builds an application suitable for deployment on a search head using Slim, the Splunk Packaging Toolkit
218
- 5. **deploy** - Deploy the security content pack to a Splunk Server
219
- 6. **docs** - Create documentation as Markdown
220
- 7. **reporting** - Create different reporting files such as a Mitre ATT&CK overlay
221
-
222
- # Shell tab-complete
223
-
224
- Leveraging the tab completion featureset of the CLI library we're using, you can generate tab completions for `contentctl` automatically, for zsh, bash, and tcsh. For additional details, you can view the docs for the library [here.](https://brentyi.github.io/tyro/tab_completion/)
225
-
226
- ### Zsh
227
- If you already have a location for your ZSH tab completions, you only need to run the generation line and can skip the folder creation, configuring the rest to fit with your shell config.
228
-
229
- ```zsh
230
- mkdir -p ~/.zfunc
231
- contentctl --tyro-write-completion zsh ~/.zfunc/_contentctl
232
- echo "fpath+=~/.zfunc" >> ~/.zshrc
233
- echo "autoload -Uz compinit && compinit" >> ~/.zshrc
234
- source ~/.zshrc
235
- ```
236
-
237
- ### Bash
238
-
239
- ```bash
240
- completion_dir=${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/
241
- mkdir -p $completion_dir
242
- contentctl --tyro-write-completion bash ${completion_dir}/_contentctl
243
- ```
244
-
245
- # Acronyms
246
- | Acronym | Meaning| Description |
247
- | --------------------- | ------------------------------------------------------- | ---- |
248
- | SOC | Security Operation Center | Description of a SoC |
249
- | DaC | Detection as Code | A systematic approach applying DevOps priciples to Detection Engineering. DaC enables Continuous Integration and Continuous Delivery of Detectionsa via automated validation, testing, and deployment |
250
- | CICD | Continuous Integration/Continuous Delivery | A modern DevOps practice that encourages users to make small, frequent changes which are automatically tested and deployed. This contrasts with legacy approaches that emphasize large changes which may be manually tested an infrequently deployed. |
251
- | ACS| Admin Config Service | The [Splunk Admin Config Service](https://docs.splunk.com/Documentation/SplunkCloud/9.0.2209/Config/ACSIntro) is a cloud-native API that provides programmatic self-service administration capabilities for Splunk Cloud Platform. One of its features, Automated Private App Vetting (APAV) enables the installation of custom app on Splunk Cloud instances. |
252
- | APAV | Automated Private App Vetting | [Automated Private App Vetting](https://docs.splunk.com/Documentation/SplunkCloud/9.0.2209/Config/ManageApps) enables admins to use the ACS API to install, upgrade, and uninstall apps directly on your Splunk Cloud Platform deployment without assistance from Splunk Support. |
253
-
254
-
255
- # License
256
- Copyright 2023 Splunk Inc.
257
-
258
- Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
259
-
260
- http://www.apache.org/licenses/LICENSE-2.0
261
-
262
- Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
263
-