contentctl 3.6.0__py3-none-any.whl → 4.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/actions/build.py +89 -0
- contentctl/actions/detection_testing/DetectionTestingManager.py +48 -49
- contentctl/actions/detection_testing/GitService.py +148 -230
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +14 -24
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +43 -17
- contentctl/actions/detection_testing/views/DetectionTestingView.py +3 -2
- contentctl/actions/detection_testing/views/DetectionTestingViewFile.py +0 -8
- contentctl/actions/doc_gen.py +1 -1
- contentctl/actions/initialize.py +28 -65
- contentctl/actions/inspect.py +260 -0
- contentctl/actions/new_content.py +106 -13
- contentctl/actions/release_notes.py +168 -144
- contentctl/actions/reporting.py +24 -13
- contentctl/actions/test.py +39 -20
- contentctl/actions/validate.py +25 -48
- contentctl/contentctl.py +196 -754
- contentctl/enrichments/attack_enrichment.py +69 -19
- contentctl/enrichments/cve_enrichment.py +28 -13
- contentctl/helper/link_validator.py +24 -26
- contentctl/helper/utils.py +7 -3
- contentctl/input/director.py +139 -201
- contentctl/input/new_content_questions.py +63 -61
- contentctl/input/sigma_converter.py +1 -2
- contentctl/input/ssa_detection_builder.py +16 -7
- contentctl/input/yml_reader.py +4 -3
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +487 -154
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +155 -51
- contentctl/objects/alert_action.py +40 -0
- contentctl/objects/atomic.py +212 -0
- contentctl/objects/baseline.py +44 -43
- contentctl/objects/baseline_tags.py +69 -20
- contentctl/objects/config.py +857 -125
- contentctl/objects/constants.py +0 -1
- contentctl/objects/correlation_search.py +1 -1
- contentctl/objects/data_source.py +2 -4
- contentctl/objects/deployment.py +61 -21
- contentctl/objects/deployment_email.py +2 -2
- contentctl/objects/deployment_notable.py +4 -4
- contentctl/objects/deployment_phantom.py +2 -2
- contentctl/objects/deployment_rba.py +3 -4
- contentctl/objects/deployment_scheduling.py +2 -3
- contentctl/objects/deployment_slack.py +2 -2
- contentctl/objects/detection.py +1 -5
- contentctl/objects/detection_tags.py +210 -119
- contentctl/objects/enums.py +312 -24
- contentctl/objects/integration_test.py +1 -1
- contentctl/objects/integration_test_result.py +0 -2
- contentctl/objects/investigation.py +62 -53
- contentctl/objects/investigation_tags.py +30 -6
- contentctl/objects/lookup.py +80 -31
- contentctl/objects/macro.py +29 -45
- contentctl/objects/mitre_attack_enrichment.py +29 -5
- contentctl/objects/observable.py +3 -7
- contentctl/objects/playbook.py +60 -30
- contentctl/objects/playbook_tags.py +45 -8
- contentctl/objects/security_content_object.py +1 -5
- contentctl/objects/ssa_detection.py +8 -4
- contentctl/objects/ssa_detection_tags.py +19 -26
- contentctl/objects/story.py +142 -44
- contentctl/objects/story_tags.py +46 -33
- contentctl/objects/unit_test.py +7 -2
- contentctl/objects/unit_test_attack_data.py +10 -19
- contentctl/objects/unit_test_baseline.py +1 -1
- contentctl/objects/unit_test_old.py +4 -3
- contentctl/objects/unit_test_result.py +5 -3
- contentctl/objects/unit_test_ssa.py +31 -0
- contentctl/output/api_json_output.py +202 -130
- contentctl/output/attack_nav_output.py +20 -9
- contentctl/output/attack_nav_writer.py +3 -3
- contentctl/output/ba_yml_output.py +3 -3
- contentctl/output/conf_output.py +125 -391
- contentctl/output/conf_writer.py +169 -31
- contentctl/output/jinja_writer.py +2 -2
- contentctl/output/json_writer.py +17 -5
- contentctl/output/new_content_yml_output.py +8 -7
- contentctl/output/svg_output.py +17 -27
- contentctl/output/templates/analyticstories_detections.j2 +8 -4
- contentctl/output/templates/analyticstories_investigations.j2 +1 -1
- contentctl/output/templates/analyticstories_stories.j2 +6 -6
- contentctl/output/templates/app.conf.j2 +2 -2
- contentctl/output/templates/app.manifest.j2 +2 -2
- contentctl/output/templates/detection_coverage.j2 +6 -8
- contentctl/output/templates/doc_detection_page.j2 +2 -2
- contentctl/output/templates/doc_detections.j2 +2 -2
- contentctl/output/templates/doc_stories.j2 +1 -1
- contentctl/output/templates/es_investigations_investigations.j2 +1 -1
- contentctl/output/templates/es_investigations_stories.j2 +1 -1
- contentctl/output/templates/header.j2 +2 -1
- contentctl/output/templates/macros.j2 +6 -10
- contentctl/output/templates/savedsearches_baselines.j2 +5 -5
- contentctl/output/templates/savedsearches_detections.j2 +36 -33
- contentctl/output/templates/savedsearches_investigations.j2 +4 -4
- contentctl/output/templates/transforms.j2 +4 -4
- contentctl/output/yml_writer.py +2 -2
- contentctl/templates/app_template/README.md +7 -0
- contentctl/{output/templates/splunk_app → templates/app_template}/default/data/ui/nav/default.xml +1 -0
- contentctl/templates/app_template/lookups/mitre_enrichment.csv +638 -0
- contentctl/templates/deployments/{00_default_anomaly.yml → escu_default_configuration_anomaly.yml} +1 -2
- contentctl/templates/deployments/{00_default_baseline.yml → escu_default_configuration_baseline.yml} +1 -2
- contentctl/templates/deployments/{00_default_correlation.yml → escu_default_configuration_correlation.yml} +2 -2
- contentctl/templates/deployments/{00_default_hunting.yml → escu_default_configuration_hunting.yml} +2 -2
- contentctl/templates/deployments/{00_default_ttp.yml → escu_default_configuration_ttp.yml} +1 -2
- contentctl/templates/detections/anomalous_usage_of_7zip.yml +0 -1
- contentctl/templates/stories/cobalt_strike.yml +0 -1
- {contentctl-3.6.0.dist-info → contentctl-4.0.2.dist-info}/METADATA +36 -15
- contentctl-4.0.2.dist-info/RECORD +168 -0
- contentctl/actions/detection_testing/DataManipulation.py +0 -149
- contentctl/actions/generate.py +0 -91
- contentctl/helper/config_handler.py +0 -75
- contentctl/input/baseline_builder.py +0 -66
- contentctl/input/basic_builder.py +0 -58
- contentctl/input/detection_builder.py +0 -370
- contentctl/input/investigation_builder.py +0 -42
- contentctl/input/new_content_generator.py +0 -95
- contentctl/input/playbook_builder.py +0 -68
- contentctl/input/story_builder.py +0 -106
- contentctl/objects/app.py +0 -214
- contentctl/objects/repo_config.py +0 -163
- contentctl/objects/test_config.py +0 -630
- contentctl/output/templates/macros_detections.j2 +0 -7
- contentctl/output/templates/splunk_app/README.md +0 -7
- contentctl-3.6.0.dist-info/RECORD +0 -176
- /contentctl/{output/templates/splunk_app → templates/app_template}/README/essoc_story_detail.txt +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/README/essoc_summary.txt +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/README/essoc_usage_dashboard.txt +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/analytic_stories.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/app.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/commands.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/content-version.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/data/ui/views/escu_summary.xml +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/data/ui/views/feedback.xml +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/distsearch.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/usage_searches.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/default/use_case_library.conf +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/metadata/default.meta +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/static/appIcon.png +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/static/appIconAlt.png +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/static/appIconAlt_2x.png +0 -0
- /contentctl/{output/templates/splunk_app → templates/app_template}/static/appIcon_2x.png +0 -0
- {contentctl-3.6.0.dist-info → contentctl-4.0.2.dist-info}/LICENSE.md +0 -0
- {contentctl-3.6.0.dist-info → contentctl-4.0.2.dist-info}/WHEEL +0 -0
- {contentctl-3.6.0.dist-info → contentctl-4.0.2.dist-info}/entry_points.txt +0 -0
|
@@ -1,23 +1,116 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
|
-
|
|
5
|
-
from
|
|
4
|
+
import questionary
|
|
5
|
+
from typing import Any
|
|
6
|
+
from contentctl.input.new_content_questions import NewContentQuestions
|
|
6
7
|
from contentctl.output.new_content_yml_output import NewContentYmlOutput
|
|
8
|
+
from contentctl.objects.config import new, NewContentType
|
|
9
|
+
import uuid
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
import pathlib
|
|
12
|
+
from contentctl.objects.abstract_security_content_objects.security_content_object_abstract import SecurityContentObject_Abstract
|
|
13
|
+
from contentctl.output.yml_writer import YmlWriter
|
|
7
14
|
|
|
15
|
+
class NewContent:
|
|
8
16
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
17
|
+
def buildDetection(self)->dict[str,Any]:
|
|
18
|
+
questions = NewContentQuestions.get_questions_detection()
|
|
19
|
+
answers = questionary.prompt(questions)
|
|
20
|
+
answers.update(answers)
|
|
21
|
+
answers['name'] = answers['detection_name']
|
|
22
|
+
answers['id'] = str(uuid.uuid4())
|
|
23
|
+
answers['version'] = 1
|
|
24
|
+
answers['date'] = datetime.today().strftime('%Y-%m-%d')
|
|
25
|
+
answers['author'] = answers['detection_author']
|
|
26
|
+
answers['data_source'] = answers['data_source']
|
|
27
|
+
answers['type'] = answers['detection_type']
|
|
28
|
+
answers['status'] = "production" #start everything as production since that's what we INTEND the content to become
|
|
29
|
+
answers['description'] = 'UPDATE_DESCRIPTION'
|
|
30
|
+
file_name = answers['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower()
|
|
31
|
+
answers['search'] = answers['detection_search'] + ' | `' + file_name + '_filter`'
|
|
32
|
+
answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT'
|
|
33
|
+
answers['known_false_positives'] = 'UPDATE_KNOWN_FALSE_POSITIVES'
|
|
34
|
+
answers['references'] = ['REFERENCE']
|
|
35
|
+
answers['tags'] = dict()
|
|
36
|
+
answers['tags']['analytic_story'] = ['UPDATE_STORY_NAME']
|
|
37
|
+
answers['tags']['asset_type'] = 'UPDATE asset_type'
|
|
38
|
+
answers['tags']['confidence'] = 'UPDATE value between 1-100'
|
|
39
|
+
answers['tags']['impact'] = 'UPDATE value between 1-100'
|
|
40
|
+
answers['tags']['message'] = 'UPDATE message'
|
|
41
|
+
answers['tags']['mitre_attack_id'] = [x.strip() for x in answers['mitre_attack_ids'].split(',')]
|
|
42
|
+
answers['tags']['observable'] = [{'name': 'UPDATE', 'type': 'UPDATE', 'role': ['UPDATE']}]
|
|
43
|
+
answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']
|
|
44
|
+
answers['tags']['required_fields'] = ['UPDATE']
|
|
45
|
+
answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100'
|
|
46
|
+
answers['tags']['security_domain'] = answers['security_domain']
|
|
47
|
+
answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
|
|
48
|
+
|
|
49
|
+
#generate the tests section
|
|
50
|
+
answers['tests'] = [
|
|
51
|
+
{
|
|
52
|
+
'name': "True Positive Test",
|
|
53
|
+
'attack_data': [
|
|
54
|
+
{
|
|
55
|
+
'data': "Enter URL for Dataset Here. This may also be a relative or absolute path on your local system for testing.",
|
|
56
|
+
"sourcetype": "UPDATE SOURCETYPE",
|
|
57
|
+
"source": "UPDATE SOURCE"
|
|
58
|
+
}
|
|
59
|
+
]
|
|
60
|
+
}
|
|
61
|
+
]
|
|
62
|
+
return answers
|
|
13
63
|
|
|
64
|
+
def buildStory(self)->dict[str,Any]:
|
|
65
|
+
questions = NewContentQuestions.get_questions_story()
|
|
66
|
+
answers = questionary.prompt(questions)
|
|
67
|
+
answers['name'] = answers['story_name']
|
|
68
|
+
answers['id'] = str(uuid.uuid4())
|
|
69
|
+
answers['version'] = 1
|
|
70
|
+
answers['date'] = datetime.today().strftime('%Y-%m-%d')
|
|
71
|
+
answers['author'] = answers['story_author']
|
|
72
|
+
answers['description'] = 'UPDATE_DESCRIPTION'
|
|
73
|
+
answers['narrative'] = 'UPDATE_NARRATIVE'
|
|
74
|
+
answers['references'] = []
|
|
75
|
+
answers['tags'] = dict()
|
|
76
|
+
answers['tags']['analytic_story'] = answers['name']
|
|
77
|
+
answers['tags']['category'] = answers['category']
|
|
78
|
+
answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']
|
|
79
|
+
answers['tags']['usecase'] = answers['usecase']
|
|
80
|
+
answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
|
|
81
|
+
return answers
|
|
82
|
+
|
|
14
83
|
|
|
15
|
-
|
|
84
|
+
def execute(self, input_dto: new) -> None:
|
|
85
|
+
if input_dto.type == NewContentType.detection:
|
|
86
|
+
content_dict = self.buildDetection()
|
|
87
|
+
subdirectory = pathlib.Path('detections') / content_dict.get('type')
|
|
88
|
+
elif input_dto.type == NewContentType.story:
|
|
89
|
+
content_dict = self.buildStory()
|
|
90
|
+
subdirectory = pathlib.Path('stories')
|
|
91
|
+
else:
|
|
92
|
+
raise Exception(f"Unsupported new content type: [{input_dto.type}]")
|
|
93
|
+
|
|
94
|
+
full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name'))
|
|
95
|
+
YmlWriter.writeYmlFile(str(full_output_path), content_dict)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewContentType) -> None:
|
|
100
|
+
if type == NewContentType.detection:
|
|
101
|
+
file_path = os.path.join(self.output_path, 'detections', subdirectory_name, self.convertNameToFileName(object['name'], object['tags']['product']))
|
|
102
|
+
output_folder = pathlib.Path(self.output_path)/'detections'/subdirectory_name
|
|
103
|
+
#make sure the output folder exists for this detection
|
|
104
|
+
output_folder.mkdir(exist_ok=True)
|
|
16
105
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
106
|
+
YmlWriter.writeYmlFile(file_path, object)
|
|
107
|
+
print("Successfully created detection " + file_path)
|
|
108
|
+
|
|
109
|
+
elif type == NewContentType.story:
|
|
110
|
+
file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product']))
|
|
111
|
+
YmlWriter.writeYmlFile(file_path, object)
|
|
112
|
+
print("Successfully created story " + file_path)
|
|
113
|
+
|
|
114
|
+
else:
|
|
115
|
+
raise(Exception(f"Object Must be Story or Detection, but is not: {object}"))
|
|
21
116
|
|
|
22
|
-
new_content_yml_output = NewContentYmlOutput(input_dto.output_path)
|
|
23
|
-
new_content_yml_output.writeObjectNewContent(new_content_generator_output_dto.obj, new_content_generator_output_dto.answers.get("detection_kind",None), input_dto.new_content_generator_input_dto.type)
|
|
@@ -1,120 +1,131 @@
|
|
|
1
1
|
import os
|
|
2
|
-
|
|
3
|
-
from dataclasses import dataclass
|
|
4
|
-
|
|
5
|
-
from contentctl.input.director import DirectorInputDto, Director, DirectorOutputDto
|
|
6
|
-
from contentctl.output.svg_output import SvgOutput
|
|
7
|
-
from contentctl.output.attack_nav_output import AttackNavOutput
|
|
2
|
+
from contentctl.objects.config import release_notes
|
|
8
3
|
from git import Repo
|
|
9
4
|
import re
|
|
10
5
|
import yaml
|
|
11
|
-
|
|
6
|
+
import pathlib
|
|
7
|
+
from typing import List, Union
|
|
12
8
|
|
|
13
9
|
|
|
14
|
-
@dataclass(frozen=True)
|
|
15
|
-
class ReleaseNotesInputDto:
|
|
16
|
-
director_input_dto: DirectorInputDto
|
|
17
10
|
|
|
18
11
|
class ReleaseNotes:
|
|
19
|
-
def create_notes(self,repo_path, file_paths):
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
12
|
+
def create_notes(self,repo_path:pathlib.Path, file_paths:List[pathlib.Path], header:str)->dict[str,Union[List[str], str]]:
|
|
13
|
+
updates:List[str] = []
|
|
14
|
+
warnings:List[str] = []
|
|
15
|
+
for file_path in file_paths:
|
|
16
|
+
# Check if the file exists
|
|
17
|
+
if file_path.exists() and file_path.is_file():
|
|
18
|
+
# Check if the file is a YAML file
|
|
19
|
+
if file_path.suffix in ['.yaml', '.yml']:
|
|
20
|
+
# Read and parse the YAML file
|
|
21
|
+
with open(file_path, 'r') as file:
|
|
22
|
+
try:
|
|
23
|
+
data = yaml.safe_load(file)
|
|
24
|
+
# Check and create story link
|
|
25
|
+
if 'name' in data and 'stories' in file_path.parts:
|
|
26
|
+
story_link = "https://research.splunk.com/stories/" + data['name']
|
|
27
|
+
story_link=story_link.replace(" ","_")
|
|
28
|
+
story_link = story_link.lower()
|
|
29
|
+
updates.append("- "+"["+f"{data['name']}"+"]"+"("+story_link+")")
|
|
30
|
+
|
|
31
|
+
if 'name' in data and'playbooks' in file_path.parts:
|
|
32
|
+
playbook_link = "https://research.splunk.com/" + str(file_path).replace(str(repo_path),"")
|
|
33
|
+
playbook_link=playbook_link.replace(".yml","/").lower()
|
|
34
|
+
updates.append("- "+"["+f"{data['name']}"+"]"+"("+playbook_link+")")
|
|
35
|
+
|
|
36
|
+
if 'name' in data and'macros' in file_path.parts:
|
|
37
|
+
updates.append("- " + f"{data['name']}")
|
|
38
|
+
|
|
39
|
+
if 'name' in data and'lookups' in file_path.parts:
|
|
40
|
+
updates.append("- " + f"{data['name']}")
|
|
41
|
+
|
|
42
|
+
# Create only SSA link when its production
|
|
43
|
+
if 'name' in data and 'id' in data and 'ssa_detections' in file_path.parts:
|
|
44
|
+
if data['status'] == "production":
|
|
45
|
+
temp_link = "https://research.splunk.com/" + str(file_path).replace(str(repo_path),"")
|
|
46
|
+
pattern = r'(?<=/)[^/]*$'
|
|
47
|
+
detection_link = re.sub(pattern, data['id'], temp_link)
|
|
48
|
+
detection_link = detection_link.replace("detections","" )
|
|
49
|
+
detection_link = detection_link.replace("ssa_/","" )
|
|
50
|
+
updates.append("- "+"["+f"{data['name']}"+"]"+"("+detection_link+")")
|
|
51
|
+
|
|
52
|
+
if data['status'] == "validation":
|
|
53
|
+
updates.append("- "+f"{data['name']}"+" (Validation Mode)")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Check and create detection link
|
|
57
|
+
if 'name' in data and 'id' in data and 'detections' in file_path.parts and not 'ssa_detections' in file_path.parts and 'detections/deprecated' not in file_path.parts:
|
|
58
|
+
|
|
59
|
+
if data['status'] == "production":
|
|
60
|
+
temp_link = "https://research.splunk.com" + str(file_path).replace(str(repo_path),"")
|
|
63
61
|
pattern = r'(?<=/)[^/]*$'
|
|
64
62
|
detection_link = re.sub(pattern, data['id'], temp_link)
|
|
65
63
|
detection_link = detection_link.replace("detections","" )
|
|
66
64
|
detection_link = detection_link.replace(".com//",".com/" )
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
65
|
+
updates.append("- "+"["+f"{data['name']}"+"]"+"("+detection_link+")")
|
|
66
|
+
|
|
67
|
+
if data['status'] == "deprecated":
|
|
68
|
+
temp_link = "https://research.splunk.com" + str(file_path).replace(str(repo_path),"")
|
|
69
|
+
pattern = r'(?<=/)[^/]*$'
|
|
70
|
+
detection_link = re.sub(pattern, data['id'], temp_link)
|
|
71
|
+
detection_link = detection_link.replace("detections","" )
|
|
72
|
+
detection_link = detection_link.replace(".com//",".com/" )
|
|
73
|
+
updates.append("- "+"["+f"{data['name']}"+"]"+"("+detection_link+")")
|
|
74
|
+
|
|
75
|
+
except yaml.YAMLError as exc:
|
|
76
|
+
raise Exception(f"Error parsing YAML file for release_notes {file_path}: {str(exc)}")
|
|
77
|
+
else:
|
|
78
|
+
warnings.append(f"Error parsing YAML file for release_notes. File not found or is not a file: {file_path}")
|
|
79
|
+
#print out all updates at once
|
|
80
|
+
success_header = f'### {header} - [{len(updates)}]'
|
|
81
|
+
warning_header = f'### {header} - [{len(warnings)}]'
|
|
82
|
+
return {'header': success_header, 'changes': sorted(updates),
|
|
83
|
+
'warning_header': warning_header, 'warnings': warnings}
|
|
84
|
+
|
|
73
85
|
|
|
74
|
-
def release_notes(self,
|
|
86
|
+
def release_notes(self, config:release_notes) -> None:
|
|
75
87
|
|
|
76
88
|
### Remove hard coded path
|
|
77
89
|
directories = ['detections/','stories/','macros/','lookups/','playbooks/','ssa_detections/']
|
|
78
|
-
|
|
79
|
-
repo = Repo(
|
|
90
|
+
|
|
91
|
+
repo = Repo(config.path)
|
|
80
92
|
# Ensure the new tag is in the tags if tags are supplied
|
|
81
93
|
|
|
82
|
-
if new_tag:
|
|
83
|
-
if new_tag not in repo.tags:
|
|
84
|
-
raise
|
|
85
|
-
if old_tag is None:
|
|
94
|
+
if config.new_tag:
|
|
95
|
+
if config.new_tag not in repo.tags:
|
|
96
|
+
raise Exception(f"new_tag {config.new_tag} does not exist in the repository. Make sure your branch nameis ")
|
|
97
|
+
if config.old_tag is None:
|
|
86
98
|
#Old tag was not supplied, so find the index of the new tag, then get the tag before it
|
|
87
99
|
tags_sorted = sorted(repo.tags, key=lambda t: t.commit.committed_datetime, reverse=True)
|
|
88
100
|
tags_names_sorted = [tag.name for tag in tags_sorted]
|
|
89
|
-
new_tag_index = tags_names_sorted.index(new_tag)
|
|
101
|
+
new_tag_index = tags_names_sorted.index(config.new_tag)
|
|
90
102
|
try:
|
|
91
|
-
old_tag = tags_names_sorted[new_tag_index+1]
|
|
103
|
+
config.old_tag = tags_names_sorted[new_tag_index+1]
|
|
92
104
|
except Exception:
|
|
93
|
-
raise
|
|
94
|
-
latest_tag = new_tag
|
|
95
|
-
previous_tag = old_tag
|
|
105
|
+
raise Exception(f"old_tag cannot be inferred. {config.new_tag} is the oldest tag in the repo!")
|
|
106
|
+
latest_tag = config.new_tag
|
|
107
|
+
previous_tag = config.old_tag
|
|
96
108
|
commit1 = repo.commit(latest_tag)
|
|
97
109
|
commit2 = repo.commit(previous_tag)
|
|
98
110
|
diff_index = commit2.diff(commit1)
|
|
99
111
|
|
|
100
112
|
# Ensure the branch is in the repo
|
|
101
|
-
if latest_branch:
|
|
113
|
+
if config.latest_branch:
|
|
102
114
|
#If a branch name is supplied, compare against develop
|
|
103
|
-
if latest_branch not in repo.branches:
|
|
104
|
-
raise ValueError(f"latest branch {latest_branch} does not exist in the repository. Make sure your branch name is correct")
|
|
105
|
-
latest_branch = latest_branch
|
|
115
|
+
if config.latest_branch not in repo.branches:
|
|
116
|
+
raise ValueError(f"latest branch {config.latest_branch} does not exist in the repository. Make sure your branch name is correct")
|
|
106
117
|
compare_against = "develop"
|
|
107
|
-
commit1 = repo.commit(latest_branch)
|
|
118
|
+
commit1 = repo.commit(config.latest_branch)
|
|
108
119
|
commit2 = repo.commit(compare_against)
|
|
109
120
|
diff_index = commit2.diff(commit1)
|
|
110
121
|
|
|
111
|
-
modified_files = []
|
|
112
|
-
added_files = []
|
|
122
|
+
modified_files:List[pathlib.Path] = []
|
|
123
|
+
added_files:List[pathlib.Path] = []
|
|
113
124
|
for diff in diff_index:
|
|
114
|
-
file_path = diff.a_path
|
|
125
|
+
file_path = pathlib.Path(diff.a_path)
|
|
115
126
|
|
|
116
127
|
# Check if the file is in the specified directories
|
|
117
|
-
if any(file_path.startswith(directory) for directory in directories):
|
|
128
|
+
if any(str(file_path).startswith(directory) for directory in directories):
|
|
118
129
|
# Check if a file is Modified
|
|
119
130
|
if diff.change_type == 'M':
|
|
120
131
|
modified_files.append(file_path)
|
|
@@ -124,91 +135,104 @@ class ReleaseNotes:
|
|
|
124
135
|
elif diff.change_type == 'A':
|
|
125
136
|
added_files.append(file_path)
|
|
126
137
|
# print(added_files)
|
|
127
|
-
detections_added = []
|
|
128
|
-
ba_detections_added = []
|
|
129
|
-
stories_added = []
|
|
130
|
-
macros_added = []
|
|
131
|
-
lookups_added = []
|
|
132
|
-
playbooks_added = []
|
|
133
|
-
detections_modified = []
|
|
134
|
-
ba_detections_modified = []
|
|
135
|
-
stories_modified = []
|
|
136
|
-
macros_modified = []
|
|
137
|
-
lookups_modified = []
|
|
138
|
-
playbooks_modified = []
|
|
138
|
+
detections_added:List[pathlib.Path] = []
|
|
139
|
+
ba_detections_added:List[pathlib.Path] = []
|
|
140
|
+
stories_added:List[pathlib.Path] = []
|
|
141
|
+
macros_added:List[pathlib.Path] = []
|
|
142
|
+
lookups_added:List[pathlib.Path] = []
|
|
143
|
+
playbooks_added:List[pathlib.Path] = []
|
|
144
|
+
detections_modified:List[pathlib.Path] = []
|
|
145
|
+
ba_detections_modified:List[pathlib.Path] = []
|
|
146
|
+
stories_modified:List[pathlib.Path] = []
|
|
147
|
+
macros_modified:List[pathlib.Path] = []
|
|
148
|
+
lookups_modified:List[pathlib.Path] = []
|
|
149
|
+
playbooks_modified:List[pathlib.Path] = []
|
|
150
|
+
detections_deprecated:List[pathlib.Path] = []
|
|
139
151
|
|
|
140
152
|
for file in modified_files:
|
|
141
|
-
file=
|
|
142
|
-
if 'detections
|
|
153
|
+
file= config.path / file
|
|
154
|
+
if 'detections' in file.parts and 'ssa_detections' not in file.parts and 'deprecated' not in file.parts:
|
|
143
155
|
detections_modified.append(file)
|
|
144
|
-
if '
|
|
156
|
+
if 'detections' in file.parts and 'ssa_detections' not in file.parts and 'deprecated' in file.parts:
|
|
157
|
+
detections_deprecated.append(file)
|
|
158
|
+
if 'stories' in file.parts:
|
|
145
159
|
stories_modified.append(file)
|
|
146
|
-
if 'macros
|
|
160
|
+
if 'macros' in file.parts:
|
|
147
161
|
macros_modified.append(file)
|
|
148
|
-
if 'lookups
|
|
162
|
+
if 'lookups' in file.parts:
|
|
149
163
|
lookups_modified.append(file)
|
|
150
|
-
if 'playbooks
|
|
164
|
+
if 'playbooks' in file.parts:
|
|
151
165
|
playbooks_modified.append(file)
|
|
152
|
-
if 'ssa_detections
|
|
166
|
+
if 'ssa_detections' in file.parts:
|
|
153
167
|
ba_detections_modified.append(file)
|
|
154
168
|
|
|
155
169
|
for file in added_files:
|
|
156
|
-
file=
|
|
157
|
-
if 'detections
|
|
170
|
+
file=config.path / file
|
|
171
|
+
if 'detections' in file.parts and 'ssa_detections' not in file.parts:
|
|
158
172
|
detections_added.append(file)
|
|
159
|
-
if 'stories
|
|
173
|
+
if 'stories' in file.parts:
|
|
160
174
|
stories_added.append(file)
|
|
161
|
-
if 'macros
|
|
175
|
+
if 'macros' in file.parts:
|
|
162
176
|
macros_added.append(file)
|
|
163
|
-
if 'lookups
|
|
177
|
+
if 'lookups' in file.parts:
|
|
164
178
|
lookups_added.append(file)
|
|
165
|
-
if 'playbooks
|
|
179
|
+
if 'playbooks' in file.parts:
|
|
166
180
|
playbooks_added.append(file)
|
|
167
|
-
if 'ssa_detections
|
|
181
|
+
if 'ssa_detections' in file.parts:
|
|
168
182
|
ba_detections_added.append(file)
|
|
169
183
|
|
|
170
|
-
if new_tag:
|
|
184
|
+
if config.new_tag:
|
|
171
185
|
|
|
172
186
|
print(f"Generating release notes - \033[92m{latest_tag}\033[0m")
|
|
173
187
|
print(f"Compared against - \033[92m{previous_tag}\033[0m")
|
|
174
188
|
print("\n## Release notes for ESCU " + latest_tag)
|
|
175
189
|
|
|
176
|
-
if latest_branch:
|
|
177
|
-
print(f"Generating release notes - \033[92m{latest_branch}\033[0m")
|
|
190
|
+
if config.latest_branch:
|
|
191
|
+
print(f"Generating release notes - \033[92m{config.latest_branch}\033[0m")
|
|
178
192
|
print(f"Compared against - \033[92m{compare_against}\033[0m")
|
|
179
|
-
print("\n## Release notes for ESCU " + latest_branch)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
self.create_notes(
|
|
195
|
-
|
|
196
|
-
self.create_notes(repo_path,lookups_modified)
|
|
197
|
-
print("\n### Playbooks Added")
|
|
198
|
-
self.create_notes(repo_path,playbooks_added)
|
|
199
|
-
print("\n### Playbooks Updated")
|
|
200
|
-
self.create_notes(repo_path,playbooks_modified)
|
|
201
|
-
|
|
202
|
-
print("\n### Other Updates\n-\n")
|
|
203
|
-
|
|
204
|
-
print("\n## BA Release Notes")
|
|
205
|
-
|
|
206
|
-
print("\n### New BA Analytics")
|
|
207
|
-
self.create_notes(repo_path,ba_detections_added)
|
|
208
|
-
|
|
209
|
-
print("\n### Updated BA Analytics")
|
|
210
|
-
self.create_notes(repo_path,ba_detections_modified)
|
|
211
|
-
|
|
212
|
-
|
|
193
|
+
print("\n## Release notes for ESCU " + config.latest_branch)
|
|
194
|
+
|
|
195
|
+
notes = [self.create_notes(config.path, stories_added, header="New Analytic Story"),
|
|
196
|
+
self.create_notes(config.path,stories_modified, header="Updated Analytic Story"),
|
|
197
|
+
self.create_notes(config.path,detections_added, header="New Analytics"),
|
|
198
|
+
self.create_notes(config.path,detections_modified, header="Updated Analytics"),
|
|
199
|
+
self.create_notes(config.path,macros_added, header="Macros Added"),
|
|
200
|
+
self.create_notes(config.path,macros_modified, header="Macros Updated"),
|
|
201
|
+
self.create_notes(config.path,lookups_added, header="Lookups Added"),
|
|
202
|
+
self.create_notes(config.path,lookups_modified, header="Lookups Updated"),
|
|
203
|
+
self.create_notes(config.path,playbooks_added, header="Playbooks Added"),
|
|
204
|
+
self.create_notes(config.path,playbooks_modified, header="Playbooks Updated"),
|
|
205
|
+
self.create_notes(config.path,detections_deprecated, header="Deprecated Analytics")]
|
|
206
|
+
|
|
207
|
+
#generate and show ba_notes in a different section
|
|
208
|
+
ba_notes = [self.create_notes(config.path,ba_detections_added, header="New BA Analytics"),
|
|
209
|
+
self.create_notes(config.path,ba_detections_modified, header="Updated BA Analytics") ]
|
|
213
210
|
|
|
211
|
+
|
|
212
|
+
def printNotes(notes:List[dict[str,Union[List[str], str]]], outfile:Union[pathlib.Path,None]=None):
|
|
213
|
+
num_changes = sum([len(note['changes']) for note in notes])
|
|
214
|
+
num_warnings = sum([len(note['warnings']) for note in notes])
|
|
215
|
+
lines:List[str] = []
|
|
216
|
+
lines.append(f"Total New and Updated Content: [{num_changes}]")
|
|
217
|
+
for note in notes:
|
|
218
|
+
lines.append("")
|
|
219
|
+
lines.append(note['header'])
|
|
220
|
+
lines+=(note['changes'])
|
|
221
|
+
|
|
222
|
+
lines.append(f"\n\nTotal Warnings: [{num_warnings}]")
|
|
223
|
+
for note in notes:
|
|
224
|
+
if len(note['warnings']) > 0:
|
|
225
|
+
lines.append(note['warning_header'])
|
|
226
|
+
lines+=note['warnings']
|
|
227
|
+
text_blob = '\n'.join(lines)
|
|
228
|
+
print(text_blob)
|
|
229
|
+
if outfile is not None:
|
|
230
|
+
with open(outfile,'w') as writer:
|
|
231
|
+
writer.write(text_blob)
|
|
232
|
+
|
|
233
|
+
printNotes(notes, config.releaseNotesFilename(f"release_notes.txt"))
|
|
234
|
+
|
|
235
|
+
print("\n\n### Other Updates\n-\n")
|
|
236
|
+
print("\n## BA Release Notes")
|
|
237
|
+
printNotes(ba_notes, config.releaseNotesFilename("ba_release_notes.txt"))
|
|
214
238
|
print(f"Release notes completed succesfully")
|
contentctl/actions/reporting.py
CHANGED
|
@@ -2,32 +2,43 @@ import os
|
|
|
2
2
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
|
|
5
|
-
from contentctl.input.director import
|
|
5
|
+
from contentctl.input.director import DirectorOutputDto
|
|
6
6
|
from contentctl.output.svg_output import SvgOutput
|
|
7
7
|
from contentctl.output.attack_nav_output import AttackNavOutput
|
|
8
|
-
|
|
8
|
+
from contentctl.objects.config import report
|
|
9
9
|
|
|
10
10
|
@dataclass(frozen=True)
|
|
11
11
|
class ReportingInputDto:
|
|
12
|
-
|
|
12
|
+
director_output_dto: DirectorOutputDto
|
|
13
|
+
config: report
|
|
13
14
|
|
|
14
15
|
class Reporting:
|
|
15
16
|
|
|
16
17
|
def execute(self, input_dto: ReportingInputDto) -> None:
|
|
17
|
-
director_output_dto = DirectorOutputDto([],[],[],[],[],[],[],[],[])
|
|
18
|
-
director = Director(director_output_dto)
|
|
19
|
-
director.execute(input_dto.director_input_dto)
|
|
20
18
|
|
|
19
|
+
|
|
20
|
+
#Ensure the reporting path exists
|
|
21
|
+
try:
|
|
22
|
+
input_dto.config.getReportingPath().mkdir(exist_ok=True,parents=True)
|
|
23
|
+
except Exception as e:
|
|
24
|
+
if input_dto.config.getReportingPath().is_file():
|
|
25
|
+
raise Exception(f"Error writing reporting: '{input_dto.config.getReportingPath()}' is a file, not a directory.")
|
|
26
|
+
else:
|
|
27
|
+
raise Exception(f"Error writing reporting : '{input_dto.config.getReportingPath()}': {str(e)}")
|
|
28
|
+
|
|
29
|
+
print("Creating GitHub Badges...")
|
|
30
|
+
#Generate GitHub Badges
|
|
21
31
|
svg_output = SvgOutput()
|
|
22
32
|
svg_output.writeObjects(
|
|
23
|
-
director_output_dto.detections,
|
|
24
|
-
|
|
25
|
-
)
|
|
33
|
+
input_dto.director_output_dto.detections,
|
|
34
|
+
input_dto.config.getReportingPath())
|
|
26
35
|
|
|
27
|
-
|
|
36
|
+
#Generate coverage json
|
|
37
|
+
print("Generating coverage.json...")
|
|
38
|
+
attack_nav_output = AttackNavOutput()
|
|
28
39
|
attack_nav_output.writeObjects(
|
|
29
|
-
director_output_dto.detections,
|
|
30
|
-
|
|
40
|
+
input_dto.director_output_dto.detections,
|
|
41
|
+
input_dto.config.getReportingPath()
|
|
31
42
|
)
|
|
32
43
|
|
|
33
|
-
print(
|
|
44
|
+
print(f"Reporting successfully written to '{input_dto.config.getReportingPath()}'")
|