contentctl 5.0.0a2__py3-none-any.whl → 5.0.0a3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contentctl/__init__.py +1 -1
- contentctl/actions/build.py +88 -55
- contentctl/actions/deploy_acs.py +29 -24
- contentctl/actions/detection_testing/DetectionTestingManager.py +66 -41
- contentctl/actions/detection_testing/GitService.py +2 -4
- contentctl/actions/detection_testing/generate_detection_coverage_badge.py +48 -30
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructure.py +163 -124
- contentctl/actions/detection_testing/infrastructures/DetectionTestingInfrastructureContainer.py +45 -32
- contentctl/actions/detection_testing/progress_bar.py +3 -0
- contentctl/actions/detection_testing/views/DetectionTestingView.py +15 -18
- contentctl/actions/detection_testing/views/DetectionTestingViewCLI.py +1 -5
- contentctl/actions/detection_testing/views/DetectionTestingViewFile.py +2 -2
- contentctl/actions/detection_testing/views/DetectionTestingViewWeb.py +1 -4
- contentctl/actions/doc_gen.py +9 -5
- contentctl/actions/initialize.py +45 -33
- contentctl/actions/inspect.py +118 -61
- contentctl/actions/new_content.py +78 -50
- contentctl/actions/release_notes.py +276 -146
- contentctl/actions/reporting.py +23 -19
- contentctl/actions/test.py +31 -25
- contentctl/actions/validate.py +54 -34
- contentctl/api.py +54 -45
- contentctl/contentctl.py +10 -10
- contentctl/enrichments/attack_enrichment.py +112 -72
- contentctl/enrichments/cve_enrichment.py +34 -28
- contentctl/enrichments/splunk_app_enrichment.py +38 -36
- contentctl/helper/link_validator.py +101 -78
- contentctl/helper/splunk_app.py +69 -41
- contentctl/helper/utils.py +58 -39
- contentctl/input/director.py +69 -37
- contentctl/input/new_content_questions.py +26 -34
- contentctl/input/yml_reader.py +22 -17
- contentctl/objects/abstract_security_content_objects/detection_abstract.py +250 -314
- contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py +58 -36
- contentctl/objects/alert_action.py +8 -8
- contentctl/objects/annotated_types.py +1 -1
- contentctl/objects/atomic.py +64 -54
- contentctl/objects/base_test.py +2 -1
- contentctl/objects/base_test_result.py +16 -8
- contentctl/objects/baseline.py +41 -30
- contentctl/objects/baseline_tags.py +29 -22
- contentctl/objects/config.py +1 -1
- contentctl/objects/constants.py +29 -58
- contentctl/objects/correlation_search.py +75 -55
- contentctl/objects/dashboard.py +55 -41
- contentctl/objects/data_source.py +13 -13
- contentctl/objects/deployment.py +44 -37
- contentctl/objects/deployment_email.py +1 -1
- contentctl/objects/deployment_notable.py +2 -1
- contentctl/objects/deployment_phantom.py +5 -5
- contentctl/objects/deployment_rba.py +1 -1
- contentctl/objects/deployment_scheduling.py +1 -1
- contentctl/objects/deployment_slack.py +1 -1
- contentctl/objects/detection.py +5 -2
- contentctl/objects/detection_metadata.py +1 -0
- contentctl/objects/detection_stanza.py +7 -2
- contentctl/objects/detection_tags.py +54 -64
- contentctl/objects/drilldown.py +66 -35
- contentctl/objects/enums.py +61 -43
- contentctl/objects/errors.py +16 -24
- contentctl/objects/integration_test.py +3 -3
- contentctl/objects/integration_test_result.py +1 -0
- contentctl/objects/investigation.py +41 -26
- contentctl/objects/investigation_tags.py +29 -17
- contentctl/objects/lookup.py +234 -113
- contentctl/objects/macro.py +55 -38
- contentctl/objects/manual_test.py +3 -3
- contentctl/objects/manual_test_result.py +1 -0
- contentctl/objects/mitre_attack_enrichment.py +17 -16
- contentctl/objects/notable_action.py +2 -1
- contentctl/objects/notable_event.py +1 -3
- contentctl/objects/playbook.py +37 -35
- contentctl/objects/playbook_tags.py +22 -16
- contentctl/objects/rba.py +14 -8
- contentctl/objects/risk_analysis_action.py +15 -11
- contentctl/objects/risk_event.py +27 -20
- contentctl/objects/risk_object.py +1 -0
- contentctl/objects/savedsearches_conf.py +9 -7
- contentctl/objects/security_content_object.py +5 -2
- contentctl/objects/story.py +45 -44
- contentctl/objects/story_tags.py +56 -44
- contentctl/objects/test_group.py +5 -2
- contentctl/objects/threat_object.py +1 -0
- contentctl/objects/throttling.py +27 -18
- contentctl/objects/unit_test.py +3 -4
- contentctl/objects/unit_test_baseline.py +4 -5
- contentctl/objects/unit_test_result.py +6 -6
- contentctl/output/api_json_output.py +22 -22
- contentctl/output/attack_nav_output.py +21 -21
- contentctl/output/attack_nav_writer.py +29 -37
- contentctl/output/conf_output.py +230 -174
- contentctl/output/data_source_writer.py +38 -25
- contentctl/output/doc_md_output.py +53 -27
- contentctl/output/jinja_writer.py +19 -15
- contentctl/output/json_writer.py +20 -8
- contentctl/output/svg_output.py +56 -38
- contentctl/output/templates/transforms.j2 +2 -2
- contentctl/output/yml_writer.py +18 -24
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.0a3.dist-info}/METADATA +1 -1
- contentctl-5.0.0a3.dist-info/RECORD +168 -0
- contentctl/actions/initialize_old.py +0 -245
- contentctl/objects/observable.py +0 -39
- contentctl-5.0.0a2.dist-info/RECORD +0 -170
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.0a3.dist-info}/LICENSE.md +0 -0
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.0a3.dist-info}/WHEEL +0 -0
- {contentctl-5.0.0a2.dist-info → contentctl-5.0.0a3.dist-info}/entry_points.txt +0 -0
contentctl/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ =
|
|
1
|
+
__version__ = "0.1.0"
|
contentctl/actions/build.py
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
import sys
|
|
2
1
|
import shutil
|
|
3
|
-
import os
|
|
4
2
|
|
|
5
3
|
from dataclasses import dataclass
|
|
6
4
|
|
|
7
|
-
from contentctl.
|
|
8
|
-
from contentctl.input.director import Director, DirectorOutputDto
|
|
5
|
+
from contentctl.input.director import DirectorOutputDto
|
|
9
6
|
from contentctl.output.conf_output import ConfOutput
|
|
10
7
|
from contentctl.output.conf_writer import ConfWriter
|
|
11
8
|
from contentctl.output.api_json_output import ApiJsonOutput
|
|
@@ -18,86 +15,122 @@ import uuid
|
|
|
18
15
|
|
|
19
16
|
from contentctl.objects.config import build
|
|
20
17
|
|
|
18
|
+
|
|
21
19
|
@dataclass(frozen=True)
|
|
22
20
|
class BuildInputDto:
|
|
23
21
|
director_output_dto: DirectorOutputDto
|
|
24
|
-
config:build
|
|
22
|
+
config: build
|
|
25
23
|
|
|
26
24
|
|
|
27
25
|
class Build:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
26
|
def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
|
|
32
27
|
if input_dto.config.build_app:
|
|
33
|
-
|
|
34
|
-
updated_conf_files:set[pathlib.Path] = set()
|
|
28
|
+
updated_conf_files: set[pathlib.Path] = set()
|
|
35
29
|
conf_output = ConfOutput(input_dto.config)
|
|
36
30
|
|
|
37
|
-
|
|
38
31
|
# Construct a path to a YML that does not actually exist.
|
|
39
32
|
# We mock this "fake" path since the YML does not exist.
|
|
40
33
|
# This ensures the checking for the existence of the CSV is correct
|
|
41
|
-
data_sources_fake_yml_path =
|
|
34
|
+
data_sources_fake_yml_path = (
|
|
35
|
+
input_dto.config.getPackageDirectoryPath()
|
|
36
|
+
/ "lookups"
|
|
37
|
+
/ "data_sources.yml"
|
|
38
|
+
)
|
|
42
39
|
|
|
43
40
|
# Construct a special lookup whose CSV is created at runtime and
|
|
44
|
-
# written directly into the lookups folder. We will delete this after a build,
|
|
41
|
+
# written directly into the lookups folder. We will delete this after a build,
|
|
45
42
|
# assuming that it is successful.
|
|
46
|
-
data_sources_lookup_csv_path =
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
43
|
+
data_sources_lookup_csv_path = (
|
|
44
|
+
input_dto.config.getPackageDirectoryPath()
|
|
45
|
+
/ "lookups"
|
|
46
|
+
/ "data_sources.csv"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
DataSourceWriter.writeDataSourceCsv(
|
|
50
|
+
input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
|
|
51
|
+
)
|
|
52
|
+
input_dto.director_output_dto.addContentToDictMappings(
|
|
53
|
+
CSVLookup.model_construct(
|
|
54
|
+
name="data_sources",
|
|
55
|
+
id=uuid.UUID("b45c1403-6e09-47b0-824f-cf6e44f15ac8"),
|
|
56
|
+
version=1,
|
|
57
|
+
author=input_dto.config.app.author_name,
|
|
58
|
+
date=datetime.date.today(),
|
|
59
|
+
description="A lookup file that will contain the data source objects for detections.",
|
|
60
|
+
lookup_type=Lookup_Type.csv,
|
|
61
|
+
file_path=data_sources_fake_yml_path,
|
|
62
|
+
)
|
|
63
|
+
)
|
|
59
64
|
updated_conf_files.update(conf_output.writeHeaders())
|
|
60
|
-
updated_conf_files.update(
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
updated_conf_files.update(
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
updated_conf_files.update(
|
|
65
|
+
updated_conf_files.update(
|
|
66
|
+
conf_output.writeLookups(input_dto.director_output_dto.lookups)
|
|
67
|
+
)
|
|
68
|
+
updated_conf_files.update(
|
|
69
|
+
conf_output.writeDetections(input_dto.director_output_dto.detections)
|
|
70
|
+
)
|
|
71
|
+
updated_conf_files.update(
|
|
72
|
+
conf_output.writeStories(input_dto.director_output_dto.stories)
|
|
73
|
+
)
|
|
74
|
+
updated_conf_files.update(
|
|
75
|
+
conf_output.writeBaselines(input_dto.director_output_dto.baselines)
|
|
76
|
+
)
|
|
77
|
+
updated_conf_files.update(
|
|
78
|
+
conf_output.writeInvestigations(
|
|
79
|
+
input_dto.director_output_dto.investigations
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
updated_conf_files.update(
|
|
83
|
+
conf_output.writeMacros(input_dto.director_output_dto.macros)
|
|
84
|
+
)
|
|
85
|
+
updated_conf_files.update(
|
|
86
|
+
conf_output.writeDashboards(input_dto.director_output_dto.dashboards)
|
|
87
|
+
)
|
|
67
88
|
updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
|
|
68
|
-
|
|
69
89
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
#Ensure that the conf file we just generated/update is syntactically valid
|
|
90
|
+
# Ensure that the conf file we just generated/update is syntactically valid
|
|
73
91
|
for conf_file in updated_conf_files:
|
|
74
|
-
ConfWriter.validateConfFile(conf_file)
|
|
75
|
-
|
|
92
|
+
ConfWriter.validateConfFile(conf_file)
|
|
93
|
+
|
|
76
94
|
conf_output.packageApp()
|
|
77
95
|
|
|
78
|
-
print(
|
|
79
|
-
|
|
96
|
+
print(
|
|
97
|
+
f"Build of '{input_dto.config.app.title}' APP successful to {input_dto.config.getPackageFilePath()}"
|
|
98
|
+
)
|
|
80
99
|
|
|
81
|
-
if input_dto.config.build_api:
|
|
100
|
+
if input_dto.config.build_api:
|
|
82
101
|
shutil.rmtree(input_dto.config.getAPIPath(), ignore_errors=True)
|
|
83
102
|
input_dto.config.getAPIPath().mkdir(parents=True)
|
|
84
|
-
api_json_output = ApiJsonOutput(
|
|
103
|
+
api_json_output = ApiJsonOutput(
|
|
104
|
+
input_dto.config.getAPIPath(), input_dto.config.app.label
|
|
105
|
+
)
|
|
85
106
|
api_json_output.writeDetections(input_dto.director_output_dto.detections)
|
|
86
107
|
api_json_output.writeStories(input_dto.director_output_dto.stories)
|
|
87
108
|
api_json_output.writeBaselines(input_dto.director_output_dto.baselines)
|
|
88
|
-
api_json_output.writeInvestigations(
|
|
109
|
+
api_json_output.writeInvestigations(
|
|
110
|
+
input_dto.director_output_dto.investigations
|
|
111
|
+
)
|
|
89
112
|
api_json_output.writeLookups(input_dto.director_output_dto.lookups)
|
|
90
113
|
api_json_output.writeMacros(input_dto.director_output_dto.macros)
|
|
91
114
|
api_json_output.writeDeployments(input_dto.director_output_dto.deployments)
|
|
92
115
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
116
|
+
# create version file for sse api
|
|
117
|
+
version_file = input_dto.config.getAPIPath() / "version.json"
|
|
118
|
+
utc_time = (
|
|
119
|
+
datetime.datetime.now(datetime.timezone.utc)
|
|
120
|
+
.replace(microsecond=0, tzinfo=None)
|
|
121
|
+
.isoformat()
|
|
122
|
+
)
|
|
123
|
+
version_dict = {
|
|
124
|
+
"version": {
|
|
125
|
+
"name": f"v{input_dto.config.app.version}",
|
|
126
|
+
"published_at": f"{utc_time}Z",
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
with open(version_file, "w") as version_f:
|
|
130
|
+
json.dump(version_dict, version_f)
|
|
131
|
+
|
|
132
|
+
print(
|
|
133
|
+
f"Build of '{input_dto.config.app.title}' API successful to {input_dto.config.getAPIPath()}"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
return input_dto.director_output_dto
|
contentctl/actions/deploy_acs.py
CHANGED
|
@@ -4,52 +4,57 @@ import pprint
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Deploy:
|
|
7
|
-
def execute(self, config: deploy_acs, appinspect_token:str) -> None:
|
|
8
|
-
|
|
9
|
-
#The following common headers are used by both Clasic and Victoria
|
|
7
|
+
def execute(self, config: deploy_acs, appinspect_token: str) -> None:
|
|
8
|
+
# The following common headers are used by both Clasic and Victoria
|
|
10
9
|
headers = {
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
"Authorization": f"Bearer {config.splunk_cloud_jwt_token}",
|
|
11
|
+
"ACS-Legal-Ack": "Y",
|
|
13
12
|
}
|
|
14
13
|
try:
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
with open(
|
|
15
|
+
config.getPackageFilePath(include_version=False), "rb"
|
|
16
|
+
) as app_data:
|
|
17
|
+
# request_data = app_data.read()
|
|
18
18
|
if config.stack_type == StackType.classic:
|
|
19
19
|
# Classic instead uses a form to store token and package
|
|
20
20
|
# https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
|
|
21
21
|
address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
|
|
22
|
-
|
|
23
|
-
form_data = {
|
|
24
|
-
|
|
25
|
-
'package': app_data
|
|
26
|
-
}
|
|
27
|
-
res = post(address, headers=headers, files = form_data)
|
|
22
|
+
|
|
23
|
+
form_data = {"token": (None, appinspect_token), "package": app_data}
|
|
24
|
+
res = post(address, headers=headers, files=form_data)
|
|
28
25
|
elif config.stack_type == StackType.victoria:
|
|
29
26
|
# Victoria uses the X-Splunk-Authorization Header
|
|
30
27
|
# It also uses --data-binary for the app content
|
|
31
28
|
# https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
|
|
32
|
-
headers.update({
|
|
29
|
+
headers.update({"X-Splunk-Authorization": appinspect_token})
|
|
33
30
|
address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
|
|
34
31
|
res = post(address, headers=headers, data=app_data.read())
|
|
35
32
|
else:
|
|
36
33
|
raise Exception(f"Unsupported stack type: '{config.stack_type}'")
|
|
37
34
|
except Exception as e:
|
|
38
|
-
raise Exception(
|
|
39
|
-
|
|
35
|
+
raise Exception(
|
|
36
|
+
f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}"
|
|
37
|
+
)
|
|
38
|
+
|
|
40
39
|
try:
|
|
41
40
|
# Request went through and completed, but may have returned a non-successful error code.
|
|
42
41
|
# This likely includes a more verbose response describing the error
|
|
43
42
|
res.raise_for_status()
|
|
44
43
|
print(res.json())
|
|
45
|
-
except Exception
|
|
44
|
+
except Exception:
|
|
46
45
|
try:
|
|
47
46
|
error_text = res.json()
|
|
48
|
-
except Exception
|
|
47
|
+
except Exception:
|
|
49
48
|
error_text = "No error text - request failed"
|
|
50
49
|
formatted_error_text = pprint.pformat(error_text)
|
|
51
|
-
print(
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
50
|
+
print(
|
|
51
|
+
"While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
|
|
52
|
+
"ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase."
|
|
53
|
+
)
|
|
54
|
+
raise Exception(
|
|
55
|
+
f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
print(
|
|
59
|
+
f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!"
|
|
60
|
+
)
|
|
@@ -1,25 +1,29 @@
|
|
|
1
|
-
from typing import List,Union
|
|
2
|
-
from contentctl.objects.config import test, test_servers, Container,Infrastructure
|
|
3
|
-
from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
from
|
|
7
|
-
|
|
1
|
+
from typing import List, Union
|
|
2
|
+
from contentctl.objects.config import test, test_servers, Container, Infrastructure
|
|
3
|
+
from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
|
|
4
|
+
DetectionTestingInfrastructure,
|
|
5
|
+
)
|
|
6
|
+
from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureContainer import (
|
|
7
|
+
DetectionTestingInfrastructureContainer,
|
|
8
|
+
)
|
|
9
|
+
from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructureServer import (
|
|
10
|
+
DetectionTestingInfrastructureServer,
|
|
11
|
+
)
|
|
8
12
|
import signal
|
|
9
13
|
import datetime
|
|
14
|
+
|
|
10
15
|
# from queue import Queue
|
|
11
16
|
from dataclasses import dataclass
|
|
17
|
+
|
|
12
18
|
# import threading
|
|
13
|
-
import ctypes
|
|
14
19
|
from contentctl.actions.detection_testing.infrastructures.DetectionTestingInfrastructure import (
|
|
15
|
-
DetectionTestingInfrastructure,
|
|
16
20
|
DetectionTestingManagerOutputDto,
|
|
17
21
|
)
|
|
18
22
|
from contentctl.actions.detection_testing.views.DetectionTestingView import (
|
|
19
23
|
DetectionTestingView,
|
|
20
24
|
)
|
|
21
25
|
from contentctl.objects.enums import PostTestBehavior
|
|
22
|
-
from pydantic import BaseModel
|
|
26
|
+
from pydantic import BaseModel
|
|
23
27
|
from contentctl.objects.detection import Detection
|
|
24
28
|
import concurrent.futures
|
|
25
29
|
import docker
|
|
@@ -27,7 +31,7 @@ import docker
|
|
|
27
31
|
|
|
28
32
|
@dataclass(frozen=False)
|
|
29
33
|
class DetectionTestingManagerInputDto:
|
|
30
|
-
config: Union[test,test_servers]
|
|
34
|
+
config: Union[test, test_servers]
|
|
31
35
|
detections: List[Detection]
|
|
32
36
|
views: list[DetectionTestingView]
|
|
33
37
|
|
|
@@ -64,15 +68,18 @@ class DetectionTestingManager(BaseModel):
|
|
|
64
68
|
print("*******************************")
|
|
65
69
|
|
|
66
70
|
signal.signal(signal.SIGINT, sigint_handler)
|
|
67
|
-
|
|
68
|
-
with concurrent.futures.ThreadPoolExecutor(
|
|
69
|
-
max_workers=len(self.input_dto.config.test_instances),
|
|
70
|
-
) as instance_pool, concurrent.futures.ThreadPoolExecutor(
|
|
71
|
-
max_workers=len(self.input_dto.views)
|
|
72
|
-
) as view_runner, concurrent.futures.ThreadPoolExecutor(
|
|
73
|
-
max_workers=len(self.input_dto.config.test_instances),
|
|
74
|
-
) as view_shutdowner:
|
|
75
71
|
|
|
72
|
+
with (
|
|
73
|
+
concurrent.futures.ThreadPoolExecutor(
|
|
74
|
+
max_workers=len(self.input_dto.config.test_instances),
|
|
75
|
+
) as instance_pool,
|
|
76
|
+
concurrent.futures.ThreadPoolExecutor(
|
|
77
|
+
max_workers=len(self.input_dto.views)
|
|
78
|
+
) as view_runner,
|
|
79
|
+
concurrent.futures.ThreadPoolExecutor(
|
|
80
|
+
max_workers=len(self.input_dto.config.test_instances),
|
|
81
|
+
) as view_shutdowner,
|
|
82
|
+
):
|
|
76
83
|
# Start all the views
|
|
77
84
|
future_views = {
|
|
78
85
|
view_runner.submit(view.setup): view for view in self.input_dto.views
|
|
@@ -86,7 +93,7 @@ class DetectionTestingManager(BaseModel):
|
|
|
86
93
|
# Wait for all instances to be set up
|
|
87
94
|
for future in concurrent.futures.as_completed(future_instances_setup):
|
|
88
95
|
try:
|
|
89
|
-
|
|
96
|
+
future.result()
|
|
90
97
|
except Exception as e:
|
|
91
98
|
self.output_dto.terminate = True
|
|
92
99
|
print(f"Error setting up container: {str(e)}")
|
|
@@ -101,7 +108,7 @@ class DetectionTestingManager(BaseModel):
|
|
|
101
108
|
# Wait for execution to finish
|
|
102
109
|
for future in concurrent.futures.as_completed(future_instances_execute):
|
|
103
110
|
try:
|
|
104
|
-
|
|
111
|
+
future.result()
|
|
105
112
|
except Exception as e:
|
|
106
113
|
self.output_dto.terminate = True
|
|
107
114
|
print(f"Error running in container: {str(e)}")
|
|
@@ -114,34 +121,43 @@ class DetectionTestingManager(BaseModel):
|
|
|
114
121
|
}
|
|
115
122
|
for future in concurrent.futures.as_completed(future_views_shutdowner):
|
|
116
123
|
try:
|
|
117
|
-
|
|
124
|
+
future.result()
|
|
118
125
|
except Exception as e:
|
|
119
126
|
print(f"Error stopping view: {str(e)}")
|
|
120
127
|
|
|
121
128
|
# Wait for original view-related threads to complete
|
|
122
129
|
for future in concurrent.futures.as_completed(future_views):
|
|
123
130
|
try:
|
|
124
|
-
|
|
131
|
+
future.result()
|
|
125
132
|
except Exception as e:
|
|
126
133
|
print(f"Error running container: {str(e)}")
|
|
127
134
|
|
|
128
135
|
return self.output_dto
|
|
129
136
|
|
|
130
137
|
def create_DetectionTestingInfrastructureObjects(self):
|
|
131
|
-
#Make sure that, if we need to, we pull the appropriate container
|
|
138
|
+
# Make sure that, if we need to, we pull the appropriate container
|
|
132
139
|
for infrastructure in self.input_dto.config.test_instances:
|
|
133
|
-
if
|
|
140
|
+
if isinstance(self.input_dto.config, test) and isinstance(
|
|
141
|
+
infrastructure, Container
|
|
142
|
+
):
|
|
134
143
|
try:
|
|
135
144
|
client = docker.from_env()
|
|
136
|
-
except Exception
|
|
137
|
-
raise Exception(
|
|
145
|
+
except Exception:
|
|
146
|
+
raise Exception(
|
|
147
|
+
"Unable to connect to docker. Are you sure that docker is running on this host?"
|
|
148
|
+
)
|
|
138
149
|
try:
|
|
139
|
-
|
|
140
|
-
|
|
150
|
+
parts = (
|
|
151
|
+
self.input_dto.config.container_settings.full_image_path.split(
|
|
152
|
+
":"
|
|
153
|
+
)
|
|
154
|
+
)
|
|
141
155
|
if len(parts) != 2:
|
|
142
|
-
raise Exception(
|
|
143
|
-
|
|
144
|
-
|
|
156
|
+
raise Exception(
|
|
157
|
+
f"Expected to find a name:tag in {self.input_dto.config.container_settings.full_image_path}, "
|
|
158
|
+
f"but instead found {parts}. Note that this path MUST include the tag, which is separated by ':'"
|
|
159
|
+
)
|
|
160
|
+
|
|
145
161
|
print(
|
|
146
162
|
f"Getting the latest version of the container image [{self.input_dto.config.container_settings.full_image_path}]...",
|
|
147
163
|
end="",
|
|
@@ -151,12 +167,15 @@ class DetectionTestingManager(BaseModel):
|
|
|
151
167
|
print("done!")
|
|
152
168
|
break
|
|
153
169
|
except Exception as e:
|
|
154
|
-
raise Exception(
|
|
170
|
+
raise Exception(
|
|
171
|
+
f"Failed to pull docker container image [{self.input_dto.config.container_settings.full_image_path}]: {str(e)}"
|
|
172
|
+
)
|
|
155
173
|
|
|
156
174
|
already_staged_container_files = False
|
|
157
175
|
for infrastructure in self.input_dto.config.test_instances:
|
|
158
|
-
|
|
159
|
-
|
|
176
|
+
if isinstance(self.input_dto.config, test) and isinstance(
|
|
177
|
+
infrastructure, Container
|
|
178
|
+
):
|
|
160
179
|
# Stage the files in the apps dir so that they can be passed directly to
|
|
161
180
|
# subsequent containers. Do this here, instead of inside each container, to
|
|
162
181
|
# avoid duplicate downloads/moves/copies
|
|
@@ -166,18 +185,24 @@ class DetectionTestingManager(BaseModel):
|
|
|
166
185
|
|
|
167
186
|
self.detectionTestingInfrastructureObjects.append(
|
|
168
187
|
DetectionTestingInfrastructureContainer(
|
|
169
|
-
global_config=self.input_dto.config,
|
|
188
|
+
global_config=self.input_dto.config,
|
|
189
|
+
infrastructure=infrastructure,
|
|
190
|
+
sync_obj=self.output_dto,
|
|
170
191
|
)
|
|
171
192
|
)
|
|
172
193
|
|
|
173
|
-
elif
|
|
194
|
+
elif isinstance(self.input_dto.config, test_servers) and isinstance(
|
|
195
|
+
infrastructure, Infrastructure
|
|
196
|
+
):
|
|
174
197
|
self.detectionTestingInfrastructureObjects.append(
|
|
175
198
|
DetectionTestingInfrastructureServer(
|
|
176
|
-
global_config=self.input_dto.config,
|
|
199
|
+
global_config=self.input_dto.config,
|
|
200
|
+
infrastructure=infrastructure,
|
|
201
|
+
sync_obj=self.output_dto,
|
|
177
202
|
)
|
|
178
203
|
)
|
|
179
204
|
|
|
180
205
|
else:
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
206
|
+
raise Exception(
|
|
207
|
+
f"Unsupported target infrastructure '{infrastructure}' and config type {self.input_dto.config}"
|
|
208
|
+
)
|
|
@@ -10,6 +10,7 @@ from pygit2.enums import DeltaStatus
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
11
11
|
from contentctl.input.director import DirectorOutputDto
|
|
12
12
|
|
|
13
|
+
from contentctl.input.director import DirectorOutputDto
|
|
13
14
|
from contentctl.objects.config import All, Changes, Selected, test_common
|
|
14
15
|
from contentctl.objects.data_source import DataSource
|
|
15
16
|
from contentctl.objects.detection import Detection
|
|
@@ -22,9 +23,6 @@ logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
|
|
22
23
|
LOGGER = logging.getLogger(__name__)
|
|
23
24
|
|
|
24
25
|
|
|
25
|
-
from contentctl.input.director import DirectorOutputDto
|
|
26
|
-
|
|
27
|
-
|
|
28
26
|
class GitService(BaseModel):
|
|
29
27
|
director: DirectorOutputDto
|
|
30
28
|
config: test_common
|
|
@@ -79,7 +77,7 @@ class GitService(BaseModel):
|
|
|
79
77
|
updated_datasources: set[DataSource] = set()
|
|
80
78
|
|
|
81
79
|
for diff in all_diffs:
|
|
82
|
-
if type(diff)
|
|
80
|
+
if type(diff) is pygit2.Patch:
|
|
83
81
|
if diff.delta.status in (
|
|
84
82
|
DeltaStatus.ADDED,
|
|
85
83
|
DeltaStatus.MODIFIED,
|
|
@@ -2,7 +2,7 @@ import argparse
|
|
|
2
2
|
import json
|
|
3
3
|
import sys
|
|
4
4
|
|
|
5
|
-
RAW_BADGE_SVG =
|
|
5
|
+
RAW_BADGE_SVG = """<?xml version="1.0"?>
|
|
6
6
|
<svg xmlns="http://www.w3.org/2000/svg" width="100" height="20">
|
|
7
7
|
<linearGradient id="a" x2="0" y2="100%">
|
|
8
8
|
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
|
|
@@ -19,47 +19,65 @@ RAW_BADGE_SVG = '''<?xml version="1.0"?>
|
|
|
19
19
|
<text x="30" y="14">{}</text>
|
|
20
20
|
<text x="80" y="14">{}</text>
|
|
21
21
|
</g>
|
|
22
|
-
</svg>
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
parser = argparse.ArgumentParser(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
parser.add_argument(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
22
|
+
</svg>"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
parser = argparse.ArgumentParser(
|
|
26
|
+
description="Use a summary.json file to generate a test coverage badge"
|
|
27
|
+
)
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"-i",
|
|
30
|
+
"--input_summary_file",
|
|
31
|
+
type=argparse.FileType("r"),
|
|
32
|
+
required=True,
|
|
33
|
+
help="Summary file to use to generate the pass percentage badge",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"-o",
|
|
37
|
+
"--output_badge_file",
|
|
38
|
+
type=argparse.FileType("w"),
|
|
39
|
+
required=True,
|
|
40
|
+
help="Name of the badge to output",
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument(
|
|
43
|
+
"-s", "--badge_string", type=str, required=True, help="Name of the badge to output"
|
|
44
|
+
)
|
|
33
45
|
|
|
34
46
|
|
|
35
47
|
try:
|
|
36
|
-
|
|
48
|
+
results = parser.parse_args()
|
|
37
49
|
except Exception as e:
|
|
38
|
-
|
|
39
|
-
|
|
50
|
+
print(f"Error parsing arguments: {str(e)}")
|
|
51
|
+
exit(1)
|
|
40
52
|
|
|
41
53
|
try:
|
|
42
|
-
|
|
54
|
+
summary_info = json.loads(results.input_summary_file.read())
|
|
43
55
|
except Exception as e:
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
elif
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
56
|
+
print(f"Error loading {results.input_summary_file.name} JSON file: {str(e)}")
|
|
57
|
+
sys.exit(1)
|
|
58
|
+
|
|
59
|
+
if "summary" not in summary_info:
|
|
60
|
+
print("Missing 'summary' key in {results.input_summary_file.name}")
|
|
61
|
+
sys.exit(1)
|
|
62
|
+
elif (
|
|
63
|
+
"PASS_RATE" not in summary_info["summary"]
|
|
64
|
+
or "TESTS_PASSED" not in summary_info["summary"]
|
|
65
|
+
):
|
|
66
|
+
print(
|
|
67
|
+
f"Missing PASS_RATE in 'summary' section of {results.input_summary_file.name}"
|
|
68
|
+
)
|
|
69
|
+
sys.exit(1)
|
|
70
|
+
pass_percent = 100 * summary_info["summary"]["PASS_RATE"]
|
|
54
71
|
|
|
55
72
|
|
|
56
73
|
try:
|
|
57
|
-
|
|
74
|
+
results.output_badge_file.write(
|
|
75
|
+
RAW_BADGE_SVG.format(results.badge_string, "{:2.1f}%".format(pass_percent))
|
|
76
|
+
)
|
|
58
77
|
except Exception as e:
|
|
59
|
-
|
|
60
|
-
|
|
78
|
+
print(f"Error generating badge: {str(e)}")
|
|
79
|
+
sys.exit(1)
|
|
61
80
|
|
|
62
81
|
|
|
63
82
|
print(f"Badge {results.output_badge_file.name} successfully generated!")
|
|
64
83
|
sys.exit(0)
|
|
65
|
-
|