devsecops-engine-tools 1.7.32__py3-none-any.whl → 1.7.33__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of devsecops-engine-tools might be problematic. Click here for more details.
- devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py +74 -17
- devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/tool_gateway.py +1 -1
- devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py +12 -6
- devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/handle_remote_config_patterns.py +0 -37
- devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py +157 -40
- devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py +13 -14
- devsecops_engine_tools/version.py +1 -1
- {devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/METADATA +2 -2
- {devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/RECORD +12 -13
- {devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/WHEEL +1 -1
- devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/find_artifacts.py +0 -100
- {devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/entry_points.txt +0 -0
- {devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/top_level.txt +0 -0
|
@@ -31,26 +31,48 @@ from devsecops_engine_tools.version import version
|
|
|
31
31
|
|
|
32
32
|
logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
|
|
33
33
|
|
|
34
|
+
|
|
34
35
|
def parse_separated_list(value, choices):
|
|
35
|
-
values = value.split(
|
|
36
|
+
values = value.split(",")
|
|
36
37
|
# Validar cada elemento de la lista
|
|
37
38
|
for val in values:
|
|
38
39
|
if val not in choices:
|
|
39
|
-
raise argparse.ArgumentTypeError(
|
|
40
|
-
|
|
40
|
+
raise argparse.ArgumentTypeError(
|
|
41
|
+
f"Invalid value: {val}. Valid values are: {', '.join(choices)}"
|
|
42
|
+
)
|
|
43
|
+
|
|
41
44
|
return values
|
|
42
45
|
|
|
46
|
+
|
|
43
47
|
def parse_choices(choices):
|
|
44
48
|
def parse_with_choices(value):
|
|
45
49
|
return parse_separated_list(value, choices)
|
|
50
|
+
|
|
46
51
|
return parse_with_choices
|
|
47
52
|
|
|
53
|
+
|
|
48
54
|
def get_inputs_from_cli(args):
|
|
49
55
|
parser = argparse.ArgumentParser()
|
|
50
|
-
parser.add_argument(
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
parser.add_argument(
|
|
56
|
+
parser.add_argument(
|
|
57
|
+
"-v", "--version", action="version", version="{version}".format(version=version)
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument(
|
|
60
|
+
"-pd",
|
|
61
|
+
"--platform_devops",
|
|
62
|
+
choices=["azure", "github", "local"],
|
|
63
|
+
type=str,
|
|
64
|
+
required=True,
|
|
65
|
+
help="Platform where is executed",
|
|
66
|
+
)
|
|
67
|
+
parser.add_argument(
|
|
68
|
+
"-rcf",
|
|
69
|
+
"--remote_config_repo",
|
|
70
|
+
type=str,
|
|
71
|
+
required=True,
|
|
72
|
+
help="Name or Folder Path of Config Repo",
|
|
73
|
+
)
|
|
74
|
+
parser.add_argument(
|
|
75
|
+
"-t",
|
|
54
76
|
"--tool",
|
|
55
77
|
choices=[
|
|
56
78
|
"engine_iac",
|
|
@@ -64,9 +86,20 @@ def get_inputs_from_cli(args):
|
|
|
64
86
|
required=True,
|
|
65
87
|
help="Tool to execute",
|
|
66
88
|
)
|
|
67
|
-
parser.add_argument(
|
|
68
|
-
|
|
69
|
-
"--
|
|
89
|
+
parser.add_argument(
|
|
90
|
+
"-fp",
|
|
91
|
+
"--folder_path",
|
|
92
|
+
type=str,
|
|
93
|
+
required=False,
|
|
94
|
+
help="Folder Path to scan, only apply engine_iac and engine_dependencies tools",
|
|
95
|
+
)
|
|
96
|
+
parser.add_argument(
|
|
97
|
+
"-p",
|
|
98
|
+
"--platform",
|
|
99
|
+
type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi"}),
|
|
100
|
+
required=False,
|
|
101
|
+
default="all",
|
|
102
|
+
help="Platform to scan, only apply engine_iac tool",
|
|
70
103
|
)
|
|
71
104
|
parser.add_argument(
|
|
72
105
|
"--use_secrets_manager",
|
|
@@ -89,10 +122,31 @@ def get_inputs_from_cli(args):
|
|
|
89
122
|
required=False,
|
|
90
123
|
help="Enable or Disable the send metrics to the driven adapter metrics",
|
|
91
124
|
)
|
|
92
|
-
parser.add_argument(
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
parser.add_argument(
|
|
125
|
+
parser.add_argument(
|
|
126
|
+
"--token_cmdb", required=False, help="Token to connect to the CMDB"
|
|
127
|
+
)
|
|
128
|
+
parser.add_argument(
|
|
129
|
+
"--token_vulnerability_management",
|
|
130
|
+
required=False,
|
|
131
|
+
help="Token to connect to the Vulnerability Management",
|
|
132
|
+
)
|
|
133
|
+
parser.add_argument(
|
|
134
|
+
"--token_engine_container",
|
|
135
|
+
required=False,
|
|
136
|
+
help="Token to execute engine_container if is necessary",
|
|
137
|
+
)
|
|
138
|
+
parser.add_argument(
|
|
139
|
+
"--token_engine_dependencies",
|
|
140
|
+
required=False,
|
|
141
|
+
help="Token to execute engine_dependencies if is necessary. If using xray as engine_dependencies tool, the token is the base64 of artifactory server config that can be obtain from jfrog cli with 'jf config export <ServerID>' command.",
|
|
142
|
+
)
|
|
143
|
+
parser.add_argument(
|
|
144
|
+
"--xray_mode",
|
|
145
|
+
choices=["scan", "audit"],
|
|
146
|
+
required=False,
|
|
147
|
+
default="scan",
|
|
148
|
+
help="Mode to execute xray, only apply engine_dependencies xray tool",
|
|
149
|
+
)
|
|
96
150
|
args = parser.parse_args()
|
|
97
151
|
return {
|
|
98
152
|
"platform_devops": args.platform_devops,
|
|
@@ -107,8 +161,10 @@ def get_inputs_from_cli(args):
|
|
|
107
161
|
"token_vulnerability_management": args.token_vulnerability_management,
|
|
108
162
|
"token_engine_container": args.token_engine_container,
|
|
109
163
|
"token_engine_dependencies": args.token_engine_dependencies,
|
|
164
|
+
"xray_mode": args.xray_mode,
|
|
110
165
|
}
|
|
111
166
|
|
|
167
|
+
|
|
112
168
|
def application_core():
|
|
113
169
|
try:
|
|
114
170
|
# Get inputs from CLI
|
|
@@ -120,18 +176,18 @@ def application_core():
|
|
|
120
176
|
devops_platform_gateway = {
|
|
121
177
|
"azure": AzureDevops(),
|
|
122
178
|
"github": GithubActions(),
|
|
123
|
-
"local": RuntimeLocal()
|
|
179
|
+
"local": RuntimeLocal(),
|
|
124
180
|
}.get(args["platform_devops"])
|
|
125
181
|
printer_table_gateway = PrinterPrettyTable()
|
|
126
182
|
metrics_manager_gateway = S3Manager()
|
|
127
|
-
|
|
183
|
+
|
|
128
184
|
init_engine_core(
|
|
129
185
|
vulnerability_management_gateway,
|
|
130
186
|
secrets_manager_gateway,
|
|
131
187
|
devops_platform_gateway,
|
|
132
188
|
printer_table_gateway,
|
|
133
189
|
metrics_manager_gateway,
|
|
134
|
-
args
|
|
190
|
+
args,
|
|
135
191
|
)
|
|
136
192
|
except Exception as e:
|
|
137
193
|
logger.error("Error engine_core: {0} ".format(str(e)))
|
|
@@ -142,5 +198,6 @@ def application_core():
|
|
|
142
198
|
)
|
|
143
199
|
print(devops_platform_gateway.result_pipeline("failed"))
|
|
144
200
|
|
|
201
|
+
|
|
145
202
|
if __name__ == "__main__":
|
|
146
203
|
application_core()
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/tool_gateway.py
CHANGED
|
@@ -4,6 +4,6 @@ from abc import ABCMeta, abstractmethod
|
|
|
4
4
|
class ToolGateway(metaclass=ABCMeta):
|
|
5
5
|
@abstractmethod
|
|
6
6
|
def run_tool_dependencies_sca(
|
|
7
|
-
self, remote_config,
|
|
7
|
+
self, remote_config, dict_args, to_scan, token
|
|
8
8
|
) -> str:
|
|
9
9
|
"run tool dependencies sca"
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py
CHANGED
|
@@ -12,15 +12,19 @@ class DependenciesScan:
|
|
|
12
12
|
tool_run: ToolGateway,
|
|
13
13
|
tool_deserializator: DeserializatorGateway,
|
|
14
14
|
remote_config,
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
dict_args,
|
|
16
|
+
exclusions,
|
|
17
|
+
pipeline_name,
|
|
18
|
+
to_scan,
|
|
17
19
|
token,
|
|
18
20
|
):
|
|
19
21
|
self.tool_run = tool_run
|
|
20
22
|
self.tool_deserializator = tool_deserializator
|
|
21
23
|
self.remote_config = remote_config
|
|
22
|
-
self.
|
|
23
|
-
self.
|
|
24
|
+
self.exclusions = exclusions
|
|
25
|
+
self.pipeline_name = pipeline_name
|
|
26
|
+
self.dict_args = dict_args
|
|
27
|
+
self.to_scan = to_scan
|
|
24
28
|
self.token = token
|
|
25
29
|
|
|
26
30
|
def process(self):
|
|
@@ -31,8 +35,10 @@ class DependenciesScan:
|
|
|
31
35
|
"""
|
|
32
36
|
return self.tool_run.run_tool_dependencies_sca(
|
|
33
37
|
self.remote_config,
|
|
34
|
-
self.
|
|
35
|
-
self.
|
|
38
|
+
self.dict_args,
|
|
39
|
+
self.exclusions,
|
|
40
|
+
self.pipeline_name,
|
|
41
|
+
self.to_scan,
|
|
36
42
|
self.token,
|
|
37
43
|
)
|
|
38
44
|
|
|
@@ -12,31 +12,6 @@ class HandleRemoteConfigPatterns:
|
|
|
12
12
|
self.exclusions = exclusions
|
|
13
13
|
self.pipeline_name = pipeline_name
|
|
14
14
|
|
|
15
|
-
def excluded_files(self):
|
|
16
|
-
"""
|
|
17
|
-
Handle excluded files.
|
|
18
|
-
|
|
19
|
-
Return: string: new regex expresion.
|
|
20
|
-
"""
|
|
21
|
-
|
|
22
|
-
pattern = self.remote_config["REGEX_EXPRESSION_EXTENSIONS"]
|
|
23
|
-
if (self.pipeline_name in self.exclusions) and (
|
|
24
|
-
self.exclusions[self.pipeline_name].get("SKIP_FILES", 0)
|
|
25
|
-
):
|
|
26
|
-
exclusion = self.exclusions[self.pipeline_name]["SKIP_FILES"]
|
|
27
|
-
if exclusion.get("files", 0):
|
|
28
|
-
excluded_file_types = exclusion["files"]
|
|
29
|
-
pattern2 = pattern
|
|
30
|
-
for ext in excluded_file_types:
|
|
31
|
-
pattern2 = (
|
|
32
|
-
pattern2.replace("|" + ext, "")
|
|
33
|
-
.replace(ext + "|", "")
|
|
34
|
-
.replace(ext, "")
|
|
35
|
-
)
|
|
36
|
-
pattern = pattern2
|
|
37
|
-
|
|
38
|
-
return pattern
|
|
39
|
-
|
|
40
15
|
def ignore_analysis_pattern(self):
|
|
41
16
|
"""
|
|
42
17
|
Handle analysis pattern.
|
|
@@ -49,18 +24,6 @@ class HandleRemoteConfigPatterns:
|
|
|
49
24
|
else:
|
|
50
25
|
return True
|
|
51
26
|
|
|
52
|
-
def bypass_archive_limits(self):
|
|
53
|
-
"""
|
|
54
|
-
Handle bypass archive limits.
|
|
55
|
-
|
|
56
|
-
Return: bool: True -> Bypass archive limits, False -> Without bypass archive limits.
|
|
57
|
-
"""
|
|
58
|
-
bypass_limits = self.remote_config["BYPASS_ARCHIVE_LIMITS"]
|
|
59
|
-
if re.match(bypass_limits, self.pipeline_name):
|
|
60
|
-
return True
|
|
61
|
-
else:
|
|
62
|
-
return False
|
|
63
|
-
|
|
64
27
|
def skip_from_exclusion(self):
|
|
65
28
|
"""
|
|
66
29
|
Handle skip tool.
|
|
@@ -8,6 +8,8 @@ import requests
|
|
|
8
8
|
import re
|
|
9
9
|
import os
|
|
10
10
|
import json
|
|
11
|
+
import shutil
|
|
12
|
+
import tarfile
|
|
11
13
|
|
|
12
14
|
from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
|
|
13
15
|
from devsecops_engine_tools.engine_utilities import settings
|
|
@@ -16,19 +18,113 @@ logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
|
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
class XrayScan(ToolGateway):
|
|
19
|
-
def
|
|
21
|
+
def excluded_files(self, remote_config, pipeline_name, exclusions):
|
|
22
|
+
pattern = remote_config["XRAY"]["REGEX_EXPRESSION_EXTENSIONS"]
|
|
23
|
+
if pipeline_name in exclusions:
|
|
24
|
+
for ex in exclusions[pipeline_name]["XRAY"]:
|
|
25
|
+
if ex.get("SKIP_FILES", 0):
|
|
26
|
+
exclusion = ex.get("SKIP_FILES")
|
|
27
|
+
if exclusion.get("files", 0):
|
|
28
|
+
excluded_file_types = exclusion["files"]
|
|
29
|
+
pattern2 = pattern
|
|
30
|
+
for ext in excluded_file_types:
|
|
31
|
+
pattern2 = (
|
|
32
|
+
pattern2.replace("|" + ext, "")
|
|
33
|
+
.replace(ext + "|", "")
|
|
34
|
+
.replace(ext, "")
|
|
35
|
+
)
|
|
36
|
+
pattern = pattern2
|
|
37
|
+
|
|
38
|
+
return pattern
|
|
39
|
+
|
|
40
|
+
def find_packages(self, pattern, packages, working_dir):
|
|
41
|
+
packages_list = []
|
|
42
|
+
files_list = []
|
|
43
|
+
extension_pattern = re.compile(pattern, re.IGNORECASE)
|
|
44
|
+
for root, dirs, files in os.walk(working_dir):
|
|
45
|
+
components = root.split(os.path.sep)
|
|
46
|
+
flag = 0
|
|
47
|
+
for package in packages:
|
|
48
|
+
if not (package in components):
|
|
49
|
+
flag = 1
|
|
50
|
+
if package in dirs:
|
|
51
|
+
packages_list.append(os.path.join(root, package))
|
|
52
|
+
if flag:
|
|
53
|
+
for file in files:
|
|
54
|
+
if extension_pattern.search(file):
|
|
55
|
+
files_list.append(os.path.join(root, file))
|
|
56
|
+
return packages_list, files_list
|
|
57
|
+
|
|
58
|
+
def compress_and_mv(self, tar_path, package):
|
|
59
|
+
try:
|
|
60
|
+
with tarfile.open(tar_path, "w") as tar:
|
|
61
|
+
tar.add(
|
|
62
|
+
package,
|
|
63
|
+
arcname=os.path.basename(package),
|
|
64
|
+
filter=lambda x: None if "/.bin/" in x.name else x,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
except subprocess.CalledProcessError as e:
|
|
68
|
+
logger.error(f"Error during {package} compression: {e}")
|
|
69
|
+
|
|
70
|
+
def move_files(self, dir_to_scan_path, finded_files):
|
|
71
|
+
for file in finded_files:
|
|
72
|
+
target = os.path.join(dir_to_scan_path, os.path.basename(file))
|
|
73
|
+
shutil.copy2(file, target)
|
|
74
|
+
logger.debug(f"File to scan: {file}")
|
|
75
|
+
|
|
76
|
+
def find_artifacts(self, to_scan, pattern, packages):
|
|
77
|
+
dir_to_scan_path = os.path.join(to_scan, "dependencies_to_scan")
|
|
78
|
+
if os.path.exists(dir_to_scan_path):
|
|
79
|
+
shutil.rmtree(dir_to_scan_path)
|
|
80
|
+
os.makedirs(dir_to_scan_path)
|
|
81
|
+
|
|
82
|
+
packages_list, files_list = self.find_packages(pattern, packages, to_scan)
|
|
83
|
+
|
|
84
|
+
for package in packages_list:
|
|
85
|
+
tar_path = os.path.join(
|
|
86
|
+
dir_to_scan_path,
|
|
87
|
+
"pkg"
|
|
88
|
+
+ str(packages_list.index(package) + 1)
|
|
89
|
+
+ "_"
|
|
90
|
+
+ os.path.basename(package)
|
|
91
|
+
+ ".tar",
|
|
92
|
+
)
|
|
93
|
+
self.compress_and_mv(tar_path, package)
|
|
94
|
+
|
|
95
|
+
if len(files_list):
|
|
96
|
+
self.move_files(dir_to_scan_path, files_list)
|
|
97
|
+
|
|
98
|
+
files = os.listdir(dir_to_scan_path)
|
|
99
|
+
files = [
|
|
100
|
+
file
|
|
101
|
+
for file in files
|
|
102
|
+
if os.path.isfile(os.path.join(dir_to_scan_path, file))
|
|
103
|
+
]
|
|
104
|
+
file_to_scan = None
|
|
105
|
+
if files:
|
|
106
|
+
file_to_scan = os.path.join(dir_to_scan_path, "file_to_scan.tar")
|
|
107
|
+
self.compress_and_mv(file_to_scan, dir_to_scan_path)
|
|
108
|
+
files_string = ", ".join(files)
|
|
109
|
+
logger.debug(f"Files to scan: {files_string}")
|
|
110
|
+
print(f"Files to scan: {files_string}")
|
|
111
|
+
else:
|
|
112
|
+
logger.warning("No artifacts found")
|
|
113
|
+
|
|
114
|
+
return file_to_scan
|
|
115
|
+
|
|
116
|
+
def install_tool_linux(self, prefix, version):
|
|
20
117
|
installed = subprocess.run(
|
|
21
|
-
["which",
|
|
118
|
+
["which", prefix],
|
|
22
119
|
stdout=subprocess.PIPE,
|
|
23
120
|
stderr=subprocess.PIPE,
|
|
24
121
|
)
|
|
25
122
|
if installed.returncode == 1:
|
|
26
|
-
command = ["chmod", "+x",
|
|
123
|
+
command = ["chmod", "+x", prefix]
|
|
27
124
|
try:
|
|
28
125
|
url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-linux-amd64/jf"
|
|
29
|
-
file = "./jf"
|
|
30
126
|
response = requests.get(url, allow_redirects=True)
|
|
31
|
-
with open(
|
|
127
|
+
with open(prefix, "wb") as archivo:
|
|
32
128
|
archivo.write(response.content)
|
|
33
129
|
subprocess.run(
|
|
34
130
|
command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
@@ -36,36 +132,34 @@ class XrayScan(ToolGateway):
|
|
|
36
132
|
except subprocess.CalledProcessError as error:
|
|
37
133
|
logger.error(f"Error during Jfrog Cli installation on Linux: {error}")
|
|
38
134
|
|
|
39
|
-
def install_tool_windows(self, version):
|
|
135
|
+
def install_tool_windows(self, prefix, version):
|
|
40
136
|
try:
|
|
41
137
|
subprocess.run(
|
|
42
|
-
[
|
|
138
|
+
[prefix, "--version"],
|
|
43
139
|
stdout=subprocess.PIPE,
|
|
44
140
|
stderr=subprocess.PIPE,
|
|
45
141
|
)
|
|
46
142
|
except:
|
|
47
143
|
try:
|
|
48
144
|
url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-windows-amd64/jf.exe"
|
|
49
|
-
exe_file = "./jf.exe"
|
|
50
145
|
response = requests.get(url, allow_redirects=True)
|
|
51
|
-
with open(
|
|
146
|
+
with open(prefix, "wb") as archivo:
|
|
52
147
|
archivo.write(response.content)
|
|
53
148
|
except subprocess.CalledProcessError as error:
|
|
54
149
|
logger.error(f"Error while Jfrog Cli installation on Windows: {error}")
|
|
55
150
|
|
|
56
|
-
def install_tool_darwin(self, version):
|
|
151
|
+
def install_tool_darwin(self, prefix, version):
|
|
57
152
|
installed = subprocess.run(
|
|
58
|
-
["which",
|
|
153
|
+
["which", prefix],
|
|
59
154
|
stdout=subprocess.PIPE,
|
|
60
155
|
stderr=subprocess.PIPE,
|
|
61
156
|
)
|
|
62
157
|
if installed.returncode == 1:
|
|
63
|
-
command = ["chmod", "+x",
|
|
158
|
+
command = ["chmod", "+x", prefix]
|
|
64
159
|
try:
|
|
65
160
|
url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-mac-386/jf"
|
|
66
|
-
file = "./jf"
|
|
67
161
|
response = requests.get(url, allow_redirects=True)
|
|
68
|
-
with open(
|
|
162
|
+
with open(prefix, "wb") as archivo:
|
|
69
163
|
archivo.write(response.content)
|
|
70
164
|
subprocess.run(
|
|
71
165
|
command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
|
@@ -95,56 +189,79 @@ class XrayScan(ToolGateway):
|
|
|
95
189
|
except subprocess.CalledProcessError as error:
|
|
96
190
|
logger.error(f"Error during Xray Server configuration: {error}")
|
|
97
191
|
|
|
98
|
-
def
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
192
|
+
def config_audit_scan(self, to_scan):
|
|
193
|
+
gradlew_path = os.path.join(to_scan, "gradlew")
|
|
194
|
+
if os.path.exists(gradlew_path):
|
|
195
|
+
os.chmod(gradlew_path, 0o755)
|
|
196
|
+
|
|
197
|
+
def scan_dependencies(self, prefix, cwd, mode, to_scan):
|
|
198
|
+
command = [
|
|
199
|
+
prefix,
|
|
200
|
+
mode,
|
|
201
|
+
"--format=json",
|
|
202
|
+
f"{to_scan}",
|
|
203
|
+
]
|
|
204
|
+
result = subprocess.run(
|
|
205
|
+
command, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
|
|
206
|
+
)
|
|
207
|
+
if result.returncode == 0:
|
|
113
208
|
scan_result = json.loads(result.stdout)
|
|
114
209
|
file_result = os.path.join(os.getcwd(), "scan_result.json")
|
|
115
210
|
with open(file_result, "w") as file:
|
|
116
211
|
json.dump(scan_result, file, indent=4)
|
|
212
|
+
if result.stdout == "null\n":
|
|
213
|
+
logger.warning(f"Xray scan returned null: {result.stderr}")
|
|
214
|
+
return None
|
|
117
215
|
return file_result
|
|
118
|
-
|
|
119
|
-
logger.error(f"Error executing
|
|
216
|
+
else:
|
|
217
|
+
logger.error(f"Error executing Xray scan: {result.stderr}")
|
|
218
|
+
return None
|
|
120
219
|
|
|
121
220
|
def run_tool_dependencies_sca(
|
|
122
221
|
self,
|
|
123
222
|
remote_config,
|
|
124
|
-
|
|
125
|
-
|
|
223
|
+
dict_args,
|
|
224
|
+
exclusion,
|
|
225
|
+
pipeline_name,
|
|
226
|
+
to_scan,
|
|
126
227
|
token,
|
|
127
228
|
):
|
|
229
|
+
if dict_args["xray_mode"] == "scan":
|
|
230
|
+
pattern = self.excluded_files(remote_config, pipeline_name, exclusion)
|
|
231
|
+
to_scan = self.find_artifacts(
|
|
232
|
+
to_scan, pattern, remote_config["XRAY"]["PACKAGES_TO_SCAN"]
|
|
233
|
+
)
|
|
234
|
+
cwd = os.getcwd()
|
|
235
|
+
if not to_scan:
|
|
236
|
+
return None
|
|
237
|
+
else:
|
|
238
|
+
self.config_audit_scan(to_scan)
|
|
239
|
+
cwd = to_scan
|
|
240
|
+
to_scan = ""
|
|
128
241
|
|
|
129
242
|
cli_version = remote_config["XRAY"]["CLI_VERSION"]
|
|
130
243
|
os_platform = platform.system()
|
|
131
244
|
|
|
132
245
|
if os_platform == "Linux":
|
|
133
|
-
|
|
134
|
-
command_prefix
|
|
246
|
+
command_prefix = os.path.join(os.path.expanduser("~"), "jf")
|
|
247
|
+
self.install_tool_linux(command_prefix, cli_version)
|
|
135
248
|
elif os_platform == "Windows":
|
|
136
|
-
|
|
137
|
-
command_prefix
|
|
249
|
+
command_prefix = os.path.join(os.path.expanduser("~"), "jf.exe")
|
|
250
|
+
self.install_tool_windows(command_prefix, cli_version)
|
|
138
251
|
elif os_platform == "Darwin":
|
|
139
|
-
command_prefix = "
|
|
140
|
-
self.install_tool_darwin(cli_version)
|
|
252
|
+
command_prefix = os.path.join(os.path.expanduser("~"), "jf")
|
|
253
|
+
self.install_tool_darwin(command_prefix, cli_version)
|
|
141
254
|
else:
|
|
142
255
|
logger.warning(f"{os_platform} is not supported.")
|
|
256
|
+
return None
|
|
143
257
|
|
|
144
258
|
self.config_server(command_prefix, token)
|
|
145
259
|
|
|
146
260
|
results_file = self.scan_dependencies(
|
|
147
|
-
command_prefix,
|
|
261
|
+
command_prefix,
|
|
262
|
+
cwd,
|
|
263
|
+
dict_args["xray_mode"],
|
|
264
|
+
to_scan,
|
|
148
265
|
)
|
|
149
266
|
|
|
150
267
|
return results_file
|
|
@@ -7,9 +7,6 @@ from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.s
|
|
|
7
7
|
from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.handle_remote_config_patterns import (
|
|
8
8
|
HandleRemoteConfigPatterns,
|
|
9
9
|
)
|
|
10
|
-
from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.find_artifacts import (
|
|
11
|
-
FindArtifacts,
|
|
12
|
-
)
|
|
13
10
|
|
|
14
11
|
import os
|
|
15
12
|
import sys
|
|
@@ -46,24 +43,26 @@ def init_engine_dependencies(
|
|
|
46
43
|
input_core = SetInputCore(remote_config, exclusions, pipeline_name, tool)
|
|
47
44
|
|
|
48
45
|
if scan_flag and not (skip_flag):
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
find_artifacts = FindArtifacts(
|
|
53
|
-
os.getcwd(), pattern, remote_config["PACKAGES_TO_SCAN"]
|
|
54
|
-
)
|
|
55
|
-
file_to_scan = find_artifacts.find_artifacts()
|
|
56
|
-
if file_to_scan:
|
|
46
|
+
to_scan = dict_args["folder_path"] if dict_args["folder_path"] else os.getcwd()
|
|
47
|
+
if os.path.exists(to_scan):
|
|
57
48
|
dependencies_sca_scan = DependenciesScan(
|
|
58
49
|
tool_run,
|
|
59
50
|
tool_deserializator,
|
|
60
51
|
remote_config,
|
|
61
|
-
|
|
62
|
-
|
|
52
|
+
dict_args,
|
|
53
|
+
exclusions,
|
|
54
|
+
pipeline_name,
|
|
55
|
+
to_scan,
|
|
63
56
|
token,
|
|
64
57
|
)
|
|
65
58
|
dependencies_scanned = dependencies_sca_scan.process()
|
|
66
|
-
deserialized =
|
|
59
|
+
deserialized = (
|
|
60
|
+
dependencies_sca_scan.deserializator(dependencies_scanned)
|
|
61
|
+
if dependencies_scanned is not None
|
|
62
|
+
else []
|
|
63
|
+
)
|
|
64
|
+
else:
|
|
65
|
+
logger.error(f"Path {to_scan} does not exist")
|
|
67
66
|
else:
|
|
68
67
|
print(f"Tool skipped by DevSecOps policy")
|
|
69
68
|
logger.info(f"Tool skipped by DevSecOps policy")
|
|
@@ -1 +1 @@
|
|
|
1
|
-
version = '1.7.
|
|
1
|
+
version = '1.7.33'
|
{devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: devsecops-engine-tools
|
|
3
|
-
Version: 1.7.
|
|
3
|
+
Version: 1.7.33
|
|
4
4
|
Summary: Tool for DevSecOps strategy
|
|
5
5
|
Home-page: https://github.com/bancolombia/devsecops-engine-tools
|
|
6
6
|
Author: Bancolombia DevSecOps Team
|
|
@@ -64,7 +64,7 @@ pip3 install devsecops-engine-tools
|
|
|
64
64
|
### Scan running - flags (CLI)
|
|
65
65
|
|
|
66
66
|
```bash
|
|
67
|
-
devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container"] --folder_path ["Folder path scan engine_iac"] --platform ["k8s","cloudformation","docker", "openapi"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"]
|
|
67
|
+
devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container"] --folder_path ["Folder path scan engine_iac"] --platform ["k8s","cloudformation","docker", "openapi"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --xray_mode ["scan", "audit"]
|
|
68
68
|
```
|
|
69
69
|
|
|
70
70
|
### Structure Remote Config
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
devsecops_engine_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
devsecops_engine_tools/version.py,sha256=
|
|
2
|
+
devsecops_engine_tools/version.py,sha256=h11L2DDrYwZBoizyDdRE9iXihnMzBYqtD28vWFs75So,19
|
|
3
3
|
devsecops_engine_tools/engine_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
devsecops_engine_tools/engine_core/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
devsecops_engine_tools/engine_core/src/applications/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
-
devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py,sha256=
|
|
6
|
+
devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py,sha256=muzsafzDgDpBcVANGfIeKAF1eE7p9uKpAoWnAMNIV9U,6678
|
|
7
7
|
devsecops_engine_tools/engine_core/src/deployment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
devsecops_engine_tools/engine_core/src/deployment/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
9
|
devsecops_engine_tools/engine_core/src/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -172,19 +172,18 @@ devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/__init__.py,sha
|
|
|
172
172
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
173
173
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
174
174
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/deserializator_gateway.py,sha256=WOJbKbeFIbQ0IR6lic2JheTPUJPPzAYsGykU1DiMmHw,286
|
|
175
|
-
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/tool_gateway.py,sha256=
|
|
175
|
+
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/tool_gateway.py,sha256=xPUpWnXJZjnaMDScIL1duIFl8apF8obD4v0XXONT-44,240
|
|
176
176
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
177
|
-
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py,sha256=
|
|
178
|
-
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/
|
|
179
|
-
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/handle_remote_config_patterns.py,sha256=ct_TinG3-UnHmG0Ic2fgPAc6mvWjiyq1bxU7RqFIZXA,2203
|
|
177
|
+
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py,sha256=8XrYyqi_s4yRPopBJhW0l5djiYOaupdnl6f7omaKG4U,1461
|
|
178
|
+
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/handle_remote_config_patterns.py,sha256=cTM8IQRZJBr5zG5nhCkTxuw2fCHDZ3wrPgQhRjG88pg,968
|
|
180
179
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py,sha256=ZMr9l1h9-JdoT7QlLN8K9VTP2VP0B_5TZGMUQ6KyIYo,2525
|
|
181
180
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
182
181
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
183
182
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
184
183
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_deserialize_output.py,sha256=Vm0pj1i6a34xXouXUU95Y04hzR--9tcMQuycR7IMUnQ,2221
|
|
185
|
-
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py,sha256=
|
|
184
|
+
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py,sha256=aY3Gfl1ALK5XZaFb3PGOSVRXNzPiBni2y_WOO2tubnY,10061
|
|
186
185
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
187
|
-
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py,sha256=
|
|
186
|
+
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py,sha256=2UbEknG_24_pO6Wba496Fbkg103mDyqx7myAWYPoThU,2570
|
|
188
187
|
devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
189
188
|
devsecops_engine_tools/engine_utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
190
189
|
devsecops_engine_tools/engine_utilities/settings.py,sha256=CPnDndwVeRgQNml3HVzvytVruDd8dTd1ICHbkMDSgTM,2144
|
|
@@ -250,8 +249,8 @@ devsecops_engine_tools/engine_utilities/utils/logger_info.py,sha256=4Mz8Bwlm9Mku
|
|
|
250
249
|
devsecops_engine_tools/engine_utilities/utils/name_conversion.py,sha256=ADJrRGaxYSDe0ZRh6VHRf53H4sXPcb-vNP_i81PUn3I,307
|
|
251
250
|
devsecops_engine_tools/engine_utilities/utils/printers.py,sha256=GAslbWaBpwP3mP6fBsgVl07TTBgcCggQTy8h2M9ibeo,612
|
|
252
251
|
devsecops_engine_tools/engine_utilities/utils/session_manager.py,sha256=yNtlT-8Legz1sHbGPH8LNYjL-LgDUE0zXG2rYjiab7U,290
|
|
253
|
-
devsecops_engine_tools-1.7.
|
|
254
|
-
devsecops_engine_tools-1.7.
|
|
255
|
-
devsecops_engine_tools-1.7.
|
|
256
|
-
devsecops_engine_tools-1.7.
|
|
257
|
-
devsecops_engine_tools-1.7.
|
|
252
|
+
devsecops_engine_tools-1.7.33.dist-info/METADATA,sha256=DEM9Bq8gpesoQ-IiHTaFD-MABTtAeAM77r_ufJNqUQE,7720
|
|
253
|
+
devsecops_engine_tools-1.7.33.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
|
|
254
|
+
devsecops_engine_tools-1.7.33.dist-info/entry_points.txt,sha256=9IjXF_7Zpgowq_SY6OSmsA9vZze18a8_AeHwkQVrgKk,131
|
|
255
|
+
devsecops_engine_tools-1.7.33.dist-info/top_level.txt,sha256=ge6y0X_xBAU1aG3EMWFtl9djbVyg5BxuSp2r2Lg6EQU,23
|
|
256
|
+
devsecops_engine_tools-1.7.33.dist-info/RECORD,,
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import tarfile
|
|
3
|
-
import subprocess
|
|
4
|
-
import shutil
|
|
5
|
-
import re
|
|
6
|
-
|
|
7
|
-
from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
|
|
8
|
-
from devsecops_engine_tools.engine_utilities import settings
|
|
9
|
-
|
|
10
|
-
logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
class FindArtifacts:
|
|
14
|
-
def __init__(
|
|
15
|
-
self,
|
|
16
|
-
working_dir,
|
|
17
|
-
pattern,
|
|
18
|
-
packages,
|
|
19
|
-
):
|
|
20
|
-
self.working_dir = working_dir
|
|
21
|
-
self.pattern = pattern
|
|
22
|
-
self.packages = packages
|
|
23
|
-
|
|
24
|
-
def find_packages(self, pattern, packages, working_dir):
|
|
25
|
-
packages_list = []
|
|
26
|
-
files_list = []
|
|
27
|
-
extension_pattern = re.compile(pattern, re.IGNORECASE)
|
|
28
|
-
for root, dirs, files in os.walk(working_dir):
|
|
29
|
-
components = root.split(os.path.sep)
|
|
30
|
-
flag = 0
|
|
31
|
-
for package in packages:
|
|
32
|
-
if not (package in components):
|
|
33
|
-
flag = 1
|
|
34
|
-
if package in dirs:
|
|
35
|
-
packages_list.append(os.path.join(root, package))
|
|
36
|
-
if flag:
|
|
37
|
-
for file in files:
|
|
38
|
-
if extension_pattern.search(file):
|
|
39
|
-
files_list.append(os.path.join(root, file))
|
|
40
|
-
return packages_list, files_list
|
|
41
|
-
|
|
42
|
-
def compress_and_mv(self, tar_path, package):
|
|
43
|
-
try:
|
|
44
|
-
with tarfile.open(tar_path, "w") as tar:
|
|
45
|
-
tar.add(
|
|
46
|
-
package,
|
|
47
|
-
arcname=os.path.basename(package),
|
|
48
|
-
filter=lambda x: None if "/.bin/" in x.name else x,
|
|
49
|
-
)
|
|
50
|
-
|
|
51
|
-
except subprocess.CalledProcessError as e:
|
|
52
|
-
logger.error(f"Error during {package} compression: {e}")
|
|
53
|
-
|
|
54
|
-
def move_files(self, dir_to_scan_path, finded_files):
|
|
55
|
-
for file in finded_files:
|
|
56
|
-
target = os.path.join(dir_to_scan_path, os.path.basename(file))
|
|
57
|
-
shutil.copy2(file, target)
|
|
58
|
-
logger.debug(f"File to scan: {file}")
|
|
59
|
-
|
|
60
|
-
def find_artifacts(self):
|
|
61
|
-
dir_to_scan_path = os.path.join(self.working_dir, "dependencies_to_scan")
|
|
62
|
-
if os.path.exists(dir_to_scan_path):
|
|
63
|
-
shutil.rmtree(dir_to_scan_path)
|
|
64
|
-
os.makedirs(dir_to_scan_path)
|
|
65
|
-
|
|
66
|
-
packages_list, files_list = self.find_packages(
|
|
67
|
-
self.pattern, self.packages, self.working_dir
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
for package in packages_list:
|
|
71
|
-
tar_path = os.path.join(
|
|
72
|
-
dir_to_scan_path,
|
|
73
|
-
"pkg"
|
|
74
|
-
+ str(packages_list.index(package) + 1)
|
|
75
|
-
+ "_"
|
|
76
|
-
+ os.path.basename(package)
|
|
77
|
-
+ ".tar",
|
|
78
|
-
)
|
|
79
|
-
self.compress_and_mv(tar_path, package)
|
|
80
|
-
|
|
81
|
-
if len(files_list):
|
|
82
|
-
self.move_files(dir_to_scan_path, files_list)
|
|
83
|
-
|
|
84
|
-
files = os.listdir(dir_to_scan_path)
|
|
85
|
-
files = [
|
|
86
|
-
file
|
|
87
|
-
for file in files
|
|
88
|
-
if os.path.isfile(os.path.join(dir_to_scan_path, file))
|
|
89
|
-
]
|
|
90
|
-
file_to_scan = None
|
|
91
|
-
if files:
|
|
92
|
-
file_to_scan = os.path.join(dir_to_scan_path, "file_to_scan.tar")
|
|
93
|
-
self.compress_and_mv(file_to_scan, dir_to_scan_path)
|
|
94
|
-
files_string = ", ".join(files)
|
|
95
|
-
logger.debug(f"Files to scan: {files_string}")
|
|
96
|
-
print(f"Files to scan: {files_string}")
|
|
97
|
-
else:
|
|
98
|
-
logger.warning("No artifacts found")
|
|
99
|
-
|
|
100
|
-
return file_to_scan
|
{devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{devsecops_engine_tools-1.7.32.dist-info → devsecops_engine_tools-1.7.33.dist-info}/top_level.txt
RENAMED
|
File without changes
|