fosslight-dependency 3.0.7__py3-none-any.whl → 4.1.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fosslight_dependency/LICENSES/LICENSE +201 -0
- fosslight_dependency/LICENSES/LicenseRef-3rd_party_licenses.txt +1254 -0
- fosslight_dependency/__init__.py +0 -1
- fosslight_dependency/_analyze_dependency.py +130 -0
- fosslight_dependency/_graph_convertor.py +67 -0
- fosslight_dependency/_help.py +79 -0
- fosslight_dependency/_package_manager.py +397 -0
- fosslight_dependency/cli.py +127 -0
- fosslight_dependency/constant.py +57 -0
- fosslight_dependency/dependency_item.py +103 -0
- fosslight_dependency/package_manager/Android.py +90 -0
- fosslight_dependency/package_manager/Cargo.py +144 -0
- fosslight_dependency/package_manager/Carthage.py +130 -0
- fosslight_dependency/package_manager/Cocoapods.py +194 -0
- fosslight_dependency/package_manager/Go.py +179 -0
- fosslight_dependency/package_manager/Gradle.py +123 -0
- fosslight_dependency/package_manager/Helm.py +106 -0
- fosslight_dependency/package_manager/Maven.py +274 -0
- fosslight_dependency/package_manager/Npm.py +296 -0
- fosslight_dependency/package_manager/Nuget.py +368 -0
- fosslight_dependency/package_manager/Pnpm.py +155 -0
- fosslight_dependency/package_manager/Pub.py +241 -0
- fosslight_dependency/package_manager/Pypi.py +395 -0
- fosslight_dependency/package_manager/Swift.py +159 -0
- fosslight_dependency/package_manager/Unity.py +118 -0
- fosslight_dependency/package_manager/Yarn.py +231 -0
- fosslight_dependency/package_manager/__init__.py +0 -0
- fosslight_dependency/run_dependency_scanner.py +393 -0
- fosslight_dependency-4.1.30.dist-info/METADATA +213 -0
- fosslight_dependency-4.1.30.dist-info/RECORD +37 -0
- {fosslight_dependency-3.0.7.dist-info → fosslight_dependency-4.1.30.dist-info}/WHEEL +1 -1
- fosslight_dependency-4.1.30.dist-info/entry_points.txt +2 -0
- fosslight_dependency-4.1.30.dist-info/licenses/LICENSES/Apache-2.0.txt +201 -0
- fosslight_dependency-4.1.30.dist-info/licenses/LICENSES/LicenseRef-3rd_party_licenses.txt +1254 -0
- fosslight_dependency-4.1.30.dist-info/licenses/LICENSES/MIT.txt +21 -0
- fosslight_dependency/_version.py +0 -1
- fosslight_dependency/analyze_dependency.py +0 -1090
- fosslight_dependency/third_party/askalono/askalono.exe +0 -0
- fosslight_dependency/third_party/askalono/askalono_macos +0 -0
- fosslight_dependency/third_party/nomos/nomossa +0 -0
- fosslight_dependency-3.0.7.dist-info/3rd_party_licenses.txt +0 -726
- fosslight_dependency-3.0.7.dist-info/METADATA +0 -51
- fosslight_dependency-3.0.7.dist-info/RECORD +0 -13
- fosslight_dependency-3.0.7.dist-info/entry_points.txt +0 -3
- {fosslight_dependency-3.0.7.dist-info → fosslight_dependency-4.1.30.dist-info/licenses}/LICENSE +0 -0
- {fosslight_dependency-3.0.7.dist-info → fosslight_dependency-4.1.30.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2021 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import logging
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import shutil
|
|
11
|
+
import yaml
|
|
12
|
+
import subprocess
|
|
13
|
+
import fosslight_util.constant as constant
|
|
14
|
+
import fosslight_dependency.constant as const
|
|
15
|
+
from fosslight_dependency._package_manager import PackageManager
|
|
16
|
+
from fosslight_dependency._package_manager import get_url_to_purl, check_license_name
|
|
17
|
+
from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
|
|
18
|
+
from fosslight_util.oss_item import OssItem
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Pub(PackageManager):
|
|
24
|
+
package_manager_name = const.PUB
|
|
25
|
+
|
|
26
|
+
dn_url = 'https://pub.dev/packages/'
|
|
27
|
+
input_file_name = 'tmp_flutter_oss_licenses.json'
|
|
28
|
+
tmp_dir = "fl_dependency_tmp_dir"
|
|
29
|
+
cur_path = ''
|
|
30
|
+
pkg_source_list = {}
|
|
31
|
+
|
|
32
|
+
def __init__(self, input_dir, output_dir):
|
|
33
|
+
super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
|
|
34
|
+
self.append_input_package_list_file(self.input_file_name)
|
|
35
|
+
|
|
36
|
+
def __del__(self):
|
|
37
|
+
if self.cur_path != '':
|
|
38
|
+
os.chdir(self.cur_path)
|
|
39
|
+
if os.path.exists(self.tmp_dir):
|
|
40
|
+
shutil.rmtree(self.tmp_dir)
|
|
41
|
+
|
|
42
|
+
def run_plugin(self):
|
|
43
|
+
if os.path.exists(self.input_file_name):
|
|
44
|
+
logger.info(f"Found {self.input_file_name}, skip the flutter cmd to analyze dependency.")
|
|
45
|
+
return True
|
|
46
|
+
|
|
47
|
+
if not os.path.exists(const.SUPPORT_PACKAE.get(self.package_manager_name)):
|
|
48
|
+
logger.error(f"Cannot find the file({const.SUPPORT_PACKAE.get(self.package_manager_name)})")
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
if os.path.exists(self.tmp_dir):
|
|
52
|
+
shutil.rmtree(self.tmp_dir)
|
|
53
|
+
|
|
54
|
+
os.mkdir(self.tmp_dir)
|
|
55
|
+
shutil.copy(const.SUPPORT_PACKAE.get(self.package_manager_name),
|
|
56
|
+
os.path.join(self.tmp_dir, const.SUPPORT_PACKAE.get(self.package_manager_name)))
|
|
57
|
+
|
|
58
|
+
self.cur_path = os.getcwd()
|
|
59
|
+
os.chdir(self.tmp_dir)
|
|
60
|
+
|
|
61
|
+
with open(const.SUPPORT_PACKAE.get(self.package_manager_name), 'r', encoding='utf8') as f:
|
|
62
|
+
tmp_yml = yaml.safe_load(f)
|
|
63
|
+
tmp_yml['dev_dependencies'] = {'flutter_oss_licenses': '^2.0.1'}
|
|
64
|
+
with open(const.SUPPORT_PACKAE.get(self.package_manager_name), 'w', encoding='utf8') as f:
|
|
65
|
+
f.write(yaml.dump(tmp_yml))
|
|
66
|
+
|
|
67
|
+
cmd = "flutter pub get"
|
|
68
|
+
ret = subprocess.call(cmd, shell=True)
|
|
69
|
+
if ret != 0:
|
|
70
|
+
logger.error(f"Failed to run: {cmd}")
|
|
71
|
+
os.chdir(self.cur_path)
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
cmd = f"flutter pub run flutter_oss_licenses:generate.dart -o {self.input_file_name} --json"
|
|
75
|
+
ret = subprocess.call(cmd, shell=True)
|
|
76
|
+
if ret != 0:
|
|
77
|
+
logger.error(f"Failed to run: {cmd}")
|
|
78
|
+
os.chdir(self.cur_path)
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
return True
|
|
82
|
+
|
|
83
|
+
def parse_pub_deps_file(self, rel_json):
|
|
84
|
+
name_version_dict = {}
|
|
85
|
+
try:
|
|
86
|
+
for p in rel_json['packages']:
|
|
87
|
+
if p['kind'] == 'root':
|
|
88
|
+
self.package_name = p['name']
|
|
89
|
+
name_version_dict[p['name']] = p['version']
|
|
90
|
+
if p['dependencies'] == []:
|
|
91
|
+
continue
|
|
92
|
+
dep_key = f"{p['name']}({p['version']})"
|
|
93
|
+
if dep_key not in self.relation_tree:
|
|
94
|
+
self.relation_tree[dep_key] = []
|
|
95
|
+
self.relation_tree[dep_key].extend(p['dependencies'])
|
|
96
|
+
self.pkg_source_list[dep_key] = p['source']
|
|
97
|
+
|
|
98
|
+
for i in self.relation_tree:
|
|
99
|
+
tmp_dep = []
|
|
100
|
+
for d in self.relation_tree[i]:
|
|
101
|
+
d_ver = name_version_dict[d]
|
|
102
|
+
tmp_dep.append(f'{d}({d_ver})')
|
|
103
|
+
self.relation_tree[i] = []
|
|
104
|
+
self.relation_tree[i].extend(tmp_dep)
|
|
105
|
+
except Exception as e:
|
|
106
|
+
logger.error(f'Failed to parse dependency tree: {e}')
|
|
107
|
+
|
|
108
|
+
def parse_oss_information(self, f_name):
|
|
109
|
+
tmp_license_txt_file_name = 'tmp_license.txt'
|
|
110
|
+
json_data = ''
|
|
111
|
+
|
|
112
|
+
with open(f_name, 'r', encoding='utf8') as pub_file:
|
|
113
|
+
json_f = json.load(pub_file)
|
|
114
|
+
|
|
115
|
+
purl_dict = {}
|
|
116
|
+
for json_data in json_f:
|
|
117
|
+
try:
|
|
118
|
+
dep_item = DependencyItem()
|
|
119
|
+
oss_item = OssItem()
|
|
120
|
+
oss_origin_name = json_data['name']
|
|
121
|
+
if oss_origin_name not in self.total_dep_list:
|
|
122
|
+
continue
|
|
123
|
+
oss_item.name = f"{self.package_manager_name}:{oss_origin_name}"
|
|
124
|
+
oss_item.version = json_data['version']
|
|
125
|
+
if oss_item.version is None:
|
|
126
|
+
oss_item.version = ''
|
|
127
|
+
oss_item.homepage = json_data['homepage']
|
|
128
|
+
if oss_item.homepage is None:
|
|
129
|
+
oss_item.homepage = json_data['repository']
|
|
130
|
+
if oss_item.homepage is None:
|
|
131
|
+
oss_item.homepage = ''
|
|
132
|
+
oss_item.download_location = f"{self.dn_url}{oss_origin_name}/versions/{oss_item.version}"
|
|
133
|
+
dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
|
|
134
|
+
if json_data['isSdk']:
|
|
135
|
+
oss_item.download_location = json_data['repository'] or json_data['homepage'] or ''
|
|
136
|
+
oss_item.comment = 'SDK'
|
|
137
|
+
purl_dict[f'{oss_origin_name}({oss_item.version})'] = dep_item.purl
|
|
138
|
+
license_txt = json_data['license']
|
|
139
|
+
if license_txt is not None:
|
|
140
|
+
oss_item.license = check_license_name(license_txt)
|
|
141
|
+
|
|
142
|
+
if self.direct_dep:
|
|
143
|
+
if oss_origin_name not in self.total_dep_list:
|
|
144
|
+
continue
|
|
145
|
+
if self.package_name == f'{oss_origin_name}({oss_item.version})':
|
|
146
|
+
oss_item.comment = 'root package'
|
|
147
|
+
else:
|
|
148
|
+
if json_data['isDirectDependency']:
|
|
149
|
+
oss_item.comment = 'direct'
|
|
150
|
+
else:
|
|
151
|
+
oss_item.comment = 'transitive'
|
|
152
|
+
|
|
153
|
+
if f'{oss_origin_name}({oss_item.version})' in self.relation_tree:
|
|
154
|
+
dep_item.depends_on_raw = self.relation_tree[f'{oss_origin_name}({oss_item.version})']
|
|
155
|
+
if f'{oss_origin_name}({oss_item.version})' in self.pkg_source_list:
|
|
156
|
+
pkg_source = self.pkg_source_list[f'{oss_origin_name}({oss_item.version})']
|
|
157
|
+
if pkg_source in ['git', 'path']:
|
|
158
|
+
oss_item.download_location = json_data['repository']
|
|
159
|
+
if oss_item.download_location is None:
|
|
160
|
+
oss_item.download_location = json_data['homepage']
|
|
161
|
+
if oss_item.download_location is None:
|
|
162
|
+
oss_item.download_location = ''
|
|
163
|
+
oss_item.comment = pkg_source
|
|
164
|
+
|
|
165
|
+
dep_item.oss_items.append(oss_item)
|
|
166
|
+
self.dep_items.append(dep_item)
|
|
167
|
+
except Exception as e:
|
|
168
|
+
logger.error(f"Fail to parse pub oss information: {e}")
|
|
169
|
+
if self.direct_dep:
|
|
170
|
+
self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
|
|
171
|
+
|
|
172
|
+
if os.path.isfile(tmp_license_txt_file_name):
|
|
173
|
+
os.remove(tmp_license_txt_file_name)
|
|
174
|
+
|
|
175
|
+
return
|
|
176
|
+
|
|
177
|
+
def parse_no_dev_command_file(self, pub_deps):
|
|
178
|
+
for line in pub_deps.split('\n'):
|
|
179
|
+
re_result = re.findall(r'\-\s(\S+)\s', line)
|
|
180
|
+
if re_result:
|
|
181
|
+
self.total_dep_list.append(re_result[0])
|
|
182
|
+
self.total_dep_list = list(set(self.total_dep_list))
|
|
183
|
+
|
|
184
|
+
def parse_direct_dependencies(self):
|
|
185
|
+
self.direct_dep = True
|
|
186
|
+
tmp_pub_deps_file = 'tmp_deps.json'
|
|
187
|
+
tmp_no_dev_deps_file = 'tmp_no_dev_deps.txt'
|
|
188
|
+
encoding_list = ['utf8', 'utf16']
|
|
189
|
+
if os.path.exists(tmp_pub_deps_file) and os.path.exists(tmp_no_dev_deps_file):
|
|
190
|
+
for encode in encoding_list:
|
|
191
|
+
try:
|
|
192
|
+
logger.info(f'Try to encode with {encode}.')
|
|
193
|
+
with open(tmp_pub_deps_file, 'r+', encoding=encode) as deps_f:
|
|
194
|
+
lines = deps_f.readlines()
|
|
195
|
+
deps_f.seek(0)
|
|
196
|
+
deps_f.truncate()
|
|
197
|
+
for num, line in enumerate(lines):
|
|
198
|
+
if line.startswith('{'):
|
|
199
|
+
first_line = num
|
|
200
|
+
break
|
|
201
|
+
deps_f.writelines(lines[first_line:])
|
|
202
|
+
deps_f.seek(0)
|
|
203
|
+
deps_l = json.load(deps_f)
|
|
204
|
+
self.parse_pub_deps_file(deps_l)
|
|
205
|
+
with open(tmp_no_dev_deps_file, 'r', encoding=encode) as no_dev_f:
|
|
206
|
+
self.parse_no_dev_command_file(no_dev_f.read())
|
|
207
|
+
logger.info('Parse tmp pub deps file.')
|
|
208
|
+
except UnicodeDecodeError as e1:
|
|
209
|
+
logger.info(f'Fail to encode with {encode}: {e1}')
|
|
210
|
+
pass
|
|
211
|
+
except Exception as e:
|
|
212
|
+
logger.error(f'Fail to parse tmp pub deps result file: {e}')
|
|
213
|
+
return False
|
|
214
|
+
else:
|
|
215
|
+
logger.info(f'Success to encode with {encode}.')
|
|
216
|
+
break
|
|
217
|
+
else:
|
|
218
|
+
try:
|
|
219
|
+
cmd = "flutter pub get"
|
|
220
|
+
ret = subprocess.call(cmd, shell=True)
|
|
221
|
+
if ret != 0:
|
|
222
|
+
logger.error(f"Failed to run: {cmd}")
|
|
223
|
+
os.chdir(self.cur_path)
|
|
224
|
+
return False
|
|
225
|
+
|
|
226
|
+
cmd = "flutter pub deps --json"
|
|
227
|
+
ret_txt = subprocess.check_output(cmd, text=True, shell=True)
|
|
228
|
+
if ret_txt is not None:
|
|
229
|
+
deps_l = json.loads(ret_txt)
|
|
230
|
+
self.parse_pub_deps_file(deps_l)
|
|
231
|
+
else:
|
|
232
|
+
return False
|
|
233
|
+
|
|
234
|
+
cmd = "flutter pub deps --no-dev -s compact"
|
|
235
|
+
ret_no_dev = subprocess.check_output(cmd, text=True, shell=True, encoding='utf8')
|
|
236
|
+
if ret_no_dev != 0:
|
|
237
|
+
self.parse_no_dev_command_file(ret_no_dev)
|
|
238
|
+
|
|
239
|
+
except Exception as e:
|
|
240
|
+
logger.error(f'Fail to run flutter command:{e}')
|
|
241
|
+
return True
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2021 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import logging
|
|
8
|
+
import subprocess
|
|
9
|
+
import json
|
|
10
|
+
import shutil
|
|
11
|
+
import copy
|
|
12
|
+
import re
|
|
13
|
+
import fosslight_util.constant as constant
|
|
14
|
+
import fosslight_dependency.constant as const
|
|
15
|
+
from fosslight_dependency._package_manager import PackageManager
|
|
16
|
+
from fosslight_dependency._package_manager import get_url_to_purl, check_license_name
|
|
17
|
+
from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
|
|
18
|
+
from fosslight_util.oss_item import OssItem
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Pypi(PackageManager):
|
|
24
|
+
package_manager_name = const.PYPI
|
|
25
|
+
|
|
26
|
+
dn_url = 'https://pypi.org/project/'
|
|
27
|
+
venv_tmp_dir = 'venv_osc_dep_tmp'
|
|
28
|
+
tmp_file_name = "tmp_pip_license_output.json"
|
|
29
|
+
tmp_deptree_file = "tmp_pipdeptree.json"
|
|
30
|
+
pip_activate_cmd = ''
|
|
31
|
+
pip_deactivate_cmd = ''
|
|
32
|
+
|
|
33
|
+
def __init__(self, input_dir, output_dir, pip_activate_cmd, pip_deactivate_cmd):
|
|
34
|
+
super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
|
|
35
|
+
|
|
36
|
+
self.pip_activate_cmd = pip_activate_cmd
|
|
37
|
+
self.pip_deactivate_cmd = pip_deactivate_cmd
|
|
38
|
+
|
|
39
|
+
def __del__(self):
|
|
40
|
+
if os.path.isfile(self.tmp_file_name):
|
|
41
|
+
os.remove(self.tmp_file_name)
|
|
42
|
+
|
|
43
|
+
shutil.rmtree(self.venv_tmp_dir, ignore_errors=True)
|
|
44
|
+
|
|
45
|
+
if os.path.isfile(self.tmp_deptree_file):
|
|
46
|
+
os.remove(self.tmp_deptree_file)
|
|
47
|
+
|
|
48
|
+
def set_pip_activate_cmd(self, pip_activate_cmd):
|
|
49
|
+
self.pip_activate_cmd = pip_activate_cmd
|
|
50
|
+
|
|
51
|
+
def set_pip_deactivate_cmd(self, pip_deactivate_cmd):
|
|
52
|
+
self.pip_deactivate_cmd = pip_deactivate_cmd
|
|
53
|
+
|
|
54
|
+
def run_plugin(self):
|
|
55
|
+
ret = True
|
|
56
|
+
|
|
57
|
+
req_f = 'requirements.txt'
|
|
58
|
+
if os.path.exists(req_f):
|
|
59
|
+
with open(req_f, encoding='utf8') as rf:
|
|
60
|
+
for rf_line in rf.readlines():
|
|
61
|
+
ret_find = rf_line.find('--extra-index-url ')
|
|
62
|
+
if ret_find == -1:
|
|
63
|
+
ret_find = rf_line.find('--index-url ')
|
|
64
|
+
if ret_find == -1:
|
|
65
|
+
continue
|
|
66
|
+
self.cover_comment += rf_line
|
|
67
|
+
|
|
68
|
+
if not self.pip_activate_cmd and not self.pip_deactivate_cmd:
|
|
69
|
+
ret = self.create_virtualenv()
|
|
70
|
+
|
|
71
|
+
if ret:
|
|
72
|
+
ret = self.start_pip_inspect()
|
|
73
|
+
|
|
74
|
+
return ret
|
|
75
|
+
|
|
76
|
+
def create_virtualenv(self):
|
|
77
|
+
ret = True
|
|
78
|
+
|
|
79
|
+
manifest_files = self.manifest_file_name
|
|
80
|
+
if not manifest_files:
|
|
81
|
+
manifest_files = copy.deepcopy(const.SUPPORT_PACKAE[self.package_manager_name])
|
|
82
|
+
self.set_manifest_file(manifest_files)
|
|
83
|
+
|
|
84
|
+
install_cmd_list = []
|
|
85
|
+
for manifest_file in manifest_files:
|
|
86
|
+
if os.path.exists(manifest_file):
|
|
87
|
+
if manifest_file == 'requirements.txt':
|
|
88
|
+
install_cmd_list.append("pip install -r requirements.txt")
|
|
89
|
+
else:
|
|
90
|
+
install_cmd_list.append("pip install .")
|
|
91
|
+
else:
|
|
92
|
+
manifest_files.remove(manifest_file)
|
|
93
|
+
self.set_manifest_file(manifest_files)
|
|
94
|
+
|
|
95
|
+
venv_path = os.path.join(self.input_dir, self.venv_tmp_dir)
|
|
96
|
+
|
|
97
|
+
if self.platform == const.WINDOWS:
|
|
98
|
+
create_venv_cmd = f"python -m venv {self.venv_tmp_dir}"
|
|
99
|
+
activate_cmd = os.path.join(self.venv_tmp_dir, "Scripts", "activate.bat")
|
|
100
|
+
cmd_separator = "&"
|
|
101
|
+
else:
|
|
102
|
+
create_venv_cmd = f"virtualenv -p python3 {self.venv_tmp_dir}"
|
|
103
|
+
activate_cmd = ". " + os.path.join(venv_path, "bin", "activate")
|
|
104
|
+
cmd_separator = ";"
|
|
105
|
+
|
|
106
|
+
if install_cmd_list:
|
|
107
|
+
install_cmd = cmd_separator.join(install_cmd_list)
|
|
108
|
+
else:
|
|
109
|
+
logger.error(const.SUPPORT_PACKAE[self.package_manager_name])
|
|
110
|
+
logger.error('Cannot create virtualenv because it cannot find: '
|
|
111
|
+
+ ', '.join(const.SUPPORT_PACKAE[self.package_manager_name]))
|
|
112
|
+
logger.error("Please run with '-a' and '-d' option.")
|
|
113
|
+
return False
|
|
114
|
+
|
|
115
|
+
deactivate_cmd = "deactivate"
|
|
116
|
+
pip_upgrade_cmd = "pip install --upgrade pip"
|
|
117
|
+
|
|
118
|
+
self.set_pip_activate_cmd(activate_cmd)
|
|
119
|
+
self.set_pip_deactivate_cmd(deactivate_cmd)
|
|
120
|
+
|
|
121
|
+
cmd_list = [create_venv_cmd, activate_cmd, install_cmd, pip_upgrade_cmd, deactivate_cmd]
|
|
122
|
+
cmd = cmd_separator.join(cmd_list)
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
cmd_ret = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE)
|
|
126
|
+
if cmd_ret.returncode != 0:
|
|
127
|
+
ret = False
|
|
128
|
+
err_msg = f"return code({cmd_ret.returncode})"
|
|
129
|
+
elif cmd_ret.stderr.decode('utf-8').strip().lower().startswith('error:'):
|
|
130
|
+
ret = False
|
|
131
|
+
err_msg = f"stderr msg({cmd_ret.stderr})"
|
|
132
|
+
except Exception as e:
|
|
133
|
+
ret = False
|
|
134
|
+
err_msg = e
|
|
135
|
+
finally:
|
|
136
|
+
try:
|
|
137
|
+
if (not ret) and (self.platform != const.WINDOWS):
|
|
138
|
+
ret = True
|
|
139
|
+
create_venv_cmd = f"python3 -m venv {self.venv_tmp_dir}"
|
|
140
|
+
|
|
141
|
+
cmd_list = [create_venv_cmd, activate_cmd, install_cmd, pip_upgrade_cmd, deactivate_cmd]
|
|
142
|
+
cmd = cmd_separator.join(cmd_list)
|
|
143
|
+
cmd_ret = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE)
|
|
144
|
+
if cmd_ret.returncode != 0:
|
|
145
|
+
ret = False
|
|
146
|
+
err_msg = f"return code({cmd_ret.returncode})"
|
|
147
|
+
elif cmd_ret.stderr.decode('utf-8').strip().lower().startswith('error:'):
|
|
148
|
+
ret = False
|
|
149
|
+
err_msg = f"stderr msg({cmd_ret.stderr})"
|
|
150
|
+
except Exception as e:
|
|
151
|
+
ret = False
|
|
152
|
+
err_msg = e
|
|
153
|
+
if ret:
|
|
154
|
+
logger.info(f"Created the temporary virtualenv({venv_path}).")
|
|
155
|
+
else:
|
|
156
|
+
logger.error(f"Failed to create virtualenv: {err_msg}")
|
|
157
|
+
|
|
158
|
+
return ret
|
|
159
|
+
|
|
160
|
+
def start_pip_inspect(self):
|
|
161
|
+
ret = True
|
|
162
|
+
pipdeptree = 'pipdeptree'
|
|
163
|
+
tmp_pip_list = "tmp_list.txt"
|
|
164
|
+
python_cmd = "python -m"
|
|
165
|
+
|
|
166
|
+
if self.pip_activate_cmd.startswith("source "):
|
|
167
|
+
tmp_activate = self.pip_activate_cmd[7:]
|
|
168
|
+
pip_activate_cmd = f". {tmp_activate}"
|
|
169
|
+
elif self.pip_activate_cmd.startswith("conda "):
|
|
170
|
+
if self.platform == const.LINUX:
|
|
171
|
+
tmp_activate = "eval \"$(conda shell.bash hook)\";"
|
|
172
|
+
pip_activate_cmd = tmp_activate + self.pip_activate_cmd
|
|
173
|
+
else:
|
|
174
|
+
pip_activate_cmd = self.pip_activate_cmd
|
|
175
|
+
|
|
176
|
+
if self.platform == const.WINDOWS:
|
|
177
|
+
command_separator = "&"
|
|
178
|
+
else:
|
|
179
|
+
command_separator = ";"
|
|
180
|
+
|
|
181
|
+
activate_command = pip_activate_cmd
|
|
182
|
+
pip_list_command = f"{python_cmd} pip freeze > {tmp_pip_list}"
|
|
183
|
+
deactivate_command = self.pip_deactivate_cmd
|
|
184
|
+
|
|
185
|
+
command_list = [activate_command, pip_list_command, deactivate_command]
|
|
186
|
+
command = command_separator.join(command_list)
|
|
187
|
+
|
|
188
|
+
exists_pipdeptree = False
|
|
189
|
+
try:
|
|
190
|
+
cmd_ret = subprocess.call(command, shell=True)
|
|
191
|
+
if cmd_ret != 0:
|
|
192
|
+
ret = False
|
|
193
|
+
err_msg = f"cmd ret code({cmd_ret})"
|
|
194
|
+
else:
|
|
195
|
+
if os.path.isfile(tmp_pip_list):
|
|
196
|
+
with open(tmp_pip_list, 'r', encoding='utf-8') as pip_list_file:
|
|
197
|
+
for pip_list in pip_list_file.readlines():
|
|
198
|
+
pip_list_name = pip_list.split('==')[0]
|
|
199
|
+
if pip_list_name == pipdeptree:
|
|
200
|
+
exists_pipdeptree = True
|
|
201
|
+
break
|
|
202
|
+
os.remove(tmp_pip_list)
|
|
203
|
+
except Exception as e:
|
|
204
|
+
ret = False
|
|
205
|
+
err_msg = str(e)
|
|
206
|
+
finally:
|
|
207
|
+
if not ret:
|
|
208
|
+
logger.error(f"Failed to freeze dependencies ({command}): {err_msg})")
|
|
209
|
+
return False
|
|
210
|
+
|
|
211
|
+
command_list = []
|
|
212
|
+
command_list.append(activate_command)
|
|
213
|
+
|
|
214
|
+
pip_inspect_command = f"{python_cmd} pip inspect > {self.tmp_file_name}"
|
|
215
|
+
command_list.append(pip_inspect_command)
|
|
216
|
+
|
|
217
|
+
if not exists_pipdeptree:
|
|
218
|
+
install_deptree_command = f"{python_cmd} pip install {pipdeptree}"
|
|
219
|
+
command_list.append(install_deptree_command)
|
|
220
|
+
uninstall_deptree_command = f"{python_cmd} pip uninstall -y {pipdeptree}"
|
|
221
|
+
pipdeptree_command = f"{pipdeptree} --json-tree -e 'pipdeptree,pip,wheel,setuptools' > {self.tmp_deptree_file}"
|
|
222
|
+
command_list.append(pipdeptree_command)
|
|
223
|
+
|
|
224
|
+
if not exists_pipdeptree:
|
|
225
|
+
command_list.append(uninstall_deptree_command)
|
|
226
|
+
|
|
227
|
+
command_list.append(deactivate_command)
|
|
228
|
+
command = command_separator.join(command_list)
|
|
229
|
+
|
|
230
|
+
try:
|
|
231
|
+
cmd_ret = subprocess.call(command, shell=True)
|
|
232
|
+
if cmd_ret == 0:
|
|
233
|
+
if os.path.exists(self.tmp_file_name):
|
|
234
|
+
self.append_input_package_list_file(self.tmp_file_name)
|
|
235
|
+
|
|
236
|
+
with open(self.tmp_file_name, 'r', encoding='utf-8') as json_f:
|
|
237
|
+
inspect_data = json.load(json_f)
|
|
238
|
+
for package in inspect_data.get('installed', []):
|
|
239
|
+
metadata = package.get('metadata', {})
|
|
240
|
+
package_name = metadata.get('name', '')
|
|
241
|
+
if package_name:
|
|
242
|
+
if package_name in ['pip', 'setuptools', 'wheel']:
|
|
243
|
+
continue
|
|
244
|
+
self.total_dep_list.append(re.sub(r"[-_.]+", "-", package_name).lower())
|
|
245
|
+
else:
|
|
246
|
+
logger.error(f"pip inspect output file not found: {self.tmp_file_name}")
|
|
247
|
+
ret = False
|
|
248
|
+
else:
|
|
249
|
+
logger.error(f"Failed to run command: {command}")
|
|
250
|
+
ret = False
|
|
251
|
+
except Exception as e:
|
|
252
|
+
ret = False
|
|
253
|
+
logger.error(f"Failed to get package information using pip inspect: {e}")
|
|
254
|
+
|
|
255
|
+
return ret
|
|
256
|
+
|
|
257
|
+
def parse_oss_information(self, f_name):
|
|
258
|
+
purl_dict = {}
|
|
259
|
+
try:
|
|
260
|
+
oss_init_name = ''
|
|
261
|
+
with open(f_name, 'r', encoding='utf-8') as json_file:
|
|
262
|
+
inspect_data = json.load(json_file)
|
|
263
|
+
|
|
264
|
+
for package in inspect_data.get('installed', []):
|
|
265
|
+
dep_item = DependencyItem()
|
|
266
|
+
oss_item = OssItem()
|
|
267
|
+
metadata = package.get('metadata', {})
|
|
268
|
+
if not metadata:
|
|
269
|
+
continue
|
|
270
|
+
|
|
271
|
+
oss_init_name = metadata.get('name', '')
|
|
272
|
+
oss_init_name = re.sub(r"[-_.]+", "-", oss_init_name).lower()
|
|
273
|
+
if oss_init_name not in self.total_dep_list:
|
|
274
|
+
continue
|
|
275
|
+
oss_item.name = f"{self.package_manager_name}:{oss_init_name}"
|
|
276
|
+
oss_item.version = metadata.get('version', '')
|
|
277
|
+
|
|
278
|
+
# license_expression > license > classifier
|
|
279
|
+
license_info = check_UNKNOWN(metadata.get('license_expression', ''))
|
|
280
|
+
if not license_info:
|
|
281
|
+
license_info = metadata.get('license', '')
|
|
282
|
+
if '\n' in license_info:
|
|
283
|
+
license_info = check_UNKNOWN(check_license_name(license_info))
|
|
284
|
+
if not license_info:
|
|
285
|
+
classifiers = metadata.get('classifier', [])
|
|
286
|
+
license_classifiers = [c for c in classifiers if c.startswith('License ::')]
|
|
287
|
+
if license_classifiers:
|
|
288
|
+
license_info_l = []
|
|
289
|
+
for license_classifier in license_classifiers:
|
|
290
|
+
if license_classifier.startswith('License :: OSI Approved ::'):
|
|
291
|
+
license_info_l.append(license_classifier.split('::')[-1].strip())
|
|
292
|
+
break
|
|
293
|
+
license_info = ','.join(license_info_l)
|
|
294
|
+
license_name = check_UNKNOWN(license_info)
|
|
295
|
+
if license_name:
|
|
296
|
+
license_name = license_name.replace(';', ',')
|
|
297
|
+
oss_item.license = license_name
|
|
298
|
+
|
|
299
|
+
oss_item.download_location = f"{self.dn_url}{oss_init_name}/{oss_item.version}"
|
|
300
|
+
|
|
301
|
+
# project_url 'source' > download_url > project_url 'homepage' > homepage
|
|
302
|
+
homepage_url = check_UNKNOWN(metadata.get('home_page', ''))
|
|
303
|
+
download_url = check_UNKNOWN(metadata.get('download_url', ''))
|
|
304
|
+
project_urls = check_UNKNOWN(metadata.get('project_url', []))
|
|
305
|
+
if project_urls:
|
|
306
|
+
priority_order = ['source', 'repository', 'github', 'code', 'source code', 'homepage']
|
|
307
|
+
for priority in priority_order:
|
|
308
|
+
for url_entry in project_urls:
|
|
309
|
+
url_entry_lower = url_entry.lower()
|
|
310
|
+
if url_entry_lower.startswith(priority):
|
|
311
|
+
download_url = url_entry.split(', ')[-1]
|
|
312
|
+
break
|
|
313
|
+
if download_url:
|
|
314
|
+
break
|
|
315
|
+
oss_item.homepage = download_url or homepage_url or f"{self.dn_url}{oss_init_name}"
|
|
316
|
+
|
|
317
|
+
dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
|
|
318
|
+
purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
|
|
319
|
+
|
|
320
|
+
direct_url = package.get('direct_url', {})
|
|
321
|
+
if direct_url:
|
|
322
|
+
oss_item.download_location = direct_url.get('url', '')
|
|
323
|
+
oss_item.homepage = oss_item.download_location
|
|
324
|
+
if not package.get('installer', ''):
|
|
325
|
+
oss_item.download_location = oss_item.homepage
|
|
326
|
+
|
|
327
|
+
if oss_init_name == self.package_name:
|
|
328
|
+
oss_item.comment = 'root package'
|
|
329
|
+
elif self.direct_dep and len(self.direct_dep_list) > 0:
|
|
330
|
+
if f'{oss_init_name}({oss_item.version})' in self.direct_dep_list:
|
|
331
|
+
oss_item.comment = 'direct'
|
|
332
|
+
else:
|
|
333
|
+
oss_item.comment = 'transitive'
|
|
334
|
+
if f'{oss_init_name}({oss_item.version})' in self.relation_tree:
|
|
335
|
+
dep_item.depends_on_raw = self.relation_tree[f'{oss_init_name}({oss_item.version})']
|
|
336
|
+
|
|
337
|
+
dep_item.oss_items.append(oss_item)
|
|
338
|
+
self.dep_items.append(dep_item)
|
|
339
|
+
|
|
340
|
+
except Exception as ex:
|
|
341
|
+
logger.warning(f"Fail to parse oss information: {oss_init_name}({ex})")
|
|
342
|
+
if self.direct_dep:
|
|
343
|
+
self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
|
|
344
|
+
return
|
|
345
|
+
|
|
346
|
+
def get_dependencies(self, dependencies, package):
|
|
347
|
+
package_name = 'package_name'
|
|
348
|
+
deps = 'dependencies'
|
|
349
|
+
installed_ver = 'installed_version'
|
|
350
|
+
|
|
351
|
+
pkg_name = re.sub(r"[-_.]+", "-", package[package_name]).lower()
|
|
352
|
+
pkg_ver = package[installed_ver]
|
|
353
|
+
dependency_list = package[deps]
|
|
354
|
+
dependencies[f"{pkg_name}({pkg_ver})"] = []
|
|
355
|
+
for dependency in dependency_list:
|
|
356
|
+
dep_name = re.sub(r"[-_.]+", "-", dependency[package_name]).lower()
|
|
357
|
+
dep_version = dependency[installed_ver]
|
|
358
|
+
dependencies[f"{pkg_name}({pkg_ver})"].append(f"{dep_name}({dep_version})")
|
|
359
|
+
if dependency[deps] != []:
|
|
360
|
+
dependencies = self.get_dependencies(dependencies, dependency)
|
|
361
|
+
return dependencies
|
|
362
|
+
|
|
363
|
+
def parse_direct_dependencies(self):
|
|
364
|
+
self.direct_dep = True
|
|
365
|
+
if not os.path.exists(self.tmp_deptree_file):
|
|
366
|
+
self.direct_dep = False
|
|
367
|
+
return
|
|
368
|
+
try:
|
|
369
|
+
with open(self.tmp_deptree_file, 'r', encoding='utf8') as f:
|
|
370
|
+
json_f = json.load(f)
|
|
371
|
+
root_package = json_f
|
|
372
|
+
if ('pyproject.toml' in self.manifest_file_name) or ('setup.py' in self.manifest_file_name):
|
|
373
|
+
direct_without_system_package = 0
|
|
374
|
+
for package in root_package:
|
|
375
|
+
package_name = re.sub(r"[-_.]+", "-", package['package_name']).lower()
|
|
376
|
+
if package_name in self.total_dep_list:
|
|
377
|
+
direct_without_system_package += 1
|
|
378
|
+
if direct_without_system_package == 1:
|
|
379
|
+
self.package_name = re.sub(r"[-_.]+", "-", json_f[0]['package_name']).lower()
|
|
380
|
+
root_package = json_f[0]['dependencies']
|
|
381
|
+
|
|
382
|
+
for package in root_package:
|
|
383
|
+
package_name = re.sub(r"[-_.]+", "-", package['package_name']).lower()
|
|
384
|
+
self.direct_dep_list.append(f"{package_name}({package['installed_version']})")
|
|
385
|
+
if package['dependencies'] == []:
|
|
386
|
+
continue
|
|
387
|
+
self.relation_tree = self.get_dependencies(self.relation_tree, package)
|
|
388
|
+
except Exception as e:
|
|
389
|
+
logger.warning(f'Fail to parse direct dependency: {e}')
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def check_UNKNOWN(text):
|
|
393
|
+
if text == ['UNKNOWN'] or text == 'UNKNOWN':
|
|
394
|
+
text = ""
|
|
395
|
+
return text
|