fosslight-scanner 1.7.31__py3-none-any.whl → 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fosslight_scanner/_run_compare.py +14 -5
- fosslight_scanner/common.py +72 -160
- fosslight_scanner/fosslight_scanner.py +40 -55
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.1.dist-info}/METADATA +6 -6
- fosslight_scanner-2.0.1.dist-info/RECORD +15 -0
- fosslight_scanner-1.7.31.dist-info/RECORD +0 -15
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.1.dist-info}/LICENSE +0 -0
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.1.dist-info}/WHEEL +0 -0
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.1.dist-info}/entry_points.txt +0 -0
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.1.dist-info}/top_level.txt +0 -0
|
@@ -14,7 +14,8 @@ from pathlib import Path
|
|
|
14
14
|
from bs4 import BeautifulSoup
|
|
15
15
|
import fosslight_util.constant as constant
|
|
16
16
|
from fosslight_util.compare_yaml import compare_yaml
|
|
17
|
-
from fosslight_util.
|
|
17
|
+
from fosslight_util.read_excel import read_oss_report
|
|
18
|
+
from fosslight_util.parsing_yaml import parsing_yml
|
|
18
19
|
|
|
19
20
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
20
21
|
ADD = "add"
|
|
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
|
|
|
255
256
|
|
|
256
257
|
result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
|
|
257
258
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
259
|
+
before_basepath = os.path.dirname(before_f)
|
|
260
|
+
after_basepath = os.path.dirname(after_f)
|
|
261
|
+
if XLSX_EXT == before_ext:
|
|
262
|
+
before_fileitems = read_oss_report(before_f, "", before_basepath)
|
|
263
|
+
elif YAML_EXT == before_ext:
|
|
264
|
+
before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
|
|
265
|
+
if XLSX_EXT == after_ext:
|
|
266
|
+
after_fileitems = read_oss_report(after_f, after_basepath)
|
|
267
|
+
elif YAML_EXT == after_ext:
|
|
268
|
+
after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
|
|
269
|
+
|
|
270
|
+
compared_result = compare_yaml(before_fileitems, after_fileitems)
|
|
262
271
|
if compared_result != '':
|
|
263
272
|
count_compared_result(compared_result)
|
|
264
273
|
ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)
|
fosslight_scanner/common.py
CHANGED
|
@@ -7,18 +7,12 @@ import os
|
|
|
7
7
|
import sys
|
|
8
8
|
import logging
|
|
9
9
|
import shutil
|
|
10
|
-
import
|
|
11
|
-
import
|
|
12
|
-
|
|
13
|
-
import fosslight_util.constant as constant
|
|
14
|
-
from fosslight_util.parsing_yaml import parsing_yml
|
|
15
|
-
from fosslight_util.write_yaml import create_yaml_with_ossitem
|
|
10
|
+
import copy
|
|
11
|
+
from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY
|
|
16
12
|
from fosslight_util.write_scancodejson import write_scancodejson
|
|
17
|
-
from fosslight_util.
|
|
18
|
-
from fosslight_util.output_format import write_output_file
|
|
19
|
-
from fosslight_util.oss_item import OssItem
|
|
13
|
+
from fosslight_util.oss_item import OssItem, FileItem
|
|
20
14
|
|
|
21
|
-
logger = logging.getLogger(
|
|
15
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
22
16
|
SRC_SHEET = 'SRC_FL_Source'
|
|
23
17
|
BIN_SHEET = 'BIN_FL_Binary'
|
|
24
18
|
BIN_EXT_HEADER = {
|
|
@@ -93,72 +87,23 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
|
|
|
93
87
|
return success, result or []
|
|
94
88
|
|
|
95
89
|
|
|
96
|
-
def
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if column_name in df.columns:
|
|
109
|
-
updated = (df[column_name] == '') | (df[column_name].isnull())
|
|
110
|
-
df.loc[updated, column_name] = oss_name
|
|
111
|
-
df.to_excel(file_path, sheet_name=sheet_name, index=False)
|
|
112
|
-
except Exception as ex:
|
|
113
|
-
logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
|
|
114
|
-
except Exception as ex:
|
|
115
|
-
logger.debug(f"overwrite_excel:{ex}")
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
|
|
119
|
-
default_oss_name='', default_oss_version='', url=''):
|
|
120
|
-
success = True
|
|
121
|
-
err_msg = ''
|
|
122
|
-
|
|
123
|
-
oss_total_list = []
|
|
124
|
-
yaml_dict = {}
|
|
125
|
-
try:
|
|
126
|
-
for mf in merge_yaml_files:
|
|
127
|
-
if os.path.exists(os.path.join(_output_dir, mf)):
|
|
128
|
-
oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
|
|
129
|
-
|
|
130
|
-
if remove_src_data:
|
|
131
|
-
existed_yaml = {}
|
|
132
|
-
for oi in oss_list:
|
|
133
|
-
oi.name = default_oss_name if oi.name == '' else oi.name
|
|
134
|
-
oi.version = default_oss_version if oi.version == '' else oi.version
|
|
135
|
-
oi.download_location = url if oi.download_location == '' else oi.download_location
|
|
136
|
-
create_yaml_with_ossitem(oi, existed_yaml)
|
|
137
|
-
with open(os.path.join(_output_dir, mf), 'w') as f:
|
|
138
|
-
yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
|
|
139
|
-
|
|
140
|
-
oss_total_list.extend(oss_list)
|
|
141
|
-
|
|
142
|
-
if oss_total_list != []:
|
|
143
|
-
for oti in oss_total_list:
|
|
144
|
-
create_yaml_with_ossitem(oti, yaml_dict)
|
|
145
|
-
with open(os.path.join(_output_dir, final_report), 'w') as f:
|
|
146
|
-
yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
|
|
147
|
-
else:
|
|
148
|
-
success = False
|
|
149
|
-
err_msg = "Output file is not created as no oss items detected."
|
|
150
|
-
except Exception as ex:
|
|
151
|
-
err_msg = ex
|
|
152
|
-
success = False
|
|
153
|
-
|
|
154
|
-
return success, err_msg
|
|
90
|
+
def update_oss_item(scan_item, oss_name, oss_version, download_loc):
|
|
91
|
+
for file_items in scan_item.file_items.values():
|
|
92
|
+
for file_item in file_items:
|
|
93
|
+
if file_item.oss_items:
|
|
94
|
+
for oi in file_item.oss_items:
|
|
95
|
+
if oi.name == '' and oi.version == '' and oi.download_location == '':
|
|
96
|
+
oi.name = oss_name
|
|
97
|
+
oi.version = oss_version
|
|
98
|
+
oi.download_location = download_loc
|
|
99
|
+
else:
|
|
100
|
+
file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
|
|
101
|
+
return scan_item
|
|
155
102
|
|
|
156
103
|
|
|
157
|
-
def create_scancodejson(
|
|
104
|
+
def create_scancodejson(all_scan_item_origin, ui_mode_report, src_path=""):
|
|
158
105
|
success = True
|
|
159
106
|
err_msg = ''
|
|
160
|
-
|
|
161
|
-
oss_total_list = []
|
|
162
107
|
root_dir = ""
|
|
163
108
|
root_strip = ""
|
|
164
109
|
try:
|
|
@@ -170,28 +115,37 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
|
|
|
170
115
|
root_dir = ""
|
|
171
116
|
|
|
172
117
|
try:
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
118
|
+
all_scan_item = copy.deepcopy(all_scan_item_origin)
|
|
119
|
+
if FOSSLIGHT_DEPENDENCY in all_scan_item.file_items:
|
|
120
|
+
del all_scan_item.file_items[FOSSLIGHT_DEPENDENCY]
|
|
176
121
|
if src_path:
|
|
177
|
-
|
|
122
|
+
fileitems_without_oss = []
|
|
123
|
+
for root, _, files in os.walk(src_path):
|
|
178
124
|
root = root.replace(root_strip, "")
|
|
179
125
|
for file in files:
|
|
126
|
+
fi_without_oss = FileItem('')
|
|
127
|
+
included = False
|
|
180
128
|
item_path = os.path.join(root, file)
|
|
181
129
|
item_path = item_path.replace(parent + os.path.sep, '', 1)
|
|
182
|
-
|
|
130
|
+
|
|
131
|
+
for file_items in all_scan_item.file_items.values():
|
|
132
|
+
for file_item in file_items:
|
|
133
|
+
if file_item.source_name_or_path:
|
|
134
|
+
if file_item.source_name_or_path == item_path:
|
|
135
|
+
included = True
|
|
136
|
+
break
|
|
183
137
|
if not included:
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
138
|
+
fi_without_oss.source_name_or_path = item_path
|
|
139
|
+
fileitems_without_oss.append(fi_without_oss)
|
|
140
|
+
if len(fileitems_without_oss) > 0:
|
|
141
|
+
all_scan_item.file_items[FOSSLIGHT_SOURCE].extend(fileitems_without_oss)
|
|
187
142
|
if root_dir:
|
|
188
|
-
for
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
143
|
+
for file_items in all_scan_item.file_items.values():
|
|
144
|
+
for fi in file_items:
|
|
145
|
+
if fi.source_name_or_path:
|
|
146
|
+
fi.source_name_or_path = os.path.join(root_dir, fi.source_name_or_path)
|
|
193
147
|
write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
|
|
194
|
-
|
|
148
|
+
all_scan_item)
|
|
195
149
|
except Exception as ex:
|
|
196
150
|
err_msg = ex
|
|
197
151
|
success = False
|
|
@@ -199,100 +153,58 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
|
|
|
199
153
|
return success, err_msg
|
|
200
154
|
|
|
201
155
|
|
|
202
|
-
def correct_scanner_result(
|
|
203
|
-
src_oss_list = []
|
|
204
|
-
bin_oss_list = []
|
|
156
|
+
def correct_scanner_result(all_scan_item):
|
|
205
157
|
duplicates = False
|
|
206
158
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
if
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
if exist_src and exist_bin:
|
|
159
|
+
keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
|
|
160
|
+
is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
|
|
161
|
+
if is_contained:
|
|
162
|
+
src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
|
|
163
|
+
bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
|
|
213
164
|
try:
|
|
214
165
|
remove_src_idx_list = []
|
|
215
|
-
for idx_src,
|
|
166
|
+
for idx_src, src_fileitem in enumerate(src_fileitems):
|
|
167
|
+
src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path, src_fileitem.exclude)
|
|
216
168
|
dup_flag = False
|
|
217
|
-
for
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
|
|
169
|
+
for bin_fileitem in bin_fileitems:
|
|
170
|
+
bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path, bin_fileitem.exclude)
|
|
171
|
+
if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
|
|
221
172
|
dup_flag = True
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
173
|
+
src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
|
|
174
|
+
bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
|
|
175
|
+
|
|
176
|
+
if src_all_licenses_non_empty and bin_empty_license_exists:
|
|
177
|
+
exclude = bin_fileitem.oss_items[0].exclude
|
|
178
|
+
bin_fileitem.oss_items = []
|
|
179
|
+
for src_oss_item in src_fileitem.oss_items:
|
|
180
|
+
src_oss_item.exclude = exclude
|
|
181
|
+
bin_fileitem.oss_items.append(src_oss_item)
|
|
182
|
+
bin_fileitem.comment = 'Loaded from SRC OSS info'
|
|
228
183
|
if dup_flag:
|
|
229
184
|
remove_src_idx_list.append(idx_src)
|
|
230
185
|
if remove_src_idx_list:
|
|
231
186
|
duplicates = True
|
|
232
187
|
for i in sorted(remove_src_idx_list, reverse=True):
|
|
233
|
-
del
|
|
188
|
+
del src_fileitems[i]
|
|
234
189
|
except Exception as ex:
|
|
235
190
|
logger.warning(f"correct the scanner result:{ex}")
|
|
236
191
|
|
|
237
192
|
try:
|
|
238
|
-
if exist_src:
|
|
239
|
-
success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
|
|
240
|
-
output_extension, SRC_SHEET)
|
|
241
|
-
if not success:
|
|
242
|
-
logger.warning(err_msg)
|
|
243
|
-
if exist_bin:
|
|
244
|
-
success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
|
|
245
|
-
output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
|
|
246
|
-
if not success:
|
|
247
|
-
logger.warning(err_msg)
|
|
248
193
|
if duplicates:
|
|
249
194
|
logger.info('Success to correct the src/bin scanner result')
|
|
250
195
|
except Exception as ex:
|
|
251
196
|
logger.warning(f"Corrected src/bin scanner result:{ex}")
|
|
252
|
-
return
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
|
|
256
|
-
new_oss_list = []
|
|
257
|
-
sheet_list = {}
|
|
258
|
-
sheet_list[sheetname] = []
|
|
259
|
-
|
|
260
|
-
for src_item in oss_list:
|
|
261
|
-
scanner_name = constant.supported_sheet_and_scanner[sheetname]
|
|
262
|
-
new_oss_list.append(src_item.get_print_array(scanner_name)[0])
|
|
263
|
-
sheet_list[sheetname].extend(new_oss_list)
|
|
264
|
-
if os.path.exists(os.path.join(output_dir, output_file)):
|
|
265
|
-
os.remove(os.path.join(output_dir, output_file))
|
|
266
|
-
success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
|
|
267
|
-
output_extension, sheet_list, extended_hdr, hidden_hdr)
|
|
268
|
-
return success, err_msg
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
|
|
272
|
-
err_reason = ''
|
|
273
|
-
oss_list = []
|
|
274
|
-
oss_file_with_fullpath = os.path.join(_output_dir, output_file)
|
|
275
|
-
|
|
276
|
-
if os.path.exists(oss_file_with_fullpath):
|
|
277
|
-
if output_extension == '.xlsx':
|
|
278
|
-
oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
|
|
279
|
-
elif output_extension == '.yaml':
|
|
280
|
-
oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
|
|
281
|
-
else:
|
|
282
|
-
err_reason = f'Not supported extension: {output_extension}'
|
|
283
|
-
if err_reason:
|
|
284
|
-
logger.info(f'get_osslist: {err_reason}')
|
|
285
|
-
return oss_list
|
|
197
|
+
return all_scan_item
|
|
286
198
|
|
|
287
199
|
|
|
288
|
-
def check_exclude_dir(
|
|
200
|
+
def check_exclude_dir(source_name_or_path, file_item_exclude):
|
|
201
|
+
if file_item_exclude:
|
|
202
|
+
return True
|
|
289
203
|
_exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
|
|
204
|
+
exclude = False
|
|
290
205
|
|
|
291
|
-
for
|
|
292
|
-
if
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
oss_item.exclude = True
|
|
297
|
-
break
|
|
298
|
-
return oss_list
|
|
206
|
+
for exclude_dir in _exclude_dirs:
|
|
207
|
+
if exclude_dir in source_name_or_path.split(os.path.sep):
|
|
208
|
+
exclude = True
|
|
209
|
+
break
|
|
210
|
+
return exclude
|
|
@@ -25,12 +25,13 @@ from fosslight_util.timer_thread import TimerThread
|
|
|
25
25
|
import fosslight_util.constant as constant
|
|
26
26
|
from fosslight_util.output_format import check_output_format
|
|
27
27
|
from fosslight_prechecker._precheck import run_lint as prechecker_lint
|
|
28
|
-
from fosslight_util.write_excel import merge_excels, merge_cover_comment
|
|
29
28
|
from fosslight_util.cover import CoverItem
|
|
29
|
+
from fosslight_util.oss_item import ScannerItem
|
|
30
|
+
from fosslight_util.output_format import write_output_file
|
|
30
31
|
|
|
31
32
|
from .common import (
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
call_analysis_api, update_oss_item,
|
|
34
|
+
correct_scanner_result, create_scancodejson
|
|
34
35
|
)
|
|
35
36
|
from ._run_compare import run_compare
|
|
36
37
|
|
|
@@ -56,7 +57,7 @@ SCANNER_MODE = [
|
|
|
56
57
|
|
|
57
58
|
|
|
58
59
|
def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
|
|
59
|
-
|
|
60
|
+
result = []
|
|
60
61
|
|
|
61
62
|
package_manager = ""
|
|
62
63
|
pip_activate_cmd = ""
|
|
@@ -90,7 +91,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
90
91
|
timer.start()
|
|
91
92
|
|
|
92
93
|
try:
|
|
93
|
-
success,
|
|
94
|
+
success, scan_item = call_analysis_api(
|
|
94
95
|
path_to_analyze, "Dependency Analysis",
|
|
95
96
|
1, run_dependency_scanner,
|
|
96
97
|
package_manager,
|
|
@@ -101,11 +102,11 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
101
102
|
github_token, path_to_exclude=path_to_exclude
|
|
102
103
|
)
|
|
103
104
|
if success:
|
|
104
|
-
|
|
105
|
+
result = scan_item
|
|
105
106
|
except Exception as ex:
|
|
106
107
|
logger.warning(f"Run dependency: {ex}")
|
|
107
108
|
|
|
108
|
-
return
|
|
109
|
+
return result
|
|
109
110
|
|
|
110
111
|
|
|
111
112
|
def source_analysis_wrapper(*args, **kwargs):
|
|
@@ -130,7 +131,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
130
131
|
source_time_out=120, binary_simple=False):
|
|
131
132
|
final_excel_dir = output_path
|
|
132
133
|
success = True
|
|
133
|
-
|
|
134
|
+
all_cover_items = []
|
|
135
|
+
all_scan_item = ScannerItem(PKG_NAME, _start_time)
|
|
134
136
|
if not remove_src_data:
|
|
135
137
|
success, final_excel_dir, result_log = init(output_path)
|
|
136
138
|
|
|
@@ -158,9 +160,6 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
158
160
|
-1, prechecker_lint,
|
|
159
161
|
abs_path, False, output_prechecker,
|
|
160
162
|
exclude_path=path_to_exclude)
|
|
161
|
-
success_file, copied_file = copy_file(output_prechecker, output_path)
|
|
162
|
-
if success_file:
|
|
163
|
-
temp_output_fiiles.append(copied_file)
|
|
164
163
|
|
|
165
164
|
if run_src:
|
|
166
165
|
try:
|
|
@@ -180,6 +179,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
180
179
|
source_print_matched_text=source_print_matched_text,
|
|
181
180
|
source_time_out=source_time_out
|
|
182
181
|
)
|
|
182
|
+
if success:
|
|
183
|
+
all_scan_item.file_items.update(result[2].file_items)
|
|
184
|
+
all_cover_items.append(result[2].cover)
|
|
183
185
|
|
|
184
186
|
else: # Run fosslight_source by using docker image
|
|
185
187
|
src_output = os.path.join("output", output_files["SRC"])
|
|
@@ -195,16 +197,22 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
195
197
|
logger.warning(f"Failed to run source analysis: {ex}")
|
|
196
198
|
|
|
197
199
|
if run_bin:
|
|
198
|
-
success,
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
200
|
+
success, result = call_analysis_api(src_path, "Binary Analysis",
|
|
201
|
+
1, binary_analysis.find_binaries,
|
|
202
|
+
abs_path,
|
|
203
|
+
os.path.join(_output_dir, output_files["BIN"]),
|
|
204
|
+
"", db_url, binary_simple,
|
|
205
|
+
correct_mode, correct_fpath,
|
|
206
|
+
path_to_exclude=path_to_exclude)
|
|
207
|
+
if success:
|
|
208
|
+
all_scan_item.file_items.update(result.file_items)
|
|
209
|
+
all_cover_items.append(result.cover)
|
|
205
210
|
|
|
206
211
|
if run_dep:
|
|
207
|
-
run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
|
|
212
|
+
dep_scanitem = run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
|
|
213
|
+
dep_arguments, path_to_exclude)
|
|
214
|
+
all_scan_item.file_items.update(dep_scanitem.file_items)
|
|
215
|
+
all_cover_items.append(dep_scanitem.cover)
|
|
208
216
|
|
|
209
217
|
else:
|
|
210
218
|
return
|
|
@@ -215,46 +223,23 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
215
223
|
try:
|
|
216
224
|
output_file_without_ext = os.path.join(final_excel_dir, output_file)
|
|
217
225
|
final_report = f"{output_file_without_ext}{output_extension}"
|
|
218
|
-
merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
|
|
219
226
|
cover = CoverItem(tool_name=PKG_NAME,
|
|
220
227
|
start_time=_start_time,
|
|
221
228
|
input_path=abs_path,
|
|
222
229
|
exclude_path=path_to_exclude,
|
|
223
230
|
simple_mode=False)
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
exist_bin = True
|
|
237
|
-
shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
|
|
238
|
-
if exist_src or exist_bin:
|
|
239
|
-
correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
|
|
240
|
-
|
|
241
|
-
if remove_src_data:
|
|
242
|
-
overwrite_excel(_output_dir, default_oss_name, "OSS Name")
|
|
243
|
-
overwrite_excel(_output_dir, default_oss_version, "OSS Version")
|
|
244
|
-
overwrite_excel(_output_dir, url, "Download Location")
|
|
245
|
-
success, err_msg = merge_excels(_output_dir, final_report, merge_files, cover)
|
|
246
|
-
|
|
247
|
-
if correct_mode:
|
|
248
|
-
if exist_src:
|
|
249
|
-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
|
|
250
|
-
os.path.join(_output_dir, output_files['SRC']))
|
|
251
|
-
if exist_bin:
|
|
252
|
-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
|
|
253
|
-
os.path.join(_output_dir, output_files['BIN']))
|
|
254
|
-
shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
|
|
255
|
-
elif output_extension == ".yaml":
|
|
256
|
-
success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
|
|
257
|
-
remove_src_data, default_oss_name, default_oss_version, url)
|
|
231
|
+
merge_comment = []
|
|
232
|
+
for ci in all_cover_items:
|
|
233
|
+
merge_comment.append(str(f'[{ci.tool_name}] {ci.comment}'))
|
|
234
|
+
cover.comment = '\n'.join(merge_comment)
|
|
235
|
+
all_scan_item.cover = cover
|
|
236
|
+
|
|
237
|
+
if correct_mode:
|
|
238
|
+
all_scan_item = correct_scanner_result(all_scan_item)
|
|
239
|
+
|
|
240
|
+
if remove_src_data:
|
|
241
|
+
all_scan_item = update_oss_item(all_scan_item, default_oss_name, default_oss_version, url)
|
|
242
|
+
success, err_msg, final_report = write_output_file(output_file_without_ext, output_extension, all_scan_item)
|
|
258
243
|
if success:
|
|
259
244
|
if os.path.isfile(final_report):
|
|
260
245
|
logger.info(f'Generated the result file: {final_report}')
|
|
@@ -266,7 +251,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
266
251
|
|
|
267
252
|
if ui_mode:
|
|
268
253
|
ui_mode_report = f"{output_file_without_ext}.json"
|
|
269
|
-
success, err_msg = create_scancodejson(
|
|
254
|
+
success, err_msg = create_scancodejson(all_scan_item, ui_mode_report, src_path)
|
|
270
255
|
if success and os.path.isfile(ui_mode_report):
|
|
271
256
|
logger.info(f'Generated the ui mode result file: {ui_mode_report}')
|
|
272
257
|
else:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fosslight-scanner
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.1
|
|
4
4
|
Summary: FOSSLight Scanner
|
|
5
5
|
Home-page: https://github.com/fosslight/fosslight_scanner
|
|
6
6
|
Author: LG Electronics
|
|
@@ -21,11 +21,11 @@ Requires-Dist: openpyxl
|
|
|
21
21
|
Requires-Dist: progress
|
|
22
22
|
Requires-Dist: pyyaml
|
|
23
23
|
Requires-Dist: beautifulsoup4
|
|
24
|
-
Requires-Dist: fosslight-util
|
|
25
|
-
Requires-Dist: fosslight-source
|
|
26
|
-
Requires-Dist: fosslight-dependency
|
|
27
|
-
Requires-Dist: fosslight-binary
|
|
28
|
-
Requires-Dist: fosslight-prechecker
|
|
24
|
+
Requires-Dist: fosslight-util>=2.0.0
|
|
25
|
+
Requires-Dist: fosslight-source>=2.0.0
|
|
26
|
+
Requires-Dist: fosslight-dependency>=4.0.0
|
|
27
|
+
Requires-Dist: fosslight-binary>=5.0.0
|
|
28
|
+
Requires-Dist: fosslight-prechecker>=4.0.0
|
|
29
29
|
|
|
30
30
|
<!--
|
|
31
31
|
Copyright (c) 2021 LG Electronics
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
fosslight_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
fosslight_scanner/_get_input.py,sha256=f78M57QU7Lhw1VfZJKvYXbGsUiYMViPz1---UDIbV9I,1396
|
|
3
|
+
fosslight_scanner/_help.py,sha256=-77CZm1es2Ib-HuISCSd4mTPxYzuLJ9NjryDI0g4otM,2921
|
|
4
|
+
fosslight_scanner/_parse_setting.py,sha256=SoIEoPGlFXAp6__eEFEMIPYPoKLvmqxXF5X_C_uFiVI,2468
|
|
5
|
+
fosslight_scanner/_run_compare.py,sha256=ufx3zldtM7sUOyQOYSgs66TY5-Gt26yE2Irz5kdkmio,10566
|
|
6
|
+
fosslight_scanner/cli.py,sha256=K088vCNTBLaiOfxW2EDkrWRhrRs7IILmDfP1HSdSI2U,6536
|
|
7
|
+
fosslight_scanner/common.py,sha256=NOfIxdKrH7MS_ptOIwY56OZJHVR9S_hfs-L2a1H9N0g,8179
|
|
8
|
+
fosslight_scanner/fosslight_scanner.py,sha256=Sp-qYnlcLw1h9wLpJoySCpTcfvMaERy11SARx_itOxY,19640
|
|
9
|
+
fosslight_scanner/resources/bom_compare.html,sha256=VocJ9bDmQQOwfGyvXatPZ4W-QddO-IlsAvKdO0nZ7pA,2240
|
|
10
|
+
fosslight_scanner-2.0.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
11
|
+
fosslight_scanner-2.0.1.dist-info/METADATA,sha256=rYCbU4NlBximKgYG1p-L3fVaCzd_oDCPDd2Gk-CO9Ho,7837
|
|
12
|
+
fosslight_scanner-2.0.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
13
|
+
fosslight_scanner-2.0.1.dist-info/entry_points.txt,sha256=0tBUtN9pKtAkuohQuhbzuohdHbeoI0-w95aWAR7J_RU,105
|
|
14
|
+
fosslight_scanner-2.0.1.dist-info/top_level.txt,sha256=43_xLb5KYpy8wOU1H2Wd2fEsWBY7Dg6ZEJJXkfT64Ak,18
|
|
15
|
+
fosslight_scanner-2.0.1.dist-info/RECORD,,
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
fosslight_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
fosslight_scanner/_get_input.py,sha256=f78M57QU7Lhw1VfZJKvYXbGsUiYMViPz1---UDIbV9I,1396
|
|
3
|
-
fosslight_scanner/_help.py,sha256=-77CZm1es2Ib-HuISCSd4mTPxYzuLJ9NjryDI0g4otM,2921
|
|
4
|
-
fosslight_scanner/_parse_setting.py,sha256=SoIEoPGlFXAp6__eEFEMIPYPoKLvmqxXF5X_C_uFiVI,2468
|
|
5
|
-
fosslight_scanner/_run_compare.py,sha256=1B39bD6uPfjIIMgbIC-sBbPJbTYgaRybZtRSqbRfzRI,10147
|
|
6
|
-
fosslight_scanner/cli.py,sha256=K088vCNTBLaiOfxW2EDkrWRhrRs7IILmDfP1HSdSI2U,6536
|
|
7
|
-
fosslight_scanner/common.py,sha256=C_RMplhWAdOWwZHfnvls2PXXH0EJp6q5SjrNqN4BH7Q,11717
|
|
8
|
-
fosslight_scanner/fosslight_scanner.py,sha256=7IZ5ZW5qzHiByR_SgoN0IMaTZK7q5U-jnybSzu9b19s,20839
|
|
9
|
-
fosslight_scanner/resources/bom_compare.html,sha256=VocJ9bDmQQOwfGyvXatPZ4W-QddO-IlsAvKdO0nZ7pA,2240
|
|
10
|
-
fosslight_scanner-1.7.31.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
11
|
-
fosslight_scanner-1.7.31.dist-info/METADATA,sha256=ks-IsMQ7ofcztBzDd9cUNL_gxK1yRbI1L3MOcLN3D2M,7842
|
|
12
|
-
fosslight_scanner-1.7.31.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
13
|
-
fosslight_scanner-1.7.31.dist-info/entry_points.txt,sha256=0tBUtN9pKtAkuohQuhbzuohdHbeoI0-w95aWAR7J_RU,105
|
|
14
|
-
fosslight_scanner-1.7.31.dist-info/top_level.txt,sha256=43_xLb5KYpy8wOU1H2Wd2fEsWBY7Dg6ZEJJXkfT64Ak,18
|
|
15
|
-
fosslight_scanner-1.7.31.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|