fosslight-scanner 1.7.31__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fosslight_scanner/_run_compare.py +14 -5
- fosslight_scanner/common.py +72 -160
- fosslight_scanner/fosslight_scanner.py +44 -73
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.0.dist-info}/METADATA +5 -6
- fosslight_scanner-2.0.0.dist-info/RECORD +15 -0
- fosslight_scanner-1.7.31.dist-info/RECORD +0 -15
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.0.dist-info}/LICENSE +0 -0
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.0.dist-info}/WHEEL +0 -0
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.0.dist-info}/entry_points.txt +0 -0
- {fosslight_scanner-1.7.31.dist-info → fosslight_scanner-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -14,7 +14,8 @@ from pathlib import Path
|
|
|
14
14
|
from bs4 import BeautifulSoup
|
|
15
15
|
import fosslight_util.constant as constant
|
|
16
16
|
from fosslight_util.compare_yaml import compare_yaml
|
|
17
|
-
from fosslight_util.
|
|
17
|
+
from fosslight_util.read_excel import read_oss_report
|
|
18
|
+
from fosslight_util.parsing_yaml import parsing_yml
|
|
18
19
|
|
|
19
20
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
20
21
|
ADD = "add"
|
|
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
|
|
|
255
256
|
|
|
256
257
|
result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
|
|
257
258
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
259
|
+
before_basepath = os.path.dirname(before_f)
|
|
260
|
+
after_basepath = os.path.dirname(after_f)
|
|
261
|
+
if XLSX_EXT == before_ext:
|
|
262
|
+
before_fileitems = read_oss_report(before_f, "", before_basepath)
|
|
263
|
+
elif YAML_EXT == before_ext:
|
|
264
|
+
before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
|
|
265
|
+
if XLSX_EXT == after_ext:
|
|
266
|
+
after_fileitems = read_oss_report(after_f, after_basepath)
|
|
267
|
+
elif YAML_EXT == after_ext:
|
|
268
|
+
after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
|
|
269
|
+
|
|
270
|
+
compared_result = compare_yaml(before_fileitems, after_fileitems)
|
|
262
271
|
if compared_result != '':
|
|
263
272
|
count_compared_result(compared_result)
|
|
264
273
|
ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)
|
fosslight_scanner/common.py
CHANGED
|
@@ -7,18 +7,12 @@ import os
|
|
|
7
7
|
import sys
|
|
8
8
|
import logging
|
|
9
9
|
import shutil
|
|
10
|
-
import
|
|
11
|
-
import
|
|
12
|
-
|
|
13
|
-
import fosslight_util.constant as constant
|
|
14
|
-
from fosslight_util.parsing_yaml import parsing_yml
|
|
15
|
-
from fosslight_util.write_yaml import create_yaml_with_ossitem
|
|
10
|
+
import copy
|
|
11
|
+
from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY
|
|
16
12
|
from fosslight_util.write_scancodejson import write_scancodejson
|
|
17
|
-
from fosslight_util.
|
|
18
|
-
from fosslight_util.output_format import write_output_file
|
|
19
|
-
from fosslight_util.oss_item import OssItem
|
|
13
|
+
from fosslight_util.oss_item import OssItem, FileItem
|
|
20
14
|
|
|
21
|
-
logger = logging.getLogger(
|
|
15
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
22
16
|
SRC_SHEET = 'SRC_FL_Source'
|
|
23
17
|
BIN_SHEET = 'BIN_FL_Binary'
|
|
24
18
|
BIN_EXT_HEADER = {
|
|
@@ -93,72 +87,23 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
|
|
|
93
87
|
return success, result or []
|
|
94
88
|
|
|
95
89
|
|
|
96
|
-
def
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if column_name in df.columns:
|
|
109
|
-
updated = (df[column_name] == '') | (df[column_name].isnull())
|
|
110
|
-
df.loc[updated, column_name] = oss_name
|
|
111
|
-
df.to_excel(file_path, sheet_name=sheet_name, index=False)
|
|
112
|
-
except Exception as ex:
|
|
113
|
-
logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
|
|
114
|
-
except Exception as ex:
|
|
115
|
-
logger.debug(f"overwrite_excel:{ex}")
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
|
|
119
|
-
default_oss_name='', default_oss_version='', url=''):
|
|
120
|
-
success = True
|
|
121
|
-
err_msg = ''
|
|
122
|
-
|
|
123
|
-
oss_total_list = []
|
|
124
|
-
yaml_dict = {}
|
|
125
|
-
try:
|
|
126
|
-
for mf in merge_yaml_files:
|
|
127
|
-
if os.path.exists(os.path.join(_output_dir, mf)):
|
|
128
|
-
oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
|
|
129
|
-
|
|
130
|
-
if remove_src_data:
|
|
131
|
-
existed_yaml = {}
|
|
132
|
-
for oi in oss_list:
|
|
133
|
-
oi.name = default_oss_name if oi.name == '' else oi.name
|
|
134
|
-
oi.version = default_oss_version if oi.version == '' else oi.version
|
|
135
|
-
oi.download_location = url if oi.download_location == '' else oi.download_location
|
|
136
|
-
create_yaml_with_ossitem(oi, existed_yaml)
|
|
137
|
-
with open(os.path.join(_output_dir, mf), 'w') as f:
|
|
138
|
-
yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
|
|
139
|
-
|
|
140
|
-
oss_total_list.extend(oss_list)
|
|
141
|
-
|
|
142
|
-
if oss_total_list != []:
|
|
143
|
-
for oti in oss_total_list:
|
|
144
|
-
create_yaml_with_ossitem(oti, yaml_dict)
|
|
145
|
-
with open(os.path.join(_output_dir, final_report), 'w') as f:
|
|
146
|
-
yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
|
|
147
|
-
else:
|
|
148
|
-
success = False
|
|
149
|
-
err_msg = "Output file is not created as no oss items detected."
|
|
150
|
-
except Exception as ex:
|
|
151
|
-
err_msg = ex
|
|
152
|
-
success = False
|
|
153
|
-
|
|
154
|
-
return success, err_msg
|
|
90
|
+
def update_oss_item(scan_item, oss_name, oss_version, download_loc):
|
|
91
|
+
for file_items in scan_item.file_items.values():
|
|
92
|
+
for file_item in file_items:
|
|
93
|
+
if file_item.oss_items:
|
|
94
|
+
for oi in file_item.oss_items:
|
|
95
|
+
if oi.name == '' and oi.version == '' and oi.download_location == '':
|
|
96
|
+
oi.name = oss_name
|
|
97
|
+
oi.version = oss_version
|
|
98
|
+
oi.download_location = download_loc
|
|
99
|
+
else:
|
|
100
|
+
file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
|
|
101
|
+
return scan_item
|
|
155
102
|
|
|
156
103
|
|
|
157
|
-
def create_scancodejson(
|
|
104
|
+
def create_scancodejson(all_scan_item_origin, ui_mode_report, src_path=""):
|
|
158
105
|
success = True
|
|
159
106
|
err_msg = ''
|
|
160
|
-
|
|
161
|
-
oss_total_list = []
|
|
162
107
|
root_dir = ""
|
|
163
108
|
root_strip = ""
|
|
164
109
|
try:
|
|
@@ -170,28 +115,37 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
|
|
|
170
115
|
root_dir = ""
|
|
171
116
|
|
|
172
117
|
try:
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
118
|
+
all_scan_item = copy.deepcopy(all_scan_item_origin)
|
|
119
|
+
if FOSSLIGHT_DEPENDENCY in all_scan_item.file_items:
|
|
120
|
+
del all_scan_item.file_items[FOSSLIGHT_DEPENDENCY]
|
|
176
121
|
if src_path:
|
|
177
|
-
|
|
122
|
+
fileitems_without_oss = []
|
|
123
|
+
for root, _, files in os.walk(src_path):
|
|
178
124
|
root = root.replace(root_strip, "")
|
|
179
125
|
for file in files:
|
|
126
|
+
fi_without_oss = FileItem('')
|
|
127
|
+
included = False
|
|
180
128
|
item_path = os.path.join(root, file)
|
|
181
129
|
item_path = item_path.replace(parent + os.path.sep, '', 1)
|
|
182
|
-
|
|
130
|
+
|
|
131
|
+
for file_items in all_scan_item.file_items.values():
|
|
132
|
+
for file_item in file_items:
|
|
133
|
+
if file_item.source_name_or_path:
|
|
134
|
+
if file_item.source_name_or_path == item_path:
|
|
135
|
+
included = True
|
|
136
|
+
break
|
|
183
137
|
if not included:
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
138
|
+
fi_without_oss.source_name_or_path = item_path
|
|
139
|
+
fileitems_without_oss.append(fi_without_oss)
|
|
140
|
+
if len(fileitems_without_oss) > 0:
|
|
141
|
+
all_scan_item.file_items[FOSSLIGHT_SOURCE].extend(fileitems_without_oss)
|
|
187
142
|
if root_dir:
|
|
188
|
-
for
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
143
|
+
for file_items in all_scan_item.file_items.values():
|
|
144
|
+
for fi in file_items:
|
|
145
|
+
if fi.source_name_or_path:
|
|
146
|
+
fi.source_name_or_path = os.path.join(root_dir, fi.source_name_or_path)
|
|
193
147
|
write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
|
|
194
|
-
|
|
148
|
+
all_scan_item)
|
|
195
149
|
except Exception as ex:
|
|
196
150
|
err_msg = ex
|
|
197
151
|
success = False
|
|
@@ -199,100 +153,58 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
|
|
|
199
153
|
return success, err_msg
|
|
200
154
|
|
|
201
155
|
|
|
202
|
-
def correct_scanner_result(
|
|
203
|
-
src_oss_list = []
|
|
204
|
-
bin_oss_list = []
|
|
156
|
+
def correct_scanner_result(all_scan_item):
|
|
205
157
|
duplicates = False
|
|
206
158
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
if
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
if exist_src and exist_bin:
|
|
159
|
+
keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
|
|
160
|
+
is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
|
|
161
|
+
if is_contained:
|
|
162
|
+
src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
|
|
163
|
+
bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
|
|
213
164
|
try:
|
|
214
165
|
remove_src_idx_list = []
|
|
215
|
-
for idx_src,
|
|
166
|
+
for idx_src, src_fileitem in enumerate(src_fileitems):
|
|
167
|
+
src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path, src_fileitem.exclude)
|
|
216
168
|
dup_flag = False
|
|
217
|
-
for
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
|
|
169
|
+
for bin_fileitem in bin_fileitems:
|
|
170
|
+
bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path, bin_fileitem.exclude)
|
|
171
|
+
if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
|
|
221
172
|
dup_flag = True
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
173
|
+
src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
|
|
174
|
+
bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
|
|
175
|
+
|
|
176
|
+
if src_all_licenses_non_empty and bin_empty_license_exists:
|
|
177
|
+
exclude = bin_fileitem.oss_items[0].exclude
|
|
178
|
+
bin_fileitem.oss_items = []
|
|
179
|
+
for src_oss_item in src_fileitem.oss_items:
|
|
180
|
+
src_oss_item.exclude = exclude
|
|
181
|
+
bin_fileitem.oss_items.append(src_oss_item)
|
|
182
|
+
bin_fileitem.comment = 'Loaded from SRC OSS info'
|
|
228
183
|
if dup_flag:
|
|
229
184
|
remove_src_idx_list.append(idx_src)
|
|
230
185
|
if remove_src_idx_list:
|
|
231
186
|
duplicates = True
|
|
232
187
|
for i in sorted(remove_src_idx_list, reverse=True):
|
|
233
|
-
del
|
|
188
|
+
del src_fileitems[i]
|
|
234
189
|
except Exception as ex:
|
|
235
190
|
logger.warning(f"correct the scanner result:{ex}")
|
|
236
191
|
|
|
237
192
|
try:
|
|
238
|
-
if exist_src:
|
|
239
|
-
success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
|
|
240
|
-
output_extension, SRC_SHEET)
|
|
241
|
-
if not success:
|
|
242
|
-
logger.warning(err_msg)
|
|
243
|
-
if exist_bin:
|
|
244
|
-
success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
|
|
245
|
-
output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
|
|
246
|
-
if not success:
|
|
247
|
-
logger.warning(err_msg)
|
|
248
193
|
if duplicates:
|
|
249
194
|
logger.info('Success to correct the src/bin scanner result')
|
|
250
195
|
except Exception as ex:
|
|
251
196
|
logger.warning(f"Corrected src/bin scanner result:{ex}")
|
|
252
|
-
return
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
|
|
256
|
-
new_oss_list = []
|
|
257
|
-
sheet_list = {}
|
|
258
|
-
sheet_list[sheetname] = []
|
|
259
|
-
|
|
260
|
-
for src_item in oss_list:
|
|
261
|
-
scanner_name = constant.supported_sheet_and_scanner[sheetname]
|
|
262
|
-
new_oss_list.append(src_item.get_print_array(scanner_name)[0])
|
|
263
|
-
sheet_list[sheetname].extend(new_oss_list)
|
|
264
|
-
if os.path.exists(os.path.join(output_dir, output_file)):
|
|
265
|
-
os.remove(os.path.join(output_dir, output_file))
|
|
266
|
-
success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
|
|
267
|
-
output_extension, sheet_list, extended_hdr, hidden_hdr)
|
|
268
|
-
return success, err_msg
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
|
|
272
|
-
err_reason = ''
|
|
273
|
-
oss_list = []
|
|
274
|
-
oss_file_with_fullpath = os.path.join(_output_dir, output_file)
|
|
275
|
-
|
|
276
|
-
if os.path.exists(oss_file_with_fullpath):
|
|
277
|
-
if output_extension == '.xlsx':
|
|
278
|
-
oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
|
|
279
|
-
elif output_extension == '.yaml':
|
|
280
|
-
oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
|
|
281
|
-
else:
|
|
282
|
-
err_reason = f'Not supported extension: {output_extension}'
|
|
283
|
-
if err_reason:
|
|
284
|
-
logger.info(f'get_osslist: {err_reason}')
|
|
285
|
-
return oss_list
|
|
197
|
+
return all_scan_item
|
|
286
198
|
|
|
287
199
|
|
|
288
|
-
def check_exclude_dir(
|
|
200
|
+
def check_exclude_dir(source_name_or_path, file_item_exclude):
|
|
201
|
+
if file_item_exclude:
|
|
202
|
+
return True
|
|
289
203
|
_exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
|
|
204
|
+
exclude = False
|
|
290
205
|
|
|
291
|
-
for
|
|
292
|
-
if
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
oss_item.exclude = True
|
|
297
|
-
break
|
|
298
|
-
return oss_list
|
|
206
|
+
for exclude_dir in _exclude_dirs:
|
|
207
|
+
if exclude_dir in source_name_or_path.split(os.path.sep):
|
|
208
|
+
exclude = True
|
|
209
|
+
break
|
|
210
|
+
return exclude
|
|
@@ -24,13 +24,13 @@ from fosslight_util.set_log import init_log
|
|
|
24
24
|
from fosslight_util.timer_thread import TimerThread
|
|
25
25
|
import fosslight_util.constant as constant
|
|
26
26
|
from fosslight_util.output_format import check_output_format
|
|
27
|
-
from fosslight_prechecker._precheck import run_lint as prechecker_lint
|
|
28
|
-
from fosslight_util.write_excel import merge_excels, merge_cover_comment
|
|
29
27
|
from fosslight_util.cover import CoverItem
|
|
28
|
+
from fosslight_util.oss_item import ScannerItem
|
|
29
|
+
from fosslight_util.output_format import write_output_file
|
|
30
30
|
|
|
31
31
|
from .common import (
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
call_analysis_api, update_oss_item,
|
|
33
|
+
correct_scanner_result, create_scancodejson
|
|
34
34
|
)
|
|
35
35
|
from ._run_compare import run_compare
|
|
36
36
|
|
|
@@ -50,13 +50,13 @@ _start_time = ""
|
|
|
50
50
|
_executed_path = ""
|
|
51
51
|
SRC_DIR_FROM_LINK_PREFIX = "fosslight_src_dir_"
|
|
52
52
|
SCANNER_MODE = [
|
|
53
|
-
"all", "compare", "
|
|
53
|
+
"all", "compare", "binary",
|
|
54
54
|
"bin", "src", "source", "dependency", "dep"
|
|
55
55
|
]
|
|
56
56
|
|
|
57
57
|
|
|
58
58
|
def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
|
|
59
|
-
|
|
59
|
+
result = []
|
|
60
60
|
|
|
61
61
|
package_manager = ""
|
|
62
62
|
pip_activate_cmd = ""
|
|
@@ -90,7 +90,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
90
90
|
timer.start()
|
|
91
91
|
|
|
92
92
|
try:
|
|
93
|
-
success,
|
|
93
|
+
success, scan_item = call_analysis_api(
|
|
94
94
|
path_to_analyze, "Dependency Analysis",
|
|
95
95
|
1, run_dependency_scanner,
|
|
96
96
|
package_manager,
|
|
@@ -101,11 +101,11 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
101
101
|
github_token, path_to_exclude=path_to_exclude
|
|
102
102
|
)
|
|
103
103
|
if success:
|
|
104
|
-
|
|
104
|
+
result = scan_item
|
|
105
105
|
except Exception as ex:
|
|
106
106
|
logger.warning(f"Run dependency: {ex}")
|
|
107
107
|
|
|
108
|
-
return
|
|
108
|
+
return result
|
|
109
109
|
|
|
110
110
|
|
|
111
111
|
def source_analysis_wrapper(*args, **kwargs):
|
|
@@ -130,7 +130,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
130
130
|
source_time_out=120, binary_simple=False):
|
|
131
131
|
final_excel_dir = output_path
|
|
132
132
|
success = True
|
|
133
|
-
|
|
133
|
+
all_cover_items = []
|
|
134
|
+
all_scan_item = ScannerItem(PKG_NAME, _start_time)
|
|
134
135
|
if not remove_src_data:
|
|
135
136
|
success, final_excel_dir, result_log = init(output_path)
|
|
136
137
|
|
|
@@ -150,17 +151,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
150
151
|
if success:
|
|
151
152
|
output_files = {"SRC": f"fosslight_src_{_start_time}{output_extension}",
|
|
152
153
|
"BIN": f"fosslight_bin_{_start_time}{output_extension}",
|
|
153
|
-
"DEP": f"fosslight_dep_{_start_time}{output_extension}"
|
|
154
|
-
"PRECHECKER": f"fosslight_lint_{_start_time}.yaml"}
|
|
155
|
-
if run_prechecker:
|
|
156
|
-
output_prechecker = os.path.join(_output_dir, output_files["PRECHECKER"])
|
|
157
|
-
success, result = call_analysis_api(src_path, "Prechecker Lint",
|
|
158
|
-
-1, prechecker_lint,
|
|
159
|
-
abs_path, False, output_prechecker,
|
|
160
|
-
exclude_path=path_to_exclude)
|
|
161
|
-
success_file, copied_file = copy_file(output_prechecker, output_path)
|
|
162
|
-
if success_file:
|
|
163
|
-
temp_output_fiiles.append(copied_file)
|
|
154
|
+
"DEP": f"fosslight_dep_{_start_time}{output_extension}"}
|
|
164
155
|
|
|
165
156
|
if run_src:
|
|
166
157
|
try:
|
|
@@ -180,6 +171,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
180
171
|
source_print_matched_text=source_print_matched_text,
|
|
181
172
|
source_time_out=source_time_out
|
|
182
173
|
)
|
|
174
|
+
if success:
|
|
175
|
+
all_scan_item.file_items.update(result[2].file_items)
|
|
176
|
+
all_cover_items.append(result[2].cover)
|
|
183
177
|
|
|
184
178
|
else: # Run fosslight_source by using docker image
|
|
185
179
|
src_output = os.path.join("output", output_files["SRC"])
|
|
@@ -195,16 +189,22 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
195
189
|
logger.warning(f"Failed to run source analysis: {ex}")
|
|
196
190
|
|
|
197
191
|
if run_bin:
|
|
198
|
-
success,
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
192
|
+
success, result = call_analysis_api(src_path, "Binary Analysis",
|
|
193
|
+
1, binary_analysis.find_binaries,
|
|
194
|
+
abs_path,
|
|
195
|
+
os.path.join(_output_dir, output_files["BIN"]),
|
|
196
|
+
"", db_url, binary_simple,
|
|
197
|
+
correct_mode, correct_fpath,
|
|
198
|
+
path_to_exclude=path_to_exclude)
|
|
199
|
+
if success:
|
|
200
|
+
all_scan_item.file_items.update(result.file_items)
|
|
201
|
+
all_cover_items.append(result.cover)
|
|
205
202
|
|
|
206
203
|
if run_dep:
|
|
207
|
-
run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
|
|
204
|
+
dep_scanitem = run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
|
|
205
|
+
dep_arguments, path_to_exclude)
|
|
206
|
+
all_scan_item.file_items.update(dep_scanitem.file_items)
|
|
207
|
+
all_cover_items.append(dep_scanitem.cover)
|
|
208
208
|
|
|
209
209
|
else:
|
|
210
210
|
return
|
|
@@ -215,46 +215,23 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
215
215
|
try:
|
|
216
216
|
output_file_without_ext = os.path.join(final_excel_dir, output_file)
|
|
217
217
|
final_report = f"{output_file_without_ext}{output_extension}"
|
|
218
|
-
merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
|
|
219
218
|
cover = CoverItem(tool_name=PKG_NAME,
|
|
220
219
|
start_time=_start_time,
|
|
221
220
|
input_path=abs_path,
|
|
222
221
|
exclude_path=path_to_exclude,
|
|
223
222
|
simple_mode=False)
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
exist_bin = True
|
|
237
|
-
shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
|
|
238
|
-
if exist_src or exist_bin:
|
|
239
|
-
correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
|
|
240
|
-
|
|
241
|
-
if remove_src_data:
|
|
242
|
-
overwrite_excel(_output_dir, default_oss_name, "OSS Name")
|
|
243
|
-
overwrite_excel(_output_dir, default_oss_version, "OSS Version")
|
|
244
|
-
overwrite_excel(_output_dir, url, "Download Location")
|
|
245
|
-
success, err_msg = merge_excels(_output_dir, final_report, merge_files, cover)
|
|
246
|
-
|
|
247
|
-
if correct_mode:
|
|
248
|
-
if exist_src:
|
|
249
|
-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
|
|
250
|
-
os.path.join(_output_dir, output_files['SRC']))
|
|
251
|
-
if exist_bin:
|
|
252
|
-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
|
|
253
|
-
os.path.join(_output_dir, output_files['BIN']))
|
|
254
|
-
shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
|
|
255
|
-
elif output_extension == ".yaml":
|
|
256
|
-
success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
|
|
257
|
-
remove_src_data, default_oss_name, default_oss_version, url)
|
|
223
|
+
merge_comment = []
|
|
224
|
+
for ci in all_cover_items:
|
|
225
|
+
merge_comment.append(str(f'[{ci.tool_name}] {ci.comment}'))
|
|
226
|
+
cover.comment = '\n'.join(merge_comment)
|
|
227
|
+
all_scan_item.cover = cover
|
|
228
|
+
|
|
229
|
+
if correct_mode:
|
|
230
|
+
all_scan_item = correct_scanner_result(all_scan_item)
|
|
231
|
+
|
|
232
|
+
if remove_src_data:
|
|
233
|
+
all_scan_item = update_oss_item(all_scan_item, default_oss_name, default_oss_version, url)
|
|
234
|
+
success, err_msg, final_report = write_output_file(output_file_without_ext, output_extension, all_scan_item)
|
|
258
235
|
if success:
|
|
259
236
|
if os.path.isfile(final_report):
|
|
260
237
|
logger.info(f'Generated the result file: {final_report}')
|
|
@@ -266,7 +243,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
266
243
|
|
|
267
244
|
if ui_mode:
|
|
268
245
|
ui_mode_report = f"{output_file_without_ext}.json"
|
|
269
|
-
success, err_msg = create_scancodejson(
|
|
246
|
+
success, err_msg = create_scancodejson(all_scan_item, ui_mode_report, src_path)
|
|
270
247
|
if success and os.path.isfile(ui_mode_report):
|
|
271
248
|
logger.info(f'Generated the ui mode result file: {ui_mode_report}')
|
|
272
249
|
else:
|
|
@@ -412,19 +389,13 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
412
389
|
run_src = False
|
|
413
390
|
run_bin = False
|
|
414
391
|
run_dep = False
|
|
415
|
-
run_prechecker = False
|
|
416
392
|
remove_downloaded_source = False
|
|
417
393
|
|
|
418
394
|
if "all" in mode_list or (not mode_list):
|
|
419
395
|
run_src = True
|
|
420
396
|
run_bin = True
|
|
421
397
|
run_dep = True
|
|
422
|
-
run_prechecker = False
|
|
423
|
-
if "prechecker" in mode_list or "reuse" in mode_list:
|
|
424
|
-
run_prechecker = True
|
|
425
398
|
else:
|
|
426
|
-
if "prechecker" in mode_list or "reuse" in mode_list:
|
|
427
|
-
run_prechecker = True
|
|
428
399
|
if "binary" in mode_list or "bin" in mode_list:
|
|
429
400
|
run_bin = True
|
|
430
401
|
if "source" in mode_list or "src" in mode_list:
|
|
@@ -432,7 +403,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
432
403
|
if "dependency" in mode_list or "dep" in mode_list:
|
|
433
404
|
run_dep = True
|
|
434
405
|
|
|
435
|
-
if run_dep or run_src or run_bin
|
|
406
|
+
if run_dep or run_src or run_bin:
|
|
436
407
|
if src_path == "" and url_to_analyze == "":
|
|
437
408
|
src_path, dep_arguments, url_to_analyze = get_input_mode(_executed_path, mode_list)
|
|
438
409
|
|
|
@@ -454,7 +425,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
454
425
|
|
|
455
426
|
if src_path != "":
|
|
456
427
|
run_scanner(src_path, dep_arguments, output_path, keep_raw_data,
|
|
457
|
-
run_src, run_bin, run_dep,
|
|
428
|
+
run_src, run_bin, run_dep, '',
|
|
458
429
|
remove_downloaded_source, {}, output_file,
|
|
459
430
|
output_extension, num_cores, db_url,
|
|
460
431
|
default_oss_name, default_oss_version, url_to_analyze,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fosslight-scanner
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: FOSSLight Scanner
|
|
5
5
|
Home-page: https://github.com/fosslight/fosslight_scanner
|
|
6
6
|
Author: LG Electronics
|
|
@@ -21,11 +21,10 @@ Requires-Dist: openpyxl
|
|
|
21
21
|
Requires-Dist: progress
|
|
22
22
|
Requires-Dist: pyyaml
|
|
23
23
|
Requires-Dist: beautifulsoup4
|
|
24
|
-
Requires-Dist: fosslight-util
|
|
25
|
-
Requires-Dist: fosslight-source
|
|
26
|
-
Requires-Dist: fosslight-dependency
|
|
27
|
-
Requires-Dist: fosslight-binary
|
|
28
|
-
Requires-Dist: fosslight-prechecker==3.0.27
|
|
24
|
+
Requires-Dist: fosslight-util>=2.0.0
|
|
25
|
+
Requires-Dist: fosslight-source>=2.0.0
|
|
26
|
+
Requires-Dist: fosslight-dependency>=4.0.0
|
|
27
|
+
Requires-Dist: fosslight-binary>=5.0.0
|
|
29
28
|
|
|
30
29
|
<!--
|
|
31
30
|
Copyright (c) 2021 LG Electronics
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
fosslight_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
fosslight_scanner/_get_input.py,sha256=f78M57QU7Lhw1VfZJKvYXbGsUiYMViPz1---UDIbV9I,1396
|
|
3
|
+
fosslight_scanner/_help.py,sha256=-77CZm1es2Ib-HuISCSd4mTPxYzuLJ9NjryDI0g4otM,2921
|
|
4
|
+
fosslight_scanner/_parse_setting.py,sha256=SoIEoPGlFXAp6__eEFEMIPYPoKLvmqxXF5X_C_uFiVI,2468
|
|
5
|
+
fosslight_scanner/_run_compare.py,sha256=ufx3zldtM7sUOyQOYSgs66TY5-Gt26yE2Irz5kdkmio,10566
|
|
6
|
+
fosslight_scanner/cli.py,sha256=K088vCNTBLaiOfxW2EDkrWRhrRs7IILmDfP1HSdSI2U,6536
|
|
7
|
+
fosslight_scanner/common.py,sha256=NOfIxdKrH7MS_ptOIwY56OZJHVR9S_hfs-L2a1H9N0g,8179
|
|
8
|
+
fosslight_scanner/fosslight_scanner.py,sha256=fIP4SJ3zN8E-QeChgmh5fNj4nO3eD-a5kVZ7SoAcz3s,18693
|
|
9
|
+
fosslight_scanner/resources/bom_compare.html,sha256=VocJ9bDmQQOwfGyvXatPZ4W-QddO-IlsAvKdO0nZ7pA,2240
|
|
10
|
+
fosslight_scanner-2.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
11
|
+
fosslight_scanner-2.0.0.dist-info/METADATA,sha256=SjHAGy0zLaC6FQxJDMsxHmcBopgW8VR4kp3fNj23yJ8,7794
|
|
12
|
+
fosslight_scanner-2.0.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
13
|
+
fosslight_scanner-2.0.0.dist-info/entry_points.txt,sha256=0tBUtN9pKtAkuohQuhbzuohdHbeoI0-w95aWAR7J_RU,105
|
|
14
|
+
fosslight_scanner-2.0.0.dist-info/top_level.txt,sha256=43_xLb5KYpy8wOU1H2Wd2fEsWBY7Dg6ZEJJXkfT64Ak,18
|
|
15
|
+
fosslight_scanner-2.0.0.dist-info/RECORD,,
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
fosslight_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
fosslight_scanner/_get_input.py,sha256=f78M57QU7Lhw1VfZJKvYXbGsUiYMViPz1---UDIbV9I,1396
|
|
3
|
-
fosslight_scanner/_help.py,sha256=-77CZm1es2Ib-HuISCSd4mTPxYzuLJ9NjryDI0g4otM,2921
|
|
4
|
-
fosslight_scanner/_parse_setting.py,sha256=SoIEoPGlFXAp6__eEFEMIPYPoKLvmqxXF5X_C_uFiVI,2468
|
|
5
|
-
fosslight_scanner/_run_compare.py,sha256=1B39bD6uPfjIIMgbIC-sBbPJbTYgaRybZtRSqbRfzRI,10147
|
|
6
|
-
fosslight_scanner/cli.py,sha256=K088vCNTBLaiOfxW2EDkrWRhrRs7IILmDfP1HSdSI2U,6536
|
|
7
|
-
fosslight_scanner/common.py,sha256=C_RMplhWAdOWwZHfnvls2PXXH0EJp6q5SjrNqN4BH7Q,11717
|
|
8
|
-
fosslight_scanner/fosslight_scanner.py,sha256=7IZ5ZW5qzHiByR_SgoN0IMaTZK7q5U-jnybSzu9b19s,20839
|
|
9
|
-
fosslight_scanner/resources/bom_compare.html,sha256=VocJ9bDmQQOwfGyvXatPZ4W-QddO-IlsAvKdO0nZ7pA,2240
|
|
10
|
-
fosslight_scanner-1.7.31.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
11
|
-
fosslight_scanner-1.7.31.dist-info/METADATA,sha256=ks-IsMQ7ofcztBzDd9cUNL_gxK1yRbI1L3MOcLN3D2M,7842
|
|
12
|
-
fosslight_scanner-1.7.31.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
13
|
-
fosslight_scanner-1.7.31.dist-info/entry_points.txt,sha256=0tBUtN9pKtAkuohQuhbzuohdHbeoI0-w95aWAR7J_RU,105
|
|
14
|
-
fosslight_scanner-1.7.31.dist-info/top_level.txt,sha256=43_xLb5KYpy8wOU1H2Wd2fEsWBY7Dg6ZEJJXkfT64Ak,18
|
|
15
|
-
fosslight_scanner-1.7.31.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|