fosslight-scanner 1.7.30__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fosslight_scanner/_parse_setting.py +23 -8
- fosslight_scanner/_run_compare.py +14 -5
- fosslight_scanner/cli.py +82 -38
- fosslight_scanner/common.py +90 -178
- fosslight_scanner/fosslight_scanner.py +103 -98
- {fosslight_scanner-1.7.30.dist-info → fosslight_scanner-2.0.0.dist-info}/METADATA +5 -6
- fosslight_scanner-2.0.0.dist-info/RECORD +15 -0
- {fosslight_scanner-1.7.30.dist-info → fosslight_scanner-2.0.0.dist-info}/WHEEL +1 -1
- fosslight_scanner-1.7.30.dist-info/RECORD +0 -15
- {fosslight_scanner-1.7.30.dist-info → fosslight_scanner-2.0.0.dist-info}/LICENSE +0 -0
- {fosslight_scanner-1.7.30.dist-info → fosslight_scanner-2.0.0.dist-info}/entry_points.txt +0 -0
- {fosslight_scanner-1.7.30.dist-info → fosslight_scanner-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -11,7 +11,7 @@ def parse_setting_json(data):
|
|
|
11
11
|
dep_argument = data.get('dep_argument', '')
|
|
12
12
|
output = data.get('output', '')
|
|
13
13
|
format = data.get('format', '')
|
|
14
|
-
link = data.get('link',
|
|
14
|
+
link = data.get('link', '')
|
|
15
15
|
db_url = data.get('db_url', '')
|
|
16
16
|
timer = data.get('timer', False)
|
|
17
17
|
raw = data.get('raw', False)
|
|
@@ -20,34 +20,49 @@ def parse_setting_json(data):
|
|
|
20
20
|
correct_fpath = data.get('correct_fpath', '')
|
|
21
21
|
ui = data.get('ui', False)
|
|
22
22
|
exclude_path = data.get('exclude', [])
|
|
23
|
-
|
|
23
|
+
selected_source_scanner = data.get('selected_source_scanner', '')
|
|
24
|
+
source_write_json_file = data.get('source_write_json_file', False)
|
|
25
|
+
source_print_matched_text = data.get('source_print_matched_text', False)
|
|
26
|
+
source_time_out = data.get('source_time_out', 120)
|
|
27
|
+
binary_simple = data.get('binary_simple', False)
|
|
24
28
|
str_lists = [mode, path, exclude_path]
|
|
25
|
-
strings = [
|
|
26
|
-
|
|
29
|
+
strings = [
|
|
30
|
+
dep_argument, output, format, db_url,
|
|
31
|
+
correct_fpath, link, selected_source_scanner
|
|
32
|
+
]
|
|
33
|
+
booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple]
|
|
34
|
+
|
|
27
35
|
is_incorrect = False
|
|
28
36
|
|
|
29
37
|
# check if json file is incorrect format
|
|
30
38
|
for i, target in enumerate(str_lists):
|
|
31
|
-
if not (isinstance(target, list) and
|
|
39
|
+
if not (isinstance(target, list) and
|
|
40
|
+
all(isinstance(item, str) for item in target)):
|
|
32
41
|
is_incorrect = True
|
|
33
42
|
str_lists[i] = []
|
|
34
43
|
|
|
35
44
|
for i, target in enumerate(strings):
|
|
36
45
|
if not isinstance(target, str):
|
|
37
46
|
is_incorrect = True
|
|
38
|
-
|
|
47
|
+
strings[i] = ''
|
|
39
48
|
|
|
40
49
|
for i, target in enumerate(booleans):
|
|
41
50
|
if not isinstance(target, bool):
|
|
42
51
|
is_incorrect = True
|
|
43
|
-
|
|
52
|
+
booleans[i] = False
|
|
44
53
|
|
|
45
54
|
if not isinstance(core, int):
|
|
46
55
|
is_incorrect = True
|
|
47
56
|
core = -1
|
|
48
57
|
|
|
58
|
+
if not isinstance(source_time_out, int):
|
|
59
|
+
is_incorrect = True
|
|
60
|
+
source_time_out = 120
|
|
61
|
+
|
|
49
62
|
if is_incorrect:
|
|
50
63
|
print('Ignoring some values with incorrect format in the setting file.')
|
|
51
64
|
|
|
52
65
|
return mode, path, dep_argument, output, format, link, db_url, timer, \
|
|
53
|
-
raw, core, no_correction, correct_fpath, ui, exclude_path
|
|
66
|
+
raw, core, no_correction, correct_fpath, ui, exclude_path, \
|
|
67
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
|
|
68
|
+
binary_simple
|
|
@@ -14,7 +14,8 @@ from pathlib import Path
|
|
|
14
14
|
from bs4 import BeautifulSoup
|
|
15
15
|
import fosslight_util.constant as constant
|
|
16
16
|
from fosslight_util.compare_yaml import compare_yaml
|
|
17
|
-
from fosslight_util.
|
|
17
|
+
from fosslight_util.read_excel import read_oss_report
|
|
18
|
+
from fosslight_util.parsing_yaml import parsing_yml
|
|
18
19
|
|
|
19
20
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
20
21
|
ADD = "add"
|
|
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
|
|
|
255
256
|
|
|
256
257
|
result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
|
|
257
258
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
259
|
+
before_basepath = os.path.dirname(before_f)
|
|
260
|
+
after_basepath = os.path.dirname(after_f)
|
|
261
|
+
if XLSX_EXT == before_ext:
|
|
262
|
+
before_fileitems = read_oss_report(before_f, "", before_basepath)
|
|
263
|
+
elif YAML_EXT == before_ext:
|
|
264
|
+
before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
|
|
265
|
+
if XLSX_EXT == after_ext:
|
|
266
|
+
after_fileitems = read_oss_report(after_f, after_basepath)
|
|
267
|
+
elif YAML_EXT == after_ext:
|
|
268
|
+
after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
|
|
269
|
+
|
|
270
|
+
compared_result = compare_yaml(before_fileitems, after_fileitems)
|
|
262
271
|
if compared_result != '':
|
|
263
272
|
count_compared_result(compared_result)
|
|
264
273
|
ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)
|
fosslight_scanner/cli.py
CHANGED
|
@@ -4,68 +4,107 @@
|
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
import sys
|
|
6
6
|
import json
|
|
7
|
+
import os
|
|
8
|
+
import os.path
|
|
7
9
|
from argparse import ArgumentParser
|
|
10
|
+
|
|
8
11
|
from ._help import print_help_msg
|
|
9
12
|
from .fosslight_scanner import run_main, PKG_NAME
|
|
10
13
|
from ._parse_setting import parse_setting_json
|
|
11
14
|
from fosslight_util.help import print_package_version
|
|
12
|
-
import os.path
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
|
|
16
18
|
raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
|
|
19
|
+
|
|
20
|
+
selected_source_scanner = "all"
|
|
21
|
+
source_write_json_file = False
|
|
22
|
+
source_print_matched_text = False
|
|
23
|
+
source_time_out = 120
|
|
24
|
+
binary_simple = False
|
|
25
|
+
|
|
17
26
|
if setting and os.path.isfile(setting):
|
|
18
27
|
try:
|
|
19
28
|
with open(setting, 'r', encoding='utf-8') as file:
|
|
20
29
|
data = json.load(file)
|
|
21
30
|
s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
|
|
22
|
-
s_no_correction, s_correct_fpath, s_ui, s_exclude_path
|
|
31
|
+
s_no_correction, s_correct_fpath, s_ui, s_exclude_path, \
|
|
32
|
+
s_selected_source_scanner, s_source_write_json_file, s_source_print_matched_text, \
|
|
33
|
+
s_source_time_out, s_binary_simple = parse_setting_json(data)
|
|
23
34
|
|
|
24
35
|
# direct cli arguments have higher priority than setting file
|
|
25
|
-
mode = mode
|
|
26
|
-
path = path
|
|
27
|
-
dep_argument = dep_argument
|
|
28
|
-
output = output
|
|
29
|
-
format = format
|
|
30
|
-
link = link
|
|
31
|
-
db_url = db_url
|
|
32
|
-
timer = timer
|
|
33
|
-
raw = raw
|
|
34
|
-
core = core if core else s_core
|
|
35
|
-
no_correction = no_correction
|
|
36
|
-
correct_fpath = correct_fpath
|
|
37
|
-
ui = ui
|
|
38
|
-
exclude_path = exclude_path
|
|
36
|
+
mode = mode or s_mode
|
|
37
|
+
path = path or s_path
|
|
38
|
+
dep_argument = dep_argument or s_dep_argument
|
|
39
|
+
output = output or s_output
|
|
40
|
+
format = format or s_format
|
|
41
|
+
link = link or s_link
|
|
42
|
+
db_url = db_url or s_db_url
|
|
43
|
+
timer = timer or s_timer
|
|
44
|
+
raw = raw or s_raw
|
|
45
|
+
core = core if core != -1 else s_core
|
|
46
|
+
no_correction = no_correction or s_no_correction
|
|
47
|
+
correct_fpath = correct_fpath or s_correct_fpath
|
|
48
|
+
ui = ui or s_ui
|
|
49
|
+
exclude_path = exclude_path or s_exclude_path
|
|
50
|
+
|
|
51
|
+
# These options are only set from the setting file, not from CLI arguments
|
|
52
|
+
selected_source_scanner = s_selected_source_scanner or selected_source_scanner
|
|
53
|
+
source_write_json_file = s_source_write_json_file
|
|
54
|
+
source_print_matched_text = s_source_print_matched_text
|
|
55
|
+
source_time_out = s_source_time_out if s_source_time_out != 120 else source_time_out
|
|
56
|
+
binary_simple = s_binary_simple
|
|
39
57
|
|
|
40
58
|
except Exception as e:
|
|
41
59
|
print(f"Cannot open setting file: {e}")
|
|
42
60
|
return mode, path, dep_argument, output, format, link, db_url, timer, \
|
|
43
|
-
raw, core, no_correction, correct_fpath, ui, exclude_path
|
|
61
|
+
raw, core, no_correction, correct_fpath, ui, exclude_path, \
|
|
62
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
|
|
63
|
+
binary_simple
|
|
44
64
|
|
|
45
65
|
|
|
46
66
|
def main():
|
|
47
|
-
parser = ArgumentParser(description='FOSSLight Scanner',
|
|
48
|
-
|
|
49
|
-
parser.add_argument('
|
|
67
|
+
parser = ArgumentParser(description='FOSSLight Scanner',
|
|
68
|
+
prog='fosslight_scanner', add_help=False)
|
|
69
|
+
parser.add_argument('mode', nargs='*',
|
|
70
|
+
help='source| dependency| binary| all| compare',
|
|
71
|
+
default="")
|
|
72
|
+
parser.add_argument('--path', '-p',
|
|
73
|
+
help='Path to analyze (In compare mode, two FOSSLight reports',
|
|
50
74
|
dest='path', nargs='+', default="")
|
|
51
|
-
parser.add_argument('--wget', '-w', help='Link to be analyzed',
|
|
52
|
-
|
|
75
|
+
parser.add_argument('--wget', '-w', help='Link to be analyzed',
|
|
76
|
+
type=str, dest='link', default="")
|
|
77
|
+
parser.add_argument('--format', '-f',
|
|
78
|
+
help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
|
|
53
79
|
type=str, dest='format', default="")
|
|
54
|
-
parser.add_argument('--output', '-o', help='Output directory or file',
|
|
55
|
-
|
|
56
|
-
parser.add_argument('--
|
|
57
|
-
|
|
58
|
-
parser.add_argument('--
|
|
59
|
-
|
|
60
|
-
parser.add_argument('--
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
parser.add_argument('--
|
|
64
|
-
|
|
80
|
+
parser.add_argument('--output', '-o', help='Output directory or file',
|
|
81
|
+
type=str, dest='output', default="")
|
|
82
|
+
parser.add_argument('--dependency', '-d', help='Dependency arguments',
|
|
83
|
+
type=str, dest='dep_argument', default="")
|
|
84
|
+
parser.add_argument('--url', '-u', help="DB Url",
|
|
85
|
+
type=str, dest='db_url', default="")
|
|
86
|
+
parser.add_argument('--core', '-c',
|
|
87
|
+
help='Number of processes to analyze source',
|
|
88
|
+
type=int, dest='core', default=-1)
|
|
89
|
+
parser.add_argument('--raw', '-r', help='Keep raw data',
|
|
90
|
+
action='store_true', dest='raw', default=False)
|
|
91
|
+
parser.add_argument('--timer', '-t', help='Hide the progress bar',
|
|
92
|
+
action='store_true', dest='timer', default=False)
|
|
93
|
+
parser.add_argument('--version', '-v', help='Print version',
|
|
94
|
+
action='store_true', dest='version', default=False)
|
|
95
|
+
parser.add_argument('--help', '-h', help='Print help message',
|
|
96
|
+
action='store_true', dest='help')
|
|
97
|
+
parser.add_argument('--exclude', '-e', help='Path to exclude from analysis',
|
|
98
|
+
dest='exclude_path', nargs='*', default=[])
|
|
99
|
+
parser.add_argument('--setting', '-s', help='Scanner json setting file',
|
|
100
|
+
type=str, dest='setting', default="")
|
|
101
|
+
parser.add_argument('--no_correction',
|
|
102
|
+
help='No correction with sbom-info.yaml',
|
|
65
103
|
action='store_true', required=False, default=False)
|
|
66
104
|
parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
|
|
67
105
|
type=str, required=False, default='')
|
|
68
|
-
parser.add_argument('--ui', help='Generate UI mode result file',
|
|
106
|
+
parser.add_argument('--ui', help='Generate UI mode result file',
|
|
107
|
+
action='store_true', required=False, default=False)
|
|
69
108
|
|
|
70
109
|
try:
|
|
71
110
|
args = parser.parse_args()
|
|
@@ -78,12 +117,17 @@ def main():
|
|
|
78
117
|
print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
|
|
79
118
|
else:
|
|
80
119
|
mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
120
|
+
ui, exclude_path, selected_source_scanner, source_write_json_file, source_print_matched_text, \
|
|
121
|
+
source_time_out, binary_simple, = set_args(
|
|
122
|
+
args.mode, args.path, args.dep_argument, args.output,
|
|
123
|
+
args.format, args.link, args.db_url, args.timer, args.raw,
|
|
124
|
+
args.core, args.no_correction, args.correct_fpath, args.ui,
|
|
125
|
+
args.setting, args.exclude_path)
|
|
84
126
|
|
|
85
127
|
run_main(mode, path, dep_argument, output, format, link, db_url, timer,
|
|
86
|
-
raw, core, not no_correction, correct_fpath, ui, exclude_path
|
|
128
|
+
raw, core, not no_correction, correct_fpath, ui, exclude_path,
|
|
129
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text,
|
|
130
|
+
source_time_out, binary_simple)
|
|
87
131
|
|
|
88
132
|
|
|
89
133
|
if __name__ == "__main__":
|
fosslight_scanner/common.py
CHANGED
|
@@ -7,24 +7,22 @@ import os
|
|
|
7
7
|
import sys
|
|
8
8
|
import logging
|
|
9
9
|
import shutil
|
|
10
|
-
import
|
|
11
|
-
import
|
|
12
|
-
import fosslight_util.constant as constant
|
|
13
|
-
from fosslight_util.parsing_yaml import parsing_yml
|
|
14
|
-
from fosslight_util.write_yaml import create_yaml_with_ossitem
|
|
10
|
+
import copy
|
|
11
|
+
from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY
|
|
15
12
|
from fosslight_util.write_scancodejson import write_scancodejson
|
|
16
|
-
from fosslight_util.
|
|
17
|
-
from fosslight_util.output_format import write_output_file
|
|
18
|
-
from fosslight_util.oss_item import OssItem
|
|
13
|
+
from fosslight_util.oss_item import OssItem, FileItem
|
|
19
14
|
|
|
20
|
-
logger = logging.getLogger(
|
|
15
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
21
16
|
SRC_SHEET = 'SRC_FL_Source'
|
|
22
17
|
BIN_SHEET = 'BIN_FL_Binary'
|
|
23
|
-
BIN_EXT_HEADER = {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
18
|
+
BIN_EXT_HEADER = {
|
|
19
|
+
'BIN_FL_Binary': [
|
|
20
|
+
'ID', 'Binary Path', 'OSS Name', 'OSS Version', 'License',
|
|
21
|
+
'Download Location', 'Homepage', 'Copyright Text', 'Exclude',
|
|
22
|
+
'Comment', 'Vulnerability Link', 'TLSH', 'SHA1'
|
|
23
|
+
]
|
|
24
|
+
}
|
|
25
|
+
BIN_HIDDEN_HEADER = {'TLSH', 'SHA1'}
|
|
28
26
|
|
|
29
27
|
|
|
30
28
|
def copy_file(source, destination):
|
|
@@ -38,16 +36,15 @@ def copy_file(source, destination):
|
|
|
38
36
|
except Exception as ex:
|
|
39
37
|
logger.debug(f"Failed to copy {source} to {destination}: {ex}")
|
|
40
38
|
return False, copied_file
|
|
41
|
-
|
|
42
|
-
return True, copied_file
|
|
39
|
+
return True, copied_file
|
|
43
40
|
|
|
44
41
|
|
|
45
42
|
def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
|
|
46
43
|
# This function will be replaced by call_analysis_api().
|
|
47
|
-
logger.info("## Start to run "+str_run_start)
|
|
44
|
+
logger.info("## Start to run " + str_run_start)
|
|
48
45
|
return_value = ""
|
|
49
46
|
try:
|
|
50
|
-
if path_to_run
|
|
47
|
+
if path_to_run:
|
|
51
48
|
logger.info(f"|--- Path to analyze : {path_to_run}")
|
|
52
49
|
os.chdir(output)
|
|
53
50
|
sys.argv = params
|
|
@@ -68,7 +65,7 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
|
|
|
68
65
|
success = True
|
|
69
66
|
result = []
|
|
70
67
|
try:
|
|
71
|
-
if path_to_run
|
|
68
|
+
if path_to_run:
|
|
72
69
|
logger.info(f"|--- Path to analyze : {path_to_run}")
|
|
73
70
|
result = func(*args, **kwargs)
|
|
74
71
|
else:
|
|
@@ -79,86 +76,34 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
|
|
|
79
76
|
success = False
|
|
80
77
|
logger.error(f"{str_run_start}:{ex}")
|
|
81
78
|
try:
|
|
82
|
-
if success:
|
|
83
|
-
if result
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
success = False
|
|
79
|
+
if success and result and return_idx >= 0:
|
|
80
|
+
if len(result) > return_idx:
|
|
81
|
+
result = result[return_idx]
|
|
82
|
+
else:
|
|
83
|
+
success = False
|
|
88
84
|
except Exception as ex:
|
|
89
85
|
logger.debug(f"Get return value:{ex}")
|
|
90
86
|
success = False
|
|
91
|
-
|
|
92
|
-
result = []
|
|
93
|
-
return success, result
|
|
87
|
+
return success, result or []
|
|
94
88
|
|
|
95
89
|
|
|
96
|
-
def
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if column_name in df.columns:
|
|
109
|
-
updated = (df[column_name] == '') | (df[column_name].isnull())
|
|
110
|
-
df.loc[updated, column_name] = oss_name
|
|
111
|
-
df.to_excel(file, sheet_name=sheet_name, index=False)
|
|
112
|
-
except Exception as ex:
|
|
113
|
-
logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
|
|
114
|
-
except Exception as ex:
|
|
115
|
-
logger.debug(f"overwrite_excel:{ex}")
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
|
|
119
|
-
default_oss_name='', default_oss_version='', url=''):
|
|
120
|
-
success = True
|
|
121
|
-
err_msg = ''
|
|
122
|
-
|
|
123
|
-
oss_total_list = []
|
|
124
|
-
yaml_dict = {}
|
|
125
|
-
try:
|
|
126
|
-
for mf in merge_yaml_files:
|
|
127
|
-
if os.path.exists(os.path.join(_output_dir, mf)):
|
|
128
|
-
oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
|
|
129
|
-
|
|
130
|
-
if remove_src_data:
|
|
131
|
-
existed_yaml = {}
|
|
132
|
-
for oi in oss_list:
|
|
133
|
-
oi.name = default_oss_name if oi.name == '' else oi.name
|
|
134
|
-
oi.version = default_oss_version if oi.version == '' else oi.version
|
|
135
|
-
oi.download_location = url if oi.download_location == '' else oi.download_location
|
|
136
|
-
create_yaml_with_ossitem(oi, existed_yaml)
|
|
137
|
-
with open(os.path.join(_output_dir, mf), 'w') as f:
|
|
138
|
-
yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
|
|
139
|
-
|
|
140
|
-
oss_total_list.extend(oss_list)
|
|
141
|
-
|
|
142
|
-
if oss_total_list != []:
|
|
143
|
-
for oti in oss_total_list:
|
|
144
|
-
create_yaml_with_ossitem(oti, yaml_dict)
|
|
145
|
-
with open(os.path.join(_output_dir, final_report), 'w') as f:
|
|
146
|
-
yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
|
|
147
|
-
else:
|
|
148
|
-
success = False
|
|
149
|
-
err_msg = "Output file is not created as no oss items detected."
|
|
150
|
-
except Exception as ex:
|
|
151
|
-
err_msg = ex
|
|
152
|
-
success = False
|
|
153
|
-
|
|
154
|
-
return success, err_msg
|
|
90
|
+
def update_oss_item(scan_item, oss_name, oss_version, download_loc):
|
|
91
|
+
for file_items in scan_item.file_items.values():
|
|
92
|
+
for file_item in file_items:
|
|
93
|
+
if file_item.oss_items:
|
|
94
|
+
for oi in file_item.oss_items:
|
|
95
|
+
if oi.name == '' and oi.version == '' and oi.download_location == '':
|
|
96
|
+
oi.name = oss_name
|
|
97
|
+
oi.version = oss_version
|
|
98
|
+
oi.download_location = download_loc
|
|
99
|
+
else:
|
|
100
|
+
file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
|
|
101
|
+
return scan_item
|
|
155
102
|
|
|
156
103
|
|
|
157
|
-
def create_scancodejson(
|
|
104
|
+
def create_scancodejson(all_scan_item_origin, ui_mode_report, src_path=""):
|
|
158
105
|
success = True
|
|
159
106
|
err_msg = ''
|
|
160
|
-
|
|
161
|
-
oss_total_list = []
|
|
162
107
|
root_dir = ""
|
|
163
108
|
root_strip = ""
|
|
164
109
|
try:
|
|
@@ -170,28 +115,37 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
|
|
|
170
115
|
root_dir = ""
|
|
171
116
|
|
|
172
117
|
try:
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
118
|
+
all_scan_item = copy.deepcopy(all_scan_item_origin)
|
|
119
|
+
if FOSSLIGHT_DEPENDENCY in all_scan_item.file_items:
|
|
120
|
+
del all_scan_item.file_items[FOSSLIGHT_DEPENDENCY]
|
|
176
121
|
if src_path:
|
|
177
|
-
|
|
122
|
+
fileitems_without_oss = []
|
|
123
|
+
for root, _, files in os.walk(src_path):
|
|
178
124
|
root = root.replace(root_strip, "")
|
|
179
125
|
for file in files:
|
|
126
|
+
fi_without_oss = FileItem('')
|
|
127
|
+
included = False
|
|
180
128
|
item_path = os.path.join(root, file)
|
|
181
129
|
item_path = item_path.replace(parent + os.path.sep, '', 1)
|
|
182
|
-
|
|
130
|
+
|
|
131
|
+
for file_items in all_scan_item.file_items.values():
|
|
132
|
+
for file_item in file_items:
|
|
133
|
+
if file_item.source_name_or_path:
|
|
134
|
+
if file_item.source_name_or_path == item_path:
|
|
135
|
+
included = True
|
|
136
|
+
break
|
|
183
137
|
if not included:
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
138
|
+
fi_without_oss.source_name_or_path = item_path
|
|
139
|
+
fileitems_without_oss.append(fi_without_oss)
|
|
140
|
+
if len(fileitems_without_oss) > 0:
|
|
141
|
+
all_scan_item.file_items[FOSSLIGHT_SOURCE].extend(fileitems_without_oss)
|
|
187
142
|
if root_dir:
|
|
188
|
-
for
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
143
|
+
for file_items in all_scan_item.file_items.values():
|
|
144
|
+
for fi in file_items:
|
|
145
|
+
if fi.source_name_or_path:
|
|
146
|
+
fi.source_name_or_path = os.path.join(root_dir, fi.source_name_or_path)
|
|
193
147
|
write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
|
|
194
|
-
|
|
148
|
+
all_scan_item)
|
|
195
149
|
except Exception as ex:
|
|
196
150
|
err_msg = ex
|
|
197
151
|
success = False
|
|
@@ -199,100 +153,58 @@ def create_scancodejson(final_report, output_extension, ui_mode_report, src_path
|
|
|
199
153
|
return success, err_msg
|
|
200
154
|
|
|
201
155
|
|
|
202
|
-
def correct_scanner_result(
|
|
203
|
-
src_oss_list = []
|
|
204
|
-
bin_oss_list = []
|
|
156
|
+
def correct_scanner_result(all_scan_item):
|
|
205
157
|
duplicates = False
|
|
206
158
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
if
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
if exist_src and exist_bin:
|
|
159
|
+
keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
|
|
160
|
+
is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
|
|
161
|
+
if is_contained:
|
|
162
|
+
src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
|
|
163
|
+
bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
|
|
213
164
|
try:
|
|
214
165
|
remove_src_idx_list = []
|
|
215
|
-
for idx_src,
|
|
166
|
+
for idx_src, src_fileitem in enumerate(src_fileitems):
|
|
167
|
+
src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path, src_fileitem.exclude)
|
|
216
168
|
dup_flag = False
|
|
217
|
-
for
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
|
|
169
|
+
for bin_fileitem in bin_fileitems:
|
|
170
|
+
bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path, bin_fileitem.exclude)
|
|
171
|
+
if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
|
|
221
172
|
dup_flag = True
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
173
|
+
src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
|
|
174
|
+
bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
|
|
175
|
+
|
|
176
|
+
if src_all_licenses_non_empty and bin_empty_license_exists:
|
|
177
|
+
exclude = bin_fileitem.oss_items[0].exclude
|
|
178
|
+
bin_fileitem.oss_items = []
|
|
179
|
+
for src_oss_item in src_fileitem.oss_items:
|
|
180
|
+
src_oss_item.exclude = exclude
|
|
181
|
+
bin_fileitem.oss_items.append(src_oss_item)
|
|
182
|
+
bin_fileitem.comment = 'Loaded from SRC OSS info'
|
|
228
183
|
if dup_flag:
|
|
229
184
|
remove_src_idx_list.append(idx_src)
|
|
230
185
|
if remove_src_idx_list:
|
|
231
186
|
duplicates = True
|
|
232
187
|
for i in sorted(remove_src_idx_list, reverse=True):
|
|
233
|
-
del
|
|
188
|
+
del src_fileitems[i]
|
|
234
189
|
except Exception as ex:
|
|
235
190
|
logger.warning(f"correct the scanner result:{ex}")
|
|
236
191
|
|
|
237
192
|
try:
|
|
238
|
-
if exist_src:
|
|
239
|
-
success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
|
|
240
|
-
output_extension, SRC_SHEET)
|
|
241
|
-
if not success:
|
|
242
|
-
logger.warning(err_msg)
|
|
243
|
-
if exist_bin:
|
|
244
|
-
success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
|
|
245
|
-
output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
|
|
246
|
-
if not success:
|
|
247
|
-
logger.warning(err_msg)
|
|
248
193
|
if duplicates:
|
|
249
194
|
logger.info('Success to correct the src/bin scanner result')
|
|
250
195
|
except Exception as ex:
|
|
251
196
|
logger.warning(f"Corrected src/bin scanner result:{ex}")
|
|
252
|
-
return
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
|
|
256
|
-
new_oss_list = []
|
|
257
|
-
sheet_list = {}
|
|
258
|
-
sheet_list[sheetname] = []
|
|
259
|
-
|
|
260
|
-
for src_item in oss_list:
|
|
261
|
-
scanner_name = constant.supported_sheet_and_scanner[sheetname]
|
|
262
|
-
new_oss_list.append(src_item.get_print_array(scanner_name)[0])
|
|
263
|
-
sheet_list[sheetname].extend(new_oss_list)
|
|
264
|
-
if os.path.exists(os.path.join(output_dir, output_file)):
|
|
265
|
-
os.remove(os.path.join(output_dir, output_file))
|
|
266
|
-
success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
|
|
267
|
-
output_extension, sheet_list, extended_hdr, hidden_hdr)
|
|
268
|
-
return success, err_msg
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
|
|
272
|
-
err_reason = ''
|
|
273
|
-
oss_list = []
|
|
274
|
-
oss_file_with_fullpath = os.path.join(_output_dir, output_file)
|
|
275
|
-
|
|
276
|
-
if os.path.exists(oss_file_with_fullpath):
|
|
277
|
-
if output_extension == '.xlsx':
|
|
278
|
-
oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
|
|
279
|
-
elif output_extension == '.yaml':
|
|
280
|
-
oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
|
|
281
|
-
else:
|
|
282
|
-
err_reason = f'Not supported extension: {output_extension}'
|
|
283
|
-
if err_reason:
|
|
284
|
-
logger.info(f'get_osslist: {err_reason}')
|
|
285
|
-
return oss_list
|
|
197
|
+
return all_scan_item
|
|
286
198
|
|
|
287
199
|
|
|
288
|
-
def check_exclude_dir(
|
|
200
|
+
def check_exclude_dir(source_name_or_path, file_item_exclude):
|
|
201
|
+
if file_item_exclude:
|
|
202
|
+
return True
|
|
289
203
|
_exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
|
|
204
|
+
exclude = False
|
|
290
205
|
|
|
291
|
-
for
|
|
292
|
-
if
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
oss_item.exclude = True
|
|
297
|
-
break
|
|
298
|
-
return oss_list
|
|
206
|
+
for exclude_dir in _exclude_dirs:
|
|
207
|
+
if exclude_dir in source_name_or_path.split(os.path.sep):
|
|
208
|
+
exclude = True
|
|
209
|
+
break
|
|
210
|
+
return exclude
|
|
@@ -4,16 +4,17 @@
|
|
|
4
4
|
# Copyright (c) 2020 LG Electronics Inc.
|
|
5
5
|
# SPDX-License-Identifier: Apache-2.0
|
|
6
6
|
import os
|
|
7
|
+
import sys
|
|
8
|
+
import re
|
|
7
9
|
import logging
|
|
8
10
|
import warnings
|
|
9
|
-
import re
|
|
10
11
|
import yaml
|
|
11
|
-
import sys
|
|
12
12
|
import shutil
|
|
13
13
|
import shlex
|
|
14
14
|
import subprocess
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
from datetime import datetime
|
|
17
|
+
|
|
17
18
|
from fosslight_binary import binary_analysis
|
|
18
19
|
from fosslight_dependency.run_dependency_scanner import run_dependency_scanner
|
|
19
20
|
from fosslight_util.download import cli_download_and_extract, compression_extension
|
|
@@ -23,14 +24,16 @@ from fosslight_util.set_log import init_log
|
|
|
23
24
|
from fosslight_util.timer_thread import TimerThread
|
|
24
25
|
import fosslight_util.constant as constant
|
|
25
26
|
from fosslight_util.output_format import check_output_format
|
|
26
|
-
from fosslight_prechecker._precheck import run_lint as prechecker_lint
|
|
27
|
-
from .common import (copy_file, call_analysis_api,
|
|
28
|
-
overwrite_excel,
|
|
29
|
-
merge_yamls, correct_scanner_result,
|
|
30
|
-
create_scancodejson)
|
|
31
|
-
from fosslight_util.write_excel import merge_excels, merge_cover_comment
|
|
32
|
-
from ._run_compare import run_compare
|
|
33
27
|
from fosslight_util.cover import CoverItem
|
|
28
|
+
from fosslight_util.oss_item import ScannerItem
|
|
29
|
+
from fosslight_util.output_format import write_output_file
|
|
30
|
+
|
|
31
|
+
from .common import (
|
|
32
|
+
call_analysis_api, update_oss_item,
|
|
33
|
+
correct_scanner_result, create_scancodejson
|
|
34
|
+
)
|
|
35
|
+
from ._run_compare import run_compare
|
|
36
|
+
|
|
34
37
|
fosslight_source_installed = True
|
|
35
38
|
try:
|
|
36
39
|
from fosslight_source.cli import run_scanners as source_analysis
|
|
@@ -46,11 +49,14 @@ _log_file = "fosslight_log_all_"
|
|
|
46
49
|
_start_time = ""
|
|
47
50
|
_executed_path = ""
|
|
48
51
|
SRC_DIR_FROM_LINK_PREFIX = "fosslight_src_dir_"
|
|
49
|
-
SCANNER_MODE = [
|
|
52
|
+
SCANNER_MODE = [
|
|
53
|
+
"all", "compare", "binary",
|
|
54
|
+
"bin", "src", "source", "dependency", "dep"
|
|
55
|
+
]
|
|
50
56
|
|
|
51
57
|
|
|
52
58
|
def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
|
|
53
|
-
|
|
59
|
+
result = []
|
|
54
60
|
|
|
55
61
|
package_manager = ""
|
|
56
62
|
pip_activate_cmd = ""
|
|
@@ -60,7 +66,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
60
66
|
github_token = ""
|
|
61
67
|
|
|
62
68
|
try:
|
|
63
|
-
if params
|
|
69
|
+
if params:
|
|
64
70
|
match_obj = re.findall(
|
|
65
71
|
r'\s*(-\s*[a|d|m|c|n|t])\s*\'([^\']+)\'\s*', params)
|
|
66
72
|
for param, value in match_obj:
|
|
@@ -84,22 +90,34 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
84
90
|
timer.start()
|
|
85
91
|
|
|
86
92
|
try:
|
|
87
|
-
success,
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
success, scan_item = call_analysis_api(
|
|
94
|
+
path_to_analyze, "Dependency Analysis",
|
|
95
|
+
1, run_dependency_scanner,
|
|
96
|
+
package_manager,
|
|
97
|
+
os.path.abspath(path_to_analyze),
|
|
98
|
+
output_file_with_path,
|
|
99
|
+
pip_activate_cmd, pip_deactivate_cmd,
|
|
100
|
+
output_custom_dir, app_name,
|
|
101
|
+
github_token, path_to_exclude=path_to_exclude
|
|
102
|
+
)
|
|
95
103
|
if success:
|
|
96
|
-
|
|
104
|
+
result = scan_item
|
|
97
105
|
except Exception as ex:
|
|
98
106
|
logger.warning(f"Run dependency: {ex}")
|
|
99
107
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
108
|
+
return result
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def source_analysis_wrapper(*args, **kwargs):
|
|
112
|
+
selected_scanner = kwargs.pop('selected_scanner', 'all')
|
|
113
|
+
source_write_json_file = kwargs.pop('source_write_json_file', False)
|
|
114
|
+
source_print_matched_text = kwargs.pop('source_print_matched_text', False)
|
|
115
|
+
source_time_out = kwargs.pop('source_time_out', 120)
|
|
116
|
+
args = list(args)
|
|
117
|
+
args.insert(2, source_write_json_file)
|
|
118
|
+
args.insert(5, source_print_matched_text)
|
|
119
|
+
|
|
120
|
+
return source_analysis(*args, selected_scanner=selected_scanner, time_out=source_time_out, **kwargs)
|
|
103
121
|
|
|
104
122
|
|
|
105
123
|
def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
@@ -107,10 +125,13 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
107
125
|
remove_src_data=True, result_log={}, output_file="",
|
|
108
126
|
output_extension="", num_cores=-1, db_url="",
|
|
109
127
|
default_oss_name="", default_oss_version="", url="",
|
|
110
|
-
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]
|
|
128
|
+
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
|
|
129
|
+
selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
|
|
130
|
+
source_time_out=120, binary_simple=False):
|
|
111
131
|
final_excel_dir = output_path
|
|
112
132
|
success = True
|
|
113
|
-
|
|
133
|
+
all_cover_items = []
|
|
134
|
+
all_scan_item = ScannerItem(PKG_NAME, _start_time)
|
|
114
135
|
if not remove_src_data:
|
|
115
136
|
success, final_excel_dir, result_log = init(output_path)
|
|
116
137
|
|
|
@@ -130,28 +151,30 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
130
151
|
if success:
|
|
131
152
|
output_files = {"SRC": f"fosslight_src_{_start_time}{output_extension}",
|
|
132
153
|
"BIN": f"fosslight_bin_{_start_time}{output_extension}",
|
|
133
|
-
"DEP": f"fosslight_dep_{_start_time}{output_extension}"
|
|
134
|
-
"PRECHECKER": f"fosslight_lint_{_start_time}.yaml"}
|
|
135
|
-
if run_prechecker:
|
|
136
|
-
output_prechecker = os.path.join(_output_dir, output_files["PRECHECKER"])
|
|
137
|
-
success, result = call_analysis_api(src_path, "Prechecker Lint",
|
|
138
|
-
-1, prechecker_lint,
|
|
139
|
-
abs_path, False, output_prechecker,
|
|
140
|
-
exclude_path=path_to_exclude)
|
|
141
|
-
success_file, copied_file = copy_file(output_prechecker, output_path)
|
|
142
|
-
if success_file:
|
|
143
|
-
temp_output_fiiles.append(copied_file)
|
|
154
|
+
"DEP": f"fosslight_dep_{_start_time}{output_extension}"}
|
|
144
155
|
|
|
145
156
|
if run_src:
|
|
146
157
|
try:
|
|
147
158
|
if fosslight_source_installed:
|
|
148
159
|
src_output = os.path.join(_output_dir, output_files["SRC"])
|
|
149
|
-
success, result = call_analysis_api(
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
160
|
+
success, result = call_analysis_api(
|
|
161
|
+
src_path,
|
|
162
|
+
"Source Analysis",
|
|
163
|
+
-1, source_analysis_wrapper,
|
|
164
|
+
abs_path,
|
|
165
|
+
src_output,
|
|
166
|
+
num_cores,
|
|
167
|
+
False,
|
|
168
|
+
path_to_exclude=path_to_exclude,
|
|
169
|
+
selected_scanner=selected_source_scanner,
|
|
170
|
+
source_write_json_file=source_write_json_file,
|
|
171
|
+
source_print_matched_text=source_print_matched_text,
|
|
172
|
+
source_time_out=source_time_out
|
|
173
|
+
)
|
|
174
|
+
if success:
|
|
175
|
+
all_scan_item.file_items.update(result[2].file_items)
|
|
176
|
+
all_cover_items.append(result[2].cover)
|
|
177
|
+
|
|
155
178
|
else: # Run fosslight_source by using docker image
|
|
156
179
|
src_output = os.path.join("output", output_files["SRC"])
|
|
157
180
|
output_rel_path = os.path.relpath(abs_path, os.getcwd())
|
|
@@ -166,16 +189,22 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
166
189
|
logger.warning(f"Failed to run source analysis: {ex}")
|
|
167
190
|
|
|
168
191
|
if run_bin:
|
|
169
|
-
success,
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
192
|
+
success, result = call_analysis_api(src_path, "Binary Analysis",
|
|
193
|
+
1, binary_analysis.find_binaries,
|
|
194
|
+
abs_path,
|
|
195
|
+
os.path.join(_output_dir, output_files["BIN"]),
|
|
196
|
+
"", db_url, binary_simple,
|
|
197
|
+
correct_mode, correct_fpath,
|
|
198
|
+
path_to_exclude=path_to_exclude)
|
|
199
|
+
if success:
|
|
200
|
+
all_scan_item.file_items.update(result.file_items)
|
|
201
|
+
all_cover_items.append(result.cover)
|
|
176
202
|
|
|
177
203
|
if run_dep:
|
|
178
|
-
run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
|
|
204
|
+
dep_scanitem = run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
|
|
205
|
+
dep_arguments, path_to_exclude)
|
|
206
|
+
all_scan_item.file_items.update(dep_scanitem.file_items)
|
|
207
|
+
all_cover_items.append(dep_scanitem.cover)
|
|
179
208
|
|
|
180
209
|
else:
|
|
181
210
|
return
|
|
@@ -186,45 +215,23 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
186
215
|
try:
|
|
187
216
|
output_file_without_ext = os.path.join(final_excel_dir, output_file)
|
|
188
217
|
final_report = f"{output_file_without_ext}{output_extension}"
|
|
189
|
-
merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
|
|
190
218
|
cover = CoverItem(tool_name=PKG_NAME,
|
|
191
219
|
start_time=_start_time,
|
|
192
220
|
input_path=abs_path,
|
|
193
|
-
exclude_path=path_to_exclude
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
|
|
208
|
-
if exist_src or exist_bin:
|
|
209
|
-
correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
|
|
210
|
-
|
|
211
|
-
if remove_src_data:
|
|
212
|
-
overwrite_excel(_output_dir, default_oss_name, "OSS Name")
|
|
213
|
-
overwrite_excel(_output_dir, default_oss_version, "OSS Version")
|
|
214
|
-
overwrite_excel(_output_dir, url, "Download Location")
|
|
215
|
-
success, err_msg = merge_excels(_output_dir, final_report, merge_files, cover)
|
|
216
|
-
|
|
217
|
-
if correct_mode:
|
|
218
|
-
if exist_src:
|
|
219
|
-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
|
|
220
|
-
os.path.join(_output_dir, output_files['SRC']))
|
|
221
|
-
if exist_bin:
|
|
222
|
-
shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
|
|
223
|
-
os.path.join(_output_dir, output_files['BIN']))
|
|
224
|
-
shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
|
|
225
|
-
elif output_extension == ".yaml":
|
|
226
|
-
success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
|
|
227
|
-
remove_src_data, default_oss_name, default_oss_version, url)
|
|
221
|
+
exclude_path=path_to_exclude,
|
|
222
|
+
simple_mode=False)
|
|
223
|
+
merge_comment = []
|
|
224
|
+
for ci in all_cover_items:
|
|
225
|
+
merge_comment.append(str(f'[{ci.tool_name}] {ci.comment}'))
|
|
226
|
+
cover.comment = '\n'.join(merge_comment)
|
|
227
|
+
all_scan_item.cover = cover
|
|
228
|
+
|
|
229
|
+
if correct_mode:
|
|
230
|
+
all_scan_item = correct_scanner_result(all_scan_item)
|
|
231
|
+
|
|
232
|
+
if remove_src_data:
|
|
233
|
+
all_scan_item = update_oss_item(all_scan_item, default_oss_name, default_oss_version, url)
|
|
234
|
+
success, err_msg, final_report = write_output_file(output_file_without_ext, output_extension, all_scan_item)
|
|
228
235
|
if success:
|
|
229
236
|
if os.path.isfile(final_report):
|
|
230
237
|
logger.info(f'Generated the result file: {final_report}')
|
|
@@ -236,7 +243,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
236
243
|
|
|
237
244
|
if ui_mode:
|
|
238
245
|
ui_mode_report = f"{output_file_without_ext}.json"
|
|
239
|
-
success, err_msg = create_scancodejson(
|
|
246
|
+
success, err_msg = create_scancodejson(all_scan_item, ui_mode_report, src_path)
|
|
240
247
|
if success and os.path.isfile(ui_mode_report):
|
|
241
248
|
logger.info(f'Generated the ui mode result file: {ui_mode_report}')
|
|
242
249
|
else:
|
|
@@ -308,7 +315,9 @@ def init(output_path="", make_outdir=True):
|
|
|
308
315
|
|
|
309
316
|
def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze,
|
|
310
317
|
db_url, hide_progressbar=False, keep_raw_data=False, num_cores=-1,
|
|
311
|
-
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]
|
|
318
|
+
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
|
|
319
|
+
selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
|
|
320
|
+
source_time_out=120, binary_simple=False):
|
|
312
321
|
global _executed_path, _start_time
|
|
313
322
|
|
|
314
323
|
output_file = ""
|
|
@@ -380,19 +389,13 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
380
389
|
run_src = False
|
|
381
390
|
run_bin = False
|
|
382
391
|
run_dep = False
|
|
383
|
-
run_prechecker = False
|
|
384
392
|
remove_downloaded_source = False
|
|
385
393
|
|
|
386
394
|
if "all" in mode_list or (not mode_list):
|
|
387
395
|
run_src = True
|
|
388
396
|
run_bin = True
|
|
389
397
|
run_dep = True
|
|
390
|
-
run_prechecker = False
|
|
391
|
-
if "prechecker" in mode_list or "reuse" in mode_list:
|
|
392
|
-
run_prechecker = True
|
|
393
398
|
else:
|
|
394
|
-
if "prechecker" in mode_list or "reuse" in mode_list:
|
|
395
|
-
run_prechecker = True
|
|
396
399
|
if "binary" in mode_list or "bin" in mode_list:
|
|
397
400
|
run_bin = True
|
|
398
401
|
if "source" in mode_list or "src" in mode_list:
|
|
@@ -400,7 +403,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
400
403
|
if "dependency" in mode_list or "dep" in mode_list:
|
|
401
404
|
run_dep = True
|
|
402
405
|
|
|
403
|
-
if run_dep or run_src or run_bin
|
|
406
|
+
if run_dep or run_src or run_bin:
|
|
404
407
|
if src_path == "" and url_to_analyze == "":
|
|
405
408
|
src_path, dep_arguments, url_to_analyze = get_input_mode(_executed_path, mode_list)
|
|
406
409
|
|
|
@@ -422,11 +425,13 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
422
425
|
|
|
423
426
|
if src_path != "":
|
|
424
427
|
run_scanner(src_path, dep_arguments, output_path, keep_raw_data,
|
|
425
|
-
run_src, run_bin, run_dep,
|
|
428
|
+
run_src, run_bin, run_dep, '',
|
|
426
429
|
remove_downloaded_source, {}, output_file,
|
|
427
430
|
output_extension, num_cores, db_url,
|
|
428
431
|
default_oss_name, default_oss_version, url_to_analyze,
|
|
429
|
-
correct_mode, correct_fpath, ui_mode, path_to_exclude
|
|
432
|
+
correct_mode, correct_fpath, ui_mode, path_to_exclude,
|
|
433
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out,
|
|
434
|
+
binary_simple)
|
|
430
435
|
|
|
431
436
|
if extract_folder:
|
|
432
437
|
shutil.rmtree(extract_folder)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: fosslight-scanner
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: FOSSLight Scanner
|
|
5
5
|
Home-page: https://github.com/fosslight/fosslight_scanner
|
|
6
6
|
Author: LG Electronics
|
|
@@ -21,11 +21,10 @@ Requires-Dist: openpyxl
|
|
|
21
21
|
Requires-Dist: progress
|
|
22
22
|
Requires-Dist: pyyaml
|
|
23
23
|
Requires-Dist: beautifulsoup4
|
|
24
|
-
Requires-Dist: fosslight-util
|
|
25
|
-
Requires-Dist: fosslight-source
|
|
26
|
-
Requires-Dist: fosslight-dependency
|
|
27
|
-
Requires-Dist: fosslight-binary
|
|
28
|
-
Requires-Dist: fosslight-prechecker >=3.0.27
|
|
24
|
+
Requires-Dist: fosslight-util>=2.0.0
|
|
25
|
+
Requires-Dist: fosslight-source>=2.0.0
|
|
26
|
+
Requires-Dist: fosslight-dependency>=4.0.0
|
|
27
|
+
Requires-Dist: fosslight-binary>=5.0.0
|
|
29
28
|
|
|
30
29
|
<!--
|
|
31
30
|
Copyright (c) 2021 LG Electronics
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
fosslight_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
fosslight_scanner/_get_input.py,sha256=f78M57QU7Lhw1VfZJKvYXbGsUiYMViPz1---UDIbV9I,1396
|
|
3
|
+
fosslight_scanner/_help.py,sha256=-77CZm1es2Ib-HuISCSd4mTPxYzuLJ9NjryDI0g4otM,2921
|
|
4
|
+
fosslight_scanner/_parse_setting.py,sha256=SoIEoPGlFXAp6__eEFEMIPYPoKLvmqxXF5X_C_uFiVI,2468
|
|
5
|
+
fosslight_scanner/_run_compare.py,sha256=ufx3zldtM7sUOyQOYSgs66TY5-Gt26yE2Irz5kdkmio,10566
|
|
6
|
+
fosslight_scanner/cli.py,sha256=K088vCNTBLaiOfxW2EDkrWRhrRs7IILmDfP1HSdSI2U,6536
|
|
7
|
+
fosslight_scanner/common.py,sha256=NOfIxdKrH7MS_ptOIwY56OZJHVR9S_hfs-L2a1H9N0g,8179
|
|
8
|
+
fosslight_scanner/fosslight_scanner.py,sha256=fIP4SJ3zN8E-QeChgmh5fNj4nO3eD-a5kVZ7SoAcz3s,18693
|
|
9
|
+
fosslight_scanner/resources/bom_compare.html,sha256=VocJ9bDmQQOwfGyvXatPZ4W-QddO-IlsAvKdO0nZ7pA,2240
|
|
10
|
+
fosslight_scanner-2.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
11
|
+
fosslight_scanner-2.0.0.dist-info/METADATA,sha256=SjHAGy0zLaC6FQxJDMsxHmcBopgW8VR4kp3fNj23yJ8,7794
|
|
12
|
+
fosslight_scanner-2.0.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
13
|
+
fosslight_scanner-2.0.0.dist-info/entry_points.txt,sha256=0tBUtN9pKtAkuohQuhbzuohdHbeoI0-w95aWAR7J_RU,105
|
|
14
|
+
fosslight_scanner-2.0.0.dist-info/top_level.txt,sha256=43_xLb5KYpy8wOU1H2Wd2fEsWBY7Dg6ZEJJXkfT64Ak,18
|
|
15
|
+
fosslight_scanner-2.0.0.dist-info/RECORD,,
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
fosslight_scanner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
fosslight_scanner/_get_input.py,sha256=f78M57QU7Lhw1VfZJKvYXbGsUiYMViPz1---UDIbV9I,1396
|
|
3
|
-
fosslight_scanner/_help.py,sha256=-77CZm1es2Ib-HuISCSd4mTPxYzuLJ9NjryDI0g4otM,2921
|
|
4
|
-
fosslight_scanner/_parse_setting.py,sha256=egFRtVYBHfRv3Z2fSQEAS7h8a1QtM8ZZydtppVmNwcw,1784
|
|
5
|
-
fosslight_scanner/_run_compare.py,sha256=1B39bD6uPfjIIMgbIC-sBbPJbTYgaRybZtRSqbRfzRI,10147
|
|
6
|
-
fosslight_scanner/cli.py,sha256=wFtQaBK3cwyF48pbslbGrWY2Mk63TXmfgAEb9I83ElY,5076
|
|
7
|
-
fosslight_scanner/common.py,sha256=09e4xpGnoD7UTAcjkmLTKgxfOgY6WsJmUccjoDiGUog,11859
|
|
8
|
-
fosslight_scanner/fosslight_scanner.py,sha256=v_fxIbD-7gDhGZddukjeySoW_yDWGKEjc62uLgPUYp8,19705
|
|
9
|
-
fosslight_scanner/resources/bom_compare.html,sha256=VocJ9bDmQQOwfGyvXatPZ4W-QddO-IlsAvKdO0nZ7pA,2240
|
|
10
|
-
fosslight_scanner-1.7.30.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
11
|
-
fosslight_scanner-1.7.30.dist-info/METADATA,sha256=VU0IaEg2dv_JRUvNPNibxPL1ZNjRtncdCjghDbh5mNI,7847
|
|
12
|
-
fosslight_scanner-1.7.30.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
13
|
-
fosslight_scanner-1.7.30.dist-info/entry_points.txt,sha256=0tBUtN9pKtAkuohQuhbzuohdHbeoI0-w95aWAR7J_RU,105
|
|
14
|
-
fosslight_scanner-1.7.30.dist-info/top_level.txt,sha256=43_xLb5KYpy8wOU1H2Wd2fEsWBY7Dg6ZEJJXkfT64Ak,18
|
|
15
|
-
fosslight_scanner-1.7.30.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|