fosslight-scanner 1.7.29__tar.gz → 1.7.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/PKG-INFO +1 -1
- fosslight_scanner-1.7.31/requirements.txt +11 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/setup.py +1 -1
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/_parse_setting.py +23 -8
- fosslight_scanner-1.7.31/src/fosslight_scanner/cli.py +134 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/common.py +24 -24
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/fosslight_scanner.py +82 -39
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/PKG-INFO +1 -1
- fosslight_scanner-1.7.31/src/fosslight_scanner.egg-info/requires.txt +11 -0
- fosslight_scanner-1.7.29/requirements.txt +0 -11
- fosslight_scanner-1.7.29/src/fosslight_scanner/cli.py +0 -90
- fosslight_scanner-1.7.29/src/fosslight_scanner.egg-info/requires.txt +0 -11
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/LICENSE +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/MANIFEST.in +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/README.md +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/setup.cfg +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/__init__.py +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/_get_input.py +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/_help.py +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/_run_compare.py +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/resources/bom_compare.html +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/SOURCES.txt +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/dependency_links.txt +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/entry_points.txt +0 -0
- {fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/top_level.txt +0 -0
|
@@ -15,7 +15,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
|
|
|
15
15
|
if __name__ == "__main__":
|
|
16
16
|
setup(
|
|
17
17
|
name='fosslight_scanner',
|
|
18
|
-
version='1.7.
|
|
18
|
+
version='1.7.31',
|
|
19
19
|
package_dir={"": "src"},
|
|
20
20
|
packages=find_packages(where='src'),
|
|
21
21
|
description='FOSSLight Scanner',
|
{fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/_parse_setting.py
RENAMED
|
@@ -11,7 +11,7 @@ def parse_setting_json(data):
|
|
|
11
11
|
dep_argument = data.get('dep_argument', '')
|
|
12
12
|
output = data.get('output', '')
|
|
13
13
|
format = data.get('format', '')
|
|
14
|
-
link = data.get('link',
|
|
14
|
+
link = data.get('link', '')
|
|
15
15
|
db_url = data.get('db_url', '')
|
|
16
16
|
timer = data.get('timer', False)
|
|
17
17
|
raw = data.get('raw', False)
|
|
@@ -20,34 +20,49 @@ def parse_setting_json(data):
|
|
|
20
20
|
correct_fpath = data.get('correct_fpath', '')
|
|
21
21
|
ui = data.get('ui', False)
|
|
22
22
|
exclude_path = data.get('exclude', [])
|
|
23
|
-
|
|
23
|
+
selected_source_scanner = data.get('selected_source_scanner', '')
|
|
24
|
+
source_write_json_file = data.get('source_write_json_file', False)
|
|
25
|
+
source_print_matched_text = data.get('source_print_matched_text', False)
|
|
26
|
+
source_time_out = data.get('source_time_out', 120)
|
|
27
|
+
binary_simple = data.get('binary_simple', False)
|
|
24
28
|
str_lists = [mode, path, exclude_path]
|
|
25
|
-
strings = [
|
|
26
|
-
|
|
29
|
+
strings = [
|
|
30
|
+
dep_argument, output, format, db_url,
|
|
31
|
+
correct_fpath, link, selected_source_scanner
|
|
32
|
+
]
|
|
33
|
+
booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple]
|
|
34
|
+
|
|
27
35
|
is_incorrect = False
|
|
28
36
|
|
|
29
37
|
# check if json file is incorrect format
|
|
30
38
|
for i, target in enumerate(str_lists):
|
|
31
|
-
if not (isinstance(target, list) and
|
|
39
|
+
if not (isinstance(target, list) and
|
|
40
|
+
all(isinstance(item, str) for item in target)):
|
|
32
41
|
is_incorrect = True
|
|
33
42
|
str_lists[i] = []
|
|
34
43
|
|
|
35
44
|
for i, target in enumerate(strings):
|
|
36
45
|
if not isinstance(target, str):
|
|
37
46
|
is_incorrect = True
|
|
38
|
-
|
|
47
|
+
strings[i] = ''
|
|
39
48
|
|
|
40
49
|
for i, target in enumerate(booleans):
|
|
41
50
|
if not isinstance(target, bool):
|
|
42
51
|
is_incorrect = True
|
|
43
|
-
|
|
52
|
+
booleans[i] = False
|
|
44
53
|
|
|
45
54
|
if not isinstance(core, int):
|
|
46
55
|
is_incorrect = True
|
|
47
56
|
core = -1
|
|
48
57
|
|
|
58
|
+
if not isinstance(source_time_out, int):
|
|
59
|
+
is_incorrect = True
|
|
60
|
+
source_time_out = 120
|
|
61
|
+
|
|
49
62
|
if is_incorrect:
|
|
50
63
|
print('Ignoring some values with incorrect format in the setting file.')
|
|
51
64
|
|
|
52
65
|
return mode, path, dep_argument, output, format, link, db_url, timer, \
|
|
53
|
-
raw, core, no_correction, correct_fpath, ui, exclude_path
|
|
66
|
+
raw, core, no_correction, correct_fpath, ui, exclude_path, \
|
|
67
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
|
|
68
|
+
binary_simple
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2022 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
import sys
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import os.path
|
|
9
|
+
from argparse import ArgumentParser
|
|
10
|
+
|
|
11
|
+
from ._help import print_help_msg
|
|
12
|
+
from .fosslight_scanner import run_main, PKG_NAME
|
|
13
|
+
from ._parse_setting import parse_setting_json
|
|
14
|
+
from fosslight_util.help import print_package_version
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
|
|
18
|
+
raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
|
|
19
|
+
|
|
20
|
+
selected_source_scanner = "all"
|
|
21
|
+
source_write_json_file = False
|
|
22
|
+
source_print_matched_text = False
|
|
23
|
+
source_time_out = 120
|
|
24
|
+
binary_simple = False
|
|
25
|
+
|
|
26
|
+
if setting and os.path.isfile(setting):
|
|
27
|
+
try:
|
|
28
|
+
with open(setting, 'r', encoding='utf-8') as file:
|
|
29
|
+
data = json.load(file)
|
|
30
|
+
s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
|
|
31
|
+
s_no_correction, s_correct_fpath, s_ui, s_exclude_path, \
|
|
32
|
+
s_selected_source_scanner, s_source_write_json_file, s_source_print_matched_text, \
|
|
33
|
+
s_source_time_out, s_binary_simple = parse_setting_json(data)
|
|
34
|
+
|
|
35
|
+
# direct cli arguments have higher priority than setting file
|
|
36
|
+
mode = mode or s_mode
|
|
37
|
+
path = path or s_path
|
|
38
|
+
dep_argument = dep_argument or s_dep_argument
|
|
39
|
+
output = output or s_output
|
|
40
|
+
format = format or s_format
|
|
41
|
+
link = link or s_link
|
|
42
|
+
db_url = db_url or s_db_url
|
|
43
|
+
timer = timer or s_timer
|
|
44
|
+
raw = raw or s_raw
|
|
45
|
+
core = core if core != -1 else s_core
|
|
46
|
+
no_correction = no_correction or s_no_correction
|
|
47
|
+
correct_fpath = correct_fpath or s_correct_fpath
|
|
48
|
+
ui = ui or s_ui
|
|
49
|
+
exclude_path = exclude_path or s_exclude_path
|
|
50
|
+
|
|
51
|
+
# These options are only set from the setting file, not from CLI arguments
|
|
52
|
+
selected_source_scanner = s_selected_source_scanner or selected_source_scanner
|
|
53
|
+
source_write_json_file = s_source_write_json_file
|
|
54
|
+
source_print_matched_text = s_source_print_matched_text
|
|
55
|
+
source_time_out = s_source_time_out if s_source_time_out != 120 else source_time_out
|
|
56
|
+
binary_simple = s_binary_simple
|
|
57
|
+
|
|
58
|
+
except Exception as e:
|
|
59
|
+
print(f"Cannot open setting file: {e}")
|
|
60
|
+
return mode, path, dep_argument, output, format, link, db_url, timer, \
|
|
61
|
+
raw, core, no_correction, correct_fpath, ui, exclude_path, \
|
|
62
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
|
|
63
|
+
binary_simple
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def main():
|
|
67
|
+
parser = ArgumentParser(description='FOSSLight Scanner',
|
|
68
|
+
prog='fosslight_scanner', add_help=False)
|
|
69
|
+
parser.add_argument('mode', nargs='*',
|
|
70
|
+
help='source| dependency| binary| all| compare',
|
|
71
|
+
default="")
|
|
72
|
+
parser.add_argument('--path', '-p',
|
|
73
|
+
help='Path to analyze (In compare mode, two FOSSLight reports',
|
|
74
|
+
dest='path', nargs='+', default="")
|
|
75
|
+
parser.add_argument('--wget', '-w', help='Link to be analyzed',
|
|
76
|
+
type=str, dest='link', default="")
|
|
77
|
+
parser.add_argument('--format', '-f',
|
|
78
|
+
help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
|
|
79
|
+
type=str, dest='format', default="")
|
|
80
|
+
parser.add_argument('--output', '-o', help='Output directory or file',
|
|
81
|
+
type=str, dest='output', default="")
|
|
82
|
+
parser.add_argument('--dependency', '-d', help='Dependency arguments',
|
|
83
|
+
type=str, dest='dep_argument', default="")
|
|
84
|
+
parser.add_argument('--url', '-u', help="DB Url",
|
|
85
|
+
type=str, dest='db_url', default="")
|
|
86
|
+
parser.add_argument('--core', '-c',
|
|
87
|
+
help='Number of processes to analyze source',
|
|
88
|
+
type=int, dest='core', default=-1)
|
|
89
|
+
parser.add_argument('--raw', '-r', help='Keep raw data',
|
|
90
|
+
action='store_true', dest='raw', default=False)
|
|
91
|
+
parser.add_argument('--timer', '-t', help='Hide the progress bar',
|
|
92
|
+
action='store_true', dest='timer', default=False)
|
|
93
|
+
parser.add_argument('--version', '-v', help='Print version',
|
|
94
|
+
action='store_true', dest='version', default=False)
|
|
95
|
+
parser.add_argument('--help', '-h', help='Print help message',
|
|
96
|
+
action='store_true', dest='help')
|
|
97
|
+
parser.add_argument('--exclude', '-e', help='Path to exclude from analysis',
|
|
98
|
+
dest='exclude_path', nargs='*', default=[])
|
|
99
|
+
parser.add_argument('--setting', '-s', help='Scanner json setting file',
|
|
100
|
+
type=str, dest='setting', default="")
|
|
101
|
+
parser.add_argument('--no_correction',
|
|
102
|
+
help='No correction with sbom-info.yaml',
|
|
103
|
+
action='store_true', required=False, default=False)
|
|
104
|
+
parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
|
|
105
|
+
type=str, required=False, default='')
|
|
106
|
+
parser.add_argument('--ui', help='Generate UI mode result file',
|
|
107
|
+
action='store_true', required=False, default=False)
|
|
108
|
+
|
|
109
|
+
try:
|
|
110
|
+
args = parser.parse_args()
|
|
111
|
+
except SystemExit:
|
|
112
|
+
sys.exit(1)
|
|
113
|
+
|
|
114
|
+
if args.help:
|
|
115
|
+
print_help_msg()
|
|
116
|
+
elif args.version:
|
|
117
|
+
print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
|
|
118
|
+
else:
|
|
119
|
+
mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
|
|
120
|
+
ui, exclude_path, selected_source_scanner, source_write_json_file, source_print_matched_text, \
|
|
121
|
+
source_time_out, binary_simple, = set_args(
|
|
122
|
+
args.mode, args.path, args.dep_argument, args.output,
|
|
123
|
+
args.format, args.link, args.db_url, args.timer, args.raw,
|
|
124
|
+
args.core, args.no_correction, args.correct_fpath, args.ui,
|
|
125
|
+
args.setting, args.exclude_path)
|
|
126
|
+
|
|
127
|
+
run_main(mode, path, dep_argument, output, format, link, db_url, timer,
|
|
128
|
+
raw, core, not no_correction, correct_fpath, ui, exclude_path,
|
|
129
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text,
|
|
130
|
+
source_time_out, binary_simple)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == "__main__":
|
|
134
|
+
main()
|
|
@@ -9,6 +9,7 @@ import logging
|
|
|
9
9
|
import shutil
|
|
10
10
|
import pandas as pd
|
|
11
11
|
import yaml
|
|
12
|
+
|
|
12
13
|
import fosslight_util.constant as constant
|
|
13
14
|
from fosslight_util.parsing_yaml import parsing_yml
|
|
14
15
|
from fosslight_util.write_yaml import create_yaml_with_ossitem
|
|
@@ -20,11 +21,14 @@ from fosslight_util.oss_item import OssItem
|
|
|
20
21
|
logger = logging.getLogger(constant.LOGGER_NAME)
|
|
21
22
|
SRC_SHEET = 'SRC_FL_Source'
|
|
22
23
|
BIN_SHEET = 'BIN_FL_Binary'
|
|
23
|
-
BIN_EXT_HEADER = {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
BIN_EXT_HEADER = {
|
|
25
|
+
'BIN_FL_Binary': [
|
|
26
|
+
'ID', 'Binary Path', 'OSS Name', 'OSS Version', 'License',
|
|
27
|
+
'Download Location', 'Homepage', 'Copyright Text', 'Exclude',
|
|
28
|
+
'Comment', 'Vulnerability Link', 'TLSH', 'SHA1'
|
|
29
|
+
]
|
|
30
|
+
}
|
|
31
|
+
BIN_HIDDEN_HEADER = {'TLSH', 'SHA1'}
|
|
28
32
|
|
|
29
33
|
|
|
30
34
|
def copy_file(source, destination):
|
|
@@ -38,16 +42,15 @@ def copy_file(source, destination):
|
|
|
38
42
|
except Exception as ex:
|
|
39
43
|
logger.debug(f"Failed to copy {source} to {destination}: {ex}")
|
|
40
44
|
return False, copied_file
|
|
41
|
-
|
|
42
|
-
return True, copied_file
|
|
45
|
+
return True, copied_file
|
|
43
46
|
|
|
44
47
|
|
|
45
48
|
def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
|
|
46
49
|
# This function will be replaced by call_analysis_api().
|
|
47
|
-
logger.info("## Start to run "+str_run_start)
|
|
50
|
+
logger.info("## Start to run " + str_run_start)
|
|
48
51
|
return_value = ""
|
|
49
52
|
try:
|
|
50
|
-
if path_to_run
|
|
53
|
+
if path_to_run:
|
|
51
54
|
logger.info(f"|--- Path to analyze : {path_to_run}")
|
|
52
55
|
os.chdir(output)
|
|
53
56
|
sys.argv = params
|
|
@@ -68,7 +71,7 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
|
|
|
68
71
|
success = True
|
|
69
72
|
result = []
|
|
70
73
|
try:
|
|
71
|
-
if path_to_run
|
|
74
|
+
if path_to_run:
|
|
72
75
|
logger.info(f"|--- Path to analyze : {path_to_run}")
|
|
73
76
|
result = func(*args, **kwargs)
|
|
74
77
|
else:
|
|
@@ -79,36 +82,33 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwa
|
|
|
79
82
|
success = False
|
|
80
83
|
logger.error(f"{str_run_start}:{ex}")
|
|
81
84
|
try:
|
|
82
|
-
if success:
|
|
83
|
-
if result
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
success = False
|
|
85
|
+
if success and result and return_idx >= 0:
|
|
86
|
+
if len(result) > return_idx:
|
|
87
|
+
result = result[return_idx]
|
|
88
|
+
else:
|
|
89
|
+
success = False
|
|
88
90
|
except Exception as ex:
|
|
89
91
|
logger.debug(f"Get return value:{ex}")
|
|
90
92
|
success = False
|
|
91
|
-
|
|
92
|
-
result = []
|
|
93
|
-
return success, result
|
|
93
|
+
return success, result or []
|
|
94
94
|
|
|
95
95
|
|
|
96
96
|
def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
|
|
97
|
-
if oss_name
|
|
97
|
+
if oss_name:
|
|
98
98
|
try:
|
|
99
99
|
files = os.listdir(excel_file_path)
|
|
100
100
|
for file in files:
|
|
101
101
|
if file.endswith(".xlsx"):
|
|
102
|
-
|
|
103
|
-
excel_file = pd.ExcelFile(
|
|
102
|
+
file_path = os.path.join(excel_file_path, file)
|
|
103
|
+
excel_file = pd.ExcelFile(file_path, engine='openpyxl')
|
|
104
104
|
|
|
105
105
|
for sheet_name in excel_file.sheet_names:
|
|
106
106
|
try:
|
|
107
|
-
df = pd.read_excel(
|
|
107
|
+
df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl')
|
|
108
108
|
if column_name in df.columns:
|
|
109
109
|
updated = (df[column_name] == '') | (df[column_name].isnull())
|
|
110
110
|
df.loc[updated, column_name] = oss_name
|
|
111
|
-
df.to_excel(
|
|
111
|
+
df.to_excel(file_path, sheet_name=sheet_name, index=False)
|
|
112
112
|
except Exception as ex:
|
|
113
113
|
logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
|
|
114
114
|
except Exception as ex:
|
{fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner/fosslight_scanner.py
RENAMED
|
@@ -4,32 +4,36 @@
|
|
|
4
4
|
# Copyright (c) 2020 LG Electronics Inc.
|
|
5
5
|
# SPDX-License-Identifier: Apache-2.0
|
|
6
6
|
import os
|
|
7
|
+
import sys
|
|
8
|
+
import re
|
|
7
9
|
import logging
|
|
8
10
|
import warnings
|
|
9
|
-
import re
|
|
10
11
|
import yaml
|
|
11
|
-
import sys
|
|
12
12
|
import shutil
|
|
13
13
|
import shlex
|
|
14
14
|
import subprocess
|
|
15
15
|
from pathlib import Path
|
|
16
16
|
from datetime import datetime
|
|
17
|
+
|
|
17
18
|
from fosslight_binary import binary_analysis
|
|
18
19
|
from fosslight_dependency.run_dependency_scanner import run_dependency_scanner
|
|
19
|
-
from fosslight_util.download import cli_download_and_extract
|
|
20
|
+
from fosslight_util.download import cli_download_and_extract, compression_extension
|
|
21
|
+
from fosslight_util.download import extract_compressed_file as extract_file
|
|
20
22
|
from ._get_input import get_input_mode
|
|
21
23
|
from fosslight_util.set_log import init_log
|
|
22
24
|
from fosslight_util.timer_thread import TimerThread
|
|
23
25
|
import fosslight_util.constant as constant
|
|
24
26
|
from fosslight_util.output_format import check_output_format
|
|
25
27
|
from fosslight_prechecker._precheck import run_lint as prechecker_lint
|
|
26
|
-
from .common import (copy_file, call_analysis_api,
|
|
27
|
-
overwrite_excel,
|
|
28
|
-
merge_yamls, correct_scanner_result,
|
|
29
|
-
create_scancodejson)
|
|
30
28
|
from fosslight_util.write_excel import merge_excels, merge_cover_comment
|
|
31
|
-
from ._run_compare import run_compare
|
|
32
29
|
from fosslight_util.cover import CoverItem
|
|
30
|
+
|
|
31
|
+
from .common import (
|
|
32
|
+
copy_file, call_analysis_api, overwrite_excel,
|
|
33
|
+
merge_yamls, correct_scanner_result, create_scancodejson
|
|
34
|
+
)
|
|
35
|
+
from ._run_compare import run_compare
|
|
36
|
+
|
|
33
37
|
fosslight_source_installed = True
|
|
34
38
|
try:
|
|
35
39
|
from fosslight_source.cli import run_scanners as source_analysis
|
|
@@ -45,7 +49,10 @@ _log_file = "fosslight_log_all_"
|
|
|
45
49
|
_start_time = ""
|
|
46
50
|
_executed_path = ""
|
|
47
51
|
SRC_DIR_FROM_LINK_PREFIX = "fosslight_src_dir_"
|
|
48
|
-
SCANNER_MODE = [
|
|
52
|
+
SCANNER_MODE = [
|
|
53
|
+
"all", "compare", "reuse", "prechecker", "binary",
|
|
54
|
+
"bin", "src", "source", "dependency", "dep"
|
|
55
|
+
]
|
|
49
56
|
|
|
50
57
|
|
|
51
58
|
def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
|
|
@@ -59,7 +66,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
59
66
|
github_token = ""
|
|
60
67
|
|
|
61
68
|
try:
|
|
62
|
-
if params
|
|
69
|
+
if params:
|
|
63
70
|
match_obj = re.findall(
|
|
64
71
|
r'\s*(-\s*[a|d|m|c|n|t])\s*\'([^\']+)\'\s*', params)
|
|
65
72
|
for param, value in match_obj:
|
|
@@ -83,22 +90,34 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
|
|
|
83
90
|
timer.start()
|
|
84
91
|
|
|
85
92
|
try:
|
|
86
|
-
success, result = call_analysis_api(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
93
|
+
success, result = call_analysis_api(
|
|
94
|
+
path_to_analyze, "Dependency Analysis",
|
|
95
|
+
1, run_dependency_scanner,
|
|
96
|
+
package_manager,
|
|
97
|
+
os.path.abspath(path_to_analyze),
|
|
98
|
+
output_file_with_path,
|
|
99
|
+
pip_activate_cmd, pip_deactivate_cmd,
|
|
100
|
+
output_custom_dir, app_name,
|
|
101
|
+
github_token, path_to_exclude=path_to_exclude
|
|
102
|
+
)
|
|
94
103
|
if success:
|
|
95
104
|
result_list = result.get('SRC_FL_Dependency')
|
|
96
105
|
except Exception as ex:
|
|
97
106
|
logger.warning(f"Run dependency: {ex}")
|
|
98
107
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
108
|
+
return result_list or []
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def source_analysis_wrapper(*args, **kwargs):
|
|
112
|
+
selected_scanner = kwargs.pop('selected_scanner', 'all')
|
|
113
|
+
source_write_json_file = kwargs.pop('source_write_json_file', False)
|
|
114
|
+
source_print_matched_text = kwargs.pop('source_print_matched_text', False)
|
|
115
|
+
source_time_out = kwargs.pop('source_time_out', 120)
|
|
116
|
+
args = list(args)
|
|
117
|
+
args.insert(2, source_write_json_file)
|
|
118
|
+
args.insert(5, source_print_matched_text)
|
|
119
|
+
|
|
120
|
+
return source_analysis(*args, selected_scanner=selected_scanner, time_out=source_time_out, **kwargs)
|
|
102
121
|
|
|
103
122
|
|
|
104
123
|
def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
@@ -106,7 +125,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
106
125
|
remove_src_data=True, result_log={}, output_file="",
|
|
107
126
|
output_extension="", num_cores=-1, db_url="",
|
|
108
127
|
default_oss_name="", default_oss_version="", url="",
|
|
109
|
-
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]
|
|
128
|
+
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
|
|
129
|
+
selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
|
|
130
|
+
source_time_out=120, binary_simple=False):
|
|
110
131
|
final_excel_dir = output_path
|
|
111
132
|
success = True
|
|
112
133
|
temp_output_fiiles = []
|
|
@@ -129,7 +150,6 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
129
150
|
if success:
|
|
130
151
|
output_files = {"SRC": f"fosslight_src_{_start_time}{output_extension}",
|
|
131
152
|
"BIN": f"fosslight_bin_{_start_time}{output_extension}",
|
|
132
|
-
"BIN_TXT": f"fosslight_binary_bin_{_start_time}.txt",
|
|
133
153
|
"DEP": f"fosslight_dep_{_start_time}{output_extension}",
|
|
134
154
|
"PRECHECKER": f"fosslight_lint_{_start_time}.yaml"}
|
|
135
155
|
if run_prechecker:
|
|
@@ -146,12 +166,21 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
146
166
|
try:
|
|
147
167
|
if fosslight_source_installed:
|
|
148
168
|
src_output = os.path.join(_output_dir, output_files["SRC"])
|
|
149
|
-
success, result = call_analysis_api(
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
169
|
+
success, result = call_analysis_api(
|
|
170
|
+
src_path,
|
|
171
|
+
"Source Analysis",
|
|
172
|
+
-1, source_analysis_wrapper,
|
|
173
|
+
abs_path,
|
|
174
|
+
src_output,
|
|
175
|
+
num_cores,
|
|
176
|
+
False,
|
|
177
|
+
path_to_exclude=path_to_exclude,
|
|
178
|
+
selected_scanner=selected_source_scanner,
|
|
179
|
+
source_write_json_file=source_write_json_file,
|
|
180
|
+
source_print_matched_text=source_print_matched_text,
|
|
181
|
+
source_time_out=source_time_out
|
|
182
|
+
)
|
|
183
|
+
|
|
155
184
|
else: # Run fosslight_source by using docker image
|
|
156
185
|
src_output = os.path.join("output", output_files["SRC"])
|
|
157
186
|
output_rel_path = os.path.relpath(abs_path, os.getcwd())
|
|
@@ -170,15 +199,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
170
199
|
1, binary_analysis.find_binaries,
|
|
171
200
|
abs_path,
|
|
172
201
|
os.path.join(_output_dir, output_files["BIN"]),
|
|
173
|
-
"", db_url,
|
|
202
|
+
"", db_url, binary_simple,
|
|
174
203
|
correct_mode, correct_fpath,
|
|
175
204
|
path_to_exclude=path_to_exclude)
|
|
176
|
-
if success:
|
|
177
|
-
output_binary_txt_raw = f"{output_files['BIN'].split('.')[0]}.txt"
|
|
178
|
-
success_file, copied_file = copy_file(os.path.join(_output_dir, output_binary_txt_raw),
|
|
179
|
-
os.path.join(output_path, output_files["BIN_TXT"]))
|
|
180
|
-
if success_file:
|
|
181
|
-
temp_output_fiiles.append(copied_file)
|
|
182
205
|
|
|
183
206
|
if run_dep:
|
|
184
207
|
run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments, path_to_exclude)
|
|
@@ -196,7 +219,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
|
|
|
196
219
|
cover = CoverItem(tool_name=PKG_NAME,
|
|
197
220
|
start_time=_start_time,
|
|
198
221
|
input_path=abs_path,
|
|
199
|
-
exclude_path=path_to_exclude
|
|
222
|
+
exclude_path=path_to_exclude,
|
|
223
|
+
simple_mode=False)
|
|
200
224
|
cover.comment = merge_cover_comment(_output_dir, merge_files)
|
|
201
225
|
|
|
202
226
|
if output_extension == ".xlsx":
|
|
@@ -314,7 +338,9 @@ def init(output_path="", make_outdir=True):
|
|
|
314
338
|
|
|
315
339
|
def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze,
|
|
316
340
|
db_url, hide_progressbar=False, keep_raw_data=False, num_cores=-1,
|
|
317
|
-
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]
|
|
341
|
+
correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
|
|
342
|
+
selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
|
|
343
|
+
source_time_out=120, binary_simple=False):
|
|
318
344
|
global _executed_path, _start_time
|
|
319
345
|
|
|
320
346
|
output_file = ""
|
|
@@ -322,6 +348,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
322
348
|
default_oss_version = ""
|
|
323
349
|
src_path = ""
|
|
324
350
|
_executed_path = os.getcwd()
|
|
351
|
+
extract_folder = ""
|
|
325
352
|
|
|
326
353
|
mode_not_supported = list(set(mode_list).difference(SCANNER_MODE))
|
|
327
354
|
if mode_not_supported:
|
|
@@ -344,6 +371,17 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
344
371
|
if isinstance(path_arg, list):
|
|
345
372
|
if len(path_arg) == 1:
|
|
346
373
|
src_path = path_arg[0]
|
|
374
|
+
|
|
375
|
+
for ext in compression_extension:
|
|
376
|
+
if src_path.endswith(ext):
|
|
377
|
+
temp_folder = f"temp_extract_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
|
378
|
+
Path(temp_folder).mkdir(parents=True, exist_ok=True)
|
|
379
|
+
|
|
380
|
+
extract_success = extract_file(src_path, temp_folder, False)
|
|
381
|
+
if extract_success:
|
|
382
|
+
src_path = os.path.join(_executed_path, temp_folder)
|
|
383
|
+
extract_folder = src_path
|
|
384
|
+
break
|
|
347
385
|
else:
|
|
348
386
|
logger.warning(f"Cannot analyze with multiple path: {path_arg}")
|
|
349
387
|
|
|
@@ -420,7 +458,12 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
|
|
|
420
458
|
remove_downloaded_source, {}, output_file,
|
|
421
459
|
output_extension, num_cores, db_url,
|
|
422
460
|
default_oss_name, default_oss_version, url_to_analyze,
|
|
423
|
-
correct_mode, correct_fpath, ui_mode, path_to_exclude
|
|
461
|
+
correct_mode, correct_fpath, ui_mode, path_to_exclude,
|
|
462
|
+
selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out,
|
|
463
|
+
binary_simple)
|
|
464
|
+
|
|
465
|
+
if extract_folder:
|
|
466
|
+
shutil.rmtree(extract_folder)
|
|
424
467
|
else:
|
|
425
468
|
logger.error("No mode has been selected for analysis.")
|
|
426
469
|
try:
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
|
-
# Copyright (c) 2022 LG Electronics Inc.
|
|
4
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
import sys
|
|
6
|
-
import json
|
|
7
|
-
from argparse import ArgumentParser
|
|
8
|
-
from ._help import print_help_msg
|
|
9
|
-
from .fosslight_scanner import run_main, PKG_NAME
|
|
10
|
-
from ._parse_setting import parse_setting_json
|
|
11
|
-
from fosslight_util.help import print_package_version
|
|
12
|
-
import os.path
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
|
|
16
|
-
raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
|
|
17
|
-
if setting and os.path.isfile(setting):
|
|
18
|
-
try:
|
|
19
|
-
with open(setting, 'r', encoding='utf-8') as file:
|
|
20
|
-
data = json.load(file)
|
|
21
|
-
s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
|
|
22
|
-
s_no_correction, s_correct_fpath, s_ui, s_exclude_path = parse_setting_json(data)
|
|
23
|
-
|
|
24
|
-
# direct cli arguments have higher priority than setting file
|
|
25
|
-
mode = mode if mode else s_mode
|
|
26
|
-
path = path if path else s_path
|
|
27
|
-
dep_argument = dep_argument if dep_argument else s_dep_argument
|
|
28
|
-
output = output if output else s_output
|
|
29
|
-
format = format if format else s_format
|
|
30
|
-
link = link if link else s_link
|
|
31
|
-
db_url = db_url if db_url else s_db_url
|
|
32
|
-
timer = timer if timer else s_timer
|
|
33
|
-
raw = raw if raw else s_raw
|
|
34
|
-
core = core if core else s_core
|
|
35
|
-
no_correction = no_correction if no_correction else s_no_correction
|
|
36
|
-
correct_fpath = correct_fpath if correct_fpath else s_correct_fpath
|
|
37
|
-
ui = ui if ui else s_ui
|
|
38
|
-
exclude_path = exclude_path if exclude_path else s_exclude_path
|
|
39
|
-
|
|
40
|
-
except Exception as e:
|
|
41
|
-
print(f"Cannot open setting file: {e}")
|
|
42
|
-
return mode, path, dep_argument, output, format, link, db_url, timer, \
|
|
43
|
-
raw, core, no_correction, correct_fpath, ui, exclude_path
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def main():
|
|
47
|
-
parser = ArgumentParser(description='FOSSLight Scanner', prog='fosslight_scanner', add_help=False)
|
|
48
|
-
parser.add_argument('mode', nargs='*', help='source| dependency| binary| all| compare', default="")
|
|
49
|
-
parser.add_argument('--path', '-p', help='Path to analyze (In compare mode, two FOSSLight reports',
|
|
50
|
-
dest='path', nargs='+', default="")
|
|
51
|
-
parser.add_argument('--wget', '-w', help='Link to be analyzed', type=str, dest='link', default="")
|
|
52
|
-
parser.add_argument('--format', '-f', help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
|
|
53
|
-
type=str, dest='format', default="")
|
|
54
|
-
parser.add_argument('--output', '-o', help='Output directory or file', type=str, dest='output', default="")
|
|
55
|
-
parser.add_argument('--dependency', '-d', help='Dependency arguments', type=str, dest='dep_argument', default="")
|
|
56
|
-
parser.add_argument('--url', '-u', help="DB Url", type=str, dest='db_url', default="")
|
|
57
|
-
parser.add_argument('--core', '-c', help='Number of processes to analyze source', type=int, dest='core', default=-1)
|
|
58
|
-
parser.add_argument('--raw', '-r', help='Keep raw data', action='store_true', dest='raw', default=False)
|
|
59
|
-
parser.add_argument('--timer', '-t', help='Hide the progress bar', action='store_true', dest='timer', default=False)
|
|
60
|
-
parser.add_argument('--version', '-v', help='Print version', action='store_true', dest='version', default=False)
|
|
61
|
-
parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help')
|
|
62
|
-
parser.add_argument('--exclude', '-e', help='Path to exclude from analysis', dest='exclude_path', nargs='*', default=[])
|
|
63
|
-
parser.add_argument('--setting', '-s', help='Scanner json setting file', type=str, dest='setting', default="")
|
|
64
|
-
parser.add_argument('--no_correction', help='No correction with sbom-info.yaml',
|
|
65
|
-
action='store_true', required=False, default=False)
|
|
66
|
-
parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
|
|
67
|
-
type=str, required=False, default='')
|
|
68
|
-
parser.add_argument('--ui', help='Generate UI mode result file', action='store_true', required=False, default=False)
|
|
69
|
-
|
|
70
|
-
try:
|
|
71
|
-
args = parser.parse_args()
|
|
72
|
-
except SystemExit:
|
|
73
|
-
sys.exit(1)
|
|
74
|
-
|
|
75
|
-
if args.help:
|
|
76
|
-
print_help_msg()
|
|
77
|
-
elif args.version:
|
|
78
|
-
print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
|
|
79
|
-
else:
|
|
80
|
-
mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
|
|
81
|
-
ui, exclude_path = set_args(args.mode, args.path, args.dep_argument, args.output, args.format,
|
|
82
|
-
args.link, args.db_url, args.timer, args.raw, args.core, args.no_correction,
|
|
83
|
-
args.correct_fpath, args.ui, args.setting, args.exclude_path)
|
|
84
|
-
|
|
85
|
-
run_main(mode, path, dep_argument, output, format, link, db_url, timer,
|
|
86
|
-
raw, core, not no_correction, correct_fpath, ui, exclude_path)
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
if __name__ == "__main__":
|
|
90
|
-
main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_scanner-1.7.29 → fosslight_scanner-1.7.31}/src/fosslight_scanner.egg-info/top_level.txt
RENAMED
|
File without changes
|