fosslight-util 1.4.47__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/PKG-INFO +1 -1
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/requirements.txt +0 -1
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/setup.py +1 -1
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/compare_yaml.py +18 -11
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/constant.py +11 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/convert_excel_to_yaml.py +1 -1
- fosslight_util-2.0.0/src/fosslight_util/correct.py +89 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/cover.py +19 -6
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/help.py +3 -4
- fosslight_util-2.0.0/src/fosslight_util/oss_item.py +219 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/output_format.py +7 -5
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/parsing_yaml.py +45 -30
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/read_excel.py +29 -35
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/set_log.py +20 -3
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/spdx_licenses.py +2 -1
- fosslight_util-2.0.0/src/fosslight_util/write_excel.py +245 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/write_opossum.py +14 -20
- fosslight_util-2.0.0/src/fosslight_util/write_scancodejson.py +64 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/write_spdx.py +30 -35
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/write_txt.py +2 -1
- fosslight_util-2.0.0/src/fosslight_util/write_yaml.py +91 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util.egg-info/PKG-INFO +1 -1
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util.egg-info/requires.txt +0 -1
- fosslight_util-1.4.47/src/fosslight_util/correct.py +0 -133
- fosslight_util-1.4.47/src/fosslight_util/oss_item.py +0 -226
- fosslight_util-1.4.47/src/fosslight_util/write_excel.py +0 -313
- fosslight_util-1.4.47/src/fosslight_util/write_scancodejson.py +0 -64
- fosslight_util-1.4.47/src/fosslight_util/write_yaml.py +0 -102
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/LICENSE +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/MANIFEST.in +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/README.md +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/setup.cfg +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/__init__.py +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/_get_downloadable_url.py +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/download.py +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/resources/frequentLicenselist.json +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/resources/frequent_license_nick_list.json +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/resources/licenses.json +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util/timer_thread.py +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util.egg-info/SOURCES.txt +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util.egg-info/dependency_links.txt +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util.egg-info/entry_points.txt +0 -0
- {fosslight_util-1.4.47 → fosslight_util-2.0.0}/src/fosslight_util.egg-info/top_level.txt +0 -0
|
@@ -4,9 +4,7 @@
|
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
|
|
6
6
|
import logging
|
|
7
|
-
import os
|
|
8
7
|
from fosslight_util.constant import LOGGER_NAME
|
|
9
|
-
from fosslight_util.parsing_yaml import parsing_yml
|
|
10
8
|
|
|
11
9
|
logger = logging.getLogger(LOGGER_NAME)
|
|
12
10
|
VERSION = 'version'
|
|
@@ -14,12 +12,16 @@ LICENSE = 'license'
|
|
|
14
12
|
NAME = 'name'
|
|
15
13
|
|
|
16
14
|
|
|
17
|
-
def compare_yaml(
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
def compare_yaml(before_fileitems, after_fileitems):
|
|
16
|
+
bf_raw = []
|
|
17
|
+
af_raw = []
|
|
18
|
+
for bf in before_fileitems:
|
|
19
|
+
bf_raw.extend(bf.get_print_json())
|
|
20
|
+
for af in after_fileitems:
|
|
21
|
+
af_raw.extend(af.get_print_json())
|
|
20
22
|
|
|
21
|
-
before_items = get_merged_item(
|
|
22
|
-
after_items = get_merged_item(
|
|
23
|
+
before_items = get_merged_item(bf_raw)
|
|
24
|
+
after_items = get_merged_item(af_raw)
|
|
23
25
|
|
|
24
26
|
new_before = []
|
|
25
27
|
for bi in before_items:
|
|
@@ -72,13 +74,18 @@ def compare_yaml(before_file, after_file):
|
|
|
72
74
|
def get_merged_item(oss_items):
|
|
73
75
|
item_list = []
|
|
74
76
|
for oi in oss_items:
|
|
75
|
-
if oi.exclude:
|
|
77
|
+
if oi.get("exclude", None):
|
|
76
78
|
continue
|
|
77
|
-
|
|
79
|
+
oi_name = oi.get("name", '')
|
|
80
|
+
oi_version = oi.get("version", '')
|
|
81
|
+
oi_license = oi.get("license", '')
|
|
82
|
+
if not (oi_name and oi_version and oi_license):
|
|
83
|
+
continue
|
|
84
|
+
item_info = {NAME: oi_name, VERSION: oi_version, LICENSE: oi_license}
|
|
78
85
|
|
|
79
|
-
filtered = next(filter(lambda oss_dict: oss_dict[NAME] ==
|
|
86
|
+
filtered = next(filter(lambda oss_dict: oss_dict[NAME] == oi_name and oss_dict[VERSION] == oi_version, item_list), None)
|
|
80
87
|
if filtered:
|
|
81
|
-
filtered[LICENSE].extend(
|
|
88
|
+
filtered[LICENSE].extend(oi_license)
|
|
82
89
|
filtered[LICENSE] = list(set(filtered[LICENSE]))
|
|
83
90
|
else:
|
|
84
91
|
item_list.append(item_info)
|
|
@@ -15,6 +15,17 @@ supported_sheet_and_scanner = {'SRC': FL_SOURCE,
|
|
|
15
15
|
f'BIN_{FL_BINARY}': FL_BINARY,
|
|
16
16
|
f'DEP_{FL_DEPENDENCY}': FL_DEPENDENCY}
|
|
17
17
|
|
|
18
|
+
FOSSLIGHT_SCANNER = 'fosslight_scanner'
|
|
19
|
+
FOSSLIGHT_SOURCE = 'fosslight_source'
|
|
20
|
+
FOSSLIGHT_DEPENDENCY = 'fosslight_dependency'
|
|
21
|
+
FOSSLIGHT_BINARY = 'fosslight_binary'
|
|
22
|
+
|
|
23
|
+
SHEET_NAME_FOR_SCANNER = {
|
|
24
|
+
FOSSLIGHT_SOURCE: 'SRC_FL_Source',
|
|
25
|
+
FOSSLIGHT_BINARY: 'BIN_FL_Binary',
|
|
26
|
+
FOSSLIGHT_DEPENDENCY: 'DEP_FL_Dependency'
|
|
27
|
+
}
|
|
28
|
+
|
|
18
29
|
# Github : https://github.com/(owner)/(repo)
|
|
19
30
|
# npm : https://www.npmjs.com/package/(package)/v/(version)
|
|
20
31
|
# npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
|
|
@@ -33,7 +33,7 @@ def find_report_file(path_to_find):
|
|
|
33
33
|
return ""
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
def convert_excel_to_yaml(oss_report_to_read, output_file, sheet_names=""):
|
|
36
|
+
def convert_excel_to_yaml(oss_report_to_read: str, output_file: str, sheet_names: str = "") -> None:
|
|
37
37
|
_file_extension = ".yaml"
|
|
38
38
|
yaml_dict = {}
|
|
39
39
|
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2023 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import copy
|
|
9
|
+
import re
|
|
10
|
+
from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE
|
|
11
|
+
from fosslight_util.parsing_yaml import parsing_yml
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
14
|
+
SBOM_INFO_YAML = r"sbom(-|_)info[\s\S]*.ya?ml"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def correct_with_yaml(correct_filepath, path_to_scan, scan_item):
|
|
18
|
+
success = True
|
|
19
|
+
msg = ""
|
|
20
|
+
correct_yaml = ""
|
|
21
|
+
if correct_filepath == "":
|
|
22
|
+
correct_filepath = path_to_scan
|
|
23
|
+
|
|
24
|
+
path_to_scan = os.path.normpath(path_to_scan)
|
|
25
|
+
correct_filepath = os.path.normpath(correct_filepath)
|
|
26
|
+
for filename in os.listdir(correct_filepath):
|
|
27
|
+
if re.search(SBOM_INFO_YAML, filename, re.IGNORECASE):
|
|
28
|
+
correct_yaml = os.path.join(correct_filepath, filename)
|
|
29
|
+
break
|
|
30
|
+
if not correct_yaml:
|
|
31
|
+
msg = f"Cannot find sbom-info.yaml in {correct_filepath}."
|
|
32
|
+
success = False
|
|
33
|
+
return success, msg, scan_item
|
|
34
|
+
|
|
35
|
+
rel_path = os.path.relpath(path_to_scan, correct_filepath)
|
|
36
|
+
|
|
37
|
+
yaml_file_list, _, err_msg = parsing_yml(correct_yaml, os.path.dirname(correct_yaml), print_log=True)
|
|
38
|
+
find_match = False
|
|
39
|
+
for scanner_name, _ in scan_item.file_items.items():
|
|
40
|
+
correct_fileitems = []
|
|
41
|
+
exclude_fileitems = []
|
|
42
|
+
for yaml_file_item in yaml_file_list:
|
|
43
|
+
yaml_path_exists = False
|
|
44
|
+
if yaml_file_item.source_name_or_path == '':
|
|
45
|
+
if scanner_name == FOSSLIGHT_SOURCE:
|
|
46
|
+
correct_item = copy.deepcopy(yaml_file_item)
|
|
47
|
+
correct_item.comment = 'Added by sbom-info.yaml'
|
|
48
|
+
correct_fileitems.append(correct_item)
|
|
49
|
+
continue
|
|
50
|
+
for idx, scan_file_item in enumerate(scan_item.file_items[scanner_name]):
|
|
51
|
+
oss_rel_path = os.path.normpath(os.path.join(rel_path, scan_file_item.source_name_or_path))
|
|
52
|
+
yi_path = yaml_file_item.source_name_or_path
|
|
53
|
+
if ((os.path.normpath(yi_path) == os.path.normpath(oss_rel_path)) or
|
|
54
|
+
((os.path.normpath(oss_rel_path).startswith(os.path.normpath(yi_path.rstrip('*')))))):
|
|
55
|
+
correct_item = copy.deepcopy(scan_file_item)
|
|
56
|
+
correct_item.exclude = yaml_file_item.exclude
|
|
57
|
+
correct_item.oss_items = copy.deepcopy(yaml_file_item.oss_items)
|
|
58
|
+
correct_item.comment = ''
|
|
59
|
+
correct_item.comment = 'Loaded from sbom-info.yaml'
|
|
60
|
+
correct_fileitems.append(correct_item)
|
|
61
|
+
|
|
62
|
+
yaml_path_exists = True
|
|
63
|
+
exclude_fileitems.append(idx)
|
|
64
|
+
|
|
65
|
+
if not yaml_path_exists:
|
|
66
|
+
correct_item = copy.deepcopy(yaml_file_item)
|
|
67
|
+
if os.path.exists(os.path.normpath(yaml_file_item.source_name_or_path)):
|
|
68
|
+
correct_item.comment = 'Loaded from sbom-info.yaml'
|
|
69
|
+
correct_fileitems.append(correct_item)
|
|
70
|
+
else:
|
|
71
|
+
if scanner_name == FOSSLIGHT_SOURCE:
|
|
72
|
+
correct_item.exclude = True
|
|
73
|
+
correct_item.comment = 'Added by sbom-info.yaml'
|
|
74
|
+
correct_fileitems.append(correct_item)
|
|
75
|
+
if correct_fileitems:
|
|
76
|
+
scan_item.append_file_items(correct_fileitems, scanner_name)
|
|
77
|
+
find_match = True
|
|
78
|
+
if exclude_fileitems:
|
|
79
|
+
exclude_fileitems = list(set(exclude_fileitems))
|
|
80
|
+
for e_idx in exclude_fileitems:
|
|
81
|
+
scan_item.file_items[scanner_name][e_idx].exclude = True
|
|
82
|
+
scan_item.file_items[scanner_name][e_idx].comment = 'Excluded by sbom-info.yaml'
|
|
83
|
+
|
|
84
|
+
if not find_match:
|
|
85
|
+
success = False
|
|
86
|
+
err_msg = 'No match items in sbom-info.yaml'
|
|
87
|
+
return success, err_msg, scan_item
|
|
88
|
+
|
|
89
|
+
return success, msg, scan_item
|
|
@@ -9,16 +9,31 @@ from fosslight_util.help import print_package_version
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class CoverItem:
|
|
12
|
-
tool_name_key = "Tool
|
|
13
|
-
tool_version_key = "Tool version"
|
|
12
|
+
tool_name_key = "Tool information"
|
|
14
13
|
start_time_key = "Start time"
|
|
15
14
|
python_ver_key = "Python version"
|
|
16
15
|
analyzed_path_key = "Analyzed path"
|
|
17
16
|
excluded_path_key = "Excluded path"
|
|
18
17
|
comment_key = "Comment"
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
PKG_NAMES = [
|
|
20
|
+
"fosslight_scanner",
|
|
21
|
+
"fosslight_source",
|
|
22
|
+
"fosslight_dependency",
|
|
23
|
+
"fosslight_binary"
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
def __init__(self, tool_name="", start_time="", input_path="", comment="", exclude_path=[], simple_mode=True):
|
|
27
|
+
if simple_mode:
|
|
28
|
+
self.tool_name = f'{tool_name} v{print_package_version(tool_name, "", False)}'
|
|
29
|
+
else:
|
|
30
|
+
first_pkg = f'{self.PKG_NAMES[0]} v{print_package_version(self.PKG_NAMES[0], "", False)}'
|
|
31
|
+
remaining_pkgs = ", ".join([
|
|
32
|
+
f'{pkg_name} v{print_package_version(pkg_name, "", False)}'
|
|
33
|
+
for pkg_name in self.PKG_NAMES[1:]
|
|
34
|
+
])
|
|
35
|
+
self.tool_name = f'{first_pkg} ({remaining_pkgs})'
|
|
36
|
+
|
|
22
37
|
if start_time:
|
|
23
38
|
date, time = start_time.split('_')
|
|
24
39
|
self.start_time = f'{date}, {time[0:2]}:{time[2:4]}'
|
|
@@ -28,7 +43,6 @@ class CoverItem:
|
|
|
28
43
|
self.exclude_path = ", ".join(exclude_path)
|
|
29
44
|
self.comment = comment
|
|
30
45
|
|
|
31
|
-
self.tool_version = print_package_version(self.tool_name, "", False)
|
|
32
46
|
self.python_version = f'{sys.version_info.major}.{sys.version_info.minor}'
|
|
33
47
|
|
|
34
48
|
def __del__(self):
|
|
@@ -37,7 +51,6 @@ class CoverItem:
|
|
|
37
51
|
def get_print_json(self):
|
|
38
52
|
json_item = {}
|
|
39
53
|
json_item[self.tool_name_key] = self.tool_name
|
|
40
|
-
json_item[self.tool_version_key] = self.tool_version
|
|
41
54
|
json_item[self.start_time_key] = self.start_time
|
|
42
55
|
json_item[self.python_ver_key] = self.python_version
|
|
43
56
|
json_item[self.analyzed_path_key] = self.input_path
|
|
@@ -35,12 +35,11 @@ _HELP_MESSAGE_DOWNLOAD = """
|
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
class PrintHelpMsg():
|
|
38
|
-
message_suffix = ""
|
|
39
38
|
|
|
40
|
-
def __init__(self, value):
|
|
39
|
+
def __init__(self, value: str = ""):
|
|
41
40
|
self.message_suffix = value
|
|
42
41
|
|
|
43
|
-
def print_help_msg(self, exitopt):
|
|
42
|
+
def print_help_msg(self, exitopt: bool) -> None:
|
|
44
43
|
print(_HELP_MESSAGE_COMMON)
|
|
45
44
|
print(self.message_suffix)
|
|
46
45
|
|
|
@@ -48,7 +47,7 @@ class PrintHelpMsg():
|
|
|
48
47
|
sys.exit()
|
|
49
48
|
|
|
50
49
|
|
|
51
|
-
def print_package_version(pkg_name, msg="", exitopt=True):
|
|
50
|
+
def print_package_version(pkg_name: str, msg: str = "", exitopt: bool = True) -> str:
|
|
52
51
|
if msg == "":
|
|
53
52
|
msg = f"{pkg_name} Version:"
|
|
54
53
|
cur_version = pkg_resources.get_distribution(pkg_name).version
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2021 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SCANNER
|
|
9
|
+
from fosslight_util.cover import CoverItem
|
|
10
|
+
from typing import List, Dict
|
|
11
|
+
|
|
12
|
+
_logger = logging.getLogger(LOGGER_NAME)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class OssItem:
|
|
16
|
+
|
|
17
|
+
def __init__(self, name="", version="", license="", dl_url=""):
|
|
18
|
+
self.name = name
|
|
19
|
+
self.version = version
|
|
20
|
+
self._license = []
|
|
21
|
+
self.license = license
|
|
22
|
+
self.download_location = dl_url
|
|
23
|
+
self.exclude = False
|
|
24
|
+
self.comment = ""
|
|
25
|
+
self.homepage = ""
|
|
26
|
+
self._copyright = ""
|
|
27
|
+
|
|
28
|
+
def __del__(self):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def license(self):
|
|
33
|
+
return self._license
|
|
34
|
+
|
|
35
|
+
@license.setter
|
|
36
|
+
def license(self, value):
|
|
37
|
+
if value != "":
|
|
38
|
+
if not isinstance(value, list):
|
|
39
|
+
value = value.split(",")
|
|
40
|
+
self._license.extend(value)
|
|
41
|
+
self._license = [item.strip() for item in self._license]
|
|
42
|
+
self._license = list(set(self._license))
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def exclude(self):
|
|
46
|
+
return self._exclude
|
|
47
|
+
|
|
48
|
+
@exclude.setter
|
|
49
|
+
def exclude(self, value):
|
|
50
|
+
if value:
|
|
51
|
+
self._exclude = True
|
|
52
|
+
else:
|
|
53
|
+
self._exclude = False
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def copyright(self):
|
|
57
|
+
return self._copyright
|
|
58
|
+
|
|
59
|
+
@copyright.setter
|
|
60
|
+
def copyright(self, value):
|
|
61
|
+
if value != "":
|
|
62
|
+
if isinstance(value, list):
|
|
63
|
+
value = "\n".join(value)
|
|
64
|
+
value = value.strip()
|
|
65
|
+
self._copyright = value
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def version(self):
|
|
69
|
+
return self._version
|
|
70
|
+
|
|
71
|
+
@version.setter
|
|
72
|
+
def version(self, value):
|
|
73
|
+
if value:
|
|
74
|
+
self._version = str(value)
|
|
75
|
+
else:
|
|
76
|
+
self._version = ""
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def comment(self):
|
|
80
|
+
return self._comment
|
|
81
|
+
|
|
82
|
+
@comment.setter
|
|
83
|
+
def comment(self, value):
|
|
84
|
+
if not value:
|
|
85
|
+
self._comment = ""
|
|
86
|
+
else:
|
|
87
|
+
if self._comment:
|
|
88
|
+
self._comment = f"{self._comment} / {value}"
|
|
89
|
+
else:
|
|
90
|
+
self._comment = value
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class FileItem:
|
|
94
|
+
def __init__(self, value):
|
|
95
|
+
self.relative_path = value
|
|
96
|
+
self.source_name_or_path = ""
|
|
97
|
+
self._exclude = False
|
|
98
|
+
self._comment = ""
|
|
99
|
+
self.is_binary = False
|
|
100
|
+
self.oss_items: List[OssItem] = []
|
|
101
|
+
|
|
102
|
+
def __del__(self):
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def exclude(self):
|
|
107
|
+
return self._exclude
|
|
108
|
+
|
|
109
|
+
@exclude.setter
|
|
110
|
+
def exclude(self, value):
|
|
111
|
+
if value:
|
|
112
|
+
self._exclude = True
|
|
113
|
+
else:
|
|
114
|
+
self._exclude = False
|
|
115
|
+
for oss in self.oss_items:
|
|
116
|
+
oss.exclude = value
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def comment(self):
|
|
120
|
+
return self._comment
|
|
121
|
+
|
|
122
|
+
@comment.setter
|
|
123
|
+
def comment(self, value):
|
|
124
|
+
if not value:
|
|
125
|
+
self._comment = ""
|
|
126
|
+
else:
|
|
127
|
+
if self._comment:
|
|
128
|
+
self._comment = f"{self._comment} / {value}"
|
|
129
|
+
else:
|
|
130
|
+
self._comment = value
|
|
131
|
+
for oss in self.oss_items:
|
|
132
|
+
oss.comment = value
|
|
133
|
+
|
|
134
|
+
def get_print_array(self):
|
|
135
|
+
items = []
|
|
136
|
+
|
|
137
|
+
for oss in self.oss_items:
|
|
138
|
+
exclude = "Exclude" if self.exclude or oss.exclude else ""
|
|
139
|
+
lic = ",".join(oss.license)
|
|
140
|
+
|
|
141
|
+
oss_item = [os.path.join(self.relative_path, self.source_name_or_path), oss.name, oss.version, lic,
|
|
142
|
+
oss.download_location, oss.homepage, oss.copyright, exclude, oss.comment]
|
|
143
|
+
items.append(oss_item)
|
|
144
|
+
return items
|
|
145
|
+
|
|
146
|
+
def get_print_json(self):
|
|
147
|
+
items = []
|
|
148
|
+
|
|
149
|
+
for oss in self.oss_items:
|
|
150
|
+
json_item = {}
|
|
151
|
+
json_item["name"] = oss.name
|
|
152
|
+
json_item["version"] = oss.version
|
|
153
|
+
|
|
154
|
+
if self.source_name_or_path != "":
|
|
155
|
+
json_item["source path"] = self.source_name_or_path
|
|
156
|
+
if len(oss.license) > 0:
|
|
157
|
+
json_item["license"] = oss.license
|
|
158
|
+
if oss.download_location != "":
|
|
159
|
+
json_item["download location"] = oss.download_location
|
|
160
|
+
if oss.homepage != "":
|
|
161
|
+
json_item["homepage"] = oss.homepage
|
|
162
|
+
if oss.copyright != "":
|
|
163
|
+
json_item["copyright text"] = oss.copyright
|
|
164
|
+
if self.exclude or oss.exclude:
|
|
165
|
+
json_item["exclude"] = True
|
|
166
|
+
if oss.comment != "":
|
|
167
|
+
json_item["comment"] = oss.comment
|
|
168
|
+
items.append(json_item)
|
|
169
|
+
return items
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def invalid(cmd):
|
|
173
|
+
_logger.info('[{}] is invalid'.format(cmd))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class ScannerItem:
|
|
177
|
+
def __init__(self, pkg_name, start_time=""):
|
|
178
|
+
self.cover = CoverItem(tool_name=pkg_name, start_time=start_time)
|
|
179
|
+
self.file_items: Dict[str, List[FileItem]] = {pkg_name: []} if pkg_name != FOSSLIGHT_SCANNER else {}
|
|
180
|
+
self.external_sheets: Dict[str, List[List[str]]] = {}
|
|
181
|
+
|
|
182
|
+
def set_cover_pathinfo(self, input_dir, path_to_exclude):
|
|
183
|
+
self.cover.input_path = input_dir
|
|
184
|
+
self.cover.exclude_path = ", ".join(path_to_exclude)
|
|
185
|
+
|
|
186
|
+
def set_cover_comment(self, value):
|
|
187
|
+
if value:
|
|
188
|
+
if self.cover.comment:
|
|
189
|
+
self.cover.comment = f"{self.cover.comment} / {value}"
|
|
190
|
+
else:
|
|
191
|
+
self.cover.comment = value
|
|
192
|
+
|
|
193
|
+
def get_cover_comment(self):
|
|
194
|
+
return [item.strip() for item in self.cover.comment.split(" / ")]
|
|
195
|
+
|
|
196
|
+
def append_file_items(self, file_item: List[FileItem], pkg_name=""):
|
|
197
|
+
if pkg_name == "":
|
|
198
|
+
if len(self.file_items.keys()) != 1:
|
|
199
|
+
_logger.error("Package name is not set. Cannot append file_item into ScannerItem.")
|
|
200
|
+
else:
|
|
201
|
+
pkg_name = list(self.file_items.keys())[0]
|
|
202
|
+
if pkg_name not in self.file_items:
|
|
203
|
+
self.file_items[pkg_name] = []
|
|
204
|
+
self.file_items[pkg_name].extend(file_item)
|
|
205
|
+
|
|
206
|
+
def get_print_array(self, scanner_name):
|
|
207
|
+
items = []
|
|
208
|
+
for file_item in self.file_items[scanner_name]:
|
|
209
|
+
items.extend(file_item.get_print_array())
|
|
210
|
+
return items
|
|
211
|
+
|
|
212
|
+
def get_print_json(self, scanner_name):
|
|
213
|
+
items = []
|
|
214
|
+
for file_item in self.file_items[scanner_name]:
|
|
215
|
+
items.extend(file_item.get_print_json())
|
|
216
|
+
return items
|
|
217
|
+
|
|
218
|
+
def __del__(self):
|
|
219
|
+
pass
|
|
@@ -6,6 +6,7 @@ import os
|
|
|
6
6
|
from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv
|
|
7
7
|
from fosslight_util.write_opossum import write_opossum
|
|
8
8
|
from fosslight_util.write_yaml import write_yaml
|
|
9
|
+
from typing import Tuple
|
|
9
10
|
|
|
10
11
|
SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'}
|
|
11
12
|
|
|
@@ -105,7 +106,8 @@ def check_output_formats(output='', formats=[], customized_format={}):
|
|
|
105
106
|
return success, msg, output_path, output_files, output_extensions
|
|
106
107
|
|
|
107
108
|
|
|
108
|
-
def write_output_file(output_file_without_ext, file_extension,
|
|
109
|
+
def write_output_file(output_file_without_ext: str, file_extension: str, scan_item, extended_header: dict = {},
|
|
110
|
+
hide_header: dict = {}) -> Tuple[bool, str, str]:
|
|
109
111
|
success = True
|
|
110
112
|
msg = ''
|
|
111
113
|
|
|
@@ -114,13 +116,13 @@ def write_output_file(output_file_without_ext, file_extension, sheet_list, exten
|
|
|
114
116
|
result_file = output_file_without_ext + file_extension
|
|
115
117
|
|
|
116
118
|
if file_extension == '.xlsx':
|
|
117
|
-
success, msg = write_result_to_excel(result_file,
|
|
119
|
+
success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
|
|
118
120
|
elif file_extension == '.csv':
|
|
119
|
-
success, msg, result_file = write_result_to_csv(result_file,
|
|
121
|
+
success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
|
|
120
122
|
elif file_extension == '.json':
|
|
121
|
-
success, msg = write_opossum(result_file,
|
|
123
|
+
success, msg = write_opossum(result_file, scan_item)
|
|
122
124
|
elif file_extension == '.yaml':
|
|
123
|
-
success, msg, result_file = write_yaml(result_file,
|
|
125
|
+
success, msg, result_file = write_yaml(result_file, scan_item, False)
|
|
124
126
|
else:
|
|
125
127
|
success = False
|
|
126
128
|
msg = f'Not supported file extension({file_extension})'
|
|
@@ -8,8 +8,8 @@ import codecs
|
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
10
|
import sys
|
|
11
|
-
from .constant import LOGGER_NAME
|
|
12
|
-
from .oss_item import OssItem
|
|
11
|
+
from fosslight_util.constant import LOGGER_NAME
|
|
12
|
+
from fosslight_util.oss_item import OssItem, FileItem
|
|
13
13
|
|
|
14
14
|
_logger = logging.getLogger(LOGGER_NAME)
|
|
15
15
|
SUPPORT_OSS_INFO_FILES = [r"oss-pkg-info[\s\S]*.ya?ml", r"sbom(-|_)info[\s\S]*.ya?ml"]
|
|
@@ -17,7 +17,7 @@ EXAMPLE_OSS_PKG_INFO_LINK = "https://github.com/fosslight/fosslight_prechecker/b
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def parsing_yml(yaml_file, base_path, print_log=True):
|
|
20
|
-
|
|
20
|
+
fileitems = []
|
|
21
21
|
license_list = []
|
|
22
22
|
idx = 1
|
|
23
23
|
err_reason = ""
|
|
@@ -38,37 +38,65 @@ def parsing_yml(yaml_file, base_path, print_log=True):
|
|
|
38
38
|
err_reason = "empty"
|
|
39
39
|
if print_log:
|
|
40
40
|
_logger.warning(f"The yaml file is empty file: {yaml_file}")
|
|
41
|
-
return
|
|
41
|
+
return fileitems, license_list, err_reason
|
|
42
42
|
|
|
43
43
|
is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT)
|
|
44
44
|
|
|
45
|
+
filepath_list = []
|
|
45
46
|
for root_element in doc:
|
|
46
47
|
oss_items = doc[root_element]
|
|
47
48
|
if oss_items:
|
|
48
49
|
if not isinstance(oss_items, list) or 'version' not in oss_items[0]:
|
|
49
50
|
raise AttributeError(f"- Ref. {EXAMPLE_OSS_PKG_INFO_LINK}")
|
|
50
51
|
for oss in oss_items:
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
52
|
+
source_paths = get_source_name_or_path_in_yaml(oss)
|
|
53
|
+
for source_path in source_paths:
|
|
54
|
+
if os.path.join(relative_path, source_path) not in filepath_list:
|
|
55
|
+
filepath_list.append(os.path.join(relative_path, source_path))
|
|
56
|
+
fileitem = FileItem(relative_path)
|
|
57
|
+
fileitem.source_name_or_path = source_path
|
|
58
|
+
fileitems.append(fileitem)
|
|
59
|
+
else:
|
|
60
|
+
fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
|
|
61
|
+
ossitem = OssItem()
|
|
62
|
+
if not is_old_format:
|
|
63
|
+
ossitem.name = root_element
|
|
64
|
+
for key, value in oss.items():
|
|
65
|
+
if key:
|
|
66
|
+
key = key.lower().strip()
|
|
67
|
+
set_value_switch(ossitem, key, value, yaml_file)
|
|
68
|
+
fileitem.oss_items.append(ossitem)
|
|
69
|
+
license_list.extend(ossitem.license)
|
|
70
|
+
idx += 1
|
|
61
71
|
except AttributeError as ex:
|
|
62
72
|
if print_log:
|
|
63
73
|
_logger.warning(f"Not supported yaml file format: {yaml_file} {ex}")
|
|
64
|
-
|
|
74
|
+
fileitems = []
|
|
65
75
|
err_reason = "not_supported"
|
|
66
76
|
except yaml.YAMLError:
|
|
67
77
|
if print_log:
|
|
68
78
|
_logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}")
|
|
69
|
-
|
|
79
|
+
fileitems = []
|
|
70
80
|
err_reason = "yaml_error"
|
|
71
|
-
|
|
81
|
+
|
|
82
|
+
return fileitems, set(license_list), err_reason
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def get_source_name_or_path_in_yaml(oss):
|
|
86
|
+
source_name_or_path = []
|
|
87
|
+
find = False
|
|
88
|
+
for key in oss.keys():
|
|
89
|
+
if key in ['file name or path', 'source name or path', 'source path',
|
|
90
|
+
'file', 'binary name', 'binary path']:
|
|
91
|
+
if isinstance(oss[key], list):
|
|
92
|
+
source_name_or_path = oss[key]
|
|
93
|
+
else:
|
|
94
|
+
source_name_or_path.append(oss[key])
|
|
95
|
+
find = True
|
|
96
|
+
break
|
|
97
|
+
if not find:
|
|
98
|
+
source_name_or_path.append('')
|
|
99
|
+
return source_name_or_path
|
|
72
100
|
|
|
73
101
|
|
|
74
102
|
def find_sbom_yaml_files(path_to_find):
|
|
@@ -101,9 +129,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
|
|
|
101
129
|
oss.download_location = value
|
|
102
130
|
elif key in ['license', 'license text']:
|
|
103
131
|
oss.license = value
|
|
104
|
-
elif key in ['file name or path', 'source name or path', 'source path',
|
|
105
|
-
'file', 'binary name', 'binary path']:
|
|
106
|
-
oss.source_name_or_path = value
|
|
107
132
|
elif key in ['copyright text', 'copyright']:
|
|
108
133
|
oss.copyright = value
|
|
109
134
|
elif key == 'exclude':
|
|
@@ -112,16 +137,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
|
|
|
112
137
|
oss.comment = value
|
|
113
138
|
elif key == 'homepage':
|
|
114
139
|
oss.homepage = value
|
|
115
|
-
elif key == 'yocto_package':
|
|
116
|
-
oss.yocto_package = value
|
|
117
|
-
elif key == 'yocto_recipe':
|
|
118
|
-
oss.yocto_recipe = value
|
|
119
|
-
elif key == 'vulnerability link':
|
|
120
|
-
oss.bin_vulnerability = value
|
|
121
|
-
elif key == 'tlsh':
|
|
122
|
-
oss.bin_tlsh = value
|
|
123
|
-
elif key == 'sha1':
|
|
124
|
-
oss.bin_sha1 = value
|
|
125
140
|
else:
|
|
126
141
|
if yaml_file != "":
|
|
127
142
|
_logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")
|