fosslight-util 1.4.34__py3-none-any.whl → 2.1.28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fosslight_util/_get_downloadable_url.py +466 -36
- fosslight_util/compare_yaml.py +20 -11
- fosslight_util/constant.py +35 -0
- fosslight_util/correct.py +46 -78
- fosslight_util/cover.py +60 -0
- fosslight_util/download.py +302 -95
- fosslight_util/exclude.py +65 -0
- fosslight_util/help.py +20 -8
- fosslight_util/oss_item.py +171 -110
- fosslight_util/output_format.py +147 -19
- fosslight_util/parsing_yaml.py +45 -23
- fosslight_util/read_excel.py +40 -39
- fosslight_util/set_log.py +30 -5
- fosslight_util/spdx_licenses.py +2 -1
- fosslight_util/write_cyclonedx.py +210 -0
- fosslight_util/write_excel.py +141 -133
- fosslight_util/write_opossum.py +14 -20
- fosslight_util/write_scancodejson.py +51 -32
- fosslight_util/write_spdx.py +162 -115
- fosslight_util/write_txt.py +2 -1
- fosslight_util/write_yaml.py +43 -49
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/METADATA +32 -24
- fosslight_util-2.1.28.dist-info/RECORD +32 -0
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/WHEEL +1 -1
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/entry_points.txt +0 -1
- fosslight_util/convert_excel_to_yaml.py +0 -69
- fosslight_util-1.4.34.dist-info/RECORD +0 -30
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info/licenses}/LICENSE +0 -0
- {fosslight_util-1.4.34.dist-info → fosslight_util-2.1.28.dist-info}/top_level.txt +0 -0
fosslight_util/write_spdx.py
CHANGED
|
@@ -8,23 +8,35 @@ import uuid
|
|
|
8
8
|
import logging
|
|
9
9
|
import re
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from
|
|
12
|
-
from spdx.document import Document
|
|
13
|
-
from spdx.package import Package
|
|
14
|
-
from spdx.relationship import Relationship
|
|
15
|
-
from spdx.license import License, LicenseConjunction
|
|
16
|
-
from spdx.utils import SPDXNone
|
|
17
|
-
from spdx.utils import NoAssert
|
|
18
|
-
from spdx.version import Version
|
|
19
|
-
from spdx.writers import json
|
|
20
|
-
from spdx.writers import yaml
|
|
21
|
-
from spdx.writers import xml
|
|
22
|
-
from spdx.writers import tagvalue
|
|
11
|
+
from datetime import datetime
|
|
23
12
|
from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick
|
|
24
|
-
|
|
13
|
+
from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER,
|
|
14
|
+
FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE)
|
|
15
|
+
from fosslight_util.oss_item import CHECKSUM_NULL, get_checksum_sha1
|
|
25
16
|
import traceback
|
|
26
17
|
|
|
27
|
-
logger = logging.getLogger(
|
|
18
|
+
logger = logging.getLogger(LOGGER_NAME)
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from spdx_tools.common.spdx_licensing import spdx_licensing
|
|
22
|
+
from spdx_tools.spdx.model import (
|
|
23
|
+
Actor,
|
|
24
|
+
ActorType,
|
|
25
|
+
Checksum,
|
|
26
|
+
ChecksumAlgorithm,
|
|
27
|
+
CreationInfo,
|
|
28
|
+
Document,
|
|
29
|
+
File,
|
|
30
|
+
Package,
|
|
31
|
+
Relationship,
|
|
32
|
+
RelationshipType,
|
|
33
|
+
SpdxNoAssertion,
|
|
34
|
+
SpdxNone
|
|
35
|
+
)
|
|
36
|
+
from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document
|
|
37
|
+
from spdx_tools.spdx.writer.write_anything import write_file
|
|
38
|
+
except Exception:
|
|
39
|
+
logger.info('No import spdx-tools')
|
|
28
40
|
|
|
29
41
|
|
|
30
42
|
def get_license_list_version():
|
|
@@ -37,20 +49,29 @@ def get_license_list_version():
|
|
|
37
49
|
return version
|
|
38
50
|
|
|
39
51
|
|
|
40
|
-
def write_spdx(output_file_without_ext, output_extension,
|
|
41
|
-
scanner_name, scanner_version, spdx_version=(2, 3)):
|
|
52
|
+
def write_spdx(output_file_without_ext, output_extension, scan_item, spdx_version='2.3'):
|
|
42
53
|
success = True
|
|
43
54
|
error_msg = ''
|
|
44
|
-
if sheet_list:
|
|
45
|
-
doc = Document(version=Version(*spdx_version),
|
|
46
|
-
data_license=License.from_identifier('CC0-1.0'),
|
|
47
|
-
namespace=f'http://spdx.org/spdxdocs/{scanner_name.lower()}-{uuid.uuid4()}',
|
|
48
|
-
name=f'SPDX Document by {scanner_name.upper()}',
|
|
49
|
-
spdx_id='SPDXRef-DOCUMENT')
|
|
50
55
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
56
|
+
if scan_item:
|
|
57
|
+
try:
|
|
58
|
+
cover_name = scan_item.cover.get_print_json()["Tool information"].split('(').pop(0).strip()
|
|
59
|
+
match = re.search(r"(.+) v([0-9.]+)", cover_name)
|
|
60
|
+
if match:
|
|
61
|
+
scanner_name = match.group(1)
|
|
62
|
+
else:
|
|
63
|
+
scanner_name = FOSSLIGHT_SCANNER
|
|
64
|
+
except Exception:
|
|
65
|
+
cover_name = FOSSLIGHT_SCANNER
|
|
66
|
+
scanner_name = FOSSLIGHT_SCANNER
|
|
67
|
+
creation_info = CreationInfo(spdx_version=f'SPDX-{spdx_version}',
|
|
68
|
+
spdx_id='SPDXRef-DOCUMENT',
|
|
69
|
+
name=f'SPDX Document by {scanner_name.upper()}',
|
|
70
|
+
data_license='CC0-1.0',
|
|
71
|
+
document_namespace=f'http://spdx.org/spdxdocs/{scanner_name.lower()}-{uuid.uuid4()}',
|
|
72
|
+
creators=[Actor(name=cover_name, actor_type=ActorType.TOOL)],
|
|
73
|
+
created=datetime.now())
|
|
74
|
+
doc = Document(creation_info=creation_info)
|
|
54
75
|
|
|
55
76
|
relation_tree = {}
|
|
56
77
|
spdx_id_packages = []
|
|
@@ -58,72 +79,104 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
|
|
|
58
79
|
output_dir = os.path.dirname(output_file_without_ext)
|
|
59
80
|
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
|
60
81
|
try:
|
|
82
|
+
file_id = 0
|
|
61
83
|
package_id = 0
|
|
62
84
|
root_package = False
|
|
63
|
-
for
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
85
|
+
for scanner_name, file_items in scan_item.file_items.items():
|
|
86
|
+
for file_item in file_items:
|
|
87
|
+
file = '' # file의 license, copyright은 oss item에서 append
|
|
88
|
+
if scanner_name in [FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE]:
|
|
89
|
+
if file_item.exclude:
|
|
90
|
+
continue
|
|
91
|
+
if file_item.checksum == CHECKSUM_NULL:
|
|
92
|
+
if os.path.exists(file_item.source_name_or_path):
|
|
93
|
+
file_item.checksum = get_checksum_sha1(file_item.source_name_or_path)
|
|
94
|
+
if file_item.checksum == CHECKSUM_NULL:
|
|
95
|
+
logger.info(f'Failed to get checksum, Skip: {file_item.source_name_or_path}')
|
|
96
|
+
continue
|
|
97
|
+
file_id += 1
|
|
98
|
+
file = File(name=file_item.source_name_or_path,
|
|
99
|
+
spdx_id=f'SPDXRef-File{file_id}',
|
|
100
|
+
checksums=[Checksum(ChecksumAlgorithm.SHA1, file_item.checksum)])
|
|
101
|
+
file_license = []
|
|
102
|
+
file_copyright = []
|
|
103
|
+
file_comment = []
|
|
104
|
+
for oss_item in file_item.oss_items:
|
|
105
|
+
oss_licenses = []
|
|
106
|
+
declared_oss_licenses = []
|
|
107
|
+
lic_comment = []
|
|
108
|
+
for oi in oss_item.license:
|
|
109
|
+
oi = check_input_license_format(oi)
|
|
110
|
+
try:
|
|
111
|
+
oi_spdx = spdx_licensing.parse(oi, validate=True)
|
|
112
|
+
oss_licenses.append(oi_spdx)
|
|
113
|
+
declared_oss_licenses.append(oi)
|
|
114
|
+
except Exception:
|
|
115
|
+
logger.debug(f'No spdx license name: {oi}')
|
|
116
|
+
lic_comment.append(oi)
|
|
117
|
+
file_comment.append(oi)
|
|
118
|
+
if oss_licenses:
|
|
119
|
+
file_license.extend(oss_licenses)
|
|
120
|
+
if oss_item.copyright != '':
|
|
121
|
+
file_copyright.append(oss_item.copyright)
|
|
122
|
+
|
|
123
|
+
if oss_item.download_location == '':
|
|
124
|
+
if scanner_name == FOSSLIGHT_DEPENDENCY:
|
|
125
|
+
download_location = SpdxNone()
|
|
126
|
+
else:
|
|
127
|
+
continue
|
|
128
|
+
else:
|
|
129
|
+
download_location = oss_item.download_location
|
|
130
|
+
if scanner_name != FOSSLIGHT_DEPENDENCY and oss_item.name == '':
|
|
131
|
+
continue
|
|
132
|
+
package_id += 1
|
|
133
|
+
package = Package(name=oss_item.name,
|
|
134
|
+
spdx_id=f'SPDXRef-Package{package_id}',
|
|
135
|
+
download_location=download_location)
|
|
136
|
+
|
|
137
|
+
if oss_item.version != '':
|
|
138
|
+
package.version = oss_item.version
|
|
139
|
+
|
|
140
|
+
if scanner_name == FOSSLIGHT_DEPENDENCY:
|
|
141
|
+
package.files_analyzed = False # If omitted, the default value of true is assumed.
|
|
142
|
+
else:
|
|
143
|
+
package.files_analyzed = True
|
|
144
|
+
if oss_item.copyright != '':
|
|
145
|
+
package.cr_text = oss_item.copyright
|
|
146
|
+
if oss_item.homepage != '':
|
|
147
|
+
package.homepage = oss_item.homepage
|
|
148
|
+
|
|
149
|
+
if declared_oss_licenses:
|
|
150
|
+
package.license_declared = spdx_licensing.parse(' AND '.join(declared_oss_licenses))
|
|
151
|
+
if lic_comment:
|
|
152
|
+
package.license_comment = ' '.join(lic_comment)
|
|
153
|
+
|
|
154
|
+
doc.packages.append(package)
|
|
155
|
+
|
|
156
|
+
if scanner_name == FOSSLIGHT_DEPENDENCY:
|
|
157
|
+
purl = file_item.purl
|
|
158
|
+
spdx_id_packages.append([purl, package.spdx_id])
|
|
159
|
+
relation_tree[purl] = {}
|
|
160
|
+
relation_tree[purl]['id'] = package.spdx_id
|
|
161
|
+
relation_tree[purl]['dep'] = []
|
|
162
|
+
if 'root package' in oss_item.comment:
|
|
163
|
+
root_package = True
|
|
164
|
+
relationship = Relationship(doc.creation_info.spdx_id,
|
|
165
|
+
RelationshipType.DESCRIBES,
|
|
166
|
+
package.spdx_id)
|
|
167
|
+
doc.relationships.append(relationship)
|
|
168
|
+
relation_tree[purl]['dep'].extend(file_item.depends_on)
|
|
169
|
+
|
|
170
|
+
if scanner_name in [FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE]:
|
|
171
|
+
if file_license:
|
|
172
|
+
file.license_info_in_file = file_license
|
|
173
|
+
if file_copyright:
|
|
174
|
+
file.copyright_text = '\n'.join(file_copyright)
|
|
175
|
+
if file_comment:
|
|
176
|
+
file.license_comment = ' '.join(file_comment)
|
|
177
|
+
doc.files.append(file)
|
|
178
|
+
|
|
179
|
+
if len(doc.packages) > 0:
|
|
127
180
|
for pkg in relation_tree:
|
|
128
181
|
if len(relation_tree[pkg]['dep']) > 0:
|
|
129
182
|
pkg_spdx_id = relation_tree[pkg]['id']
|
|
@@ -133,18 +186,18 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
|
|
|
133
186
|
if ans is None:
|
|
134
187
|
continue
|
|
135
188
|
rel_pkg_spdx_id = ans[1]
|
|
136
|
-
relationship = Relationship(
|
|
137
|
-
doc.
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
189
|
+
relationship = Relationship(pkg_spdx_id, RelationshipType.DEPENDS_ON, rel_pkg_spdx_id)
|
|
190
|
+
doc.relationships.append(relationship)
|
|
191
|
+
if not root_package:
|
|
192
|
+
root_package = Package(name='root package',
|
|
193
|
+
spdx_id='SPDXRef-ROOT-PACKAGE',
|
|
194
|
+
download_location=SpdxNoAssertion())
|
|
195
|
+
root_package.files_analyzed = False
|
|
196
|
+
root_package.license_declared = SpdxNoAssertion()
|
|
197
|
+
doc.packages.append(root_package)
|
|
198
|
+
relationship = Relationship(doc.creation_info.spdx_id, RelationshipType.DESCRIBES, root_package.spdx_id)
|
|
199
|
+
doc.relationships.append(relationship)
|
|
200
|
+
|
|
148
201
|
except Exception as e:
|
|
149
202
|
success = False
|
|
150
203
|
error_msg = f'Failed to create spdx document object:{e}, {traceback.format_exc()}'
|
|
@@ -152,24 +205,18 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
|
|
|
152
205
|
success = False
|
|
153
206
|
error_msg = 'No item to write in output file.'
|
|
154
207
|
|
|
208
|
+
validation_messages = validate_full_spdx_document(doc)
|
|
209
|
+
for message in validation_messages:
|
|
210
|
+
logger.warning(message.validation_message)
|
|
211
|
+
logger.warning(message.context)
|
|
212
|
+
|
|
213
|
+
# assert validation_messages == []
|
|
214
|
+
|
|
155
215
|
result_file = ''
|
|
156
216
|
if success:
|
|
157
217
|
result_file = output_file_without_ext + output_extension
|
|
158
218
|
try:
|
|
159
|
-
|
|
160
|
-
if result_file.endswith(".tag"):
|
|
161
|
-
writer_module = tagvalue
|
|
162
|
-
elif result_file.endswith(".json"):
|
|
163
|
-
writer_module = json
|
|
164
|
-
elif result_file.endswith(".xml"):
|
|
165
|
-
writer_module = xml
|
|
166
|
-
elif result_file.endswith(".yaml"):
|
|
167
|
-
writer_module = yaml
|
|
168
|
-
else:
|
|
169
|
-
raise Exception("FileType Not Supported")
|
|
170
|
-
|
|
171
|
-
with open(result_file, out_mode) as out:
|
|
172
|
-
writer_module.write_document(doc, out, True)
|
|
219
|
+
write_file(doc, result_file)
|
|
173
220
|
except Exception as e:
|
|
174
221
|
success = False
|
|
175
222
|
error_msg = f'Failed to write spdx document: {e}'
|
fosslight_util/write_txt.py
CHANGED
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
import os
|
|
6
6
|
from pathlib import Path
|
|
7
|
+
from typing import Tuple
|
|
7
8
|
|
|
8
9
|
|
|
9
|
-
def write_txt_file(file_to_create, str_to_write):
|
|
10
|
+
def write_txt_file(file_to_create: str, str_to_write: str) -> Tuple[bool, str]:
|
|
10
11
|
success = True
|
|
11
12
|
error_msg = ""
|
|
12
13
|
try:
|
fosslight_util/write_yaml.py
CHANGED
|
@@ -2,55 +2,49 @@
|
|
|
2
2
|
# -*- coding: utf-8 -*-
|
|
3
3
|
# Copyright (c) 2022 LG Electronics Inc.
|
|
4
4
|
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
|
|
6
5
|
import yaml
|
|
7
6
|
import logging
|
|
8
7
|
import os
|
|
9
|
-
import
|
|
8
|
+
import json
|
|
10
9
|
from pathlib import Path
|
|
11
|
-
|
|
12
|
-
from
|
|
13
|
-
from fosslight_util.write_excel import _EMPTY_ITEM_MSG
|
|
10
|
+
from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER
|
|
11
|
+
from typing import Tuple
|
|
14
12
|
|
|
15
|
-
_logger = logging.getLogger(
|
|
13
|
+
_logger = logging.getLogger(LOGGER_NAME)
|
|
16
14
|
|
|
17
15
|
|
|
18
|
-
def write_yaml(output_file,
|
|
16
|
+
def write_yaml(output_file, scan_item, separate_yaml=False) -> Tuple[bool, str, str]:
|
|
19
17
|
success = True
|
|
20
18
|
error_msg = ""
|
|
21
19
|
output = ""
|
|
22
20
|
|
|
23
21
|
try:
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if sheet_name not in constant.supported_sheet_and_scanner.keys():
|
|
37
|
-
continue
|
|
38
|
-
if not separate_yaml:
|
|
39
|
-
merge_sheet.extend(sheet_contents)
|
|
40
|
-
else:
|
|
41
|
-
output_file = f'{separate_output_file}_{sheet_name}.yaml'
|
|
42
|
-
convert_sheet_to_yaml(sheet_contents, output_file)
|
|
43
|
-
output_files.append(output_file)
|
|
22
|
+
output_files = []
|
|
23
|
+
output_dir = os.path.dirname(output_file)
|
|
24
|
+
|
|
25
|
+
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
|
26
|
+
if separate_yaml:
|
|
27
|
+
filename = os.path.splitext(os.path.basename(output_file))[0]
|
|
28
|
+
separate_output_file = os.path.join(output_dir, filename)
|
|
29
|
+
|
|
30
|
+
merge_sheet = []
|
|
31
|
+
for scanner_name, _ in scan_item.file_items.items():
|
|
32
|
+
sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
|
|
33
|
+
json_contents = scan_item.get_print_json(scanner_name)
|
|
44
34
|
|
|
45
35
|
if not separate_yaml:
|
|
46
|
-
|
|
36
|
+
merge_sheet.extend(json_contents)
|
|
37
|
+
else:
|
|
38
|
+
output_file = f'{separate_output_file}_{sheet_name}.yaml'
|
|
39
|
+
remove_duplicates_and_dump_yaml(json_contents, output_file)
|
|
47
40
|
output_files.append(output_file)
|
|
48
41
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
42
|
+
if not separate_yaml:
|
|
43
|
+
remove_duplicates_and_dump_yaml(merge_sheet, output_file)
|
|
44
|
+
output_files.append(output_file)
|
|
45
|
+
|
|
46
|
+
if output_files:
|
|
47
|
+
output = ", ".join(output_files)
|
|
54
48
|
except Exception as ex:
|
|
55
49
|
error_msg = str(ex)
|
|
56
50
|
success = False
|
|
@@ -61,37 +55,37 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
|
|
|
61
55
|
return success, error_msg, output
|
|
62
56
|
|
|
63
57
|
|
|
64
|
-
def
|
|
65
|
-
|
|
58
|
+
def remove_duplicates_and_dump_yaml(json_contents, output_file):
|
|
59
|
+
unique_json_strings = {json.dumps(e, sort_keys=True) for e in json_contents}
|
|
60
|
+
unique_json_contents = [json.loads(e) for e in unique_json_strings]
|
|
66
61
|
|
|
67
62
|
yaml_dict = {}
|
|
68
|
-
for
|
|
69
|
-
|
|
70
|
-
item.set_sheet_item(sheet_item)
|
|
71
|
-
create_yaml_with_ossitem(item, yaml_dict)
|
|
63
|
+
for uitem in unique_json_contents:
|
|
64
|
+
create_yaml_with_ossitem(uitem, yaml_dict)
|
|
72
65
|
|
|
73
66
|
with open(output_file, 'w') as f:
|
|
74
67
|
yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
|
|
75
68
|
|
|
76
69
|
|
|
77
70
|
def create_yaml_with_ossitem(item, yaml_dict):
|
|
78
|
-
|
|
79
|
-
item_name = item_json.pop("name")
|
|
71
|
+
item_name = item.pop("name")
|
|
80
72
|
|
|
81
73
|
if item_name not in yaml_dict.keys():
|
|
82
74
|
yaml_dict[item_name] = []
|
|
83
75
|
merged = False
|
|
84
76
|
for oss_info in yaml_dict[item_name]:
|
|
85
|
-
if oss_info.get('version', '') == item.version and \
|
|
86
|
-
oss_info.get('license', []) == item.license and \
|
|
87
|
-
oss_info.get('copyright text', '') == item.copyright and \
|
|
88
|
-
oss_info.get('homepage', '') == item.homepage and \
|
|
89
|
-
oss_info.get('download location', '') == item.
|
|
90
|
-
oss_info.get('exclude', False) == item.exclude:
|
|
91
|
-
oss_info.get('source
|
|
77
|
+
if oss_info.get('version', '') == item.get('version', '') and \
|
|
78
|
+
oss_info.get('license', []) == item.get('license', []) and \
|
|
79
|
+
oss_info.get('copyright text', '') == item.get('copyright text', '') and \
|
|
80
|
+
oss_info.get('homepage', '') == item.get('homepage', '') and \
|
|
81
|
+
oss_info.get('download location', '') == item.get('download location', '') and \
|
|
82
|
+
oss_info.get('exclude', False) == item.get('exclude', False):
|
|
83
|
+
if isinstance(oss_info.get('source path', []), str):
|
|
84
|
+
oss_info['source path'] = [oss_info.get('source path', '')]
|
|
85
|
+
oss_info.get('source path', []).append(item.get('source path', ''))
|
|
92
86
|
oss_info.pop('comment', None)
|
|
93
87
|
merged = True
|
|
94
88
|
break
|
|
95
89
|
|
|
96
90
|
if not merged:
|
|
97
|
-
yaml_dict[item_name].append(
|
|
91
|
+
yaml_dict[item_name].append(item)
|
|
@@ -1,24 +1,20 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name:
|
|
3
|
-
Version: 1.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fosslight_util
|
|
3
|
+
Version: 2.1.28
|
|
4
4
|
Summary: FOSSLight Util
|
|
5
5
|
Home-page: https://github.com/fosslight/fosslight_util
|
|
6
|
+
Download-URL: https://github.com/fosslight/fosslight_util
|
|
6
7
|
Author: LG Electronics
|
|
7
8
|
License: Apache-2.0
|
|
8
|
-
Download-URL: https://github.com/fosslight/fosslight_util
|
|
9
|
-
Platform: UNKNOWN
|
|
10
9
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
10
|
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
16
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
17
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
14
|
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
19
16
|
Requires-Dist: XlsxWriter
|
|
20
17
|
Requires-Dist: pandas
|
|
21
|
-
Requires-Dist: xlrd ==1.2.0
|
|
22
18
|
Requires-Dist: openpyxl
|
|
23
19
|
Requires-Dist: progress
|
|
24
20
|
Requires-Dist: PyYAML
|
|
@@ -27,13 +23,22 @@ Requires-Dist: coloredlogs
|
|
|
27
23
|
Requires-Dist: python3-wget
|
|
28
24
|
Requires-Dist: beautifulsoup4
|
|
29
25
|
Requires-Dist: jsonmerge
|
|
30
|
-
Requires-Dist: spdx-tools
|
|
31
|
-
Requires-Dist:
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist:
|
|
35
|
-
Requires-Dist:
|
|
36
|
-
|
|
26
|
+
Requires-Dist: spdx-tools==0.8.*; sys_platform == "linux"
|
|
27
|
+
Requires-Dist: setuptools>=65.5.1
|
|
28
|
+
Requires-Dist: numpy
|
|
29
|
+
Requires-Dist: requests
|
|
30
|
+
Requires-Dist: GitPython
|
|
31
|
+
Requires-Dist: cyclonedx-python-lib==8.5.*; sys_platform == "linux"
|
|
32
|
+
Dynamic: author
|
|
33
|
+
Dynamic: classifier
|
|
34
|
+
Dynamic: description
|
|
35
|
+
Dynamic: description-content-type
|
|
36
|
+
Dynamic: download-url
|
|
37
|
+
Dynamic: home-page
|
|
38
|
+
Dynamic: license
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
Dynamic: requires-dist
|
|
41
|
+
Dynamic: summary
|
|
37
42
|
|
|
38
43
|
<!--
|
|
39
44
|
Copyright (c) 2021 LG Electronics
|
|
@@ -66,7 +71,7 @@ It is a package that supports common utils used by FOSSLight Scanner.
|
|
|
66
71
|
|
|
67
72
|
## 📋 Prerequisite
|
|
68
73
|
|
|
69
|
-
FOSSLight Util needs a Python 3.
|
|
74
|
+
FOSSLight Util needs a Python 3.10+.
|
|
70
75
|
|
|
71
76
|
## 🎉 How to install
|
|
72
77
|
|
|
@@ -80,7 +85,7 @@ $ pip3 install fosslight_util
|
|
|
80
85
|
|
|
81
86
|
Three modules can be called. Please refer to each file for detailed calling method.
|
|
82
87
|
|
|
83
|
-
|
|
88
|
+
|
|
84
89
|
### 1. Setup logger (tests/test_log.py)
|
|
85
90
|
```
|
|
86
91
|
from fosslight_util.set_log import init_log
|
|
@@ -101,7 +106,7 @@ def test():
|
|
|
101
106
|
logger.warning("TESTING - Print log")
|
|
102
107
|
```
|
|
103
108
|
|
|
104
|
-
|
|
109
|
+
|
|
105
110
|
### 2. Write result files (tests/test_output_format.py)
|
|
106
111
|
```
|
|
107
112
|
from fosslight_util.output_format import write_output_file
|
|
@@ -117,7 +122,7 @@ def test():
|
|
|
117
122
|
'0.4.3', 'Apache-2.0', 'https://github.com/jpeddicord/askalono', '', 'Copyright (c) 2018 Amazon.com, Inc. or its affiliates.', '', '']]}
|
|
118
123
|
success, msg = write_output_file('test_result/excel/FOSSLight-Report', '.xlsx', sheet_contents)
|
|
119
124
|
```
|
|
120
|
-
|
|
125
|
+
|
|
121
126
|
### 3. Get spdx licenses (tests/test_spdx_licenses.py)
|
|
122
127
|
```
|
|
123
128
|
from fosslight_util.spdx_licenses import get_spdx_licenses_json
|
|
@@ -151,6 +156,7 @@ If you give a link, the source is downloaded to the target directory through git
|
|
|
151
156
|
|
|
152
157
|
#### How it works
|
|
153
158
|
1. Try git clone.
|
|
159
|
+
1-1. If the link is ssh-url, convert to https-url.
|
|
154
160
|
2. If git clone fails, download it with wget and extract the compressed file.
|
|
155
161
|
3. After extracting the compressed file, delete the compressed file.
|
|
156
162
|
|
|
@@ -164,7 +170,11 @@ If you give a link, the source is downloaded to the target directory through git
|
|
|
164
170
|
|
|
165
171
|
#### How to run
|
|
166
172
|
```
|
|
167
|
-
$ fosslight_download
|
|
173
|
+
$ fosslight_download -s "https://github.com/LGE-OSS/example" -t target_dir/
|
|
174
|
+
```
|
|
175
|
+
If you want to try with private repository, set your github token like below.
|
|
176
|
+
```
|
|
177
|
+
$ fosslight_download -s "https://my_github_token@github.com/Foo/private_repo -t target_dir/"
|
|
168
178
|
```
|
|
169
179
|
|
|
170
180
|
## 👏 How to report issue
|
|
@@ -178,5 +188,3 @@ Please report any ideas or bugs to improve by creating an issue in [fosslight_ut
|
|
|
178
188
|
FOSSLight Util is released under [Apache-2.0][l].
|
|
179
189
|
|
|
180
190
|
[l]: https://github.com/fosslight/fosslight_util/blob/main/LICENSE
|
|
181
|
-
|
|
182
|
-
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
fosslight_util/_get_downloadable_url.py,sha256=GNfGLJPQVHrU_eYse1flE42AgYUviGNKWZnq9sfUY9c,24473
|
|
3
|
+
fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE4,2916
|
|
4
|
+
fosslight_util/constant.py,sha256=3BzJtyxC0o5IFAhYaUW-DeTKkA6f5tDJFJTb0k5ji9Y,2418
|
|
5
|
+
fosslight_util/correct.py,sha256=1WEAL-9_KhjFPLucPhv0PNN3K7avm0z8mU6sTuSyeHM,3864
|
|
6
|
+
fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
|
|
7
|
+
fosslight_util/download.py,sha256=t6-5NAcvCOfmi9TM7O1yp-9X11MiuKd8JdzNEZtEmqQ,20967
|
|
8
|
+
fosslight_util/exclude.py,sha256=fDmBsZJ_F7O9Oh2T-07R03XNbElo1tFaf_z01KfSAqU,2399
|
|
9
|
+
fosslight_util/help.py,sha256=iyWmAaUQSHJtWv5mjFv0f3YoDVlDgEqdsDDEyImEUNc,2646
|
|
10
|
+
fosslight_util/oss_item.py,sha256=8890JHb5ZoKQWAwN7Fl8badnlYatJtF4MVJz1rdS4yQ,6938
|
|
11
|
+
fosslight_util/output_format.py,sha256=BP23LspxawDZ_a99oWLVKWUQ-G7P5uoUpjEXhkRFKwc,8801
|
|
12
|
+
fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
|
|
13
|
+
fosslight_util/read_excel.py,sha256=-QvrdxaNqYOpIm1H7ZqIEh5NLvFPymZo6BAOZcQmQug,5263
|
|
14
|
+
fosslight_util/set_log.py,sha256=AbcLFLvY9GSOYSN0a110wO5gNcyc8KKnNjl7GxHEW9A,4008
|
|
15
|
+
fosslight_util/spdx_licenses.py,sha256=GvMNe_D4v2meapTVwPu2BJXInnTo3_gIzg669eJhUu0,3691
|
|
16
|
+
fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
|
|
17
|
+
fosslight_util/write_cyclonedx.py,sha256=hq817j-0OM89B8jtZKgHgvVa0YEaYHlz_8R5vNpe21I,9662
|
|
18
|
+
fosslight_util/write_excel.py,sha256=QUIMCnmEKJoSpri5RctBcKLvhDShLdZUP_dhHv-sVy8,10165
|
|
19
|
+
fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
|
|
20
|
+
fosslight_util/write_scancodejson.py,sha256=dMCjTtUnNR5BCL6gBCleDT8bTSAN5Gg2RAfimmkGXUE,2692
|
|
21
|
+
fosslight_util/write_spdx.py,sha256=Ov9jBlfVrkWIymcfAxbupUxDZKfCOZZGOPZ4v-x230M,12108
|
|
22
|
+
fosslight_util/write_txt.py,sha256=BEFjYBppqk1CITx-fUN4vfvKv0XCs1GXWtc2Iu-etU4,629
|
|
23
|
+
fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,3238
|
|
24
|
+
fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
|
|
25
|
+
fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
|
|
26
|
+
fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
|
|
27
|
+
fosslight_util-2.1.28.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
28
|
+
fosslight_util-2.1.28.dist-info/METADATA,sha256=9aponw8jVAsmq4_4hIgpvIYCyzSfkDcPR7h9a3tg9mo,6367
|
|
29
|
+
fosslight_util-2.1.28.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
|
|
30
|
+
fosslight_util-2.1.28.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
|
|
31
|
+
fosslight_util-2.1.28.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
|
|
32
|
+
fosslight_util-2.1.28.dist-info/RECORD,,
|