fosslight-source 2.3.2__tar.gz → 2.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fosslight_source-2.3.2/src/fosslight_source.egg-info → fosslight_source-2.3.3}/PKG-INFO +2 -2
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/pyproject.toml +2 -2
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_help.py +1 -0
- fosslight_source-2.3.3/src/fosslight_source/_merge.py +269 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_scan_item.py +1 -3
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/cli.py +57 -29
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_scancode.py +6 -12
- {fosslight_source-2.3.2 → fosslight_source-2.3.3/src/fosslight_source.egg-info}/PKG-INFO +2 -2
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/SOURCES.txt +1 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/requires.txt +1 -1
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/LICENSE +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/MANIFEST.in +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/README.md +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/setup.cfg +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/__init__.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_kb_client.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_license_matched.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_parsing_scancode_file_item.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_parsing_scanoss_file.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_manifest_extractor.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_scanoss.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_spdx_extractor.py +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/dependency_links.txt +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/entry_points.txt +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/top_level.txt +0 -0
- {fosslight_source-2.3.2 → fosslight_source-2.3.3}/tests/test_tox.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_source
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.3
|
|
4
4
|
Summary: FOSSLight Source Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -20,7 +20,7 @@ Requires-Dist: setuptools<=80.10.2
|
|
|
20
20
|
Requires-Dist: pyparsing
|
|
21
21
|
Requires-Dist: scanoss>=1.19.0
|
|
22
22
|
Requires-Dist: XlsxWriter
|
|
23
|
-
Requires-Dist: fosslight_util>=2.
|
|
23
|
+
Requires-Dist: fosslight_util>=2.2.2
|
|
24
24
|
Requires-Dist: PyYAML
|
|
25
25
|
Requires-Dist: wheel>=0.38.1
|
|
26
26
|
Requires-Dist: intbitset
|
|
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "fosslight_source"
|
|
10
|
-
version = "2.3.
|
|
10
|
+
version = "2.3.3"
|
|
11
11
|
description = "FOSSLight Source Scanner"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = "Apache-2.0"
|
|
@@ -29,7 +29,7 @@ dependencies = [
|
|
|
29
29
|
"pyparsing",
|
|
30
30
|
"scanoss>=1.19.0",
|
|
31
31
|
"XlsxWriter",
|
|
32
|
-
"fosslight_util>=2.
|
|
32
|
+
"fosslight_util>=2.2.2",
|
|
33
33
|
"PyYAML",
|
|
34
34
|
"wheel>=0.38.1",
|
|
35
35
|
"intbitset",
|
|
@@ -39,6 +39,7 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
|
|
|
39
39
|
-c <number> Number of CPU cores/threads to use for scanning
|
|
40
40
|
-t <seconds> Timeout in seconds for ScanCode scanning
|
|
41
41
|
-j Generate raw scanner results in JSON format
|
|
42
|
+
--no_merge Keep source paths file-based without folder merge
|
|
42
43
|
--no_correction Skip OSS information correction with sbom-info.yaml
|
|
43
44
|
--correct_fpath <path> Path to custom sbom-info.yaml file
|
|
44
45
|
--hide_progress Hide the progress bar during scanning
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# Copyright (c) 2020 LG Electronics Inc.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
from collections import Counter
|
|
8
|
+
|
|
9
|
+
from fosslight_util.constant import COMMENT_DELIMITER
|
|
10
|
+
|
|
11
|
+
from ._scan_item import SourceItem
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _get_source_rows_to_print(source_items: list) -> list:
|
|
15
|
+
source_rows = []
|
|
16
|
+
for source_item in source_items:
|
|
17
|
+
source_rows.extend(source_item.get_print_array())
|
|
18
|
+
return source_rows
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _add_pre_merge_sheet(scan_item, pre_merge_sheet_name: str, header_row: list, pkg_name: str) -> None:
|
|
22
|
+
external_sheets = getattr(scan_item, "external_sheets", {}) or {}
|
|
23
|
+
external_sheets[pre_merge_sheet_name] = [
|
|
24
|
+
header_row,
|
|
25
|
+
*_get_source_rows_to_print(scan_item.file_items.get(pkg_name, [])),
|
|
26
|
+
]
|
|
27
|
+
scan_item.external_sheets = external_sheets
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _normalize_merge_text(value: str) -> str:
|
|
31
|
+
return value.strip() if value else ""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _iter_merge_values(values) -> list:
|
|
35
|
+
if not values:
|
|
36
|
+
return []
|
|
37
|
+
if isinstance(values, str):
|
|
38
|
+
return [values]
|
|
39
|
+
return values
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _normalize_merge_licenses(licenses) -> tuple:
|
|
43
|
+
return tuple(sorted([lic.strip() for lic in _iter_merge_values(licenses) if lic and lic.strip()]))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _normalize_merge_downloads(downloads) -> tuple:
|
|
47
|
+
if not downloads:
|
|
48
|
+
return ()
|
|
49
|
+
if isinstance(downloads, str):
|
|
50
|
+
downloads = [downloads]
|
|
51
|
+
return tuple(sorted([dl.strip() for dl in downloads if dl and dl.strip()]))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _iter_merge_rows(item: SourceItem):
|
|
55
|
+
if item.oss_items:
|
|
56
|
+
for oss_item in item.oss_items:
|
|
57
|
+
name = _normalize_merge_text(oss_item.name) or _normalize_merge_text(item.oss_name)
|
|
58
|
+
version = _normalize_merge_text(oss_item.version) or _normalize_merge_text(item.oss_version)
|
|
59
|
+
licenses = _normalize_merge_licenses(oss_item.license or item.licenses)
|
|
60
|
+
downloads = _normalize_merge_downloads(oss_item.download_location or item.download_location)
|
|
61
|
+
yield name, version, licenses, downloads
|
|
62
|
+
else:
|
|
63
|
+
yield (
|
|
64
|
+
_normalize_merge_text(item.oss_name),
|
|
65
|
+
_normalize_merge_text(item.oss_version),
|
|
66
|
+
_normalize_merge_licenses(item.licenses),
|
|
67
|
+
_normalize_merge_downloads(item.download_location),
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _get_item_oss_name(item: SourceItem) -> str:
|
|
72
|
+
for name, _, _, _ in _iter_merge_rows(item):
|
|
73
|
+
if name:
|
|
74
|
+
return name
|
|
75
|
+
return ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _get_item_oss_version(item: SourceItem) -> str:
|
|
79
|
+
for _, version, _, _ in _iter_merge_rows(item):
|
|
80
|
+
if version:
|
|
81
|
+
return version
|
|
82
|
+
return ""
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _get_merge_licenses(scan_item: SourceItem) -> tuple:
|
|
86
|
+
for _, _, licenses, _ in _iter_merge_rows(scan_item):
|
|
87
|
+
if licenses:
|
|
88
|
+
return licenses
|
|
89
|
+
return ()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _get_merge_download_locations(scan_item: SourceItem) -> tuple:
|
|
93
|
+
for _, _, _, downloads in _iter_merge_rows(scan_item):
|
|
94
|
+
if downloads:
|
|
95
|
+
return downloads
|
|
96
|
+
return ()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _get_merge_field_value(scan_items: list, value_getter):
|
|
100
|
+
for scan_item in scan_items:
|
|
101
|
+
value = value_getter(scan_item)
|
|
102
|
+
if value:
|
|
103
|
+
return value
|
|
104
|
+
return ""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _get_top_merge_values(scan_items: list, value_getter) -> list:
|
|
108
|
+
values = []
|
|
109
|
+
for scan_item in scan_items:
|
|
110
|
+
for value in _iter_merge_values(value_getter(scan_item)):
|
|
111
|
+
normalized_value = _normalize_merge_text(value)
|
|
112
|
+
if normalized_value:
|
|
113
|
+
values.append(normalized_value)
|
|
114
|
+
return [value for value, _ in Counter(values).most_common(3)]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _can_merge_folder(scan_items: list) -> bool:
|
|
118
|
+
if len(scan_items) <= 1:
|
|
119
|
+
return False
|
|
120
|
+
|
|
121
|
+
ref_name = None
|
|
122
|
+
ref_version = None
|
|
123
|
+
ref_licenses = None
|
|
124
|
+
ref_downloads = None
|
|
125
|
+
|
|
126
|
+
for item in scan_items:
|
|
127
|
+
for name, version, licenses, downloads in _iter_merge_rows(item):
|
|
128
|
+
if name:
|
|
129
|
+
if ref_name is None:
|
|
130
|
+
ref_name = name
|
|
131
|
+
elif name != ref_name:
|
|
132
|
+
return False
|
|
133
|
+
|
|
134
|
+
if version:
|
|
135
|
+
if ref_version is None:
|
|
136
|
+
ref_version = version
|
|
137
|
+
elif version != ref_version:
|
|
138
|
+
return False
|
|
139
|
+
|
|
140
|
+
if licenses:
|
|
141
|
+
if ref_licenses is None:
|
|
142
|
+
ref_licenses = licenses
|
|
143
|
+
elif licenses != ref_licenses:
|
|
144
|
+
return False
|
|
145
|
+
|
|
146
|
+
if downloads:
|
|
147
|
+
if ref_downloads is None:
|
|
148
|
+
ref_downloads = downloads
|
|
149
|
+
elif downloads != ref_downloads:
|
|
150
|
+
return False
|
|
151
|
+
|
|
152
|
+
return True
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _get_merged_comments(scan_items: list) -> str:
|
|
156
|
+
comments = []
|
|
157
|
+
for item in scan_items:
|
|
158
|
+
val = item.comment
|
|
159
|
+
if val:
|
|
160
|
+
parts = [p.strip() for p in val.split(COMMENT_DELIMITER) if p.strip()]
|
|
161
|
+
for p in parts:
|
|
162
|
+
if p not in comments:
|
|
163
|
+
comments.append(p)
|
|
164
|
+
if not comments:
|
|
165
|
+
return ""
|
|
166
|
+
|
|
167
|
+
return COMMENT_DELIMITER.join(comments)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _create_merged_item(scan_items: list, merge_path: str) -> SourceItem:
|
|
171
|
+
merged_path = f"{merge_path} ({len(scan_items)})"
|
|
172
|
+
merged_item = SourceItem(merged_path)
|
|
173
|
+
merged_item.oss_name = _get_merge_field_value(scan_items, lambda item: _normalize_merge_text(_get_item_oss_name(item)))
|
|
174
|
+
merged_item.oss_version = _get_merge_field_value(scan_items, lambda item: _normalize_merge_text(_get_item_oss_version(item)))
|
|
175
|
+
merged_licenses = _get_merge_field_value(scan_items, _get_merge_licenses)
|
|
176
|
+
merged_item.licenses = list(merged_licenses) if merged_licenses else []
|
|
177
|
+
merged_downloads = _get_merge_field_value(scan_items, _get_merge_download_locations)
|
|
178
|
+
merged_item.download_location = list(merged_downloads) if merged_downloads else []
|
|
179
|
+
merged_copyrights = _get_top_merge_values(scan_items, lambda item: item.copyright)
|
|
180
|
+
merged_item.copyright = merged_copyrights if merged_copyrights else []
|
|
181
|
+
merged_comments = _get_merged_comments(scan_items)
|
|
182
|
+
if merged_comments:
|
|
183
|
+
merged_item.comment = merged_comments
|
|
184
|
+
merged_item.set_oss_item()
|
|
185
|
+
return merged_item
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def merge_results_by_folder(scan_result: list) -> list:
|
|
189
|
+
"""
|
|
190
|
+
Merge output rows within the same folder when OSS name, OSS version, license,
|
|
191
|
+
and download location are compatible.
|
|
192
|
+
|
|
193
|
+
A field is compatible when all non-empty values across rows are identical
|
|
194
|
+
(empty values are ignored). Each oss_item is treated as its own row. All eligible
|
|
195
|
+
rows in the folder must be compatible together; rows are not grouped by key subsets.
|
|
196
|
+
"""
|
|
197
|
+
# Build a folder tree first so merge never jumps straight to root ".".
|
|
198
|
+
merge_tree = {"items": [], "children": {}}
|
|
199
|
+
|
|
200
|
+
for scan_item in scan_result:
|
|
201
|
+
normalized_path = os.path.normpath(scan_item.source_name_or_path).replace("\\", "/")
|
|
202
|
+
path_parts = [part for part in normalized_path.split("/") if part and part != "."]
|
|
203
|
+
current_node = merge_tree
|
|
204
|
+
|
|
205
|
+
for folder_name in path_parts[:-1]:
|
|
206
|
+
current_node = current_node["children"].setdefault(folder_name, {"items": [], "children": {}})
|
|
207
|
+
current_node["items"].append(scan_item)
|
|
208
|
+
|
|
209
|
+
def merge_node(merge_node_item: dict, merge_path: str = "", depth: int = 0) -> tuple:
|
|
210
|
+
child_finalized = []
|
|
211
|
+
child_unfinalized = []
|
|
212
|
+
for folder_name, child_node in merge_node_item["children"].items():
|
|
213
|
+
child_path = f"{merge_path}/{folder_name}" if merge_path else folder_name
|
|
214
|
+
fin, unfin = merge_node(child_node, child_path, depth + 1)
|
|
215
|
+
child_finalized.extend(fin)
|
|
216
|
+
child_unfinalized.extend(unfin)
|
|
217
|
+
|
|
218
|
+
local_excluded = [item for item in merge_node_item["items"] if item.exclude]
|
|
219
|
+
local_eligible = [item for item in merge_node_item["items"] if not item.exclude]
|
|
220
|
+
|
|
221
|
+
all_eligible_candidates = list(local_eligible)
|
|
222
|
+
for _, g_items in child_unfinalized:
|
|
223
|
+
all_eligible_candidates.extend(g_items)
|
|
224
|
+
|
|
225
|
+
# We can merge under the current node if depth > 0 and we are combining multiple sources:
|
|
226
|
+
# e.g., local files + child groups, or multiple child groups, or multiple local files.
|
|
227
|
+
can_merge_here = (depth > 0) and (len(local_eligible) + len(child_unfinalized) > 1)
|
|
228
|
+
|
|
229
|
+
if can_merge_here:
|
|
230
|
+
if _can_merge_folder(all_eligible_candidates):
|
|
231
|
+
new_finalized = child_finalized + local_excluded
|
|
232
|
+
new_unfinalized = [(merge_path, all_eligible_candidates)]
|
|
233
|
+
return new_finalized, new_unfinalized
|
|
234
|
+
else:
|
|
235
|
+
# Compatibility broke at this level. We must finalize the subtrees.
|
|
236
|
+
finalized_child_groups = []
|
|
237
|
+
for c_path, c_items in child_unfinalized:
|
|
238
|
+
if len(c_items) > 1 and _can_merge_folder(c_items):
|
|
239
|
+
finalized_child_groups.append(_create_merged_item(c_items, c_path))
|
|
240
|
+
else:
|
|
241
|
+
finalized_child_groups.extend(c_items)
|
|
242
|
+
|
|
243
|
+
finalized_local = []
|
|
244
|
+
if len(local_eligible) > 1 and _can_merge_folder(local_eligible):
|
|
245
|
+
finalized_local.append(_create_merged_item(local_eligible, merge_path))
|
|
246
|
+
else:
|
|
247
|
+
finalized_local.extend(local_eligible)
|
|
248
|
+
|
|
249
|
+
new_finalized = child_finalized + local_excluded + finalized_child_groups + finalized_local
|
|
250
|
+
new_unfinalized = []
|
|
251
|
+
return new_finalized, new_unfinalized
|
|
252
|
+
else:
|
|
253
|
+
# We cannot merge or don't need to merge at this level (e.g. depth == 0 or only 1 source).
|
|
254
|
+
# We propagate everything up as unfinalized.
|
|
255
|
+
new_finalized = child_finalized + local_excluded
|
|
256
|
+
new_unfinalized = list(child_unfinalized)
|
|
257
|
+
if local_eligible:
|
|
258
|
+
new_unfinalized.append((merge_path, local_eligible))
|
|
259
|
+
return new_finalized, new_unfinalized
|
|
260
|
+
|
|
261
|
+
fin, unfin = merge_node(merge_tree)
|
|
262
|
+
finalized_results = list(fin)
|
|
263
|
+
for path, items in unfin:
|
|
264
|
+
if path and len(items) > 1 and _can_merge_folder(items):
|
|
265
|
+
finalized_results.append(_create_merged_item(items, path))
|
|
266
|
+
else:
|
|
267
|
+
finalized_results.extend(items)
|
|
268
|
+
|
|
269
|
+
return finalized_results
|
|
@@ -48,7 +48,6 @@ class SourceItem(FileItem):
|
|
|
48
48
|
self.source_name_or_path = value
|
|
49
49
|
self.is_license_text = False
|
|
50
50
|
self.is_manifest_file = False
|
|
51
|
-
self.license_reference = ""
|
|
52
51
|
self.scanoss_reference = {}
|
|
53
52
|
self.matched_lines = "" # Only for SCANOSS results
|
|
54
53
|
self.fileURL = "" # Only for SCANOSS results
|
|
@@ -206,8 +205,7 @@ class SourceItem(FileItem):
|
|
|
206
205
|
for item in self.oss_items:
|
|
207
206
|
print_rows.append([self.source_name_or_path, item.name, item.version, ",".join(item.license),
|
|
208
207
|
item.download_location, "",
|
|
209
|
-
item.copyright, "Exclude" if self.exclude else "", item.comment
|
|
210
|
-
self.license_reference])
|
|
208
|
+
item.copyright, "Exclude" if self.exclude else "", item.comment])
|
|
211
209
|
return print_rows
|
|
212
210
|
|
|
213
211
|
def __eq__(self, other: object) -> bool:
|
|
@@ -12,12 +12,12 @@ import logging
|
|
|
12
12
|
import re
|
|
13
13
|
import urllib.request
|
|
14
14
|
import urllib.error
|
|
15
|
-
from datetime import datetime
|
|
16
15
|
import fosslight_util.constant as constant
|
|
17
16
|
from fosslight_util.set_log import init_log
|
|
18
17
|
from ._help import print_version, print_help_msg_source_scanner
|
|
19
18
|
from ._license_matched import get_license_list_to_print
|
|
20
19
|
from fosslight_util.output_format import check_output_formats_v2, write_output_file
|
|
20
|
+
from fosslight_util.write_excel import get_header_row
|
|
21
21
|
from fosslight_util.correct import correct_with_yaml
|
|
22
22
|
from fosslight_util.parsing_yaml import SUPPORT_OSS_INFO_FILES
|
|
23
23
|
from .run_scancode import run_scan
|
|
@@ -31,19 +31,20 @@ from .run_spdx_extractor import get_spdx_downloads
|
|
|
31
31
|
from .run_manifest_extractor import get_manifest_licenses
|
|
32
32
|
from ._scan_item import SourceItem, resolve_kb_config, is_notice_file
|
|
33
33
|
from ._kb_client import fetch_origin_urls_via_scan_job
|
|
34
|
+
from fosslight_util.cover import dump_result_log
|
|
35
|
+
from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename
|
|
34
36
|
from fosslight_util.oss_item import ScannerItem
|
|
35
37
|
from typing import Optional, Tuple
|
|
36
38
|
from ._scan_item import is_manifest_file
|
|
37
39
|
import shutil
|
|
40
|
+
from ._merge import (
|
|
41
|
+
_add_pre_merge_sheet,
|
|
42
|
+
merge_results_by_folder
|
|
43
|
+
)
|
|
38
44
|
|
|
39
45
|
|
|
40
46
|
SRC_SHEET_NAME = 'SRC_FL_Source'
|
|
41
|
-
|
|
42
|
-
'OSS Version', 'License', 'Download Location',
|
|
43
|
-
'Homepage', 'Copyright Text', 'Exclude', 'Comment']}
|
|
44
|
-
MERGED_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
|
|
45
|
-
'OSS Version', 'License', 'Download Location',
|
|
46
|
-
'Homepage', 'Copyright Text', 'Exclude', 'Comment', 'license_reference']}
|
|
47
|
+
PRE_MERGE_SHEET_NAME = '.SRC_FL_Source_no_merge'
|
|
47
48
|
KB_REFERENCE_HEADER = ['ID', 'Source Path', 'KB Origin URL', 'Evidence']
|
|
48
49
|
ALL_MODE = 'all'
|
|
49
50
|
SCANNER_TYPE = ['kb', 'scancode', 'scanoss', ALL_MODE]
|
|
@@ -88,6 +89,7 @@ def main() -> None:
|
|
|
88
89
|
parser.add_argument('--hide_progress', action='store_true', required=False)
|
|
89
90
|
parser.add_argument('--kb_url', type=str, required=False, default="")
|
|
90
91
|
parser.add_argument('--kb_token', type=str, required=False, default="")
|
|
92
|
+
parser.add_argument('--no_merge', action='store_true', required=False)
|
|
91
93
|
|
|
92
94
|
args = parser.parse_args()
|
|
93
95
|
|
|
@@ -118,19 +120,19 @@ def main() -> None:
|
|
|
118
120
|
hide_progress = args.hide_progress
|
|
119
121
|
kb_url = args.kb_url
|
|
120
122
|
kb_token = args.kb_token
|
|
123
|
+
merge_by_folder = not args.no_merge
|
|
121
124
|
|
|
122
125
|
time_out = args.timeout
|
|
123
126
|
core = args.cores
|
|
124
127
|
|
|
125
128
|
if os.path.isdir(path_to_scan):
|
|
126
|
-
result = []
|
|
127
129
|
result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
|
|
128
130
|
print_matched_text, formats, time_out, correct_mode, correct_filepath,
|
|
129
131
|
selected_scanner, path_to_exclude, hide_progress=hide_progress,
|
|
130
|
-
kb_url=kb_url, kb_token=kb_token
|
|
132
|
+
kb_url=kb_url, kb_token=kb_token,
|
|
133
|
+
merge_by_folder=merge_by_folder)
|
|
131
134
|
|
|
132
135
|
_result_log["Scan Result"] = result[1]
|
|
133
|
-
|
|
134
136
|
try:
|
|
135
137
|
logger.info(yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True))
|
|
136
138
|
except Exception as ex:
|
|
@@ -148,7 +150,7 @@ def create_report_file(
|
|
|
148
150
|
output_extensions: list = [], correct_mode: bool = True,
|
|
149
151
|
correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [],
|
|
150
152
|
formats: list = [], api_limit_exceed: bool = False, files_count: int = 0, final_output_path: str = "",
|
|
151
|
-
run_kb_msg: str = ""
|
|
153
|
+
run_kb_msg: str = "", merge_by_folder: bool = True
|
|
152
154
|
) -> 'ScannerItem':
|
|
153
155
|
"""
|
|
154
156
|
Create report files for given scanned result.
|
|
@@ -158,9 +160,9 @@ def create_report_file(
|
|
|
158
160
|
:param license_list: matched text (only for scancode).
|
|
159
161
|
:param need_license: if requested, output matched text (only for scancode).
|
|
160
162
|
"""
|
|
161
|
-
extended_header = {}
|
|
162
163
|
sheet_list = {}
|
|
163
164
|
_json_ext = ".json"
|
|
165
|
+
name_time = timestamp_for_filename(_start_time)
|
|
164
166
|
|
|
165
167
|
output_path = os.path.abspath(output_path)
|
|
166
168
|
|
|
@@ -178,19 +180,19 @@ def create_report_file(
|
|
|
178
180
|
to_remove.append(i)
|
|
179
181
|
else:
|
|
180
182
|
if formats[i].startswith('spdx'):
|
|
181
|
-
output_files[i] = f"fosslight_spdx_src_{
|
|
183
|
+
output_files[i] = f"fosslight_spdx_src_{name_time}"
|
|
182
184
|
elif formats[i].startswith('cyclonedx'):
|
|
183
|
-
output_files[i] = f'fosslight_cyclonedx_src_{
|
|
185
|
+
output_files[i] = f'fosslight_cyclonedx_src_{name_time}'
|
|
184
186
|
else:
|
|
185
187
|
if output_extension == _json_ext:
|
|
186
|
-
output_files[i] = f"fosslight_opossum_src_{
|
|
188
|
+
output_files[i] = f"fosslight_opossum_src_{name_time}"
|
|
187
189
|
else:
|
|
188
|
-
output_files[i] = f"fosslight_report_src_{
|
|
190
|
+
output_files[i] = f"fosslight_report_src_{name_time}"
|
|
189
191
|
else:
|
|
190
192
|
if output_extension == _json_ext:
|
|
191
|
-
output_files[i] = f"fosslight_opossum_src_{
|
|
193
|
+
output_files[i] = f"fosslight_opossum_src_{name_time}"
|
|
192
194
|
else:
|
|
193
|
-
output_files[i] = f"fosslight_report_src_{
|
|
195
|
+
output_files[i] = f"fosslight_report_src_{name_time}"
|
|
194
196
|
for index in sorted(to_remove, reverse=True):
|
|
195
197
|
# remove elements of spdx format on windows
|
|
196
198
|
del output_files[index]
|
|
@@ -228,11 +230,6 @@ def create_report_file(
|
|
|
228
230
|
sheet_list = {}
|
|
229
231
|
scan_item.append_file_items(merged_result, PKG_NAME)
|
|
230
232
|
|
|
231
|
-
if selected_scanner == 'scanoss':
|
|
232
|
-
extended_header = SCANOSS_HEADER
|
|
233
|
-
else:
|
|
234
|
-
extended_header = MERGED_HEADER
|
|
235
|
-
|
|
236
233
|
if need_license:
|
|
237
234
|
if selected_scanner == 'scancode':
|
|
238
235
|
sheet_list["scancode_reference"] = get_license_list_to_print(license_list)
|
|
@@ -258,12 +255,24 @@ def create_report_file(
|
|
|
258
255
|
scan_item = correct_item
|
|
259
256
|
logger.info("Success to correct with yaml.")
|
|
260
257
|
|
|
258
|
+
finish_time = current_timestamp_utc()
|
|
259
|
+
scan_item.set_cover_finish_time(finish_time)
|
|
260
|
+
|
|
261
|
+
if merged_result and merge_by_folder:
|
|
262
|
+
_add_pre_merge_sheet(scan_item, PRE_MERGE_SHEET_NAME, get_header_row(SRC_SHEET_NAME), PKG_NAME)
|
|
263
|
+
_merge_start = time.time()
|
|
264
|
+
scan_item.file_items[PKG_NAME] = merge_results_by_folder(scan_item.file_items[PKG_NAME])
|
|
265
|
+
logger.debug(f"[TIMING] merge_results_by_folder: {time.time() - _merge_start:.4f}s")
|
|
266
|
+
|
|
261
267
|
combined_paths_and_files = [os.path.join(output_path, file) for file in output_files]
|
|
262
268
|
results = []
|
|
263
269
|
for combined_path_and_file, output_extension, output_format in zip(combined_paths_and_files, output_extensions, formats):
|
|
264
270
|
# if need_license and output_extension == _json_ext and "scanoss_reference" in sheet_list:
|
|
265
271
|
# del sheet_list["scanoss_reference"]
|
|
266
|
-
|
|
272
|
+
result = write_output_file(
|
|
273
|
+
combined_path_and_file, output_extension, scan_item, hide_header="", format=output_format
|
|
274
|
+
)
|
|
275
|
+
results.append(result)
|
|
267
276
|
for success, msg, result_file in results:
|
|
268
277
|
final_result_file = result_file.replace(output_path, final_output_path)
|
|
269
278
|
if success:
|
|
@@ -492,7 +501,8 @@ def run_scanners(
|
|
|
492
501
|
formats: list = [], time_out: int = 120,
|
|
493
502
|
correct_mode: bool = True, correct_filepath: str = "",
|
|
494
503
|
selected_scanner: str = ALL_MODE, path_to_exclude: list = [],
|
|
495
|
-
all_exclude_mode: tuple = (), hide_progress: bool = False,
|
|
504
|
+
all_exclude_mode: tuple = (), hide_progress: bool = False,
|
|
505
|
+
kb_url: str = "", kb_token: str = "", merge_by_folder: bool = True
|
|
496
506
|
) -> Tuple[bool, str, 'ScannerItem', list, list]:
|
|
497
507
|
"""
|
|
498
508
|
Run Scancode and scanoss.py for the given path.
|
|
@@ -513,7 +523,8 @@ def run_scanners(
|
|
|
513
523
|
"""
|
|
514
524
|
global logger
|
|
515
525
|
|
|
516
|
-
start_time =
|
|
526
|
+
start_time = current_timestamp_utc()
|
|
527
|
+
file_time = timestamp_for_filename(start_time)
|
|
517
528
|
scancode_result = []
|
|
518
529
|
scanoss_result = []
|
|
519
530
|
merged_result = []
|
|
@@ -529,13 +540,13 @@ def run_scanners(
|
|
|
529
540
|
if output_path == "":
|
|
530
541
|
output_path = os.getcwd()
|
|
531
542
|
final_output_path = output_path
|
|
532
|
-
output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{
|
|
543
|
+
output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{file_time}')
|
|
533
544
|
publish_temp_output = False
|
|
534
545
|
logger = None
|
|
535
546
|
publish_ok = True
|
|
536
547
|
|
|
537
548
|
try:
|
|
538
|
-
logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{
|
|
549
|
+
logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{file_time}.txt"),
|
|
539
550
|
True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
|
|
540
551
|
|
|
541
552
|
logger.info(f"Tool Info : {result_log['Tool Info']}")
|
|
@@ -545,6 +556,11 @@ def run_scanners(
|
|
|
545
556
|
print_matched_text = False
|
|
546
557
|
|
|
547
558
|
if success:
|
|
559
|
+
has_sbom_format = any(f.startswith('spdx') or f.startswith('cyclonedx') for f in formats) if formats else False
|
|
560
|
+
if has_sbom_format and merge_by_folder:
|
|
561
|
+
logger.info("SPDX/CycloneDX format does not support merge. Merging is not performed.")
|
|
562
|
+
merge_by_folder = False
|
|
563
|
+
|
|
548
564
|
if all_exclude_mode and len(all_exclude_mode) == 4:
|
|
549
565
|
(excluded_path_with_default_exclusion,
|
|
550
566
|
excluded_path_without_dot,
|
|
@@ -598,7 +614,8 @@ def run_scanners(
|
|
|
598
614
|
scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
|
|
599
615
|
print_matched_text, output_path, output_files, output_extensions, correct_mode,
|
|
600
616
|
correct_filepath, path_to_scan, excluded_path_without_dot, formats,
|
|
601
|
-
api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg
|
|
617
|
+
api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg,
|
|
618
|
+
merge_by_folder)
|
|
602
619
|
else:
|
|
603
620
|
print_help_msg_source_scanner()
|
|
604
621
|
result_log[RESULT_KEY] = "Unsupported scanner"
|
|
@@ -619,6 +636,17 @@ def run_scanners(
|
|
|
619
636
|
else "Failed to publish scan artifacts"
|
|
620
637
|
)
|
|
621
638
|
|
|
639
|
+
if logger:
|
|
640
|
+
if scan_item and not isinstance(scan_item, list):
|
|
641
|
+
result_log["Running time"] = scan_item.cover.running_time
|
|
642
|
+
else:
|
|
643
|
+
finish_time = current_timestamp_utc()
|
|
644
|
+
result_log["Running time"] = format_running_time(start_time, finish_time)
|
|
645
|
+
try:
|
|
646
|
+
logger.info(dump_result_log(result_log))
|
|
647
|
+
except Exception as ex:
|
|
648
|
+
logger.warning(f"Failed to print result log. {ex}")
|
|
649
|
+
|
|
622
650
|
return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result
|
|
623
651
|
|
|
624
652
|
|
|
@@ -7,11 +7,10 @@ import os
|
|
|
7
7
|
import multiprocessing
|
|
8
8
|
import warnings
|
|
9
9
|
import logging
|
|
10
|
-
import yaml
|
|
11
10
|
from scancode import cli
|
|
12
|
-
from datetime import datetime
|
|
13
11
|
import fosslight_util.constant as constant
|
|
14
12
|
from fosslight_util.set_log import init_log
|
|
13
|
+
from fosslight_util.time import current_timestamp_utc, timestamp_for_filename
|
|
15
14
|
from ._parsing_scancode_file_item import parsing_file_item
|
|
16
15
|
from ._parsing_scancode_file_item import get_error_from_header
|
|
17
16
|
from fosslight_util.output_format import check_output_formats_v2
|
|
@@ -168,12 +167,12 @@ def run_scan(
|
|
|
168
167
|
|
|
169
168
|
success = True
|
|
170
169
|
msg = ""
|
|
171
|
-
_str_final_result_log = ""
|
|
172
170
|
_result_log = {}
|
|
173
171
|
result_list = []
|
|
174
172
|
license_list = []
|
|
175
173
|
_json_ext = ".json"
|
|
176
|
-
_start_time =
|
|
174
|
+
_start_time = current_timestamp_utc()
|
|
175
|
+
_file_time = timestamp_for_filename(_start_time)
|
|
177
176
|
|
|
178
177
|
if not correct_filepath:
|
|
179
178
|
correct_filepath = path_to_scan
|
|
@@ -189,9 +188,9 @@ def run_scan(
|
|
|
189
188
|
for i, output_extension in enumerate(output_extensions):
|
|
190
189
|
if output_files[i] is None or output_files[i] == "":
|
|
191
190
|
if output_extension == _json_ext:
|
|
192
|
-
output_files[i] = f"fosslight_opossum_src_{
|
|
191
|
+
output_files[i] = f"fosslight_opossum_src_{_file_time}"
|
|
193
192
|
else:
|
|
194
|
-
output_files[i] = f"fosslight_report_src_{
|
|
193
|
+
output_files[i] = f"fosslight_report_src_{_file_time}"
|
|
195
194
|
|
|
196
195
|
if _write_json_file:
|
|
197
196
|
output_json_file = os.path.join(output_path, "scancode_raw_result.json")
|
|
@@ -199,7 +198,7 @@ def run_scan(
|
|
|
199
198
|
output_json_file = ""
|
|
200
199
|
|
|
201
200
|
if not called_by_cli:
|
|
202
|
-
logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{
|
|
201
|
+
logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_file_time}.txt"),
|
|
203
202
|
True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
|
|
204
203
|
|
|
205
204
|
logger.info(f"Tool Info : {_result_log['Tool Info']}")
|
|
@@ -292,11 +291,6 @@ def run_scan(
|
|
|
292
291
|
scan_result_msg = str(success) if msg == "" else f"{success}, {msg}"
|
|
293
292
|
_result_log["Scan Result"] = scan_result_msg
|
|
294
293
|
_result_log["Output Directory"] = output_path
|
|
295
|
-
try:
|
|
296
|
-
_str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)
|
|
297
|
-
logger.info(_str_final_result_log)
|
|
298
|
-
except Exception as ex:
|
|
299
|
-
logger.warning(f"Failed to print result log. {ex}")
|
|
300
294
|
|
|
301
295
|
if not success:
|
|
302
296
|
logger.error(f"Failed to run: {scan_result_msg}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fosslight_source
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.3
|
|
4
4
|
Summary: FOSSLight Source Scanner
|
|
5
5
|
Author: LG Electronics
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -20,7 +20,7 @@ Requires-Dist: setuptools<=80.10.2
|
|
|
20
20
|
Requires-Dist: pyparsing
|
|
21
21
|
Requires-Dist: scanoss>=1.19.0
|
|
22
22
|
Requires-Dist: XlsxWriter
|
|
23
|
-
Requires-Dist: fosslight_util>=2.
|
|
23
|
+
Requires-Dist: fosslight_util>=2.2.2
|
|
24
24
|
Requires-Dist: PyYAML
|
|
25
25
|
Requires-Dist: wheel>=0.38.1
|
|
26
26
|
Requires-Dist: intbitset
|
|
@@ -6,6 +6,7 @@ src/fosslight_source/__init__.py
|
|
|
6
6
|
src/fosslight_source/_help.py
|
|
7
7
|
src/fosslight_source/_kb_client.py
|
|
8
8
|
src/fosslight_source/_license_matched.py
|
|
9
|
+
src/fosslight_source/_merge.py
|
|
9
10
|
src/fosslight_source/_parsing_scancode_file_item.py
|
|
10
11
|
src/fosslight_source/_parsing_scanoss_file.py
|
|
11
12
|
src/fosslight_source/_scan_item.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_parsing_scanoss_file.py
RENAMED
|
File without changes
|
{fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_manifest_extractor.py
RENAMED
|
File without changes
|
|
File without changes
|
{fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_spdx_extractor.py
RENAMED
|
File without changes
|
{fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|