fosslight-source 2.3.2__tar.gz → 2.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {fosslight_source-2.3.2/src/fosslight_source.egg-info → fosslight_source-2.3.3}/PKG-INFO +2 -2
  2. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/pyproject.toml +2 -2
  3. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_help.py +1 -0
  4. fosslight_source-2.3.3/src/fosslight_source/_merge.py +269 -0
  5. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_scan_item.py +1 -3
  6. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/cli.py +57 -29
  7. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_scancode.py +6 -12
  8. {fosslight_source-2.3.2 → fosslight_source-2.3.3/src/fosslight_source.egg-info}/PKG-INFO +2 -2
  9. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/SOURCES.txt +1 -0
  10. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/requires.txt +1 -1
  11. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/LICENSE +0 -0
  12. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/MANIFEST.in +0 -0
  13. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/README.md +0 -0
  14. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/setup.cfg +0 -0
  15. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/__init__.py +0 -0
  16. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_kb_client.py +0 -0
  17. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_license_matched.py +0 -0
  18. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_parsing_scancode_file_item.py +0 -0
  19. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/_parsing_scanoss_file.py +0 -0
  20. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_manifest_extractor.py +0 -0
  21. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_scanoss.py +0 -0
  22. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source/run_spdx_extractor.py +0 -0
  23. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/dependency_links.txt +0 -0
  24. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/entry_points.txt +0 -0
  25. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/src/fosslight_source.egg-info/top_level.txt +0 -0
  26. {fosslight_source-2.3.2 → fosslight_source-2.3.3}/tests/test_tox.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fosslight_source
3
- Version: 2.3.2
3
+ Version: 2.3.3
4
4
  Summary: FOSSLight Source Scanner
5
5
  Author: LG Electronics
6
6
  License-Expression: Apache-2.0
@@ -20,7 +20,7 @@ Requires-Dist: setuptools<=80.10.2
20
20
  Requires-Dist: pyparsing
21
21
  Requires-Dist: scanoss>=1.19.0
22
22
  Requires-Dist: XlsxWriter
23
- Requires-Dist: fosslight_util>=2.1.37
23
+ Requires-Dist: fosslight_util>=2.2.2
24
24
  Requires-Dist: PyYAML
25
25
  Requires-Dist: wheel>=0.38.1
26
26
  Requires-Dist: intbitset
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
7
7
 
8
8
  [project]
9
9
  name = "fosslight_source"
10
- version = "2.3.2"
10
+ version = "2.3.3"
11
11
  description = "FOSSLight Source Scanner"
12
12
  readme = "README.md"
13
13
  license = "Apache-2.0"
@@ -29,7 +29,7 @@ dependencies = [
29
29
  "pyparsing",
30
30
  "scanoss>=1.19.0",
31
31
  "XlsxWriter",
32
- "fosslight_util>=2.1.37",
32
+ "fosslight_util>=2.2.2",
33
33
  "PyYAML",
34
34
  "wheel>=0.38.1",
35
35
  "intbitset",
@@ -39,6 +39,7 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
39
39
  -c <number> Number of CPU cores/threads to use for scanning
40
40
  -t <seconds> Timeout in seconds for ScanCode scanning
41
41
  -j Generate raw scanner results in JSON format
42
+ --no_merge Keep source paths file-based without folder merge
42
43
  --no_correction Skip OSS information correction with sbom-info.yaml
43
44
  --correct_fpath <path> Path to custom sbom-info.yaml file
44
45
  --hide_progress Hide the progress bar during scanning
@@ -0,0 +1,269 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2020 LG Electronics Inc.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ import os
7
+ from collections import Counter
8
+
9
+ from fosslight_util.constant import COMMENT_DELIMITER
10
+
11
+ from ._scan_item import SourceItem
12
+
13
+
14
+ def _get_source_rows_to_print(source_items: list) -> list:
15
+ source_rows = []
16
+ for source_item in source_items:
17
+ source_rows.extend(source_item.get_print_array())
18
+ return source_rows
19
+
20
+
21
+ def _add_pre_merge_sheet(scan_item, pre_merge_sheet_name: str, header_row: list, pkg_name: str) -> None:
22
+ external_sheets = getattr(scan_item, "external_sheets", {}) or {}
23
+ external_sheets[pre_merge_sheet_name] = [
24
+ header_row,
25
+ *_get_source_rows_to_print(scan_item.file_items.get(pkg_name, [])),
26
+ ]
27
+ scan_item.external_sheets = external_sheets
28
+
29
+
30
+ def _normalize_merge_text(value: str) -> str:
31
+ return value.strip() if value else ""
32
+
33
+
34
+ def _iter_merge_values(values) -> list:
35
+ if not values:
36
+ return []
37
+ if isinstance(values, str):
38
+ return [values]
39
+ return values
40
+
41
+
42
+ def _normalize_merge_licenses(licenses) -> tuple:
43
+ return tuple(sorted([lic.strip() for lic in _iter_merge_values(licenses) if lic and lic.strip()]))
44
+
45
+
46
+ def _normalize_merge_downloads(downloads) -> tuple:
47
+ if not downloads:
48
+ return ()
49
+ if isinstance(downloads, str):
50
+ downloads = [downloads]
51
+ return tuple(sorted([dl.strip() for dl in downloads if dl and dl.strip()]))
52
+
53
+
54
+ def _iter_merge_rows(item: SourceItem):
55
+ if item.oss_items:
56
+ for oss_item in item.oss_items:
57
+ name = _normalize_merge_text(oss_item.name) or _normalize_merge_text(item.oss_name)
58
+ version = _normalize_merge_text(oss_item.version) or _normalize_merge_text(item.oss_version)
59
+ licenses = _normalize_merge_licenses(oss_item.license or item.licenses)
60
+ downloads = _normalize_merge_downloads(oss_item.download_location or item.download_location)
61
+ yield name, version, licenses, downloads
62
+ else:
63
+ yield (
64
+ _normalize_merge_text(item.oss_name),
65
+ _normalize_merge_text(item.oss_version),
66
+ _normalize_merge_licenses(item.licenses),
67
+ _normalize_merge_downloads(item.download_location),
68
+ )
69
+
70
+
71
+ def _get_item_oss_name(item: SourceItem) -> str:
72
+ for name, _, _, _ in _iter_merge_rows(item):
73
+ if name:
74
+ return name
75
+ return ""
76
+
77
+
78
+ def _get_item_oss_version(item: SourceItem) -> str:
79
+ for _, version, _, _ in _iter_merge_rows(item):
80
+ if version:
81
+ return version
82
+ return ""
83
+
84
+
85
+ def _get_merge_licenses(scan_item: SourceItem) -> tuple:
86
+ for _, _, licenses, _ in _iter_merge_rows(scan_item):
87
+ if licenses:
88
+ return licenses
89
+ return ()
90
+
91
+
92
+ def _get_merge_download_locations(scan_item: SourceItem) -> tuple:
93
+ for _, _, _, downloads in _iter_merge_rows(scan_item):
94
+ if downloads:
95
+ return downloads
96
+ return ()
97
+
98
+
99
+ def _get_merge_field_value(scan_items: list, value_getter):
100
+ for scan_item in scan_items:
101
+ value = value_getter(scan_item)
102
+ if value:
103
+ return value
104
+ return ""
105
+
106
+
107
+ def _get_top_merge_values(scan_items: list, value_getter) -> list:
108
+ values = []
109
+ for scan_item in scan_items:
110
+ for value in _iter_merge_values(value_getter(scan_item)):
111
+ normalized_value = _normalize_merge_text(value)
112
+ if normalized_value:
113
+ values.append(normalized_value)
114
+ return [value for value, _ in Counter(values).most_common(3)]
115
+
116
+
117
+ def _can_merge_folder(scan_items: list) -> bool:
118
+ if len(scan_items) <= 1:
119
+ return False
120
+
121
+ ref_name = None
122
+ ref_version = None
123
+ ref_licenses = None
124
+ ref_downloads = None
125
+
126
+ for item in scan_items:
127
+ for name, version, licenses, downloads in _iter_merge_rows(item):
128
+ if name:
129
+ if ref_name is None:
130
+ ref_name = name
131
+ elif name != ref_name:
132
+ return False
133
+
134
+ if version:
135
+ if ref_version is None:
136
+ ref_version = version
137
+ elif version != ref_version:
138
+ return False
139
+
140
+ if licenses:
141
+ if ref_licenses is None:
142
+ ref_licenses = licenses
143
+ elif licenses != ref_licenses:
144
+ return False
145
+
146
+ if downloads:
147
+ if ref_downloads is None:
148
+ ref_downloads = downloads
149
+ elif downloads != ref_downloads:
150
+ return False
151
+
152
+ return True
153
+
154
+
155
+ def _get_merged_comments(scan_items: list) -> str:
156
+ comments = []
157
+ for item in scan_items:
158
+ val = item.comment
159
+ if val:
160
+ parts = [p.strip() for p in val.split(COMMENT_DELIMITER) if p.strip()]
161
+ for p in parts:
162
+ if p not in comments:
163
+ comments.append(p)
164
+ if not comments:
165
+ return ""
166
+
167
+ return COMMENT_DELIMITER.join(comments)
168
+
169
+
170
+ def _create_merged_item(scan_items: list, merge_path: str) -> SourceItem:
171
+ merged_path = f"{merge_path} ({len(scan_items)})"
172
+ merged_item = SourceItem(merged_path)
173
+ merged_item.oss_name = _get_merge_field_value(scan_items, lambda item: _normalize_merge_text(_get_item_oss_name(item)))
174
+ merged_item.oss_version = _get_merge_field_value(scan_items, lambda item: _normalize_merge_text(_get_item_oss_version(item)))
175
+ merged_licenses = _get_merge_field_value(scan_items, _get_merge_licenses)
176
+ merged_item.licenses = list(merged_licenses) if merged_licenses else []
177
+ merged_downloads = _get_merge_field_value(scan_items, _get_merge_download_locations)
178
+ merged_item.download_location = list(merged_downloads) if merged_downloads else []
179
+ merged_copyrights = _get_top_merge_values(scan_items, lambda item: item.copyright)
180
+ merged_item.copyright = merged_copyrights if merged_copyrights else []
181
+ merged_comments = _get_merged_comments(scan_items)
182
+ if merged_comments:
183
+ merged_item.comment = merged_comments
184
+ merged_item.set_oss_item()
185
+ return merged_item
186
+
187
+
188
+ def merge_results_by_folder(scan_result: list) -> list:
189
+ """
190
+ Merge output rows within the same folder when OSS name, OSS version, license,
191
+ and download location are compatible.
192
+
193
+ A field is compatible when all non-empty values across rows are identical
194
+ (empty values are ignored). Each oss_item is treated as its own row. All eligible
195
+ rows in the folder must be compatible together; rows are not grouped by key subsets.
196
+ """
197
+ # Build a folder tree first so merge never jumps straight to root ".".
198
+ merge_tree = {"items": [], "children": {}}
199
+
200
+ for scan_item in scan_result:
201
+ normalized_path = os.path.normpath(scan_item.source_name_or_path).replace("\\", "/")
202
+ path_parts = [part for part in normalized_path.split("/") if part and part != "."]
203
+ current_node = merge_tree
204
+
205
+ for folder_name in path_parts[:-1]:
206
+ current_node = current_node["children"].setdefault(folder_name, {"items": [], "children": {}})
207
+ current_node["items"].append(scan_item)
208
+
209
+ def merge_node(merge_node_item: dict, merge_path: str = "", depth: int = 0) -> tuple:
210
+ child_finalized = []
211
+ child_unfinalized = []
212
+ for folder_name, child_node in merge_node_item["children"].items():
213
+ child_path = f"{merge_path}/{folder_name}" if merge_path else folder_name
214
+ fin, unfin = merge_node(child_node, child_path, depth + 1)
215
+ child_finalized.extend(fin)
216
+ child_unfinalized.extend(unfin)
217
+
218
+ local_excluded = [item for item in merge_node_item["items"] if item.exclude]
219
+ local_eligible = [item for item in merge_node_item["items"] if not item.exclude]
220
+
221
+ all_eligible_candidates = list(local_eligible)
222
+ for _, g_items in child_unfinalized:
223
+ all_eligible_candidates.extend(g_items)
224
+
225
+ # We can merge under the current node if depth > 0 and we are combining multiple sources:
226
+ # e.g., local files + child groups, or multiple child groups, or multiple local files.
227
+ can_merge_here = (depth > 0) and (len(local_eligible) + len(child_unfinalized) > 1)
228
+
229
+ if can_merge_here:
230
+ if _can_merge_folder(all_eligible_candidates):
231
+ new_finalized = child_finalized + local_excluded
232
+ new_unfinalized = [(merge_path, all_eligible_candidates)]
233
+ return new_finalized, new_unfinalized
234
+ else:
235
+ # Compatibility broke at this level. We must finalize the subtrees.
236
+ finalized_child_groups = []
237
+ for c_path, c_items in child_unfinalized:
238
+ if len(c_items) > 1 and _can_merge_folder(c_items):
239
+ finalized_child_groups.append(_create_merged_item(c_items, c_path))
240
+ else:
241
+ finalized_child_groups.extend(c_items)
242
+
243
+ finalized_local = []
244
+ if len(local_eligible) > 1 and _can_merge_folder(local_eligible):
245
+ finalized_local.append(_create_merged_item(local_eligible, merge_path))
246
+ else:
247
+ finalized_local.extend(local_eligible)
248
+
249
+ new_finalized = child_finalized + local_excluded + finalized_child_groups + finalized_local
250
+ new_unfinalized = []
251
+ return new_finalized, new_unfinalized
252
+ else:
253
+ # We cannot merge or don't need to merge at this level (e.g. depth == 0 or only 1 source).
254
+ # We propagate everything up as unfinalized.
255
+ new_finalized = child_finalized + local_excluded
256
+ new_unfinalized = list(child_unfinalized)
257
+ if local_eligible:
258
+ new_unfinalized.append((merge_path, local_eligible))
259
+ return new_finalized, new_unfinalized
260
+
261
+ fin, unfin = merge_node(merge_tree)
262
+ finalized_results = list(fin)
263
+ for path, items in unfin:
264
+ if path and len(items) > 1 and _can_merge_folder(items):
265
+ finalized_results.append(_create_merged_item(items, path))
266
+ else:
267
+ finalized_results.extend(items)
268
+
269
+ return finalized_results
@@ -48,7 +48,6 @@ class SourceItem(FileItem):
48
48
  self.source_name_or_path = value
49
49
  self.is_license_text = False
50
50
  self.is_manifest_file = False
51
- self.license_reference = ""
52
51
  self.scanoss_reference = {}
53
52
  self.matched_lines = "" # Only for SCANOSS results
54
53
  self.fileURL = "" # Only for SCANOSS results
@@ -206,8 +205,7 @@ class SourceItem(FileItem):
206
205
  for item in self.oss_items:
207
206
  print_rows.append([self.source_name_or_path, item.name, item.version, ",".join(item.license),
208
207
  item.download_location, "",
209
- item.copyright, "Exclude" if self.exclude else "", item.comment,
210
- self.license_reference])
208
+ item.copyright, "Exclude" if self.exclude else "", item.comment])
211
209
  return print_rows
212
210
 
213
211
  def __eq__(self, other: object) -> bool:
@@ -12,12 +12,12 @@ import logging
12
12
  import re
13
13
  import urllib.request
14
14
  import urllib.error
15
- from datetime import datetime
16
15
  import fosslight_util.constant as constant
17
16
  from fosslight_util.set_log import init_log
18
17
  from ._help import print_version, print_help_msg_source_scanner
19
18
  from ._license_matched import get_license_list_to_print
20
19
  from fosslight_util.output_format import check_output_formats_v2, write_output_file
20
+ from fosslight_util.write_excel import get_header_row
21
21
  from fosslight_util.correct import correct_with_yaml
22
22
  from fosslight_util.parsing_yaml import SUPPORT_OSS_INFO_FILES
23
23
  from .run_scancode import run_scan
@@ -31,19 +31,20 @@ from .run_spdx_extractor import get_spdx_downloads
31
31
  from .run_manifest_extractor import get_manifest_licenses
32
32
  from ._scan_item import SourceItem, resolve_kb_config, is_notice_file
33
33
  from ._kb_client import fetch_origin_urls_via_scan_job
34
+ from fosslight_util.cover import dump_result_log
35
+ from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename
34
36
  from fosslight_util.oss_item import ScannerItem
35
37
  from typing import Optional, Tuple
36
38
  from ._scan_item import is_manifest_file
37
39
  import shutil
40
+ from ._merge import (
41
+ _add_pre_merge_sheet,
42
+ merge_results_by_folder
43
+ )
38
44
 
39
45
 
40
46
  SRC_SHEET_NAME = 'SRC_FL_Source'
41
- SCANOSS_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
42
- 'OSS Version', 'License', 'Download Location',
43
- 'Homepage', 'Copyright Text', 'Exclude', 'Comment']}
44
- MERGED_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
45
- 'OSS Version', 'License', 'Download Location',
46
- 'Homepage', 'Copyright Text', 'Exclude', 'Comment', 'license_reference']}
47
+ PRE_MERGE_SHEET_NAME = '.SRC_FL_Source_no_merge'
47
48
  KB_REFERENCE_HEADER = ['ID', 'Source Path', 'KB Origin URL', 'Evidence']
48
49
  ALL_MODE = 'all'
49
50
  SCANNER_TYPE = ['kb', 'scancode', 'scanoss', ALL_MODE]
@@ -88,6 +89,7 @@ def main() -> None:
88
89
  parser.add_argument('--hide_progress', action='store_true', required=False)
89
90
  parser.add_argument('--kb_url', type=str, required=False, default="")
90
91
  parser.add_argument('--kb_token', type=str, required=False, default="")
92
+ parser.add_argument('--no_merge', action='store_true', required=False)
91
93
 
92
94
  args = parser.parse_args()
93
95
 
@@ -118,19 +120,19 @@ def main() -> None:
118
120
  hide_progress = args.hide_progress
119
121
  kb_url = args.kb_url
120
122
  kb_token = args.kb_token
123
+ merge_by_folder = not args.no_merge
121
124
 
122
125
  time_out = args.timeout
123
126
  core = args.cores
124
127
 
125
128
  if os.path.isdir(path_to_scan):
126
- result = []
127
129
  result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
128
130
  print_matched_text, formats, time_out, correct_mode, correct_filepath,
129
131
  selected_scanner, path_to_exclude, hide_progress=hide_progress,
130
- kb_url=kb_url, kb_token=kb_token)
132
+ kb_url=kb_url, kb_token=kb_token,
133
+ merge_by_folder=merge_by_folder)
131
134
 
132
135
  _result_log["Scan Result"] = result[1]
133
-
134
136
  try:
135
137
  logger.info(yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True))
136
138
  except Exception as ex:
@@ -148,7 +150,7 @@ def create_report_file(
148
150
  output_extensions: list = [], correct_mode: bool = True,
149
151
  correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [],
150
152
  formats: list = [], api_limit_exceed: bool = False, files_count: int = 0, final_output_path: str = "",
151
- run_kb_msg: str = ""
153
+ run_kb_msg: str = "", merge_by_folder: bool = True
152
154
  ) -> 'ScannerItem':
153
155
  """
154
156
  Create report files for given scanned result.
@@ -158,9 +160,9 @@ def create_report_file(
158
160
  :param license_list: matched text (only for scancode).
159
161
  :param need_license: if requested, output matched text (only for scancode).
160
162
  """
161
- extended_header = {}
162
163
  sheet_list = {}
163
164
  _json_ext = ".json"
165
+ name_time = timestamp_for_filename(_start_time)
164
166
 
165
167
  output_path = os.path.abspath(output_path)
166
168
 
@@ -178,19 +180,19 @@ def create_report_file(
178
180
  to_remove.append(i)
179
181
  else:
180
182
  if formats[i].startswith('spdx'):
181
- output_files[i] = f"fosslight_spdx_src_{_start_time}"
183
+ output_files[i] = f"fosslight_spdx_src_{name_time}"
182
184
  elif formats[i].startswith('cyclonedx'):
183
- output_files[i] = f'fosslight_cyclonedx_src_{_start_time}'
185
+ output_files[i] = f'fosslight_cyclonedx_src_{name_time}'
184
186
  else:
185
187
  if output_extension == _json_ext:
186
- output_files[i] = f"fosslight_opossum_src_{_start_time}"
188
+ output_files[i] = f"fosslight_opossum_src_{name_time}"
187
189
  else:
188
- output_files[i] = f"fosslight_report_src_{_start_time}"
190
+ output_files[i] = f"fosslight_report_src_{name_time}"
189
191
  else:
190
192
  if output_extension == _json_ext:
191
- output_files[i] = f"fosslight_opossum_src_{_start_time}"
193
+ output_files[i] = f"fosslight_opossum_src_{name_time}"
192
194
  else:
193
- output_files[i] = f"fosslight_report_src_{_start_time}"
195
+ output_files[i] = f"fosslight_report_src_{name_time}"
194
196
  for index in sorted(to_remove, reverse=True):
195
197
  # remove elements of spdx format on windows
196
198
  del output_files[index]
@@ -228,11 +230,6 @@ def create_report_file(
228
230
  sheet_list = {}
229
231
  scan_item.append_file_items(merged_result, PKG_NAME)
230
232
 
231
- if selected_scanner == 'scanoss':
232
- extended_header = SCANOSS_HEADER
233
- else:
234
- extended_header = MERGED_HEADER
235
-
236
233
  if need_license:
237
234
  if selected_scanner == 'scancode':
238
235
  sheet_list["scancode_reference"] = get_license_list_to_print(license_list)
@@ -258,12 +255,24 @@ def create_report_file(
258
255
  scan_item = correct_item
259
256
  logger.info("Success to correct with yaml.")
260
257
 
258
+ finish_time = current_timestamp_utc()
259
+ scan_item.set_cover_finish_time(finish_time)
260
+
261
+ if merged_result and merge_by_folder:
262
+ _add_pre_merge_sheet(scan_item, PRE_MERGE_SHEET_NAME, get_header_row(SRC_SHEET_NAME), PKG_NAME)
263
+ _merge_start = time.time()
264
+ scan_item.file_items[PKG_NAME] = merge_results_by_folder(scan_item.file_items[PKG_NAME])
265
+ logger.debug(f"[TIMING] merge_results_by_folder: {time.time() - _merge_start:.4f}s")
266
+
261
267
  combined_paths_and_files = [os.path.join(output_path, file) for file in output_files]
262
268
  results = []
263
269
  for combined_path_and_file, output_extension, output_format in zip(combined_paths_and_files, output_extensions, formats):
264
270
  # if need_license and output_extension == _json_ext and "scanoss_reference" in sheet_list:
265
271
  # del sheet_list["scanoss_reference"]
266
- results.append(write_output_file(combined_path_and_file, output_extension, scan_item, extended_header, "", output_format))
272
+ result = write_output_file(
273
+ combined_path_and_file, output_extension, scan_item, hide_header="", format=output_format
274
+ )
275
+ results.append(result)
267
276
  for success, msg, result_file in results:
268
277
  final_result_file = result_file.replace(output_path, final_output_path)
269
278
  if success:
@@ -492,7 +501,8 @@ def run_scanners(
492
501
  formats: list = [], time_out: int = 120,
493
502
  correct_mode: bool = True, correct_filepath: str = "",
494
503
  selected_scanner: str = ALL_MODE, path_to_exclude: list = [],
495
- all_exclude_mode: tuple = (), hide_progress: bool = False, kb_url: str = "", kb_token: str = ""
504
+ all_exclude_mode: tuple = (), hide_progress: bool = False,
505
+ kb_url: str = "", kb_token: str = "", merge_by_folder: bool = True
496
506
  ) -> Tuple[bool, str, 'ScannerItem', list, list]:
497
507
  """
498
508
  Run Scancode and scanoss.py for the given path.
@@ -513,7 +523,8 @@ def run_scanners(
513
523
  """
514
524
  global logger
515
525
 
516
- start_time = datetime.now().strftime('%y%m%d_%H%M')
526
+ start_time = current_timestamp_utc()
527
+ file_time = timestamp_for_filename(start_time)
517
528
  scancode_result = []
518
529
  scanoss_result = []
519
530
  merged_result = []
@@ -529,13 +540,13 @@ def run_scanners(
529
540
  if output_path == "":
530
541
  output_path = os.getcwd()
531
542
  final_output_path = output_path
532
- output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{start_time}')
543
+ output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{file_time}')
533
544
  publish_temp_output = False
534
545
  logger = None
535
546
  publish_ok = True
536
547
 
537
548
  try:
538
- logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
549
+ logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{file_time}.txt"),
539
550
  True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
540
551
 
541
552
  logger.info(f"Tool Info : {result_log['Tool Info']}")
@@ -545,6 +556,11 @@ def run_scanners(
545
556
  print_matched_text = False
546
557
 
547
558
  if success:
559
+ has_sbom_format = any(f.startswith('spdx') or f.startswith('cyclonedx') for f in formats) if formats else False
560
+ if has_sbom_format and merge_by_folder:
561
+ logger.info("SPDX/CycloneDX format does not support merge. Merging is not performed.")
562
+ merge_by_folder = False
563
+
548
564
  if all_exclude_mode and len(all_exclude_mode) == 4:
549
565
  (excluded_path_with_default_exclusion,
550
566
  excluded_path_without_dot,
@@ -598,7 +614,8 @@ def run_scanners(
598
614
  scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
599
615
  print_matched_text, output_path, output_files, output_extensions, correct_mode,
600
616
  correct_filepath, path_to_scan, excluded_path_without_dot, formats,
601
- api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg)
617
+ api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg,
618
+ merge_by_folder)
602
619
  else:
603
620
  print_help_msg_source_scanner()
604
621
  result_log[RESULT_KEY] = "Unsupported scanner"
@@ -619,6 +636,17 @@ def run_scanners(
619
636
  else "Failed to publish scan artifacts"
620
637
  )
621
638
 
639
+ if logger:
640
+ if scan_item and not isinstance(scan_item, list):
641
+ result_log["Running time"] = scan_item.cover.running_time
642
+ else:
643
+ finish_time = current_timestamp_utc()
644
+ result_log["Running time"] = format_running_time(start_time, finish_time)
645
+ try:
646
+ logger.info(dump_result_log(result_log))
647
+ except Exception as ex:
648
+ logger.warning(f"Failed to print result log. {ex}")
649
+
622
650
  return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result
623
651
 
624
652
 
@@ -7,11 +7,10 @@ import os
7
7
  import multiprocessing
8
8
  import warnings
9
9
  import logging
10
- import yaml
11
10
  from scancode import cli
12
- from datetime import datetime
13
11
  import fosslight_util.constant as constant
14
12
  from fosslight_util.set_log import init_log
13
+ from fosslight_util.time import current_timestamp_utc, timestamp_for_filename
15
14
  from ._parsing_scancode_file_item import parsing_file_item
16
15
  from ._parsing_scancode_file_item import get_error_from_header
17
16
  from fosslight_util.output_format import check_output_formats_v2
@@ -168,12 +167,12 @@ def run_scan(
168
167
 
169
168
  success = True
170
169
  msg = ""
171
- _str_final_result_log = ""
172
170
  _result_log = {}
173
171
  result_list = []
174
172
  license_list = []
175
173
  _json_ext = ".json"
176
- _start_time = datetime.now().strftime('%y%m%d_%H%M')
174
+ _start_time = current_timestamp_utc()
175
+ _file_time = timestamp_for_filename(_start_time)
177
176
 
178
177
  if not correct_filepath:
179
178
  correct_filepath = path_to_scan
@@ -189,9 +188,9 @@ def run_scan(
189
188
  for i, output_extension in enumerate(output_extensions):
190
189
  if output_files[i] is None or output_files[i] == "":
191
190
  if output_extension == _json_ext:
192
- output_files[i] = f"fosslight_opossum_src_{_start_time}"
191
+ output_files[i] = f"fosslight_opossum_src_{_file_time}"
193
192
  else:
194
- output_files[i] = f"fosslight_report_src_{_start_time}"
193
+ output_files[i] = f"fosslight_report_src_{_file_time}"
195
194
 
196
195
  if _write_json_file:
197
196
  output_json_file = os.path.join(output_path, "scancode_raw_result.json")
@@ -199,7 +198,7 @@ def run_scan(
199
198
  output_json_file = ""
200
199
 
201
200
  if not called_by_cli:
202
- logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
201
+ logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_file_time}.txt"),
203
202
  True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
204
203
 
205
204
  logger.info(f"Tool Info : {_result_log['Tool Info']}")
@@ -292,11 +291,6 @@ def run_scan(
292
291
  scan_result_msg = str(success) if msg == "" else f"{success}, {msg}"
293
292
  _result_log["Scan Result"] = scan_result_msg
294
293
  _result_log["Output Directory"] = output_path
295
- try:
296
- _str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)
297
- logger.info(_str_final_result_log)
298
- except Exception as ex:
299
- logger.warning(f"Failed to print result log. {ex}")
300
294
 
301
295
  if not success:
302
296
  logger.error(f"Failed to run: {scan_result_msg}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fosslight_source
3
- Version: 2.3.2
3
+ Version: 2.3.3
4
4
  Summary: FOSSLight Source Scanner
5
5
  Author: LG Electronics
6
6
  License-Expression: Apache-2.0
@@ -20,7 +20,7 @@ Requires-Dist: setuptools<=80.10.2
20
20
  Requires-Dist: pyparsing
21
21
  Requires-Dist: scanoss>=1.19.0
22
22
  Requires-Dist: XlsxWriter
23
- Requires-Dist: fosslight_util>=2.1.37
23
+ Requires-Dist: fosslight_util>=2.2.2
24
24
  Requires-Dist: PyYAML
25
25
  Requires-Dist: wheel>=0.38.1
26
26
  Requires-Dist: intbitset
@@ -6,6 +6,7 @@ src/fosslight_source/__init__.py
6
6
  src/fosslight_source/_help.py
7
7
  src/fosslight_source/_kb_client.py
8
8
  src/fosslight_source/_license_matched.py
9
+ src/fosslight_source/_merge.py
9
10
  src/fosslight_source/_parsing_scancode_file_item.py
10
11
  src/fosslight_source/_parsing_scanoss_file.py
11
12
  src/fosslight_source/_scan_item.py
@@ -2,7 +2,7 @@ setuptools<=80.10.2
2
2
  pyparsing
3
3
  scanoss>=1.19.0
4
4
  XlsxWriter
5
- fosslight_util>=2.1.37
5
+ fosslight_util>=2.2.2
6
6
  PyYAML
7
7
  wheel>=0.38.1
8
8
  intbitset