fosslight-source 2.3.2__tar.gz → 2.3.4__tar.gz

{fosslight_source-2.3.2/src/fosslight_source.egg-info → fosslight_source-2.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.3.2
+Version: 2.3.4
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -20,12 +20,13 @@ Requires-Dist: setuptools<=80.10.2
 Requires-Dist: pyparsing
 Requires-Dist: scanoss>=1.19.0
 Requires-Dist: XlsxWriter
-Requires-Dist: fosslight_util>=2.1.37
+Requires-Dist: fosslight_util>=2.2.2
 Requires-Dist: PyYAML
 Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
 Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
+Requires-Dist: lxml>=6.0.1
 Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fosslight_source"
-version = "2.3.2"
+version = "2.3.4"
 description = "FOSSLight Source Scanner"
 readme = "README.md"
 license = "Apache-2.0"
@@ -29,12 +29,13 @@ dependencies = [
     "pyparsing",
     "scanoss>=1.19.0",
     "XlsxWriter",
-    "fosslight_util>=2.1.37",
+    "fosslight_util>=2.2.2",
     "PyYAML",
     "wheel>=0.38.1",
     "intbitset",
     "fosslight_binary>=5.1.22",
     "scancode-toolkit>=32.0.2",
+    "lxml>=6.0.1",
     # cryptography 49.x does not provide macOS x86_64 wheels, causing source builds to require OpenSSL/pkg-config.
     "cryptography<49; platform_system == 'Darwin' and platform_machine == 'x86_64'",
     "fingerprints==1.2.3",

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/src/fosslight_source/_help.py

@@ -39,6 +39,7 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
     -c <number>            Number of CPU cores/threads to use for scanning
     -t <seconds>           Timeout in seconds for ScanCode scanning
     -j                     Generate raw scanner results in JSON format
+    --no_merge             Keep source paths file-based without folder merge
     --no_correction        Skip OSS information correction with sbom-info.yaml
     --correct_fpath <path> Path to custom sbom-info.yaml file
     --hide_progress        Hide the progress bar during scanning

fosslight_source-2.3.4/src/fosslight_source/_merge.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from collections import Counter
+
+from fosslight_util.constant import COMMENT_DELIMITER
+
+from ._scan_item import SourceItem
+
+
+def _get_source_rows_to_print(source_items: list) -> list:
+    source_rows = []
+    for source_item in source_items:
+        source_rows.extend(source_item.get_print_array())
+    return source_rows
+
+
+def _add_pre_merge_sheet(scan_item, pre_merge_sheet_name: str, header_row: list, pkg_name: str) -> None:
+    external_sheets = getattr(scan_item, "external_sheets", {}) or {}
+    external_sheets[pre_merge_sheet_name] = [
+        header_row,
+        *_get_source_rows_to_print(scan_item.file_items.get(pkg_name, [])),
+    ]
+    scan_item.external_sheets = external_sheets
+
+
+def _normalize_merge_text(value: str) -> str:
+    return value.strip() if value else ""
+
+
+def _iter_merge_values(values) -> list:
+    if not values:
+        return []
+    if isinstance(values, str):
+        return [values]
+    return values
+
+
+def _normalize_merge_licenses(licenses) -> tuple:
+    return tuple(sorted([lic.strip() for lic in _iter_merge_values(licenses) if lic and lic.strip()]))
+
+
+def _normalize_merge_downloads(downloads) -> tuple:
+    if not downloads:
+        return ()
+    if isinstance(downloads, str):
+        downloads = [downloads]
+    return tuple(sorted([dl.strip() for dl in downloads if dl and dl.strip()]))
+
+
+def _iter_merge_rows(item: SourceItem):
+    if item.oss_items:
+        for oss_item in item.oss_items:
+            name = _normalize_merge_text(oss_item.name) or _normalize_merge_text(item.oss_name)
+            version = _normalize_merge_text(oss_item.version) or _normalize_merge_text(item.oss_version)
+            licenses = _normalize_merge_licenses(oss_item.license or item.licenses)
+            downloads = _normalize_merge_downloads(oss_item.download_location or item.download_location)
+            yield name, version, licenses, downloads
+    else:
+        yield (
+            _normalize_merge_text(item.oss_name),
+            _normalize_merge_text(item.oss_version),
+            _normalize_merge_licenses(item.licenses),
+            _normalize_merge_downloads(item.download_location),
+        )
+
+
+def _get_item_oss_name(item: SourceItem) -> str:
+    for name, _, _, _ in _iter_merge_rows(item):
+        if name:
+            return name
+    return ""
+
+
+def _get_item_oss_version(item: SourceItem) -> str:
+    for _, version, _, _ in _iter_merge_rows(item):
+        if version:
+            return version
+    return ""
+
+
+def _get_merge_licenses(scan_item: SourceItem) -> tuple:
+    for _, _, licenses, _ in _iter_merge_rows(scan_item):
+        if licenses:
+            return licenses
+    return ()
+
+
+def _get_merge_download_locations(scan_item: SourceItem) -> tuple:
+    for _, _, _, downloads in _iter_merge_rows(scan_item):
+        if downloads:
+            return downloads
+    return ()
+
+
+def _get_merge_field_value(scan_items: list, value_getter):
+    for scan_item in scan_items:
+        value = value_getter(scan_item)
+        if value:
+            return value
+    return ""
+
+
+def _get_top_merge_values(scan_items: list, value_getter) -> list:
+    values = []
+    for scan_item in scan_items:
+        for value in _iter_merge_values(value_getter(scan_item)):
+            normalized_value = _normalize_merge_text(value)
+            if normalized_value:
+                values.append(normalized_value)
+    return [value for value, _ in Counter(values).most_common(3)]
+
+
+def _can_merge_folder(scan_items: list) -> bool:
+    if len(scan_items) <= 1:
+        return False
+
+    ref_name = None
+    ref_version = None
+    ref_licenses = None
+    ref_downloads = None
+
+    for item in scan_items:
+        for name, version, licenses, downloads in _iter_merge_rows(item):
+            if name:
+                if ref_name is None:
+                    ref_name = name
+                elif name != ref_name:
+                    return False
+
+            if version:
+                if ref_version is None:
+                    ref_version = version
+                elif version != ref_version:
+                    return False
+
+            if licenses:
+                if ref_licenses is None:
+                    ref_licenses = licenses
+                elif licenses != ref_licenses:
+                    return False
+
+            if downloads:
+                if ref_downloads is None:
+                    ref_downloads = downloads
+                elif downloads != ref_downloads:
+                    return False
+
+    return True
+
+
+def _get_merged_comments(scan_items: list) -> str:
+    comments = []
+    for item in scan_items:
+        val = item.comment
+        if val:
+            parts = [p.strip() for p in val.split(COMMENT_DELIMITER) if p.strip()]
+            for p in parts:
+                if p not in comments:
+                    comments.append(p)
+    if not comments:
+        return ""
+
+    return COMMENT_DELIMITER.join(comments)
+
+
+def _create_merged_item(scan_items: list, merge_path: str) -> SourceItem:
+    merged_path = f"{merge_path} ({len(scan_items)})"
+    merged_item = SourceItem(merged_path)
+    merged_item.oss_name = _get_merge_field_value(scan_items, lambda item: _normalize_merge_text(_get_item_oss_name(item)))
+    merged_item.oss_version = _get_merge_field_value(scan_items, lambda item: _normalize_merge_text(_get_item_oss_version(item)))
+    merged_licenses = _get_merge_field_value(scan_items, _get_merge_licenses)
+    merged_item.licenses = list(merged_licenses) if merged_licenses else []
+    merged_downloads = _get_merge_field_value(scan_items, _get_merge_download_locations)
+    merged_item.download_location = list(merged_downloads) if merged_downloads else []
+    merged_copyrights = _get_top_merge_values(scan_items, lambda item: item.copyright)
+    merged_item.copyright = merged_copyrights if merged_copyrights else []
+    merged_comments = _get_merged_comments(scan_items)
+    if merged_comments:
+        merged_item.comment = merged_comments
+    merged_item.set_oss_item()
+    return merged_item
+
+
+def merge_results_by_folder(scan_result: list) -> list:
+    """
+    Merge output rows within the same folder when OSS name, OSS version, license,
+    and download location are compatible.
+
+    A field is compatible when all non-empty values across rows are identical
+    (empty values are ignored). Each oss_item is treated as its own row. All eligible
+    rows in the folder must be compatible together; rows are not grouped by key subsets.
+    """
+    # Build a folder tree first so merge never jumps straight to root ".".
+    merge_tree = {"items": [], "children": {}}
+
+    for scan_item in scan_result:
+        normalized_path = os.path.normpath(scan_item.source_name_or_path).replace("\\", "/")
+        path_parts = [part for part in normalized_path.split("/") if part and part != "."]
+        current_node = merge_tree
+
+        for folder_name in path_parts[:-1]:
+            current_node = current_node["children"].setdefault(folder_name, {"items": [], "children": {}})
+        current_node["items"].append(scan_item)
+
+    def merge_node(merge_node_item: dict, merge_path: str = "", depth: int = 0) -> tuple:
+        child_finalized = []
+        child_unfinalized = []
+        for folder_name, child_node in merge_node_item["children"].items():
+            child_path = f"{merge_path}/{folder_name}" if merge_path else folder_name
+            fin, unfin = merge_node(child_node, child_path, depth + 1)
+            child_finalized.extend(fin)
+            child_unfinalized.extend(unfin)
+
+        local_excluded = [item for item in merge_node_item["items"] if item.exclude]
+        local_eligible = [item for item in merge_node_item["items"] if not item.exclude]
+
+        all_eligible_candidates = list(local_eligible)
+        for _, g_items in child_unfinalized:
+            all_eligible_candidates.extend(g_items)
+
+        # We can merge under the current node if depth > 0 and we are combining multiple sources:
+        # e.g., local files + child groups, or multiple child groups, or multiple local files.
+        can_merge_here = (depth > 0) and (len(local_eligible) + len(child_unfinalized) > 1)
+
+        if can_merge_here:
+            if _can_merge_folder(all_eligible_candidates):
+                new_finalized = child_finalized + local_excluded
+                new_unfinalized = [(merge_path, all_eligible_candidates)]
+                return new_finalized, new_unfinalized
+            else:
+                # Compatibility broke at this level. We must finalize the subtrees.
+                finalized_child_groups = []
+                for c_path, c_items in child_unfinalized:
+                    if len(c_items) > 1 and _can_merge_folder(c_items):
+                        finalized_child_groups.append(_create_merged_item(c_items, c_path))
+                    else:
+                        finalized_child_groups.extend(c_items)
+
+                finalized_local = []
+                if len(local_eligible) > 1 and _can_merge_folder(local_eligible):
+                    finalized_local.append(_create_merged_item(local_eligible, merge_path))
+                else:
+                    finalized_local.extend(local_eligible)
+
+                new_finalized = child_finalized + local_excluded + finalized_child_groups + finalized_local
+                new_unfinalized = []
+                return new_finalized, new_unfinalized
+        else:
+            # We cannot merge or don't need to merge at this level (e.g. depth == 0 or only 1 source).
+            # We propagate everything up as unfinalized.
+            new_finalized = child_finalized + local_excluded
+            new_unfinalized = list(child_unfinalized)
+            if local_eligible:
+                new_unfinalized.append((merge_path, local_eligible))
+            return new_finalized, new_unfinalized
+
+    fin, unfin = merge_node(merge_tree)
+    finalized_results = list(fin)
+    for path, items in unfin:
+        if path and len(items) > 1 and _can_merge_folder(items):
+            finalized_results.append(_create_merged_item(items, path))
+        else:
+            finalized_results.extend(items)
+
+    return finalized_results

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/src/fosslight_source/_scan_item.py

@@ -48,7 +48,6 @@ class SourceItem(FileItem):
         self.source_name_or_path = value
         self.is_license_text = False
         self.is_manifest_file = False
-        self.license_reference = ""
         self.scanoss_reference = {}
         self.matched_lines = ""  # Only for SCANOSS results
         self.fileURL = ""  # Only for SCANOSS results
@@ -206,8 +205,7 @@ class SourceItem(FileItem):
         for item in self.oss_items:
             print_rows.append([self.source_name_or_path, item.name, item.version, ",".join(item.license),
                                item.download_location, "",
-                               item.copyright, "Exclude" if self.exclude else "", item.comment,
-                               self.license_reference])
+                               item.copyright, "Exclude" if self.exclude else "", item.comment])
         return print_rows
 
     def __eq__(self, other: object) -> bool:

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/src/fosslight_source/cli.py

@@ -12,12 +12,12 @@ import logging
 import re
 import urllib.request
 import urllib.error
-from datetime import datetime
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
 from ._help import print_version, print_help_msg_source_scanner
 from ._license_matched import get_license_list_to_print
 from fosslight_util.output_format import check_output_formats_v2, write_output_file
+from fosslight_util.write_excel import get_header_row
 from fosslight_util.correct import correct_with_yaml
 from fosslight_util.parsing_yaml import SUPPORT_OSS_INFO_FILES
 from .run_scancode import run_scan
@@ -31,19 +31,20 @@ from .run_spdx_extractor import get_spdx_downloads
 from .run_manifest_extractor import get_manifest_licenses
 from ._scan_item import SourceItem, resolve_kb_config, is_notice_file
 from ._kb_client import fetch_origin_urls_via_scan_job
+from fosslight_util.cover import dump_result_log
+from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename
 from fosslight_util.oss_item import ScannerItem
 from typing import Optional, Tuple
 from ._scan_item import is_manifest_file
 import shutil
+from ._merge import (
+    _add_pre_merge_sheet,
+    merge_results_by_folder
+)
 
 
 SRC_SHEET_NAME = 'SRC_FL_Source'
-SCANOSS_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
-                                   'OSS Version', 'License', 'Download Location',
-                                   'Homepage', 'Copyright Text', 'Exclude', 'Comment']}
-MERGED_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
-                                  'OSS Version', 'License', 'Download Location',
-                                  'Homepage', 'Copyright Text', 'Exclude', 'Comment', 'license_reference']}
+PRE_MERGE_SHEET_NAME = '.SRC_FL_Source_no_merge'
 KB_REFERENCE_HEADER = ['ID', 'Source Path', 'KB Origin URL', 'Evidence']
 ALL_MODE = 'all'
 SCANNER_TYPE = ['kb', 'scancode', 'scanoss', ALL_MODE]
@@ -88,6 +89,7 @@ def main() -> None:
     parser.add_argument('--hide_progress', action='store_true', required=False)
     parser.add_argument('--kb_url', type=str, required=False, default="")
     parser.add_argument('--kb_token', type=str, required=False, default="")
+    parser.add_argument('--no_merge', action='store_true', required=False)
 
     args = parser.parse_args()
 
@@ -118,19 +120,19 @@ def main() -> None:
     hide_progress = args.hide_progress
     kb_url = args.kb_url
     kb_token = args.kb_token
+    merge_by_folder = not args.no_merge
 
     time_out = args.timeout
     core = args.cores
 
     if os.path.isdir(path_to_scan):
-        result = []
         result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
                               print_matched_text, formats, time_out, correct_mode, correct_filepath,
                               selected_scanner, path_to_exclude, hide_progress=hide_progress,
-                              kb_url=kb_url, kb_token=kb_token)
+                              kb_url=kb_url, kb_token=kb_token,
+                              merge_by_folder=merge_by_folder)
 
         _result_log["Scan Result"] = result[1]
-
         try:
             logger.info(yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True))
         except Exception as ex:
@@ -148,7 +150,7 @@ def create_report_file(
     output_extensions: list = [], correct_mode: bool = True,
     correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [],
     formats: list = [], api_limit_exceed: bool = False, files_count: int = 0, final_output_path: str = "",
-    run_kb_msg: str = ""
+    run_kb_msg: str = "", merge_by_folder: bool = True
 ) -> 'ScannerItem':
     """
     Create report files for given scanned result.
@@ -158,9 +160,9 @@ def create_report_file(
     :param license_list: matched text (only for scancode).
     :param need_license: if requested, output matched text (only for scancode).
     """
-    extended_header = {}
     sheet_list = {}
     _json_ext = ".json"
+    name_time = timestamp_for_filename(_start_time)
 
     output_path = os.path.abspath(output_path)
 
@@ -178,19 +180,19 @@ def create_report_file(
                             to_remove.append(i)
                         else:
                             if formats[i].startswith('spdx'):
-                                output_files[i] = f"fosslight_spdx_src_{_start_time}"
+                                output_files[i] = f"fosslight_spdx_src_{name_time}"
                             elif formats[i].startswith('cyclonedx'):
-                                output_files[i] = f'fosslight_cyclonedx_src_{_start_time}'
+                                output_files[i] = f'fosslight_cyclonedx_src_{name_time}'
                     else:
                         if output_extension == _json_ext:
-                            output_files[i] = f"fosslight_opossum_src_{_start_time}"
+                            output_files[i] = f"fosslight_opossum_src_{name_time}"
                         else:
-                            output_files[i] = f"fosslight_report_src_{_start_time}"
+                            output_files[i] = f"fosslight_report_src_{name_time}"
                 else:
                     if output_extension == _json_ext:
-                        output_files[i] = f"fosslight_opossum_src_{_start_time}"
+                        output_files[i] = f"fosslight_opossum_src_{name_time}"
                     else:
-                        output_files[i] = f"fosslight_report_src_{_start_time}"
+                        output_files[i] = f"fosslight_report_src_{name_time}"
         for index in sorted(to_remove, reverse=True):
             # remove elements of spdx format on windows
             del output_files[index]
@@ -228,11 +230,6 @@ def create_report_file(
         sheet_list = {}
         scan_item.append_file_items(merged_result, PKG_NAME)
 
-        if selected_scanner == 'scanoss':
-            extended_header = SCANOSS_HEADER
-        else:
-            extended_header = MERGED_HEADER
-
         if need_license:
             if selected_scanner == 'scancode':
                 sheet_list["scancode_reference"] = get_license_list_to_print(license_list)
@@ -258,12 +255,24 @@ def create_report_file(
             scan_item = correct_item
             logger.info("Success to correct with yaml.")
 
+    finish_time = current_timestamp_utc()
+    scan_item.set_cover_finish_time(finish_time)
+
+    if merged_result and merge_by_folder:
+        _add_pre_merge_sheet(scan_item, PRE_MERGE_SHEET_NAME, get_header_row(SRC_SHEET_NAME), PKG_NAME)
+        _merge_start = time.time()
+        scan_item.file_items[PKG_NAME] = merge_results_by_folder(scan_item.file_items[PKG_NAME])
+        logger.debug(f"[TIMING] merge_results_by_folder: {time.time() - _merge_start:.4f}s")
+
     combined_paths_and_files = [os.path.join(output_path, file) for file in output_files]
     results = []
     for combined_path_and_file, output_extension, output_format in zip(combined_paths_and_files, output_extensions, formats):
         # if need_license and output_extension == _json_ext and "scanoss_reference" in sheet_list:
         #     del sheet_list["scanoss_reference"]
-        results.append(write_output_file(combined_path_and_file, output_extension, scan_item, extended_header, "", output_format))
+        result = write_output_file(
+            combined_path_and_file, output_extension, scan_item, hide_header="", format=output_format
+        )
+        results.append(result)
     for success, msg, result_file in results:
         final_result_file = result_file.replace(output_path, final_output_path)
         if success:
@@ -492,7 +501,8 @@ def run_scanners(
     formats: list = [], time_out: int = 120,
     correct_mode: bool = True, correct_filepath: str = "",
     selected_scanner: str = ALL_MODE, path_to_exclude: list = [],
-    all_exclude_mode: tuple = (), hide_progress: bool = False, kb_url: str = "", kb_token: str = ""
+    all_exclude_mode: tuple = (), hide_progress: bool = False,
+    kb_url: str = "", kb_token: str = "", merge_by_folder: bool = True
 ) -> Tuple[bool, str, 'ScannerItem', list, list]:
     """
     Run Scancode and scanoss.py for the given path.
@@ -513,7 +523,8 @@ def run_scanners(
     """
     global logger
 
-    start_time = datetime.now().strftime('%y%m%d_%H%M')
+    start_time = current_timestamp_utc()
+    file_time = timestamp_for_filename(start_time)
     scancode_result = []
     scanoss_result = []
     merged_result = []
@@ -529,13 +540,13 @@ def run_scanners(
     if output_path == "":
         output_path = os.getcwd()
     final_output_path = output_path
-    output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{start_time}')
+    output_path = os.path.join(os.path.dirname(output_path), f'.fosslight_temp_{file_time}')
     publish_temp_output = False
     logger = None
     publish_ok = True
 
     try:
-        logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
+        logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{file_time}.txt"),
                                       True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
 
         logger.info(f"Tool Info : {result_log['Tool Info']}")
@@ -545,6 +556,11 @@ def run_scanners(
             print_matched_text = False
 
         if success:
+            has_sbom_format = any(f.startswith('spdx') or f.startswith('cyclonedx') for f in formats) if formats else False
+            if has_sbom_format and merge_by_folder:
+                logger.info("SPDX/CycloneDX format does not support merge. Merging is not performed.")
+                merge_by_folder = False
+
             if all_exclude_mode and len(all_exclude_mode) == 4:
                 (excluded_path_with_default_exclusion,
                  excluded_path_without_dot,
@@ -556,7 +572,7 @@ def run_scanners(
                  excluded_path_without_dot,
                  excluded_files,
                  cnt_file_except_skipped) = get_excluded_paths(path_to_scan, path_to_exclude_with_filename)
-                logger.debug(f"Skipped paths: {excluded_path_with_default_exclusion}")
+                logger.debug(f"Skipped paths count: {len(excluded_path_with_default_exclusion)}")
 
             if not selected_scanner:
                 selected_scanner = ALL_MODE
@@ -564,8 +580,8 @@ def run_scanners(
                 success, result_log[RESULT_KEY], scancode_result, license_list = run_scan(
                     path_to_scan, output_file_name, write_json_file, num_cores, True,
                     print_matched_text, formats, called_by_cli, time_out, correct_mode,
-                    correct_filepath, excluded_path_with_default_exclusion,
-                    excluded_files, hide_progress,
+                    correct_filepath, path_to_exclude,
+                    hide_progress=hide_progress,
                 )
             excluded_files = set(excluded_files) if excluded_files else set()
             if selected_scanner in ['scanoss', ALL_MODE]:
@@ -582,7 +598,7 @@ def run_scanners(
                         run_kb_msg = f"KB({kb_url}) Unreachable"
 
                 spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
-                merged_result, kb_status_message, kb_requested_count, kb_returned_count = merge_results(
+                merged_result, kb_status_message, kb_requested_count, _ = merge_results(
                     scancode_result, scanoss_result, spdx_downloads,
                     path_to_scan, run_kb, manifest_licenses, excluded_files,
                     hide_progress, kb_url, kb_token,
@@ -590,15 +606,13 @@ def run_scanners(
                 if kb_status_message:
                     run_kb_msg = f"KB({kb_url}) {kb_status_message}"
                 elif run_kb and kb_requested_count > 0:
-                    run_kb_msg = (
-                        f"KB({kb_url}) response : {kb_returned_count}/"
-                        f" requested: {kb_requested_count}"
-                    )
+                    run_kb_msg = f"KB({kb_url}) : Completed"
                 mark_oss_info_correction_files_as_excluded(merged_result)
                 scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                                print_matched_text, output_path, output_files, output_extensions, correct_mode,
                                                correct_filepath, path_to_scan, excluded_path_without_dot, formats,
-                                               api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg)
+                                               api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg,
+                                               merge_by_folder)
             else:
                 print_help_msg_source_scanner()
                 result_log[RESULT_KEY] = "Unsupported scanner"
@@ -619,6 +633,17 @@ def run_scanners(
             else "Failed to publish scan artifacts"
         )
 
+    if logger:
+        if scan_item and not isinstance(scan_item, list):
+            result_log["Running time"] = scan_item.cover.running_time
+        else:
+            finish_time = current_timestamp_utc()
+            result_log["Running time"] = format_running_time(start_time, finish_time)
+        try:
+            logger.info(dump_result_log(result_log))
+        except Exception as ex:
+            logger.warning(f"Failed to print result log. {ex}")
+
     return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result
 
 

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/src/fosslight_source/run_scancode.py

@@ -7,11 +7,10 @@ import os
 import multiprocessing
 import warnings
 import logging
-import yaml
 from scancode import cli
-from datetime import datetime
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
+from fosslight_util.time import current_timestamp_utc, timestamp_for_filename
 from ._parsing_scancode_file_item import parsing_file_item
 from ._parsing_scancode_file_item import get_error_from_header
 from fosslight_util.output_format import check_output_formats_v2
@@ -23,7 +22,7 @@ from fosslight_util.exclude import (
     PACKAGE_DIRECTORY,
 )
 from commoncode.fileset import is_included
-from typing import Tuple, Iterable
+from typing import Callable, Tuple
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -63,6 +62,96 @@ def _apply_scancode_unset_workaround(kwargs: dict) -> None:
         logger.debug("scancode UNSET workaround skipped: %s", ex)
 
 
+_WILDCARD_EXTENSIONS = {
+    "png", "mp3", "wav", "comp", "bin", "o", "db", "tflite",
+    "ttf", "exe", "dll", "jpg", "jpeg", "gif",
+    "zip", "tar", "tgz", "gz",
+    "bmp", "webp", "ico",
+} | {ext.lower() for ext in EXCLUDE_FILE_EXTENSION}
+_SKIP_DIR_NAMES = frozenset(name.lower() for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY)
+_SKIP_EXTS = frozenset(ext.lower() for ext in EXCLUDE_FILE_EXTENSION)
+
+
+def _normalize_custom_pattern(pattern: str, abs_path_to_scan: str) -> set:
+    pat = pattern.replace('\\', '/').strip()
+    if not pat:
+        return set()
+
+    patterns_to_add = {pat}
+
+    if pat.endswith("/**"):
+        base = pat[:-3].rstrip("/")
+        if base:
+            patterns_to_add.add(base)
+    elif pat.endswith("/*"):
+        base = pat[:-2].rstrip("/")
+        if base:
+            patterns_to_add.add(base)
+            patterns_to_add.add(f"{base}/**")
+    elif pat.endswith("/"):
+        base = pat.rstrip("/")
+        if base:
+            patterns_to_add.add(base)
+            patterns_to_add.add(f"{base}/**")
+            patterns_to_add.add(f"{base}/*")
+    else:
+        full_path = os.path.join(abs_path_to_scan, pat)
+        if os.path.isdir(full_path):
+            patterns_to_add.add(f"{pat}/**")
+            patterns_to_add.add(f"{pat}/*")
+
+    return patterns_to_add
+
+
+def _expand_custom_exclude_pattern(pattern: str, abs_path_to_scan: str) -> set:
+    exclude_path_normalized = os.path.normpath(
+        pattern.replace('\\', '/').strip()
+    ).replace("\\", "/")
+    if not exclude_path_normalized:
+        return set()
+
+    patterns = set(_normalize_custom_pattern(exclude_path_normalized, abs_path_to_scan))
+
+    if exclude_path_normalized.endswith("/**"):
+        base_dir = exclude_path_normalized[:-3].rstrip("/")
+        if base_dir:
+            full_exclude_path = os.path.join(abs_path_to_scan, base_dir)
+            if os.path.isdir(full_exclude_path):
+                patterns.add(base_dir)
+            patterns.add(exclude_path_normalized)
+        else:
+            patterns.add(exclude_path_normalized)
+        return patterns
+
+    has_glob_chars = any(char in exclude_path_normalized for char in ['*', '?', '['])
+    if has_glob_chars:
+        patterns.add(exclude_path_normalized)
+        return patterns
+
+    full_exclude_path = os.path.join(abs_path_to_scan, exclude_path_normalized)
+    if os.path.isdir(full_exclude_path):
+        base_path = exclude_path_normalized.rstrip("/")
+        if base_path:
+            patterns.add(base_path)
+            patterns.add(f"{base_path}/**")
+        else:
+            patterns.add(exclude_path_normalized)
+    elif os.path.isfile(full_exclude_path):
+        ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower()
+        if ext in _WILDCARD_EXTENSIONS:
+            patterns.add(f"*.{ext}")
+        else:
+            patterns.add(f"**/{exclude_path_normalized}")
+    else:
+        ext = os.path.splitext(exclude_path_normalized)[1].lstrip('.').lower()
+        if ext in _WILDCARD_EXTENSIONS:
+            patterns.add(f"*.{ext}")
+        else:
+            patterns.add(exclude_path_normalized)
+
+    return patterns
+
+
 def _directory_ignore_pattern(dir_name: str) -> str:
     """Path-based glob for a directory name (avoids matching the scan root itself)."""
     normalized = dir_name.strip().strip("/").replace("\\", "/")
@@ -71,7 +160,10 @@ def _directory_ignore_pattern(dir_name: str) -> str:
     return f"**/{normalized}/**"
 
 
-def _default_scancode_coarse_ignore_patterns() -> frozenset:
+def _default_scancode_coarse_ignore_patterns(
+    path_to_exclude: list = None,
+    abs_path_to_scan: str = ""
+) -> frozenset:
     """
     Coarse ignore patterns aligned with fosslight_util.get_excluded_paths() rules.
     Directory names use path-based globs (e.g. **/tests/**) so they do not match
@@ -84,73 +176,93 @@ def _default_scancode_coarse_ignore_patterns() -> frozenset:
         patterns.add(f"*.{ext}")
     for name in EXCLUDE_FILENAME:
         patterns.add(name)
+
+    for pattern in path_to_exclude or []:
+        if os.path.isabs(pattern):
+            pattern = os.path.relpath(pattern, abs_path_to_scan)
+        patterns.update(_expand_custom_exclude_pattern(pattern, abs_path_to_scan))
+
     return frozenset(patterns)
 
 
-def _is_covered_by_coarse_ignore(rel_path: str, coarse_patterns: Iterable[str]) -> bool:
-    excludes = {pattern: "" for pattern in coarse_patterns}
+def _to_excludes_dict(patterns) -> dict:
+    return {pattern: "exclude" for pattern in patterns}
+
+
+def _is_path_covered(rel_path: str, excludes: dict) -> bool:
     return not is_included(rel_path, includes={}, excludes=excludes)
 
 
-def _add_path_to_exclude_pattern(
+def _add_ignore_pattern(
     patterns: set,
-    exclude_path: str,
-    abs_path_to_scan: str,
+    excludes: dict,
+    pattern: str,
+    *,
+    sample_path: str = None,
+) -> bool:
+    if pattern in patterns:
+        return False
+    if sample_path and _is_path_covered(sample_path, excludes):
+        return False
+    patterns.add(pattern)
+    excludes[pattern] = "exclude"
+    return True
+
+
+def _make_pre_scan_skip_filter(
     coarse_patterns: frozenset,
+) -> Tuple[dict, Callable[[str, str], bool], Callable[[str, str], bool]]:
+    excludes = _to_excludes_dict(coarse_patterns)
+
+    def should_skip_dir(dir_name: str, rel_dir: str) -> bool:
+        if dir_name.startswith('.'):
+            return True
+        if dir_name.lower() in _SKIP_DIR_NAMES:
+            return True
+        return _is_path_covered(f"{rel_dir}/_", excludes)
+
+    def should_skip_file(file_name: str, rel_path: str) -> bool:
+        if file_name.startswith('.'):
+            return True
+        ext = os.path.splitext(file_name)[1].lstrip('.').lower()
+        if ext in _SKIP_EXTS:
+            return True
+        return _is_path_covered(rel_path, excludes)
+
+    return excludes, should_skip_dir, should_skip_file
+
+
+def _add_binary_ignore_patterns(
+    patterns: set,
+    excludes: dict,
+    binary_paths: list,
 ) -> None:
-    exclude_path_normalized = os.path.normpath(exclude_path).replace("\\", "/")
+    extensions = set()
+    no_ext_paths = []
 
-    if exclude_path_normalized.endswith("/**"):
-        base_dir = exclude_path_normalized[:-3].rstrip("/")
-        if base_dir:
-            full_exclude_path = os.path.join(abs_path_to_scan, base_dir)
-            if os.path.isdir(full_exclude_path):
-                patterns.add(base_dir)
-                patterns.add(exclude_path_normalized)
-            else:
-                patterns.add(exclude_path_normalized)
+    for rel_path in binary_paths:
+        ext = os.path.splitext(rel_path)[1].lstrip('.').lower()
+        if ext:
+            extensions.add(ext)
         else:
-            patterns.add(exclude_path_normalized)
-        return
+            no_ext_paths.append(rel_path)
 
-    has_glob_chars = any(char in exclude_path_normalized for char in ['*', '?', '['])
-    if has_glob_chars:
-        patterns.add(exclude_path_normalized)
-        return
+    for ext in extensions:
+        _add_ignore_pattern(patterns, excludes, f"*.{ext}")
 
-    full_exclude_path = os.path.join(abs_path_to_scan, exclude_path_normalized)
-    if os.path.isdir(full_exclude_path):
-        base_path = exclude_path_normalized.rstrip("/")
-        if base_path:
-            patterns.add(base_path)
-            patterns.add(f"{base_path}/**")
-        else:
-            patterns.add(exclude_path_normalized)
-    elif os.path.isfile(full_exclude_path):
-        if not _is_covered_by_coarse_ignore(exclude_path_normalized, coarse_patterns):
-            patterns.add(f"**/{exclude_path_normalized}")
-    else:
-        patterns.add(exclude_path_normalized)
+    for rel_path in no_ext_paths:
+        _add_ignore_pattern(
+            patterns, excludes, f"**/{rel_path}", sample_path=rel_path
+        )
 
 
 def _build_scancode_ignore_patterns(
-    path_to_exclude: list,
-    abs_path_to_scan: str,
+    coarse_patterns: frozenset,
     binary_paths: list,
 ) -> tuple:
-    coarse_patterns = _default_scancode_coarse_ignore_patterns()
     patterns = set(coarse_patterns)
-
-    for path in path_to_exclude or []:
-        if os.path.isabs(path):
-            exclude_path = os.path.relpath(path, abs_path_to_scan)
-        else:
-            exclude_path = path
-        _add_path_to_exclude_pattern(patterns, exclude_path, abs_path_to_scan, coarse_patterns)
-
-    for rel_path in binary_paths:
-        patterns.add(f"**/{rel_path}")
-
+    excludes = _to_excludes_dict(coarse_patterns)
+    _add_binary_ignore_patterns(patterns, excludes, binary_paths)
     return tuple(sorted(patterns))
 
 
@@ -168,12 +280,12 @@ def run_scan(
 
     success = True
     msg = ""
-    _str_final_result_log = ""
     _result_log = {}
     result_list = []
     license_list = []
     _json_ext = ".json"
-    _start_time = datetime.now().strftime('%y%m%d_%H%M')
+    _start_time = current_timestamp_utc()
+    _file_time = timestamp_for_filename(_start_time)
 
     if not correct_filepath:
         correct_filepath = path_to_scan
@@ -189,9 +301,9 @@ def run_scan(
             for i, output_extension in enumerate(output_extensions):
                 if output_files[i] is None or output_files[i] == "":
                     if output_extension == _json_ext:
-                        output_files[i] = f"fosslight_opossum_src_{_start_time}"
+                        output_files[i] = f"fosslight_opossum_src_{_file_time}"
                     else:
-                        output_files[i] = f"fosslight_report_src_{_start_time}"
+                        output_files[i] = f"fosslight_report_src_{_file_time}"
 
         if _write_json_file:
             output_json_file = os.path.join(output_path, "scancode_raw_result.json")
@@ -199,8 +311,13 @@ def run_scan(
             output_json_file = ""
 
         if not called_by_cli:
-            logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
-                                           True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
+            log_file_path = os.path.join(
+                output_path, f"fosslight_log_src_{_file_time}.txt"
+            )
+            logger, _result_log = init_log(
+                log_file_path, True, logging.INFO, logging.DEBUG,
+                _PKG_NAME, path_to_scan, path_to_exclude
+            )
 
             logger.info(f"Tool Info : {_result_log['Tool Info']}")
 
@@ -215,21 +332,36 @@ def run_scan(
                 pretty_params["output_file"] = output_file_name
                 abs_path_to_scan = os.path.abspath(path_to_scan)
                 binary_paths = []
-                for root, _, files in os.walk(path_to_scan):
+                coarse_patterns = _default_scancode_coarse_ignore_patterns(
+                    path_to_exclude, abs_path_to_scan
+                )
+                _, should_skip_dir, should_skip_file = _make_pre_scan_skip_filter(
+                    coarse_patterns
+                )
+
+                for root, dirs, files in os.walk(path_to_scan):
+                    rel_root = os.path.relpath(root, abs_path_to_scan).replace("\\", "/")
+                    dirs[:] = [
+                        d for d in dirs
+                        if not should_skip_dir(
+                            d, d if rel_root == "." else f"{rel_root}/{d}"
+                        )
+                    ]
                     for name in files:
+                        rel_path = name if rel_root == "." else f"{rel_root}/{name}"
+                        if should_skip_file(name, rel_path):
+                            continue
                         full_path = os.path.join(root, name)
                         try:
                             if not check_binary(full_path, True):
                                 continue
                         except Exception:
                             continue
-                        rel_path = os.path.relpath(full_path, abs_path_to_scan)
-                        rel_norm = os.path.normpath(rel_path).replace("\\", "/")
-                        binary_paths.append(rel_norm)
-                        logger.debug(f"Excluded binary from scancode: {rel_norm}")
+                        binary_paths.append(rel_path)
+                        logger.debug(f"Excluded binary from scancode: {rel_path}")
 
                 ignore_tuple = _build_scancode_ignore_patterns(
-                    path_to_exclude, abs_path_to_scan, binary_paths
+                    coarse_patterns, binary_paths
                 )
                 logger.debug(f"Scancode ignore patterns: {len(ignore_tuple)}")
 
@@ -292,11 +424,6 @@ def run_scan(
     scan_result_msg = str(success) if msg == "" else f"{success}, {msg}"
     _result_log["Scan Result"] = scan_result_msg
     _result_log["Output Directory"] = output_path
-    try:
-        _str_final_result_log = yaml.safe_dump(_result_log, allow_unicode=True, sort_keys=True)
-        logger.info(_str_final_result_log)
-    except Exception as ex:
-        logger.warning(f"Failed to print result log. {ex}")
 
     if not success:
         logger.error(f"Failed to run: {scan_result_msg}")

{fosslight_source-2.3.2 → fosslight_source-2.3.4/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.3.2
+Version: 2.3.4
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -20,12 +20,13 @@ Requires-Dist: setuptools<=80.10.2
 Requires-Dist: pyparsing
 Requires-Dist: scanoss>=1.19.0
 Requires-Dist: XlsxWriter
-Requires-Dist: fosslight_util>=2.1.37
+Requires-Dist: fosslight_util>=2.2.2
 Requires-Dist: PyYAML
 Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
 Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
+Requires-Dist: lxml>=6.0.1
 Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/src/fosslight_source.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ src/fosslight_source/__init__.py
 src/fosslight_source/_help.py
 src/fosslight_source/_kb_client.py
 src/fosslight_source/_license_matched.py
+src/fosslight_source/_merge.py
 src/fosslight_source/_parsing_scancode_file_item.py
 src/fosslight_source/_parsing_scanoss_file.py
 src/fosslight_source/_scan_item.py

{fosslight_source-2.3.2 → fosslight_source-2.3.4}/src/fosslight_source.egg-info/requires.txt

@@ -2,12 +2,13 @@ setuptools<=80.10.2
 pyparsing
 scanoss>=1.19.0
 XlsxWriter
-fosslight_util>=2.1.37
+fosslight_util>=2.2.2
 PyYAML
 wheel>=0.38.1
 intbitset
 fosslight_binary>=5.1.22
 scancode-toolkit>=32.0.2
+lxml>=6.0.1
 fingerprints==1.2.3
 normality==2.6.1
 tqdm