fosslight-source 2.3.4__tar.gz → 2.3.6__tar.gz

{fosslight_source-2.3.4/src/fosslight_source.egg-info → fosslight_source-2.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.3.4
+Version: 2.3.6
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -24,7 +24,6 @@ Requires-Dist: fosslight_util>=2.2.2
 Requires-Dist: PyYAML
 Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
-Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
 Requires-Dist: lxml>=6.0.1
 Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fosslight_source"
-version = "2.3.4"
+version = "2.3.6"
 description = "FOSSLight Source Scanner"
 readme = "README.md"
 license = "Apache-2.0"
@@ -33,7 +33,6 @@ dependencies = [
     "PyYAML",
     "wheel>=0.38.1",
     "intbitset",
-    "fosslight_binary>=5.1.22",
     "scancode-toolkit>=32.0.2",
     "lxml>=6.0.1",
     # cryptography 49.x does not provide macOS x86_64 wheels, causing source builds to require OpenSSL/pkg-config.
@@ -56,6 +55,9 @@ Download = "https://github.com/fosslight/fosslight_source_scanner"
 [project.scripts]
 fosslight_source = "fosslight_source.cli:main"
 
+[project.entry-points."scancode_pre_scan"]
+ignore-binaries = "fosslight_source._scancode_ignore_binaries:IgnoreBinaries"
+
 [tool.setuptools]
 package-dir = {"" = "src"}
 

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source/_help.py

@@ -37,7 +37,7 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
     ────────────────────────────────────────────────────────────────────
     -s <mode>              Choose mode: scancode, scanoss, kb, or all(default)
     -c <number>            Number of CPU cores/threads to use for scanning
-    -t <seconds>           Timeout in seconds for ScanCode scanning
+    -t <seconds>           Timeout in seconds for ScanCode and SCANOSS scanning
     -j                     Generate raw scanner results in JSON format
     --no_merge             Keep source paths file-based without folder merge
     --no_correction        Skip OSS information correction with sbom-info.yaml

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source/_parsing_scancode_file_item.py

@@ -73,7 +73,9 @@ def get_error_from_header(header_item: list) -> Tuple[bool, str]:
     return has_error, str_error
 
 
-def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = False) -> Tuple[bool, list, list, dict]:
+def parsing_scancode_32_earlier(
+    scancode_file_list: list, has_error: bool = False, ui_mode: bool = False
+) -> Tuple[bool, list, list, dict]:
     rc = True
     msg = []
     scancode_file_item = []
@@ -118,7 +120,10 @@ def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = Fals
 
                     # Set the license value
                     license_detected = []
-                    if licenses is None or licenses == "":
+                    if not licenses:
+                        licenses = []
+                    # Keep license and/or copyright findings; UI keeps finding-less files too.
+                    if not licenses and not copyright_value_list and not ui_mode:
                         continue
 
                     license_expression_list = file.get("license_expressions", {})
@@ -191,6 +196,9 @@ def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = Fals
                                 set(license_expression_list))
                             result_item.comment = ','.join(license_expression_list)
 
+                        scancode_file_item.append(result_item)
+                    elif copyright_value_list or ui_mode:
+                        result_item.copyright = copyright_value_list
                         scancode_file_item.append(result_item)
             except Exception as ex:
                 msg.append(f"Error Parsing item: {ex}")
@@ -223,7 +231,7 @@ def get_license_expression_spdx(license_expression: str) -> str:
 
 
 def parsing_scancode_32_later(
-    scancode_file_list: list, has_error: bool = False
+    scancode_file_list: list, has_error: bool = False, ui_mode: bool = False
 ) -> Tuple[bool, list, list, dict]:
     rc = True
     msg = []
@@ -258,9 +266,10 @@ def parsing_scancode_32_later(
                         copyright_value_list.append(copyright_data)
                 license_detected = []
                 licenses = file.get("license_detections", [])
-                if not licenses:
+                # Keep license and/or copyright findings; UI keeps finding-less files too.
+                if not licenses and not copyright_value_list and not ui_mode:
                     continue
-                for lic in licenses:
+                for lic in licenses or []:
                     matched_lic_list = lic.get("matches", [])
                     for matched_lic in matched_lic_list:
                         found_lic_list = matched_lic.get("license_expression", "")
@@ -322,7 +331,8 @@ def parsing_scancode_32_later(
 
 
 def parsing_file_item(
-    scancode_file_list: list, has_error: bool, need_matched_license: bool = False
+    scancode_file_list: list, has_error: bool, need_matched_license: bool = False,
+    ui_mode: bool = False
 ) -> Tuple[bool, list, list, dict]:
 
     rc = True
@@ -330,9 +340,13 @@ def parsing_file_item(
 
     first_item = next(iter(scancode_file_list or []), {})
     if "licenses" in first_item:
-        rc, scancode_file_item, msg, license_list = parsing_scancode_32_earlier(scancode_file_list, has_error)
+        rc, scancode_file_item, msg, license_list = parsing_scancode_32_earlier(
+            scancode_file_list, has_error, ui_mode
+        )
     else:
-        rc, scancode_file_item, msg, license_list = parsing_scancode_32_later(scancode_file_list, has_error)
+        rc, scancode_file_item, msg, license_list = parsing_scancode_32_later(
+            scancode_file_list, has_error, ui_mode
+        )
     if not need_matched_license:
         license_list = {}
     return rc, scancode_file_item, msg, license_list

fosslight_source-2.3.6/src/fosslight_source/_scancode_ignore_binaries.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2018 nexB Inc. and others.
+# Copyright (c) 2026 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Vendored from scancode-ignore-binaries (aboutcode-org/scancode-plugins)
+# so PyPI installs do not need a GitHub git dependency.
+# SPDX-PackageDownloadLocation: https://github.com/aboutcode-org/scancode-plugins/tree/main/misc/scancode-ignore-binaries
+
+from plugincode.pre_scan import PreScanPlugin
+from plugincode.pre_scan import pre_scan_impl
+from commoncode.cliutils import PluggableCommandLineOption
+from commoncode.cliutils import PRE_SCAN_GROUP
+from typecode.contenttype import get_type
+
+
+@pre_scan_impl
+class IgnoreBinaries(PreScanPlugin):
+    """
+    Ignore binary files.
+    """
+
+    options = [
+        PluggableCommandLineOption(
+            ('--ignore-binaries',),
+            is_flag=True,
+            help='Ignore binary files.',
+            sort_order=10,
+            help_group=PRE_SCAN_GROUP,
+        )
+    ]
+
+    def is_enabled(self, ignore_binaries, **kwargs):
+        return ignore_binaries
+
+    def process_codebase(self, codebase, ignore_binaries, **kwargs):
+        """
+        Remove binary Resources from the resource tree.
+        """
+        if not ignore_binaries:
+            return
+
+        resources_to_remove = []
+        for resource in codebase.walk():
+            if not resource.is_file:
+                continue
+            if is_binary(resource.location):
+                resources_to_remove.append(resource)
+
+        for resource in resources_to_remove:
+            resource.remove(codebase)
+
+
+def is_binary(location):
+    """
+    Return True if the resource at location is a binary file.
+    """
+    t = get_type(location)
+    return (
+        t.is_binary
+        or t.is_archive
+        or t.is_media
+        or t.is_office_doc
+        or t.is_compressed
+        or t.is_filesystem
+        or t.is_winexe
+        or t.is_elf
+        or t.is_java_class
+        or t.is_data
+    )

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source/cli.py

@@ -90,6 +90,7 @@ def main() -> None:
     parser.add_argument('--kb_url', type=str, required=False, default="")
     parser.add_argument('--kb_token', type=str, required=False, default="")
     parser.add_argument('--no_merge', action='store_true', required=False)
+    parser.add_argument('--ui', action='store_true', required=False)
 
     args = parser.parse_args()
 
@@ -121,6 +122,7 @@ def main() -> None:
     kb_url = args.kb_url
     kb_token = args.kb_token
     merge_by_folder = not args.no_merge
+    ui_mode = args.ui
 
     time_out = args.timeout
     core = args.cores
@@ -130,7 +132,7 @@ def main() -> None:
                               print_matched_text, formats, time_out, correct_mode, correct_filepath,
                               selected_scanner, path_to_exclude, hide_progress=hide_progress,
                               kb_url=kb_url, kb_token=kb_token,
-                              merge_by_folder=merge_by_folder)
+                              merge_by_folder=merge_by_folder, ui_mode=ui_mode)
 
         _result_log["Scan Result"] = result[1]
         try:
@@ -149,7 +151,7 @@ def create_report_file(
     output_path: str = "", output_files: list = [],
     output_extensions: list = [], correct_mode: bool = True,
     correct_filepath: str = "", path_to_scan: str = "", path_to_exclude: list = [],
-    formats: list = [], api_limit_exceed: bool = False, files_count: int = 0, final_output_path: str = "",
+    formats: list = [], scanoss_skipped: bool = False, files_count: int = 0, final_output_path: str = "",
     run_kb_msg: str = "", merge_by_folder: bool = True
 ) -> 'ScannerItem':
     """
@@ -216,10 +218,14 @@ def create_report_file(
         else:
             scan_item.set_cover_comment("(No OSS detected.)")
 
-    if api_limit_exceed:
-        scan_item.set_cover_comment("SCANOSS skipped (API limits)")
+    if scanoss_skipped:
+        is_kb_success = run_kb_msg != "" and run_kb_msg.endswith("Completed")
+        if is_kb_success:
+            scan_item.set_cover_comment("SCANOSS replaced with KB")
+        else:
+            scan_item.set_cover_comment("SCANOSS skipped")
 
-    if run_kb_msg:
+    if run_kb_msg and not run_kb_msg.endswith("Completed"):
         scan_item.set_cover_comment(run_kb_msg)
     display_mode = selected_scanner
     if selected_scanner == ALL_MODE:
@@ -390,11 +396,20 @@ def _collect_kb_file_hashes(
 def merge_results(
     scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {},
     path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {},
-    excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = ""
+    excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = "",
+    ui_mode: bool = False
 ) -> tuple[list, Optional[str], int, int]:
 
     """
     Merge scanner results and spdx parsing result.
+
+    ScanOSS items for files already present in scancode_result (same path) do not
+    replace the ScanCode license; only OSS name, version, and download location
+    from ScanOSS are applied. ScanOSS-only files are appended in full.
+
+    When ui_mode is False, items with both empty download location and empty
+    license are removed after merging.
+
     :param scancode_result: list of scancode results in SourceItem.
     :param scanoss_result: list of scanoss results in SourceItem.
     :param spdx_downloads: dictionary of spdx parsed results.
@@ -403,12 +418,27 @@ def merge_results(
     :param excluded_files: set of relative paths to exclude from KB-only file discovery.
     :param kb_url: KB API base URL.
     :param kb_token: KB API bearer token.
+    :param ui_mode: if False, drop items with no download location and no license.
     :return: (merged_result, kb failure message, requested file_hash count, returned match count).
     """
     if excluded_files is None:
         excluded_files = set()
 
-    scancode_result.extend([item for item in scanoss_result if item not in scancode_result])
+    # Merge ScanOSS into ScanCode results.
+    # When ScanCode already detected a license for the same file, keep that license
+    # and only load OSS name, version, and download location from ScanOSS.
+    # Files found only by ScanOSS are appended as-is (including ScanOSS licenses).
+    for scanoss_item in scanoss_result:
+        if scanoss_item in scancode_result:
+            scancode_item = scancode_result[scancode_result.index(scanoss_item)]
+            if scanoss_item.oss_name:
+                scancode_item.oss_name = scanoss_item.oss_name
+            if scanoss_item.oss_version:
+                scancode_item.oss_version = scanoss_item.oss_version
+            if scanoss_item.download_location:
+                scancode_item.download_location = scanoss_item.download_location
+        else:
+            scancode_result.append(scanoss_item)
 
     # If download loc. in SPDX form found, overwrite the scanner result.
     # If scanner result doesn't exist, create a new row.
@@ -424,19 +454,16 @@ def merge_results(
     if manifest_licenses:
         for file_name, licenses in manifest_licenses.items():
             valid_licenses = [lic.strip() for lic in licenses if isinstance(lic, str) and lic.strip()]
-            if not valid_licenses:
+            # Non-UI: skip manifests with no extracted licenses.
+            # UI: keep/create the row and mark is_manifest_file even without licenses.
+            if not valid_licenses and not ui_mode:
                 continue
-            if file_name in scancode_result:
-                merged_result_item = scancode_result[scancode_result.index(file_name)]
+            item = _get_or_append_source_item(scancode_result, file_name)
+            item.is_manifest_file = True
+            if valid_licenses:
                 # overwrite existing detected licenses with manifest-provided licenses
-                merged_result_item.licenses = []  # clear existing licenses (setter clears when value falsy)
-                merged_result_item.licenses = valid_licenses
-                merged_result_item.is_manifest_file = True
-            else:
-                new_result_item = SourceItem(file_name)
-                new_result_item.licenses = valid_licenses
-                new_result_item.is_manifest_file = True
-                scancode_result.append(new_result_item)
+                item.licenses = []  # clear existing licenses (setter clears when value falsy)
+                item.licenses = valid_licenses
 
     kb_origin_urls: dict[str, str] = {}
     kb_status_message: Optional[str] = None
@@ -465,9 +492,36 @@ def merge_results(
             if extra_item.download_location:
                 scancode_result.append(extra_item)
 
+    if not ui_mode:
+        scancode_result[:] = [
+            item for item in scancode_result
+            if not _has_empty_download_and_license(item)
+        ]
+
     return scancode_result, kb_status_message, kb_requested_count, kb_returned_count
 
 
+def _get_or_append_source_item(scancode_result: list, file_name: str) -> SourceItem:
+    if file_name in scancode_result:
+        return scancode_result[scancode_result.index(file_name)]
+    item = SourceItem(file_name)
+    scancode_result.append(item)
+    return item
+
+
+def _has_empty_download_and_license(item: SourceItem) -> bool:
+    downloads = item.download_location
+    if isinstance(downloads, str):
+        has_download = bool(downloads.strip())
+    else:
+        has_download = bool(downloads) and any(bool(d and str(d).strip()) for d in downloads)
+
+    licenses = item.licenses
+    has_license = bool(licenses) and any(bool(lic and str(lic).strip()) for lic in licenses)
+
+    return (not has_download) and (not has_license)
+
+
 def _finalize_temp_output(
     temp_output_path: str,
     final_output_path: str,
@@ -502,7 +556,8 @@ def run_scanners(
     correct_mode: bool = True, correct_filepath: str = "",
     selected_scanner: str = ALL_MODE, path_to_exclude: list = [],
     all_exclude_mode: tuple = (), hide_progress: bool = False,
-    kb_url: str = "", kb_token: str = "", merge_by_folder: bool = True
+    kb_url: str = "", kb_token: str = "", merge_by_folder: bool = True,
+    ui_mode: bool = False
 ) -> Tuple[bool, str, 'ScannerItem', list, list]:
     """
     Run Scancode and scanoss.py for the given path.
@@ -532,7 +587,7 @@ def run_scanners(
     spdx_downloads = {}
     result_log = {}
     scan_item = []
-    api_limit_exceed = False
+    scanoss_skipped = False
     kb_url, kb_token = resolve_kb_config(kb_url, kb_token)
 
     success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats)
@@ -582,12 +637,15 @@ def run_scanners(
                     print_matched_text, formats, called_by_cli, time_out, correct_mode,
                     correct_filepath, path_to_exclude,
                     hide_progress=hide_progress,
+                    ui_mode=ui_mode,
                 )
             excluded_files = set(excluded_files) if excluded_files else set()
             if selected_scanner in ['scanoss', ALL_MODE]:
-                scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_path, formats, True, num_cores,
-                                                                  excluded_path_with_default_exclusion, excluded_files,
-                                                                  write_json_file, hide_progress)
+                scanoss_result, scanoss_skipped = run_scanoss_py(
+                    path_to_scan, output_path, formats, True, num_cores,
+                    excluded_path_with_default_exclusion, excluded_files,
+                    write_json_file, hide_progress, timeout=time_out
+                )
 
             run_kb_msg = ""
             if selected_scanner in SCANNER_TYPE:
@@ -602,6 +660,7 @@ def run_scanners(
                     scancode_result, scanoss_result, spdx_downloads,
                     path_to_scan, run_kb, manifest_licenses, excluded_files,
                     hide_progress, kb_url, kb_token,
+                    ui_mode=ui_mode,
                 )
                 if kb_status_message:
                     run_kb_msg = f"KB({kb_url}) {kb_status_message}"
@@ -611,7 +670,7 @@ def run_scanners(
                 scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                                print_matched_text, output_path, output_files, output_extensions, correct_mode,
                                                correct_filepath, path_to_scan, excluded_path_without_dot, formats,
-                                               api_limit_exceed, cnt_file_except_skipped, final_output_path, run_kb_msg,
+                                               scanoss_skipped, cnt_file_except_skipped, final_output_path, run_kb_msg,
                                                merge_by_folder)
             else:
                 print_help_msg_source_scanner()
@@ -653,7 +712,7 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> dict:
 
     - Traverse files with exclusions applied
     - spdx_downloads: {rel_path: [download_urls]}
-    - manifest_licenses: {rel_path: [license_names]}
+    - manifest_licenses: {rel_path: [license_names]} (empty list if extraction failed)
 
     :return: (spdx_downloads, manifest_licenses)
     """
@@ -673,9 +732,7 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> dict:
                 spdx_downloads[rel_path_file] = downloads
 
             if is_manifest_file(file_path):
-                licenses = get_manifest_licenses(file_path)
-                if licenses:
-                    manifest_licenses[rel_path_file] = licenses
+                manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or []
 
     return spdx_downloads, manifest_licenses
 

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source/run_scancode.py

@@ -7,6 +7,8 @@ import os
 import multiprocessing
 import warnings
 import logging
+from typing import Tuple
+
 from scancode import cli
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
@@ -14,15 +16,12 @@ from fosslight_util.time import current_timestamp_utc, timestamp_for_filename
 from ._parsing_scancode_file_item import parsing_file_item
 from ._parsing_scancode_file_item import get_error_from_header
 from fosslight_util.output_format import check_output_formats_v2
-from fosslight_binary.binary_analysis import check_binary
 from fosslight_util.exclude import (
     EXCLUDE_DIRECTORY,
     EXCLUDE_FILE_EXTENSION,
     EXCLUDE_FILENAME,
     PACKAGE_DIRECTORY,
 )
-from commoncode.fileset import is_included
-from typing import Callable, Tuple
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 warnings.filterwarnings("ignore", category=FutureWarning)
@@ -68,8 +67,6 @@ _WILDCARD_EXTENSIONS = {
     "zip", "tar", "tgz", "gz",
     "bmp", "webp", "ico",
 } | {ext.lower() for ext in EXCLUDE_FILE_EXTENSION}
-_SKIP_DIR_NAMES = frozenset(name.lower() for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY)
-_SKIP_EXTS = frozenset(ext.lower() for ext in EXCLUDE_FILE_EXTENSION)
 
 
 def _normalize_custom_pattern(pattern: str, abs_path_to_scan: str) -> set:
@@ -160,14 +157,15 @@ def _directory_ignore_pattern(dir_name: str) -> str:
     return f"**/{normalized}/**"
 
 
-def _default_scancode_coarse_ignore_patterns(
+def _default_scancode_ignore_patterns(
     path_to_exclude: list = None,
     abs_path_to_scan: str = ""
-) -> frozenset:
+) -> tuple:
     """
     Coarse ignore patterns aligned with fosslight_util.get_excluded_paths() rules.
     Directory names use path-based globs (e.g. **/tests/**) so they do not match
     the scan root directory name itself.
+    Binary files are excluded separately via scancode --ignore-binaries.
     """
     patterns = {".*"}
     for name in PACKAGE_DIRECTORY + EXCLUDE_DIRECTORY:
@@ -182,87 +180,6 @@ def _default_scancode_coarse_ignore_patterns(
             pattern = os.path.relpath(pattern, abs_path_to_scan)
         patterns.update(_expand_custom_exclude_pattern(pattern, abs_path_to_scan))
 
-    return frozenset(patterns)
-
-
-def _to_excludes_dict(patterns) -> dict:
-    return {pattern: "exclude" for pattern in patterns}
-
-
-def _is_path_covered(rel_path: str, excludes: dict) -> bool:
-    return not is_included(rel_path, includes={}, excludes=excludes)
-
-
-def _add_ignore_pattern(
-    patterns: set,
-    excludes: dict,
-    pattern: str,
-    *,
-    sample_path: str = None,
-) -> bool:
-    if pattern in patterns:
-        return False
-    if sample_path and _is_path_covered(sample_path, excludes):
-        return False
-    patterns.add(pattern)
-    excludes[pattern] = "exclude"
-    return True
-
-
-def _make_pre_scan_skip_filter(
-    coarse_patterns: frozenset,
-) -> Tuple[dict, Callable[[str, str], bool], Callable[[str, str], bool]]:
-    excludes = _to_excludes_dict(coarse_patterns)
-
-    def should_skip_dir(dir_name: str, rel_dir: str) -> bool:
-        if dir_name.startswith('.'):
-            return True
-        if dir_name.lower() in _SKIP_DIR_NAMES:
-            return True
-        return _is_path_covered(f"{rel_dir}/_", excludes)
-
-    def should_skip_file(file_name: str, rel_path: str) -> bool:
-        if file_name.startswith('.'):
-            return True
-        ext = os.path.splitext(file_name)[1].lstrip('.').lower()
-        if ext in _SKIP_EXTS:
-            return True
-        return _is_path_covered(rel_path, excludes)
-
-    return excludes, should_skip_dir, should_skip_file
-
-
-def _add_binary_ignore_patterns(
-    patterns: set,
-    excludes: dict,
-    binary_paths: list,
-) -> None:
-    extensions = set()
-    no_ext_paths = []
-
-    for rel_path in binary_paths:
-        ext = os.path.splitext(rel_path)[1].lstrip('.').lower()
-        if ext:
-            extensions.add(ext)
-        else:
-            no_ext_paths.append(rel_path)
-
-    for ext in extensions:
-        _add_ignore_pattern(patterns, excludes, f"*.{ext}")
-
-    for rel_path in no_ext_paths:
-        _add_ignore_pattern(
-            patterns, excludes, f"**/{rel_path}", sample_path=rel_path
-        )
-
-
-def _build_scancode_ignore_patterns(
-    coarse_patterns: frozenset,
-    binary_paths: list,
-) -> tuple:
-    patterns = set(coarse_patterns)
-    excludes = _to_excludes_dict(coarse_patterns)
-    _add_binary_ignore_patterns(patterns, excludes, binary_paths)
     return tuple(sorted(patterns))
 
 
@@ -273,7 +190,8 @@ def run_scan(
     formats: list = [], called_by_cli: bool = False,
     time_out: int = 120, correct_mode: bool = True,
     correct_filepath: str = "", path_to_exclude: list = [],
-    excluded_files: list = [], hide_progress: bool = False
+    excluded_files: list = [], hide_progress: bool = False,
+    ui_mode: bool = False
 ) -> Tuple[bool, str, list, list]:
     if not called_by_cli:
         global logger
@@ -331,38 +249,9 @@ def run_scan(
                 pretty_params["path_to_exclude"] = path_to_exclude
                 pretty_params["output_file"] = output_file_name
                 abs_path_to_scan = os.path.abspath(path_to_scan)
-                binary_paths = []
-                coarse_patterns = _default_scancode_coarse_ignore_patterns(
+                ignore_tuple = _default_scancode_ignore_patterns(
                     path_to_exclude, abs_path_to_scan
                 )
-                _, should_skip_dir, should_skip_file = _make_pre_scan_skip_filter(
-                    coarse_patterns
-                )
-
-                for root, dirs, files in os.walk(path_to_scan):
-                    rel_root = os.path.relpath(root, abs_path_to_scan).replace("\\", "/")
-                    dirs[:] = [
-                        d for d in dirs
-                        if not should_skip_dir(
-                            d, d if rel_root == "." else f"{rel_root}/{d}"
-                        )
-                    ]
-                    for name in files:
-                        rel_path = name if rel_root == "." else f"{rel_root}/{name}"
-                        if should_skip_file(name, rel_path):
-                            continue
-                        full_path = os.path.join(root, name)
-                        try:
-                            if not check_binary(full_path, True):
-                                continue
-                        except Exception:
-                            continue
-                        binary_paths.append(rel_path)
-                        logger.debug(f"Excluded binary from scancode: {rel_path}")
-
-                ignore_tuple = _build_scancode_ignore_patterns(
-                    coarse_patterns, binary_paths
-                )
                 logger.debug(f"Scancode ignore patterns: {len(ignore_tuple)}")
 
                 kwargs = {
@@ -374,12 +263,14 @@ def run_scan(
                     "processes": num_cores,
                     "pretty_params": pretty_params,
                     "output_json_pp": output_json_file,
-                    "only_findings": True,
+                    # Non-UI: only files with findings (license/copyright). UI: all files.
+                    "only_findings": not ui_mode,
                     "license_text": True,
                     "url": True,
                     "timeout": time_out,
                     "include": (),
                     "ignore": ignore_tuple,
+                    "ignore_binaries": True,
                     "quiet": hide_progress
                 }
                 _apply_scancode_unset_workaround(kwargs)
@@ -395,8 +286,9 @@ def run_scan(
                             _result_log["Error_files"] = error_msg
                             msg = "Failed to analyze :" + error_msg
                     if "files" in results:
-                        rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"],
-                                                                                       has_error, need_license)
+                        rc, result_list, parsing_msg, license_list = parsing_file_item(
+                            results["files"], has_error, need_license, ui_mode=ui_mode
+                        )
                         if parsing_msg:
                             _result_log["Parsing Log"] = parsing_msg
                         if rc:
@@ -404,12 +296,6 @@ def run_scan(
                                 success = True
                             result_list = sorted(
                                 result_list, key=lambda row: (''.join(row.licenses)))
-
-                            for scan_item in result_list:
-                                if os.path.isdir(scan_item.source_name_or_path):
-                                    continue
-                                if check_binary(os.path.join(path_to_scan, scan_item.source_name_or_path), True):
-                                    scan_item.exclude = True
             except Exception as ex:
                 success = False
                 msg = str(ex)

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source/run_scanoss.py

@@ -31,7 +31,8 @@ def get_scanoss_extra_info(scanned_result: dict) -> list:
 def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
                    called_by_cli: bool = False, num_threads: int = -1,
                    path_to_exclude: list = [], excluded_files: set = None,
-                   write_json_file: bool = False, hide_progress: bool = False) -> Tuple[list, bool]:
+                   write_json_file: bool = False, hide_progress: bool = False,
+                   timeout: int = 120) -> Tuple[list, bool]:
     """
     Run scanoss.py for the given path.
 
@@ -40,23 +41,25 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
     :param format: Output file format (not being used except when calling check_output_format).
     :param called_by_cli: if not called by cli, initialize logger.
     :param write_json_file: if requested, keep the raw files.
+    :param timeout: timeout in seconds for SCANOSS API request.
     :return scanoss_file_list: list of ScanItem (scanned result by files).
     """
 
     scanoss_file_list = []
-    api_limit_exceed = False
+    scanoss_skipped = False
     try:
         importlib_metadata.distribution("scanoss")
     except Exception as error:
         logger.warning(f"{error}. Skipping scan with scanoss.")
         logger.warning("Please install scanoss and dataclasses before run fosslight_source with scanoss option.")
-        return scanoss_file_list, api_limit_exceed
+        return scanoss_file_list, scanoss_skipped
 
     output_json_file = os.path.join(output_path, SCANOSS_OUTPUT_FILE)
     output_wfp_file = os.path.join(output_path, SCANOSS_RESULT_FILE)
     if os.path.exists(output_json_file):
         os.remove(output_json_file)
 
+    output_buffer = io.StringIO()
     try:
         logger.debug(f"|---Running SCANOSS on {path_to_scan}")
         scanoss_settings = ScanossSettings()
@@ -66,15 +69,77 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
             scan_output=output_json_file,
             scan_options=ScanType.SCAN_SNIPPETS.value,
             nb_threads=num_threads if num_threads > 0 else 10,
-            scanoss_settings=scanoss_settings
+            scanoss_settings=scanoss_settings,
+            timeout=timeout,
+            retry=0
         )
-        output_buffer = io.StringIO()
+
+        # Check API connectivity & API Limit using dummy WFP POST
+        try:
+            logger.debug(f"|---Checking SCANOSS API connectivity to {scanner.scanoss_api.url}")
+            dummy_wfp = "file=72214db4e1e543018d1bafe86ea3b444,21,dummy.txt\nfh2=b200cd2eff5d535886e598b3a833aab5\n"
+            ping_response = scanner.scanoss_api.session.post(
+                scanner.scanoss_api.url,
+                files={'file': ('dummy.wfp', dummy_wfp)},
+                headers=scanner.scanoss_api.headers,
+                timeout=5
+            )
+            if ping_response.status_code != 200:
+                response_text = ping_response.text.lower()
+                is_limit = ping_response.status_code in [429, 503]
+                is_limit = is_limit or "rate limit" in response_text
+                is_limit = is_limit or "limits being exceeded" in response_text
+                if is_limit:
+                    logger.debug(f"[SCANOSS] API Limit Exceeded: HTTP {ping_response.status_code}")
+                elif ping_response.status_code in [401, 403]:
+                    logger.debug(f"[SCANOSS] Authentication Failed: HTTP {ping_response.status_code}")
+                else:
+                    logger.debug(f"[SCANOSS] API is not ready: HTTP {ping_response.status_code}")
+                scanoss_skipped = True
+                return scanoss_file_list, scanoss_skipped
+        except Exception as ping_error:
+            logger.debug(f"[SCANOSS] Connection failed to {scanner.scanoss_api.url}: {ping_error}")
+            scanoss_skipped = True
+            return scanoss_file_list, scanoss_skipped
+
         with contextlib.redirect_stdout(output_buffer), contextlib.redirect_stderr(output_buffer):
             scanner.scan_folder_with_options(scan_dir=path_to_scan)
-        captured_output = output_buffer.getvalue()
-        api_limit_exceed = "due to service limits being exceeded" in captured_output
+    except Exception as error:
+        logger.debug(f"SCANOSS execution failed: {error}")
 
-        if os.path.isfile(output_json_file):
+    captured_output = output_buffer.getvalue()
+    if captured_output:
+        for line in captured_output.splitlines():
+            line_strip = line.strip()
+            if line_strip.startswith("ERROR:") or "rejected" in line_strip:
+                logger.debug(f"[SCANOSS] {line_strip}")
+
+        api_limit_patterns = [
+            "due to service limits being exceeded",
+            "service limits/rate limit being exceeded",
+            "Rate limit exceeded",
+            "HTTP 429"
+        ]
+        timeout_patterns = [
+            "The SCANOSS API request timed out",
+            "Service unavailable (HTTP 503)",
+            "The SCANOSS API is currently unavailable",
+            "ConnectionError communicating with",
+            "The SCANOSS API request failed",
+            "Connection aborted",
+            "RemoteDisconnected"
+        ]
+        api_limit_exceed = any(p in captured_output for p in api_limit_patterns)
+        timeout_occurred = any(p in captured_output for p in timeout_patterns)
+        if timeout_occurred or api_limit_exceed:
+            scanoss_skipped = True
+            if api_limit_exceed:
+                logger.debug("SCANOSS skipped (API Limit Exceeded)")
+            elif timeout_occurred:
+                logger.debug("SCANOSS skipped (Timeout)")
+
+    if os.path.isfile(output_json_file):
+        try:
             logger.debug("|---SCANOSS Parsing")
             with open(output_json_file, "r") as st_json:
                 st_python = json.load(st_json)
@@ -86,8 +151,8 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
             with open(output_json_file, "r") as st_json:
                 st_python = json.load(st_json)
                 scanoss_file_list = parsing_scan_result(st_python, excluded_files)
-    except Exception as error:
-        logger.debug(f"SCANOSS Parsing {path_to_scan}: {error}")
+        except Exception as error:
+            logger.debug(f"SCANOSS Parsing {path_to_scan}: {error}")
 
     if not write_json_file:
         if os.path.isfile(output_json_file):
@@ -97,4 +162,4 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
 
     logger.info(f"|---Number of files detected with SCANOSS: {(len(scanoss_file_list))}")
 
-    return scanoss_file_list, api_limit_exceed
+    return scanoss_file_list, scanoss_skipped

{fosslight_source-2.3.4 → fosslight_source-2.3.6/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.3.4
+Version: 2.3.6
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -24,7 +24,6 @@ Requires-Dist: fosslight_util>=2.2.2
 Requires-Dist: PyYAML
 Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
-Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
 Requires-Dist: lxml>=6.0.1
 Requires-Dist: cryptography<49; platform_system == "Darwin" and platform_machine == "x86_64"

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ src/fosslight_source/_merge.py
 src/fosslight_source/_parsing_scancode_file_item.py
 src/fosslight_source/_parsing_scanoss_file.py
 src/fosslight_source/_scan_item.py
+src/fosslight_source/_scancode_ignore_binaries.py
 src/fosslight_source/cli.py
 src/fosslight_source/run_manifest_extractor.py
 src/fosslight_source/run_scancode.py

fosslight_source-2.3.6/src/fosslight_source.egg-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+fosslight_source = fosslight_source.cli:main
+
+[scancode_pre_scan]
+ignore-binaries = fosslight_source._scancode_ignore_binaries:IgnoreBinaries

{fosslight_source-2.3.4 → fosslight_source-2.3.6}/src/fosslight_source.egg-info/requires.txt

@@ -6,7 +6,6 @@ fosslight_util>=2.2.2
 PyYAML
 wheel>=0.38.1
 intbitset
-fosslight_binary>=5.1.22
 scancode-toolkit>=32.0.2
 lxml>=6.0.1
 fingerprints==1.2.3

fosslight_source-2.3.4/src/fosslight_source.egg-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-fosslight_source = fosslight_source.cli:main