fosslight-source 2.2.13__tar.gz → 2.2.15__tar.gz

{fosslight_source-2.2.13/src/fosslight_source.egg-info → fosslight_source-2.2.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.13
+Version: 2.2.15
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -24,11 +24,12 @@ Requires-Dist: fosslight_util>=2.1.37
 Requires-Dist: PyYAML
 Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
-Requires-Dist: fosslight_binary>=5.0.0
+Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1
 Requires-Dist: psycopg2-binary>=2.9.10; python_version >= "3.13"
+Requires-Dist: tqdm
 Dynamic: license-file
 
 <!--

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fosslight_source"
-version = "2.2.13"
+version = "2.2.15"
 description = "FOSSLight Source Scanner"
 readme = "README.md"
 license = "Apache-2.0"
@@ -33,12 +33,13 @@ dependencies = [
     "PyYAML",
     "wheel>=0.38.1",
     "intbitset",
-    "fosslight_binary>=5.0.0",
+    "fosslight_binary>=5.1.22",
     "scancode-toolkit>=32.0.2",
     "fingerprints==1.2.3",
     "normality==2.6.1",
     # Python 3.13+ needs psycopg2-binary 2.9.10+ (has wheels; 2.9.9 builds fail with _PyInterpreterState_Get)
     "psycopg2-binary>=2.9.10; python_version >= '3.13'",
+    "tqdm",
 ]
 
 [project.optional-dependencies]

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source/_help.py

@@ -41,6 +41,7 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
     -j                     Generate raw scanner results in JSON format
     --no_correction        Skip OSS information correction with sbom-info.yaml
     --correct_fpath <path> Path to custom sbom-info.yaml file
+    --hide_progress        Hide the progress bar during scanning
 
     💡 Examples
     ────────────────────────────────────────────────────────────────────

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source/_scan_item.py

@@ -19,7 +19,15 @@ replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-", "lic
 _notice_filename = ['licen[cs]e[s]?', 'notice[s]?', 'legal', 'copyright[s]?', 'copying*', 'patent[s]?', 'unlicen[cs]e', 'eula',
                     '[a,l]?gpl[-]?[1-3]?[.,-,_]?[0-1]?', 'mit', 'bsd[-]?[0-4]?', 'bsd[-]?[0-4][-]?clause[s]?',
                     'apache[-,_]?[1-2]?[.,-,_]?[0-2]?']
-_manifest_filename = [r'.*\.pom$', r'package\.json$', r'setup\.py$', r'setup\.cfg$', r'.*\.podspec$', r'Cargo\.toml$']
+_manifest_filename = [
+    r'.*\.pom$',
+    r'package\.json$',
+    r'setup\.py$',
+    r'setup\.cfg$',
+    r'.*\.podspec$',
+    r'Cargo\.toml$',
+    r'huggingface_hub_metadata\.json$',
+]
 MAX_LICENSE_LENGTH = 200
 MAX_LICENSE_TOTAL_LENGTH = 600
 SUBSTRING_LICENSE_COMMENT = "Maximum character limit (License)"

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source/cli.py

@@ -14,7 +14,6 @@ import urllib.error
 from datetime import datetime
 import fosslight_util.constant as constant
 from fosslight_util.set_log import init_log
-from fosslight_util.timer_thread import TimerThread
 from ._help import print_version, print_help_msg_source_scanner
 from ._license_matched import get_license_list_to_print
 from fosslight_util.output_format import check_output_formats_v2, write_output_file
@@ -81,6 +80,7 @@ def main() -> None:
     parser.add_argument('-e', '--exclude', nargs='*', required=False, default=[])
     parser.add_argument('--no_correction', action='store_true', required=False)
     parser.add_argument('--correct_fpath', nargs=1, type=str, required=False)
+    parser.add_argument('--hide_progress', action='store_true', required=False)
 
     args = parser.parse_args()
 
@@ -108,19 +108,16 @@ def main() -> None:
     correct_filepath = path_to_scan
     if args.correct_fpath:
         correct_filepath = ''.join(args.correct_fpath)
+    hide_progress = args.hide_progress
 
     time_out = args.timeout
     core = args.cores
 
-    timer = TimerThread()
-    timer.setDaemon(True)
-    timer.start()
-
     if os.path.isdir(path_to_scan):
         result = []
         result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
                               print_matched_text, formats, time_out, correct_mode, correct_filepath,
-                              selected_scanner, path_to_exclude)
+                              selected_scanner, path_to_exclude, hide_progress=hide_progress)
 
         _result_log["Scan Result"] = result[1]
 
@@ -318,7 +315,7 @@ def get_kb_reference_to_print(merged_result: list) -> list:
 def merge_results(
     scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {},
     path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {},
-    excluded_files: set = None
+    excluded_files: set = None, hide_progress: bool = False
 ) -> list:
 
     """
@@ -349,15 +346,18 @@ def merge_results(
                 scancode_result.append(new_result_item)
     if manifest_licenses:
         for file_name, licenses in manifest_licenses.items():
+            valid_licenses = [lic.strip() for lic in licenses if isinstance(lic, str) and lic.strip()]
+            if not valid_licenses:
+                continue
             if file_name in scancode_result:
                 merged_result_item = scancode_result[scancode_result.index(file_name)]
                 # overwrite existing detected licenses with manifest-provided licenses
                 merged_result_item.licenses = []  # clear existing licenses (setter clears when value falsy)
-                merged_result_item.licenses = licenses
+                merged_result_item.licenses = valid_licenses
                 merged_result_item.is_manifest_file = True
             else:
                 new_result_item = SourceItem(file_name)
-                new_result_item.licenses = licenses
+                new_result_item.licenses = valid_licenses
                 new_result_item.is_manifest_file = True
                 scancode_result.append(new_result_item)
 
@@ -367,19 +367,24 @@ def merge_results(
     # Add OSSItem for files in path_to_scan that are not in scancode_result
     # when KB returns an origin URL for their MD5 hash (skip excluded_files)
     if run_kb:
+        import tqdm
         abs_path_to_scan = os.path.abspath(path_to_scan)
         scancode_paths = {item.source_name_or_path for item in scancode_result}
+
+        files_to_scan = []
         for root, _dirs, files in os.walk(path_to_scan):
             for file in files:
-                file_path = os.path.join(root, file)
-                rel_path = os.path.relpath(file_path, abs_path_to_scan).replace("\\", "/")
-                if rel_path in scancode_paths or rel_path in excluded_files:
-                    continue
-                extra_item = SourceItem(rel_path)
-                extra_item.set_oss_item(path_to_scan, run_kb)
-                if extra_item.download_location:
-                    scancode_result.append(extra_item)
-                    scancode_paths.add(rel_path)
+                files_to_scan.append(os.path.join(root, file))
+
+        for file_path in tqdm.tqdm(files_to_scan, desc="KB Scanning", disable=hide_progress):
+            rel_path = os.path.relpath(file_path, abs_path_to_scan).replace("\\", "/")
+            if rel_path in scancode_paths or rel_path in excluded_files:
+                continue
+            extra_item = SourceItem(rel_path)
+            extra_item.set_oss_item(path_to_scan, run_kb)
+            if extra_item.download_location:
+                scancode_result.append(extra_item)
+                scancode_paths.add(rel_path)
 
     return scancode_result
 
@@ -391,7 +396,7 @@ def run_scanners(
     formats: list = [], time_out: int = 120,
     correct_mode: bool = True, correct_filepath: str = "",
     selected_scanner: str = ALL_MODE, path_to_exclude: list = [],
-    all_exclude_mode: tuple = ()
+    all_exclude_mode: tuple = (), hide_progress: bool = False
 ) -> Tuple[bool, str, 'ScannerItem', list, list]:
     """
     Run Scancode and scanoss.py for the given path.
@@ -430,6 +435,8 @@ def run_scanners(
     logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
                                   True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
 
+    logger.info(f"Tool Info : {result_log['Tool Info']}")
+
     if '.xlsx' not in output_extensions and print_matched_text:
         logger.warning("-m option is only available for excel.")
         print_matched_text = False
@@ -456,12 +463,12 @@ def run_scanners(
                                                                                       print_matched_text, formats, called_by_cli,
                                                                                       time_out, correct_mode, correct_filepath,
                                                                                       excluded_path_with_default_exclusion,
-                                                                                      excluded_files)
+                                                                                      excluded_files, hide_progress)
         excluded_files = set(excluded_files) if excluded_files else set()
         if selected_scanner in ['scanoss', ALL_MODE]:
             scanoss_result, api_limit_exceed = run_scanoss_py(path_to_scan, output_path, formats, True, num_cores,
                                                               excluded_path_with_default_exclusion, excluded_files,
-                                                              write_json_file)
+                                                              write_json_file, hide_progress)
 
         run_kb_msg = ""
         if selected_scanner in SCANNER_TYPE:
@@ -475,7 +482,7 @@ def run_scanners(
 
             spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
             merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads,
-                                          path_to_scan, run_kb, manifest_licenses, excluded_files)
+                                          path_to_scan, run_kb, manifest_licenses, excluded_files, hide_progress)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,
                                            correct_filepath, path_to_scan, excluded_path_without_dot, formats,

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source/run_manifest_extractor.py

@@ -207,6 +207,49 @@ def get_licenses_from_cargo_toml(file_path: str) -> list[str]:
     return []
 
 
+def get_licenses_from_huggingface_metadata(file_path: str) -> list[str]:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except Exception as ex:
+        logger.info(f"Failed to read huggingface_hub_metadata.json {file_path}: {ex}")
+        return []
+
+    if not isinstance(data, dict):
+        return []
+
+    licenses: list[str] = []
+
+    def append_license(value):
+        if isinstance(value, str):
+            token = value.strip()
+            if token and token not in licenses:
+                licenses.append(token)
+        elif isinstance(value, list):
+            for item in value:
+                append_license(item)
+
+    # Hugging Face model API commonly returns top-level `license`
+    append_license(data.get('license'))
+
+    # Some metadata may include cardData/license variants
+    card_data = data.get('cardData')
+    if isinstance(card_data, dict):
+        append_license(card_data.get('license'))
+        append_license(card_data.get('licenses'))
+
+    # Many Hub API responses expose license only via tags, e.g. "license:apache-2.0".
+    tags = data.get('tags')
+    if isinstance(tags, list):
+        for tag in tags:
+            if isinstance(tag, str):
+                prefix = 'license:'
+                if tag.lower().startswith(prefix):
+                    append_license(tag[len(prefix):].strip())
+
+    return licenses
+
+
 def get_manifest_licenses(file_path: str) -> list[str]:
     if file_path.endswith('.pom'):
         try:
@@ -247,3 +290,9 @@ def get_manifest_licenses(file_path: str) -> list[str]:
         except Exception as ex:
             logger.info(f"Failed to extract license from Cargo.toml {file_path}: {ex}")
             return []
+    elif os.path.basename(file_path).lower() == 'huggingface_hub_metadata.json':
+        try:
+            return get_licenses_from_huggingface_metadata(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from huggingface_hub_metadata.json {file_path}: {ex}")
+            return []

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source/run_scancode.py

@@ -63,7 +63,7 @@ def run_scan(
     formats: list = [], called_by_cli: bool = False,
     time_out: int = 120, correct_mode: bool = True,
     correct_filepath: str = "", path_to_exclude: list = [],
-    excluded_files: list = []
+    excluded_files: list = [], hide_progress: bool = False
 ) -> Tuple[bool, str, list, list]:
     if not called_by_cli:
         global logger
@@ -103,6 +103,9 @@ def run_scan(
         if not called_by_cli:
             logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
                                            True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
+
+            logger.info(f"Tool Info : {_result_log['Tool Info']}")
+
         num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores
 
         if os.path.isdir(path_to_scan):
@@ -113,8 +116,8 @@ def run_scan(
                 pretty_params["path_to_exclude"] = path_to_exclude
                 pretty_params["output_file"] = output_file_name
                 total_files_to_excluded = []
+                abs_path_to_scan = os.path.abspath(path_to_scan)
                 if path_to_exclude:
-                    abs_path_to_scan = os.path.abspath(path_to_scan)
                     for path in path_to_exclude:
                         if os.path.isabs(path):
                             exclude_path = os.path.relpath(path, abs_path_to_scan)
@@ -156,6 +159,19 @@ def run_scan(
                             else:
                                 total_files_to_excluded.append(exclude_path_normalized)
 
+                for root, _, files in os.walk(path_to_scan):
+                    for name in files:
+                        full_path = os.path.join(root, name)
+                        try:
+                            if not check_binary(full_path, True):
+                                continue
+                        except Exception:
+                            continue
+                        rel_path = os.path.relpath(full_path, abs_path_to_scan)
+                        rel_norm = os.path.normpath(rel_path).replace("\\", "/")
+                        excluded_files.append(rel_norm)
+                        logger.debug(f"Excluded binary from scancode: {rel_norm}")
+
                 if excluded_files:
                     total_files_to_excluded.extend(f"**/{file_path}" for file_path in excluded_files)
 
@@ -176,7 +192,8 @@ def run_scan(
                     "url": True,
                     "timeout": time_out,
                     "include": (),
-                    "ignore": ignore_tuple
+                    "ignore": ignore_tuple,
+                    "quiet": hide_progress
                 }
 
                 _apply_scancode_unset_workaround(kwargs)
@@ -206,7 +223,7 @@ def run_scan(
                             for scan_item in result_list:
                                 if os.path.isdir(scan_item.source_name_or_path):
                                     continue
-                                if check_binary(os.path.join(path_to_scan, scan_item.source_name_or_path)):
+                                if check_binary(os.path.join(path_to_scan, scan_item.source_name_or_path), True):
                                     scan_item.exclude = True
             except Exception as ex:
                 success = False

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source/run_scanoss.py

@@ -31,7 +31,7 @@ def get_scanoss_extra_info(scanned_result: dict) -> list:
 def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
                    called_by_cli: bool = False, num_threads: int = -1,
                    path_to_exclude: list = [], excluded_files: set = None,
-                   write_json_file: bool = False) -> Tuple[list, bool]:
+                   write_json_file: bool = False, hide_progress: bool = False) -> Tuple[list, bool]:
     """
     Run scanoss.py for the given path.
 
@@ -58,6 +58,7 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
         os.remove(output_json_file)
 
     try:
+        logger.debug(f"|---Running SCANOSS on {path_to_scan}")
         scanoss_settings = ScanossSettings()
         scanner = Scanner(
             ignore_cert_errors=True,
@@ -67,7 +68,6 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
             nb_threads=num_threads if num_threads > 0 else 10,
             scanoss_settings=scanoss_settings
         )
-
         output_buffer = io.StringIO()
         with contextlib.redirect_stdout(output_buffer), contextlib.redirect_stderr(output_buffer):
             scanner.scan_folder_with_options(scan_dir=path_to_scan)
@@ -75,6 +75,7 @@ def run_scanoss_py(path_to_scan: str, output_path: str = "", format: list = [],
         api_limit_exceed = "due to service limits being exceeded" in captured_output
 
         if os.path.isfile(output_json_file):
+            logger.debug("|---SCANOSS Parsing")
             with open(output_json_file, "r") as st_json:
                 st_python = json.load(st_json)
                 for key_to_exclude in excluded_files:

{fosslight_source-2.2.13 → fosslight_source-2.2.15/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.13
+Version: 2.2.15
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -24,11 +24,12 @@ Requires-Dist: fosslight_util>=2.1.37
 Requires-Dist: PyYAML
 Requires-Dist: wheel>=0.38.1
 Requires-Dist: intbitset
-Requires-Dist: fosslight_binary>=5.0.0
+Requires-Dist: fosslight_binary>=5.1.22
 Requires-Dist: scancode-toolkit>=32.0.2
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1
 Requires-Dist: psycopg2-binary>=2.9.10; python_version >= "3.13"
+Requires-Dist: tqdm
 Dynamic: license-file
 
 <!--

{fosslight_source-2.2.13 → fosslight_source-2.2.15}/src/fosslight_source.egg-info/requires.txt

@@ -6,10 +6,11 @@ fosslight_util>=2.1.37
 PyYAML
 wheel>=0.38.1
 intbitset
-fosslight_binary>=5.0.0
+fosslight_binary>=5.1.22
 scancode-toolkit>=32.0.2
 fingerprints==1.2.3
 normality==2.6.1
+tqdm
 
 [:python_version >= "3.13"]
 psycopg2-binary>=2.9.10