fosslight-source 2.2.14__tar.gz → 2.2.16__tar.gz

{fosslight_source-2.2.14/src/fosslight_source.egg-info → fosslight_source-2.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.14
+Version: 2.2.16
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -29,6 +29,7 @@ Requires-Dist: scancode-toolkit>=32.0.2
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1
 Requires-Dist: psycopg2-binary>=2.9.10; python_version >= "3.13"
+Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Dynamic: license-file
 

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fosslight_source"
-version = "2.2.14"
+version = "2.2.16"
 description = "FOSSLight Source Scanner"
 readme = "README.md"
 license = "Apache-2.0"
@@ -39,6 +39,7 @@ dependencies = [
     "normality==2.6.1",
     # Python 3.13+ needs psycopg2-binary 2.9.10+ (has wheels; 2.9.9 builds fail with _PyInterpreterState_Get)
     "psycopg2-binary>=2.9.10; python_version >= '3.13'",
+    "tomli; python_version < '3.11'",
     "tqdm",
 ]
 

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/src/fosslight_source/_help.py

@@ -42,6 +42,8 @@ _HELP_MESSAGE_SOURCE_SCANNER = f"""
     --no_correction        Skip OSS information correction with sbom-info.yaml
     --correct_fpath <path> Path to custom sbom-info.yaml file
     --hide_progress        Hide the progress bar during scanning
+    --kb_url <url>         KB API URL (priority: parameter > KB_URL env > default)
+    --kb_token <token>     KB bearer token (priority: parameter > KB_TOKEN env)
 
     💡 Examples
     ────────────────────────────────────────────────────────────────────

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/src/fosslight_source/_scan_item.py

@@ -19,11 +19,30 @@ replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-", "lic
 _notice_filename = ['licen[cs]e[s]?', 'notice[s]?', 'legal', 'copyright[s]?', 'copying*', 'patent[s]?', 'unlicen[cs]e', 'eula',
                     '[a,l]?gpl[-]?[1-3]?[.,-,_]?[0-1]?', 'mit', 'bsd[-]?[0-4]?', 'bsd[-]?[0-4][-]?clause[s]?',
                     'apache[-,_]?[1-2]?[.,-,_]?[0-2]?']
-_manifest_filename = [r'.*\.pom$', r'package\.json$', r'setup\.py$', r'setup\.cfg$', r'.*\.podspec$', r'Cargo\.toml$']
+_manifest_filename = [
+    r'.*\.pom$',
+    r'package\.json$',
+    r'setup\.py$',
+    r'setup\.cfg$',
+    r'pyproject\.toml$',
+    r'.*\.podspec$',
+    r'Cargo\.toml$',
+    r'huggingface_hub_metadata\.json$',
+]
 MAX_LICENSE_LENGTH = 200
 MAX_LICENSE_TOTAL_LENGTH = 600
 SUBSTRING_LICENSE_COMMENT = "Maximum character limit (License)"
-KB_URL = "http://fosslight-kb.lge.com/"
+DEFAULT_KB_URL = "http://fosslight-kb.lge.com/"
+
+
+def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> tuple[str, str]:
+    url = (kb_url or os.environ.get("KB_URL", DEFAULT_KB_URL)).strip() or DEFAULT_KB_URL
+
+    token = (kb_token or "").strip()
+    if not token:
+        token = (os.environ.get("KB_TOKEN") or "").strip()
+
+    return f"{url.rstrip('/')}/", token
 
 
 class SourceItem(FileItem):
@@ -105,15 +124,21 @@ class SourceItem(FileItem):
             logger.debug(f"Failed to compute MD5 for {self.source_name_or_path}: {e}")
         return md5_hex, wfp
 
-    def _get_origin_url_from_md5_hash(self, md5_hash: str, wfp: str = "") -> str:
+    def _get_origin_url_from_md5_hash(
+        self, md5_hash: str, wfp: str = "", kb_url: str = DEFAULT_KB_URL, kb_token: str = ""
+    ) -> str:
         """Return origin_url from KB API."""
         try:
             payload = {"file_hash": md5_hash}
             if wfp and wfp.strip():
                 payload["wfp_base64"] = base64.b64encode(wfp.strip().encode("utf-8")).decode("ascii")
-            request = urllib.request.Request(f"{KB_URL}query", data=json.dumps(payload).encode('utf-8'), method='POST')
+            request = urllib.request.Request(
+                f"{kb_url}query", data=json.dumps(payload).encode('utf-8'), method='POST'
+            )
             request.add_header('Accept', 'application/json')
             request.add_header('Content-Type', 'application/json')
+            if kb_token:
+                request.add_header('Authorization', f'Bearer {kb_token}')
 
             with urllib.request.urlopen(request, timeout=10) as response:
                 data = json.loads(response.read().decode())
@@ -170,7 +195,9 @@ class SourceItem(FileItem):
             logger.debug(f"Failed to extract OSS info from URL {url}: {e}")
             return "", "", ""
 
-    def set_oss_item(self, path_to_scan: str = "", run_kb: bool = False) -> None:
+    def set_oss_item(
+        self, path_to_scan: str = "", run_kb: bool = False, kb_url: str = DEFAULT_KB_URL, kb_token: str = ""
+    ) -> None:
         self.oss_items = []
         if self.download_location:
             for url in self.download_location:
@@ -183,7 +210,7 @@ class SourceItem(FileItem):
             if run_kb and not self.is_license_text:
                 md5_hash, wfp = self._get_hash(path_to_scan)
                 if md5_hash:
-                    origin_url = self._get_origin_url_from_md5_hash(md5_hash, wfp)
+                    origin_url = self._get_origin_url_from_md5_hash(md5_hash, wfp, kb_url, kb_token)
                     if origin_url:
                         self.kb_origin_url = origin_url
                         self.kb_evidence = "exact_match"

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/src/fosslight_source/cli.py

@@ -9,6 +9,7 @@ import platform
 import time
 import warnings
 import logging
+import re
 import urllib.request
 import urllib.error
 from datetime import datetime
@@ -18,6 +19,7 @@ from ._help import print_version, print_help_msg_source_scanner
 from ._license_matched import get_license_list_to_print
 from fosslight_util.output_format import check_output_formats_v2, write_output_file
 from fosslight_util.correct import correct_with_yaml
+from fosslight_util.parsing_yaml import SUPPORT_OSS_INFO_FILES
 from .run_scancode import run_scan
 from fosslight_util.exclude import get_excluded_paths
 from .run_scanoss import run_scanoss_py
@@ -26,7 +28,7 @@ import yaml
 import argparse
 from .run_spdx_extractor import get_spdx_downloads
 from .run_manifest_extractor import get_manifest_licenses
-from ._scan_item import SourceItem, KB_URL
+from ._scan_item import SourceItem, resolve_kb_config
 from fosslight_util.oss_item import ScannerItem
 from typing import Tuple
 from ._scan_item import is_manifest_file
@@ -43,6 +45,7 @@ MERGED_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
 KB_REFERENCE_HEADER = ['ID', 'Source Path', 'KB Origin URL', 'Evidence']
 ALL_MODE = 'all'
 SCANNER_TYPE = ['kb', 'scancode', 'scanoss', ALL_MODE]
+OSS_INFO_CORRECTION_COMMENT = "Excluded because it's OSS info correction file"
 
 
 logger = logging.getLogger(constant.LOGGER_NAME)
@@ -81,6 +84,8 @@ def main() -> None:
     parser.add_argument('--no_correction', action='store_true', required=False)
     parser.add_argument('--correct_fpath', nargs=1, type=str, required=False)
     parser.add_argument('--hide_progress', action='store_true', required=False)
+    parser.add_argument('--kb_url', type=str, required=False, default="")
+    parser.add_argument('--kb_token', type=str, required=False, default="")
 
     args = parser.parse_args()
 
@@ -109,6 +114,8 @@ def main() -> None:
     if args.correct_fpath:
         correct_filepath = ''.join(args.correct_fpath)
     hide_progress = args.hide_progress
+    kb_url = args.kb_url
+    kb_token = args.kb_token
 
     time_out = args.timeout
     core = args.cores
@@ -117,7 +124,8 @@ def main() -> None:
         result = []
         result = run_scanners(path_to_scan, output_file_name, write_json_file, core, True,
                               print_matched_text, formats, time_out, correct_mode, correct_filepath,
-                              selected_scanner, path_to_exclude, hide_progress=hide_progress)
+                              selected_scanner, path_to_exclude, hide_progress=hide_progress,
+                              kb_url=kb_url, kb_token=kb_token)
 
         _result_log["Scan Result"] = result[1]
 
@@ -265,10 +273,12 @@ def create_report_file(
     return scan_item
 
 
-def check_kb_server_reachable() -> bool:
+def check_kb_server_reachable(kb_url: str, kb_token: str = "") -> bool:
     for attempt in range(3):
         try:
-            request = urllib.request.Request(f"{KB_URL}health", method='GET')
+            request = urllib.request.Request(f"{kb_url}health", method='GET')
+            if kb_token:
+                request.add_header('Authorization', f'Bearer {kb_token}')
             with urllib.request.urlopen(request, timeout=10) as response:
                 logger.debug(f"KB server is reachable. Response status: {response.status}")
                 return True
@@ -312,10 +322,18 @@ def get_kb_reference_to_print(merged_result: list) -> list:
     return data
 
 
+def mark_oss_info_correction_files_as_excluded(scan_results: list) -> None:
+    for item in scan_results:
+        file_name = os.path.basename(item.source_name_or_path).lower()
+        if any(re.search(pattern, file_name, re.IGNORECASE) for pattern in SUPPORT_OSS_INFO_FILES):
+            item.exclude = True
+            item.comment = OSS_INFO_CORRECTION_COMMENT
+
+
 def merge_results(
     scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {},
     path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {},
-    excluded_files: set = None, hide_progress: bool = False
+    excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = ""
 ) -> list:
 
     """
@@ -326,6 +344,8 @@ def merge_results(
     :param path_to_scan: path to the scanned directory for constructing absolute file paths.
     :param run_kb: if True, load kb result.
     :param excluded_files: set of relative paths to exclude from KB-only file discovery.
+    :param kb_url: KB API base URL.
+    :param kb_token: KB API bearer token.
     :return merged_result: list of merged result in SourceItem.
     """
     if excluded_files is None:
@@ -346,20 +366,23 @@ def merge_results(
                 scancode_result.append(new_result_item)
     if manifest_licenses:
         for file_name, licenses in manifest_licenses.items():
+            valid_licenses = [lic.strip() for lic in licenses if isinstance(lic, str) and lic.strip()]
+            if not valid_licenses:
+                continue
             if file_name in scancode_result:
                 merged_result_item = scancode_result[scancode_result.index(file_name)]
                 # overwrite existing detected licenses with manifest-provided licenses
                 merged_result_item.licenses = []  # clear existing licenses (setter clears when value falsy)
-                merged_result_item.licenses = licenses
+                merged_result_item.licenses = valid_licenses
                 merged_result_item.is_manifest_file = True
             else:
                 new_result_item = SourceItem(file_name)
-                new_result_item.licenses = licenses
+                new_result_item.licenses = valid_licenses
                 new_result_item.is_manifest_file = True
                 scancode_result.append(new_result_item)
 
     for item in scancode_result:
-        item.set_oss_item(path_to_scan, run_kb)
+        item.set_oss_item(path_to_scan, run_kb, kb_url, kb_token)
 
     # Add OSSItem for files in path_to_scan that are not in scancode_result
     # when KB returns an origin URL for their MD5 hash (skip excluded_files)
@@ -378,7 +401,7 @@ def merge_results(
             if rel_path in scancode_paths or rel_path in excluded_files:
                 continue
             extra_item = SourceItem(rel_path)
-            extra_item.set_oss_item(path_to_scan, run_kb)
+            extra_item.set_oss_item(path_to_scan, run_kb, kb_url, kb_token)
             if extra_item.download_location:
                 scancode_result.append(extra_item)
                 scancode_paths.add(rel_path)
@@ -393,7 +416,7 @@ def run_scanners(
     formats: list = [], time_out: int = 120,
     correct_mode: bool = True, correct_filepath: str = "",
     selected_scanner: str = ALL_MODE, path_to_exclude: list = [],
-    all_exclude_mode: tuple = (), hide_progress: bool = False
+    all_exclude_mode: tuple = (), hide_progress: bool = False, kb_url: str = "", kb_token: str = ""
 ) -> Tuple[bool, str, 'ScannerItem', list, list]:
     """
     Run Scancode and scanoss.py for the given path.
@@ -405,6 +428,8 @@ def run_scanners(
     :param called_by_cli: if not called by cli, initialize logger.
     :param print_matched_text: if requested, output matched text (only for scancode).
     :param format: output format (excel, csv, opossum).
+    :param kb_url: KB API base URL. If empty, read KB_URL environment variable, then use default.
+    :param kb_token: KB API bearer token. If empty, read KB_TOKEN environment variable.
     :return success: success or failure of scancode.
     :return result_log["Scan Result"]:
     :return merged_result: merged scan result of scancode and scanoss.
@@ -421,6 +446,7 @@ def run_scanners(
     result_log = {}
     scan_item = []
     api_limit_exceed = False
+    kb_url, kb_token = resolve_kb_config(kb_url, kb_token)
 
     success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats)
 
@@ -432,6 +458,8 @@ def run_scanners(
     logger, result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{start_time}.txt"),
                                   True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_scan, path_to_exclude)
 
+    logger.info(f"Tool Info : {result_log['Tool Info']}")
+
     if '.xlsx' not in output_extensions and print_matched_text:
         logger.warning("-m option is only available for excel.")
         print_matched_text = False
@@ -469,15 +497,17 @@ def run_scanners(
         if selected_scanner in SCANNER_TYPE:
             run_kb = True if selected_scanner in ['kb', ALL_MODE] else False
             if run_kb:
-                if not check_kb_server_reachable():
+                if not check_kb_server_reachable(kb_url, kb_token):
                     run_kb = False
-                    run_kb_msg = "KB Unreachable"
+                    run_kb_msg = f"KB({kb_url}) Unreachable"
                 else:
-                    run_kb_msg = "KB Enabled"
+                    run_kb_msg = f"KB({kb_url}) Enabled"
 
             spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
             merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads,
-                                          path_to_scan, run_kb, manifest_licenses, excluded_files, hide_progress)
+                                          path_to_scan, run_kb, manifest_licenses, excluded_files,
+                                          hide_progress, kb_url, kb_token)
+            mark_oss_info_correction_files_as_excluded(merged_result)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,
                                            correct_filepath, path_to_scan, excluded_path_without_dot, formats,

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/src/fosslight_source/run_manifest_extractor.py

@@ -125,6 +125,63 @@ def get_licenses_from_setup_py(file_path: str) -> list[str]:
     return _split_spdx_expression(value)
 
 
+def get_licenses_from_pyproject_toml(file_path: str) -> list[str]:
+    try:
+        data = None
+        try:
+            import tomllib as toml_loader  # Python 3.11+
+            with open(file_path, 'rb') as f:
+                data = toml_loader.load(f)
+        except Exception:
+            try:
+                import tomli as toml_loader  # Backport
+                with open(file_path, 'rb') as f:
+                    data = toml_loader.load(f)
+            except Exception:
+                data = None
+
+        if isinstance(data, dict):
+            project_tbl = data.get('project') or {}
+            license_value = project_tbl.get('license')
+            if isinstance(license_value, str) and license_value.strip():
+                return [license_value.strip()]
+            if isinstance(license_value, dict):
+                text_value = license_value.get('text')
+                if isinstance(text_value, str) and text_value.strip():
+                    return [text_value.strip()]
+                if license_value.get('file'):
+                    return []
+    except Exception as ex:
+        logger.info(f"Failed to parse pyproject.toml via toml parser for {file_path}: {ex}")
+
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        project_match = re.search(r'^\s*\[project\]\s*(.*?)(?=^\s*\[|\Z)', content, flags=re.MULTILINE | re.DOTALL)
+        if not project_match:
+            return []
+        block = project_match.group(1)
+        m = re.search(r'^\s*license\s*=\s*(?P<q>"""|\'\'\'|"|\')(?P<val>.*?)(?P=q)', block,
+                      flags=re.MULTILINE | re.DOTALL)
+        if m:
+            val = m.group('val').strip()
+            if val:
+                return [val]
+        m2 = re.search(r'^\s*license\s*=\s*\{[^}]*?\btext\s*=\s*(?P<q>"""|\'\'\'|"|\')(?P<val>.*?)(?P=q)',
+                       block, flags=re.MULTILINE | re.DOTALL)
+        if m2:
+            val = m2.group('val').strip()
+            if val:
+                return [val]
+        m3 = re.search(r'^\s*license\s*=\s*\{[^}]*?\bfile\s*=', block, flags=re.MULTILINE | re.DOTALL)
+        if m3:
+            return []
+    except Exception as ex:
+        logger.info(f"Failed to parse pyproject.toml {file_path}: {ex}")
+        return []
+    return []
+
+
 def get_licenses_from_podspec(file_path: str) -> list[str]:
     try:
         with open(file_path, 'r', encoding='utf-8') as f:
@@ -207,6 +264,49 @@ def get_licenses_from_cargo_toml(file_path: str) -> list[str]:
     return []
 
 
+def get_licenses_from_huggingface_metadata(file_path: str) -> list[str]:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except Exception as ex:
+        logger.info(f"Failed to read huggingface_hub_metadata.json {file_path}: {ex}")
+        return []
+
+    if not isinstance(data, dict):
+        return []
+
+    licenses: list[str] = []
+
+    def append_license(value):
+        if isinstance(value, str):
+            token = value.strip()
+            if token and token not in licenses:
+                licenses.append(token)
+        elif isinstance(value, list):
+            for item in value:
+                append_license(item)
+
+    # Hugging Face model API commonly returns top-level `license`
+    append_license(data.get('license'))
+
+    # Some metadata may include cardData/license variants
+    card_data = data.get('cardData')
+    if isinstance(card_data, dict):
+        append_license(card_data.get('license'))
+        append_license(card_data.get('licenses'))
+
+    # Many Hub API responses expose license only via tags, e.g. "license:apache-2.0".
+    tags = data.get('tags')
+    if isinstance(tags, list):
+        for tag in tags:
+            if isinstance(tag, str):
+                prefix = 'license:'
+                if tag.lower().startswith(prefix):
+                    append_license(tag[len(prefix):].strip())
+
+    return licenses
+
+
 def get_manifest_licenses(file_path: str) -> list[str]:
     if file_path.endswith('.pom'):
         try:
@@ -235,6 +335,12 @@ def get_manifest_licenses(file_path: str) -> list[str]:
         except Exception as ex:
             logger.info(f"Failed to extract license from setup.py {file_path}: {ex}")
             return []
+    elif os.path.basename(file_path).lower() == 'pyproject.toml':
+        try:
+            return get_licenses_from_pyproject_toml(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from pyproject.toml {file_path}: {ex}")
+            return []
     elif os.path.basename(file_path).lower().endswith('.podspec'):
         try:
             return get_licenses_from_podspec(file_path)
@@ -247,3 +353,9 @@ def get_manifest_licenses(file_path: str) -> list[str]:
         except Exception as ex:
             logger.info(f"Failed to extract license from Cargo.toml {file_path}: {ex}")
             return []
+    elif os.path.basename(file_path).lower() == 'huggingface_hub_metadata.json':
+        try:
+            return get_licenses_from_huggingface_metadata(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from huggingface_hub_metadata.json {file_path}: {ex}")
+            return []

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/src/fosslight_source/run_scancode.py

@@ -103,6 +103,9 @@ def run_scan(
         if not called_by_cli:
             logger, _result_log = init_log(os.path.join(output_path, f"fosslight_log_src_{_start_time}.txt"),
                                            True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_scan, path_to_exclude)
+
+            logger.info(f"Tool Info : {_result_log['Tool Info']}")
+
         num_cores = multiprocessing.cpu_count() - 1 if num_cores < 0 else num_cores
 
         if os.path.isdir(path_to_scan):
@@ -113,6 +116,7 @@ def run_scan(
                 pretty_params["path_to_exclude"] = path_to_exclude
                 pretty_params["output_file"] = output_file_name
                 total_files_to_excluded = []
+                binary_files_to_exclude = []
                 abs_path_to_scan = os.path.abspath(path_to_scan)
                 if path_to_exclude:
                     for path in path_to_exclude:
@@ -166,11 +170,12 @@ def run_scan(
                             continue
                         rel_path = os.path.relpath(full_path, abs_path_to_scan)
                         rel_norm = os.path.normpath(rel_path).replace("\\", "/")
-                        excluded_files.append(rel_norm)
+                        binary_files_to_exclude.append(rel_norm)
                         logger.debug(f"Excluded binary from scancode: {rel_norm}")
 
-                if excluded_files:
-                    total_files_to_excluded.extend(f"**/{file_path}" for file_path in excluded_files)
+                all_excluded_for_scancode = list(excluded_files) + binary_files_to_exclude
+                if all_excluded_for_scancode:
+                    total_files_to_excluded.extend(f"**/{file_path}" for file_path in all_excluded_for_scancode)
 
                 total_files_to_excluded = sorted(list(set(total_files_to_excluded)))
                 ignore_tuple = tuple(total_files_to_excluded)

{fosslight_source-2.2.14 → fosslight_source-2.2.16/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.14
+Version: 2.2.16
 Summary: FOSSLight Source Scanner
 Author: LG Electronics
 License-Expression: Apache-2.0
@@ -29,6 +29,7 @@ Requires-Dist: scancode-toolkit>=32.0.2
 Requires-Dist: fingerprints==1.2.3
 Requires-Dist: normality==2.6.1
 Requires-Dist: psycopg2-binary>=2.9.10; python_version >= "3.13"
+Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Dynamic: license-file
 

{fosslight_source-2.2.14 → fosslight_source-2.2.16}/src/fosslight_source.egg-info/requires.txt

@@ -12,5 +12,8 @@ fingerprints==1.2.3
 normality==2.6.1
 tqdm
 
+[:python_version < "3.11"]
+tomli
+
 [:python_version >= "3.13"]
 psycopg2-binary>=2.9.10