fosslight-source 2.2.1__tar.gz → 2.2.2__tar.gz

{fosslight_source-2.2.1/src/fosslight_source.egg-info → fosslight_source-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.1
+Version: 2.2.2
 Summary: FOSSLight Source Scanner
 Home-page: https://github.com/fosslight/fosslight_source_scanner
 Download-URL: https://github.com/fosslight/fosslight_source_scanner

{fosslight_source-2.2.1 → fosslight_source-2.2.2}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_source',
-        version='2.2.1',
+        version='2.2.2',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Source Scanner',

{fosslight_source-2.2.1 → fosslight_source-2.2.2}/src/fosslight_source/_parsing_scancode_file_item.py

@@ -7,12 +7,10 @@ import os
 import logging
 import re
 import fosslight_util.constant as constant
-from fosslight_util.get_pom_license import get_license_from_pom
 from ._license_matched import MatchedLicense
 from ._scan_item import SourceItem
 from ._scan_item import replace_word
 from ._scan_item import is_notice_file
-from ._scan_item import is_manifest_file
 from typing import Tuple
 
 logger = logging.getLogger(constant.LOGGER_NAME)
@@ -181,35 +179,6 @@ def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = Fals
                     if len(license_detected) > 0:
                         result_item.licenses = license_detected
 
-                        detected_without_pom = []
-                        if is_manifest_file(file_path) and len(license_detected) > 0:
-                            result_item.is_manifest_file = True
-                            if file_path.endswith('.pom'):
-                                try:
-                                    pom_licenses = get_license_from_pom(pom_path=file_path, check_parent=False)
-                                    normalize_pom_licenses = []
-                                    if pom_licenses:
-                                        pom_license_list = pom_licenses.split(', ')
-                                        for pom_license in pom_license_list:
-                                            if pom_license not in license_detected:
-                                                for lic_matched_key, lic_info in license_list.items():
-                                                    if hasattr(lic_info, 'matched_text') and lic_info.matched_text:
-                                                        matched_txt = str(lic_info.matched_text).replace(',', '')
-                                                        if pom_license in matched_txt:
-                                                            normalize_pom_licenses.append(lic_info.license)
-                                                            break
-                                            else:
-                                                normalize_pom_licenses.append(pom_license)
-                                    detected_without_pom = list(set(license_detected) - set(normalize_pom_licenses))
-                                    if detected_without_pom:
-                                        result_item.comment = f"Detected: {', '.join(detected_without_pom)}"
-                                        result_item.licenses = []
-                                        result_item.licenses = normalize_pom_licenses
-                                        if not normalize_pom_licenses:
-                                            result_item.exclude = True
-                                except Exception as ex:
-                                    logger.info(f"Failed to extract license from POM {file_path}: {ex}")
-
                         # Remove copyright info for license text file of GPL family
                         if should_remove_copyright_for_gpl_license_text(license_detected, result_item.is_license_text):
                             logger.debug(f"Removing copyright for GPL family license text file: {file_path}")
@@ -217,7 +186,7 @@ def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = Fals
                         else:
                             result_item.copyright = copyright_value_list
 
-                        if len(license_expression_list) > 0 and not detected_without_pom:
+                        if len(license_expression_list) > 0:
                             license_expression_list = list(
                                 set(license_expression_list))
                             result_item.comment = ','.join(license_expression_list)
@@ -314,35 +283,6 @@ def parsing_scancode_32_later(
                     file.get("percentage_of_license_text", 0) > 90 and not is_source_file
                 )
 
-                detected_without_pom = []
-                if is_manifest_file(file_path) and len(license_detected) > 0:
-                    result_item.is_manifest_file = True
-                    if file_path.endswith('.pom'):
-                        try:
-                            pom_licenses = get_license_from_pom(pom_path=file_path, check_parent=False)
-                            normalize_pom_licenses = []
-                            if pom_licenses:
-                                pom_license_list = pom_licenses.split(', ')
-                                for pom_license in pom_license_list:
-                                    if pom_license not in license_detected:
-                                        for lic_matched_key, lic_info in license_list.items():
-                                            if hasattr(lic_info, 'matched_text') and lic_info.matched_text:
-                                                matched_txt = str(lic_info.matched_text).replace(',', '')
-                                                if pom_license in matched_txt:
-                                                    normalize_pom_licenses.append(lic_info.license)
-                                                    break
-                                    else:
-                                        normalize_pom_licenses.append(pom_license)
-                            detected_without_pom = list(set(license_detected) - set(normalize_pom_licenses))
-                            if detected_without_pom:
-                                result_item.comment = f"Detected: {', '.join(detected_without_pom)}"
-                                result_item.licenses = []
-                                result_item.licenses = normalize_pom_licenses
-                                if not normalize_pom_licenses:
-                                    result_item.exclude = True
-                        except Exception as ex:
-                            logger.info(f"Failed to extract license from POM {file_path}: {ex}")
-
                 # Remove copyright info for license text file of GPL family
                 if should_remove_copyright_for_gpl_license_text(license_detected, result_item.is_license_text):
                     logger.debug(f"Removing copyright for GPL family license text file: {file_path}")
@@ -350,7 +290,7 @@ def parsing_scancode_32_later(
                 else:
                     result_item.copyright = copyright_value_list
 
-                if len(license_detected) > 1 and not detected_without_pom:
+                if len(license_detected) > 1:
                     license_expression_spdx = file.get("detected_license_expression_spdx", "")
                     license_expression = file.get("detected_license_expression", "")
                     if license_expression_spdx:

{fosslight_source-2.2.1 → fosslight_source-2.2.2}/src/fosslight_source/_scan_item.py

@@ -18,7 +18,7 @@ replace_word = ["-only", "-old-style", "-or-later", "licenseref-scancode-", "lic
 _notice_filename = ['licen[cs]e[s]?', 'notice[s]?', 'legal', 'copyright[s]?', 'copying*', 'patent[s]?', 'unlicen[cs]e', 'eula',
                     '[a,l]?gpl[-]?[1-3]?[.,-,_]?[0-1]?', 'mit', 'bsd[-]?[0-4]?', 'bsd[-]?[0-4][-]?clause[s]?',
                     'apache[-,_]?[1-2]?[.,-,_]?[0-2]?']
-_manifest_filename = [r'.*\.pom$', r'package\.json$', r'setup\.py$', r'pubspec\.yaml$', r'.*\.podspec$', r'Cargo\.toml$']
+_manifest_filename = [r'.*\.pom$', r'package\.json$', r'setup\.py$', r'setup\.cfg$', r'.*\.podspec$', r'Cargo\.toml$']
 MAX_LICENSE_LENGTH = 200
 MAX_LICENSE_TOTAL_LENGTH = 600
 SUBSTRING_LICENSE_COMMENT = "Maximum character limit (License)"

{fosslight_source-2.2.1 → fosslight_source-2.2.2}/src/fosslight_source/cli.py

@@ -25,9 +25,12 @@ from .run_scanoss import get_scanoss_extra_info
 import yaml
 import argparse
 from .run_spdx_extractor import get_spdx_downloads
+from .run_manifest_extractor import get_manifest_licenses
 from ._scan_item import SourceItem, KB_URL
 from fosslight_util.oss_item import ScannerItem
 from typing import Tuple
+from ._scan_item import is_manifest_file
+
 
 SRC_SHEET_NAME = 'SRC_FL_Source'
 SCANOSS_HEADER = {SRC_SHEET_NAME: ['ID', 'Source Path', 'OSS Name',
@@ -265,7 +268,7 @@ def check_kb_server_reachable() -> bool:
 
 def merge_results(
     scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {},
-    path_to_scan: str = "", run_kb: bool = False
+    path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {}
 ) -> list:
 
     """
@@ -291,6 +294,19 @@ def merge_results(
                 new_result_item = SourceItem(file_name)
                 new_result_item.download_location = download_location
                 scancode_result.append(new_result_item)
+    if manifest_licenses:
+        for file_name, licenses in manifest_licenses.items():
+            if file_name in scancode_result:
+                merged_result_item = scancode_result[scancode_result.index(file_name)]
+                # overwrite existing detected licenses with manifest-provided licenses
+                merged_result_item.licenses = []  # clear existing licenses (setter clears when value falsy)
+                merged_result_item.licenses = licenses
+                merged_result_item.is_manifest_file = True
+            else:
+                new_result_item = SourceItem(file_name)
+                new_result_item.licenses = licenses
+                new_result_item.is_manifest_file = True
+                scancode_result.append(new_result_item)
     if run_kb and not check_kb_server_reachable():
         run_kb = False
     if run_kb:
@@ -369,8 +385,9 @@ def run_scanners(
                                                               num_cores, excluded_path_with_default_exclusion, excluded_files)
         if selected_scanner in SCANNER_TYPE:
             run_kb = True if selected_scanner in ['kb', 'all'] else False
-            spdx_downloads = get_spdx_downloads(path_to_scan, excluded_files)
-            merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads, path_to_scan, run_kb)
+            spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files)
+            merged_result = merge_results(scancode_result, scanoss_result, spdx_downloads,
+                                          path_to_scan, run_kb, manifest_licenses)
             scan_item = create_report_file(start_time, merged_result, license_list, scanoss_result, selected_scanner,
                                            print_matched_text, output_path, output_files, output_extensions, correct_mode,
                                            correct_filepath, path_to_scan, excluded_path_without_dot, formats,
@@ -385,5 +402,38 @@ def run_scanners(
     return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result
 
 
+def metadata_collector(path_to_scan: str, excluded_files: set) -> dict:
+    """
+    Collect metadata for merging.
+
+    - Traverse files with exclusions applied
+    - spdx_downloads: {rel_path: [download_urls]}
+    - manifest_licenses: {rel_path: [license_names]}
+
+    :return: (spdx_downloads, manifest_licenses)
+    """
+    abs_path_to_scan = os.path.abspath(path_to_scan)
+    spdx_downloads = {}
+    manifest_licenses = {}
+
+    for root, dirs, files in os.walk(path_to_scan):
+        for file in files:
+            file_path = os.path.join(root, file)
+            rel_path_file = os.path.relpath(file_path, abs_path_to_scan).replace('\\', '/')
+            if rel_path_file in excluded_files:
+                continue
+
+            downloads = get_spdx_downloads(file_path)
+            if downloads:
+                spdx_downloads[rel_path_file] = downloads
+
+            if is_manifest_file(file_path):
+                licenses = get_manifest_licenses(file_path)
+                if licenses:
+                    manifest_licenses[rel_path_file] = licenses
+
+    return spdx_downloads, manifest_licenses
+
+
 if __name__ == '__main__':
     main()

fosslight_source-2.2.2/src/fosslight_source/run_manifest_extractor.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+import json
+import re
+import logging
+from fosslight_util.get_pom_license import get_license_from_pom
+import fosslight_util.constant as constant
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+def _split_spdx_expression(value: str) -> list[str]:
+    parts = re.split(r'\s+(?:OR|AND)\s+|[|]{2}|&&', value, flags=re.IGNORECASE)
+    tokens: list[str] = []
+    for part in parts:
+        token = part.strip().strip('()')
+        if token:
+            tokens.append(token)
+    unique: list[str] = []
+    for t in tokens:
+        if t not in unique:
+            unique.append(t)
+    return unique
+
+
+def get_licenses_from_package_json(file_path: str) -> list[str]:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except Exception as ex:
+        logger.info(f"Failed to read package.json {file_path}: {ex}")
+        return []
+
+    if not isinstance(data, dict):
+        return []
+
+    licenses: list[str] = []
+    license_field = data.get('license')
+
+    if isinstance(license_field, str):
+        value = license_field.strip()
+        if value.upper() == 'UNLICENSED':
+            return []
+        if value.upper().startswith('SEE LICENSE IN'):
+            return []
+        licenses.extend(_split_spdx_expression(value))
+    elif isinstance(license_field, dict):
+        type_val = license_field.get('type')
+        if isinstance(type_val, str):
+            type_val = type_val.strip()
+            if type_val and type_val.upper() != 'UNLICENSED':
+                licenses.append(type_val)
+
+    if not licenses:
+        legacy = data.get('licenses')
+        if isinstance(legacy, list):
+            for item in legacy:
+                if isinstance(item, str):
+                    token = item.strip()
+                    if token and token.upper() != 'UNLICENSED':
+                        licenses.append(token)
+                elif isinstance(item, dict):
+                    t = item.get('type')
+                    if isinstance(t, str):
+                        t = t.strip()
+                        if t and t.upper() != 'UNLICENSED':
+                            licenses.append(t)
+
+    unique: list[str] = []
+    for lic in licenses:
+        if lic not in unique:
+            unique.append(lic)
+    return unique
+
+
+def get_licenses_from_setup_cfg(file_path: str) -> list[str]:
+    try:
+        import configparser
+        parser = configparser.ConfigParser()
+        parser.read(file_path, encoding='utf-8')
+        if parser.has_section('metadata'):
+            license_value = parser.get('metadata', 'license', fallback='').strip()
+            if license_value:
+                return _split_spdx_expression(license_value)
+    except Exception as ex:
+        logger.info(f"Failed to parse setup.cfg with configparser for {file_path}: {ex}")
+
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        meta_match = re.search(r'^\s*\[metadata\]\s*(.*?)(?=^\s*\[|\Z)', content, flags=re.MULTILINE | re.DOTALL)
+        if not meta_match:
+            return []
+        block = meta_match.group(1)
+        m = re.search(r'^\s*license\s*=\s*(.+)$', block, flags=re.MULTILINE)
+        if not m:
+            return []
+        val = m.group(1).strip()
+        if (len(val) >= 2) and ((val[0] == val[-1]) and val[0] in ('"', "'")):
+            val = val[1:-1].strip()
+        if not val:
+            return []
+        return _split_spdx_expression(val)
+    except Exception as ex:
+        logger.info(f"Failed to parse setup.cfg {file_path} via regex fallback: {ex}")
+        return []
+
+
+def get_licenses_from_setup_py(file_path: str) -> list[str]:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except Exception as ex:
+        logger.info(f"Failed to read setup.py {file_path}: {ex}")
+        return []
+
+    match = re.search(r'license\s*=\s*([\'"]{1,3})(.+?)\1', content, flags=re.IGNORECASE | re.DOTALL)
+    if not match:
+        return []
+    value = match.group(2).strip()
+    if not value:
+        return []
+
+    return _split_spdx_expression(value)
+
+
+def get_licenses_from_podspec(file_path: str) -> list[str]:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+    except Exception as ex:
+        logger.info(f"Failed to read podspec {file_path}: {ex}")
+        return []
+
+    m = re.search(r'\blicense\s*=\s*([\'"])(.+?)\1', content, flags=re.IGNORECASE)
+    if m:
+        value = m.group(2).strip()
+        if value:
+            return _split_spdx_expression(value)
+
+    m = re.search(r'\blicense\s*=\s*\{[^}]*?:type\s*=>\s*([\'"])(.+?)\1', content, flags=re.IGNORECASE | re.DOTALL)
+    if m:
+        value = m.group(2).strip()
+        if value:
+            return _split_spdx_expression(value)
+
+    m = re.search(r'\blicense\s*=\s*\{[^}]*?:type\s*=>\s*:(\w+)', content, flags=re.IGNORECASE | re.DOTALL)
+    if m:
+        value = m.group(1).strip()
+        if value:
+            return _split_spdx_expression(value)
+
+    m = re.search(r'\blicense\s*=\s*:(\w+)', content, flags=re.DOTALL | re.IGNORECASE)
+    if m:
+        value = m.group(1).strip()
+        if value:
+            return _split_spdx_expression(value)
+
+    return []
+
+
+def get_licenses_from_cargo_toml(file_path: str) -> list[str]:
+    try:
+        data = None
+        try:
+            import tomllib as toml_loader  # Python 3.11+
+            with open(file_path, 'rb') as f:
+                data = toml_loader.load(f)
+        except Exception:
+            try:
+                import tomli as toml_loader  # Backport
+                with open(file_path, 'rb') as f:
+                    data = toml_loader.load(f)
+            except Exception:
+                data = None
+
+        if isinstance(data, dict):
+            package_tbl = data.get('package') or {}
+            license_value = package_tbl.get('license')
+            if isinstance(license_value, str) and license_value.strip():
+                return _split_spdx_expression(license_value.strip())
+            if package_tbl.get('license-file'):
+                return []
+    except Exception as ex:
+        logger.info(f"Failed to parse Cargo.toml via toml parser for {file_path}: {ex}")
+
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        pkg_match = re.search(r'^\s*\[package\]\s*(.*?)(?=^\s*\[|\Z)', content, flags=re.MULTILINE | re.DOTALL)
+        if not pkg_match:
+            return []
+        block = pkg_match.group(1)
+        m = re.search(r'^\s*license\s*=\s*(?P<q>"""|\'\'\'|"|\')(?P<val>.*?)(?P=q)', block, flags=re.MULTILINE | re.DOTALL)
+        if m:
+            val = m.group('val').strip()
+            if val:
+                return _split_spdx_expression(val)
+        m2 = re.search(r'^\s*license-file\s*=\s*(?:"""|\'\'\'|"|\')(.*?)(?:"""|\'\'\'|"|\')', block,
+                       flags=re.MULTILINE | re.DOTALL)
+        if m2:
+            return []
+    except Exception as ex:
+        logger.info(f"Failed to parse Cargo.toml {file_path}: {ex}")
+        return []
+    return []
+
+
+def get_manifest_licenses(file_path: str) -> list[str]:
+    if file_path.endswith('.pom'):
+        try:
+            pom_licenses = get_license_from_pom(group_id='', artifact_id='', version='', pom_path=file_path, check_parent=True)
+            if not pom_licenses:
+                return []
+            return [x.strip() for x in pom_licenses.split(', ') if x.strip()]
+        except Exception as ex:
+            logger.info(f"Failed to extract license from POM {file_path}: {ex}")
+            return []
+    elif os.path.basename(file_path).lower() == 'package.json':
+        try:
+            return get_licenses_from_package_json(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from package.json {file_path}: {ex}")
+            return []
+    elif os.path.basename(file_path).lower() == 'setup.cfg':
+        try:
+            return get_licenses_from_setup_cfg(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from setup.cfg {file_path}: {ex}")
+            return []
+    elif os.path.basename(file_path).lower() == 'setup.py':
+        try:
+            return get_licenses_from_setup_py(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from setup.py {file_path}: {ex}")
+            return []
+    elif os.path.basename(file_path).lower().endswith('.podspec'):
+        try:
+            return get_licenses_from_podspec(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from podspec {file_path}: {ex}")
+            return []
+    elif os.path.basename(file_path).lower() == 'cargo.toml':
+        try:
+            return get_licenses_from_cargo_toml(file_path)
+        except Exception as ex:
+            logger.info(f"Failed to extract license from Cargo.toml {file_path}: {ex}")
+            return []

fosslight_source-2.2.2/src/fosslight_source/run_spdx_extractor.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import re
+import fosslight_util.constant as constant
+import mmap
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+def get_spdx_downloads(file_path: str) -> list[str]:
+    results = []
+    find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
+    try:
+        if os.path.getsize(file_path) > 0:
+            with open(file_path, "r") as f:
+                with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mmap_obj:
+                    for word in find_word.findall(mmap_obj):
+                        results.append(word.decode('utf-8'))
+    except Exception as ex:
+        logger.warning(f"Failed to extract SPDX download location. {file_path}, {ex}")
+    return results

{fosslight_source-2.2.1 → fosslight_source-2.2.2/src/fosslight_source.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_source
-Version: 2.2.1
+Version: 2.2.2
 Summary: FOSSLight Source Scanner
 Home-page: https://github.com/fosslight/fosslight_source_scanner
 Download-URL: https://github.com/fosslight/fosslight_source_scanner

{fosslight_source-2.2.1 → fosslight_source-2.2.2}/src/fosslight_source.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ src/fosslight_source/_parsing_scancode_file_item.py
 src/fosslight_source/_parsing_scanoss_file.py
 src/fosslight_source/_scan_item.py
 src/fosslight_source/cli.py
+src/fosslight_source/run_manifest_extractor.py
 src/fosslight_source/run_scancode.py
 src/fosslight_source/run_scanoss.py
 src/fosslight_source/run_spdx_extractor.py

fosslight_source-2.2.1/src/fosslight_source/run_spdx_extractor.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2023 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-import logging
-import re
-import fosslight_util.constant as constant
-import mmap
-
-logger = logging.getLogger(constant.LOGGER_NAME)
-
-
-def get_spdx_downloads(path_to_scan: str, path_to_exclude: set = None) -> dict:
-    download_dict = {}
-    find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
-    abs_path_to_scan = os.path.abspath(path_to_scan)
-
-    for root, dirs, files in os.walk(path_to_scan):
-        for file in files:
-            file_path = os.path.join(root, file)
-            rel_path_file = os.path.relpath(file_path, abs_path_to_scan).replace('\\', '/')
-            if rel_path_file in path_to_exclude:
-                continue
-            try:
-                if os.path.getsize(file_path) > 0:
-                    with open(file_path, "r") as f:
-                        with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mmap_obj:
-                            for word in find_word.findall(mmap_obj):
-                                if rel_path_file in download_dict:
-                                    download_dict[rel_path_file].append(word.decode('utf-8'))
-                                else:
-                                    download_dict[rel_path_file] = [word.decode('utf-8')]
-            except Exception as ex:
-                logger.warning(f"Failed to extract SPDX download location. {rel_path_file}, {ex}")
-    return download_dict