fosslight-util 2.1.7__py3-none-any.whl → 2.1.18__py3-none-any.whl

fosslight_util/_get_downloadable_url.py

@@ -27,7 +27,7 @@ def extract_name_version_from_link(link):
                 origin_name = match.group(1)
                 if (key == "pypi") or (key == "pypi2"):
                     oss_name = f"pypi:{origin_name}"
-                    oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
+                    oss_name = re.sub(r"[-_.]+", "-", oss_name)
                     oss_version = match.group(2)
                 elif key == "maven":
                     artifact = match.group(2)
@@ -42,12 +42,20 @@ def extract_name_version_from_link(link):
                     oss_version = match.group(2)
                 elif key == "cocoapods":
                     oss_name = f"cocoapods:{origin_name}"
+                elif key == "go":
+                    if origin_name.endswith('/'):
+                        origin_name = origin_name[:-1]
+                    oss_name = f"go:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "cargo":
+                    oss_name = f"cargo:{origin_name}"
+                    oss_version = match.group(2)
             except Exception as ex:
                 logger.info(f"extract_name_version_from_link {key}:{ex}")
             if oss_name and (not oss_version):
-                if key in ["pypi", "maven", "npm", "npm2", "pub"]:
+                if key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
                     oss_version, link = get_latest_package_version(link, key, origin_name)
-                    logger.debug(f'Try to download with the latest version:{link}')
+                    logger.info(f'Try to download with the latest version:{link}')
             break
     return oss_name, oss_version, link, key
 
@@ -76,8 +84,13 @@ def get_latest_package_version(link, pkg_type, oss_name):
             if pub_response.status_code == 200:
                 find_version = pub_response.json().get('latest').get('version')
             link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
+        elif pkg_type == 'go':
+            go_response = requests.get(f'https://proxy.golang.org/{oss_name}/@latest')
+            if go_response.status_code == 200:
+                find_version = go_response.json().get('Version')
+            link_with_version = f'https://pkg.go.dev/{oss_name}@{find_version}'
     except Exception as e:
-        logger.debug(f'Fail to get latest package version({link}:{e})')
+        logger.info(f'Fail to get latest package version({link}:{e})')
     return find_version, link_with_version
 
 
@@ -98,8 +111,66 @@ def get_downloadable_url(link):
         ret, result_link = get_download_location_for_npm(new_link)
     elif pkg_type == "pub":
         ret, result_link = get_download_location_for_pub(new_link)
+    elif pkg_type == "go":
+        ret, result_link = get_download_location_for_go(new_link)
+    elif pkg_type == "cargo":
+        ret, result_link = get_download_location_for_cargo(new_link)
+    return ret, result_link, oss_name, oss_version, pkg_type
 
-    return ret, result_link, oss_name, oss_version
+
+def get_download_location_for_cargo(link):
+    # get the url for downloading source file: https://crates.io/api/v1/crates/<name>/<version>/download
+    ret = False
+    new_link = ''
+    host = 'https://crates.io/api/v1/crates'
+
+    try:
+        dn_loc_re = re.findall(r'crates.io\/crates\/([^\/]+)\/?([^\/]*)', link)
+        if dn_loc_re:
+            oss_name = dn_loc_re[0][0]
+            oss_version = dn_loc_re[0][1]
+
+            new_link = f'{host}/{oss_name}/{oss_version}/download'
+            res = urlopen(new_link)
+            if res.getcode() == 200:
+                ret = True
+            else:
+                logger.warning(f'Cannot find the valid link for cargo (url:{new_link}')
+    except Exception as error:
+        ret = False
+        logger.warning(f'Cannot find the link for cargo (url:{link}({(new_link)})): {error}')
+
+    return ret, new_link
+
+
+def get_download_location_for_go(link):
+    # get the url for downloading source file: https://proxy.golang.org/<module>/@v/VERSION.zip
+    ret = False
+    new_link = ''
+    host = 'https://proxy.golang.org'
+
+    try:
+        dn_loc_re = re.findall(r'pkg.go.dev\/([^\@]+)\@?([^\/]*)', link)
+        if dn_loc_re:
+            oss_name = dn_loc_re[0][0]
+            if oss_name.endswith('/'):
+                oss_name = oss_name[:-1]
+            oss_version = dn_loc_re[0][1]
+
+            new_link = f'{host}/{oss_name}/@v/{oss_version}.zip'
+        try:
+            res = urlopen(new_link)
+            if res.getcode() == 200:
+                ret = True
+            else:
+                logger.warning(f'Cannot find the valid link for go (url:{new_link}')
+        except Exception as e:
+            logger.warning(f'Fail to find the valid link for go (url:{new_link}: {e}')
+    except Exception as error:
+        ret = False
+        logger.warning(f'Cannot find the link for go (url:{link}({(new_link)})): {error}')
+
+    return ret, new_link
 
 
 def get_download_location_for_pypi(link):
@@ -111,7 +182,7 @@ def get_download_location_for_pypi(link):
     try:
         dn_loc_re = re.findall(r'pypi.org\/project\/?([^\/]*)\/?([^\/]*)', link)
         oss_name = dn_loc_re[0][0]
-        oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
+        oss_name = re.sub(r"[-_.]+", "-", oss_name)
         oss_version = dn_loc_re[0][1]
 
         new_link = f'{host}/packages/source/{oss_name[0]}/{oss_name}/{oss_name}-{oss_version}.tar.gz'
@@ -121,8 +192,8 @@ def get_download_location_for_pypi(link):
                 ret = True
             else:
                 logger.warning(f'Cannot find the valid link for pypi (url:{new_link}')
-        except Exception as e:
-            oss_name = re.sub(r"[-]+", "_", oss_name).lower()
+        except Exception:
+            oss_name = re.sub(r"[-]+", "_", oss_name)
             new_link = f'{host}/packages/source/{oss_name[0]}/{oss_name}/{oss_name}-{oss_version}.tar.gz'
             res = urlopen(new_link)
             if res.getcode() == 200:

fosslight_util/constant.py

@@ -35,6 +35,7 @@ SHEET_NAME_FOR_SCANNER = {
 # pub: https://pub.dev/packages/(package)/versions/(version)
 # Cocoapods : https://cocoapods.org/(package)
 # go : https://pkg.go.dev/(package_name_with_slash)@(version)
+# cargo : https://crates.io/crates/(crate_name)/(version)
 PKG_PATTERN = {
     "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
     "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
@@ -43,5 +44,6 @@ PKG_PATTERN = {
     "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
     "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
     "cocoapods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)',
-    "go": r'https?:\/\/pkg.go.dev\/([^\@]+)\@?v?([^\/]*)'
+    "go": r'https?:\/\/pkg.go.dev\/([^\@]+)\@?v?([^\/]*)',
+    "cargo": r'https?:\/\/crates\.io\/crates\/([^\/]+)\/?([^\/]*)',
 }

fosslight_util/correct.py

@@ -61,17 +61,15 @@ def correct_with_yaml(correct_filepath, path_to_scan, scan_item):
 
                     yaml_path_exists = True
                     exclude_fileitems.append(idx)
-
-            if not yaml_path_exists:
+            if scanner_name == FOSSLIGHT_SOURCE and not yaml_path_exists:
                 correct_item = copy.deepcopy(yaml_file_item)
                 if os.path.exists(os.path.normpath(yaml_file_item.source_name_or_path)):
                     correct_item.comment = 'Loaded from sbom-info.yaml'
                     correct_fileitems.append(correct_item)
                 else:
-                    if scanner_name == FOSSLIGHT_SOURCE:
-                        correct_item.exclude = True
-                        correct_item.comment = 'Added by sbom-info.yaml'
-                        correct_fileitems.append(correct_item)
+                    correct_item.exclude = True
+                    correct_item.comment = 'Added by sbom-info.yaml'
+                    correct_fileitems.append(correct_item)
         if correct_fileitems:
             scan_item.append_file_items(correct_fileitems, scanner_name)
             find_match = True

fosslight_util/download.py

@@ -26,6 +26,7 @@ import platform
 import subprocess
 import re
 from typing import Tuple
+import urllib.parse
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
@@ -94,7 +95,8 @@ def parse_src_link(src_link):
 
 def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_to: str = "",
                              compressed_only: bool = False, ssh_key: str = "",
-                             id: str = "", git_token: str = "") -> Tuple[bool, str, str, str]:
+                             id: str = "", git_token: str = "",
+                             called_cli: bool = True) -> Tuple[bool, str, str, str]:
     global logger
 
     success = True
@@ -123,8 +125,11 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
             is_rubygems = src_info.get("rubygems", False)
 
             # General download (git clone, wget)
-            success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir, checkout_to,
-                                                                         tag, branch, ssh_key, id, git_token)
+            success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir,
+                                                                         checkout_to,
+                                                                         tag, branch,
+                                                                         ssh_key, id, git_token,
+                                                                         called_cli)
             link = change_ssh_link_to_https(link)
             if (not is_rubygems) and (not success_git):
                 if os.path.isfile(target_dir):
@@ -202,34 +207,44 @@ def get_github_token(git_url):
     return github_token
 
 
-def download_git_repository(refs_to_checkout, git_url, target_dir, tag):
+def download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli=True):
     success = False
     oss_version = ""
-    clone_default_branch_flag = False
 
     logger.info(f"Download git url :{git_url}")
+    env = os.environ.copy()
+    if not called_cli:
+        env["GIT_TERMINAL_PROMPT"] = "0"
     if refs_to_checkout:
         try:
             # gitPython uses the branch argument the same whether you check out to a branch or a tag.
-            repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout)
-            success = True
+            Repo.clone_from(git_url, target_dir, branch=refs_to_checkout, env=env)
+            if any(Path(target_dir).iterdir()):
+                success = True
+                oss_version = refs_to_checkout
+                logger.info(f"Files found in {target_dir} after clone.")
+            else:
+                logger.info(f"No files found in {target_dir} after clone.")
+                success = False
         except GitCommandError as error:
-            logger.debug(f"Git checkout error:{error}")
+            logger.info(f"Git checkout error:{error}")
+            success = False
+        except Exception as e:
+            logger.info(f"Repo.clone_from error:{e}")
             success = False
 
     if not success:
-        repo = Repo.clone_from(git_url, target_dir)
-        clone_default_branch_flag = True
-        success = True
-
-    if refs_to_checkout != tag or clone_default_branch_flag:
-        oss_version = repo.active_branch.name
-    else:
-        oss_version = repo.git.describe('--tags')
+        Repo.clone_from(git_url, target_dir, env=env)
+        if any(Path(target_dir).iterdir()):
+            success = True
+        else:
+            logger.info(f"No files found in {target_dir} after clone.")
+            success = False
     return success, oss_version
 
 
-def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", ssh_key="", id="", git_token=""):
+def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="",
+                       ssh_key="", id="", git_token="", called_cli=True):
     oss_name = get_github_ossname(git_url)
     refs_to_checkout = decide_checkout(checkout_to, tag, branch)
     msg = ""
@@ -253,17 +268,19 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", s
                 logger.info(f"Download git with ssh_key:{git_url}")
                 git_ssh_cmd = f'ssh -i {ssh_key}'
                 with Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
-                    success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag)
+                    success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli)
             else:
                 if id and git_token:
                     try:
                         m = re.match(r"^(ht|f)tp(s?)\:\/\/", git_url)
                         protocol = m.group()
                         if protocol:
-                            git_url = git_url.replace(protocol, f"{protocol}{id}:{git_token}@")
+                            encoded_git_token = urllib.parse.quote(git_token, safe='')
+                            encoded_id = urllib.parse.quote(id, safe='')
+                            git_url = git_url.replace(protocol, f"{protocol}{encoded_id}:{encoded_git_token}@")
                     except Exception as error:
                         logger.info(f"Failed to insert id, token to git url:{error}")
-                success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag)
+                success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli)
 
             logger.info(f"git checkout: {oss_version}")
             refs_to_checkout = oss_version
@@ -297,7 +314,7 @@ def download_wget(link, target_dir, compressed_only):
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link, oss_name, oss_version = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link)
         if ret and new_link:
             link = new_link
 
@@ -306,6 +323,9 @@ def download_wget(link, target_dir, compressed_only):
                 if link.endswith(ext):
                     success = True
                     break
+            if not success:
+                if pkg_type == 'cargo':
+                    success = True
         else:
             success = True
 
@@ -313,7 +333,11 @@ def download_wget(link, target_dir, compressed_only):
             raise Exception('Not supported compression type (link:{0})'.format(link))
 
         logger.info(f"wget: {link}")
-        downloaded_file = wget.download(link, target_dir)
+        if pkg_type == 'cargo':
+            outfile = os.path.join(target_dir, f'{oss_name}.tar.gz')
+            downloaded_file = wget.download(link, out=outfile)
+        else:
+            downloaded_file = wget.download(link, target_dir)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:

fosslight_util/exclude.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import fnmatch
+from typing import List
+
+
+def excluding_files(patterns: List[str], path_to_scan: str) -> List[str]:
+    excluded_paths = set()
+
+    # Normalize patterns: e.g., 'sample/', 'sample/*' -> 'sample'
+    # Replace backslash with slash
+    normalized_patterns = []
+    for pattern in patterns:
+        pattern = pattern.replace('\\', '/')
+        if pattern.endswith('/') or pattern.endswith('/*'):
+            pattern = pattern.rstrip('/*')
+        normalized_patterns.append(pattern)
+
+    # Traverse directories
+    for root, dirs, files in os.walk(path_to_scan):
+        remove_dir_list = []
+
+        # (1) Directory matching
+        for d in dirs:
+            dir_name = d
+            dir_path = os.path.relpath(os.path.join(root, d), path_to_scan).replace('\\', '/')
+            matched = False
+
+            for pat in normalized_patterns:
+                # Match directory name
+                if fnmatch.fnmatch(dir_name, pat):
+                    matched = True
+
+                # Match the full relative path
+                if not matched:
+                    if fnmatch.fnmatch(dir_path, pat) or fnmatch.fnmatch(dir_path, pat + "/*"):
+                        matched = True
+
+                # If matched, exclude all files under this directory and stop checking patterns
+                if matched:
+                    sub_root_path = os.path.join(root, d)
+                    for sr, _, sf in os.walk(sub_root_path):
+                        for sub_file in sf:
+                            sub_file_path = os.path.relpath(os.path.join(sr, sub_file), path_to_scan)
+                            excluded_paths.add(sub_file_path.replace('\\', '/'))
+                    remove_dir_list.append(d)
+                    break
+
+        # (1-2) Prune matched directories from further traversal
+        for rd in remove_dir_list:
+            dirs.remove(rd)
+
+        # (2) File matching
+        for f in files:
+            file_path = os.path.relpath(os.path.join(root, f), path_to_scan).replace('\\', '/')
+            for pat in normalized_patterns:
+                if fnmatch.fnmatch(file_path, pat) or fnmatch.fnmatch(file_path, pat + "/*"):
+                    excluded_paths.add(file_path)
+                    break
+
+    return sorted(excluded_paths)

fosslight_util/help.py

@@ -3,7 +3,10 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import sys
-import pkg_resources
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    from importlib_metadata import version, PackageNotFoundError  # Python <3.8
 
 _HELP_MESSAGE_COMMON = """
          _______  _______  _______  _______  ___      ___           __
@@ -50,7 +53,10 @@ class PrintHelpMsg():
 def print_package_version(pkg_name: str, msg: str = "", exitopt: bool = True) -> str:
     if msg == "":
         msg = f"{pkg_name} Version:"
-    cur_version = pkg_resources.get_distribution(pkg_name).version
+    try:
+        cur_version = version(pkg_name)
+    except PackageNotFoundError:
+        cur_version = "unknown"
 
     if exitopt:
         print(f'{msg} {cur_version}')

fosslight_util/output_format.py

@@ -48,7 +48,8 @@ def check_output_format(output='', format='', customized_format={}):
                 if format:
                     if output_extension != basename_extension:
                         success = False
-                        msg = f"(-o & -f option) Enter the same extension of output file(-o:'{output}') with format(-f:'{format}')."
+                        msg = f"(-o & -f option) Enter the same extension of output file(-o:'{output}') \
+                                with format(-f:'{format}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
@@ -96,7 +97,8 @@ def check_output_formats(output='', formats=[], customized_format={}):
                 if formats:
                     if basename_extension not in output_extensions:
                         success = False
-                        msg = f"(-o & -f option) The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
+                        msg = f"(-o & -f option) The format of output file(-o:'{output}') \
+                                should be in the format list(-f:'{formats}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
@@ -145,7 +147,8 @@ def check_output_formats_v2(output='', formats=[], customized_format={}):
                 if formats:
                     if basename_extension not in output_extensions:
                         success = False
-                        msg = f"(-o & -f option) The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
+                        msg = f"(-o & -f option) The format of output file(-o:'{output}') \
+                                should be in the format list(-f:'{formats}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
@@ -157,6 +160,7 @@ def check_output_formats_v2(output='', formats=[], customized_format={}):
     if not output_extensions:
         output_extensions = ['.xlsx']
     if not formats:
+        formats = []
         for ext in output_extensions:
             for key, value in support_format.items():
                 if value == ext:

fosslight_util/set_log.py

@@ -6,7 +6,6 @@
 import logging
 import os
 from pathlib import Path
-import pkg_resources
 import sys
 import platform
 from . import constant as constant
@@ -15,6 +14,11 @@ import coloredlogs
 from typing import Tuple
 from logging import Logger
 
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    from importlib_metadata import version, PackageNotFoundError  # Python <3.8
+
 
 def init_check_latest_version(pkg_version="", main_package_name=""):
 
@@ -92,9 +96,11 @@ def init_log(log_file: str, create_file: bool = True, stream_log_level: int = lo
     if main_package_name != "":
         pkg_info = main_package_name
         try:
-            pkg_version = pkg_resources.get_distribution(main_package_name).version
+            pkg_version = version(main_package_name)
             init_check_latest_version(pkg_version, main_package_name)
             pkg_info = main_package_name + " v" + pkg_version
+        except PackageNotFoundError:
+            logger.debug('Cannot check the version: Package not found')
         except Exception as error:
             logger.debug('Cannot check the version:' + str(error))
         _result_log["Tool Info"] = pkg_info

fosslight_util/write_cyclonedx.py

@@ -5,16 +5,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-import sys
 import logging
 import re
-import json
 from pathlib import Path
-from datetime import datetime
-from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick
 from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER,
-                                     FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE)
-from fosslight_util.oss_item import CHECKSUM_NULL, get_checksum_sha1
+                                     FOSSLIGHT_SOURCE)
 import traceback
 
 logger = logging.getLogger(LOGGER_NAME)
@@ -27,14 +22,11 @@ try:
     from cyclonedx.model import XsUri, ExternalReferenceType
     from cyclonedx.model.bom import Bom
     from cyclonedx.model.component import Component, ComponentType, HashAlgorithm, HashType, ExternalReference
-    from cyclonedx.model.contact import OrganizationalEntity
     from cyclonedx.output import make_outputter, BaseOutput
     from cyclonedx.output.json import JsonV1Dot6
     from cyclonedx.schema import OutputFormat, SchemaVersion
-    from cyclonedx.validation import make_schemabased_validator
     from cyclonedx.validation.json import JsonStrictValidator
     from cyclonedx.output.json import Json as JsonOutputter
-    from cyclonedx.output.xml import Xml as XmlOutputter
     from cyclonedx.validation.xml import XmlValidator
 except Exception:
     logger.info('No import cyclonedx-python-lib')
@@ -66,7 +58,6 @@ def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
                                                             type=ComponentType.APPLICATION,
                                                             bom_ref=str(comp_id))
         relation_tree = {}
-        bom_ref_packages = []
 
         output_dir = os.path.dirname(output_file_without_ext)
         Path(output_dir).mkdir(parents=True, exist_ok=True)
@@ -82,7 +73,7 @@ def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
                         comp_type = ComponentType.LIBRARY
 
                     for oss_item in file_item.oss_items:
-                        if oss_item.name == '':
+                        if oss_item.name == '' or oss_item.name == '-':
                             if scanner_name == FOSSLIGHT_DEPENDENCY:
                                 continue
                             else:
@@ -102,7 +93,8 @@ def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
                         if scanner_name == FOSSLIGHT_DEPENDENCY and file_item.purl:
                             comp.purl = PackageURL.from_string(file_item.purl)
                         if scanner_name != FOSSLIGHT_DEPENDENCY:
-                            comp.hashes = [HashType(alg=HashAlgorithm.SHA_1, content=file_item.checksum)]
+                            if file_item.checksum != '0':
+                                comp.hashes = [HashType(alg=HashAlgorithm.SHA_1, content=file_item.checksum)]
 
                         if oss_item.download_location != '':
                             comp.external_references = [ExternalReference(url=XsUri(oss_item.download_location),
@@ -113,7 +105,7 @@ def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
                             try:
                                 oss_licenses.append(lc_factory.make_from_string(ol))
                             except Exception:
-                                logger.info(f'No spdx license name: {oi}')
+                                logger.info(f'No spdx license name: {ol}')
                         if oss_licenses:
                             comp.licenses = oss_licenses
 
@@ -192,9 +184,9 @@ def write_cyclonedx_json(bom, result_file):
         except MissingOptionalDependencyException as error:
             logger.debug(f'JSON-validation was skipped due to {error}')
     except Exception as e:
+        logger.warning(f'Fail to write cyclonedx json: {e}')
         success = False
     return success
-        
 
 
 def write_cyclonedx_xml(bom, result_file):
@@ -213,5 +205,6 @@ def write_cyclonedx_xml(bom, result_file):
         except MissingOptionalDependencyException as error:
             logger.debug(f'XML-validation was skipped due to {error}')
     except Exception as e:
+        logger.warning(f'Fail to write cyclonedx xml: {e}')
         success = False
-    return success
+    return success

fosslight_util/write_excel.py

@@ -34,6 +34,7 @@ IDX_FILE = 0
 IDX_EXCLUDE = 7
 logger = logging.getLogger(LOGGER_NAME)
 COVER_SHEET_NAME = 'Scanner Info'
+MAX_EXCEL_URL_LENGTH = 255
 
 
 def get_header_row(sheet_name, extended_header={}):
@@ -181,7 +182,10 @@ def write_result_to_sheet(worksheet, sheet_contents):
     for row_item in sheet_contents:
         worksheet.write(row, 0, row)
         for col_num, value in enumerate(row_item):
-            worksheet.write(row, col_num + 1, str(value))
+            if len(value) > MAX_EXCEL_URL_LENGTH and (value.startswith("http://") or value.startswith("https://")):
+                worksheet.write_string(row, col_num + 1, str(value))
+            else:
+                worksheet.write(row, col_num + 1, str(value))
         row += 1
 
 

fosslight_util/write_scancodejson.py

@@ -6,7 +6,7 @@
 import logging
 import os
 import json
-from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_DEPENDENCY
 from fosslight_util.oss_item import ScannerItem
 from typing import List
 
@@ -20,22 +20,27 @@ def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[Sca
     json_output['summary'] = {}
     json_output['license_detections'] = []
     json_output['files'] = []
+    json_output['dependencies'] = []
 
-    for file_items in oss_list.file_items.values():
+    for scanner, file_items in oss_list.file_items.items():
         for fi in file_items:
-            if fi.exclude:
-                continue
-            if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
-                continue
-            if not fi.source_name_or_path:
-                fi.source_name_or_path = EMPTY_FILE_PATH
-            json_output['files'] = add_item_in_files(fi, json_output['files'])
+            if scanner == FOSSLIGHT_DEPENDENCY:
+                json_output['dependencies'] = add_item_in_deps(fi, json_output['dependencies'])
+            else:
+                if fi.exclude:
+                    continue
+                if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
+                    continue
+                if not fi.source_name_or_path:
+                    fi.source_name_or_path = EMPTY_FILE_PATH
+                json_output['files'] = add_item_in_files(fi, json_output['files'])
 
     with open(os.path.join(output_dir, output_filename), 'w') as f:
         json.dump(json_output, f, sort_keys=False, indent=4)
 
 
-def append_oss_item_in_filesitem(oss_items, files_item):
+def get_oss_item_list(oss_items):
+    scan_oss_items = []
     for oi in oss_items:
         if oi.exclude:
             continue
@@ -46,9 +51,9 @@ def append_oss_item_in_filesitem(oss_items, files_item):
         oss_item['copyright'] = oi.copyright
         oss_item['download_location'] = oi.download_location
         oss_item['comment'] = oi.comment
-        files_item['oss'].append(oss_item)
+        scan_oss_items.append(oss_item)
 
-    return files_item
+    return scan_oss_items
 
 
 def add_item_in_files(file_item, files_list):
@@ -57,8 +62,20 @@ def add_item_in_files(file_item, files_list):
     files_item['name'] = os.path.basename(file_item.source_name_or_path)
     files_item['is_binary'] = file_item.is_binary
     files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(file_item.source_name_or_path))
-    files_item['oss'] = []
-    files_item = append_oss_item_in_filesitem(file_item.oss_items, files_item)
+    files_item['oss'] = get_oss_item_list(file_item.oss_items)
+
     files_list.append(files_item)
 
     return files_list
+
+
+def add_item_in_deps(file_item, deps_list):
+    deps_item = {}
+    deps_item['purl'] = file_item.purl
+    deps_item['scope'] = 'dependencies'
+    deps_item['depends_on'] = file_item.depends_on
+    deps_item['oss'] = get_oss_item_list(file_item.oss_items)
+
+    deps_list.append(deps_item)
+
+    return deps_list

{fosslight_util-2.1.7.dist-info → fosslight_util-2.1.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.1.7
+Version: 2.1.18
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

fosslight_util-2.1.18.dist-info/RECORD

@@ -0,0 +1,32 @@
+fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fosslight_util/_get_downloadable_url.py,sha256=SqXCESg1GVXhCpObZoceXdFJVecDrDp-7qW88w0QsCY,12091
+fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE4,2916
+fosslight_util/constant.py,sha256=zElnWOzXt020sYiFTiRQn8ZjZyZpL3aPmfAqfQLcxJk,2278
+fosslight_util/correct.py,sha256=1WEAL-9_KhjFPLucPhv0PNN3K7avm0z8mU6sTuSyeHM,3864
+fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
+fosslight_util/download.py,sha256=V3EBgXt-xHarJZFrskXHaX4Ij81ZPjj6hzvtmpKu7xE,17642
+fosslight_util/exclude.py,sha256=fDmBsZJ_F7O9Oh2T-07R03XNbElo1tFaf_z01KfSAqU,2399
+fosslight_util/help.py,sha256=Bmyz-eFP0X0qUfgFPrWiuyUPE0TLQfWjgfHTzJBIInc,2377
+fosslight_util/oss_item.py,sha256=8W2HlwqGH3l1iPPdvycrRYKsBSBpqAkqYyYtBVPgMtY,6868
+fosslight_util/output_format.py,sha256=BP23LspxawDZ_a99oWLVKWUQ-G7P5uoUpjEXhkRFKwc,8801
+fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
+fosslight_util/read_excel.py,sha256=-QvrdxaNqYOpIm1H7ZqIEh5NLvFPymZo6BAOZcQmQug,5263
+fosslight_util/set_log.py,sha256=AbcLFLvY9GSOYSN0a110wO5gNcyc8KKnNjl7GxHEW9A,4008
+fosslight_util/spdx_licenses.py,sha256=GvMNe_D4v2meapTVwPu2BJXInnTo3_gIzg669eJhUu0,3691
+fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
+fosslight_util/write_cyclonedx.py,sha256=hq817j-0OM89B8jtZKgHgvVa0YEaYHlz_8R5vNpe21I,9662
+fosslight_util/write_excel.py,sha256=QUIMCnmEKJoSpri5RctBcKLvhDShLdZUP_dhHv-sVy8,10165
+fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
+fosslight_util/write_scancodejson.py,sha256=dMCjTtUnNR5BCL6gBCleDT8bTSAN5Gg2RAfimmkGXUE,2692
+fosslight_util/write_spdx.py,sha256=Ov9jBlfVrkWIymcfAxbupUxDZKfCOZZGOPZ4v-x230M,12108
+fosslight_util/write_txt.py,sha256=BEFjYBppqk1CITx-fUN4vfvKv0XCs1GXWtc2Iu-etU4,629
+fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,3238
+fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
+fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
+fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
+fosslight_util-2.1.18.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.18.dist-info/METADATA,sha256=eiYd-q6MOwvzp-SPII-NU2MPNQs6rdm9PgPZEWdweFY,6500
+fosslight_util-2.1.18.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+fosslight_util-2.1.18.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-2.1.18.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.18.dist-info/RECORD,,

fosslight_util-2.1.7.dist-info/RECORD

@@ -1,31 +0,0 @@
-fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_util/_get_downloadable_url.py,sha256=tZz7UTUuLAbz2muXocOb9epMY_6RXIt-GEM8jhN0c3o,9315
-fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE4,2916
-fosslight_util/constant.py,sha256=Ig3ACm9_QirE4389Wt-IfxOqRkVOUjqGnX1B05z2Byo,2151
-fosslight_util/correct.py,sha256=3iUipan8ZX8sbyIIGAPtMkAGvZ4YucjeJwx1K1Bx_z4,3897
-fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
-fosslight_util/download.py,sha256=bCKvW76XJTnKMAUW5sJZxg_wBUhiybXovJuL04W4P4c,16364
-fosslight_util/help.py,sha256=M3_XahUkP794US9Q0NS6ujmGvrFFnKBHsTU95Fg1KpA,2181
-fosslight_util/oss_item.py,sha256=8W2HlwqGH3l1iPPdvycrRYKsBSBpqAkqYyYtBVPgMtY,6868
-fosslight_util/output_format.py,sha256=AdQVzCpar6n3PCDtijLWbJyFAGKQVE7JhLXlHPtITH4,8678
-fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
-fosslight_util/read_excel.py,sha256=-QvrdxaNqYOpIm1H7ZqIEh5NLvFPymZo6BAOZcQmQug,5263
-fosslight_util/set_log.py,sha256=Xpa94AiOyGEK8ucaYkvkAllvlen1Pq_d6UG6kPYBYBc,3780
-fosslight_util/spdx_licenses.py,sha256=GvMNe_D4v2meapTVwPu2BJXInnTo3_gIzg669eJhUu0,3691
-fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
-fosslight_util/write_cyclonedx.py,sha256=N6r32zj_ikoGkZa9xoreKqdsncoOfFKnV7lHpeiiEhI,9902
-fosslight_util/write_excel.py,sha256=G0fIslbWoOtWZCJxbBGLCpUKbhmwrrqhI5PHwRw8_44,9931
-fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
-fosslight_util/write_scancodejson.py,sha256=81n7cWNYoyIKE_V4Kx5YtL2CgjMPIjoKdnSU3inkpJY,2163
-fosslight_util/write_spdx.py,sha256=Ov9jBlfVrkWIymcfAxbupUxDZKfCOZZGOPZ4v-x230M,12108
-fosslight_util/write_txt.py,sha256=BEFjYBppqk1CITx-fUN4vfvKv0XCs1GXWtc2Iu-etU4,629
-fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,3238
-fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
-fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
-fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.1.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.1.7.dist-info/METADATA,sha256=_SbQsVJDKbUmd3C0i1fs7C-B9z92g_SLHWrQVb2mi_s,6499
-fosslight_util-2.1.7.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-fosslight_util-2.1.7.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-2.1.7.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.1.7.dist-info/RECORD,,