fosslight-util 2.0.0__py3-none-any.whl → 2.1.29__py3-none-any.whl

fosslight_util/oss_item.py

@@ -5,11 +5,13 @@
 
 import logging
 import os
+import hashlib
 from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SCANNER
 from fosslight_util.cover import CoverItem
 from typing import List, Dict
 
 _logger = logging.getLogger(LOGGER_NAME)
+CHECKSUM_NULL = "0"
 
 
 class OssItem:
@@ -58,10 +60,12 @@ class OssItem:
 
     @copyright.setter
     def copyright(self, value):
-        if value != "":
+        if value:
             if isinstance(value, list):
-                value = "\n".join(value)
-            value = value.strip()
+                value = list(set(value))
+            else:
+                value = set(value.split("\n"))
+            value = "\n".join(value).strip()
         self._copyright = value
 
     @property
@@ -98,6 +102,7 @@ class FileItem:
         self._comment = ""
         self.is_binary = False
         self.oss_items: List[OssItem] = []
+        self.checksum = CHECKSUM_NULL
 
     def __del__(self):
         pass
@@ -169,6 +174,22 @@ class FileItem:
         return items
 
 
+def get_checksum_sha1(source_name_or_path) -> str:
+    checksum = CHECKSUM_NULL
+    try:
+        checksum = str(hashlib.sha1(source_name_or_path.encode()).hexdigest())
+    except Exception:
+        try:
+            f = open(source_name_or_path, "rb")
+            byte = f.read()
+            checksum = str(hashlib.sha1(byte).hexdigest())
+            f.close()
+        except Exception as ex:
+            _logger.info(f"(Error) Get_checksum: {ex}")
+
+    return checksum
+
+
 def invalid(cmd):
     _logger.info('[{}] is invalid'.format(cmd))
 

fosslight_util/output_format.py

@@ -3,12 +3,17 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import os
+import platform
 from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv
 from fosslight_util.write_opossum import write_opossum
 from fosslight_util.write_yaml import write_yaml
+from fosslight_util.write_spdx import write_spdx
+from fosslight_util.write_cyclonedx import write_cyclonedx
 from typing import Tuple
 
-SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'}
+SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml',
+                  'spdx-yaml': '.yaml', 'spdx-json': '.json', 'spdx-xml': '.xml',
+                  'spdx-tag': '.tag', 'cyclonedx-json': '.json', 'cyclonedx-xml': '.xml'}
 
 
 def check_output_format(output='', format='', customized_format={}):
@@ -27,7 +32,7 @@ def check_output_format(output='', format='', customized_format={}):
         format = format.lower()
         if format not in list(support_format.keys()):
             success = False
-            msg = 'Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+            msg = '(-f option) Enter the supported format: ' + ', '.join(list(support_format.keys()))
         else:
             output_extension = support_format[format]
 
@@ -43,11 +48,12 @@ def check_output_format(output='', format='', customized_format={}):
                 if format:
                     if output_extension != basename_extension:
                         success = False
-                        msg = f"Enter the same extension of output file(-o:'{output}') with format(-f:'{format}')."
+                        msg = f"(-o & -f option) Enter the same extension of output file(-o:'{output}') \
+                                with format(-f:'{format}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
-                        msg = 'Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                        msg = '(-o option) Enter the supported file extension: ' + ', '.join(list(support_format.values()))
                 if success:
                     output_file = basename_file
                     output_extension = basename_extension
@@ -75,7 +81,7 @@ def check_output_formats(output='', formats=[], customized_format={}):
         for format in formats:
             if format not in list(support_format.keys()):
                 success = False
-                msg = 'Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+                msg = '(-f option) Enter the supported format: ' + ', '.join(list(support_format.keys()))
             else:
                 output_extensions.append(support_format[format])
 
@@ -91,11 +97,12 @@ def check_output_formats(output='', formats=[], customized_format={}):
                 if formats:
                     if basename_extension not in output_extensions:
                         success = False
-                        msg = f"The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
+                        msg = f"(-o & -f option) The format of output file(-o:'{output}') \
+                                should be in the format list(-f:'{formats}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
-                        msg = 'Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                        msg = '(-o option) Enter the supported file extension: ' + ', '.join(list(support_format.values()))
                     output_extensions.append(basename_extension)
                 output_files = [basename_file for _ in range(len(output_extensions))]
             else:
@@ -106,8 +113,64 @@ def check_output_formats(output='', formats=[], customized_format={}):
     return success, msg, output_path, output_files, output_extensions
 
 
+def check_output_formats_v2(output='', formats=[], customized_format={}):
+    success = True
+    msg = ''
+    output_path = ''
+    output_files = []
+    output_extensions = []
+
+    if customized_format:
+        support_format = customized_format
+    else:
+        support_format = SUPPORT_FORMAT
+
+    if formats:
+        # If -f option exist
+        formats = [format.lower() for format in formats]
+        for format in formats:
+            if format not in list(support_format.keys()):
+                success = False
+                msg = '(-f option) Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+            else:
+                output_extensions.append(support_format[format])
+
+    if success:
+        if output != '':
+            basename_extension = ''
+            if not os.path.isdir(output):
+                output_path = os.path.dirname(output)
+
+                basename = os.path.basename(output)
+                basename_file, basename_extension = os.path.splitext(basename)
+            if basename_extension:
+                if formats:
+                    if basename_extension not in output_extensions:
+                        success = False
+                        msg = f"(-o & -f option) The format of output file(-o:'{output}') \
+                                should be in the format list(-f:'{formats}')."
+                else:
+                    if basename_extension not in support_format.values():
+                        success = False
+                        msg = '(-o option) Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                    output_extensions.append(basename_extension)
+                output_files = [basename_file for _ in range(len(output_extensions))]
+            else:
+                output_path = output
+    if not output_extensions:
+        output_extensions = ['.xlsx']
+    if not formats:
+        formats = []
+        for ext in output_extensions:
+            for key, value in support_format.items():
+                if value == ext:
+                    formats.append(key)
+                    break
+    return success, msg, output_path, output_files, output_extensions, formats
+
+
 def write_output_file(output_file_without_ext: str, file_extension: str, scan_item, extended_header: dict = {},
-                      hide_header: dict = {}) -> Tuple[bool, str, str]:
+                      hide_header: dict = {}, format: str = '', spdx_version: str = '2.3') -> Tuple[bool, str, str]:
     success = True
     msg = ''
 
@@ -115,16 +178,35 @@ def write_output_file(output_file_without_ext: str, file_extension: str, scan_it
         file_extension = '.xlsx'
     result_file = output_file_without_ext + file_extension
 
-    if file_extension == '.xlsx':
-        success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
-    elif file_extension == '.csv':
-        success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
-    elif file_extension == '.json':
-        success, msg = write_opossum(result_file, scan_item)
-    elif file_extension == '.yaml':
-        success, msg, result_file = write_yaml(result_file, scan_item, False)
+    if format:
+        if format == 'excel':
+            success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
+        elif format == 'csv':
+            success, msg, _ = write_result_to_csv(result_file, scan_item, False, extended_header)
+        elif format == 'opossum':
+            success, msg = write_opossum(result_file, scan_item)
+        elif format == 'yaml':
+            success, msg, _ = write_yaml(result_file, scan_item, False)
+        elif format.startswith('spdx') or format.startswith('cyclonedx'):
+            if platform.system() == 'Windows' or platform.system() == 'Darwin':
+                success = False
+                msg = f'{platform.system()} not support spdx format.'
+            else:
+                if format.startswith('spdx'):
+                    success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version)
+                elif format.startswith('cyclonedx'):
+                    success, msg, _ = write_cyclonedx(output_file_without_ext, file_extension, scan_item)
     else:
-        success = False
-        msg = f'Not supported file extension({file_extension})'
+        if file_extension == '.xlsx':
+            success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
+        elif file_extension == '.csv':
+            success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
+        elif file_extension == '.json':
+            success, msg = write_opossum(result_file, scan_item)
+        elif file_extension == '.yaml':
+            success, msg, result_file = write_yaml(result_file, scan_item, False)
+        else:
+            success = False
+            msg = f'(-f option) Not supported file extension({file_extension})'
 
     return success, msg, result_file

fosslight_util/set_log.py

@@ -6,7 +6,6 @@
 import logging
 import os
 from pathlib import Path
-import pkg_resources
 import sys
 import platform
 from . import constant as constant
@@ -15,6 +14,11 @@ import coloredlogs
 from typing import Tuple
 from logging import Logger
 
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    from importlib_metadata import version, PackageNotFoundError  # Python <3.8
+
 
 def init_check_latest_version(pkg_version="", main_package_name=""):
 
@@ -92,9 +96,11 @@ def init_log(log_file: str, create_file: bool = True, stream_log_level: int = lo
     if main_package_name != "":
         pkg_info = main_package_name
         try:
-            pkg_version = pkg_resources.get_distribution(main_package_name).version
+            pkg_version = version(main_package_name)
             init_check_latest_version(pkg_version, main_package_name)
             pkg_info = main_package_name + " v" + pkg_version
+        except PackageNotFoundError:
+            logger.debug('Cannot check the version: Package not found')
         except Exception as error:
             logger.debug('Cannot check the version:' + str(error))
         _result_log["Tool Info"] = pkg_info

fosslight_util/write_cyclonedx.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 LG Electronics Inc.
+# Copyright (c) OWASP Foundation.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import re
+from pathlib import Path
+from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER,
+                                     FOSSLIGHT_SOURCE)
+import traceback
+
+logger = logging.getLogger(LOGGER_NAME)
+
+try:
+    from packageurl import PackageURL
+    from cyclonedx.builder.this import this_component as cdx_lib_component
+    from cyclonedx.exception import MissingOptionalDependencyException
+    from cyclonedx.factory.license import LicenseFactory
+    from cyclonedx.model import XsUri, ExternalReferenceType
+    from cyclonedx.model.bom import Bom
+    from cyclonedx.model.component import Component, ComponentType, HashAlgorithm, HashType, ExternalReference
+    from cyclonedx.output import make_outputter, BaseOutput
+    from cyclonedx.output.json import JsonV1Dot6
+    from cyclonedx.schema import OutputFormat, SchemaVersion
+    from cyclonedx.validation.json import JsonStrictValidator
+    from cyclonedx.output.json import Json as JsonOutputter
+    from cyclonedx.validation.xml import XmlValidator
+except Exception:
+    logger.info('No import cyclonedx-python-lib')
+
+
+def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
+    success = True
+    error_msg = ''
+
+    bom = Bom()
+    if scan_item:
+        try:
+            cover_name = scan_item.cover.get_print_json()["Tool information"].split('(').pop(0).strip()
+            match = re.search(r"(.+) v([0-9.]+)", cover_name)
+            if match:
+                scanner_name = match.group(1)
+            else:
+                scanner_name = FOSSLIGHT_SCANNER
+        except Exception:
+            cover_name = FOSSLIGHT_SCANNER
+            scanner_name = FOSSLIGHT_SCANNER
+
+        lc_factory = LicenseFactory()
+        bom.metadata.tools.components.add(cdx_lib_component())
+        bom.metadata.tools.components.add(Component(name=scanner_name.upper(),
+                                                    type=ComponentType.APPLICATION))
+        comp_id = 0
+        bom.metadata.component = root_component = Component(name='Root Component',
+                                                            type=ComponentType.APPLICATION,
+                                                            bom_ref=str(comp_id))
+        relation_tree = {}
+
+        output_dir = os.path.dirname(output_file_without_ext)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        try:
+            root_package = False
+            for scanner_name, file_items in scan_item.file_items.items():
+                for file_item in file_items:
+                    if file_item.exclude:
+                        continue
+                    if scanner_name == FOSSLIGHT_SOURCE:
+                        comp_type = ComponentType.FILE
+                    else:
+                        comp_type = ComponentType.LIBRARY
+
+                    for oss_item in file_item.oss_items:
+                        if oss_item.name == '' or oss_item.name == '-':
+                            if scanner_name == FOSSLIGHT_DEPENDENCY:
+                                continue
+                            else:
+                                comp_name = file_item.source_name_or_path
+                        else:
+                            comp_name = oss_item.name
+
+                        comp_id += 1
+                        comp = Component(type=comp_type,
+                                         name=comp_name,
+                                         bom_ref=str(comp_id))
+
+                        if oss_item.version != '':
+                            comp.version = oss_item.version
+                        if oss_item.copyright != '':
+                            comp.copyright = oss_item.copyright
+                        if scanner_name == FOSSLIGHT_DEPENDENCY and file_item.purl:
+                            comp.purl = PackageURL.from_string(file_item.purl)
+                        if scanner_name != FOSSLIGHT_DEPENDENCY:
+                            if file_item.checksum != '0':
+                                comp.hashes = [HashType(alg=HashAlgorithm.SHA_1, content=file_item.checksum)]
+
+                        if oss_item.download_location != '':
+                            comp.external_references = [ExternalReference(url=XsUri(oss_item.download_location),
+                                                                          type=ExternalReferenceType.WEBSITE)]
+
+                        oss_licenses = []
+                        for ol in oss_item.license:
+                            try:
+                                oss_licenses.append(lc_factory.make_from_string(ol))
+                            except Exception:
+                                logger.info(f'No spdx license name: {ol}')
+                        if oss_licenses:
+                            comp.licenses = oss_licenses
+
+                        root_package = False
+                        if scanner_name == FOSSLIGHT_DEPENDENCY:
+                            if oss_item.comment:
+                                oss_comment = oss_item.comment.split('/')
+                                for oc in oss_comment:
+                                    if oc in ['direct', 'transitive', 'root package']:
+                                        if oc == 'direct':
+                                            bom.register_dependency(root_component, [comp])
+                                        elif oc == 'root package':
+                                            root_package = True
+                                            root_component.name = comp_name
+                                            root_component.type = comp_type
+                                            comp_id -= 1
+                            else:
+                                bom.register_dependency(root_component, [comp])
+                            if len(file_item.depends_on) > 0:
+                                purl = file_item.purl
+                                relation_tree[purl] = []
+                                relation_tree[purl].extend(file_item.depends_on)
+
+                        if not root_package:
+                            bom.components.add(comp)
+
+            if len(bom.components) > 0:
+                for comp_purl in relation_tree:
+                    comp = bom.get_component_by_purl(PackageURL.from_string(comp_purl))
+                    if comp:
+                        dep_comp_list = []
+                        for dep_comp_purl in relation_tree[comp_purl]:
+                            dep_comp = bom.get_component_by_purl(PackageURL.from_string(dep_comp_purl))
+                            if dep_comp:
+                                dep_comp_list.append(dep_comp)
+                        bom.register_dependency(comp, dep_comp_list)
+
+        except Exception as e:
+            success = False
+            error_msg = f'Failed to create CycloneDX document object:{e}, {traceback.format_exc()}'
+    else:
+        success = False
+        error_msg = 'No item to write in output file.'
+
+    result_file = ''
+    if success:
+        result_file = output_file_without_ext + output_extension
+        try:
+            if output_extension == '.json':
+                write_cyclonedx_json(bom, result_file)
+            elif output_extension == '.xml':
+                write_cyclonedx_xml(bom, result_file)
+            else:
+                success = False
+                error_msg = f'Not supported output_extension({output_extension})'
+        except Exception as e:
+            success = False
+            error_msg = f'Failed to write CycloneDX document: {e}'
+            if os.path.exists(result_file):
+                os.remove(result_file)
+
+    return success, error_msg, result_file
+
+
+def write_cyclonedx_json(bom, result_file):
+    success = True
+    try:
+        my_json_outputter: 'JsonOutputter' = JsonV1Dot6(bom)
+        my_json_outputter.output_to_file(result_file)
+        serialized_json = my_json_outputter.output_as_string(indent=2)
+        my_json_validator = JsonStrictValidator(SchemaVersion.V1_6)
+        try:
+            validation_errors = my_json_validator.validate_str(serialized_json)
+            if validation_errors:
+                logger.warning(f'JSON invalid, ValidationError: {repr(validation_errors)}')
+        except MissingOptionalDependencyException as error:
+            logger.debug(f'JSON-validation was skipped due to {error}')
+    except Exception as e:
+        logger.warning(f'Fail to write cyclonedx json: {e}')
+        success = False
+    return success
+
+
+def write_cyclonedx_xml(bom, result_file):
+    success = True
+    try:
+        my_xml_outputter: BaseOutput = make_outputter(bom=bom,
+                                                      output_format=OutputFormat.XML,
+                                                      schema_version=SchemaVersion.V1_6)
+        my_xml_outputter.output_to_file(filename=result_file)
+        serialized_xml = my_xml_outputter.output_as_string(indent=2)
+        my_xml_validator = XmlValidator(SchemaVersion.V1_6)
+        try:
+            validation_errors = my_xml_validator.validate_str(serialized_xml)
+            if validation_errors:
+                logger.warning(f'XML invalid, ValidationError: {repr(validation_errors)}')
+        except MissingOptionalDependencyException as error:
+            logger.debug(f'XML-validation was skipped due to {error}')
+    except Exception as e:
+        logger.warning(f'Fail to write cyclonedx xml: {e}')
+        success = False
+    return success

fosslight_util/write_excel.py

@@ -34,6 +34,7 @@ IDX_FILE = 0
 IDX_EXCLUDE = 7
 logger = logging.getLogger(LOGGER_NAME)
 COVER_SHEET_NAME = 'Scanner Info'
+MAX_EXCEL_URL_LENGTH = 255
 
 
 def get_header_row(sheet_name, extended_header={}):
@@ -181,7 +182,10 @@ def write_result_to_sheet(worksheet, sheet_contents):
     for row_item in sheet_contents:
         worksheet.write(row, 0, row)
         for col_num, value in enumerate(row_item):
-            worksheet.write(row, col_num + 1, str(value))
+            if len(value) > MAX_EXCEL_URL_LENGTH and (value.startswith("http://") or value.startswith("https://")):
+                worksheet.write_string(row, col_num + 1, str(value))
+            else:
+                worksheet.write(row, col_num + 1, str(value))
         row += 1
 
 

fosslight_util/write_scancodejson.py

@@ -6,7 +6,7 @@
 import logging
 import os
 import json
-from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_DEPENDENCY
 from fosslight_util.oss_item import ScannerItem
 from typing import List
 
@@ -20,22 +20,27 @@ def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[Sca
     json_output['summary'] = {}
     json_output['license_detections'] = []
     json_output['files'] = []
+    json_output['dependencies'] = []
 
-    for file_items in oss_list.file_items.values():
+    for scanner, file_items in oss_list.file_items.items():
         for fi in file_items:
-            if fi.exclude:
-                continue
-            if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
-                continue
-            if not fi.source_name_or_path:
-                fi.source_name_or_path = EMPTY_FILE_PATH
-            json_output['files'] = add_item_in_files(fi, json_output['files'])
+            if scanner == FOSSLIGHT_DEPENDENCY:
+                json_output['dependencies'] = add_item_in_deps(fi, json_output['dependencies'])
+            else:
+                if fi.exclude:
+                    continue
+                if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
+                    continue
+                if not fi.source_name_or_path:
+                    fi.source_name_or_path = EMPTY_FILE_PATH
+                json_output['files'] = add_item_in_files(fi, json_output['files'])
 
     with open(os.path.join(output_dir, output_filename), 'w') as f:
         json.dump(json_output, f, sort_keys=False, indent=4)
 
 
-def append_oss_item_in_filesitem(oss_items, files_item):
+def get_oss_item_list(oss_items):
+    scan_oss_items = []
     for oi in oss_items:
         if oi.exclude:
             continue
@@ -46,9 +51,9 @@ def append_oss_item_in_filesitem(oss_items, files_item):
         oss_item['copyright'] = oi.copyright
         oss_item['download_location'] = oi.download_location
         oss_item['comment'] = oi.comment
-        files_item['oss'].append(oss_item)
+        scan_oss_items.append(oss_item)
 
-    return files_item
+    return scan_oss_items
 
 
 def add_item_in_files(file_item, files_list):
@@ -57,8 +62,20 @@ def add_item_in_files(file_item, files_list):
     files_item['name'] = os.path.basename(file_item.source_name_or_path)
     files_item['is_binary'] = file_item.is_binary
     files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(file_item.source_name_or_path))
-    files_item['oss'] = []
-    files_item = append_oss_item_in_filesitem(file_item.oss_items, files_item)
+    files_item['oss'] = get_oss_item_list(file_item.oss_items)
+
     files_list.append(files_item)
 
     return files_list
+
+
+def add_item_in_deps(file_item, deps_list):
+    deps_item = {}
+    deps_item['purl'] = file_item.purl
+    deps_item['scope'] = 'dependencies'
+    deps_item['depends_on'] = file_item.depends_on
+    deps_item['oss'] = get_oss_item_list(file_item.oss_items)
+
+    deps_list.append(deps_item)
+
+    return deps_list