fosslight-util 1.4.34__py3-none-any.whl → 2.1.28__py3-none-any.whl

fosslight_util/read_excel.py

@@ -3,26 +3,25 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import logging
-import xlrd
+from typing import List, Dict, Any
+import pandas as pd
 import json
 from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.oss_item import OssItem
+from fosslight_util.oss_item import OssItem, FileItem
 from fosslight_util.parsing_yaml import set_value_switch
 
 logger = logging.getLogger(LOGGER_NAME)
 IDX_CANNOT_FOUND = -1
 PREFIX_BIN = "bin"
-xlrd.xlsx.ensure_elementtree_imported(False, None)
-xlrd.xlsx.Element_has_iter = True
+SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
 
 
-def read_oss_report(excel_file, sheet_names=""):
-    _oss_report_items = []
-    xl_sheets = {}
-    all_sheet_to_read = []
-    not_matched_sheet = []
+def read_oss_report(excel_file: str, sheet_names: str = "", basepath: str = "") -> List[FileItem]:
+    fileitems: List[FileItem] = []
+    xl_sheets: Dict[str, Any] = {}
+    all_sheet_to_read: List[str] = []
+    not_matched_sheet: List[str] = []
     any_sheet_matched = False
-    SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
     if sheet_names:
         sheet_name_prefix_match = False
         sheet_name_to_read = sheet_names.split(",")
@@ -32,9 +31,8 @@ def read_oss_report(excel_file, sheet_names=""):
 
     try:
         logger.info(f"Read data from : {excel_file}")
-        xl_workbook = xlrd.open_workbook(excel_file)
-        all_sheet_in_excel = xl_workbook.sheet_names()
-
+        xl_workbook = pd.ExcelFile(excel_file, engine='openpyxl')
+        all_sheet_in_excel = xl_workbook.sheet_names
         for sheet_to_read in sheet_name_to_read:
             try:
                 any_sheet_matched = False
@@ -44,10 +42,9 @@ def read_oss_report(excel_file, sheet_names=""):
                     sheet_name_lower = sheet_name.lower()
                     if (sheet_name_prefix_match and sheet_name_lower.startswith(sheet_to_read_lower)) \
                        or sheet_to_read_lower == sheet_name_lower:
-                        sheet = xl_workbook.sheet_by_name(sheet_name)
-                        if sheet:
-                            xl_sheets[sheet_name] = sheet
-                            any_sheet_matched = True
+                        sheet = pd.read_excel(excel_file, sheet_name=sheet_name, engine='openpyxl', na_values='')
+                        xl_sheets[sheet_name] = sheet.fillna('')
+                        any_sheet_matched = True
                 if not any_sheet_matched:
                     not_matched_sheet.append(sheet_to_read)
             except Exception as error:
@@ -60,11 +57,14 @@ def read_oss_report(excel_file, sheet_names=""):
         elif (not sheet_name_prefix_match) and not_matched_sheet:
             logger.warning(f"Not matched sheet name: {not_matched_sheet}")
 
+        filepath_list = []
         for sheet_name, xl_sheet in xl_sheets.items():
             _item_idx = {
                 "ID": IDX_CANNOT_FOUND,
                 "Source Name or Path": IDX_CANNOT_FOUND,
+                "Source Path": IDX_CANNOT_FOUND,
                 "Binary Name": IDX_CANNOT_FOUND,
+                "Binary Path": IDX_CANNOT_FOUND,
                 "OSS Name": IDX_CANNOT_FOUND,
                 "OSS Version": IDX_CANNOT_FOUND,
                 "License": IDX_CANNOT_FOUND,
@@ -73,48 +73,49 @@ def read_oss_report(excel_file, sheet_names=""):
                 "Exclude": IDX_CANNOT_FOUND,
                 "Copyright Text": IDX_CANNOT_FOUND,
                 "Comment": IDX_CANNOT_FOUND,
-                "File Name or Path": IDX_CANNOT_FOUND
+                "File Name or Path": IDX_CANNOT_FOUND,
+                "Vulnerability Link": IDX_CANNOT_FOUND,
+                "TLSH": IDX_CANNOT_FOUND,
+                "SHA1": IDX_CANNOT_FOUND
             }
-            num_cols = xl_sheet.ncols
-            num_rows = xl_sheet.nrows
-            MAX_FIND_HEADER_COLUMN = 5 if num_rows > 5 else num_rows
-            DATA_START_ROW_IDX = 1
-            for row_idx in range(0, MAX_FIND_HEADER_COLUMN):
-                for col_idx in range(row_idx, num_cols):
-                    cell_obj = xl_sheet.cell(row_idx, col_idx)
-                    if cell_obj.value in _item_idx:
-                        _item_idx[cell_obj.value] = col_idx
 
-                if len([key for key, value in _item_idx.items() if value != IDX_CANNOT_FOUND]) > 3:
-                    DATA_START_ROW_IDX = row_idx + 1
-                    break
+            for index, value in enumerate(xl_sheet.columns.tolist()):
+                _item_idx[value] = index
 
             # Get all values, iterating through rows and columns
             column_keys = json.loads(json.dumps(_item_idx))
 
             is_bin = True if sheet_name.lower().startswith(PREFIX_BIN) else False
 
-            for row_idx in range(DATA_START_ROW_IDX, xl_sheet.nrows):
-                item = OssItem("")
-                item.is_binary = is_bin
+            for row_idx, row in xl_sheet.iterrows():
                 valid_row = True
                 load_data_cnt = 0
-
+                source_path = row[1]
+                if source_path not in filepath_list:
+                    filepath_list.append(source_path)
+                    fileitem = FileItem(basepath)
+                    fileitem.source_name_or_path = source_path
+                    fileitems.append(fileitem)
+                else:
+                    fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+                fileitem.is_binary = is_bin
+                ossitem = OssItem()
                 for column_key, column_idx in column_keys.items():
                     if column_idx != IDX_CANNOT_FOUND:
-                        cell_obj = xl_sheet.cell(row_idx, column_idx)
-                        cell_value = cell_obj.value
+                        cell_obj = xl_sheet.iloc[row_idx, column_idx]
+                        cell_value = cell_obj
+
                         if cell_value != "":
                             if column_key != "ID":
                                 if column_key:
                                     column_key = column_key.lower().strip()
-                                set_value_switch(item, column_key, cell_value)
+                                set_value_switch(ossitem, column_key, cell_value)
                                 load_data_cnt += 1
                             else:
                                 valid_row = False if cell_value == "-" else True
                 if valid_row and load_data_cnt > 0:
-                    _oss_report_items.append(item)
+                    fileitem.oss_items.append(ossitem)
 
     except Exception as error:
         logger.error(f"Parsing a OSS Report: {error}")
-    return _oss_report_items
+    return fileitems

fosslight_util/set_log.py

@@ -6,12 +6,18 @@
 import logging
 import os
 from pathlib import Path
-import pkg_resources
 import sys
 import platform
 from . import constant as constant
 from lastversion import lastversion
 import coloredlogs
+from typing import Tuple
+from logging import Logger
+
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    from importlib_metadata import version, PackageNotFoundError  # Python <3.8
 
 
 def init_check_latest_version(pkg_version="", main_package_name=""):
@@ -32,6 +38,21 @@ def init_check_latest_version(pkg_version="", main_package_name=""):
         logger.debug('Cannot check the latest version:' + str(error))
 
 
+def get_os_version():
+
+    logger = logging.getLogger(constant.LOGGER_NAME)
+
+    os_version = platform.system() + " " + platform.release()
+    if os_version == "Windows 10":
+        try:
+            windows_build = sys.getwindowsversion().build
+            if windows_build >= 22000:
+                os_version = "Windows 11"
+        except Exception as error:
+            logger.debug(str(error))
+    return os_version
+
+
 class CustomAdapter(logging.LoggerAdapter):
     def __init__(self, logger, extra):
         super(CustomAdapter, self).__init__(logger, {})
@@ -41,8 +62,8 @@ class CustomAdapter(logging.LoggerAdapter):
         return '[%s] %s' % (self.extra, msg), kwargs
 
 
-def init_log(log_file, create_file=True, stream_log_level=logging.INFO,
-             file_log_level=logging.DEBUG, main_package_name="", path_to_analyze=""):
+def init_log(log_file: str, create_file: bool = True, stream_log_level: int = logging.INFO, file_log_level: int = logging.DEBUG,
+             main_package_name: str = "", path_to_analyze: str = "", path_to_exclude: list = []) -> Tuple[Logger, dict]:
 
     logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -70,18 +91,22 @@ def init_log(log_file, create_file=True, stream_log_level=logging.INFO,
     _result_log = {
         "Tool Info": main_package_name,
         "Python version": _PYTHON_VERSION,
-        "OS": platform.system()+" "+platform.release(),
+        "OS": get_os_version(),
     }
     if main_package_name != "":
         pkg_info = main_package_name
         try:
-            pkg_version = pkg_resources.get_distribution(main_package_name).version
+            pkg_version = version(main_package_name)
             init_check_latest_version(pkg_version, main_package_name)
             pkg_info = main_package_name + " v" + pkg_version
+        except PackageNotFoundError:
+            logger.debug('Cannot check the version: Package not found')
         except Exception as error:
             logger.debug('Cannot check the version:' + str(error))
         _result_log["Tool Info"] = pkg_info
     if path_to_analyze != "":
         _result_log["Path to analyze"] = path_to_analyze
+    if path_to_exclude != []:
+        _result_log["Path to exclude"] = ", ".join(path_to_exclude)
 
     return logger, _result_log

fosslight_util/spdx_licenses.py

@@ -8,6 +8,7 @@ import os
 import sys
 import json
 import traceback
+from typing import Tuple
 
 _resources_dir = 'resources'
 _licenses_json_file = 'licenses.json'
@@ -34,7 +35,7 @@ def get_license_from_nick():
     return licenses
 
 
-def get_spdx_licenses_json():
+def get_spdx_licenses_json() -> Tuple[bool, str, str]:
     success = True
     error_msg = ''
     licenses = ''

fosslight_util/write_cyclonedx.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 LG Electronics Inc.
+# Copyright (c) OWASP Foundation.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import re
+from pathlib import Path
+from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER,
+                                     FOSSLIGHT_SOURCE)
+import traceback
+
+logger = logging.getLogger(LOGGER_NAME)
+
+try:
+    from packageurl import PackageURL
+    from cyclonedx.builder.this import this_component as cdx_lib_component
+    from cyclonedx.exception import MissingOptionalDependencyException
+    from cyclonedx.factory.license import LicenseFactory
+    from cyclonedx.model import XsUri, ExternalReferenceType
+    from cyclonedx.model.bom import Bom
+    from cyclonedx.model.component import Component, ComponentType, HashAlgorithm, HashType, ExternalReference
+    from cyclonedx.output import make_outputter, BaseOutput
+    from cyclonedx.output.json import JsonV1Dot6
+    from cyclonedx.schema import OutputFormat, SchemaVersion
+    from cyclonedx.validation.json import JsonStrictValidator
+    from cyclonedx.output.json import Json as JsonOutputter
+    from cyclonedx.validation.xml import XmlValidator
+except Exception:
+    logger.info('No import cyclonedx-python-lib')
+
+
+def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
+    success = True
+    error_msg = ''
+
+    bom = Bom()
+    if scan_item:
+        try:
+            cover_name = scan_item.cover.get_print_json()["Tool information"].split('(').pop(0).strip()
+            match = re.search(r"(.+) v([0-9.]+)", cover_name)
+            if match:
+                scanner_name = match.group(1)
+            else:
+                scanner_name = FOSSLIGHT_SCANNER
+        except Exception:
+            cover_name = FOSSLIGHT_SCANNER
+            scanner_name = FOSSLIGHT_SCANNER
+
+        lc_factory = LicenseFactory()
+        bom.metadata.tools.components.add(cdx_lib_component())
+        bom.metadata.tools.components.add(Component(name=scanner_name.upper(),
+                                                    type=ComponentType.APPLICATION))
+        comp_id = 0
+        bom.metadata.component = root_component = Component(name='Root Component',
+                                                            type=ComponentType.APPLICATION,
+                                                            bom_ref=str(comp_id))
+        relation_tree = {}
+
+        output_dir = os.path.dirname(output_file_without_ext)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        try:
+            root_package = False
+            for scanner_name, file_items in scan_item.file_items.items():
+                for file_item in file_items:
+                    if file_item.exclude:
+                        continue
+                    if scanner_name == FOSSLIGHT_SOURCE:
+                        comp_type = ComponentType.FILE
+                    else:
+                        comp_type = ComponentType.LIBRARY
+
+                    for oss_item in file_item.oss_items:
+                        if oss_item.name == '' or oss_item.name == '-':
+                            if scanner_name == FOSSLIGHT_DEPENDENCY:
+                                continue
+                            else:
+                                comp_name = file_item.source_name_or_path
+                        else:
+                            comp_name = oss_item.name
+
+                        comp_id += 1
+                        comp = Component(type=comp_type,
+                                         name=comp_name,
+                                         bom_ref=str(comp_id))
+
+                        if oss_item.version != '':
+                            comp.version = oss_item.version
+                        if oss_item.copyright != '':
+                            comp.copyright = oss_item.copyright
+                        if scanner_name == FOSSLIGHT_DEPENDENCY and file_item.purl:
+                            comp.purl = PackageURL.from_string(file_item.purl)
+                        if scanner_name != FOSSLIGHT_DEPENDENCY:
+                            if file_item.checksum != '0':
+                                comp.hashes = [HashType(alg=HashAlgorithm.SHA_1, content=file_item.checksum)]
+
+                        if oss_item.download_location != '':
+                            comp.external_references = [ExternalReference(url=XsUri(oss_item.download_location),
+                                                                          type=ExternalReferenceType.WEBSITE)]
+
+                        oss_licenses = []
+                        for ol in oss_item.license:
+                            try:
+                                oss_licenses.append(lc_factory.make_from_string(ol))
+                            except Exception:
+                                logger.info(f'No spdx license name: {ol}')
+                        if oss_licenses:
+                            comp.licenses = oss_licenses
+
+                        root_package = False
+                        if scanner_name == FOSSLIGHT_DEPENDENCY:
+                            if oss_item.comment:
+                                oss_comment = oss_item.comment.split('/')
+                                for oc in oss_comment:
+                                    if oc in ['direct', 'transitive', 'root package']:
+                                        if oc == 'direct':
+                                            bom.register_dependency(root_component, [comp])
+                                        elif oc == 'root package':
+                                            root_package = True
+                                            root_component.name = comp_name
+                                            root_component.type = comp_type
+                                            comp_id -= 1
+                            else:
+                                bom.register_dependency(root_component, [comp])
+                            if len(file_item.depends_on) > 0:
+                                purl = file_item.purl
+                                relation_tree[purl] = []
+                                relation_tree[purl].extend(file_item.depends_on)
+
+                        if not root_package:
+                            bom.components.add(comp)
+
+            if len(bom.components) > 0:
+                for comp_purl in relation_tree:
+                    comp = bom.get_component_by_purl(PackageURL.from_string(comp_purl))
+                    if comp:
+                        dep_comp_list = []
+                        for dep_comp_purl in relation_tree[comp_purl]:
+                            dep_comp = bom.get_component_by_purl(PackageURL.from_string(dep_comp_purl))
+                            if dep_comp:
+                                dep_comp_list.append(dep_comp)
+                        bom.register_dependency(comp, dep_comp_list)
+
+        except Exception as e:
+            success = False
+            error_msg = f'Failed to create CycloneDX document object:{e}, {traceback.format_exc()}'
+    else:
+        success = False
+        error_msg = 'No item to write in output file.'
+
+    result_file = ''
+    if success:
+        result_file = output_file_without_ext + output_extension
+        try:
+            if output_extension == '.json':
+                write_cyclonedx_json(bom, result_file)
+            elif output_extension == '.xml':
+                write_cyclonedx_xml(bom, result_file)
+            else:
+                success = False
+                error_msg = f'Not supported output_extension({output_extension})'
+        except Exception as e:
+            success = False
+            error_msg = f'Failed to write CycloneDX document: {e}'
+            if os.path.exists(result_file):
+                os.remove(result_file)
+
+    return success, error_msg, result_file
+
+
+def write_cyclonedx_json(bom, result_file):
+    success = True
+    try:
+        my_json_outputter: 'JsonOutputter' = JsonV1Dot6(bom)
+        my_json_outputter.output_to_file(result_file)
+        serialized_json = my_json_outputter.output_as_string(indent=2)
+        my_json_validator = JsonStrictValidator(SchemaVersion.V1_6)
+        try:
+            validation_errors = my_json_validator.validate_str(serialized_json)
+            if validation_errors:
+                logger.warning(f'JSON invalid, ValidationError: {repr(validation_errors)}')
+        except MissingOptionalDependencyException as error:
+            logger.debug(f'JSON-validation was skipped due to {error}')
+    except Exception as e:
+        logger.warning(f'Fail to write cyclonedx json: {e}')
+        success = False
+    return success
+
+
+def write_cyclonedx_xml(bom, result_file):
+    success = True
+    try:
+        my_xml_outputter: BaseOutput = make_outputter(bom=bom,
+                                                      output_format=OutputFormat.XML,
+                                                      schema_version=SchemaVersion.V1_6)
+        my_xml_outputter.output_to_file(filename=result_file)
+        serialized_xml = my_xml_outputter.output_as_string(indent=2)
+        my_xml_validator = XmlValidator(SchemaVersion.V1_6)
+        try:
+            validation_errors = my_xml_validator.validate_str(serialized_xml)
+            if validation_errors:
+                logger.warning(f'XML invalid, ValidationError: {repr(validation_errors)}')
+        except MissingOptionalDependencyException as error:
+            logger.debug(f'XML-validation was skipped due to {error}')
+    except Exception as e:
+        logger.warning(f'Fail to write cyclonedx xml: {e}')
+        success = False
+    return success