fosslight-util 1.4.48__py3-none-any.whl → 2.0.1__py3-none-any.whl

fosslight_util/output_format.py

@@ -3,11 +3,16 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import os
+import platform
 from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv
 from fosslight_util.write_opossum import write_opossum
 from fosslight_util.write_yaml import write_yaml
+from fosslight_util.write_spdx import write_spdx
+from typing import Tuple
 
-SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'}
+SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml',
+                  'spdx-yaml': '.yaml', 'spdx-json': '.json', 'spdx-xml': '.xml',
+                  'spdx-tag': '.tag'}
 
 
 def check_output_format(output='', format='', customized_format={}):
@@ -105,7 +110,62 @@ def check_output_formats(output='', formats=[], customized_format={}):
     return success, msg, output_path, output_files, output_extensions
 
 
-def write_output_file(output_file_without_ext, file_extension, sheet_list, extended_header={}, hide_header={}, cover=""):
+def check_output_formats_v2(output='', formats=[], customized_format={}):
+    success = True
+    msg = ''
+    output_path = ''
+    output_files = []
+    output_extensions = []
+
+    if customized_format:
+        support_format = customized_format
+    else:
+        support_format = SUPPORT_FORMAT
+
+    if formats:
+        # If -f option exist
+        formats = [format.lower() for format in formats]
+        for format in formats:
+            if format not in list(support_format.keys()):
+                success = False
+                msg = 'Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+            else:
+                output_extensions.append(support_format[format])
+
+    if success:
+        if output != '':
+            basename_extension = ''
+            if not os.path.isdir(output):
+                output_path = os.path.dirname(output)
+
+                basename = os.path.basename(output)
+                basename_file, basename_extension = os.path.splitext(basename)
+            if basename_extension:
+                if formats:
+                    if basename_extension not in output_extensions:
+                        success = False
+                        msg = f"The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
+                else:
+                    if basename_extension not in support_format.values():
+                        success = False
+                        msg = 'Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                    output_extensions.append(basename_extension)
+                output_files = [basename_file for _ in range(len(output_extensions))]
+            else:
+                output_path = output
+    if not output_extensions:
+        output_extensions = ['.xlsx']
+    if not formats:
+        for ext in output_extensions:
+            for key, value in support_format.items():
+                if value == ext:
+                    formats.append(key)
+                    break
+    return success, msg, output_path, output_files, output_extensions, formats
+
+
+def write_output_file(output_file_without_ext: str, file_extension: str, scan_item, extended_header: dict = {},
+                      hide_header: dict = {}, format: str = '', spdx_version: str = '2.3') -> Tuple[bool, str, str]:
     success = True
     msg = ''
 
@@ -113,16 +173,32 @@ def write_output_file(output_file_without_ext, file_extension, sheet_list, exten
         file_extension = '.xlsx'
     result_file = output_file_without_ext + file_extension
 
-    if file_extension == '.xlsx':
-        success, msg = write_result_to_excel(result_file, sheet_list, extended_header, hide_header, cover)
-    elif file_extension == '.csv':
-        success, msg, result_file = write_result_to_csv(result_file, sheet_list, False, extended_header)
-    elif file_extension == '.json':
-        success, msg = write_opossum(result_file, sheet_list)
-    elif file_extension == '.yaml':
-        success, msg, result_file = write_yaml(result_file, sheet_list, False)
+    if format:
+        if format == 'excel':
+            success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
+        elif format == 'csv':
+            success, msg, _ = write_result_to_csv(result_file, scan_item, False, extended_header)
+        elif format == 'opossum':
+            success, msg = write_opossum(result_file, scan_item)
+        elif format == 'yaml':
+            success, msg, _ = write_yaml(result_file, scan_item, False)
+        elif format.startswith('spdx'):
+            if platform.system() != 'Windows':
+                success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version)
+            else:
+                success = False
+                msg = 'Windows not support spdx format.'
     else:
-        success = False
-        msg = f'Not supported file extension({file_extension})'
+        if file_extension == '.xlsx':
+            success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
+        elif file_extension == '.csv':
+            success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
+        elif file_extension == '.json':
+            success, msg = write_opossum(result_file, scan_item)
+        elif file_extension == '.yaml':
+            success, msg, result_file = write_yaml(result_file, scan_item, False)
+        else:
+            success = False
+            msg = f'Not supported file extension({file_extension})'
 
     return success, msg, result_file

fosslight_util/parsing_yaml.py

@@ -8,8 +8,8 @@ import codecs
 import os
 import re
 import sys
-from .constant import LOGGER_NAME
-from .oss_item import OssItem
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import OssItem, FileItem
 
 _logger = logging.getLogger(LOGGER_NAME)
 SUPPORT_OSS_INFO_FILES = [r"oss-pkg-info[\s\S]*.ya?ml", r"sbom(-|_)info[\s\S]*.ya?ml"]
@@ -17,7 +17,7 @@ EXAMPLE_OSS_PKG_INFO_LINK = "https://github.com/fosslight/fosslight_prechecker/b
 
 
 def parsing_yml(yaml_file, base_path, print_log=True):
-    oss_list = []
+    fileitems = []
     license_list = []
     idx = 1
     err_reason = ""
@@ -38,37 +38,65 @@ def parsing_yml(yaml_file, base_path, print_log=True):
             err_reason = "empty"
             if print_log:
                 _logger.warning(f"The yaml file is empty file: {yaml_file}")
-            return oss_list, license_list, err_reason
+            return fileitems, license_list, err_reason
 
         is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT)
 
+        filepath_list = []
         for root_element in doc:
             oss_items = doc[root_element]
             if oss_items:
                 if not isinstance(oss_items, list) or 'version' not in oss_items[0]:
                     raise AttributeError(f"- Ref. {EXAMPLE_OSS_PKG_INFO_LINK}")
                 for oss in oss_items:
-                    item = OssItem(relative_path)
-                    if not is_old_format:
-                        item.name = root_element
-                    for key, value in oss.items():
-                        if key:
-                            key = key.lower().strip()
-                        set_value_switch(item, key, value, yaml_file)
-                    oss_list.append(item)
-                    license_list.extend(item.license)
-                    idx += 1
+                    source_paths = get_source_name_or_path_in_yaml(oss)
+                    for source_path in source_paths:
+                        if os.path.join(relative_path, source_path) not in filepath_list:
+                            filepath_list.append(os.path.join(relative_path, source_path))
+                            fileitem = FileItem(relative_path)
+                            fileitem.source_name_or_path = source_path
+                            fileitems.append(fileitem)
+                        else:
+                            fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+                        ossitem = OssItem()
+                        if not is_old_format:
+                            ossitem.name = root_element
+                        for key, value in oss.items():
+                            if key:
+                                key = key.lower().strip()
+                            set_value_switch(ossitem, key, value, yaml_file)
+                        fileitem.oss_items.append(ossitem)
+                        license_list.extend(ossitem.license)
+                        idx += 1
     except AttributeError as ex:
         if print_log:
             _logger.warning(f"Not supported yaml file format: {yaml_file} {ex}")
-        oss_list = []
+        fileitems = []
         err_reason = "not_supported"
     except yaml.YAMLError:
         if print_log:
             _logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}")
-        oss_list = []
+        fileitems = []
         err_reason = "yaml_error"
-    return oss_list, set(license_list), err_reason
+
+    return fileitems, set(license_list), err_reason
+
+
+def get_source_name_or_path_in_yaml(oss):
+    source_name_or_path = []
+    find = False
+    for key in oss.keys():
+        if key in ['file name or path', 'source name or path', 'source path',
+                   'file', 'binary name', 'binary path']:
+            if isinstance(oss[key], list):
+                source_name_or_path = oss[key]
+            else:
+                source_name_or_path.append(oss[key])
+            find = True
+            break
+    if not find:
+        source_name_or_path.append('')
+    return source_name_or_path
 
 
 def find_sbom_yaml_files(path_to_find):
@@ -101,9 +129,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.download_location = value
     elif key in ['license', 'license text']:
         oss.license = value
-    elif key in ['file name or path', 'source name or path', 'source path',
-                 'file', 'binary name', 'binary path']:
-        oss.source_name_or_path = value
     elif key in ['copyright text', 'copyright']:
         oss.copyright = value
     elif key == 'exclude':
@@ -112,16 +137,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.comment = value
     elif key == 'homepage':
         oss.homepage = value
-    elif key == 'yocto_package':
-        oss.yocto_package = value
-    elif key == 'yocto_recipe':
-        oss.yocto_recipe = value
-    elif key == 'vulnerability link':
-        oss.bin_vulnerability = value
-    elif key == 'tlsh':
-        oss.bin_tlsh = value
-    elif key == 'sha1':
-        oss.bin_sha1 = value
     else:
         if yaml_file != "":
             _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")

fosslight_util/read_excel.py

@@ -4,27 +4,24 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 from typing import List, Dict, Any
-import xlrd
+import pandas as pd
 import json
 from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.oss_item import OssItem
+from fosslight_util.oss_item import OssItem, FileItem
 from fosslight_util.parsing_yaml import set_value_switch
 
 logger = logging.getLogger(LOGGER_NAME)
 IDX_CANNOT_FOUND = -1
 PREFIX_BIN = "bin"
 SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
-xlrd.xlsx.ensure_elementtree_imported(False, None)
-xlrd.xlsx.Element_has_iter = True
 
 
-def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
-    oss_report_items: List[OssItem] = []
+def read_oss_report(excel_file: str, sheet_names: str = "", basepath: str = "") -> List[FileItem]:
+    fileitems: List[FileItem] = []
     xl_sheets: Dict[str, Any] = {}
     all_sheet_to_read: List[str] = []
     not_matched_sheet: List[str] = []
     any_sheet_matched = False
-
     if sheet_names:
         sheet_name_prefix_match = False
         sheet_name_to_read = sheet_names.split(",")
@@ -34,9 +31,8 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
 
     try:
         logger.info(f"Read data from : {excel_file}")
-        xl_workbook = xlrd.open_workbook(excel_file)
-        all_sheet_in_excel = xl_workbook.sheet_names()
-
+        xl_workbook = pd.ExcelFile(excel_file, engine='openpyxl')
+        all_sheet_in_excel = xl_workbook.sheet_names
         for sheet_to_read in sheet_name_to_read:
             try:
                 any_sheet_matched = False
@@ -46,10 +42,9 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                     sheet_name_lower = sheet_name.lower()
                     if (sheet_name_prefix_match and sheet_name_lower.startswith(sheet_to_read_lower)) \
                        or sheet_to_read_lower == sheet_name_lower:
-                        sheet = xl_workbook.sheet_by_name(sheet_name)
-                        if sheet:
-                            xl_sheets[sheet_name] = sheet
-                            any_sheet_matched = True
+                        sheet = pd.read_excel(excel_file, sheet_name=sheet_name, engine='openpyxl', na_values='')
+                        xl_sheets[sheet_name] = sheet.fillna('')
+                        any_sheet_matched = True
                 if not any_sheet_matched:
                     not_matched_sheet.append(sheet_to_read)
             except Exception as error:
@@ -62,6 +57,7 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
         elif (not sheet_name_prefix_match) and not_matched_sheet:
             logger.warning(f"Not matched sheet name: {not_matched_sheet}")
 
+        filepath_list = []
         for sheet_name, xl_sheet in xl_sheets.items():
             _item_idx = {
                 "ID": IDX_CANNOT_FOUND,
@@ -82,46 +78,44 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                 "TLSH": IDX_CANNOT_FOUND,
                 "SHA1": IDX_CANNOT_FOUND
             }
-            num_cols = xl_sheet.ncols
-            num_rows = xl_sheet.nrows
-            MAX_FIND_HEADER_COLUMN = 5 if num_rows > 5 else num_rows
-            DATA_START_ROW_IDX = 1
-            for row_idx in range(0, MAX_FIND_HEADER_COLUMN):
-                for col_idx in range(row_idx, num_cols):
-                    cell_obj = xl_sheet.cell(row_idx, col_idx)
-                    if cell_obj.value in _item_idx:
-                        _item_idx[cell_obj.value] = col_idx
 
-                if len([key for key, value in _item_idx.items() if value != IDX_CANNOT_FOUND]) > 3:
-                    DATA_START_ROW_IDX = row_idx + 1
-                    break
+            for index, value in enumerate(xl_sheet.columns.tolist()):
+                _item_idx[value] = index
 
             # Get all values, iterating through rows and columns
             column_keys = json.loads(json.dumps(_item_idx))
 
             is_bin = True if sheet_name.lower().startswith(PREFIX_BIN) else False
 
-            for row_idx in range(DATA_START_ROW_IDX, xl_sheet.nrows):
-                item = OssItem("")
-                item.is_binary = is_bin
+            for row_idx, row in xl_sheet.iterrows():
                 valid_row = True
                 load_data_cnt = 0
-
+                source_path = row[1]
+                if source_path not in filepath_list:
+                    filepath_list.append(source_path)
+                    fileitem = FileItem(basepath)
+                    fileitem.source_name_or_path = source_path
+                    fileitems.append(fileitem)
+                else:
+                    fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+                fileitem.is_binary = is_bin
+                ossitem = OssItem()
                 for column_key, column_idx in column_keys.items():
                     if column_idx != IDX_CANNOT_FOUND:
-                        cell_obj = xl_sheet.cell(row_idx, column_idx)
-                        cell_value = cell_obj.value
+                        cell_obj = xl_sheet.iloc[row_idx, column_idx]
+                        cell_value = cell_obj
+
                         if cell_value != "":
                             if column_key != "ID":
                                 if column_key:
                                     column_key = column_key.lower().strip()
-                                set_value_switch(item, column_key, cell_value)
+                                set_value_switch(ossitem, column_key, cell_value)
                                 load_data_cnt += 1
                             else:
                                 valid_row = False if cell_value == "-" else True
                 if valid_row and load_data_cnt > 0:
-                    oss_report_items.append(item)
+                    fileitem.oss_items.append(ossitem)
 
     except Exception as error:
         logger.error(f"Parsing a OSS Report: {error}")
-    return oss_report_items
+    return fileitems

fosslight_util/set_log.py

@@ -12,6 +12,8 @@ import platform
 from . import constant as constant
 from lastversion import lastversion
 import coloredlogs
+from typing import Tuple
+from logging import Logger
 
 
 def init_check_latest_version(pkg_version="", main_package_name=""):
@@ -32,6 +34,21 @@ def init_check_latest_version(pkg_version="", main_package_name=""):
         logger.debug('Cannot check the latest version:' + str(error))
 
 
+def get_os_version():
+
+    logger = logging.getLogger(constant.LOGGER_NAME)
+
+    os_version = platform.system() + " " + platform.release()
+    if os_version == "Windows 10":
+        try:
+            windows_build = sys.getwindowsversion().build
+            if windows_build >= 22000:
+                os_version = "Windows 11"
+        except Exception as error:
+            logger.debug(str(error))
+    return os_version
+
+
 class CustomAdapter(logging.LoggerAdapter):
     def __init__(self, logger, extra):
         super(CustomAdapter, self).__init__(logger, {})
@@ -41,8 +58,8 @@ class CustomAdapter(logging.LoggerAdapter):
         return '[%s] %s' % (self.extra, msg), kwargs
 
 
-def init_log(log_file, create_file=True, stream_log_level=logging.INFO,
-             file_log_level=logging.DEBUG, main_package_name="", path_to_analyze="", path_to_exclude=[]):
+def init_log(log_file: str, create_file: bool = True, stream_log_level: int = logging.INFO, file_log_level: int = logging.DEBUG,
+             main_package_name: str = "", path_to_analyze: str = "", path_to_exclude: list = []) -> Tuple[Logger, dict]:
 
     logger = logging.getLogger(constant.LOGGER_NAME)
 
@@ -70,7 +87,7 @@ def init_log(log_file, create_file=True, stream_log_level=logging.INFO,
     _result_log = {
         "Tool Info": main_package_name,
         "Python version": _PYTHON_VERSION,
-        "OS": platform.system()+" "+platform.release(),
+        "OS": get_os_version(),
     }
     if main_package_name != "":
         pkg_info = main_package_name

fosslight_util/spdx_licenses.py

@@ -8,6 +8,7 @@ import os
 import sys
 import json
 import traceback
+from typing import Tuple
 
 _resources_dir = 'resources'
 _licenses_json_file = 'licenses.json'
@@ -34,7 +35,7 @@ def get_license_from_nick():
     return licenses
 
 
-def get_spdx_licenses_json():
+def get_spdx_licenses_json() -> Tuple[bool, str, str]:
     success = True
     error_msg = ''
     licenses = ''