fosslight-util 1.4.47__py3-none-any.whl → 2.0.0__py3-none-any.whl

fosslight_util/oss_item.py

@@ -5,46 +5,41 @@
 
 import logging
 import os
-from fosslight_util.constant import LOGGER_NAME, FL_DEPENDENCY, FL_BINARY
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SCANNER
+from fosslight_util.cover import CoverItem
+from typing import List, Dict
 
 _logger = logging.getLogger(LOGGER_NAME)
 
 
 class OssItem:
-    def __init__(self, value):
-        self._name = ""
-        self._version = ""
+
+    def __init__(self, name="", version="", license="", dl_url=""):
+        self.name = name
+        self.version = version
         self._license = []
-        self._copyright = ""
+        self.license = license
+        self.download_location = dl_url
+        self.exclude = False
         self.comment = ""
-        self._exclude = False
         self.homepage = ""
-        self.relative_path = value
-        self._source_name_or_path = []
-        self.download_location = ""
-        self._yocto_recipe = []
-        self._yocto_package = []
-        self.is_binary = False
-        self._depends_on = []
-        self.purl = ""
-        self.bin_vulnerability = ""
-        self.bin_tlsh = ""
-        self.bin_sha1 = ""
+        self._copyright = ""
 
     def __del__(self):
         pass
 
     @property
-    def copyright(self):
-        return self._copyright
+    def license(self):
+        return self._license
 
-    @copyright.setter
-    def copyright(self, value):
+    @license.setter
+    def license(self, value):
         if value != "":
-            if isinstance(value, list):
-                value = "\n".join(value)
-            value = value.strip()
-        self._copyright = value
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._license.extend(value)
+        self._license = [item.strip() for item in self._license]
+        self._license = list(set(self._license))
 
     @property
     def exclude(self):
@@ -58,13 +53,16 @@ class OssItem:
             self._exclude = False
 
     @property
-    def name(self):
-        return self._name
+    def copyright(self):
+        return self._copyright
 
-    @name.setter
-    def name(self, value):
+    @copyright.setter
+    def copyright(self, value):
         if value != "":
-            self._name = value
+            if isinstance(value, list):
+                value = "\n".join(value)
+            value = value.strip()
+        self._copyright = value
 
     @property
     def version(self):
@@ -78,149 +76,144 @@ class OssItem:
             self._version = ""
 
     @property
-    def license(self):
-        return self._license
-
-    @license.setter
-    def license(self, value):
-        if not isinstance(value, list):
-            value = value.split(",")
-        self._license.extend(value)
-        self._license = [item.strip() for item in self._license]
-        self._license = list(set(self._license))
-
-    @property
-    def source_name_or_path(self):
-        return self._source_name_or_path
+    def comment(self):
+        return self._comment
 
-    @source_name_or_path.setter
-    def source_name_or_path(self, value):
+    @comment.setter
+    def comment(self, value):
         if not value:
-            self._source_name_or_path = []
+            self._comment = ""
         else:
-            if not isinstance(value, list):
-                value = value.split(",")
-            self._source_name_or_path.extend(value)
-            self._source_name_or_path = [item.strip() for item in self._source_name_or_path]
-            self._source_name_or_path = list(set(self._source_name_or_path))
+            if self._comment:
+                self._comment = f"{self._comment} / {value}"
+            else:
+                self._comment = value
 
-    @property
-    def yocto_recipe(self):
-        return self._yocto_recipe
 
-    @yocto_recipe.setter
-    def yocto_recipe(self, value):
-        if not isinstance(value, list):
-            value = value.split(",")
-        self._yocto_recipe.extend(value)
-        self._yocto_recipe = [item.strip() for item in self._yocto_recipe]
-        self._yocto_recipe = list(set(self._yocto_recipe))
+class FileItem:
+    def __init__(self, value):
+        self.relative_path = value
+        self.source_name_or_path = ""
+        self._exclude = False
+        self._comment = ""
+        self.is_binary = False
+        self.oss_items: List[OssItem] = []
+
+    def __del__(self):
+        pass
 
     @property
-    def yocto_package(self):
-        return self._yocto_package
+    def exclude(self):
+        return self._exclude
 
-    @yocto_package.setter
-    def yocto_package(self, value):
-        if not isinstance(value, list):
-            value = value.split(",")
-        self._yocto_package.extend(value)
-        self._yocto_package = [item.strip() for item in self._yocto_package]
-        self._yocto_package = list(set(self._yocto_package))
+    @exclude.setter
+    def exclude(self, value):
+        if value:
+            self._exclude = True
+        else:
+            self._exclude = False
+        for oss in self.oss_items:
+            oss.exclude = value
 
     @property
-    def depends_on(self):
-        return self._depends_on
+    def comment(self):
+        return self._comment
 
-    @depends_on.setter
-    def depends_on(self, value):
+    @comment.setter
+    def comment(self, value):
         if not value:
-            self._depends_on = []
-        else:
-            if not isinstance(value, list):
-                value = value.split(",")
-            self._depends_on.extend(value)
-            self._depends_on = [item.strip() for item in self._depends_on]
-            self._depends_on = list(set(self._depends_on))
-
-    def set_sheet_item(self, item, scanner_name=''):
-        if len(item) < 9:
-            _logger.warning(f"sheet list is too short ({len(item)}): {item}")
-            return
-        if scanner_name == FL_DEPENDENCY:
-            self.purl = item[0]
+            self._comment = ""
         else:
-            self.source_name_or_path = item[0]
-        self.name = item[1]
-        self.version = item[2]
-        self.license = item[3]
-        self.download_location = item[4]
-        self.homepage = item[5]
-        self.copyright = item[6]
-        self.exclude = item[7]
-        self.comment = item[8]
-
-        if len(item) >= 10 and scanner_name == FL_DEPENDENCY:
-            self.depends_on = item[9]
-        if len(item) >= 10 and scanner_name == FL_BINARY:
-            self.bin_vulnerability = item[9]
-            if len(item) >= 12:
-                self.bin_tlsh = item[10]
-                self.bin_sha1 = item[11]
-
-    def get_print_array(self, scanner_name=''):
+            if self._comment:
+                self._comment = f"{self._comment} / {value}"
+            else:
+                self._comment = value
+            for oss in self.oss_items:
+                oss.comment = value
+
+    def get_print_array(self):
         items = []
-        if scanner_name != FL_DEPENDENCY:
-            if len(self.source_name_or_path) == 0:
-                self.source_name_or_path.append("")
-        if len(self.license) == 0:
-            self.license.append("")
-
-        exclude = "Exclude" if self.exclude else ""
-        lic = ",".join(self.license)
-        if scanner_name == FL_DEPENDENCY:
-            items = [self.purl, self.name, self.version, lic,
-                     self.download_location, self.homepage, self.copyright, exclude, self.comment]
-            if len(self.depends_on) > 0:
-                items.append(",".join(self.depends_on))
-        else:
-            for source_name_or_path in self.source_name_or_path:
-                if scanner_name == FL_BINARY:
-                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
-                                self.download_location, self.homepage, self.copyright, exclude, self.comment,
-                                self.bin_vulnerability, self.bin_tlsh, self.bin_sha1]
-                else:
-                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
-                                self.download_location, self.homepage, self.copyright, exclude, self.comment]
-                items.append(oss_item)
+
+        for oss in self.oss_items:
+            exclude = "Exclude" if self.exclude or oss.exclude else ""
+            lic = ",".join(oss.license)
+
+            oss_item = [os.path.join(self.relative_path, self.source_name_or_path), oss.name, oss.version, lic,
+                        oss.download_location, oss.homepage, oss.copyright, exclude, oss.comment]
+            items.append(oss_item)
         return items
 
     def get_print_json(self):
-        json_item = {}
-        json_item["name"] = self.name
-
-        json_item["version"] = self.version
-        if len(self.source_name_or_path) > 0:
-            json_item["source path"] = self.source_name_or_path
-        if len(self.license) > 0:
-            json_item["license"] = self.license
-        if self.download_location != "":
-            json_item["download location"] = self.download_location
-        if self.homepage != "":
-            json_item["homepage"] = self.homepage
-        if self.copyright != "":
-            json_item["copyright text"] = self.copyright
-        if self.exclude:
-            json_item["exclude"] = self.exclude
-        if self.comment != "":
-            json_item["comment"] = self.comment
-        if len(self.depends_on) > 0:
-            json_item["depends on"] = self.depends_on
-        if self.purl != "":
-            json_item["package url"] = self.purl
-
-        return json_item
+        items = []
+
+        for oss in self.oss_items:
+            json_item = {}
+            json_item["name"] = oss.name
+            json_item["version"] = oss.version
+
+            if self.source_name_or_path != "":
+                json_item["source path"] = self.source_name_or_path
+            if len(oss.license) > 0:
+                json_item["license"] = oss.license
+            if oss.download_location != "":
+                json_item["download location"] = oss.download_location
+            if oss.homepage != "":
+                json_item["homepage"] = oss.homepage
+            if oss.copyright != "":
+                json_item["copyright text"] = oss.copyright
+            if self.exclude or oss.exclude:
+                json_item["exclude"] = True
+            if oss.comment != "":
+                json_item["comment"] = oss.comment
+            items.append(json_item)
+        return items
 
 
 def invalid(cmd):
     _logger.info('[{}] is invalid'.format(cmd))
+
+
+class ScannerItem:
+    def __init__(self, pkg_name, start_time=""):
+        self.cover = CoverItem(tool_name=pkg_name, start_time=start_time)
+        self.file_items: Dict[str, List[FileItem]] = {pkg_name: []} if pkg_name != FOSSLIGHT_SCANNER else {}
+        self.external_sheets: Dict[str, List[List[str]]] = {}
+
+    def set_cover_pathinfo(self, input_dir, path_to_exclude):
+        self.cover.input_path = input_dir
+        self.cover.exclude_path = ", ".join(path_to_exclude)
+
+    def set_cover_comment(self, value):
+        if value:
+            if self.cover.comment:
+                self.cover.comment = f"{self.cover.comment} / {value}"
+            else:
+                self.cover.comment = value
+
+    def get_cover_comment(self):
+        return [item.strip() for item in self.cover.comment.split(" / ")]
+
+    def append_file_items(self, file_item: List[FileItem], pkg_name=""):
+        if pkg_name == "":
+            if len(self.file_items.keys()) != 1:
+                _logger.error("Package name is not set. Cannot append file_item into ScannerItem.")
+            else:
+                pkg_name = list(self.file_items.keys())[0]
+        if pkg_name not in self.file_items:
+            self.file_items[pkg_name] = []
+        self.file_items[pkg_name].extend(file_item)
+
+    def get_print_array(self, scanner_name):
+        items = []
+        for file_item in self.file_items[scanner_name]:
+            items.extend(file_item.get_print_array())
+        return items
+
+    def get_print_json(self, scanner_name):
+        items = []
+        for file_item in self.file_items[scanner_name]:
+            items.extend(file_item.get_print_json())
+        return items
+
+    def __del__(self):
+        pass

fosslight_util/output_format.py

@@ -6,6 +6,7 @@ import os
 from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv
 from fosslight_util.write_opossum import write_opossum
 from fosslight_util.write_yaml import write_yaml
+from typing import Tuple
 
 SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'}
 
@@ -105,7 +106,8 @@ def check_output_formats(output='', formats=[], customized_format={}):
     return success, msg, output_path, output_files, output_extensions
 
 
-def write_output_file(output_file_without_ext, file_extension, sheet_list, extended_header={}, hide_header={}, cover=""):
+def write_output_file(output_file_without_ext: str, file_extension: str, scan_item, extended_header: dict = {},
+                      hide_header: dict = {}) -> Tuple[bool, str, str]:
     success = True
     msg = ''
 
@@ -114,13 +116,13 @@ def write_output_file(output_file_without_ext, file_extension, sheet_list, exten
     result_file = output_file_without_ext + file_extension
 
     if file_extension == '.xlsx':
-        success, msg = write_result_to_excel(result_file, sheet_list, extended_header, hide_header, cover)
+        success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
     elif file_extension == '.csv':
-        success, msg, result_file = write_result_to_csv(result_file, sheet_list)
+        success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
     elif file_extension == '.json':
-        success, msg = write_opossum(result_file, sheet_list)
+        success, msg = write_opossum(result_file, scan_item)
     elif file_extension == '.yaml':
-        success, msg, result_file = write_yaml(result_file, sheet_list, False)
+        success, msg, result_file = write_yaml(result_file, scan_item, False)
     else:
         success = False
         msg = f'Not supported file extension({file_extension})'

fosslight_util/parsing_yaml.py

@@ -8,8 +8,8 @@ import codecs
 import os
 import re
 import sys
-from .constant import LOGGER_NAME
-from .oss_item import OssItem
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import OssItem, FileItem
 
 _logger = logging.getLogger(LOGGER_NAME)
 SUPPORT_OSS_INFO_FILES = [r"oss-pkg-info[\s\S]*.ya?ml", r"sbom(-|_)info[\s\S]*.ya?ml"]
@@ -17,7 +17,7 @@ EXAMPLE_OSS_PKG_INFO_LINK = "https://github.com/fosslight/fosslight_prechecker/b
 
 
 def parsing_yml(yaml_file, base_path, print_log=True):
-    oss_list = []
+    fileitems = []
     license_list = []
     idx = 1
     err_reason = ""
@@ -38,37 +38,65 @@ def parsing_yml(yaml_file, base_path, print_log=True):
             err_reason = "empty"
             if print_log:
                 _logger.warning(f"The yaml file is empty file: {yaml_file}")
-            return oss_list, license_list, err_reason
+            return fileitems, license_list, err_reason
 
         is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT)
 
+        filepath_list = []
         for root_element in doc:
             oss_items = doc[root_element]
             if oss_items:
                 if not isinstance(oss_items, list) or 'version' not in oss_items[0]:
                     raise AttributeError(f"- Ref. {EXAMPLE_OSS_PKG_INFO_LINK}")
                 for oss in oss_items:
-                    item = OssItem(relative_path)
-                    if not is_old_format:
-                        item.name = root_element
-                    for key, value in oss.items():
-                        if key:
-                            key = key.lower().strip()
-                        set_value_switch(item, key, value, yaml_file)
-                    oss_list.append(item)
-                    license_list.extend(item.license)
-                    idx += 1
+                    source_paths = get_source_name_or_path_in_yaml(oss)
+                    for source_path in source_paths:
+                        if os.path.join(relative_path, source_path) not in filepath_list:
+                            filepath_list.append(os.path.join(relative_path, source_path))
+                            fileitem = FileItem(relative_path)
+                            fileitem.source_name_or_path = source_path
+                            fileitems.append(fileitem)
+                        else:
+                            fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+                        ossitem = OssItem()
+                        if not is_old_format:
+                            ossitem.name = root_element
+                        for key, value in oss.items():
+                            if key:
+                                key = key.lower().strip()
+                            set_value_switch(ossitem, key, value, yaml_file)
+                        fileitem.oss_items.append(ossitem)
+                        license_list.extend(ossitem.license)
+                        idx += 1
     except AttributeError as ex:
         if print_log:
             _logger.warning(f"Not supported yaml file format: {yaml_file} {ex}")
-        oss_list = []
+        fileitems = []
         err_reason = "not_supported"
     except yaml.YAMLError:
         if print_log:
             _logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}")
-        oss_list = []
+        fileitems = []
         err_reason = "yaml_error"
-    return oss_list, set(license_list), err_reason
+
+    return fileitems, set(license_list), err_reason
+
+
+def get_source_name_or_path_in_yaml(oss):
+    source_name_or_path = []
+    find = False
+    for key in oss.keys():
+        if key in ['file name or path', 'source name or path', 'source path',
+                   'file', 'binary name', 'binary path']:
+            if isinstance(oss[key], list):
+                source_name_or_path = oss[key]
+            else:
+                source_name_or_path.append(oss[key])
+            find = True
+            break
+    if not find:
+        source_name_or_path.append('')
+    return source_name_or_path
 
 
 def find_sbom_yaml_files(path_to_find):
@@ -101,9 +129,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.download_location = value
     elif key in ['license', 'license text']:
         oss.license = value
-    elif key in ['file name or path', 'source name or path', 'source path',
-                 'file', 'binary name', 'binary path']:
-        oss.source_name_or_path = value
     elif key in ['copyright text', 'copyright']:
         oss.copyright = value
     elif key == 'exclude':
@@ -112,16 +137,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.comment = value
     elif key == 'homepage':
         oss.homepage = value
-    elif key == 'yocto_package':
-        oss.yocto_package = value
-    elif key == 'yocto_recipe':
-        oss.yocto_recipe = value
-    elif key == 'vulnerability link':
-        oss.bin_vulnerability = value
-    elif key == 'tlsh':
-        oss.bin_tlsh = value
-    elif key == 'sha1':
-        oss.bin_sha1 = value
     else:
         if yaml_file != "":
             _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")

fosslight_util/read_excel.py

@@ -4,27 +4,24 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 from typing import List, Dict, Any
-import xlrd
+import pandas as pd
 import json
 from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.oss_item import OssItem
+from fosslight_util.oss_item import OssItem, FileItem
 from fosslight_util.parsing_yaml import set_value_switch
 
 logger = logging.getLogger(LOGGER_NAME)
 IDX_CANNOT_FOUND = -1
 PREFIX_BIN = "bin"
 SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
-xlrd.xlsx.ensure_elementtree_imported(False, None)
-xlrd.xlsx.Element_has_iter = True
 
 
-def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
-    oss_report_items: List[OssItem] = []
+def read_oss_report(excel_file: str, sheet_names: str = "", basepath: str = "") -> List[FileItem]:
+    fileitems: List[FileItem] = []
     xl_sheets: Dict[str, Any] = {}
     all_sheet_to_read: List[str] = []
     not_matched_sheet: List[str] = []
     any_sheet_matched = False
-
     if sheet_names:
         sheet_name_prefix_match = False
         sheet_name_to_read = sheet_names.split(",")
@@ -34,9 +31,8 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
 
     try:
         logger.info(f"Read data from : {excel_file}")
-        xl_workbook = xlrd.open_workbook(excel_file)
-        all_sheet_in_excel = xl_workbook.sheet_names()
-
+        xl_workbook = pd.ExcelFile(excel_file, engine='openpyxl')
+        all_sheet_in_excel = xl_workbook.sheet_names
         for sheet_to_read in sheet_name_to_read:
             try:
                 any_sheet_matched = False
@@ -46,10 +42,9 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                     sheet_name_lower = sheet_name.lower()
                     if (sheet_name_prefix_match and sheet_name_lower.startswith(sheet_to_read_lower)) \
                        or sheet_to_read_lower == sheet_name_lower:
-                        sheet = xl_workbook.sheet_by_name(sheet_name)
-                        if sheet:
-                            xl_sheets[sheet_name] = sheet
-                            any_sheet_matched = True
+                        sheet = pd.read_excel(excel_file, sheet_name=sheet_name, engine='openpyxl', na_values='')
+                        xl_sheets[sheet_name] = sheet.fillna('')
+                        any_sheet_matched = True
                 if not any_sheet_matched:
                     not_matched_sheet.append(sheet_to_read)
             except Exception as error:
@@ -62,6 +57,7 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
         elif (not sheet_name_prefix_match) and not_matched_sheet:
             logger.warning(f"Not matched sheet name: {not_matched_sheet}")
 
+        filepath_list = []
         for sheet_name, xl_sheet in xl_sheets.items():
             _item_idx = {
                 "ID": IDX_CANNOT_FOUND,
@@ -82,46 +78,44 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                 "TLSH": IDX_CANNOT_FOUND,
                 "SHA1": IDX_CANNOT_FOUND
             }
-            num_cols = xl_sheet.ncols
-            num_rows = xl_sheet.nrows
-            MAX_FIND_HEADER_COLUMN = 5 if num_rows > 5 else num_rows
-            DATA_START_ROW_IDX = 1
-            for row_idx in range(0, MAX_FIND_HEADER_COLUMN):
-                for col_idx in range(row_idx, num_cols):
-                    cell_obj = xl_sheet.cell(row_idx, col_idx)
-                    if cell_obj.value in _item_idx:
-                        _item_idx[cell_obj.value] = col_idx
 
-                if len([key for key, value in _item_idx.items() if value != IDX_CANNOT_FOUND]) > 3:
-                    DATA_START_ROW_IDX = row_idx + 1
-                    break
+            for index, value in enumerate(xl_sheet.columns.tolist()):
+                _item_idx[value] = index
 
             # Get all values, iterating through rows and columns
             column_keys = json.loads(json.dumps(_item_idx))
 
             is_bin = True if sheet_name.lower().startswith(PREFIX_BIN) else False
 
-            for row_idx in range(DATA_START_ROW_IDX, xl_sheet.nrows):
-                item = OssItem("")
-                item.is_binary = is_bin
+            for row_idx, row in xl_sheet.iterrows():
                 valid_row = True
                 load_data_cnt = 0
-
+                source_path = row[1]
+                if source_path not in filepath_list:
+                    filepath_list.append(source_path)
+                    fileitem = FileItem(basepath)
+                    fileitem.source_name_or_path = source_path
+                    fileitems.append(fileitem)
+                else:
+                    fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+                fileitem.is_binary = is_bin
+                ossitem = OssItem()
                 for column_key, column_idx in column_keys.items():
                     if column_idx != IDX_CANNOT_FOUND:
-                        cell_obj = xl_sheet.cell(row_idx, column_idx)
-                        cell_value = cell_obj.value
+                        cell_obj = xl_sheet.iloc[row_idx, column_idx]
+                        cell_value = cell_obj
+
                         if cell_value != "":
                             if column_key != "ID":
                                 if column_key:
                                     column_key = column_key.lower().strip()
-                                set_value_switch(item, column_key, cell_value)
+                                set_value_switch(ossitem, column_key, cell_value)
                                 load_data_cnt += 1
                             else:
                                 valid_row = False if cell_value == "-" else True
                 if valid_row and load_data_cnt > 0:
-                    oss_report_items.append(item)
+                    fileitem.oss_items.append(ossitem)
 
     except Exception as error:
         logger.error(f"Parsing a OSS Report: {error}")
-    return oss_report_items
+    return fileitems