fosslight-util 1.4.39__tar.gz → 1.4.41__tar.gz

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_util
-Version: 1.4.39
+Version: 1.4.41
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='1.4.39',
+        version='1.4.41',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/correct.py

@@ -48,12 +48,13 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
         if sheet_name not in constant.supported_sheet_and_scanner.keys():
             continue
         correct_contents = copy.deepcopy(sheet_contents)
+        scanner_name = constant.supported_sheet_and_scanner[sheet_name]
         for idx, oss_raw_item in enumerate(sheet_contents):
             if len(oss_raw_item) < 9:
                 logger.warning(f"sheet list is too short ({len(oss_raw_item)}): {oss_raw_item}")
                 continue
             oss_item = OssItem('')
-            oss_item.set_sheet_item(oss_raw_item)
+            oss_item.set_sheet_item(oss_raw_item, scanner_name)
 
             matched_yi = []
             oss_rel_path = os.path.normpath(os.path.join(rel_path, oss_item.source_name_or_path[0]))
@@ -75,13 +76,13 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
                     if matched_oss_item.comment:
                         matched_oss_item.comment += '/'
                     matched_oss_item.comment += 'Loaded from sbom-info.yaml'
-                    matched_oss_array = matched_oss_item.get_print_array()[0]
+                    matched_oss_array = matched_oss_item.get_print_array(scanner_name)[0]
                     correct_contents.append(matched_oss_array)
                 oss_item.exclude = True
                 if oss_item.comment:
                     oss_item.comment += '/'
                 oss_item.comment += 'Excluded by sbom-info.yaml'
-                correct_contents[idx] = oss_item.get_print_array()[0]
+                correct_contents[idx] = oss_item.get_print_array(scanner_name)[0]
 
         if sheet_name == 'SRC_FL_Source':
             for n_idx, ni in enumerate(matched_yaml):

fosslight_util-1.4.41/src/fosslight_util/cover.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+from fosslight_util.help import print_package_version
+
+
+class CoverItem:
+    tool_name_key = "Tool name"
+    tool_version_key = "Tool version"
+    start_time_key = "Start time"
+    python_ver_key = "Python version"
+    analyzed_path_key = "Analyzed path"
+    comment_key = "Comment"
+
+    def __init__(self, tool_name="", start_time="", input_path="", comment=""):
+        self.tool_name = tool_name
+        if start_time:
+            date, time = start_time.split('_')
+            self.start_time = f'{date}, {time[0:2]}:{time[2:4]}'
+        else:
+            self.start_time = ""
+        self.input_path = os.path.abspath(input_path)
+        self.comment = comment
+
+        self.tool_version = print_package_version(self.tool_name, "", False)
+        self.python_version = f'{sys.version_info.major}.{sys.version_info.minor}'
+
+    def __del__(self):
+        pass
+
+    def get_print_json(self):
+        json_item = {}
+        json_item[self.tool_name_key] = self.tool_name
+        json_item[self.tool_version_key] = self.tool_version
+        json_item[self.start_time_key] = self.start_time
+        json_item[self.python_ver_key] = self.python_version
+        json_item[self.analyzed_path_key] = self.input_path
+        json_item[self.comment_key] = self.comment
+
+        return json_item

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/download.py

@@ -62,7 +62,7 @@ def change_src_link_to_https(src_link):
 
 
 def parse_src_link(src_link):
-    src_info = {}
+    src_info = {"url": src_link}
     src_link_changed = ""
     if src_link.startswith("git://") or src_link.startswith("https://") or src_link.startswith("http://"):
         src_link_split = src_link.split(';')
@@ -79,7 +79,7 @@ def parse_src_link(src_link):
         src_info["url"] = src_link_changed
         src_info["branch"] = branch_info
         src_info["tag"] = tag_info
-        return src_info
+    return src_info
 
 
 def main():

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/help.py

@@ -52,10 +52,12 @@ def print_package_version(pkg_name, msg="", exitopt=True):
     if msg == "":
         msg = f"{pkg_name} Version:"
     cur_version = pkg_resources.get_distribution(pkg_name).version
-    print(f'{msg} {cur_version}')
 
     if exitopt:
+        print(f'{msg} {cur_version}')
         sys.exit(0)
+    else:
+        return cur_version
 
 
 def print_help_msg_download(exitOpt=True):

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/oss_item.py

@@ -5,7 +5,7 @@
 
 import logging
 import os
-from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.constant import LOGGER_NAME, FL_DEPENDENCY, FL_BINARY
 
 _logger = logging.getLogger(LOGGER_NAME)
 
@@ -25,6 +25,11 @@ class OssItem:
         self._yocto_recipe = []
         self._yocto_package = []
         self.is_binary = False
+        self._depends_on = []
+        self.purl = ""
+        self.bin_vulnerability = ""
+        self.bin_tlsh = ""
+        self.bin_sha1 = ""
 
     def __del__(self):
         pass
@@ -123,11 +128,29 @@ class OssItem:
         self._yocto_package = [item.strip() for item in self._yocto_package]
         self._yocto_package = list(set(self._yocto_package))
 
-    def set_sheet_item(self, item):
+    @property
+    def depends_on(self):
+        return self._depends_on
+
+    @depends_on.setter
+    def depends_on(self, value):
+        if not value:
+            self._depends_on = []
+        else:
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._depends_on.extend(value)
+            self._depends_on = [item.strip() for item in self._depends_on]
+            self._depends_on = list(set(self._depends_on))
+
+    def set_sheet_item(self, item, scanner_name=''):
         if len(item) < 9:
             _logger.warning(f"sheet list is too short ({len(item)}): {item}")
             return
-        self.source_name_or_path = item[0]
+        if scanner_name == FL_DEPENDENCY:
+            self.purl = item[0]
+        else:
+            self.source_name_or_path = item[0]
         self.name = item[1]
         self.version = item[2]
         self.license = item[3]
@@ -137,19 +160,39 @@ class OssItem:
         self.exclude = item[7]
         self.comment = item[8]
 
-    def get_print_array(self):
+        if len(item) >= 10 and scanner_name == FL_DEPENDENCY:
+            self.depends_on = item[9]
+        if len(item) >= 10 and scanner_name == FL_BINARY:
+            self.bin_vulnerability = item[9]
+            if len(item) >= 12:
+                self.bin_tlsh = item[10]
+                self.bin_sha1 = item[11]
+
+    def get_print_array(self, scanner_name=''):
         items = []
-        if len(self.source_name_or_path) == 0:
-            self.source_name_or_path.append("")
+        if scanner_name != FL_DEPENDENCY:
+            if len(self.source_name_or_path) == 0:
+                self.source_name_or_path.append("")
         if len(self.license) == 0:
             self.license.append("")
 
         exclude = "Exclude" if self.exclude else ""
-
-        for source_name_or_path in self.source_name_or_path:
-            lic = ",".join(self.license)
-            items.append([os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
-                          self.download_location, self.homepage, self.copyright, exclude, self.comment])
+        lic = ",".join(self.license)
+        if scanner_name == FL_DEPENDENCY:
+            items = [self.purl, self.name, self.version, lic,
+                     self.download_location, self.homepage, self.copyright, exclude, self.comment]
+            if len(self.depends_on) > 0:
+                items.append(",".join(self.depends_on))
+        else:
+            for source_name_or_path in self.source_name_or_path:
+                if scanner_name == FL_BINARY:
+                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
+                                self.download_location, self.homepage, self.copyright, exclude, self.comment,
+                                self.bin_vulnerability, self.bin_tlsh, self.bin_sha1]
+                else:
+                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
+                                self.download_location, self.homepage, self.copyright, exclude, self.comment]
+                items.append(oss_item)
         return items
 
     def get_print_json(self):
@@ -171,6 +214,10 @@ class OssItem:
             json_item["exclude"] = self.exclude
         if self.comment != "":
             json_item["comment"] = self.comment
+        if len(self.depends_on) > 0:
+            json_item["depends on"] = self.depends_on
+        if self.purl != "":
+            json_item["purl"] = self.purl
 
         return json_item
 

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/output_format.py

@@ -3,7 +3,7 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import os
-from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv, remove_empty_sheet
+from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv
 from fosslight_util.write_opossum import write_opossum
 from fosslight_util.write_yaml import write_yaml
 
@@ -56,29 +56,24 @@ def check_output_format(output='', format='', customized_format={}):
     return success, msg, output_path, output_file, output_extension
 
 
-def write_output_file(output_file_without_ext, file_extension, sheet_list, extended_header={}, hide_header={}):
+def write_output_file(output_file_without_ext, file_extension, sheet_list, extended_header={}, hide_header={}, cover=""):
     success = True
     msg = ''
 
-    is_not_null, sheet_list = remove_empty_sheet(sheet_list)
-    if is_not_null:
-        if file_extension == '':
-            file_extension = '.xlsx'
-        result_file = output_file_without_ext + file_extension
+    if file_extension == '':
+        file_extension = '.xlsx'
+    result_file = output_file_without_ext + file_extension
 
-        if file_extension == '.xlsx':
-            success, msg = write_result_to_excel(result_file, sheet_list, extended_header, hide_header)
-        elif file_extension == '.csv':
-            success, msg, result_file = write_result_to_csv(result_file, sheet_list)
-        elif file_extension == '.json':
-            success, msg = write_opossum(result_file, sheet_list)
-        elif file_extension == '.yaml':
-            success, msg, result_file = write_yaml(result_file, sheet_list, False)
-        else:
-            success = False
-            msg = f'Not supported file extension({file_extension})'
+    if file_extension == '.xlsx':
+        success, msg = write_result_to_excel(result_file, sheet_list, extended_header, hide_header, cover)
+    elif file_extension == '.csv':
+        success, msg, result_file = write_result_to_csv(result_file, sheet_list)
+    elif file_extension == '.json':
+        success, msg = write_opossum(result_file, sheet_list)
+    elif file_extension == '.yaml':
+        success, msg, result_file = write_yaml(result_file, sheet_list, False)
     else:
-        result_file = ""
-        msg = "Nothing is detected from the scanner so output file is not generated."
+        success = False
+        msg = f'Not supported file extension({file_extension})'
 
     return success, msg, result_file

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/parsing_yaml.py

@@ -115,6 +115,12 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.yocto_package = value
     elif key == 'yocto_recipe':
         oss.yocto_recipe = value
+    elif key == 'vulnerability link':
+        oss.bin_vulnerability = value
+    elif key == 'tlsh':
+        oss.bin_tlsh = value
+    elif key == 'sha1':
+        oss.bin_sha1 = value
     else:
         if yaml_file != "":
             _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/read_excel.py

@@ -75,7 +75,10 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                 "Exclude": IDX_CANNOT_FOUND,
                 "Copyright Text": IDX_CANNOT_FOUND,
                 "Comment": IDX_CANNOT_FOUND,
-                "File Name or Path": IDX_CANNOT_FOUND
+                "File Name or Path": IDX_CANNOT_FOUND,
+                "Vulnerability Link": IDX_CANNOT_FOUND,
+                "TLSH": IDX_CANNOT_FOUND,
+                "SHA1": IDX_CANNOT_FOUND
             }
             num_cols = xl_sheet.ncols
             num_rows = xl_sheet.nrows

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/write_excel.py

@@ -13,6 +13,7 @@ import copy
 from pathlib import Path
 import fosslight_util.constant as constant
 from jsonmerge import merge
+from fosslight_util.cover import CoverItem
 
 _HEADER = {'BIN (': ['ID', 'Binary Name', 'Source Code Path',
                      'NOTICE.html', 'OSS Name', 'OSS Version',
@@ -31,6 +32,7 @@ _EMPTY_ITEM_MSG = "* There is no item"\
 IDX_FILE = 0
 IDX_EXCLUDE = 7
 logger = logging.getLogger(constant.LOGGER_NAME)
+COVER_SHEET_NAME = 'Scanner Info'
 
 
 def write_excel_and_csv(filename_without_extension, sheet_list, ignore_os=False, extended_header={}, hide_header={}):
@@ -105,9 +107,9 @@ def get_header_row(sheet_name, sheet_content, extended_header={}):
             if sheet_name.startswith(header_key):
                 selected_header = merged_headers[header_key]
                 break
-
-    if not selected_header:
-        selected_header = sheet_content.pop(0)
+    if len(sheet_content) > 0:
+        if not selected_header:
+            selected_header = sheet_content.pop(0)
     return selected_header, sheet_content
 
 
@@ -166,16 +168,18 @@ def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, ex
     return success, error_msg, output
 
 
-def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_header={}):
+def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_header={}, cover=""):
     success = True
     error_msg = ""
 
     try:
-        if sheet_list:
-            output_dir = os.path.dirname(out_file_name)
-            Path(output_dir).mkdir(parents=True, exist_ok=True)
+        output_dir = os.path.dirname(out_file_name)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
 
-            workbook = xlsxwriter.Workbook(out_file_name)
+        workbook = xlsxwriter.Workbook(out_file_name)
+        if cover:
+            write_cover_sheet(workbook, cover)
+        if sheet_list:
             for sheet_name, sheet_contents in sheet_list.items():
                 selected_header, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
                 try:
@@ -188,13 +192,33 @@ def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_he
 
                 if hide_header:
                     hide_column(worksheet, selected_header, hide_header)
-            workbook.close()
+        workbook.close()
     except Exception as ex:
         error_msg = str(ex)
         success = False
     return success, error_msg
 
 
+def write_cover_sheet(workbook, cover):
+    worksheet = workbook.add_worksheet(COVER_SHEET_NAME)
+
+    format_bold = workbook.add_format({'bold': True})
+    worksheet.merge_range('A1:B1', 'About the scanner', format_bold)
+
+    key_format = workbook.add_format({'bold': True, 'font_color': 'white', 'bg_color': 'navy'})
+    item_format = workbook.add_format()
+    item_format.set_text_wrap()
+
+    cover_json = cover.get_print_json()
+    row = 1
+    for item in cover_json:
+        worksheet.write(row, 0, item, key_format)
+        worksheet.write(row, 1, cover_json[item], item_format)
+        row += 1
+    worksheet.set_column(0, 0, 30)
+    worksheet.set_column(1, 1, 100)
+
+
 def write_result_to_sheet(worksheet, sheet_contents):
     row = 1
     for row_item in sheet_contents:
@@ -221,7 +245,33 @@ def create_worksheet(workbook, sheet_name, header_row):
     return worksheet
 
 
-def merge_excels(find_excel_dir, final_out, merge_files=''):
+def merge_cover_comment(find_excel_dir, merge_files=''):
+    FIND_EXTENSION = '.xlsx'
+    merge_comment = []
+    cover_comment = ''
+    try:
+        files = os.listdir(find_excel_dir)
+
+        if len([name for name in files if name.endswith(FIND_EXTENSION)]) > 0:
+            for file in files:
+                if merge_files:
+                    if file not in merge_files:
+                        continue
+                if file.endswith(FIND_EXTENSION):
+                    file = os.path.join(find_excel_dir, file)
+                    df_excel = pd.read_excel(file, sheet_name=COVER_SHEET_NAME, index_col=0, engine='openpyxl')
+                    if not df_excel.empty:
+                        tool_name = df_excel.loc[CoverItem.tool_name_key].values[0]
+                        comment = df_excel.loc[CoverItem.comment_key].values[0]
+                        merge_comment.append(str(f"[{tool_name}] {comment}"))
+            cover_comment = '\n'.join(merge_comment)
+    except Exception as ex:
+        logger.warning(f'Fail to merge comment of Scanner info: {str(ex)}')
+
+    return cover_comment
+
+
+def merge_excels(find_excel_dir, final_out, merge_files='', cover=''):
     success = True
     msg = ""
     FIND_EXTENSION = '.xlsx'
@@ -231,7 +281,7 @@ def merge_excels(find_excel_dir, final_out, merge_files=''):
 
         if len([name for name in files if name.endswith(FIND_EXTENSION)]) > 0:
             writer = pd.ExcelWriter(final_out)
-
+            write_cover_sheet(writer.book, cover)
             for file in files:
                 if merge_files:
                     if file not in merge_files:
@@ -243,13 +293,18 @@ def merge_excels(find_excel_dir, final_out, merge_files=''):
                     excel_file = pd.ExcelFile(file, engine='openpyxl')
 
                     for sheet_name in excel_file.sheet_names:
-                        sheet_name_to_copy = f"{f_short_name}_{sheet_name}"
+                        if sheet_name == COVER_SHEET_NAME:
+                            continue
                         df_excel = pd.read_excel(
                             file, sheet_name=sheet_name, engine='openpyxl')
-                        if sheet_name not in added_sheet_names:
-                            sheet_name_to_copy = sheet_name
-                        df_excel.to_excel(writer, sheet_name_to_copy,
-                                          index=False)
+                        if sheet_name in added_sheet_names:
+                            sheet_name = f"{f_short_name}_{sheet_name}"
+                        df_excel.to_excel(writer, sheet_name, index=False)
+                        added_sheet_names.append(sheet_name)
+
+                        if sheet_name == 'BIN_FL_Binary':
+                            bin_sheet = writer.sheets[sheet_name]
+                            bin_sheet.set_column("L:M", None, None, {"hidden": True})  # 'TLSH', 'SHA1' column hide
             writer.close()
     except Exception as ex:
         msg = str(ex)

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util/write_yaml.py

@@ -35,11 +35,16 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
             for sheet_name, sheet_contents in sheet_list.items():
                 if sheet_name not in constant.supported_sheet_and_scanner.keys():
                     continue
+                scanner_name = constant.supported_sheet_and_scanner[sheet_name]
+                sheet_contents_with_scanner = []
+                for i in sheet_contents:
+                    i.insert(0, scanner_name)
+                    sheet_contents_with_scanner.append(i)
                 if not separate_yaml:
-                    merge_sheet.extend(sheet_contents)
+                    merge_sheet.extend(sheet_contents_with_scanner)
                 else:
                     output_file = f'{separate_output_file}_{sheet_name}.yaml'
-                    convert_sheet_to_yaml(sheet_contents, output_file)
+                    convert_sheet_to_yaml(sheet_contents_with_scanner, output_file)
                     output_files.append(output_file)
 
             if not separate_yaml:
@@ -61,13 +66,13 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
     return success, error_msg, output
 
 
-def convert_sheet_to_yaml(sheet_contents, output_file):
-    sheet_contents = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents))]
+def convert_sheet_to_yaml(sheet_contents_with_scanner, output_file):
+    sheet_contents_with_scanner = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents_with_scanner))]
 
     yaml_dict = {}
-    for sheet_item in sheet_contents:
+    for sheet_item in sheet_contents_with_scanner:
         item = OssItem('')
-        item.set_sheet_item(sheet_item)
+        item.set_sheet_item(sheet_item[1:], sheet_item[0])
         create_yaml_with_ossitem(item, yaml_dict)
 
     with open(output_file, 'w') as f:

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 1.4.39
+Version: 1.4.41
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-1.4.39 → fosslight_util-1.4.41}/src/fosslight_util.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ src/fosslight_util/compare_yaml.py
 src/fosslight_util/constant.py
 src/fosslight_util/convert_excel_to_yaml.py
 src/fosslight_util/correct.py
+src/fosslight_util/cover.py
 src/fosslight_util/download.py
 src/fosslight_util/help.py
 src/fosslight_util/oss_item.py