fosslight-util 1.4.34__py3-none-any.whl → 2.1.28__py3-none-any.whl

fosslight_util/write_excel.py

@@ -7,151 +7,102 @@ import csv
 import time
 import logging
 import os
-import platform
 import pandas as pd
-import copy
 from pathlib import Path
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER, FOSSLIGHT_BINARY
 from jsonmerge import merge
 
-_HEADER = {'BIN (': ['ID', 'Binary Name', 'Source Code Path',
+_HEADER = {'BIN (': ['ID', 'Binary Path', 'Source Code Path',
                      'NOTICE.html', 'OSS Name', 'OSS Version',
                      'License', 'Download Location', 'Homepage',
                      'Copyright Text', 'Exclude', 'Comment'],
-           'SRC': ['ID', 'Source Name or Path', 'OSS Name',
-                   'OSS Version', 'License',  'Download Location',
-                   'Homepage', 'Copyright Text', 'Exclude',
-                   'Comment'],
-           'BIN': ['ID', 'Binary Name', 'OSS Name', 'OSS Version',
+           'SRC': ['ID', 'Source Path', 'OSS Name', 'OSS Version',
                    'License', 'Download Location', 'Homepage',
-                   'Copyright Text', 'Exclude', 'Comment']}
+                   'Copyright Text', 'Exclude', 'Comment'],
+           'BIN': ['ID', 'Binary Path', 'OSS Name', 'OSS Version',
+                   'License', 'Download Location', 'Homepage',
+                   'Copyright Text', 'Exclude', 'Comment',
+                   'Vulnerability Link', 'TLSH', 'SHA1'],
+           'DEP': ['ID', 'Package URL', 'OSS Name', 'OSS Version',
+                   'License', 'Download Location', 'Homepage',
+                   'Copyright Text', 'Exclude', 'Comment',
+                   'Depends On']}
+
+BIN_HIDE_HEADER = {'TLSH', "SHA1"}
 _OUTPUT_FILE_PREFIX = "FOSSLight-Report_"
-_EMPTY_ITEM_MSG = "* There is no item"\
-                    " to print in FOSSLight-Report.\n"
 IDX_FILE = 0
 IDX_EXCLUDE = 7
-logger = logging.getLogger(constant.LOGGER_NAME)
-
-
-def write_excel_and_csv(filename_without_extension, sheet_list, ignore_os=False, extended_header={}):
-    success = True
-    error_msg = ""
-    success_csv = True
-    error_msg_csv = ""
-    output_files = ""
-    output_csv = ""
-
-    is_not_null, sheet_list = remove_empty_sheet(sheet_list)
-
-    if is_not_null:
-        output_dir = os.path.dirname(filename_without_extension)
-        Path(output_dir).mkdir(parents=True, exist_ok=True)
+logger = logging.getLogger(LOGGER_NAME)
+COVER_SHEET_NAME = 'Scanner Info'
+MAX_EXCEL_URL_LENGTH = 255
 
-        success, error_msg = write_result_to_excel(f"{filename_without_extension}.xlsx", sheet_list, extended_header)
-
-        if ignore_os or platform.system() != "Windows":
-            success_csv, error_msg_csv, output_csv = write_result_to_csv(f"{filename_without_extension}.csv",
-                                                                         sheet_list, True, extended_header)
-        if success:
-            output_files = f"{filename_without_extension}.xlsx"
-        else:
-            error_msg = "[Error] Writing excel:" + error_msg
-        if success_csv:
-            if output_csv:
-                output_files = f"{output_files}, {output_csv}" if output_files else output_csv
-        else:
-            error_msg += "\n[Error] Writing csv:" + error_msg_csv
-    else:
-        success = False
-        error_msg = _EMPTY_ITEM_MSG
-
-    return (success and success_csv), error_msg, output_files
-
-
-def remove_empty_sheet(sheet_items):
-    skip_sheet_name = []
-    cnt_sheet_to_print = 0
-    final_sheet_to_print = {}
-    success = False
-    try:
-        if sheet_items:
-            for sheet_name, sheet_content in sheet_items.items():
-                if len(sheet_content) > 0:
-                    final_sheet_to_print[sheet_name] = sheet_content
-                    cnt_sheet_to_print += 1
-                else:
-                    skip_sheet_name.append(sheet_name)
-            if cnt_sheet_to_print != 0:
-                success = True
-                if len(skip_sheet_name) > 0:
-                    logger.warn("* Empty sheet(not printed):" + str(skip_sheet_name))
-    except Exception as ex:
-        logger.warn("* Warning:"+str(ex))
-
-    return success, final_sheet_to_print
 
-
-def get_header_row(sheet_name, sheet_content, extended_header={}):
+def get_header_row(sheet_name, extended_header={}):
     selected_header = []
 
     merged_headers = merge(_HEADER, extended_header)
 
-    selected_header = merged_headers.get(sheet_name)
+    selected_header = merged_headers.get(sheet_name, [])
     if not selected_header:
         for header_key in merged_headers.keys():
             if sheet_name.startswith(header_key):
                 selected_header = merged_headers[header_key]
                 break
-
-    if not selected_header:
-        selected_header = sheet_content.pop(0)
-    return selected_header, sheet_content
+    return selected_header
 
 
-def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, extended_header={}):
+def write_result_to_csv(output_file, scan_item, separate_sheet=False, extended_header={}):
     success = True
     error_msg = ""
     file_extension = ".csv"
     output = ""
 
     try:
-        sheet_list = copy.deepcopy(sheet_list_origin)
-        if sheet_list:
-            output_files = []
-            output_dir = os.path.dirname(output_file)
-            Path(output_dir).mkdir(parents=True, exist_ok=True)
-            if separate_sheet:
-                filename = os.path.splitext(os.path.basename(output_file))[0]
-                separate_output_file = os.path.join(output_dir, filename)
-
-            merge_sheet = []
-            for sheet_name, sheet_contents in sheet_list.items():
-                row_num = 1
-                header_row, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
-
-                if not separate_sheet:
-                    merge_sheet.extend(sheet_content_without_header)
-                    if sheet_name == list(sheet_list.keys())[-1]:
-                        sheet_content_without_header = merge_sheet
-                    else:
-                        continue
+        output_files = []
+        output_dir = os.path.dirname(output_file)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        if separate_sheet:
+            filename = os.path.splitext(os.path.basename(output_file))[0]
+            separate_output_file = os.path.join(output_dir, filename)
+
+        merge_sheet = []
+        for scanner_name, _ in scan_item.file_items.items():
+            row_num = 1
+            sheet_name = ""
+            if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
+                sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+            elif extended_header:
+                sheet_name = list(extended_header.keys())[0]
+            sheet_content_without_header = scan_item.get_print_array(scanner_name)
+            header_row = get_header_row(sheet_name, extended_header)
+
+            if 'Copyright Text' in header_row:
+                idx = header_row.index('Copyright Text')-1
+                for item in sheet_content_without_header:
+                    item[idx] = item[idx].replace('\n', ', ')
+            if not separate_sheet:
+                merge_sheet.extend(sheet_content_without_header)
+                if scanner_name == list(scan_item.file_items.keys())[-1]:
+                    sheet_content_without_header = merge_sheet
                 else:
-                    output_file = separate_output_file + "_" + sheet_name + file_extension
-                try:
-                    sheet_content_without_header = sorted(sheet_content_without_header,
-                                                          key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
-                except Exception:
-                    pass
-                with open(output_file, 'w', newline='') as file:
-                    writer = csv.writer(file, delimiter='\t')
-                    writer.writerow(header_row)
-                    for row_item in sheet_content_without_header:
-                        row_item.insert(0, row_num)
-                        writer.writerow(row_item)
-                        row_num += 1
-                output_files.append(output_file)
-            if output_files:
-                output = ", ".join(output_files)
+                    continue
+            else:
+                output_file = separate_output_file + "_" + sheet_name + file_extension
+            try:
+                sheet_content_without_header = sorted(sheet_content_without_header,
+                                                      key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
+            except Exception:
+                pass
+            with open(output_file, 'w', newline='') as file:
+                writer = csv.writer(file, delimiter='\t')
+                writer.writerow(header_row)
+                for row_item in sheet_content_without_header:
+                    row_item.insert(0, row_num)
+                    writer.writerow(row_item)
+                    row_num += 1
+            output_files.append(output_file)
+        if output_files:
+            output = ", ".join(output_files)
     except Exception as ex:
         error_msg = str(ex)
         success = False
@@ -159,52 +110,104 @@ def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, ex
     return success, error_msg, output
 
 
-def write_result_to_excel(out_file_name, sheet_list, extended_header={}):
+def write_result_to_excel(out_file_name, scan_item, extended_header={}, hide_header={}):
     success = True
     error_msg = ""
 
     try:
-        if sheet_list:
-            output_dir = os.path.dirname(out_file_name)
-            Path(output_dir).mkdir(parents=True, exist_ok=True)
+        output_dir = os.path.dirname(out_file_name)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
 
-            workbook = xlsxwriter.Workbook(out_file_name)
-            for sheet_name, sheet_contents in sheet_list.items():
-                selected_header, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
+        workbook = xlsxwriter.Workbook(out_file_name)
+        write_cover_sheet(workbook, scan_item.cover)
+        if scan_item.file_items and len(scan_item.file_items.keys()) > 0:
+            for scanner_name, _ in scan_item.file_items.items():
+                sheet_name = ""
+                if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
+                    sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+                elif extended_header:
+                    sheet_name = list(extended_header.keys())[0]
+                sheet_content_without_header = scan_item.get_print_array(scanner_name)
+                selected_header = get_header_row(sheet_name, extended_header)
                 try:
                     sheet_content_without_header = sorted(sheet_content_without_header,
                                                           key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
                 except Exception:
                     pass
+                if sheet_name:
+                    worksheet = create_worksheet(workbook, sheet_name, selected_header)
+                    write_result_to_sheet(worksheet, sheet_content_without_header)
+                    if (scanner_name == FOSSLIGHT_BINARY) and (not hide_header):
+                        hide_header = BIN_HIDE_HEADER
+                    if hide_header:
+                        hide_column(worksheet, selected_header, hide_header)
+
+        for sheet_name, content in scan_item.external_sheets.items():
+            if len(content) > 0:
+                selected_header = content.pop(0)
                 worksheet = create_worksheet(workbook, sheet_name, selected_header)
-                write_result_to_sheet(worksheet, sheet_content_without_header)
-            workbook.close()
+                write_result_to_sheet(worksheet, content)
+                if hide_header:
+                    hide_column(worksheet, selected_header, hide_header)
+
+        workbook.close()
     except Exception as ex:
         error_msg = str(ex)
         success = False
     return success, error_msg
 
 
+def write_cover_sheet(workbook, cover):
+    worksheet = workbook.add_worksheet(COVER_SHEET_NAME)
+
+    format_bold = workbook.add_format({'bold': True})
+    worksheet.merge_range('A1:B1', 'About the scanner', format_bold)
+
+    key_format = workbook.add_format({'bold': True, 'font_color': 'white', 'bg_color': 'navy'})
+    item_format = workbook.add_format()
+    item_format.set_text_wrap()
+
+    cover_json = cover.get_print_json()
+    row = 1
+    for item in cover_json:
+        worksheet.write(row, 0, item, key_format)
+        worksheet.write(row, 1, cover_json[item], item_format)
+        row += 1
+    worksheet.set_column(0, 0, 30)
+    worksheet.set_column(1, 1, 100)
+
+
 def write_result_to_sheet(worksheet, sheet_contents):
     row = 1
     for row_item in sheet_contents:
         worksheet.write(row, 0, row)
         for col_num, value in enumerate(row_item):
-            worksheet.write(row, col_num + 1, str(value))
+            if len(value) > MAX_EXCEL_URL_LENGTH and (value.startswith("http://") or value.startswith("https://")):
+                worksheet.write_string(row, col_num + 1, str(value))
+            else:
+                worksheet.write(row, col_num + 1, str(value))
         row += 1
 
 
+def hide_column(worksheet, selected_header, hide_header):
+    for col_idx, sel_hd in enumerate(selected_header):
+        for hide_hd in hide_header:
+            if str(sel_hd).lower() == str(hide_hd).lower():
+                worksheet.set_column(col_idx, col_idx, None, None, {"hidden": True})
+
+
 def create_worksheet(workbook, sheet_name, header_row):
     if len(sheet_name) > 31:
         current_time = str(time.time())
         sheet_name = current_time
     worksheet = workbook.add_worksheet(sheet_name)
-    for col_num, value in enumerate(header_row):
-        worksheet.write(0, col_num, value)
+    if header_row:
+        for col_num, value in enumerate(header_row):
+            worksheet.write(0, col_num, value)
     return worksheet
 
 
-def merge_excels(find_excel_dir, final_out, merge_files=''):
+def merge_excels(find_excel_dir, final_out, merge_files='', cover=''):
     success = True
     msg = ""
     FIND_EXTENSION = '.xlsx'
@@ -214,7 +217,7 @@ def merge_excels(find_excel_dir, final_out, merge_files=''):
 
         if len([name for name in files if name.endswith(FIND_EXTENSION)]) > 0:
             writer = pd.ExcelWriter(final_out)
-
+            write_cover_sheet(writer.book, cover)
             for file in files:
                 if merge_files:
                     if file not in merge_files:
@@ -226,13 +229,18 @@ def merge_excels(find_excel_dir, final_out, merge_files=''):
                     excel_file = pd.ExcelFile(file, engine='openpyxl')
 
                     for sheet_name in excel_file.sheet_names:
-                        sheet_name_to_copy = f"{f_short_name}_{sheet_name}"
+                        if sheet_name == COVER_SHEET_NAME:
+                            continue
                         df_excel = pd.read_excel(
                             file, sheet_name=sheet_name, engine='openpyxl')
-                        if sheet_name not in added_sheet_names:
-                            sheet_name_to_copy = sheet_name
-                        df_excel.to_excel(writer, sheet_name_to_copy,
-                                          index=False)
+                        if sheet_name in added_sheet_names:
+                            sheet_name = f"{f_short_name}_{sheet_name}"
+                        df_excel.to_excel(writer, sheet_name, index=False)
+                        added_sheet_names.append(sheet_name)
+
+                        if sheet_name == 'BIN_FL_Binary':
+                            bin_sheet = writer.sheets[sheet_name]
+                            bin_sheet.set_column("L:M", None, None, {"hidden": True})  # 'TLSH', 'SHA1' column hide
             writer.close()
     except Exception as ex:
         msg = str(ex)

fosslight_util/write_opossum.py

@@ -11,9 +11,8 @@ import logging
 from datetime import datetime
 from pathlib import Path
 import traceback
-from typing import Dict, Optional
-
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SOURCE
+from typing import Dict, Optional, Tuple
 
 
 PACKAGE = {
@@ -30,7 +29,7 @@ PACKAGE = {
 }
 
 _attributionConfidence = 80
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 
 
 class AttributionItem():
@@ -51,7 +50,7 @@ class AttributionItem():
             self.excludeFromNotice = False
 
         self.source_name = source_name
-        if source_name == constant.FL_DEPENDENCY:
+        if source_name == FOSSLIGHT_DEPENDENCY:
             self.preSelected = True
         else:
             self.preSelected = False
@@ -113,12 +112,12 @@ class Attribution(AttributionItem):
         dict[licenseName] = self.licenseName
         dict[preSelected] = self.preSelected
 
-        if self.source_name == constant.FL_SOURCE or constant.FL_BINARY:
+        if self.source_name == FOSSLIGHT_SOURCE or FOSSLIGHT_BINARY:
             dict[copyright] = self.copyright
             dict[packageName] = self.packageName
             dict[packageVersion] = self.packageVersion
             dict[url] = self.url
-        elif self.source_name == constant.FL_DEPENDENCY:
+        elif self.source_name == FOSSLIGHT_DEPENDENCY:
             dict[copyright] = self.copyright
             dict[packageName] = self.packageName
             dict[packageVersion] = self.packageVersion
@@ -165,7 +164,7 @@ def make_frequentlicenses():
     return frequentLicenses, success, error_msg
 
 
-def write_opossum(filename, sheet_list):
+def write_opossum(filename: str, scan_item) -> Tuple[bool, str]:
     success = True
     error_msg = ''
     dict = {}
@@ -176,7 +175,7 @@ def write_opossum(filename, sheet_list):
     _filesWithChildren_key = 'filesWithChildren'
     _attributionBreakpoints_key = 'attributionBreakpoints'
 
-    if sheet_list:
+    if scan_item:
         output_dir = os.path.dirname(filename)
         Path(output_dir).mkdir(parents=True, exist_ok=True)
 
@@ -189,14 +188,9 @@ def write_opossum(filename, sheet_list):
         filesWithChildren_list = []
         attributionBreakpoints_list = []
         try:
-            for sheet_name, sheet_contents in sheet_list.items():
-                if sheet_name in constant.supported_sheet_and_scanner.keys():
-                    scanner = constant.supported_sheet_and_scanner.get(sheet_name)
-                else:
-                    logger.warning("Not supported scanner(sheet_name):" + sheet_name)
-                    continue
-
-                ret_resources_attribution = make_resources_and_attributions(sheet_contents, scanner, resources, fc_list)
+            for scanner_name, _ in scan_item.file_items.items():
+                sheet_contents = scan_item.get_print_array(scanner_name)
+                ret_resources_attribution = make_resources_and_attributions(sheet_contents, scanner_name, resources, fc_list)
                 success, rsc, ea, ra, fl, ab = ret_resources_attribution
                 if success:
                     dict[_resources_key].update(rsc)
@@ -255,14 +249,14 @@ def make_resources_and_attributions(sheet_items, scanner, resources, fc_list):
             items = items[0:9]
             path, oss_name, oss_version, license, url, homepage, copyright, exclude, comment = items
 
-            if scanner == constant.FL_SOURCE:
+            if scanner == FOSSLIGHT_SOURCE:
                 if (os.path.join(os.sep, path) + os.sep) not in fc_list:
                     resources = make_resources(path, resources)
                 attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url)
-            elif scanner == constant.FL_BINARY:
+            elif scanner == FOSSLIGHT_BINARY:
                 resources = make_resources(path, resources)
                 attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url)
-            elif scanner == constant.FL_DEPENDENCY:
+            elif scanner == FOSSLIGHT_DEPENDENCY:
                 try:
                     packageType = PACKAGE[path]
                 except Exception:

fosslight_util/write_scancodejson.py

@@ -6,57 +6,76 @@
 import logging
 import os
 import json
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_DEPENDENCY
+from fosslight_util.oss_item import ScannerItem
+from typing import List
 
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 EMPTY_FILE_PATH = '-'
 
 
-def write_scancodejson(output_dir, output_filename, oss_list):
+def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[ScannerItem]):
     json_output = {}
     json_output['headers'] = []
     json_output['summary'] = {}
     json_output['license_detections'] = []
     json_output['files'] = []
+    json_output['dependencies'] = []
 
-    for oi in oss_list:
-        if oi.exclude:
-            continue
-        if not oi.source_name_or_path:
-            oi.source_name_or_path = EMPTY_FILE_PATH
-        for item_path in oi.source_name_or_path:
-            filtered = next(filter(lambda x: x['path'] == item_path, json_output['files']), None)
-            if filtered:
-                append_oss_item_in_filesitem(oi, filtered)
+    for scanner, file_items in oss_list.file_items.items():
+        for fi in file_items:
+            if scanner == FOSSLIGHT_DEPENDENCY:
+                json_output['dependencies'] = add_item_in_deps(fi, json_output['dependencies'])
             else:
-                json_output['files'] = add_item_in_files(oi, item_path, json_output['files'])
+                if fi.exclude:
+                    continue
+                if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
+                    continue
+                if not fi.source_name_or_path:
+                    fi.source_name_or_path = EMPTY_FILE_PATH
+                json_output['files'] = add_item_in_files(fi, json_output['files'])
+
     with open(os.path.join(output_dir, output_filename), 'w') as f:
         json.dump(json_output, f, sort_keys=False, indent=4)
 
 
-def append_oss_item_in_filesitem(item, files_item):
-    if item.is_binary:
-        files_item['is_binary'] = item.is_binary
-    if item.name or item.version or item.license or item.copyright or item.download_location or item.comment:
+def get_oss_item_list(oss_items):
+    scan_oss_items = []
+    for oi in oss_items:
+        if oi.exclude:
+            continue
         oss_item = {}
-        oss_item['name'] = item.name
-        oss_item['version'] = item.version
-        oss_item['license'] = item.license
-        oss_item['copyright'] = item.copyright
-        oss_item['download_location'] = item.download_location
-        oss_item['comment'] = item.comment
-        files_item['oss'].append(oss_item)
-    return files_item
+        oss_item['name'] = oi.name
+        oss_item['version'] = oi.version
+        oss_item['license'] = oi.license
+        oss_item['copyright'] = oi.copyright
+        oss_item['download_location'] = oi.download_location
+        oss_item['comment'] = oi.comment
+        scan_oss_items.append(oss_item)
+
+    return scan_oss_items
 
 
-def add_item_in_files(item, item_path, files_list):
+def add_item_in_files(file_item, files_list):
     files_item = {}
-    files_item['path'] = item_path
-    files_item['name'] = os.path.basename(item_path)
-    files_item['is_binary'] = item.is_binary
-    files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(item_path))
-    files_item['oss'] = []
-    files_item = append_oss_item_in_filesitem(item, files_item)
+    files_item['path'] = file_item.source_name_or_path
+    files_item['name'] = os.path.basename(file_item.source_name_or_path)
+    files_item['is_binary'] = file_item.is_binary
+    files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(file_item.source_name_or_path))
+    files_item['oss'] = get_oss_item_list(file_item.oss_items)
+
     files_list.append(files_item)
 
     return files_list
+
+
+def add_item_in_deps(file_item, deps_list):
+    deps_item = {}
+    deps_item['purl'] = file_item.purl
+    deps_item['scope'] = 'dependencies'
+    deps_item['depends_on'] = file_item.depends_on
+    deps_item['oss'] = get_oss_item_list(file_item.oss_items)
+
+    deps_list.append(deps_item)
+
+    return deps_list