fosslight-util 1.4.48__py3-none-any.whl → 2.0.1__py3-none-any.whl

fosslight_util/write_excel.py

@@ -7,160 +7,101 @@ import csv
 import time
 import logging
 import os
-import platform
 import pandas as pd
-import copy
 from pathlib import Path
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER, FOSSLIGHT_BINARY
 from jsonmerge import merge
-from fosslight_util.cover import CoverItem
 
 _HEADER = {'BIN (': ['ID', 'Binary Path', 'Source Code Path',
                      'NOTICE.html', 'OSS Name', 'OSS Version',
                      'License', 'Download Location', 'Homepage',
                      'Copyright Text', 'Exclude', 'Comment'],
-           'SRC': ['ID', 'Source Path', 'OSS Name',
-                   'OSS Version', 'License',  'Download Location',
-                   'Homepage', 'Copyright Text', 'Exclude',
-                   'Comment'],
+           'SRC': ['ID', 'Source Path', 'OSS Name', 'OSS Version',
+                   'License', 'Download Location', 'Homepage',
+                   'Copyright Text', 'Exclude', 'Comment'],
            'BIN': ['ID', 'Binary Path', 'OSS Name', 'OSS Version',
                    'License', 'Download Location', 'Homepage',
-                   'Copyright Text', 'Exclude', 'Comment']}
+                   'Copyright Text', 'Exclude', 'Comment',
+                   'Vulnerability Link', 'TLSH', 'SHA1'],
+           'DEP': ['ID', 'Package URL', 'OSS Name', 'OSS Version',
+                   'License', 'Download Location', 'Homepage',
+                   'Copyright Text', 'Exclude', 'Comment',
+                   'Depends On']}
+
+BIN_HIDE_HEADER = {'TLSH', "SHA1"}
 _OUTPUT_FILE_PREFIX = "FOSSLight-Report_"
-_EMPTY_ITEM_MSG = "* There is no item"\
-                    " to print in FOSSLight-Report.\n"
 IDX_FILE = 0
 IDX_EXCLUDE = 7
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 COVER_SHEET_NAME = 'Scanner Info'
 
 
-def write_excel_and_csv(filename_without_extension, sheet_list, ignore_os=False, extended_header={}, hide_header={}):
-    success = True
-    error_msg = ""
-    success_csv = True
-    error_msg_csv = ""
-    output_files = ""
-    output_csv = ""
-
-    is_not_null, sheet_list = remove_empty_sheet(sheet_list)
-
-    if is_not_null:
-        output_dir = os.path.dirname(filename_without_extension)
-        Path(output_dir).mkdir(parents=True, exist_ok=True)
-
-        success, error_msg = write_result_to_excel(f"{filename_without_extension}.xlsx",
-                                                   sheet_list,
-                                                   extended_header,
-                                                   hide_header)
-
-        if ignore_os or platform.system() != "Windows":
-            success_csv, error_msg_csv, output_csv = write_result_to_csv(f"{filename_without_extension}.csv",
-                                                                         sheet_list, True, extended_header)
-        if success:
-            output_files = f"{filename_without_extension}.xlsx"
-        else:
-            error_msg = "[Error] Writing excel:" + error_msg
-        if success_csv:
-            if output_csv:
-                output_files = f"{output_files}, {output_csv}" if output_files else output_csv
-        else:
-            error_msg += "\n[Error] Writing csv:" + error_msg_csv
-    else:
-        success = False
-        error_msg = _EMPTY_ITEM_MSG
-
-    return (success and success_csv), error_msg, output_files
-
-
-def remove_empty_sheet(sheet_items):
-    skip_sheet_name = []
-    cnt_sheet_to_print = 0
-    final_sheet_to_print = {}
-    success = False
-    try:
-        if sheet_items:
-            for sheet_name, sheet_content in sheet_items.items():
-                if len(sheet_content) > 0:
-                    final_sheet_to_print[sheet_name] = sheet_content
-                    cnt_sheet_to_print += 1
-                else:
-                    skip_sheet_name.append(sheet_name)
-            if cnt_sheet_to_print != 0:
-                success = True
-                if len(skip_sheet_name) > 0:
-                    logger.warn("* Empty sheet(not printed):" + str(skip_sheet_name))
-    except Exception as ex:
-        logger.warn("* Warning:"+str(ex))
-
-    return success, final_sheet_to_print
-
-
-def get_header_row(sheet_name, sheet_content, extended_header={}):
+def get_header_row(sheet_name, extended_header={}):
     selected_header = []
 
     merged_headers = merge(_HEADER, extended_header)
 
-    selected_header = merged_headers.get(sheet_name)
+    selected_header = merged_headers.get(sheet_name, [])
     if not selected_header:
         for header_key in merged_headers.keys():
             if sheet_name.startswith(header_key):
                 selected_header = merged_headers[header_key]
                 break
-    if len(sheet_content) > 0:
-        if not selected_header:
-            selected_header = sheet_content.pop(0)
-    return selected_header, sheet_content
+    return selected_header
 
 
-def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, extended_header={}):
+def write_result_to_csv(output_file, scan_item, separate_sheet=False, extended_header={}):
     success = True
     error_msg = ""
     file_extension = ".csv"
     output = ""
 
     try:
-        sheet_list = copy.deepcopy(sheet_list_origin)
-        if sheet_list:
-            output_files = []
-            output_dir = os.path.dirname(output_file)
-            Path(output_dir).mkdir(parents=True, exist_ok=True)
-            if separate_sheet:
-                filename = os.path.splitext(os.path.basename(output_file))[0]
-                separate_output_file = os.path.join(output_dir, filename)
-
-            merge_sheet = []
-            for sheet_name, sheet_contents in sheet_list.items():
-                row_num = 1
-                header_row, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
-
-                if 'Copyright Text' in header_row:
-                    idx = header_row.index('Copyright Text')-1
-                    for item in sheet_content_without_header:
-                        item[idx] = item[idx].replace('\n', ', ')
-                if not separate_sheet:
-                    merge_sheet.extend(sheet_content_without_header)
-                    if sheet_name == list(sheet_list.keys())[-1]:
-                        sheet_content_without_header = merge_sheet
-                    else:
-                        continue
+        output_files = []
+        output_dir = os.path.dirname(output_file)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        if separate_sheet:
+            filename = os.path.splitext(os.path.basename(output_file))[0]
+            separate_output_file = os.path.join(output_dir, filename)
+
+        merge_sheet = []
+        for scanner_name, _ in scan_item.file_items.items():
+            row_num = 1
+            sheet_name = ""
+            if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
+                sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+            elif extended_header:
+                sheet_name = list(extended_header.keys())[0]
+            sheet_content_without_header = scan_item.get_print_array(scanner_name)
+            header_row = get_header_row(sheet_name, extended_header)
+
+            if 'Copyright Text' in header_row:
+                idx = header_row.index('Copyright Text')-1
+                for item in sheet_content_without_header:
+                    item[idx] = item[idx].replace('\n', ', ')
+            if not separate_sheet:
+                merge_sheet.extend(sheet_content_without_header)
+                if scanner_name == list(scan_item.file_items.keys())[-1]:
+                    sheet_content_without_header = merge_sheet
                 else:
-                    output_file = separate_output_file + "_" + sheet_name + file_extension
-                try:
-                    sheet_content_without_header = sorted(sheet_content_without_header,
-                                                          key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
-                except Exception:
-                    pass
-                with open(output_file, 'w', newline='') as file:
-                    writer = csv.writer(file, delimiter='\t')
-                    writer.writerow(header_row)
-                    for row_item in sheet_content_without_header:
-                        row_item.insert(0, row_num)
-                        writer.writerow(row_item)
-                        row_num += 1
-                output_files.append(output_file)
-            if output_files:
-                output = ", ".join(output_files)
+                    continue
+            else:
+                output_file = separate_output_file + "_" + sheet_name + file_extension
+            try:
+                sheet_content_without_header = sorted(sheet_content_without_header,
+                                                      key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
+            except Exception:
+                pass
+            with open(output_file, 'w', newline='') as file:
+                writer = csv.writer(file, delimiter='\t')
+                writer.writerow(header_row)
+                for row_item in sheet_content_without_header:
+                    row_item.insert(0, row_num)
+                    writer.writerow(row_item)
+                    row_num += 1
+            output_files.append(output_file)
+        if output_files:
+            output = ", ".join(output_files)
     except Exception as ex:
         error_msg = str(ex)
         success = False
@@ -168,7 +109,7 @@ def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, ex
     return success, error_msg, output
 
 
-def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_header={}, cover=""):
+def write_result_to_excel(out_file_name, scan_item, extended_header={}, hide_header={}):
     success = True
     error_msg = ""
 
@@ -177,21 +118,37 @@ def write_result_to_excel(out_file_name, sheet_list, extended_header={}, hide_he
         Path(output_dir).mkdir(parents=True, exist_ok=True)
 
         workbook = xlsxwriter.Workbook(out_file_name)
-        if cover:
-            write_cover_sheet(workbook, cover)
-        if sheet_list:
-            for sheet_name, sheet_contents in sheet_list.items():
-                selected_header, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
+        write_cover_sheet(workbook, scan_item.cover)
+        if scan_item.file_items and len(scan_item.file_items.keys()) > 0:
+            for scanner_name, _ in scan_item.file_items.items():
+                sheet_name = ""
+                if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
+                    sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+                elif extended_header:
+                    sheet_name = list(extended_header.keys())[0]
+                sheet_content_without_header = scan_item.get_print_array(scanner_name)
+                selected_header = get_header_row(sheet_name, extended_header)
                 try:
                     sheet_content_without_header = sorted(sheet_content_without_header,
                                                           key=lambda x: (x[IDX_EXCLUDE], x[IDX_FILE] == "", x[IDX_FILE]))
                 except Exception:
                     pass
+                if sheet_name:
+                    worksheet = create_worksheet(workbook, sheet_name, selected_header)
+                    write_result_to_sheet(worksheet, sheet_content_without_header)
+                    if (scanner_name == FOSSLIGHT_BINARY) and (not hide_header):
+                        hide_header = BIN_HIDE_HEADER
+                    if hide_header:
+                        hide_column(worksheet, selected_header, hide_header)
+
+        for sheet_name, content in scan_item.external_sheets.items():
+            if len(content) > 0:
+                selected_header = content.pop(0)
                 worksheet = create_worksheet(workbook, sheet_name, selected_header)
-                write_result_to_sheet(worksheet, sheet_content_without_header)
-
+                write_result_to_sheet(worksheet, content)
                 if hide_header:
                     hide_column(worksheet, selected_header, hide_header)
+
         workbook.close()
     except Exception as ex:
         error_msg = str(ex)
@@ -240,37 +197,12 @@ def create_worksheet(workbook, sheet_name, header_row):
         current_time = str(time.time())
         sheet_name = current_time
     worksheet = workbook.add_worksheet(sheet_name)
-    for col_num, value in enumerate(header_row):
-        worksheet.write(0, col_num, value)
+    if header_row:
+        for col_num, value in enumerate(header_row):
+            worksheet.write(0, col_num, value)
     return worksheet
 
 
-def merge_cover_comment(find_excel_dir, merge_files=''):
-    FIND_EXTENSION = '.xlsx'
-    merge_comment = []
-    cover_comment = ''
-    try:
-        files = os.listdir(find_excel_dir)
-
-        if len([name for name in files if name.endswith(FIND_EXTENSION)]) > 0:
-            for file in files:
-                if merge_files:
-                    if file not in merge_files:
-                        continue
-                if file.endswith(FIND_EXTENSION):
-                    file = os.path.join(find_excel_dir, file)
-                    df_excel = pd.read_excel(file, sheet_name=COVER_SHEET_NAME, index_col=0, engine='openpyxl')
-                    if not df_excel.empty:
-                        tool_name = df_excel.loc[CoverItem.tool_name_key].values[0]
-                        comment = df_excel.loc[CoverItem.comment_key].values[0]
-                        merge_comment.append(str(f"[{tool_name}] {comment}"))
-            cover_comment = '\n'.join(merge_comment)
-    except Exception as ex:
-        logger.warning(f'Fail to merge comment of Scanner info: {str(ex)}')
-
-    return cover_comment
-
-
 def merge_excels(find_excel_dir, final_out, merge_files='', cover=''):
     success = True
     msg = ""

fosslight_util/write_opossum.py

@@ -11,9 +11,8 @@ import logging
 from datetime import datetime
 from pathlib import Path
 import traceback
-from typing import Dict, Optional
-
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SOURCE
+from typing import Dict, Optional, Tuple
 
 
 PACKAGE = {
@@ -30,7 +29,7 @@ PACKAGE = {
 }
 
 _attributionConfidence = 80
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 
 
 class AttributionItem():
@@ -51,7 +50,7 @@ class AttributionItem():
             self.excludeFromNotice = False
 
         self.source_name = source_name
-        if source_name == constant.FL_DEPENDENCY:
+        if source_name == FOSSLIGHT_DEPENDENCY:
             self.preSelected = True
         else:
             self.preSelected = False
@@ -113,12 +112,12 @@ class Attribution(AttributionItem):
         dict[licenseName] = self.licenseName
         dict[preSelected] = self.preSelected
 
-        if self.source_name == constant.FL_SOURCE or constant.FL_BINARY:
+        if self.source_name == FOSSLIGHT_SOURCE or FOSSLIGHT_BINARY:
             dict[copyright] = self.copyright
             dict[packageName] = self.packageName
             dict[packageVersion] = self.packageVersion
             dict[url] = self.url
-        elif self.source_name == constant.FL_DEPENDENCY:
+        elif self.source_name == FOSSLIGHT_DEPENDENCY:
             dict[copyright] = self.copyright
             dict[packageName] = self.packageName
             dict[packageVersion] = self.packageVersion
@@ -165,7 +164,7 @@ def make_frequentlicenses():
     return frequentLicenses, success, error_msg
 
 
-def write_opossum(filename, sheet_list):
+def write_opossum(filename: str, scan_item) -> Tuple[bool, str]:
     success = True
     error_msg = ''
     dict = {}
@@ -176,7 +175,7 @@ def write_opossum(filename, sheet_list):
     _filesWithChildren_key = 'filesWithChildren'
     _attributionBreakpoints_key = 'attributionBreakpoints'
 
-    if sheet_list:
+    if scan_item:
         output_dir = os.path.dirname(filename)
         Path(output_dir).mkdir(parents=True, exist_ok=True)
 
@@ -189,14 +188,9 @@ def write_opossum(filename, sheet_list):
         filesWithChildren_list = []
         attributionBreakpoints_list = []
         try:
-            for sheet_name, sheet_contents in sheet_list.items():
-                if sheet_name in constant.supported_sheet_and_scanner.keys():
-                    scanner = constant.supported_sheet_and_scanner.get(sheet_name)
-                else:
-                    logger.warning("Not supported scanner(sheet_name):" + sheet_name)
-                    continue
-
-                ret_resources_attribution = make_resources_and_attributions(sheet_contents, scanner, resources, fc_list)
+            for scanner_name, _ in scan_item.file_items.items():
+                sheet_contents = scan_item.get_print_array(scanner_name)
+                ret_resources_attribution = make_resources_and_attributions(sheet_contents, scanner_name, resources, fc_list)
                 success, rsc, ea, ra, fl, ab = ret_resources_attribution
                 if success:
                     dict[_resources_key].update(rsc)
@@ -255,14 +249,14 @@ def make_resources_and_attributions(sheet_items, scanner, resources, fc_list):
             items = items[0:9]
             path, oss_name, oss_version, license, url, homepage, copyright, exclude, comment = items
 
-            if scanner == constant.FL_SOURCE:
+            if scanner == FOSSLIGHT_SOURCE:
                 if (os.path.join(os.sep, path) + os.sep) not in fc_list:
                     resources = make_resources(path, resources)
                 attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url)
-            elif scanner == constant.FL_BINARY:
+            elif scanner == FOSSLIGHT_BINARY:
                 resources = make_resources(path, resources)
                 attribution = Attribution(scanner, license, exclude, copyright, oss_name, oss_version, url)
-            elif scanner == constant.FL_DEPENDENCY:
+            elif scanner == FOSSLIGHT_DEPENDENCY:
                 try:
                     packageType = PACKAGE[path]
                 except Exception:

fosslight_util/write_scancodejson.py

@@ -6,59 +6,59 @@
 import logging
 import os
 import json
-import fosslight_util.constant as constant
-from fosslight_util.oss_item import OssItem
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import ScannerItem
 from typing import List
 
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 EMPTY_FILE_PATH = '-'
 
 
-def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[OssItem]):
+def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[ScannerItem]):
     json_output = {}
     json_output['headers'] = []
     json_output['summary'] = {}
     json_output['license_detections'] = []
     json_output['files'] = []
 
-    for oi in oss_list:
-        if oi.exclude:
-            continue
-        if not oi.source_name_or_path:
-            oi.source_name_or_path = EMPTY_FILE_PATH
-        for item_path in oi.source_name_or_path:
-            filtered = next(filter(lambda x: x['path'] == item_path, json_output['files']), None)
-            if filtered:
-                append_oss_item_in_filesitem(oi, filtered)
-            else:
-                json_output['files'] = add_item_in_files(oi, item_path, json_output['files'])
+    for file_items in oss_list.file_items.values():
+        for fi in file_items:
+            if fi.exclude:
+                continue
+            if fi.oss_items and (all(oss_item.exclude for oss_item in fi.oss_items)):
+                continue
+            if not fi.source_name_or_path:
+                fi.source_name_or_path = EMPTY_FILE_PATH
+            json_output['files'] = add_item_in_files(fi, json_output['files'])
+
     with open(os.path.join(output_dir, output_filename), 'w') as f:
         json.dump(json_output, f, sort_keys=False, indent=4)
 
 
-def append_oss_item_in_filesitem(item, files_item):
-    if item.is_binary:
-        files_item['is_binary'] = item.is_binary
-    if item.name or item.version or item.license or item.copyright or item.download_location or item.comment:
+def append_oss_item_in_filesitem(oss_items, files_item):
+    for oi in oss_items:
+        if oi.exclude:
+            continue
         oss_item = {}
-        oss_item['name'] = item.name
-        oss_item['version'] = item.version
-        oss_item['license'] = item.license
-        oss_item['copyright'] = item.copyright
-        oss_item['download_location'] = item.download_location
-        oss_item['comment'] = item.comment
+        oss_item['name'] = oi.name
+        oss_item['version'] = oi.version
+        oss_item['license'] = oi.license
+        oss_item['copyright'] = oi.copyright
+        oss_item['download_location'] = oi.download_location
+        oss_item['comment'] = oi.comment
         files_item['oss'].append(oss_item)
+
     return files_item
 
 
-def add_item_in_files(item, item_path, files_list):
+def add_item_in_files(file_item, files_list):
     files_item = {}
-    files_item['path'] = item_path
-    files_item['name'] = os.path.basename(item_path)
-    files_item['is_binary'] = item.is_binary
-    files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(item_path))
+    files_item['path'] = file_item.source_name_or_path
+    files_item['name'] = os.path.basename(file_item.source_name_or_path)
+    files_item['is_binary'] = file_item.is_binary
+    files_item['base_name'], files_item['extension'] = os.path.splitext(os.path.basename(file_item.source_name_or_path))
     files_item['oss'] = []
-    files_item = append_oss_item_in_filesitem(item, files_item)
+    files_item = append_oss_item_in_filesitem(file_item.oss_items, files_item)
     files_list.append(files_item)
 
     return files_list