fosslight-scanner 1.7.31__tar.gz → 2.0.1__tar.gz

{fosslight_scanner-1.7.31 → fosslight_scanner-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_scanner
-Version: 1.7.31
+Version: 2.0.1
 Summary: FOSSLight Scanner
 Home-page: https://github.com/fosslight/fosslight_scanner
 Author: LG Electronics

fosslight_scanner-2.0.1/requirements.txt

@@ -0,0 +1,11 @@
+future
+pandas
+openpyxl
+progress
+pyyaml
+beautifulsoup4
+fosslight_util>=2.0.0
+fosslight_source>=2.0.0
+fosslight_dependency>=4.0.0
+fosslight_binary>=5.0.0
+fosslight_prechecker>=4.0.0

{fosslight_scanner-1.7.31 → fosslight_scanner-2.0.1}/setup.py

@@ -15,7 +15,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_scanner',
-        version='1.7.31',
+        version='2.0.1',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Scanner',

{fosslight_scanner-1.7.31 → fosslight_scanner-2.0.1}/src/fosslight_scanner/_run_compare.py

@@ -14,7 +14,8 @@ from pathlib import Path
 from bs4 import BeautifulSoup
 import fosslight_util.constant as constant
 from fosslight_util.compare_yaml import compare_yaml
-from fosslight_util.convert_excel_to_yaml import convert_excel_to_yaml
+from fosslight_util.read_excel import read_oss_report
+from fosslight_util.parsing_yaml import parsing_yml
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 ADD = "add"
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
 
     result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
 
-    if before_ext == XLSX_EXT:
-        convert_excel_to_yaml(before_f, before_yaml)
-        convert_excel_to_yaml(after_f, after_yaml)
-    compared_result = compare_yaml(before_yaml, after_yaml)
+    before_basepath = os.path.dirname(before_f)
+    after_basepath = os.path.dirname(after_f)
+    if XLSX_EXT == before_ext:
+        before_fileitems = read_oss_report(before_f, "", before_basepath)
+    elif YAML_EXT == before_ext:
+        before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
+    if XLSX_EXT == after_ext:
+        after_fileitems = read_oss_report(after_f, after_basepath)
+    elif YAML_EXT == after_ext:
+        after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
+
+    compared_result = compare_yaml(before_fileitems, after_fileitems)
     if compared_result != '':
         count_compared_result(compared_result)
         ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)

fosslight_scanner-2.0.1/src/fosslight_scanner/common.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# FOSSLight Scanner
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+import sys
+import logging
+import shutil
+import copy
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY
+from fosslight_util.write_scancodejson import write_scancodejson
+from fosslight_util.oss_item import OssItem, FileItem
+
+logger = logging.getLogger(LOGGER_NAME)
+SRC_SHEET = 'SRC_FL_Source'
+BIN_SHEET = 'BIN_FL_Binary'
+BIN_EXT_HEADER = {
+    'BIN_FL_Binary': [
+        'ID', 'Binary Path', 'OSS Name', 'OSS Version', 'License',
+        'Download Location', 'Homepage', 'Copyright Text', 'Exclude',
+        'Comment', 'Vulnerability Link', 'TLSH', 'SHA1'
+    ]
+}
+BIN_HIDDEN_HEADER = {'TLSH', 'SHA1'}
+
+
+def copy_file(source, destination):
+    copied_file = ""
+    try:
+        shutil.copy(source, destination)
+        if os.path.isdir(destination):
+            copied_file = os.path.join(destination, os.path.basename(source))
+        else:
+            copied_file = destination
+    except Exception as ex:
+        logger.debug(f"Failed to copy {source} to {destination}: {ex}")
+        return False, copied_file
+    return True, copied_file
+
+
+def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
+    # This function will be replaced by call_analysis_api().
+    logger.info("## Start to run " + str_run_start)
+    return_value = ""
+    try:
+        if path_to_run:
+            logger.info(f"|--- Path to analyze : {path_to_run}")
+            os.chdir(output)
+            sys.argv = params
+            return_value = func()
+            os.chdir(exe_path)
+        else:
+            logger.info("Analyzing path is missing...")
+    except SystemExit:
+        pass
+    except Exception as ex:
+        logger.error(f"{str_run_start}:{ex}")
+    return return_value
+
+
+def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwargs):
+    # return_idx == -1 : Raw return value itself
+    logger.info(f"## Start to run {str_run_start}")
+    success = True
+    result = []
+    try:
+        if path_to_run:
+            logger.info(f"|--- Path to analyze : {path_to_run}")
+            result = func(*args, **kwargs)
+        else:
+            logger.info("Analyzing path is missing...")
+    except SystemExit:
+        success = False
+    except Exception as ex:
+        success = False
+        logger.error(f"{str_run_start}:{ex}")
+    try:
+        if success and result and return_idx >= 0:
+            if len(result) > return_idx:
+                result = result[return_idx]
+            else:
+                success = False
+    except Exception as ex:
+        logger.debug(f"Get return value:{ex}")
+        success = False
+    return success, result or []
+
+
+def update_oss_item(scan_item, oss_name, oss_version, download_loc):
+    for file_items in scan_item.file_items.values():
+        for file_item in file_items:
+            if file_item.oss_items:
+                for oi in file_item.oss_items:
+                    if oi.name == '' and oi.version == '' and oi.download_location == '':
+                        oi.name = oss_name
+                        oi.version = oss_version
+                        oi.download_location = download_loc
+            else:
+                file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
+    return scan_item
+
+
+def create_scancodejson(all_scan_item_origin, ui_mode_report, src_path=""):
+    success = True
+    err_msg = ''
+    root_dir = ""
+    root_strip = ""
+    try:
+        src_path = os.path.abspath(src_path)
+        parent = os.path.basename(src_path)
+        root_strip = src_path.replace(parent, "")
+        root_dir = parent
+    except Exception:
+        root_dir = ""
+
+    try:
+        all_scan_item = copy.deepcopy(all_scan_item_origin)
+        if FOSSLIGHT_DEPENDENCY in all_scan_item.file_items:
+            del all_scan_item.file_items[FOSSLIGHT_DEPENDENCY]
+        if src_path:
+            fileitems_without_oss = []
+            for root, _, files in os.walk(src_path):
+                root = root.replace(root_strip, "")
+                for file in files:
+                    fi_without_oss = FileItem('')
+                    included = False
+                    item_path = os.path.join(root, file)
+                    item_path = item_path.replace(parent + os.path.sep, '', 1)
+
+                    for file_items in all_scan_item.file_items.values():
+                        for file_item in file_items:
+                            if file_item.source_name_or_path:
+                                if file_item.source_name_or_path == item_path:
+                                    included = True
+                                    break
+                    if not included:
+                        fi_without_oss.source_name_or_path = item_path
+                        fileitems_without_oss.append(fi_without_oss)
+            if len(fileitems_without_oss) > 0:
+                all_scan_item.file_items[FOSSLIGHT_SOURCE].extend(fileitems_without_oss)
+        if root_dir:
+            for file_items in all_scan_item.file_items.values():
+                for fi in file_items:
+                    if fi.source_name_or_path:
+                        fi.source_name_or_path = os.path.join(root_dir, fi.source_name_or_path)
+        write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
+                           all_scan_item)
+    except Exception as ex:
+        err_msg = ex
+        success = False
+
+    return success, err_msg
+
+
+def correct_scanner_result(all_scan_item):
+    duplicates = False
+
+    keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
+    is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
+    if is_contained:
+        src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
+        bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
+        try:
+            remove_src_idx_list = []
+            for idx_src, src_fileitem in enumerate(src_fileitems):
+                src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path, src_fileitem.exclude)
+                dup_flag = False
+                for bin_fileitem in bin_fileitems:
+                    bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path, bin_fileitem.exclude)
+                    if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
+                        dup_flag = True
+                        src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
+                        bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
+
+                        if src_all_licenses_non_empty and bin_empty_license_exists:
+                            exclude = bin_fileitem.oss_items[0].exclude
+                            bin_fileitem.oss_items = []
+                            for src_oss_item in src_fileitem.oss_items:
+                                src_oss_item.exclude = exclude
+                                bin_fileitem.oss_items.append(src_oss_item)
+                            bin_fileitem.comment = 'Loaded from SRC OSS info'
+                if dup_flag:
+                    remove_src_idx_list.append(idx_src)
+            if remove_src_idx_list:
+                duplicates = True
+                for i in sorted(remove_src_idx_list, reverse=True):
+                    del src_fileitems[i]
+        except Exception as ex:
+            logger.warning(f"correct the scanner result:{ex}")
+
+    try:
+        if duplicates:
+            logger.info('Success to correct the src/bin scanner result')
+    except Exception as ex:
+        logger.warning(f"Corrected src/bin scanner result:{ex}")
+    return all_scan_item
+
+
+def check_exclude_dir(source_name_or_path, file_item_exclude):
+    if file_item_exclude:
+        return True
+    _exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
+    exclude = False
+
+    for exclude_dir in _exclude_dirs:
+        if exclude_dir in source_name_or_path.split(os.path.sep):
+            exclude = True
+            break
+    return exclude

{fosslight_scanner-1.7.31 → fosslight_scanner-2.0.1}/src/fosslight_scanner/fosslight_scanner.py

@@ -25,12 +25,13 @@ from fosslight_util.timer_thread import TimerThread
 import fosslight_util.constant as constant
 from fosslight_util.output_format import check_output_format
 from fosslight_prechecker._precheck import run_lint as prechecker_lint
-from fosslight_util.write_excel import merge_excels, merge_cover_comment
 from fosslight_util.cover import CoverItem
+from fosslight_util.oss_item import ScannerItem
+from fosslight_util.output_format import write_output_file
 
 from .common import (
-    copy_file, call_analysis_api, overwrite_excel,
-    merge_yamls, correct_scanner_result, create_scancodejson
+    call_analysis_api, update_oss_item,
+    correct_scanner_result, create_scancodejson
 )
 from ._run_compare import run_compare
 
@@ -56,7 +57,7 @@ SCANNER_MODE = [
 
 
 def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
-    result_list = []
+    result = []
 
     package_manager = ""
     pip_activate_cmd = ""
@@ -90,7 +91,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
     timer.start()
 
     try:
-        success, result = call_analysis_api(
+        success, scan_item = call_analysis_api(
             path_to_analyze, "Dependency Analysis",
             1, run_dependency_scanner,
             package_manager,
@@ -101,11 +102,11 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
             github_token, path_to_exclude=path_to_exclude
         )
         if success:
-            result_list = result.get('SRC_FL_Dependency')
+            result = scan_item
     except Exception as ex:
         logger.warning(f"Run dependency: {ex}")
 
-    return result_list or []
+    return result
 
 
 def source_analysis_wrapper(*args, **kwargs):
@@ -130,7 +131,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 source_time_out=120, binary_simple=False):
     final_excel_dir = output_path
     success = True
-    temp_output_fiiles = []
+    all_cover_items = []
+    all_scan_item = ScannerItem(PKG_NAME, _start_time)
     if not remove_src_data:
         success, final_excel_dir, result_log = init(output_path)
 
@@ -158,9 +160,6 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                                     -1, prechecker_lint,
                                                     abs_path, False, output_prechecker,
                                                     exclude_path=path_to_exclude)
-                success_file, copied_file = copy_file(output_prechecker, output_path)
-                if success_file:
-                    temp_output_fiiles.append(copied_file)
 
             if run_src:
                 try:
@@ -180,6 +179,9 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                                     source_print_matched_text=source_print_matched_text,
                                     source_time_out=source_time_out
                         )
+                        if success:
+                            all_scan_item.file_items.update(result[2].file_items)
+                            all_cover_items.append(result[2].cover)
 
                     else:  # Run fosslight_source by using docker image
                         src_output = os.path.join("output", output_files["SRC"])
@@ -195,16 +197,22 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                     logger.warning(f"Failed to run source analysis: {ex}")
 
             if run_bin:
-                success, _ = call_analysis_api(src_path, "Binary Analysis",
-                                               1, binary_analysis.find_binaries,
-                                               abs_path,
-                                               os.path.join(_output_dir, output_files["BIN"]),
-                                               "", db_url, binary_simple,
-                                               correct_mode, correct_fpath,
-                                               path_to_exclude=path_to_exclude)
+                success, result = call_analysis_api(src_path, "Binary Analysis",
+                                                    1, binary_analysis.find_binaries,
+                                                    abs_path,
+                                                    os.path.join(_output_dir, output_files["BIN"]),
+                                                    "", db_url, binary_simple,
+                                                    correct_mode, correct_fpath,
+                                                    path_to_exclude=path_to_exclude)
+                if success:
+                    all_scan_item.file_items.update(result.file_items)
+                    all_cover_items.append(result.cover)
 
             if run_dep:
-                run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments, path_to_exclude)
+                dep_scanitem = run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
+                                              dep_arguments, path_to_exclude)
+                all_scan_item.file_items.update(dep_scanitem.file_items)
+                all_cover_items.append(dep_scanitem.cover)
 
         else:
             return
@@ -215,46 +223,23 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
     try:
         output_file_without_ext = os.path.join(final_excel_dir, output_file)
         final_report = f"{output_file_without_ext}{output_extension}"
-        merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
         cover = CoverItem(tool_name=PKG_NAME,
                           start_time=_start_time,
                           input_path=abs_path,
                           exclude_path=path_to_exclude,
                           simple_mode=False)
-        cover.comment = merge_cover_comment(_output_dir, merge_files)
-
-        if output_extension == ".xlsx":
-            tmp_dir = f"tmp_{datetime.now().strftime('%y%m%d_%H%M')}"
-            exist_src = False
-            exist_bin = False
-            if correct_mode:
-                os.makedirs(os.path.join(_output_dir, tmp_dir), exist_ok=True)
-                if os.path.exists(os.path.join(_output_dir, output_files['SRC'])):
-                    exist_src = True
-                    shutil.copy2(os.path.join(_output_dir, output_files['SRC']), os.path.join(_output_dir, tmp_dir))
-                if os.path.exists(os.path.join(_output_dir, output_files['BIN'])):
-                    exist_bin = True
-                    shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
-                if exist_src or exist_bin:
-                    correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
-
-            if remove_src_data:
-                overwrite_excel(_output_dir, default_oss_name, "OSS Name")
-                overwrite_excel(_output_dir, default_oss_version, "OSS Version")
-                overwrite_excel(_output_dir, url, "Download Location")
-            success, err_msg = merge_excels(_output_dir, final_report, merge_files, cover)
-
-            if correct_mode:
-                if exist_src:
-                    shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
-                                os.path.join(_output_dir, output_files['SRC']))
-                if exist_bin:
-                    shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
-                                os.path.join(_output_dir, output_files['BIN']))
-                shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
-        elif output_extension == ".yaml":
-            success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
-                                           remove_src_data, default_oss_name, default_oss_version, url)
+        merge_comment = []
+        for ci in all_cover_items:
+            merge_comment.append(str(f'[{ci.tool_name}] {ci.comment}'))
+        cover.comment = '\n'.join(merge_comment)
+        all_scan_item.cover = cover
+
+        if correct_mode:
+            all_scan_item = correct_scanner_result(all_scan_item)
+
+        if remove_src_data:
+            all_scan_item = update_oss_item(all_scan_item, default_oss_name, default_oss_version, url)
+        success, err_msg, final_report = write_output_file(output_file_without_ext, output_extension, all_scan_item)
         if success:
             if os.path.isfile(final_report):
                 logger.info(f'Generated the result file: {final_report}')
@@ -266,7 +251,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
 
         if ui_mode:
             ui_mode_report = f"{output_file_without_ext}.json"
-            success, err_msg = create_scancodejson(final_report, output_extension, ui_mode_report, src_path)
+            success, err_msg = create_scancodejson(all_scan_item, ui_mode_report, src_path)
             if success and os.path.isfile(ui_mode_report):
                 logger.info(f'Generated the ui mode result file: {ui_mode_report}')
             else:

{fosslight_scanner-1.7.31 → fosslight_scanner-2.0.1}/src/fosslight_scanner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-scanner
-Version: 1.7.31
+Version: 2.0.1
 Summary: FOSSLight Scanner
 Home-page: https://github.com/fosslight/fosslight_scanner
 Author: LG Electronics

fosslight_scanner-2.0.1/src/fosslight_scanner.egg-info/requires.txt

@@ -0,0 +1,11 @@
+future
+pandas
+openpyxl
+progress
+pyyaml
+beautifulsoup4
+fosslight_util>=2.0.0
+fosslight_source>=2.0.0
+fosslight_dependency>=4.0.0
+fosslight_binary>=5.0.0
+fosslight_prechecker>=4.0.0

fosslight_scanner-1.7.31/requirements.txt

@@ -1,11 +0,0 @@
-future
-pandas
-openpyxl
-progress
-pyyaml
-beautifulsoup4
-fosslight_util~=1.4.48
-fosslight_source~=1.7.8
-fosslight_dependency~=3.15.1
-fosslight_binary~=4.1.30
-fosslight_prechecker==3.0.27

fosslight_scanner-1.7.31/src/fosslight_scanner/common.py

@@ -1,298 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# FOSSLight Scanner
-# Copyright (c) 2020 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-import os
-import sys
-import logging
-import shutil
-import pandas as pd
-import yaml
-
-import fosslight_util.constant as constant
-from fosslight_util.parsing_yaml import parsing_yml
-from fosslight_util.write_yaml import create_yaml_with_ossitem
-from fosslight_util.write_scancodejson import write_scancodejson
-from fosslight_util.read_excel import read_oss_report
-from fosslight_util.output_format import write_output_file
-from fosslight_util.oss_item import OssItem
-
-logger = logging.getLogger(constant.LOGGER_NAME)
-SRC_SHEET = 'SRC_FL_Source'
-BIN_SHEET = 'BIN_FL_Binary'
-BIN_EXT_HEADER = {
-    'BIN_FL_Binary': [
-        'ID', 'Binary Path', 'OSS Name', 'OSS Version', 'License',
-        'Download Location', 'Homepage', 'Copyright Text', 'Exclude',
-        'Comment', 'Vulnerability Link', 'TLSH', 'SHA1'
-    ]
-}
-BIN_HIDDEN_HEADER = {'TLSH', 'SHA1'}
-
-
-def copy_file(source, destination):
-    copied_file = ""
-    try:
-        shutil.copy(source, destination)
-        if os.path.isdir(destination):
-            copied_file = os.path.join(destination, os.path.basename(source))
-        else:
-            copied_file = destination
-    except Exception as ex:
-        logger.debug(f"Failed to copy {source} to {destination}: {ex}")
-        return False, copied_file
-    return True, copied_file
-
-
-def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
-    # This function will be replaced by call_analysis_api().
-    logger.info("## Start to run " + str_run_start)
-    return_value = ""
-    try:
-        if path_to_run:
-            logger.info(f"|--- Path to analyze : {path_to_run}")
-            os.chdir(output)
-            sys.argv = params
-            return_value = func()
-            os.chdir(exe_path)
-        else:
-            logger.info("Analyzing path is missing...")
-    except SystemExit:
-        pass
-    except Exception as ex:
-        logger.error(f"{str_run_start}:{ex}")
-    return return_value
-
-
-def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwargs):
-    # return_idx == -1 : Raw return value itself
-    logger.info(f"## Start to run {str_run_start}")
-    success = True
-    result = []
-    try:
-        if path_to_run:
-            logger.info(f"|--- Path to analyze : {path_to_run}")
-            result = func(*args, **kwargs)
-        else:
-            logger.info("Analyzing path is missing...")
-    except SystemExit:
-        success = False
-    except Exception as ex:
-        success = False
-        logger.error(f"{str_run_start}:{ex}")
-    try:
-        if success and result and return_idx >= 0:
-            if len(result) > return_idx:
-                result = result[return_idx]
-            else:
-                success = False
-    except Exception as ex:
-        logger.debug(f"Get return value:{ex}")
-        success = False
-    return success, result or []
-
-
-def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
-    if oss_name:
-        try:
-            files = os.listdir(excel_file_path)
-            for file in files:
-                if file.endswith(".xlsx"):
-                    file_path = os.path.join(excel_file_path, file)
-                    excel_file = pd.ExcelFile(file_path, engine='openpyxl')
-
-                    for sheet_name in excel_file.sheet_names:
-                        try:
-                            df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl')
-                            if column_name in df.columns:
-                                updated = (df[column_name] == '') | (df[column_name].isnull())
-                                df.loc[updated, column_name] = oss_name
-                                df.to_excel(file_path, sheet_name=sheet_name, index=False)
-                        except Exception as ex:
-                            logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
-        except Exception as ex:
-            logger.debug(f"overwrite_excel:{ex}")
-
-
-def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
-                default_oss_name='', default_oss_version='', url=''):
-    success = True
-    err_msg = ''
-
-    oss_total_list = []
-    yaml_dict = {}
-    try:
-        for mf in merge_yaml_files:
-            if os.path.exists(os.path.join(_output_dir, mf)):
-                oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
-
-                if remove_src_data:
-                    existed_yaml = {}
-                    for oi in oss_list:
-                        oi.name = default_oss_name if oi.name == '' else oi.name
-                        oi.version = default_oss_version if oi.version == '' else oi.version
-                        oi.download_location = url if oi.download_location == '' else oi.download_location
-                        create_yaml_with_ossitem(oi, existed_yaml)
-                    with open(os.path.join(_output_dir, mf), 'w') as f:
-                        yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
-
-                oss_total_list.extend(oss_list)
-
-        if oss_total_list != []:
-            for oti in oss_total_list:
-                create_yaml_with_ossitem(oti, yaml_dict)
-            with open(os.path.join(_output_dir, final_report), 'w') as f:
-                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
-        else:
-            success = False
-            err_msg = "Output file is not created as no oss items detected."
-    except Exception as ex:
-        err_msg = ex
-        success = False
-
-    return success, err_msg
-
-
-def create_scancodejson(final_report, output_extension, ui_mode_report, src_path=""):
-    success = True
-    err_msg = ''
-
-    oss_total_list = []
-    root_dir = ""
-    root_strip = ""
-    try:
-        src_path = os.path.abspath(src_path)
-        parent = os.path.basename(src_path)
-        root_strip = src_path.replace(parent, "")
-        root_dir = parent
-    except Exception:
-        root_dir = ""
-
-    try:
-        item_without_oss = OssItem("")
-        oss_total_list = get_osslist(os.path.dirname(final_report), os.path.basename(final_report),
-                                     output_extension, '')
-        if src_path:
-            for root, dirs, files in os.walk(src_path):
-                root = root.replace(root_strip, "")
-                for file in files:
-                    item_path = os.path.join(root, file)
-                    item_path = item_path.replace(parent + os.path.sep, '', 1)
-                    included = any(item_path in x.source_name_or_path for x in oss_total_list)
-                    if not included:
-                        item_without_oss.source_name_or_path = item_path
-            if len(item_without_oss.source_name_or_path) > 0:
-                oss_total_list.append(item_without_oss)
-        if root_dir:
-            for oss in oss_total_list:
-                tmp_path_list = oss.source_name_or_path
-                oss.source_name_or_path = ""
-                oss.source_name_or_path = [os.path.join(root_dir, path) for path in tmp_path_list]
-
-        write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
-                           oss_total_list)
-    except Exception as ex:
-        err_msg = ex
-        success = False
-
-    return success, err_msg
-
-
-def correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin):
-    src_oss_list = []
-    bin_oss_list = []
-    duplicates = False
-
-    if exist_src:
-        src_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['SRC'], output_extension, SRC_SHEET))
-    if exist_bin:
-        bin_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['BIN'], output_extension, BIN_SHEET))
-
-    if exist_src and exist_bin:
-        try:
-            remove_src_idx_list = []
-            for idx_src, src_item in enumerate(src_oss_list):
-                dup_flag = False
-                for bin_item in bin_oss_list:
-                    if (not src_item.source_name_or_path):
-                        continue
-                    if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
-                        dup_flag = True
-                        if not bin_item.license and src_item.license:
-                            src_item.exclude = bin_item.exclude
-                            bin_item.set_sheet_item(src_item.get_print_array(constant.FL_BINARY)[0])
-                            if bin_item.comment:
-                                bin_item.comment += '/'
-                            bin_item.comment += 'Loaded from SRC OSS info'
-                if dup_flag:
-                    remove_src_idx_list.append(idx_src)
-            if remove_src_idx_list:
-                duplicates = True
-                for i in sorted(remove_src_idx_list, reverse=True):
-                    del src_oss_list[i]
-        except Exception as ex:
-            logger.warning(f"correct the scanner result:{ex}")
-
-    try:
-        if exist_src:
-            success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
-                                                         output_extension, SRC_SHEET)
-            if not success:
-                logger.warning(err_msg)
-        if exist_bin:
-            success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
-                                                         output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
-            if not success:
-                logger.warning(err_msg)
-        if duplicates:
-            logger.info('Success to correct the src/bin scanner result')
-    except Exception as ex:
-        logger.warning(f"Corrected src/bin scanner result:{ex}")
-    return
-
-
-def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
-    new_oss_list = []
-    sheet_list = {}
-    sheet_list[sheetname] = []
-
-    for src_item in oss_list:
-        scanner_name = constant.supported_sheet_and_scanner[sheetname]
-        new_oss_list.append(src_item.get_print_array(scanner_name)[0])
-    sheet_list[sheetname].extend(new_oss_list)
-    if os.path.exists(os.path.join(output_dir, output_file)):
-        os.remove(os.path.join(output_dir, output_file))
-    success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
-                                            output_extension, sheet_list, extended_hdr, hidden_hdr)
-    return success, err_msg
-
-
-def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
-    err_reason = ''
-    oss_list = []
-    oss_file_with_fullpath = os.path.join(_output_dir, output_file)
-
-    if os.path.exists(oss_file_with_fullpath):
-        if output_extension == '.xlsx':
-            oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
-        elif output_extension == '.yaml':
-            oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
-        else:
-            err_reason = f'Not supported extension: {output_extension}'
-    if err_reason:
-        logger.info(f'get_osslist: {err_reason}')
-    return oss_list
-
-
-def check_exclude_dir(oss_list):
-    _exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
-
-    for oss_item in oss_list:
-        if not oss_item.source_name_or_path:
-            continue
-        for exclude_dir in _exclude_dirs:
-            if exclude_dir in oss_item.source_name_or_path[0].split(os.path.sep):
-                oss_item.exclude = True
-                break
-    return oss_list

fosslight_scanner-1.7.31/src/fosslight_scanner.egg-info/requires.txt

@@ -1,11 +0,0 @@
-future
-pandas
-openpyxl
-progress
-pyyaml
-beautifulsoup4
-fosslight_util~=1.4.48
-fosslight_source~=1.7.8
-fosslight_dependency~=3.15.1
-fosslight_binary~=4.1.30
-fosslight_prechecker==3.0.27