fosslight-scanner 1.7.30__tar.gz → 2.0.0__tar.gz

{fosslight_scanner-1.7.30 → fosslight_scanner-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_scanner
-Version: 1.7.30
+Version: 2.0.0
 Summary: FOSSLight Scanner
 Home-page: https://github.com/fosslight/fosslight_scanner
 Author: LG Electronics

fosslight_scanner-2.0.0/requirements.txt

@@ -0,0 +1,10 @@
+future
+pandas
+openpyxl
+progress
+pyyaml
+beautifulsoup4
+fosslight_util>=2.0.0
+fosslight_source>=2.0.0
+fosslight_dependency>=4.0.0
+fosslight_binary>=5.0.0

{fosslight_scanner-1.7.30 → fosslight_scanner-2.0.0}/setup.py

@@ -15,7 +15,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_scanner',
-        version='1.7.30',
+        version='2.0.0',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Scanner',

{fosslight_scanner-1.7.30 → fosslight_scanner-2.0.0}/src/fosslight_scanner/_parse_setting.py

@@ -11,7 +11,7 @@ def parse_setting_json(data):
     dep_argument = data.get('dep_argument', '')
     output = data.get('output', '')
     format = data.get('format', '')
-    link = data.get('link', "")
+    link = data.get('link', '')
     db_url = data.get('db_url', '')
     timer = data.get('timer', False)
     raw = data.get('raw', False)
@@ -20,34 +20,49 @@ def parse_setting_json(data):
     correct_fpath = data.get('correct_fpath', '')
     ui = data.get('ui', False)
     exclude_path = data.get('exclude', [])
-
+    selected_source_scanner = data.get('selected_source_scanner', '')
+    source_write_json_file = data.get('source_write_json_file', False)
+    source_print_matched_text = data.get('source_print_matched_text', False)
+    source_time_out = data.get('source_time_out', 120)
+    binary_simple = data.get('binary_simple', False)
     str_lists = [mode, path, exclude_path]
-    strings = [dep_argument, output, format, db_url, correct_fpath, link]
-    booleans = [timer, raw, no_correction, ui]
+    strings = [
+        dep_argument, output, format, db_url,
+        correct_fpath, link, selected_source_scanner
+    ]
+    booleans = [timer, raw, no_correction, ui, source_write_json_file, source_print_matched_text, binary_simple]
+
     is_incorrect = False
 
     # check if json file is incorrect format
     for i, target in enumerate(str_lists):
-        if not (isinstance(target, list) and all(isinstance(item, str) for item in target)):
+        if not (isinstance(target, list) and
+                all(isinstance(item, str) for item in target)):
             is_incorrect = True
             str_lists[i] = []
 
     for i, target in enumerate(strings):
         if not isinstance(target, str):
             is_incorrect = True
-            str_lists[i] = ''
+            strings[i] = ''
 
     for i, target in enumerate(booleans):
         if not isinstance(target, bool):
             is_incorrect = True
-            str_lists[i] = False
+            booleans[i] = False
 
     if not isinstance(core, int):
         is_incorrect = True
         core = -1
 
+    if not isinstance(source_time_out, int):
+        is_incorrect = True
+        source_time_out = 120
+
     if is_incorrect:
         print('Ignoring some values with incorrect format in the setting file.')
 
     return mode, path, dep_argument, output, format, link, db_url, timer, \
-        raw, core, no_correction, correct_fpath, ui, exclude_path
+        raw, core, no_correction, correct_fpath, ui, exclude_path, \
+        selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
+        binary_simple

{fosslight_scanner-1.7.30 → fosslight_scanner-2.0.0}/src/fosslight_scanner/_run_compare.py

@@ -14,7 +14,8 @@ from pathlib import Path
 from bs4 import BeautifulSoup
 import fosslight_util.constant as constant
 from fosslight_util.compare_yaml import compare_yaml
-from fosslight_util.convert_excel_to_yaml import convert_excel_to_yaml
+from fosslight_util.read_excel import read_oss_report
+from fosslight_util.parsing_yaml import parsing_yml
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 ADD = "add"
@@ -255,10 +256,18 @@ def run_compare(before_f, after_f, output_path, output_file, file_ext, _start_ti
 
     result_file = get_comparison_result_filename(output_path, output_file, file_ext, _start_time)
 
-    if before_ext == XLSX_EXT:
-        convert_excel_to_yaml(before_f, before_yaml)
-        convert_excel_to_yaml(after_f, after_yaml)
-    compared_result = compare_yaml(before_yaml, after_yaml)
+    before_basepath = os.path.dirname(before_f)
+    after_basepath = os.path.dirname(after_f)
+    if XLSX_EXT == before_ext:
+        before_fileitems = read_oss_report(before_f, "", before_basepath)
+    elif YAML_EXT == before_ext:
+        before_fileitems, _, _ = parsing_yml(before_yaml, before_basepath)
+    if XLSX_EXT == after_ext:
+        after_fileitems = read_oss_report(after_f, after_basepath)
+    elif YAML_EXT == after_ext:
+        after_fileitems, _, _ = parsing_yml(after_yaml, after_basepath)
+
+    compared_result = compare_yaml(before_fileitems, after_fileitems)
     if compared_result != '':
         count_compared_result(compared_result)
         ret, result_file = write_compared_result(result_file, compared_result, file_ext, before_yaml, after_yaml)

fosslight_scanner-2.0.0/src/fosslight_scanner/cli.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2022 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import sys
+import json
+import os
+import os.path
+from argparse import ArgumentParser
+
+from ._help import print_help_msg
+from .fosslight_scanner import run_main, PKG_NAME
+from ._parse_setting import parse_setting_json
+from fosslight_util.help import print_package_version
+
+
+def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
+             raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
+
+    selected_source_scanner = "all"
+    source_write_json_file = False
+    source_print_matched_text = False
+    source_time_out = 120
+    binary_simple = False
+
+    if setting and os.path.isfile(setting):
+        try:
+            with open(setting, 'r', encoding='utf-8') as file:
+                data = json.load(file)
+            s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
+                s_no_correction, s_correct_fpath, s_ui, s_exclude_path, \
+                s_selected_source_scanner, s_source_write_json_file, s_source_print_matched_text, \
+                s_source_time_out, s_binary_simple = parse_setting_json(data)
+
+            # direct cli arguments have higher priority than setting file
+            mode = mode or s_mode
+            path = path or s_path
+            dep_argument = dep_argument or s_dep_argument
+            output = output or s_output
+            format = format or s_format
+            link = link or s_link
+            db_url = db_url or s_db_url
+            timer = timer or s_timer
+            raw = raw or s_raw
+            core = core if core != -1 else s_core
+            no_correction = no_correction or s_no_correction
+            correct_fpath = correct_fpath or s_correct_fpath
+            ui = ui or s_ui
+            exclude_path = exclude_path or s_exclude_path
+
+            # These options are only set from the setting file, not from CLI arguments
+            selected_source_scanner = s_selected_source_scanner or selected_source_scanner
+            source_write_json_file = s_source_write_json_file
+            source_print_matched_text = s_source_print_matched_text
+            source_time_out = s_source_time_out if s_source_time_out != 120 else source_time_out
+            binary_simple = s_binary_simple
+
+        except Exception as e:
+            print(f"Cannot open setting file: {e}")
+    return mode, path, dep_argument, output, format, link, db_url, timer, \
+        raw, core, no_correction, correct_fpath, ui, exclude_path, \
+        selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out, \
+        binary_simple
+
+
+def main():
+    parser = ArgumentParser(description='FOSSLight Scanner',
+                            prog='fosslight_scanner', add_help=False)
+    parser.add_argument('mode', nargs='*',
+                        help='source| dependency| binary| all| compare',
+                        default="")
+    parser.add_argument('--path', '-p',
+                        help='Path to analyze (In compare mode, two FOSSLight reports',
+                        dest='path', nargs='+', default="")
+    parser.add_argument('--wget', '-w', help='Link to be analyzed',
+                        type=str, dest='link', default="")
+    parser.add_argument('--format', '-f',
+                        help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
+                        type=str, dest='format', default="")
+    parser.add_argument('--output', '-o', help='Output directory or file',
+                        type=str, dest='output', default="")
+    parser.add_argument('--dependency', '-d', help='Dependency arguments',
+                        type=str, dest='dep_argument', default="")
+    parser.add_argument('--url', '-u', help="DB Url",
+                        type=str, dest='db_url', default="")
+    parser.add_argument('--core', '-c',
+                        help='Number of processes to analyze source',
+                        type=int, dest='core', default=-1)
+    parser.add_argument('--raw', '-r', help='Keep raw data',
+                        action='store_true', dest='raw', default=False)
+    parser.add_argument('--timer', '-t', help='Hide the progress bar',
+                        action='store_true', dest='timer', default=False)
+    parser.add_argument('--version', '-v', help='Print version',
+                        action='store_true', dest='version', default=False)
+    parser.add_argument('--help', '-h', help='Print help message',
+                        action='store_true', dest='help')
+    parser.add_argument('--exclude', '-e', help='Path to exclude from analysis',
+                        dest='exclude_path', nargs='*', default=[])
+    parser.add_argument('--setting', '-s', help='Scanner json setting file',
+                        type=str, dest='setting', default="")
+    parser.add_argument('--no_correction',
+                        help='No correction with sbom-info.yaml',
+                        action='store_true', required=False, default=False)
+    parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
+                        type=str, required=False, default='')
+    parser.add_argument('--ui', help='Generate UI mode result file',
+                        action='store_true', required=False, default=False)
+
+    try:
+        args = parser.parse_args()
+    except SystemExit:
+        sys.exit(1)
+
+    if args.help:
+        print_help_msg()
+    elif args.version:
+        print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
+    else:
+        mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
+            ui, exclude_path, selected_source_scanner, source_write_json_file, source_print_matched_text, \
+            source_time_out, binary_simple, = set_args(
+                args.mode, args.path, args.dep_argument, args.output,
+                args.format, args.link, args.db_url, args.timer, args.raw,
+                args.core, args.no_correction, args.correct_fpath, args.ui,
+                args.setting, args.exclude_path)
+
+        run_main(mode, path, dep_argument, output, format, link, db_url, timer,
+                 raw, core, not no_correction, correct_fpath, ui, exclude_path,
+                 selected_source_scanner, source_write_json_file, source_print_matched_text,
+                 source_time_out, binary_simple)
+
+
+if __name__ == "__main__":
+    main()

fosslight_scanner-2.0.0/src/fosslight_scanner/common.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# FOSSLight Scanner
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+import sys
+import logging
+import shutil
+import copy
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY
+from fosslight_util.write_scancodejson import write_scancodejson
+from fosslight_util.oss_item import OssItem, FileItem
+
+logger = logging.getLogger(LOGGER_NAME)
+SRC_SHEET = 'SRC_FL_Source'
+BIN_SHEET = 'BIN_FL_Binary'
+BIN_EXT_HEADER = {
+    'BIN_FL_Binary': [
+        'ID', 'Binary Path', 'OSS Name', 'OSS Version', 'License',
+        'Download Location', 'Homepage', 'Copyright Text', 'Exclude',
+        'Comment', 'Vulnerability Link', 'TLSH', 'SHA1'
+    ]
+}
+BIN_HIDDEN_HEADER = {'TLSH', 'SHA1'}
+
+
+def copy_file(source, destination):
+    copied_file = ""
+    try:
+        shutil.copy(source, destination)
+        if os.path.isdir(destination):
+            copied_file = os.path.join(destination, os.path.basename(source))
+        else:
+            copied_file = destination
+    except Exception as ex:
+        logger.debug(f"Failed to copy {source} to {destination}: {ex}")
+        return False, copied_file
+    return True, copied_file
+
+
+def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
+    # This function will be replaced by call_analysis_api().
+    logger.info("## Start to run " + str_run_start)
+    return_value = ""
+    try:
+        if path_to_run:
+            logger.info(f"|--- Path to analyze : {path_to_run}")
+            os.chdir(output)
+            sys.argv = params
+            return_value = func()
+            os.chdir(exe_path)
+        else:
+            logger.info("Analyzing path is missing...")
+    except SystemExit:
+        pass
+    except Exception as ex:
+        logger.error(f"{str_run_start}:{ex}")
+    return return_value
+
+
+def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwargs):
+    # return_idx == -1 : Raw return value itself
+    logger.info(f"## Start to run {str_run_start}")
+    success = True
+    result = []
+    try:
+        if path_to_run:
+            logger.info(f"|--- Path to analyze : {path_to_run}")
+            result = func(*args, **kwargs)
+        else:
+            logger.info("Analyzing path is missing...")
+    except SystemExit:
+        success = False
+    except Exception as ex:
+        success = False
+        logger.error(f"{str_run_start}:{ex}")
+    try:
+        if success and result and return_idx >= 0:
+            if len(result) > return_idx:
+                result = result[return_idx]
+            else:
+                success = False
+    except Exception as ex:
+        logger.debug(f"Get return value:{ex}")
+        success = False
+    return success, result or []
+
+
+def update_oss_item(scan_item, oss_name, oss_version, download_loc):
+    for file_items in scan_item.file_items.values():
+        for file_item in file_items:
+            if file_item.oss_items:
+                for oi in file_item.oss_items:
+                    if oi.name == '' and oi.version == '' and oi.download_location == '':
+                        oi.name = oss_name
+                        oi.version = oss_version
+                        oi.download_location = download_loc
+            else:
+                file_item.oss_items.append(OssItem(oss_name, oss_version, '', download_loc))
+    return scan_item
+
+
+def create_scancodejson(all_scan_item_origin, ui_mode_report, src_path=""):
+    success = True
+    err_msg = ''
+    root_dir = ""
+    root_strip = ""
+    try:
+        src_path = os.path.abspath(src_path)
+        parent = os.path.basename(src_path)
+        root_strip = src_path.replace(parent, "")
+        root_dir = parent
+    except Exception:
+        root_dir = ""
+
+    try:
+        all_scan_item = copy.deepcopy(all_scan_item_origin)
+        if FOSSLIGHT_DEPENDENCY in all_scan_item.file_items:
+            del all_scan_item.file_items[FOSSLIGHT_DEPENDENCY]
+        if src_path:
+            fileitems_without_oss = []
+            for root, _, files in os.walk(src_path):
+                root = root.replace(root_strip, "")
+                for file in files:
+                    fi_without_oss = FileItem('')
+                    included = False
+                    item_path = os.path.join(root, file)
+                    item_path = item_path.replace(parent + os.path.sep, '', 1)
+
+                    for file_items in all_scan_item.file_items.values():
+                        for file_item in file_items:
+                            if file_item.source_name_or_path:
+                                if file_item.source_name_or_path == item_path:
+                                    included = True
+                                    break
+                    if not included:
+                        fi_without_oss.source_name_or_path = item_path
+                        fileitems_without_oss.append(fi_without_oss)
+            if len(fileitems_without_oss) > 0:
+                all_scan_item.file_items[FOSSLIGHT_SOURCE].extend(fileitems_without_oss)
+        if root_dir:
+            for file_items in all_scan_item.file_items.values():
+                for fi in file_items:
+                    if fi.source_name_or_path:
+                        fi.source_name_or_path = os.path.join(root_dir, fi.source_name_or_path)
+        write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
+                           all_scan_item)
+    except Exception as ex:
+        err_msg = ex
+        success = False
+
+    return success, err_msg
+
+
+def correct_scanner_result(all_scan_item):
+    duplicates = False
+
+    keys_needed = {FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY}
+    is_contained = keys_needed.issubset(all_scan_item.file_items.keys())
+    if is_contained:
+        src_fileitems = all_scan_item.file_items[FOSSLIGHT_SOURCE]
+        bin_fileitems = all_scan_item.file_items[FOSSLIGHT_BINARY]
+        try:
+            remove_src_idx_list = []
+            for idx_src, src_fileitem in enumerate(src_fileitems):
+                src_fileitem.exclude = check_exclude_dir(src_fileitem.source_name_or_path, src_fileitem.exclude)
+                dup_flag = False
+                for bin_fileitem in bin_fileitems:
+                    bin_fileitem.exclude = check_exclude_dir(bin_fileitem.source_name_or_path, bin_fileitem.exclude)
+                    if src_fileitem.source_name_or_path == bin_fileitem.source_name_or_path:
+                        dup_flag = True
+                        src_all_licenses_non_empty = all(oss_item.license for oss_item in src_fileitem.oss_items)
+                        bin_empty_license_exists = all(not oss_item.license for oss_item in bin_fileitem.oss_items)
+
+                        if src_all_licenses_non_empty and bin_empty_license_exists:
+                            exclude = bin_fileitem.oss_items[0].exclude
+                            bin_fileitem.oss_items = []
+                            for src_oss_item in src_fileitem.oss_items:
+                                src_oss_item.exclude = exclude
+                                bin_fileitem.oss_items.append(src_oss_item)
+                            bin_fileitem.comment = 'Loaded from SRC OSS info'
+                if dup_flag:
+                    remove_src_idx_list.append(idx_src)
+            if remove_src_idx_list:
+                duplicates = True
+                for i in sorted(remove_src_idx_list, reverse=True):
+                    del src_fileitems[i]
+        except Exception as ex:
+            logger.warning(f"correct the scanner result:{ex}")
+
+    try:
+        if duplicates:
+            logger.info('Success to correct the src/bin scanner result')
+    except Exception as ex:
+        logger.warning(f"Corrected src/bin scanner result:{ex}")
+    return all_scan_item
+
+
+def check_exclude_dir(source_name_or_path, file_item_exclude):
+    if file_item_exclude:
+        return True
+    _exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
+    exclude = False
+
+    for exclude_dir in _exclude_dirs:
+        if exclude_dir in source_name_or_path.split(os.path.sep):
+            exclude = True
+            break
+    return exclude

{fosslight_scanner-1.7.30 → fosslight_scanner-2.0.0}/src/fosslight_scanner/fosslight_scanner.py

@@ -4,16 +4,17 @@
 # Copyright (c) 2020 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import os
+import sys
+import re
 import logging
 import warnings
-import re
 import yaml
-import sys
 import shutil
 import shlex
 import subprocess
 from pathlib import Path
 from datetime import datetime
+
 from fosslight_binary import binary_analysis
 from fosslight_dependency.run_dependency_scanner import run_dependency_scanner
 from fosslight_util.download import cli_download_and_extract, compression_extension
@@ -23,14 +24,16 @@ from fosslight_util.set_log import init_log
 from fosslight_util.timer_thread import TimerThread
 import fosslight_util.constant as constant
 from fosslight_util.output_format import check_output_format
-from fosslight_prechecker._precheck import run_lint as prechecker_lint
-from .common import (copy_file, call_analysis_api,
-                     overwrite_excel,
-                     merge_yamls, correct_scanner_result,
-                     create_scancodejson)
-from fosslight_util.write_excel import merge_excels, merge_cover_comment
-from ._run_compare import run_compare
 from fosslight_util.cover import CoverItem
+from fosslight_util.oss_item import ScannerItem
+from fosslight_util.output_format import write_output_file
+
+from .common import (
+    call_analysis_api, update_oss_item,
+    correct_scanner_result, create_scancodejson
+)
+from ._run_compare import run_compare
+
 fosslight_source_installed = True
 try:
     from fosslight_source.cli import run_scanners as source_analysis
@@ -46,11 +49,14 @@ _log_file = "fosslight_log_all_"
 _start_time = ""
 _executed_path = ""
 SRC_DIR_FROM_LINK_PREFIX = "fosslight_src_dir_"
-SCANNER_MODE = ["all", "compare", "reuse", "prechecker", "binary", "bin", "src", "source", "dependency", "dep"]
+SCANNER_MODE = [
+    "all", "compare", "binary",
+    "bin", "src", "source", "dependency", "dep"
+]
 
 
 def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]):
-    result_list = []
+    result = []
 
     package_manager = ""
     pip_activate_cmd = ""
@@ -60,7 +66,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
     github_token = ""
 
     try:
-        if params != "":
+        if params:
             match_obj = re.findall(
                 r'\s*(-\s*[a|d|m|c|n|t])\s*\'([^\']+)\'\s*', params)
             for param, value in match_obj:
@@ -84,22 +90,34 @@ def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_ex
     timer.start()
 
     try:
-        success, result = call_analysis_api(path_to_analyze, "Dependency Analysis",
-                                            1, run_dependency_scanner,
-                                            package_manager,
-                                            os.path.abspath(path_to_analyze),
-                                            output_file_with_path,
-                                            pip_activate_cmd, pip_deactivate_cmd,
-                                            output_custom_dir, app_name,
-                                            github_token, path_to_exclude=path_to_exclude)
+        success, scan_item = call_analysis_api(
+            path_to_analyze, "Dependency Analysis",
+            1, run_dependency_scanner,
+            package_manager,
+            os.path.abspath(path_to_analyze),
+            output_file_with_path,
+            pip_activate_cmd, pip_deactivate_cmd,
+            output_custom_dir, app_name,
+            github_token, path_to_exclude=path_to_exclude
+        )
         if success:
-            result_list = result.get('SRC_FL_Dependency')
+            result = scan_item
     except Exception as ex:
         logger.warning(f"Run dependency: {ex}")
 
-    if not result_list:
-        result_list = []
-    return result_list
+    return result
+
+
+def source_analysis_wrapper(*args, **kwargs):
+    selected_scanner = kwargs.pop('selected_scanner', 'all')
+    source_write_json_file = kwargs.pop('source_write_json_file', False)
+    source_print_matched_text = kwargs.pop('source_print_matched_text', False)
+    source_time_out = kwargs.pop('source_time_out', 120)
+    args = list(args)
+    args.insert(2, source_write_json_file)
+    args.insert(5, source_print_matched_text)
+
+    return source_analysis(*args, selected_scanner=selected_scanner, time_out=source_time_out, **kwargs)
 
 
 def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
@@ -107,10 +125,13 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                 remove_src_data=True, result_log={}, output_file="",
                 output_extension="", num_cores=-1, db_url="",
                 default_oss_name="", default_oss_version="", url="",
-                correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]):
+                correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
+                selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
+                source_time_out=120, binary_simple=False):
     final_excel_dir = output_path
     success = True
-    temp_output_fiiles = []
+    all_cover_items = []
+    all_scan_item = ScannerItem(PKG_NAME, _start_time)
     if not remove_src_data:
         success, final_excel_dir, result_log = init(output_path)
 
@@ -130,28 +151,30 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
         if success:
             output_files = {"SRC": f"fosslight_src_{_start_time}{output_extension}",
                             "BIN": f"fosslight_bin_{_start_time}{output_extension}",
-                            "DEP": f"fosslight_dep_{_start_time}{output_extension}",
-                            "PRECHECKER": f"fosslight_lint_{_start_time}.yaml"}
-            if run_prechecker:
-                output_prechecker = os.path.join(_output_dir, output_files["PRECHECKER"])
-                success, result = call_analysis_api(src_path, "Prechecker Lint",
-                                                    -1, prechecker_lint,
-                                                    abs_path, False, output_prechecker,
-                                                    exclude_path=path_to_exclude)
-                success_file, copied_file = copy_file(output_prechecker, output_path)
-                if success_file:
-                    temp_output_fiiles.append(copied_file)
+                            "DEP": f"fosslight_dep_{_start_time}{output_extension}"}
 
             if run_src:
                 try:
                     if fosslight_source_installed:
                         src_output = os.path.join(_output_dir, output_files["SRC"])
-                        success, result = call_analysis_api(src_path, "Source Analysis",
-                                                            -1, source_analysis,
-                                                            abs_path,
-                                                            src_output,
-                                                            False, num_cores, False,
-                                                            path_to_exclude=path_to_exclude)
+                        success, result = call_analysis_api(
+                                    src_path,
+                                    "Source Analysis",
+                                    -1, source_analysis_wrapper,
+                                    abs_path,
+                                    src_output,
+                                    num_cores,
+                                    False,
+                                    path_to_exclude=path_to_exclude,
+                                    selected_scanner=selected_source_scanner,
+                                    source_write_json_file=source_write_json_file,
+                                    source_print_matched_text=source_print_matched_text,
+                                    source_time_out=source_time_out
+                        )
+                        if success:
+                            all_scan_item.file_items.update(result[2].file_items)
+                            all_cover_items.append(result[2].cover)
+
                     else:  # Run fosslight_source by using docker image
                         src_output = os.path.join("output", output_files["SRC"])
                         output_rel_path = os.path.relpath(abs_path, os.getcwd())
@@ -166,16 +189,22 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
                     logger.warning(f"Failed to run source analysis: {ex}")
 
             if run_bin:
-                success, _ = call_analysis_api(src_path, "Binary Analysis",
-                                               1, binary_analysis.find_binaries,
-                                               abs_path,
-                                               os.path.join(_output_dir, output_files["BIN"]),
-                                               "", db_url, False,
-                                               correct_mode, correct_fpath,
-                                               path_to_exclude=path_to_exclude)
+                success, result = call_analysis_api(src_path, "Binary Analysis",
+                                                    1, binary_analysis.find_binaries,
+                                                    abs_path,
+                                                    os.path.join(_output_dir, output_files["BIN"]),
+                                                    "", db_url, binary_simple,
+                                                    correct_mode, correct_fpath,
+                                                    path_to_exclude=path_to_exclude)
+                if success:
+                    all_scan_item.file_items.update(result.file_items)
+                    all_cover_items.append(result.cover)
 
             if run_dep:
-                run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments, path_to_exclude)
+                dep_scanitem = run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]),
+                                              dep_arguments, path_to_exclude)
+                all_scan_item.file_items.update(dep_scanitem.file_items)
+                all_cover_items.append(dep_scanitem.cover)
 
         else:
             return
@@ -186,45 +215,23 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
     try:
         output_file_without_ext = os.path.join(final_excel_dir, output_file)
         final_report = f"{output_file_without_ext}{output_extension}"
-        merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]]
         cover = CoverItem(tool_name=PKG_NAME,
                           start_time=_start_time,
                           input_path=abs_path,
-                          exclude_path=path_to_exclude)
-        cover.comment = merge_cover_comment(_output_dir, merge_files)
-
-        if output_extension == ".xlsx":
-            tmp_dir = f"tmp_{datetime.now().strftime('%y%m%d_%H%M')}"
-            exist_src = False
-            exist_bin = False
-            if correct_mode:
-                os.makedirs(os.path.join(_output_dir, tmp_dir), exist_ok=True)
-                if os.path.exists(os.path.join(_output_dir, output_files['SRC'])):
-                    exist_src = True
-                    shutil.copy2(os.path.join(_output_dir, output_files['SRC']), os.path.join(_output_dir, tmp_dir))
-                if os.path.exists(os.path.join(_output_dir, output_files['BIN'])):
-                    exist_bin = True
-                    shutil.copy2(os.path.join(_output_dir, output_files['BIN']), os.path.join(_output_dir, tmp_dir))
-                if exist_src or exist_bin:
-                    correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin)
-
-            if remove_src_data:
-                overwrite_excel(_output_dir, default_oss_name, "OSS Name")
-                overwrite_excel(_output_dir, default_oss_version, "OSS Version")
-                overwrite_excel(_output_dir, url, "Download Location")
-            success, err_msg = merge_excels(_output_dir, final_report, merge_files, cover)
-
-            if correct_mode:
-                if exist_src:
-                    shutil.move(os.path.join(_output_dir, tmp_dir, output_files['SRC']),
-                                os.path.join(_output_dir, output_files['SRC']))
-                if exist_bin:
-                    shutil.move(os.path.join(_output_dir, tmp_dir, output_files['BIN']),
-                                os.path.join(_output_dir, output_files['BIN']))
-                shutil.rmtree(os.path.join(_output_dir, tmp_dir), ignore_errors=True)
-        elif output_extension == ".yaml":
-            success, err_msg = merge_yamls(_output_dir, merge_files, final_report,
-                                           remove_src_data, default_oss_name, default_oss_version, url)
+                          exclude_path=path_to_exclude,
+                          simple_mode=False)
+        merge_comment = []
+        for ci in all_cover_items:
+            merge_comment.append(str(f'[{ci.tool_name}] {ci.comment}'))
+        cover.comment = '\n'.join(merge_comment)
+        all_scan_item.cover = cover
+
+        if correct_mode:
+            all_scan_item = correct_scanner_result(all_scan_item)
+
+        if remove_src_data:
+            all_scan_item = update_oss_item(all_scan_item, default_oss_name, default_oss_version, url)
+        success, err_msg, final_report = write_output_file(output_file_without_ext, output_extension, all_scan_item)
         if success:
             if os.path.isfile(final_report):
                 logger.info(f'Generated the result file: {final_report}')
@@ -236,7 +243,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False,
 
         if ui_mode:
             ui_mode_report = f"{output_file_without_ext}.json"
-            success, err_msg = create_scancodejson(final_report, output_extension, ui_mode_report, src_path)
+            success, err_msg = create_scancodejson(all_scan_item, ui_mode_report, src_path)
             if success and os.path.isfile(ui_mode_report):
                 logger.info(f'Generated the ui mode result file: {ui_mode_report}')
             else:
@@ -308,7 +315,9 @@ def init(output_path="", make_outdir=True):
 
 def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze,
              db_url, hide_progressbar=False, keep_raw_data=False, num_cores=-1,
-             correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]):
+             correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[],
+             selected_source_scanner="all", source_write_json_file=False, source_print_matched_text=False,
+             source_time_out=120, binary_simple=False):
     global _executed_path, _start_time
 
     output_file = ""
@@ -380,19 +389,13 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
             run_src = False
             run_bin = False
             run_dep = False
-            run_prechecker = False
             remove_downloaded_source = False
 
             if "all" in mode_list or (not mode_list):
                 run_src = True
                 run_bin = True
                 run_dep = True
-                run_prechecker = False
-                if "prechecker" in mode_list or "reuse" in mode_list:
-                    run_prechecker = True
             else:
-                if "prechecker" in mode_list or "reuse" in mode_list:
-                    run_prechecker = True
                 if "binary" in mode_list or "bin" in mode_list:
                     run_bin = True
                 if "source" in mode_list or "src" in mode_list:
@@ -400,7 +403,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
                 if "dependency" in mode_list or "dep" in mode_list:
                     run_dep = True
 
-            if run_dep or run_src or run_bin or run_prechecker:
+            if run_dep or run_src or run_bin:
                 if src_path == "" and url_to_analyze == "":
                     src_path, dep_arguments, url_to_analyze = get_input_mode(_executed_path, mode_list)
 
@@ -422,11 +425,13 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format
 
                 if src_path != "":
                     run_scanner(src_path, dep_arguments, output_path, keep_raw_data,
-                                run_src, run_bin, run_dep, run_prechecker,
+                                run_src, run_bin, run_dep, '',
                                 remove_downloaded_source, {}, output_file,
                                 output_extension, num_cores, db_url,
                                 default_oss_name, default_oss_version, url_to_analyze,
-                                correct_mode, correct_fpath, ui_mode, path_to_exclude)
+                                correct_mode, correct_fpath, ui_mode, path_to_exclude,
+                                selected_source_scanner, source_write_json_file, source_print_matched_text, source_time_out,
+                                binary_simple)
 
                 if extract_folder:
                     shutil.rmtree(extract_folder)

{fosslight_scanner-1.7.30 → fosslight_scanner-2.0.0}/src/fosslight_scanner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-scanner
-Version: 1.7.30
+Version: 2.0.0
 Summary: FOSSLight Scanner
 Home-page: https://github.com/fosslight/fosslight_scanner
 Author: LG Electronics

fosslight_scanner-2.0.0/src/fosslight_scanner.egg-info/requires.txt

@@ -0,0 +1,10 @@
+future
+pandas
+openpyxl
+progress
+pyyaml
+beautifulsoup4
+fosslight_util>=2.0.0
+fosslight_source>=2.0.0
+fosslight_dependency>=4.0.0
+fosslight_binary>=5.0.0

fosslight_scanner-1.7.30/requirements.txt

@@ -1,11 +0,0 @@
-future
-pandas
-openpyxl
-progress
-pyyaml
-beautifulsoup4
-fosslight_util>=1.4.45
-fosslight_source>=1.7.8
-fosslight_dependency>=3.15.1
-fosslight_binary>=4.1.30
-fosslight_prechecker>=3.0.27

fosslight_scanner-1.7.30/src/fosslight_scanner/cli.py

@@ -1,90 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2022 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-import sys
-import json
-from argparse import ArgumentParser
-from ._help import print_help_msg
-from .fosslight_scanner import run_main, PKG_NAME
-from ._parse_setting import parse_setting_json
-from fosslight_util.help import print_package_version
-import os.path
-
-
-def set_args(mode, path, dep_argument, output, format, link, db_url, timer,
-             raw, core, no_correction, correct_fpath, ui, setting, exclude_path):
-    if setting and os.path.isfile(setting):
-        try:
-            with open(setting, 'r', encoding='utf-8') as file:
-                data = json.load(file)
-            s_mode, s_path, s_dep_argument, s_output, s_format, s_link, s_db_url, s_timer, s_raw, s_core, \
-                s_no_correction, s_correct_fpath, s_ui, s_exclude_path = parse_setting_json(data)
-
-            # direct cli arguments have higher priority than setting file
-            mode = mode if mode else s_mode
-            path = path if path else s_path
-            dep_argument = dep_argument if dep_argument else s_dep_argument
-            output = output if output else s_output
-            format = format if format else s_format
-            link = link if link else s_link
-            db_url = db_url if db_url else s_db_url
-            timer = timer if timer else s_timer
-            raw = raw if raw else s_raw
-            core = core if core else s_core
-            no_correction = no_correction if no_correction else s_no_correction
-            correct_fpath = correct_fpath if correct_fpath else s_correct_fpath
-            ui = ui if ui else s_ui
-            exclude_path = exclude_path if exclude_path else s_exclude_path
-
-        except Exception as e:
-            print(f"Cannot open setting file: {e}")
-    return mode, path, dep_argument, output, format, link, db_url, timer, \
-        raw, core, no_correction, correct_fpath, ui, exclude_path
-
-
-def main():
-    parser = ArgumentParser(description='FOSSLight Scanner', prog='fosslight_scanner', add_help=False)
-    parser.add_argument('mode', nargs='*', help='source| dependency| binary| all| compare', default="")
-    parser.add_argument('--path', '-p', help='Path to analyze (In compare mode, two FOSSLight reports',
-                        dest='path', nargs='+', default="")
-    parser.add_argument('--wget', '-w', help='Link to be analyzed', type=str, dest='link', default="")
-    parser.add_argument('--format', '-f', help='Scanner output file format (excel,yaml), Compare mode (excel,html,yaml,json)',
-                        type=str, dest='format', default="")
-    parser.add_argument('--output', '-o', help='Output directory or file', type=str, dest='output', default="")
-    parser.add_argument('--dependency', '-d', help='Dependency arguments', type=str, dest='dep_argument', default="")
-    parser.add_argument('--url', '-u', help="DB Url", type=str, dest='db_url', default="")
-    parser.add_argument('--core', '-c', help='Number of processes to analyze source', type=int, dest='core', default=-1)
-    parser.add_argument('--raw', '-r', help='Keep raw data', action='store_true', dest='raw', default=False)
-    parser.add_argument('--timer', '-t', help='Hide the progress bar', action='store_true', dest='timer', default=False)
-    parser.add_argument('--version', '-v', help='Print version', action='store_true', dest='version', default=False)
-    parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help')
-    parser.add_argument('--exclude', '-e', help='Path to exclude from analysis', dest='exclude_path', nargs='*', default=[])
-    parser.add_argument('--setting', '-s', help='Scanner json setting file', type=str, dest='setting', default="")
-    parser.add_argument('--no_correction', help='No correction with sbom-info.yaml',
-                        action='store_true', required=False, default=False)
-    parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml',
-                        type=str, required=False, default='')
-    parser.add_argument('--ui', help='Generate UI mode result file', action='store_true', required=False, default=False)
-
-    try:
-        args = parser.parse_args()
-    except SystemExit:
-        sys.exit(1)
-
-    if args.help:
-        print_help_msg()
-    elif args.version:
-        print_package_version(PKG_NAME, "FOSSLight Scanner Version:")
-    else:
-        mode, path, dep_argument, output, format, link, db_url, timer, raw, core, no_correction, correct_fpath, \
-          ui, exclude_path = set_args(args.mode, args.path, args.dep_argument, args.output, args.format,
-                                      args.link, args.db_url, args.timer, args.raw, args.core, args.no_correction,
-                                      args.correct_fpath, args.ui, args.setting, args.exclude_path)
-
-        run_main(mode, path, dep_argument, output, format, link, db_url, timer,
-                 raw, core, not no_correction, correct_fpath, ui, exclude_path)
-
-
-if __name__ == "__main__":
-    main()

fosslight_scanner-1.7.30/src/fosslight_scanner/common.py

@@ -1,298 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# FOSSLight Scanner
-# Copyright (c) 2020 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-import os
-import sys
-import logging
-import shutil
-import pandas as pd
-import yaml
-import fosslight_util.constant as constant
-from fosslight_util.parsing_yaml import parsing_yml
-from fosslight_util.write_yaml import create_yaml_with_ossitem
-from fosslight_util.write_scancodejson import write_scancodejson
-from fosslight_util.read_excel import read_oss_report
-from fosslight_util.output_format import write_output_file
-from fosslight_util.oss_item import OssItem
-
-logger = logging.getLogger(constant.LOGGER_NAME)
-SRC_SHEET = 'SRC_FL_Source'
-BIN_SHEET = 'BIN_FL_Binary'
-BIN_EXT_HEADER = {'BIN_FL_Binary': ['ID', 'Binary Path', 'OSS Name',
-                                    'OSS Version', 'License', 'Download Location',
-                                    'Homepage', 'Copyright Text', 'Exclude',
-                                    'Comment', 'Vulnerability Link', 'TLSH', 'SHA1']}
-BIN_HIDDEN_HEADER = {'TLSH', "SHA1"}
-
-
-def copy_file(source, destination):
-    copied_file = ""
-    try:
-        shutil.copy(source, destination)
-        if os.path.isdir(destination):
-            copied_file = os.path.join(destination, os.path.basename(source))
-        else:
-            copied_file = destination
-    except Exception as ex:
-        logger.debug(f"Failed to copy {source} to {destination}: {ex}")
-        return False, copied_file
-    else:
-        return True, copied_file
-
-
-def run_analysis(path_to_run, params, func, str_run_start, output, exe_path):
-    # This function will be replaced by call_analysis_api().
-    logger.info("## Start to run "+str_run_start)
-    return_value = ""
-    try:
-        if path_to_run != "":
-            logger.info(f"|--- Path to analyze : {path_to_run}")
-            os.chdir(output)
-            sys.argv = params
-            return_value = func()
-            os.chdir(exe_path)
-        else:
-            logger.info("Analyzing path is missing...")
-    except SystemExit:
-        pass
-    except Exception as ex:
-        logger.error(f"{str_run_start}:{ex}")
-    return return_value
-
-
-def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwargs):
-    # return_idx == -1 : Raw return value itself
-    logger.info(f"## Start to run {str_run_start}")
-    success = True
-    result = []
-    try:
-        if path_to_run != "":
-            logger.info(f"|--- Path to analyze : {path_to_run}")
-            result = func(*args, **kwargs)
-        else:
-            logger.info("Analyzing path is missing...")
-    except SystemExit:
-        success = False
-    except Exception as ex:
-        success = False
-        logger.error(f"{str_run_start}:{ex}")
-    try:
-        if success:
-            if result and return_idx >= 0:
-                if len(result) > return_idx:
-                    result = result[return_idx]
-                else:
-                    success = False
-    except Exception as ex:
-        logger.debug(f"Get return value:{ex}")
-        success = False
-    if not result:
-        result = []
-    return success, result
-
-
-def overwrite_excel(excel_file_path, oss_name, column_name='OSS Name'):
-    if oss_name != "":
-        try:
-            files = os.listdir(excel_file_path)
-            for file in files:
-                if file.endswith(".xlsx"):
-                    file = os.path.join(excel_file_path, file)
-                    excel_file = pd.ExcelFile(file, engine='openpyxl')
-
-                    for sheet_name in excel_file.sheet_names:
-                        try:
-                            df = pd.read_excel(file, sheet_name=sheet_name, engine='openpyxl')
-                            if column_name in df.columns:
-                                updated = (df[column_name] == '') | (df[column_name].isnull())
-                                df.loc[updated, column_name] = oss_name
-                                df.to_excel(file, sheet_name=sheet_name, index=False)
-                        except Exception as ex:
-                            logger.debug(f"overwrite_sheet {sheet_name}:{ex}")
-        except Exception as ex:
-            logger.debug(f"overwrite_excel:{ex}")
-
-
-def merge_yamls(_output_dir, merge_yaml_files, final_report, remove_src_data=False,
-                default_oss_name='', default_oss_version='', url=''):
-    success = True
-    err_msg = ''
-
-    oss_total_list = []
-    yaml_dict = {}
-    try:
-        for mf in merge_yaml_files:
-            if os.path.exists(os.path.join(_output_dir, mf)):
-                oss_list, _, _ = parsing_yml(os.path.join(_output_dir, mf), _output_dir)
-
-                if remove_src_data:
-                    existed_yaml = {}
-                    for oi in oss_list:
-                        oi.name = default_oss_name if oi.name == '' else oi.name
-                        oi.version = default_oss_version if oi.version == '' else oi.version
-                        oi.download_location = url if oi.download_location == '' else oi.download_location
-                        create_yaml_with_ossitem(oi, existed_yaml)
-                    with open(os.path.join(_output_dir, mf), 'w') as f:
-                        yaml.dump(existed_yaml, f, default_flow_style=False, sort_keys=False)
-
-                oss_total_list.extend(oss_list)
-
-        if oss_total_list != []:
-            for oti in oss_total_list:
-                create_yaml_with_ossitem(oti, yaml_dict)
-            with open(os.path.join(_output_dir, final_report), 'w') as f:
-                yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
-        else:
-            success = False
-            err_msg = "Output file is not created as no oss items detected."
-    except Exception as ex:
-        err_msg = ex
-        success = False
-
-    return success, err_msg
-
-
-def create_scancodejson(final_report, output_extension, ui_mode_report, src_path=""):
-    success = True
-    err_msg = ''
-
-    oss_total_list = []
-    root_dir = ""
-    root_strip = ""
-    try:
-        src_path = os.path.abspath(src_path)
-        parent = os.path.basename(src_path)
-        root_strip = src_path.replace(parent, "")
-        root_dir = parent
-    except Exception:
-        root_dir = ""
-
-    try:
-        item_without_oss = OssItem("")
-        oss_total_list = get_osslist(os.path.dirname(final_report), os.path.basename(final_report),
-                                     output_extension, '')
-        if src_path:
-            for root, dirs, files in os.walk(src_path):
-                root = root.replace(root_strip, "")
-                for file in files:
-                    item_path = os.path.join(root, file)
-                    item_path = item_path.replace(parent + os.path.sep, '', 1)
-                    included = any(item_path in x.source_name_or_path for x in oss_total_list)
-                    if not included:
-                        item_without_oss.source_name_or_path = item_path
-            if len(item_without_oss.source_name_or_path) > 0:
-                oss_total_list.append(item_without_oss)
-        if root_dir:
-            for oss in oss_total_list:
-                tmp_path_list = oss.source_name_or_path
-                oss.source_name_or_path = ""
-                oss.source_name_or_path = [os.path.join(root_dir, path) for path in tmp_path_list]
-
-        write_scancodejson(os.path.dirname(ui_mode_report), os.path.basename(ui_mode_report),
-                           oss_total_list)
-    except Exception as ex:
-        err_msg = ex
-        success = False
-
-    return success, err_msg
-
-
-def correct_scanner_result(_output_dir, output_files, output_extension, exist_src, exist_bin):
-    src_oss_list = []
-    bin_oss_list = []
-    duplicates = False
-
-    if exist_src:
-        src_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['SRC'], output_extension, SRC_SHEET))
-    if exist_bin:
-        bin_oss_list = check_exclude_dir(get_osslist(_output_dir, output_files['BIN'], output_extension, BIN_SHEET))
-
-    if exist_src and exist_bin:
-        try:
-            remove_src_idx_list = []
-            for idx_src, src_item in enumerate(src_oss_list):
-                dup_flag = False
-                for bin_item in bin_oss_list:
-                    if (not src_item.source_name_or_path):
-                        continue
-                    if src_item.source_name_or_path[0] == bin_item.source_name_or_path[0]:
-                        dup_flag = True
-                        if not bin_item.license and src_item.license:
-                            src_item.exclude = bin_item.exclude
-                            bin_item.set_sheet_item(src_item.get_print_array(constant.FL_BINARY)[0])
-                            if bin_item.comment:
-                                bin_item.comment += '/'
-                            bin_item.comment += 'Loaded from SRC OSS info'
-                if dup_flag:
-                    remove_src_idx_list.append(idx_src)
-            if remove_src_idx_list:
-                duplicates = True
-                for i in sorted(remove_src_idx_list, reverse=True):
-                    del src_oss_list[i]
-        except Exception as ex:
-            logger.warning(f"correct the scanner result:{ex}")
-
-    try:
-        if exist_src:
-            success, err_msg = write_output_with_osslist(src_oss_list, _output_dir, output_files['SRC'],
-                                                         output_extension, SRC_SHEET)
-            if not success:
-                logger.warning(err_msg)
-        if exist_bin:
-            success, err_msg = write_output_with_osslist(bin_oss_list, _output_dir, output_files['BIN'],
-                                                         output_extension, BIN_SHEET, BIN_EXT_HEADER, BIN_HIDDEN_HEADER)
-            if not success:
-                logger.warning(err_msg)
-        if duplicates:
-            logger.info('Success to correct the src/bin scanner result')
-    except Exception as ex:
-        logger.warning(f"Corrected src/bin scanner result:{ex}")
-    return
-
-
-def write_output_with_osslist(oss_list, output_dir, output_file, output_extension, sheetname, extended_hdr={}, hidden_hdr={}):
-    new_oss_list = []
-    sheet_list = {}
-    sheet_list[sheetname] = []
-
-    for src_item in oss_list:
-        scanner_name = constant.supported_sheet_and_scanner[sheetname]
-        new_oss_list.append(src_item.get_print_array(scanner_name)[0])
-    sheet_list[sheetname].extend(new_oss_list)
-    if os.path.exists(os.path.join(output_dir, output_file)):
-        os.remove(os.path.join(output_dir, output_file))
-    success, err_msg, _ = write_output_file(os.path.join(output_dir, output_file).rstrip(output_extension),
-                                            output_extension, sheet_list, extended_hdr, hidden_hdr)
-    return success, err_msg
-
-
-def get_osslist(_output_dir, output_file, output_extension, sheet_name=''):
-    err_reason = ''
-    oss_list = []
-    oss_file_with_fullpath = os.path.join(_output_dir, output_file)
-
-    if os.path.exists(oss_file_with_fullpath):
-        if output_extension == '.xlsx':
-            oss_list = read_oss_report(oss_file_with_fullpath, sheet_name)
-        elif output_extension == '.yaml':
-            oss_list, _, err_reason = parsing_yml(oss_file_with_fullpath, _output_dir)
-        else:
-            err_reason = f'Not supported extension: {output_extension}'
-    if err_reason:
-        logger.info(f'get_osslist: {err_reason}')
-    return oss_list
-
-
-def check_exclude_dir(oss_list):
-    _exclude_dirs = ["venv", "node_modules", "Pods", "Carthage"]
-
-    for oss_item in oss_list:
-        if not oss_item.source_name_or_path:
-            continue
-        for exclude_dir in _exclude_dirs:
-            if exclude_dir in oss_item.source_name_or_path[0].split(os.path.sep):
-                oss_item.exclude = True
-                break
-    return oss_list

fosslight_scanner-1.7.30/src/fosslight_scanner.egg-info/requires.txt

@@ -1,11 +0,0 @@
-future
-pandas
-openpyxl
-progress
-pyyaml
-beautifulsoup4
-fosslight_util>=1.4.45
-fosslight_source>=1.7.8
-fosslight_dependency>=3.15.1
-fosslight_binary>=4.1.30
-fosslight_prechecker>=3.0.27