fosslight-util 1.4.48__tar.gz → 2.0.0__tar.gz

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_util
-Version: 1.4.48
+Version: 2.0.0
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/requirements.txt

@@ -1,6 +1,5 @@
 XlsxWriter
 pandas
-xlrd==1.2.0
 openpyxl
 progress
 PyYAML

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='1.4.48',
+        version='2.0.0',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/src/fosslight_util/compare_yaml.py

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
-import os
 from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.parsing_yaml import parsing_yml
 
 logger = logging.getLogger(LOGGER_NAME)
 VERSION = 'version'
@@ -14,12 +12,16 @@ LICENSE = 'license'
 NAME = 'name'
 
 
-def compare_yaml(before_file, after_file):
-    before_oss_items, _, _ = parsing_yml(before_file, os.path.dirname(before_file))
-    after_oss_items, _, _ = parsing_yml(after_file, os.path.dirname(after_file))
+def compare_yaml(before_fileitems, after_fileitems):
+    bf_raw = []
+    af_raw = []
+    for bf in before_fileitems:
+        bf_raw.extend(bf.get_print_json())
+    for af in after_fileitems:
+        af_raw.extend(af.get_print_json())
 
-    before_items = get_merged_item(before_oss_items)
-    after_items = get_merged_item(after_oss_items)
+    before_items = get_merged_item(bf_raw)
+    after_items = get_merged_item(af_raw)
 
     new_before = []
     for bi in before_items:
@@ -72,13 +74,18 @@ def compare_yaml(before_file, after_file):
 def get_merged_item(oss_items):
     item_list = []
     for oi in oss_items:
-        if oi.exclude:
+        if oi.get("exclude", None):
             continue
-        item_info = {NAME: oi.name, VERSION: oi.version, LICENSE: oi.license}
+        oi_name = oi.get("name", '')
+        oi_version = oi.get("version", '')
+        oi_license = oi.get("license", '')
+        if not (oi_name and oi_version and oi_license):
+            continue
+        item_info = {NAME: oi_name, VERSION: oi_version, LICENSE: oi_license}
 
-        filtered = next(filter(lambda oss_dict: oss_dict[NAME] == oi.name and oss_dict[VERSION] == oi.version, item_list), None)
+        filtered = next(filter(lambda oss_dict: oss_dict[NAME] == oi_name and oss_dict[VERSION] == oi_version, item_list), None)
         if filtered:
-            filtered[LICENSE].extend(oi.license)
+            filtered[LICENSE].extend(oi_license)
             filtered[LICENSE] = list(set(filtered[LICENSE]))
         else:
             item_list.append(item_info)

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/src/fosslight_util/constant.py

@@ -15,6 +15,17 @@ supported_sheet_and_scanner = {'SRC': FL_SOURCE,
                                f'BIN_{FL_BINARY}': FL_BINARY,
                                f'DEP_{FL_DEPENDENCY}': FL_DEPENDENCY}
 
+FOSSLIGHT_SCANNER = 'fosslight_scanner'
+FOSSLIGHT_SOURCE = 'fosslight_source'
+FOSSLIGHT_DEPENDENCY = 'fosslight_dependency'
+FOSSLIGHT_BINARY = 'fosslight_binary'
+
+SHEET_NAME_FOR_SCANNER = {
+    FOSSLIGHT_SOURCE: 'SRC_FL_Source',
+    FOSSLIGHT_BINARY: 'BIN_FL_Binary',
+    FOSSLIGHT_DEPENDENCY: 'DEP_FL_Dependency'
+}
+
 # Github : https://github.com/(owner)/(repo)
 # npm : https://www.npmjs.com/package/(package)/v/(version)
 # npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/src/fosslight_util/convert_excel_to_yaml.py

@@ -33,7 +33,7 @@ def find_report_file(path_to_find):
     return ""
 
 
-def convert_excel_to_yaml(oss_report_to_read, output_file, sheet_names=""):
+def convert_excel_to_yaml(oss_report_to_read: str, output_file: str, sheet_names: str = "") -> None:
     _file_extension = ".yaml"
     yaml_dict = {}
 

fosslight_util-2.0.0/src/fosslight_util/correct.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+import copy
+import re
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SOURCE
+from fosslight_util.parsing_yaml import parsing_yml
+
+logger = logging.getLogger(LOGGER_NAME)
+SBOM_INFO_YAML = r"sbom(-|_)info[\s\S]*.ya?ml"
+
+
+def correct_with_yaml(correct_filepath, path_to_scan, scan_item):
+    success = True
+    msg = ""
+    correct_yaml = ""
+    if correct_filepath == "":
+        correct_filepath = path_to_scan
+
+    path_to_scan = os.path.normpath(path_to_scan)
+    correct_filepath = os.path.normpath(correct_filepath)
+    for filename in os.listdir(correct_filepath):
+        if re.search(SBOM_INFO_YAML, filename, re.IGNORECASE):
+            correct_yaml = os.path.join(correct_filepath, filename)
+            break
+    if not correct_yaml:
+        msg = f"Cannot find sbom-info.yaml in {correct_filepath}."
+        success = False
+        return success, msg, scan_item
+
+    rel_path = os.path.relpath(path_to_scan, correct_filepath)
+
+    yaml_file_list, _, err_msg = parsing_yml(correct_yaml, os.path.dirname(correct_yaml), print_log=True)
+    find_match = False
+    for scanner_name, _ in scan_item.file_items.items():
+        correct_fileitems = []
+        exclude_fileitems = []
+        for yaml_file_item in yaml_file_list:
+            yaml_path_exists = False
+            if yaml_file_item.source_name_or_path == '':
+                if scanner_name == FOSSLIGHT_SOURCE:
+                    correct_item = copy.deepcopy(yaml_file_item)
+                    correct_item.comment = 'Added by sbom-info.yaml'
+                    correct_fileitems.append(correct_item)
+                continue
+            for idx, scan_file_item in enumerate(scan_item.file_items[scanner_name]):
+                oss_rel_path = os.path.normpath(os.path.join(rel_path, scan_file_item.source_name_or_path))
+                yi_path = yaml_file_item.source_name_or_path
+                if ((os.path.normpath(yi_path) == os.path.normpath(oss_rel_path)) or
+                   ((os.path.normpath(oss_rel_path).startswith(os.path.normpath(yi_path.rstrip('*')))))):
+                    correct_item = copy.deepcopy(scan_file_item)
+                    correct_item.exclude = yaml_file_item.exclude
+                    correct_item.oss_items = copy.deepcopy(yaml_file_item.oss_items)
+                    correct_item.comment = ''
+                    correct_item.comment = 'Loaded from sbom-info.yaml'
+                    correct_fileitems.append(correct_item)
+
+                    yaml_path_exists = True
+                    exclude_fileitems.append(idx)
+
+            if not yaml_path_exists:
+                correct_item = copy.deepcopy(yaml_file_item)
+                if os.path.exists(os.path.normpath(yaml_file_item.source_name_or_path)):
+                    correct_item.comment = 'Loaded from sbom-info.yaml'
+                    correct_fileitems.append(correct_item)
+                else:
+                    if scanner_name == FOSSLIGHT_SOURCE:
+                        correct_item.exclude = True
+                        correct_item.comment = 'Added by sbom-info.yaml'
+                        correct_fileitems.append(correct_item)
+        if correct_fileitems:
+            scan_item.append_file_items(correct_fileitems, scanner_name)
+            find_match = True
+        if exclude_fileitems:
+            exclude_fileitems = list(set(exclude_fileitems))
+            for e_idx in exclude_fileitems:
+                scan_item.file_items[scanner_name][e_idx].exclude = True
+                scan_item.file_items[scanner_name][e_idx].comment = 'Excluded by sbom-info.yaml'
+
+    if not find_match:
+        success = False
+        err_msg = 'No match items in sbom-info.yaml'
+        return success, err_msg, scan_item
+
+    return success, msg, scan_item

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/src/fosslight_util/help.py

@@ -35,12 +35,11 @@ _HELP_MESSAGE_DOWNLOAD = """
 
 
 class PrintHelpMsg():
-    message_suffix = ""
 
-    def __init__(self, value):
+    def __init__(self, value: str = ""):
         self.message_suffix = value
 
-    def print_help_msg(self, exitopt):
+    def print_help_msg(self, exitopt: bool) -> None:
         print(_HELP_MESSAGE_COMMON)
         print(self.message_suffix)
 
@@ -48,7 +47,7 @@ class PrintHelpMsg():
             sys.exit()
 
 
-def print_package_version(pkg_name, msg="", exitopt=True):
+def print_package_version(pkg_name: str, msg: str = "", exitopt: bool = True) -> str:
     if msg == "":
         msg = f"{pkg_name} Version:"
     cur_version = pkg_resources.get_distribution(pkg_name).version

fosslight_util-2.0.0/src/fosslight_util/oss_item.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SCANNER
+from fosslight_util.cover import CoverItem
+from typing import List, Dict
+
+_logger = logging.getLogger(LOGGER_NAME)
+
+
+class OssItem:
+
+    def __init__(self, name="", version="", license="", dl_url=""):
+        self.name = name
+        self.version = version
+        self._license = []
+        self.license = license
+        self.download_location = dl_url
+        self.exclude = False
+        self.comment = ""
+        self.homepage = ""
+        self._copyright = ""
+
+    def __del__(self):
+        pass
+
+    @property
+    def license(self):
+        return self._license
+
+    @license.setter
+    def license(self, value):
+        if value != "":
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._license.extend(value)
+        self._license = [item.strip() for item in self._license]
+        self._license = list(set(self._license))
+
+    @property
+    def exclude(self):
+        return self._exclude
+
+    @exclude.setter
+    def exclude(self, value):
+        if value:
+            self._exclude = True
+        else:
+            self._exclude = False
+
+    @property
+    def copyright(self):
+        return self._copyright
+
+    @copyright.setter
+    def copyright(self, value):
+        if value != "":
+            if isinstance(value, list):
+                value = "\n".join(value)
+            value = value.strip()
+        self._copyright = value
+
+    @property
+    def version(self):
+        return self._version
+
+    @version.setter
+    def version(self, value):
+        if value:
+            self._version = str(value)
+        else:
+            self._version = ""
+
+    @property
+    def comment(self):
+        return self._comment
+
+    @comment.setter
+    def comment(self, value):
+        if not value:
+            self._comment = ""
+        else:
+            if self._comment:
+                self._comment = f"{self._comment} / {value}"
+            else:
+                self._comment = value
+
+
+class FileItem:
+    def __init__(self, value):
+        self.relative_path = value
+        self.source_name_or_path = ""
+        self._exclude = False
+        self._comment = ""
+        self.is_binary = False
+        self.oss_items: List[OssItem] = []
+
+    def __del__(self):
+        pass
+
+    @property
+    def exclude(self):
+        return self._exclude
+
+    @exclude.setter
+    def exclude(self, value):
+        if value:
+            self._exclude = True
+        else:
+            self._exclude = False
+        for oss in self.oss_items:
+            oss.exclude = value
+
+    @property
+    def comment(self):
+        return self._comment
+
+    @comment.setter
+    def comment(self, value):
+        if not value:
+            self._comment = ""
+        else:
+            if self._comment:
+                self._comment = f"{self._comment} / {value}"
+            else:
+                self._comment = value
+            for oss in self.oss_items:
+                oss.comment = value
+
+    def get_print_array(self):
+        items = []
+
+        for oss in self.oss_items:
+            exclude = "Exclude" if self.exclude or oss.exclude else ""
+            lic = ",".join(oss.license)
+
+            oss_item = [os.path.join(self.relative_path, self.source_name_or_path), oss.name, oss.version, lic,
+                        oss.download_location, oss.homepage, oss.copyright, exclude, oss.comment]
+            items.append(oss_item)
+        return items
+
+    def get_print_json(self):
+        items = []
+
+        for oss in self.oss_items:
+            json_item = {}
+            json_item["name"] = oss.name
+            json_item["version"] = oss.version
+
+            if self.source_name_or_path != "":
+                json_item["source path"] = self.source_name_or_path
+            if len(oss.license) > 0:
+                json_item["license"] = oss.license
+            if oss.download_location != "":
+                json_item["download location"] = oss.download_location
+            if oss.homepage != "":
+                json_item["homepage"] = oss.homepage
+            if oss.copyright != "":
+                json_item["copyright text"] = oss.copyright
+            if self.exclude or oss.exclude:
+                json_item["exclude"] = True
+            if oss.comment != "":
+                json_item["comment"] = oss.comment
+            items.append(json_item)
+        return items
+
+
+def invalid(cmd):
+    _logger.info('[{}] is invalid'.format(cmd))
+
+
+class ScannerItem:
+    def __init__(self, pkg_name, start_time=""):
+        self.cover = CoverItem(tool_name=pkg_name, start_time=start_time)
+        self.file_items: Dict[str, List[FileItem]] = {pkg_name: []} if pkg_name != FOSSLIGHT_SCANNER else {}
+        self.external_sheets: Dict[str, List[List[str]]] = {}
+
+    def set_cover_pathinfo(self, input_dir, path_to_exclude):
+        self.cover.input_path = input_dir
+        self.cover.exclude_path = ", ".join(path_to_exclude)
+
+    def set_cover_comment(self, value):
+        if value:
+            if self.cover.comment:
+                self.cover.comment = f"{self.cover.comment} / {value}"
+            else:
+                self.cover.comment = value
+
+    def get_cover_comment(self):
+        return [item.strip() for item in self.cover.comment.split(" / ")]
+
+    def append_file_items(self, file_item: List[FileItem], pkg_name=""):
+        if pkg_name == "":
+            if len(self.file_items.keys()) != 1:
+                _logger.error("Package name is not set. Cannot append file_item into ScannerItem.")
+            else:
+                pkg_name = list(self.file_items.keys())[0]
+        if pkg_name not in self.file_items:
+            self.file_items[pkg_name] = []
+        self.file_items[pkg_name].extend(file_item)
+
+    def get_print_array(self, scanner_name):
+        items = []
+        for file_item in self.file_items[scanner_name]:
+            items.extend(file_item.get_print_array())
+        return items
+
+    def get_print_json(self, scanner_name):
+        items = []
+        for file_item in self.file_items[scanner_name]:
+            items.extend(file_item.get_print_json())
+        return items
+
+    def __del__(self):
+        pass

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/src/fosslight_util/output_format.py

@@ -6,6 +6,7 @@ import os
 from fosslight_util.write_excel import write_result_to_excel, write_result_to_csv
 from fosslight_util.write_opossum import write_opossum
 from fosslight_util.write_yaml import write_yaml
+from typing import Tuple
 
 SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml'}
 
@@ -105,7 +106,8 @@ def check_output_formats(output='', formats=[], customized_format={}):
     return success, msg, output_path, output_files, output_extensions
 
 
-def write_output_file(output_file_without_ext, file_extension, sheet_list, extended_header={}, hide_header={}, cover=""):
+def write_output_file(output_file_without_ext: str, file_extension: str, scan_item, extended_header: dict = {},
+                      hide_header: dict = {}) -> Tuple[bool, str, str]:
     success = True
     msg = ''
 
@@ -114,13 +116,13 @@ def write_output_file(output_file_without_ext, file_extension, sheet_list, exten
     result_file = output_file_without_ext + file_extension
 
     if file_extension == '.xlsx':
-        success, msg = write_result_to_excel(result_file, sheet_list, extended_header, hide_header, cover)
+        success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
     elif file_extension == '.csv':
-        success, msg, result_file = write_result_to_csv(result_file, sheet_list, False, extended_header)
+        success, msg, result_file = write_result_to_csv(result_file, scan_item, False, extended_header)
     elif file_extension == '.json':
-        success, msg = write_opossum(result_file, sheet_list)
+        success, msg = write_opossum(result_file, scan_item)
     elif file_extension == '.yaml':
-        success, msg, result_file = write_yaml(result_file, sheet_list, False)
+        success, msg, result_file = write_yaml(result_file, scan_item, False)
     else:
         success = False
         msg = f'Not supported file extension({file_extension})'

{fosslight_util-1.4.48 → fosslight_util-2.0.0}/src/fosslight_util/parsing_yaml.py

@@ -8,8 +8,8 @@ import codecs
 import os
 import re
 import sys
-from .constant import LOGGER_NAME
-from .oss_item import OssItem
+from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import OssItem, FileItem
 
 _logger = logging.getLogger(LOGGER_NAME)
 SUPPORT_OSS_INFO_FILES = [r"oss-pkg-info[\s\S]*.ya?ml", r"sbom(-|_)info[\s\S]*.ya?ml"]
@@ -17,7 +17,7 @@ EXAMPLE_OSS_PKG_INFO_LINK = "https://github.com/fosslight/fosslight_prechecker/b
 
 
 def parsing_yml(yaml_file, base_path, print_log=True):
-    oss_list = []
+    fileitems = []
     license_list = []
     idx = 1
     err_reason = ""
@@ -38,37 +38,65 @@ def parsing_yml(yaml_file, base_path, print_log=True):
             err_reason = "empty"
             if print_log:
                 _logger.warning(f"The yaml file is empty file: {yaml_file}")
-            return oss_list, license_list, err_reason
+            return fileitems, license_list, err_reason
 
         is_old_format = any(x in doc for x in OLD_YAML_ROOT_ELEMENT)
 
+        filepath_list = []
         for root_element in doc:
             oss_items = doc[root_element]
             if oss_items:
                 if not isinstance(oss_items, list) or 'version' not in oss_items[0]:
                     raise AttributeError(f"- Ref. {EXAMPLE_OSS_PKG_INFO_LINK}")
                 for oss in oss_items:
-                    item = OssItem(relative_path)
-                    if not is_old_format:
-                        item.name = root_element
-                    for key, value in oss.items():
-                        if key:
-                            key = key.lower().strip()
-                        set_value_switch(item, key, value, yaml_file)
-                    oss_list.append(item)
-                    license_list.extend(item.license)
-                    idx += 1
+                    source_paths = get_source_name_or_path_in_yaml(oss)
+                    for source_path in source_paths:
+                        if os.path.join(relative_path, source_path) not in filepath_list:
+                            filepath_list.append(os.path.join(relative_path, source_path))
+                            fileitem = FileItem(relative_path)
+                            fileitem.source_name_or_path = source_path
+                            fileitems.append(fileitem)
+                        else:
+                            fileitem = next((i for i in fileitems if i.source_name_or_path == source_path), None)
+                        ossitem = OssItem()
+                        if not is_old_format:
+                            ossitem.name = root_element
+                        for key, value in oss.items():
+                            if key:
+                                key = key.lower().strip()
+                            set_value_switch(ossitem, key, value, yaml_file)
+                        fileitem.oss_items.append(ossitem)
+                        license_list.extend(ossitem.license)
+                        idx += 1
     except AttributeError as ex:
         if print_log:
             _logger.warning(f"Not supported yaml file format: {yaml_file} {ex}")
-        oss_list = []
+        fileitems = []
         err_reason = "not_supported"
     except yaml.YAMLError:
         if print_log:
             _logger.warning(f"Error to parse yaml - skip to parse yaml file: {yaml_file}")
-        oss_list = []
+        fileitems = []
         err_reason = "yaml_error"
-    return oss_list, set(license_list), err_reason
+
+    return fileitems, set(license_list), err_reason
+
+
+def get_source_name_or_path_in_yaml(oss):
+    source_name_or_path = []
+    find = False
+    for key in oss.keys():
+        if key in ['file name or path', 'source name or path', 'source path',
+                   'file', 'binary name', 'binary path']:
+            if isinstance(oss[key], list):
+                source_name_or_path = oss[key]
+            else:
+                source_name_or_path.append(oss[key])
+            find = True
+            break
+    if not find:
+        source_name_or_path.append('')
+    return source_name_or_path
 
 
 def find_sbom_yaml_files(path_to_find):
@@ -101,9 +129,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.download_location = value
     elif key in ['license', 'license text']:
         oss.license = value
-    elif key in ['file name or path', 'source name or path', 'source path',
-                 'file', 'binary name', 'binary path']:
-        oss.source_name_or_path = value
     elif key in ['copyright text', 'copyright']:
         oss.copyright = value
     elif key == 'exclude':
@@ -112,16 +137,6 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.comment = value
     elif key == 'homepage':
         oss.homepage = value
-    elif key == 'yocto_package':
-        oss.yocto_package = value
-    elif key == 'yocto_recipe':
-        oss.yocto_recipe = value
-    elif key == 'vulnerability link':
-        oss.bin_vulnerability = value
-    elif key == 'tlsh':
-        oss.bin_tlsh = value
-    elif key == 'sha1':
-        oss.bin_sha1 = value
     else:
         if yaml_file != "":
             _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")