fosslight-binary 4.1.33__tar.gz → 5.0.0__tar.gz

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_binary
-Version: 4.1.33
+Version: 5.0.0
 Summary: FOSSLight Binary Scanner
 Home-page: https://github.com/fosslight/fosslight_binary_scanner
 Author: LG Electronics

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/requirements.txt

@@ -8,5 +8,5 @@ py-tlsh
 pytz
 XlsxWriter
 PyYAML
-fosslight_util~=1.4.47
+fosslight_util>=2.0.0
 dependency-check

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/setup.py

@@ -33,7 +33,7 @@ if __name__ == "__main__":
 
     setup(
         name=_PACKAEG_NAME,
-        version='4.1.33',
+        version='5.0.0',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Binary Scanner',

fosslight_binary-5.0.0/src/fosslight_binary/_binary.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+from fosslight_util.oss_item import FileItem
+
+EXCLUDE_TRUE_VALUE = "Exclude"
+TLSH_CHECKSUM_NULL = "0"
+
+
+class VulnerabilityItem:
+    file_path = ""
+    vul_id = ""
+    nvd_url = ""
+
+    def __init__(self, file_path, id, url):
+        self.file_path = file_path
+        self.vul_id = id
+        self.nvd_url = url
+
+
+class BinaryItem(FileItem):
+    def __init__(self, value):
+        super().__init__("")
+        self.exclude = False
+        self.source_name_or_path = ""
+        self.checksum = TLSH_CHECKSUM_NULL
+        self.tlsh = TLSH_CHECKSUM_NULL
+        self.vulnerability_items = []
+        self.binary_name_without_path = ""
+        self.bin_name_with_path = value
+        self.found_in_owasp = False
+        self.is_binary = True
+
+    def __del__(self):
+        pass
+
+    def set_oss_items(self, new_oss_list, exclude=False, exclude_msg=""):
+        if exclude:
+            for oss in new_oss_list:
+                oss.exclude = True
+                oss.comment = exclude_msg
+        # Append New input OSS
+        self.oss_items.extend(new_oss_list)
+
+    def get_vulnerability_items(self):
+        nvd_url = [vul_item.nvd_url for vul_item in self.vulnerability_items]
+        return ", ".join(nvd_url)
+
+    def get_print_binary_only(self):
+        return (self.source_name_or_path + "\t" + self.checksum + "\t" + self.tlsh)
+
+    def get_print_array(self):
+        items = []
+        if self.oss_items:
+            for oss in self.oss_items:
+                lic = ",".join(oss.license)
+                exclude = EXCLUDE_TRUE_VALUE if (self.exclude or oss.exclude) else ""
+                nvd_url = self.get_vulnerability_items()
+                items.append([self.source_name_or_path, oss.name, oss.version,
+                              lic, oss.download_location, oss.homepage,
+                              oss.copyright, exclude, oss.comment,
+                              nvd_url, self.tlsh, self.checksum])
+        else:
+            exclude = EXCLUDE_TRUE_VALUE if self.exclude else ""
+            items.append([self.source_name_or_path, '',
+                          '', '', '', '', '', exclude, self.comment, '',
+                          self.tlsh, self.checksum])
+        return items
+
+    def get_print_json(self):
+        items = []
+        if self.oss_items:
+            for oss in self.oss_items:
+                json_item = {}
+                json_item["name"] = oss.name
+                json_item["version"] = oss.version
+
+                if self.source_name_or_path:
+                    json_item["source path"] = self.source_name_or_path
+                if len(oss.license) > 0:
+                    json_item["license"] = oss.license
+                if oss.download_location:
+                    json_item["download location"] = oss.download_location
+                if oss.homepage:
+                    json_item["homepage"] = oss.homepage
+                if oss.copyright:
+                    json_item["copyright text"] = oss.copyright
+                if self.exclude or oss.exclude:
+                    json_item["exclude"] = True
+                if oss.comment:
+                    json_item["comment"] = oss.comment
+                items.append(json_item)
+        else:
+            json_item = {}
+            if self.source_name_or_path:
+                json_item["source path"] = self.source_name_or_path
+            if self.exclude:
+                json_item["exclude"] = True
+            if self.comment:
+                json_item["comment"] = self.comment
+        return items

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/src/fosslight_binary/_binary_dao.py

@@ -8,7 +8,8 @@ import logging
 import psycopg2
 import pandas as pd
 from urllib.parse import urlparse
-from ._binary import _TLSH_CHECKSUM_NULL, OssItem
+from ._binary import TLSH_CHECKSUM_NULL
+from fosslight_util.oss_item import OssItem
 import fosslight_util.constant as constant
 
 columns = ['filename', 'pathname', 'checksum', 'tlshchecksum', 'ossname',
@@ -43,10 +44,10 @@ def get_oss_info_from_db(bin_info_list, dburl=""):
                     if not item.found_in_owasp:
                         oss_from_db = OssItem(row['ossname'], row['ossversion'], row['license'])
                         bin_oss_items.append(oss_from_db)
-                        item.set_comment("Binary DB result")
 
                 if bin_oss_items:
                     item.set_oss_items(bin_oss_items)
+                    item.comment = "Binary DB result"
 
     disconnect_lge_bin_db()
     return bin_info_list, _cnt_auto_identified
@@ -97,14 +98,14 @@ def get_oss_info_by_tlsh_and_filename(file_name, checksum_value, tlsh_value):
             sql_statement_filename, ['tlshchecksum'])
         if df_result is None or len(df_result) <= 0:
             final_result_item = ""
-        elif tlsh_value == _TLSH_CHECKSUM_NULL:  # Couldn't get the tlsh of a file.
+        elif tlsh_value == TLSH_CHECKSUM_NULL:  # Couldn't get the tlsh of a file.
             final_result_item = ""
         else:
             matched_tlsh = ""
             matched_tlsh_diff = -1
             for row in df_result.tlshchecksum:
                 try:
-                    if row != _TLSH_CHECKSUM_NULL:
+                    if row != TLSH_CHECKSUM_NULL:
                         tlsh_diff = tlsh.diff(row, tlsh_value)
                         if tlsh_diff <= 120:  # MATCHED
                             if (matched_tlsh_diff < 0) or (tlsh_diff < matched_tlsh_diff):

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/src/fosslight_binary/_jar_analysis.py

@@ -8,7 +8,8 @@ import json
 import os
 import sys
 import fosslight_util.constant as constant
-from ._binary import BinaryItem, OssItem, VulnerabilityItem
+from ._binary import BinaryItem, VulnerabilityItem
+from fosslight_util.oss_item import OssItem
 from dependency_check import run as dependency_check_run
 
 
@@ -63,21 +64,21 @@ def merge_binary_list(owasp_items, vulnerability_items, bin_list):
     for key, value in owasp_items.items():
         found = False
         for bin in bin_list:
-            if bin.binary_strip_root == key:
+            if bin.source_name_or_path == key:
                 for oss in value:
                     if oss.name and oss.license:
                         bin.found_in_owasp = True
                         break
                 bin.set_oss_items(value)
-                if vulnerability_items is not None:
-                    bin.set_vulnerability_items(vulnerability_items.get(key))
+                if vulnerability_items and vulnerability_items.get(key):
+                    bin.vulnerability_items.extend(vulnerability_items.get(key))
                 found = True
                 break
 
         if not found:
             bin_item = BinaryItem(os.path.abspath(key))
             bin_item.binary_name_without_path = os.path.basename(key)
-            bin_item.binary_strip_root = key
+            bin_item.source_name_or_path = key
             bin_item.set_oss_items(value)
             not_found_bin.append(bin_item)
 
@@ -261,7 +262,7 @@ def analyze_jar_file(path_to_find_bin, path_to_exclude):
 
             if oss_name != "" or oss_ver != "" or oss_license != "" or oss_dl_url != "":
                 oss = OssItem(oss_name, oss_ver, oss_license, oss_dl_url)
-                oss.set_comment("OWASP result")
+                oss.comment = "OWASP result"
 
                 remove_owasp_item = owasp_items.get(file_with_path)
                 if remove_owasp_item:

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/src/fosslight_binary/binary_analysis.py

@@ -15,12 +15,15 @@ from fosslight_util.set_log import init_log
 import fosslight_util.constant as constant
 from fosslight_util.output_format import check_output_formats, write_output_file
 from ._binary_dao import get_oss_info_from_db
-from ._binary import BinaryItem
+from ._binary import BinaryItem, TLSH_CHECKSUM_NULL
 from ._jar_analysis import analyze_jar_file, merge_binary_list
 from fosslight_util.correct import correct_with_yaml
-from fosslight_util.cover import CoverItem
+from fosslight_util.oss_item import ScannerItem
+import hashlib
+import tlsh
+from io import open
 
-_PKG_NAME = "fosslight_binary"
+PKG_NAME = "fosslight_binary"
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 _REMOVE_FILE_EXTENSION = ['qm', 'xlsx', 'pdf', 'pptx', 'jfif', 'docx', 'doc', 'whl',
@@ -40,11 +43,31 @@ _root_path = ""
 _start_time = ""
 windows = False
 BYTES = 2048
-
 BIN_EXT_HEADER = {'BIN_FL_Binary': ['ID', 'Binary Path', 'OSS Name',
                                     'OSS Version', 'License', 'Download Location',
                                     'Homepage', 'Copyright Text', 'Exclude',
                                     'Comment', 'Vulnerability Link', 'TLSH', 'SHA1']}
+HIDE_HEADER = {'TLSH', "SHA1"}
+
+
+def get_checksum_and_tlsh(bin_with_path):
+    checksum_value = TLSH_CHECKSUM_NULL
+    tlsh_value = TLSH_CHECKSUM_NULL
+    error_msg = ""
+    try:
+        f = open(bin_with_path, "rb")
+        byte = f.read()
+        sha1_hash = hashlib.sha1(byte)
+        checksum_value = str(sha1_hash.hexdigest())
+        try:
+            tlsh_value = str(tlsh.hash(byte))
+        except:
+            tlsh_value = TLSH_CHECKSUM_NULL
+        f.close()
+    except Exception as ex:
+        error_msg = f"(Error) Get_checksum, tlsh: {ex}"
+
+    return checksum_value, tlsh_value, error_msg
 
 
 def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]):
@@ -53,7 +76,7 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]):
     _json_ext = ".json"
     _start_time = datetime.now().strftime('%y%m%d_%H%M')
     _result_log = {
-        "Tool Info": _PKG_NAME
+        "Tool Info": PKG_NAME
     }
 
     _root_path = path_to_find_bin
@@ -83,7 +106,8 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]):
         sys.exit(1)
 
     log_file = os.path.join(output_path, f"fosslight_log_bin_{_start_time}.txt")
-    logger, _result_log = init_log(log_file, True, logging.INFO, logging.DEBUG, _PKG_NAME, path_to_find_bin, path_to_exclude)
+    logger, _result_log = init_log(log_file, True, logging.INFO, logging.DEBUG,
+                                   PKG_NAME, path_to_find_bin, path_to_exclude)
 
     if not success:
         error_occured(error_msg=msg,
@@ -121,15 +145,15 @@ def get_file_list(path_to_find, abs_path_to_exclude):
             bin_with_path = os.path.join(root, file)
             bin_item = BinaryItem(bin_with_path)
             bin_item.binary_name_without_path = file
-            bin_item.binary_strip_root = bin_with_path.replace(
+            bin_item.source_name_or_path = bin_with_path.replace(
                 _root_path, '', 1)
 
             if any(dir_name in dir_path for dir_name in _EXCLUDE_DIR):
-                bin_item.set_exclude(True)
+                bin_item.exclude = True
             elif file.lower() in _EXCLUDE_FILE:
-                bin_item.set_exclude(True)
+                bin_item.exclude = True
             elif extension in _EXCLUDE_FILE_EXTENSION:
-                bin_item.set_exclude(True)
+                bin_item.exclude = True
             bin_list.append(bin_item)
             file_cnt += 1
     return file_cnt, bin_list, found_jar
@@ -146,11 +170,10 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
     db_loaded_cnt = 0
     success_to_write = False
     writing_msg = ""
-    hide_header = {'TLSH', "SHA1"}
-    content_list = []
     results = []
     bin_list = []
     base_dir_name = os.path.basename(path_to_find_bin)
+    scan_item = ScannerItem(PKG_NAME, "")
     abs_path_to_exclude = [os.path.abspath(os.path.join(base_dir_name, path)) for path in path_to_exclude if path.strip() != ""]
 
     if not os.path.isdir(path_to_find_bin):
@@ -168,12 +191,10 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
                       exit=True)
     total_bin_cnt = len(return_list)
     if simple_mode:
-        bin_list = [bin.bin_name for bin in return_list]
+        bin_list = [bin.bin_name_with_path for bin in return_list]
     else:
-        cover = CoverItem(tool_name=_PKG_NAME,
-                          start_time=_start_time,
-                          input_path=path_to_find_bin,
-                          exclude_path=path_to_exclude)
+        scan_item = ScannerItem(PKG_NAME, _start_time)
+        scan_item.set_cover_pathinfo(path_to_find_bin, path_to_exclude)
         try:
             # Run OWASP Dependency-check
             if found_jar:
@@ -185,25 +206,23 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
                     logger.warning("Could not find OSS information for some jar files.")
 
             return_list, db_loaded_cnt = get_oss_info_from_db(return_list, dburl)
-            return_list = sorted(return_list, key=lambda row: (row.bin_name))
-
-            sheet_list = {}
-            for item in return_list:
-                content_list.extend(item.get_oss_report())
-            sheet_list["BIN_FL_Binary"] = content_list
+            return_list = sorted(return_list, key=lambda row: (row.bin_name_with_path))
+            scan_item.append_file_items(return_list, PKG_NAME)
             if correct_mode:
-                success, msg_correct, correct_list = correct_with_yaml(correct_filepath, path_to_find_bin, sheet_list)
+                success, msg_correct, correct_list = correct_with_yaml(correct_filepath, path_to_find_bin, scan_item)
                 if not success:
                     logger.info(f"No correction with yaml: {msg_correct}")
                 else:
-                    sheet_list = correct_list
+                    return_list = correct_list
                     logger.info("Success to correct with yaml.")
-            cover.comment = f"Total number of binaries: {total_bin_cnt} "
+
+            scan_item.set_cover_comment(f"Total number of binaries: {total_bin_cnt}")
             if total_bin_cnt == 0:
-                cover.comment += "(No binary detected.) "
-            cover.comment += f"/ Total number of files: {total_file_cnt}"
+                scan_item.set_cover_comment("(No binary detected.) ")
+            scan_item.set_cover_comment(f"Total number of files: {total_file_cnt}")
+
             for combined_path_and_file, output_extension in zip(result_reports, output_extensions):
-                results.append(write_output_file(combined_path_and_file, output_extension, sheet_list, BIN_EXT_HEADER, hide_header, cover))
+                results.append(write_output_file(combined_path_and_file, output_extension, scan_item, BIN_EXT_HEADER, HIDE_HEADER))
 
         except Exception as ex:
             error_occured(error_msg=str(ex), exit=False)
@@ -214,8 +233,8 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
                     logger.info(f"Output file :{result_file}")
                 else:
                     logger.warning(f"{writing_msg}")
-                if cover.comment:
-                    logger.info(cover.comment)
+                for row in scan_item.get_cover_comment():
+                    logger.info(row)
             else:
                 logger.error(f"Fail to generate result file.:{writing_msg}")
 
@@ -227,21 +246,21 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F
     except Exception as ex:
         error_occured(error_msg=f"Print log : {ex}", exit=False)
 
-    return success_to_write, content_list
+    return success_to_write, scan_item
 
 
 def return_bin_only(file_list, need_checksum_tlsh=True):
     for file_item in file_list:
         try:
-            if check_binary(file_item.bin_name):
+            if check_binary(file_item.bin_name_with_path):
                 if need_checksum_tlsh:
-                    error, error_msg = file_item.set_checksum_tlsh()
-                    if error:
+                    file_item.checksum, file_item.tlsh, error_msg = get_checksum_and_tlsh(file_item.bin_name_with_path)
+                    if error_msg:
                         error_occured(error_msg=error_msg, exit=False)
                 yield file_item
         except Exception as ex:
             logger.debug(f"Exception in get_file_list: {ex}")
-            file_item.set_comment("Exclude or delete if it is not binary.")
+            file_item.comment = "Exclude or delete if it is not binary."
             yield file_item
 
 

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/src/fosslight_binary.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-binary
-Version: 4.1.33
+Version: 5.0.0
 Summary: FOSSLight Binary Scanner
 Home-page: https://github.com/fosslight/fosslight_binary_scanner
 Author: LG Electronics

{fosslight_binary-4.1.33 → fosslight_binary-5.0.0}/src/fosslight_binary.egg-info/requires.txt

@@ -8,7 +8,7 @@ py-tlsh
 pytz
 XlsxWriter
 PyYAML
-fosslight_util~=1.4.47
+fosslight_util>=2.0.0
 dependency-check
 
 [:"darwin" in sys_platform]

fosslight_binary-4.1.33/src/fosslight_binary/_binary.py

@@ -1,168 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-import hashlib
-import tlsh
-from io import open
-
-_EXCLUDE_TRUE_VALUE = "Exclude"
-_TLSH_CHECKSUM_NULL = "0"
-
-
-class OssItem:
-    name = ""
-    version = ""
-    license = ""
-    dl_url = ""
-    comment = ""
-    exclude = False
-
-    def __init__(self, name, version, license, dl_url=""):
-        self.name = name
-        self.version = version
-        self.license = license
-        self.dl_url = dl_url
-        self.exclude = False
-        self.comment = ""
-
-    def set_comment(self, value):
-        if self.comment:
-            self.comment = f"{self.comment} / {value}"
-        else:
-            self.comment = value
-
-    def set_exclude(self, value):
-        self.exclude = value
-
-    def get_comment(self):
-        return self.comment
-
-
-class VulnerabilityItem:
-    file_path = ""
-    vul_id = ""
-    nvd_url = ""
-
-    def __init__(self, file_path, id, url):
-        self.file_path = file_path
-        self.vul_id = id
-        self.nvd_url = url
-
-
-class BinaryItem:
-    bin_name = ""
-    binary_name_without_path = ""
-    binary_strip_root = ""  # Value of binary name column
-    tlsh = _TLSH_CHECKSUM_NULL
-    checksum = _TLSH_CHECKSUM_NULL
-    oss_items = []
-    vulnerability_items = []
-    exclude = False
-    comment = ""
-    found_in_owasp = False
-
-    def __init__(self, value):
-        self.exclude = False
-        self.binary_strip_root = ""
-        self.checksum = _TLSH_CHECKSUM_NULL
-        self.tlsh = _TLSH_CHECKSUM_NULL
-        self.oss_items = []
-        self.vulnerability_items = []
-        self.binary_name_without_path = ""
-        self.set_bin_name(value)
-
-    def __del__(self):
-        pass
-
-    def set_oss_items(self, new_oss_list, exclude=False, exclude_msg=""):
-        if exclude:
-            for oss in new_oss_list:
-                oss.set_exclude(True)
-                oss.set_comment(exclude_msg)
-        # Append New input OSS
-        self.oss_items.extend(new_oss_list)
-
-    def get_oss_items(self):
-        return self.oss_items
-
-    def set_vulnerability_items(self, vul_list):
-        if vul_list is not None:
-            self.vulnerability_items.extend(vul_list)
-
-    def get_vulnerability_items(self):
-        nvd_url = [vul_item.nvd_url for vul_item in self.vulnerability_items]
-        return ", ".join(nvd_url)
-
-    def set_comment(self, value):
-        if self.comment:
-            self.comment = f"{self.comment} / {value}"
-        else:
-            self.comment = value
-
-    def set_bin_name(self, value):
-        self.bin_name = value
-
-    def set_exclude(self, value):
-        self.exclude = value
-
-    def set_checksum(self, value):
-        self.checksum = value
-
-    def set_tlsh(self, value):
-        self.tlsh = value
-
-    def get_comment(self):
-        return self.comment
-
-    def get_print_binary_only(self):
-        return (self.binary_strip_root + "\t" + self.checksum + "\t" + self.tlsh)
-
-    def get_oss_report(self):
-        comment = ""
-        if len(self.oss_items) > 0:
-            for oss in self.oss_items:
-                exclude = _EXCLUDE_TRUE_VALUE if (self.exclude or oss.exclude) else ""
-                nvd_url = self.get_vulnerability_items()
-
-                if self.comment:
-                    if oss.comment:
-                        comment = f"{self.comment} / {oss.comment}"
-                    else:
-                        comment = self.comment
-                else:
-                    comment = oss.comment
-
-                yield [self.binary_strip_root, oss.name, oss.version,
-                       oss.license, oss.dl_url, '', '', exclude, comment,
-                       nvd_url, self.tlsh, self.checksum]
-        else:
-            exclude = _EXCLUDE_TRUE_VALUE if self.exclude else ""
-            yield [self.binary_strip_root, '',
-                   '', '', '', '', '', exclude, self.comment, '', self.tlsh, self.checksum]
-
-    def set_checksum_tlsh(self):
-        self.checksum, self.tlsh, error, msg = get_checksum_and_tlsh(
-            self.bin_name)
-        return error, msg
-
-
-def get_checksum_and_tlsh(bin_with_path):
-    checksum_value = _TLSH_CHECKSUM_NULL
-    tlsh_value = _TLSH_CHECKSUM_NULL
-    error_msg = ""
-    error = False
-    try:
-        f = open(bin_with_path, "rb")
-        byte = f.read()
-        sha1_hash = hashlib.sha1(byte)
-        checksum_value = str(sha1_hash.hexdigest())
-        try:
-            tlsh_value = str(tlsh.hash(byte))
-        except:
-            tlsh_value = _TLSH_CHECKSUM_NULL
-        f.close()
-    except Exception as ex:
-        error_msg = f"(Error) Get_checksum, tlsh: {ex}"
-        error = True
-    return checksum_value, tlsh_value, error, error_msg