fosslight-util 1.4.40__py3-none-any.whl → 1.4.42__py3-none-any.whl

fosslight_util/correct.py

@@ -48,12 +48,13 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
         if sheet_name not in constant.supported_sheet_and_scanner.keys():
             continue
         correct_contents = copy.deepcopy(sheet_contents)
+        scanner_name = constant.supported_sheet_and_scanner[sheet_name]
         for idx, oss_raw_item in enumerate(sheet_contents):
             if len(oss_raw_item) < 9:
                 logger.warning(f"sheet list is too short ({len(oss_raw_item)}): {oss_raw_item}")
                 continue
             oss_item = OssItem('')
-            oss_item.set_sheet_item(oss_raw_item)
+            oss_item.set_sheet_item(oss_raw_item, scanner_name)
 
             matched_yi = []
             oss_rel_path = os.path.normpath(os.path.join(rel_path, oss_item.source_name_or_path[0]))
@@ -75,13 +76,13 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
                     if matched_oss_item.comment:
                         matched_oss_item.comment += '/'
                     matched_oss_item.comment += 'Loaded from sbom-info.yaml'
-                    matched_oss_array = matched_oss_item.get_print_array()[0]
+                    matched_oss_array = matched_oss_item.get_print_array(scanner_name)[0]
                     correct_contents.append(matched_oss_array)
                 oss_item.exclude = True
                 if oss_item.comment:
                     oss_item.comment += '/'
                 oss_item.comment += 'Excluded by sbom-info.yaml'
-                correct_contents[idx] = oss_item.get_print_array()[0]
+                correct_contents[idx] = oss_item.get_print_array(scanner_name)[0]
 
         if sheet_name == 'SRC_FL_Source':
             for n_idx, ni in enumerate(matched_yaml):

fosslight_util/download.py

@@ -61,13 +61,21 @@ def change_src_link_to_https(src_link):
     return src_link
 
 
+def change_ssh_link_to_https(src_link):
+    src_link = src_link.replace("git@github.com:", "https://github.com/")
+    return src_link
+
+
 def parse_src_link(src_link):
-    src_info = {}
+    src_info = {"url": src_link}
     src_link_changed = ""
-    if src_link.startswith("git://") or src_link.startswith("https://") or src_link.startswith("http://"):
+    if src_link.startswith("git://") or src_link.startswith("git@") \
+            or src_link.startswith("https://") or src_link.startswith("http://"):
         src_link_split = src_link.split(';')
         if src_link.startswith("git://github.com/"):
             src_link_changed = change_src_link_to_https(src_link_split[0])
+        elif src_link.startswith("git@github.com:"):
+            src_link_changed = change_ssh_link_to_https(src_link_split[0])
         else:
             if "rubygems.org" in src_link:
                 src_info["rubygems"] = True
@@ -79,7 +87,7 @@ def parse_src_link(src_link):
         src_info["url"] = src_link_changed
         src_info["branch"] = branch_info
         src_info["tag"] = tag_info
-        return src_info
+    return src_info
 
 
 def main():
@@ -205,11 +213,24 @@ def get_github_ossname(link):
     return oss_name
 
 
+def get_github_token(git_url):
+    github_token = ""
+    pattern = r'https://(.*?)@'
+    search = re.search(pattern, git_url)
+    if search:
+        github_token = search.group(1)
+    return github_token
+
+
 def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
     ref_to_checkout = decide_checkout(checkout_to, tag, branch)
     msg = ""
     oss_name = get_github_ossname(git_url)
     oss_version = ""
+    github_token = get_github_token(git_url)
+    callbacks = None
+    if github_token != "":
+        callbacks = git.RemoteCallbacks(credentials=git.UserPass("foo", github_token))  # username is not used, so set to dummy
 
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -221,7 +242,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
         Path(target_dir).mkdir(parents=True, exist_ok=True)
         repo = git.clone_repository(git_url, target_dir,
                                     bare=False, repository=None,
-                                    remote=None, callbacks=None)
+                                    remote=None, callbacks=callbacks)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:

fosslight_util/oss_item.py

@@ -5,7 +5,7 @@
 
 import logging
 import os
-from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.constant import LOGGER_NAME, FL_DEPENDENCY, FL_BINARY
 
 _logger = logging.getLogger(LOGGER_NAME)
 
@@ -25,6 +25,11 @@ class OssItem:
         self._yocto_recipe = []
         self._yocto_package = []
         self.is_binary = False
+        self._depends_on = []
+        self.purl = ""
+        self.bin_vulnerability = ""
+        self.bin_tlsh = ""
+        self.bin_sha1 = ""
 
     def __del__(self):
         pass
@@ -123,11 +128,29 @@ class OssItem:
         self._yocto_package = [item.strip() for item in self._yocto_package]
         self._yocto_package = list(set(self._yocto_package))
 
-    def set_sheet_item(self, item):
+    @property
+    def depends_on(self):
+        return self._depends_on
+
+    @depends_on.setter
+    def depends_on(self, value):
+        if not value:
+            self._depends_on = []
+        else:
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._depends_on.extend(value)
+            self._depends_on = [item.strip() for item in self._depends_on]
+            self._depends_on = list(set(self._depends_on))
+
+    def set_sheet_item(self, item, scanner_name=''):
         if len(item) < 9:
             _logger.warning(f"sheet list is too short ({len(item)}): {item}")
             return
-        self.source_name_or_path = item[0]
+        if scanner_name == FL_DEPENDENCY:
+            self.purl = item[0]
+        else:
+            self.source_name_or_path = item[0]
         self.name = item[1]
         self.version = item[2]
         self.license = item[3]
@@ -137,19 +160,39 @@ class OssItem:
         self.exclude = item[7]
         self.comment = item[8]
 
-    def get_print_array(self):
+        if len(item) >= 10 and scanner_name == FL_DEPENDENCY:
+            self.depends_on = item[9]
+        if len(item) >= 10 and scanner_name == FL_BINARY:
+            self.bin_vulnerability = item[9]
+            if len(item) >= 12:
+                self.bin_tlsh = item[10]
+                self.bin_sha1 = item[11]
+
+    def get_print_array(self, scanner_name=''):
         items = []
-        if len(self.source_name_or_path) == 0:
-            self.source_name_or_path.append("")
+        if scanner_name != FL_DEPENDENCY:
+            if len(self.source_name_or_path) == 0:
+                self.source_name_or_path.append("")
         if len(self.license) == 0:
             self.license.append("")
 
         exclude = "Exclude" if self.exclude else ""
-
-        for source_name_or_path in self.source_name_or_path:
-            lic = ",".join(self.license)
-            items.append([os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
-                          self.download_location, self.homepage, self.copyright, exclude, self.comment])
+        lic = ",".join(self.license)
+        if scanner_name == FL_DEPENDENCY:
+            items = [self.purl, self.name, self.version, lic,
+                     self.download_location, self.homepage, self.copyright, exclude, self.comment]
+            if len(self.depends_on) > 0:
+                items.append(",".join(self.depends_on))
+        else:
+            for source_name_or_path in self.source_name_or_path:
+                if scanner_name == FL_BINARY:
+                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
+                                self.download_location, self.homepage, self.copyright, exclude, self.comment,
+                                self.bin_vulnerability, self.bin_tlsh, self.bin_sha1]
+                else:
+                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
+                                self.download_location, self.homepage, self.copyright, exclude, self.comment]
+                items.append(oss_item)
         return items
 
     def get_print_json(self):
@@ -171,6 +214,10 @@ class OssItem:
             json_item["exclude"] = self.exclude
         if self.comment != "":
             json_item["comment"] = self.comment
+        if len(self.depends_on) > 0:
+            json_item["depends on"] = self.depends_on
+        if self.purl != "":
+            json_item["purl"] = self.purl
 
         return json_item
 

fosslight_util/parsing_yaml.py

@@ -115,6 +115,12 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.yocto_package = value
     elif key == 'yocto_recipe':
         oss.yocto_recipe = value
+    elif key == 'vulnerability link':
+        oss.bin_vulnerability = value
+    elif key == 'tlsh':
+        oss.bin_tlsh = value
+    elif key == 'sha1':
+        oss.bin_sha1 = value
     else:
         if yaml_file != "":
             _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")

fosslight_util/read_excel.py

@@ -75,7 +75,10 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                 "Exclude": IDX_CANNOT_FOUND,
                 "Copyright Text": IDX_CANNOT_FOUND,
                 "Comment": IDX_CANNOT_FOUND,
-                "File Name or Path": IDX_CANNOT_FOUND
+                "File Name or Path": IDX_CANNOT_FOUND,
+                "Vulnerability Link": IDX_CANNOT_FOUND,
+                "TLSH": IDX_CANNOT_FOUND,
+                "SHA1": IDX_CANNOT_FOUND
             }
             num_cols = xl_sheet.ncols
             num_rows = xl_sheet.nrows

fosslight_util/write_excel.py

@@ -301,6 +301,10 @@ def merge_excels(find_excel_dir, final_out, merge_files='', cover=''):
                             sheet_name = f"{f_short_name}_{sheet_name}"
                         df_excel.to_excel(writer, sheet_name, index=False)
                         added_sheet_names.append(sheet_name)
+
+                        if sheet_name == 'BIN_FL_Binary':
+                            bin_sheet = writer.sheets[sheet_name]
+                            bin_sheet.set_column("L:M", None, None, {"hidden": True})  # 'TLSH', 'SHA1' column hide
             writer.close()
     except Exception as ex:
         msg = str(ex)

fosslight_util/write_yaml.py

@@ -35,11 +35,16 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
             for sheet_name, sheet_contents in sheet_list.items():
                 if sheet_name not in constant.supported_sheet_and_scanner.keys():
                     continue
+                scanner_name = constant.supported_sheet_and_scanner[sheet_name]
+                sheet_contents_with_scanner = []
+                for i in sheet_contents:
+                    i.insert(0, scanner_name)
+                    sheet_contents_with_scanner.append(i)
                 if not separate_yaml:
-                    merge_sheet.extend(sheet_contents)
+                    merge_sheet.extend(sheet_contents_with_scanner)
                 else:
                     output_file = f'{separate_output_file}_{sheet_name}.yaml'
-                    convert_sheet_to_yaml(sheet_contents, output_file)
+                    convert_sheet_to_yaml(sheet_contents_with_scanner, output_file)
                     output_files.append(output_file)
 
             if not separate_yaml:
@@ -61,13 +66,13 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
     return success, error_msg, output
 
 
-def convert_sheet_to_yaml(sheet_contents, output_file):
-    sheet_contents = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents))]
+def convert_sheet_to_yaml(sheet_contents_with_scanner, output_file):
+    sheet_contents_with_scanner = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents_with_scanner))]
 
     yaml_dict = {}
-    for sheet_item in sheet_contents:
+    for sheet_item in sheet_contents_with_scanner:
         item = OssItem('')
-        item.set_sheet_item(sheet_item)
+        item.set_sheet_item(sheet_item[1:], sheet_item[0])
         create_yaml_with_ossitem(item, yaml_dict)
 
     with open(output_file, 'w') as f:

{fosslight_util-1.4.40.dist-info → fosslight_util-1.4.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 1.4.40
+Version: 1.4.42
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -152,6 +152,7 @@ If you give a link, the source is downloaded to the target directory through git
 
 #### How it works
 1. Try git clone.
+1-1. If the link is ssh-url, convert to https-url.
 2. If git clone fails, download it with wget and extract the compressed file.
 3. After extracting the compressed file, delete the compressed file.
 
@@ -165,7 +166,11 @@ If you give a link, the source is downloaded to the target directory through git
 
 #### How to run
 ```
-$ fosslight_download  -s "https://github.com/LGE-OSS/example" -t target_dir/
+$ fosslight_download -s "https://github.com/LGE-OSS/example" -t target_dir/
+```
+If you want to try with private repository, set your github token like below.
+```
+$ fosslight_download -s "https://my_github_token@github.com/Foo/private_repo -t target_dir/"
 ```
 
 ## 👏 How to report issue

{fosslight_util-1.4.40.dist-info → fosslight_util-1.4.42.dist-info}/RECORD

@@ -3,29 +3,29 @@ fosslight_util/_get_downloadable_url.py,sha256=63ZPI4KCpUFgL4oheKm8zvekuCRzpwNkV
 fosslight_util/compare_yaml.py,sha256=0bapoyS0yrzkiK70K57E8-3wZ_D6mZAJ-24Eq9TYBlk,2632
 fosslight_util/constant.py,sha256=j9uhncoC2Fn4j4ATNsjUoS91nVXhyGzU6xLyWFj941A,1834
 fosslight_util/convert_excel_to_yaml.py,sha256=7ZsAMMQJIEXrmcl_28nSHvFpGMi1ZiRZYpEfI5O8vP8,2298
-fosslight_util/correct.py,sha256=RQ70zaQ_xuo9Mo0zIcb6pRQwt96bsimQJWkYldE2vbg,5224
+fosslight_util/correct.py,sha256=v4OL8XssYuatcP9G0YhnLChWUiJP8rwoyWDeMmQGT0E,5334
 fosslight_util/cover.py,sha256=21l-BS3ZJ8nDxMLHKGT9igB3XbCme55A0eQf01jbISM,1436
-fosslight_util/download.py,sha256=tYsY5rb3fRCwqCDzZzF7wY_kzpMO4cz3G6B7ApdEolA,13732
+fosslight_util/download.py,sha256=4_s1zPEpxPQ7FNkDo0f6MdrQLMOVdjifDO7Y5psqAq8,14484
 fosslight_util/help.py,sha256=xhAf43fuRLOU3TE7jqaxVTizVn4aANDvHaHYQAdaqYc,2154
-fosslight_util/oss_item.py,sha256=Gjw-aI4gZGYrCbAkCL35lXlgA2w2F2plgOUmenQ1Cig,5238
+fosslight_util/oss_item.py,sha256=JxGokv79z_6Ca4-Aquc_zZEvRqu-xmBunP8Lvy6AIjY,7268
 fosslight_util/output_format.py,sha256=kJpuTuS3XTrsdUd2gsl5GWto5KtJfSej-qkD9azZFdQ,2982
-fosslight_util/parsing_yaml.py,sha256=zmetMiZprni_YMxUfj_Nx-nXm0sT31Ck_V0nWFS6ZmE,4332
-fosslight_util/read_excel.py,sha256=eN2aNkJv3Y2071f6lHfQTHx5IFJe0imxH3TeRO_kodc,5154
+fosslight_util/parsing_yaml.py,sha256=R2aecMLDoWR-bBdD8qiW0OTn7F5ZC5YNd1Of0YYEJxU,4514
+fosslight_util/read_excel.py,sha256=21T12bSaB69Yp19n-o2qpHMJNWUFoytneaNneRp3qCA,5294
 fosslight_util/set_log.py,sha256=8cFGpr4HMk4grHo3a5keR4V1xWBscbjowYPxyrh42ks,3140
 fosslight_util/spdx_licenses.py,sha256=r90hUY4_T-XrHIJHLx1Ox3gWZ3qzdZj9rJFo7AwmkPE,3641
 fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
-fosslight_util/write_excel.py,sha256=8-qZhCM31KzLrNwysX_hF9ZCSzSUOzkZmTPeekKVKJg,12065
+fosslight_util/write_excel.py,sha256=FJFmCs6_vJTYNDgp-Cwk0ylkMTjN3u3_Wgxy0yoiGqo,12306
 fosslight_util/write_opossum.py,sha256=PGJV5DysNJvIFbzsyGXxh_kRcvZuHAOmLs-WlXP8qMI,11831
 fosslight_util/write_scancodejson.py,sha256=CSKjuwbA04nK5ogXklNsCBgGDZXPr805T8KPLgvx71U,2282
 fosslight_util/write_spdx.py,sha256=B_aHv9vScgZI5gHo5Hd56ckNWOHdyAQebRV54bTx9ec,9542
 fosslight_util/write_txt.py,sha256=Ms8E2wbc0U84IYP0dxTAbIfX5PzpQG3xblu8lv1w8J0,574
-fosslight_util/write_yaml.py,sha256=ppBBErAXbXw8jtJ9j7BDeQHHzDlsTRt62XR0GgTcr70,3294
+fosslight_util/write_yaml.py,sha256=iJcG3ItggoBp8p3m8PAZwULehKo3BlH1qW2rZzlDv8g,3665
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-1.4.40.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-1.4.40.dist-info/METADATA,sha256=o4BdKiS8Caz_wUgyy2_REDR9uQ-vYmkgw-Njxyjdo2A,6189
-fosslight_util-1.4.40.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-fosslight_util-1.4.40.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-1.4.40.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-1.4.40.dist-info/RECORD,,
+fosslight_util-1.4.42.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-1.4.42.dist-info/METADATA,sha256=Gwh1UV3hbzKi2Urxb-w9bJzw9uA1YzrTySAD-7C4ohU,6418
+fosslight_util-1.4.42.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+fosslight_util-1.4.42.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-1.4.42.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-1.4.42.dist-info/RECORD,,