fosslight-util 1.4.40__tar.gz → 1.4.42__tar.gz

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_util
-Version: 1.4.40
+Version: 1.4.42
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -122,6 +122,7 @@ Description: <!--
         
         #### How it works
         1. Try git clone.
+        1-1. If the link is ssh-url, convert to https-url.
         2. If git clone fails, download it with wget and extract the compressed file.
         3. After extracting the compressed file, delete the compressed file.
         
@@ -135,7 +136,11 @@ Description: <!--
         
         #### How to run
         ```
-        $ fosslight_download  -s "https://github.com/LGE-OSS/example" -t target_dir/
+        $ fosslight_download -s "https://github.com/LGE-OSS/example" -t target_dir/
+        ```
+        If you want to try with private repository, set your github token like below.
+        ```
+        $ fosslight_download -s "https://my_github_token@github.com/Foo/private_repo -t target_dir/"
         ```
         
         ## 👏 How to report issue

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/README.md

@@ -114,6 +114,7 @@ If you give a link, the source is downloaded to the target directory through git
 
 #### How it works
 1. Try git clone.
+1-1. If the link is ssh-url, convert to https-url.
 2. If git clone fails, download it with wget and extract the compressed file.
 3. After extracting the compressed file, delete the compressed file.
 
@@ -127,7 +128,11 @@ If you give a link, the source is downloaded to the target directory through git
 
 #### How to run
 ```
-$ fosslight_download  -s "https://github.com/LGE-OSS/example" -t target_dir/
+$ fosslight_download -s "https://github.com/LGE-OSS/example" -t target_dir/
+```
+If you want to try with private repository, set your github token like below.
+```
+$ fosslight_download -s "https://my_github_token@github.com/Foo/private_repo -t target_dir/"
 ```
 
 ## 👏 How to report issue

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='1.4.40',
+        version='1.4.42',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/correct.py

@@ -48,12 +48,13 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
         if sheet_name not in constant.supported_sheet_and_scanner.keys():
             continue
         correct_contents = copy.deepcopy(sheet_contents)
+        scanner_name = constant.supported_sheet_and_scanner[sheet_name]
         for idx, oss_raw_item in enumerate(sheet_contents):
             if len(oss_raw_item) < 9:
                 logger.warning(f"sheet list is too short ({len(oss_raw_item)}): {oss_raw_item}")
                 continue
             oss_item = OssItem('')
-            oss_item.set_sheet_item(oss_raw_item)
+            oss_item.set_sheet_item(oss_raw_item, scanner_name)
 
             matched_yi = []
             oss_rel_path = os.path.normpath(os.path.join(rel_path, oss_item.source_name_or_path[0]))
@@ -75,13 +76,13 @@ def correct_with_yaml(correct_filepath, path_to_scan, scanner_oss_list):
                     if matched_oss_item.comment:
                         matched_oss_item.comment += '/'
                     matched_oss_item.comment += 'Loaded from sbom-info.yaml'
-                    matched_oss_array = matched_oss_item.get_print_array()[0]
+                    matched_oss_array = matched_oss_item.get_print_array(scanner_name)[0]
                     correct_contents.append(matched_oss_array)
                 oss_item.exclude = True
                 if oss_item.comment:
                     oss_item.comment += '/'
                 oss_item.comment += 'Excluded by sbom-info.yaml'
-                correct_contents[idx] = oss_item.get_print_array()[0]
+                correct_contents[idx] = oss_item.get_print_array(scanner_name)[0]
 
         if sheet_name == 'SRC_FL_Source':
             for n_idx, ni in enumerate(matched_yaml):

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/download.py

@@ -61,13 +61,21 @@ def change_src_link_to_https(src_link):
     return src_link
 
 
+def change_ssh_link_to_https(src_link):
+    src_link = src_link.replace("git@github.com:", "https://github.com/")
+    return src_link
+
+
 def parse_src_link(src_link):
-    src_info = {}
+    src_info = {"url": src_link}
     src_link_changed = ""
-    if src_link.startswith("git://") or src_link.startswith("https://") or src_link.startswith("http://"):
+    if src_link.startswith("git://") or src_link.startswith("git@") \
+            or src_link.startswith("https://") or src_link.startswith("http://"):
         src_link_split = src_link.split(';')
         if src_link.startswith("git://github.com/"):
             src_link_changed = change_src_link_to_https(src_link_split[0])
+        elif src_link.startswith("git@github.com:"):
+            src_link_changed = change_ssh_link_to_https(src_link_split[0])
         else:
             if "rubygems.org" in src_link:
                 src_info["rubygems"] = True
@@ -79,7 +87,7 @@ def parse_src_link(src_link):
         src_info["url"] = src_link_changed
         src_info["branch"] = branch_info
         src_info["tag"] = tag_info
-        return src_info
+    return src_info
 
 
 def main():
@@ -205,11 +213,24 @@ def get_github_ossname(link):
     return oss_name
 
 
+def get_github_token(git_url):
+    github_token = ""
+    pattern = r'https://(.*?)@'
+    search = re.search(pattern, git_url)
+    if search:
+        github_token = search.group(1)
+    return github_token
+
+
 def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
     ref_to_checkout = decide_checkout(checkout_to, tag, branch)
     msg = ""
     oss_name = get_github_ossname(git_url)
     oss_version = ""
+    github_token = get_github_token(git_url)
+    callbacks = None
+    if github_token != "":
+        callbacks = git.RemoteCallbacks(credentials=git.UserPass("foo", github_token))  # username is not used, so set to dummy
 
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -221,7 +242,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
         Path(target_dir).mkdir(parents=True, exist_ok=True)
         repo = git.clone_repository(git_url, target_dir,
                                     bare=False, repository=None,
-                                    remote=None, callbacks=None)
+                                    remote=None, callbacks=callbacks)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/oss_item.py

@@ -5,7 +5,7 @@
 
 import logging
 import os
-from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.constant import LOGGER_NAME, FL_DEPENDENCY, FL_BINARY
 
 _logger = logging.getLogger(LOGGER_NAME)
 
@@ -25,6 +25,11 @@ class OssItem:
         self._yocto_recipe = []
         self._yocto_package = []
         self.is_binary = False
+        self._depends_on = []
+        self.purl = ""
+        self.bin_vulnerability = ""
+        self.bin_tlsh = ""
+        self.bin_sha1 = ""
 
     def __del__(self):
         pass
@@ -123,11 +128,29 @@ class OssItem:
         self._yocto_package = [item.strip() for item in self._yocto_package]
         self._yocto_package = list(set(self._yocto_package))
 
-    def set_sheet_item(self, item):
+    @property
+    def depends_on(self):
+        return self._depends_on
+
+    @depends_on.setter
+    def depends_on(self, value):
+        if not value:
+            self._depends_on = []
+        else:
+            if not isinstance(value, list):
+                value = value.split(",")
+            self._depends_on.extend(value)
+            self._depends_on = [item.strip() for item in self._depends_on]
+            self._depends_on = list(set(self._depends_on))
+
+    def set_sheet_item(self, item, scanner_name=''):
         if len(item) < 9:
             _logger.warning(f"sheet list is too short ({len(item)}): {item}")
             return
-        self.source_name_or_path = item[0]
+        if scanner_name == FL_DEPENDENCY:
+            self.purl = item[0]
+        else:
+            self.source_name_or_path = item[0]
         self.name = item[1]
         self.version = item[2]
         self.license = item[3]
@@ -137,19 +160,39 @@ class OssItem:
         self.exclude = item[7]
         self.comment = item[8]
 
-    def get_print_array(self):
+        if len(item) >= 10 and scanner_name == FL_DEPENDENCY:
+            self.depends_on = item[9]
+        if len(item) >= 10 and scanner_name == FL_BINARY:
+            self.bin_vulnerability = item[9]
+            if len(item) >= 12:
+                self.bin_tlsh = item[10]
+                self.bin_sha1 = item[11]
+
+    def get_print_array(self, scanner_name=''):
         items = []
-        if len(self.source_name_or_path) == 0:
-            self.source_name_or_path.append("")
+        if scanner_name != FL_DEPENDENCY:
+            if len(self.source_name_or_path) == 0:
+                self.source_name_or_path.append("")
         if len(self.license) == 0:
             self.license.append("")
 
         exclude = "Exclude" if self.exclude else ""
-
-        for source_name_or_path in self.source_name_or_path:
-            lic = ",".join(self.license)
-            items.append([os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
-                          self.download_location, self.homepage, self.copyright, exclude, self.comment])
+        lic = ",".join(self.license)
+        if scanner_name == FL_DEPENDENCY:
+            items = [self.purl, self.name, self.version, lic,
+                     self.download_location, self.homepage, self.copyright, exclude, self.comment]
+            if len(self.depends_on) > 0:
+                items.append(",".join(self.depends_on))
+        else:
+            for source_name_or_path in self.source_name_or_path:
+                if scanner_name == FL_BINARY:
+                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
+                                self.download_location, self.homepage, self.copyright, exclude, self.comment,
+                                self.bin_vulnerability, self.bin_tlsh, self.bin_sha1]
+                else:
+                    oss_item = [os.path.join(self.relative_path, source_name_or_path), self.name, self.version, lic,
+                                self.download_location, self.homepage, self.copyright, exclude, self.comment]
+                items.append(oss_item)
         return items
 
     def get_print_json(self):
@@ -171,6 +214,10 @@ class OssItem:
             json_item["exclude"] = self.exclude
         if self.comment != "":
             json_item["comment"] = self.comment
+        if len(self.depends_on) > 0:
+            json_item["depends on"] = self.depends_on
+        if self.purl != "":
+            json_item["purl"] = self.purl
 
         return json_item
 

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/parsing_yaml.py

@@ -115,6 +115,12 @@ def set_value_switch(oss, key, value, yaml_file=""):
         oss.yocto_package = value
     elif key == 'yocto_recipe':
         oss.yocto_recipe = value
+    elif key == 'vulnerability link':
+        oss.bin_vulnerability = value
+    elif key == 'tlsh':
+        oss.bin_tlsh = value
+    elif key == 'sha1':
+        oss.bin_sha1 = value
     else:
         if yaml_file != "":
             _logger.debug(f"file:{yaml_file} - key:{key} cannot be parsed")

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/read_excel.py

@@ -75,7 +75,10 @@ def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
                 "Exclude": IDX_CANNOT_FOUND,
                 "Copyright Text": IDX_CANNOT_FOUND,
                 "Comment": IDX_CANNOT_FOUND,
-                "File Name or Path": IDX_CANNOT_FOUND
+                "File Name or Path": IDX_CANNOT_FOUND,
+                "Vulnerability Link": IDX_CANNOT_FOUND,
+                "TLSH": IDX_CANNOT_FOUND,
+                "SHA1": IDX_CANNOT_FOUND
             }
             num_cols = xl_sheet.ncols
             num_rows = xl_sheet.nrows

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/write_excel.py

@@ -301,6 +301,10 @@ def merge_excels(find_excel_dir, final_out, merge_files='', cover=''):
                             sheet_name = f"{f_short_name}_{sheet_name}"
                         df_excel.to_excel(writer, sheet_name, index=False)
                         added_sheet_names.append(sheet_name)
+
+                        if sheet_name == 'BIN_FL_Binary':
+                            bin_sheet = writer.sheets[sheet_name]
+                            bin_sheet.set_column("L:M", None, None, {"hidden": True})  # 'TLSH', 'SHA1' column hide
             writer.close()
     except Exception as ex:
         msg = str(ex)

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util/write_yaml.py

@@ -35,11 +35,16 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
             for sheet_name, sheet_contents in sheet_list.items():
                 if sheet_name not in constant.supported_sheet_and_scanner.keys():
                     continue
+                scanner_name = constant.supported_sheet_and_scanner[sheet_name]
+                sheet_contents_with_scanner = []
+                for i in sheet_contents:
+                    i.insert(0, scanner_name)
+                    sheet_contents_with_scanner.append(i)
                 if not separate_yaml:
-                    merge_sheet.extend(sheet_contents)
+                    merge_sheet.extend(sheet_contents_with_scanner)
                 else:
                     output_file = f'{separate_output_file}_{sheet_name}.yaml'
-                    convert_sheet_to_yaml(sheet_contents, output_file)
+                    convert_sheet_to_yaml(sheet_contents_with_scanner, output_file)
                     output_files.append(output_file)
 
             if not separate_yaml:
@@ -61,13 +66,13 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
     return success, error_msg, output
 
 
-def convert_sheet_to_yaml(sheet_contents, output_file):
-    sheet_contents = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents))]
+def convert_sheet_to_yaml(sheet_contents_with_scanner, output_file):
+    sheet_contents_with_scanner = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents_with_scanner))]
 
     yaml_dict = {}
-    for sheet_item in sheet_contents:
+    for sheet_item in sheet_contents_with_scanner:
         item = OssItem('')
-        item.set_sheet_item(sheet_item)
+        item.set_sheet_item(sheet_item[1:], sheet_item[0])
         create_yaml_with_ossitem(item, yaml_dict)
 
     with open(output_file, 'w') as f:

{fosslight_util-1.4.40 → fosslight_util-1.4.42}/src/fosslight_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 1.4.40
+Version: 1.4.42
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -122,6 +122,7 @@ Description: <!--
         
         #### How it works
         1. Try git clone.
+        1-1. If the link is ssh-url, convert to https-url.
         2. If git clone fails, download it with wget and extract the compressed file.
         3. After extracting the compressed file, delete the compressed file.
         
@@ -135,7 +136,11 @@ Description: <!--
         
         #### How to run
         ```
-        $ fosslight_download  -s "https://github.com/LGE-OSS/example" -t target_dir/
+        $ fosslight_download -s "https://github.com/LGE-OSS/example" -t target_dir/
+        ```
+        If you want to try with private repository, set your github token like below.
+        ```
+        $ fosslight_download -s "https://my_github_token@github.com/Foo/private_repo -t target_dir/"
         ```
         
         ## 👏 How to report issue