fosslight-util 2.1.0__tar.gz → 2.1.2__tar.gz

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_util
-Version: 2.1.0
+Version: 2.1.2
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/requirements.txt

@@ -8,7 +8,7 @@ coloredlogs
 python3-wget
 beautifulsoup4
 jsonmerge
-spdx-tools>=0.8.2;sys_platform!="win32"
+spdx-tools==0.8.*;sys_platform=="linux"
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
 numpy; python_version < '3.8'
 numpy>=1.22.2; python_version >= '3.8'

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='2.1.0',
+        version='2.1.2',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/src/fosslight_util/compare_yaml.py

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+from typing import List, Dict
 from fosslight_util.constant import LOGGER_NAME
+from fosslight_util.oss_item import FileItem
 
 logger = logging.getLogger(LOGGER_NAME)
 VERSION = 'version'
@@ -12,7 +14,7 @@ LICENSE = 'license'
 NAME = 'name'
 
 
-def compare_yaml(before_fileitems, after_fileitems):
+def compare_yaml(before_fileitems: List[FileItem], after_fileitems: List[FileItem]) -> Dict[str, List]:
     bf_raw = []
     af_raw = []
     for bf in before_fileitems:

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/src/fosslight_util/download.py

@@ -10,7 +10,7 @@ import zipfile
 import logging
 import argparse
 import shutil
-from git import Repo, GitCommandError
+from git import Repo, GitCommandError, Git
 import bz2
 import contextlib
 from datetime import datetime
@@ -77,7 +77,7 @@ def parse_src_link(src_link):
         if src_link.startswith("git://github.com/"):
             src_link_changed = change_src_link_to_https(src_link_split[0])
         elif src_link.startswith("git@github.com:"):
-            src_link_changed = change_ssh_link_to_https(src_link_split[0])
+            src_link_changed = src_link_split[0]
         else:
             if "rubygems.org" in src_link:
                 src_info["rubygems"] = True
@@ -92,39 +92,9 @@ def parse_src_link(src_link):
     return src_info
 
 
-def main():
-    parser = argparse.ArgumentParser(description='FOSSLight Downloader', prog='fosslight_download', add_help=False)
-    parser.add_argument('-h', '--help', help='Print help message', action='store_true', dest='help')
-    parser.add_argument('-s', '--source', help='Source link to download', type=str, dest='source')
-    parser.add_argument('-t', '--target_dir', help='Target directory', type=str, dest='target_dir', default="")
-    parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
-
-    src_link = ""
-    target_dir = os.getcwd()
-    log_dir = os.getcwd()
-
-    try:
-        args = parser.parse_args()
-    except SystemExit:
-        sys.exit(0)
-
-    if args.help:
-        print_help_msg_download()
-    if args.source:
-        src_link = args.source
-    if args.target_dir:
-        target_dir = args.target_dir
-    if args.log_dir:
-        log_dir = args.log_dir
-
-    if not src_link:
-        print_help_msg_download()
-    else:
-        cli_download_and_extract(src_link, target_dir, log_dir)
-
-
 def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_to: str = "",
-                             compressed_only: bool = False) -> Tuple[bool, str, str, str]:
+                             compressed_only: bool = False, ssh_key: str = "",
+                             id: str = "", git_token: str = "") -> Tuple[bool, str, str, str]:
     global logger
 
     success = True
@@ -152,7 +122,9 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
             is_rubygems = src_info.get("rubygems", False)
 
             # General download (git clone, wget)
-            success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir, checkout_to, tag, branch)
+            success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir, checkout_to,
+                                                                         tag, branch, ssh_key, id, git_token)
+            link = change_ssh_link_to_https(link)
             if (not is_rubygems) and (not success_git):
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
@@ -229,11 +201,38 @@ def get_github_token(git_url):
     return github_token
 
 
-def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
+def download_git_repository(refs_to_checkout, git_url, target_dir, tag):
+    success = False
+    oss_version = ""
+    clone_default_branch_flag = False
+
+    logger.info(f"Download git url :{git_url}")
+    if refs_to_checkout:
+        try:
+            # gitPython uses the branch argument the same whether you check out to a branch or a tag.
+            repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout)
+            success = True
+        except GitCommandError as error:
+            logger.debug(f"Git checkout error:{error}")
+            success = False
+
+    if not success:
+        repo = Repo.clone_from(git_url, target_dir)
+        clone_default_branch_flag = True
+        success = True
+
+    if refs_to_checkout != tag or clone_default_branch_flag:
+        oss_version = repo.active_branch.name
+    else:
+        oss_version = repo.git.describe('--tags')
+    return success, oss_version
+
+
+def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="", ssh_key="", id="", git_token=""):
     oss_name = get_github_ossname(git_url)
     refs_to_checkout = decide_checkout(checkout_to, tag, branch)
-    clone_default_branch_flag = False
     msg = ""
+    success = True
 
     try:
         if platform.system() != "Windows":
@@ -244,36 +243,40 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             alarm.start()
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
-        if refs_to_checkout != "":
-            try:
-                # gitPython uses the branch argument the same whether you check out to a branch or a tag.
-                repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout)
-            except GitCommandError as error:
-                error_msg = error.args[2].decode("utf-8")
-                if "Remote branch " + refs_to_checkout + " not found in upstream origin" in error_msg:
-                    # clone default branch, when non-existent branch or tag entered
-                    repo = Repo.clone_from(git_url, target_dir)
-                    clone_default_branch_flag = True
-        else:
-            repo = Repo.clone_from(git_url, target_dir)
-            clone_default_branch_flag = True
 
-        if refs_to_checkout != tag or clone_default_branch_flag:
-            oss_version = repo.active_branch.name
-        else:
-            oss_version = repo.git.describe('--tags')
-        logger.info(f"git checkout: {oss_version}")
-
-        if platform.system() != "Windows":
-            signal.alarm(0)
+        if git_url.startswith("ssh:") and not ssh_key:
+            msg = "Private git needs ssh_key"
+            success = False
         else:
-            del alarm
+            if ssh_key:
+                logger.info(f"Download git with ssh_key:{git_url}")
+                git_ssh_cmd = f'ssh -i {ssh_key}'
+                with Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
+                    success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag)
+            else:
+                if id and git_token:
+                    try:
+                        m = re.match(r"^(ht|f)tp(s?)\:\/\/", git_url)
+                        protocol = m.group()
+                        if protocol:
+                            git_url = git_url.replace(protocol, f"{protocol}{id}:{git_token}@")
+                    except Exception as error:
+                        logger.info(f"Failed to insert id, token to git url:{error}")
+                success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag)
+
+            logger.info(f"git checkout: {oss_version}")
+            refs_to_checkout = oss_version
+
+            if platform.system() != "Windows":
+                signal.alarm(0)
+            else:
+                del alarm
     except Exception as error:
+        success = False
         logger.warning(f"git clone - failed: {error}")
         msg = str(error)
-        return False, msg, oss_name, refs_to_checkout
 
-    return True, msg, oss_name, oss_version
+    return success, msg, oss_name, refs_to_checkout
 
 
 def download_wget(link, target_dir, compressed_only):
@@ -444,5 +447,36 @@ def gem_download(link, target_dir, checkout_to):
     return success
 
 
+def main():
+    parser = argparse.ArgumentParser(description='FOSSLight Downloader', prog='fosslight_download', add_help=False)
+    parser.add_argument('-h', '--help', help='Print help message', action='store_true', dest='help')
+    parser.add_argument('-s', '--source', help='Source link to download', type=str, dest='source')
+    parser.add_argument('-t', '--target_dir', help='Target directory', type=str, dest='target_dir', default="")
+    parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
+
+    src_link = ""
+    target_dir = os.getcwd()
+    log_dir = os.getcwd()
+
+    try:
+        args = parser.parse_args()
+    except SystemExit:
+        sys.exit(0)
+
+    if args.help:
+        print_help_msg_download()
+    if args.source:
+        src_link = args.source
+    if args.target_dir:
+        target_dir = args.target_dir
+    if args.log_dir:
+        log_dir = args.log_dir
+
+    if not src_link:
+        print_help_msg_download()
+    else:
+        cli_download_and_extract(src_link, target_dir, log_dir)
+
+
 if __name__ == '__main__':
     main()

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/src/fosslight_util/output_format.py

@@ -31,7 +31,7 @@ def check_output_format(output='', format='', customized_format={}):
         format = format.lower()
         if format not in list(support_format.keys()):
             success = False
-            msg = 'Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+            msg = '(-f option) Enter the supported format: ' + ', '.join(list(support_format.keys()))
         else:
             output_extension = support_format[format]
 
@@ -47,11 +47,11 @@ def check_output_format(output='', format='', customized_format={}):
                 if format:
                     if output_extension != basename_extension:
                         success = False
-                        msg = f"Enter the same extension of output file(-o:'{output}') with format(-f:'{format}')."
+                        msg = f"(-o & -f option) Enter the same extension of output file(-o:'{output}') with format(-f:'{format}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
-                        msg = 'Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                        msg = '(-o option) Enter the supported file extension: ' + ', '.join(list(support_format.values()))
                 if success:
                     output_file = basename_file
                     output_extension = basename_extension
@@ -79,7 +79,7 @@ def check_output_formats(output='', formats=[], customized_format={}):
         for format in formats:
             if format not in list(support_format.keys()):
                 success = False
-                msg = 'Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+                msg = '(-f option) Enter the supported format: ' + ', '.join(list(support_format.keys()))
             else:
                 output_extensions.append(support_format[format])
 
@@ -95,11 +95,11 @@ def check_output_formats(output='', formats=[], customized_format={}):
                 if formats:
                     if basename_extension not in output_extensions:
                         success = False
-                        msg = f"The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
+                        msg = f"(-o & -f option) The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
-                        msg = 'Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                        msg = '(-o option) Enter the supported file extension: ' + ', '.join(list(support_format.values()))
                     output_extensions.append(basename_extension)
                 output_files = [basename_file for _ in range(len(output_extensions))]
             else:
@@ -128,7 +128,7 @@ def check_output_formats_v2(output='', formats=[], customized_format={}):
         for format in formats:
             if format not in list(support_format.keys()):
                 success = False
-                msg = 'Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
+                msg = '(-f option) Enter the supported format with -f option: ' + ', '.join(list(support_format.keys()))
             else:
                 output_extensions.append(support_format[format])
 
@@ -144,11 +144,11 @@ def check_output_formats_v2(output='', formats=[], customized_format={}):
                 if formats:
                     if basename_extension not in output_extensions:
                         success = False
-                        msg = f"The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
+                        msg = f"(-o & -f option) The format of output file(-o:'{output}') should be in the format list(-f:'{formats}')."
                 else:
                     if basename_extension not in support_format.values():
                         success = False
-                        msg = 'Enter the supported file extension: ' + ', '.join(list(support_format.values()))
+                        msg = '(-o option) Enter the supported file extension: ' + ', '.join(list(support_format.values()))
                     output_extensions.append(basename_extension)
                 output_files = [basename_file for _ in range(len(output_extensions))]
             else:
@@ -183,11 +183,11 @@ def write_output_file(output_file_without_ext: str, file_extension: str, scan_it
         elif format == 'yaml':
             success, msg, _ = write_yaml(result_file, scan_item, False)
         elif format.startswith('spdx'):
-            if platform.system() != 'Windows':
-                success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version)
-            else:
+            if platform.system() == 'Windows' or platform.system() == 'Darwin':
                 success = False
-                msg = 'Windows not support spdx format.'
+                msg = f'{platform.system()} not support spdx format.'
+            else:
+                success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version)
     else:
         if file_extension == '.xlsx':
             success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)
@@ -199,6 +199,6 @@ def write_output_file(output_file_without_ext: str, file_extension: str, scan_it
             success, msg, result_file = write_yaml(result_file, scan_item, False)
         else:
             success = False
-            msg = f'Not supported file extension({file_extension})'
+            msg = f'(-f option) Not supported file extension({file_extension})'
 
     return success, msg, result_file

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/src/fosslight_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.1.0
+Version: 2.1.2
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/src/fosslight_util.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ src/fosslight_util/__init__.py
 src/fosslight_util/_get_downloadable_url.py
 src/fosslight_util/compare_yaml.py
 src/fosslight_util/constant.py
-src/fosslight_util/convert_excel_to_yaml.py
 src/fosslight_util/correct.py
 src/fosslight_util/cover.py
 src/fosslight_util/download.py

{fosslight_util-2.1.0 → fosslight_util-2.1.2}/src/fosslight_util.egg-info/requires.txt

@@ -25,5 +25,5 @@ pygit2==1.6.1
 [:python_version>='3.7']
 pygit2>=1.10.1
 
-[:sys_platform != "win32"]
-spdx-tools>=0.8.2
+[:sys_platform == "linux"]
+spdx-tools==0.8.*

fosslight_util-2.1.0/src/fosslight_util/convert_excel_to_yaml.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2022 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-import os
-import sys
-import logging
-import yaml
-import re
-from fosslight_util.constant import LOGGER_NAME
-from fosslight_util.write_yaml import create_yaml_with_ossitem
-from fosslight_util.read_excel import read_oss_report
-
-logger = logging.getLogger(LOGGER_NAME)
-
-
-def find_report_file(path_to_find):
-    file_to_find = ["FOSSLight-Report.xlsx", "OSS-Report.xlsx"]
-
-    try:
-        for file in file_to_find:
-            file_with_path = os.path.join(path_to_find, file)
-            if os.path.isfile(file_with_path):
-                return file
-        for root, dirs, files in os.walk(path_to_find):
-            for file in files:
-                file_name = file.lower()
-                p = re.compile(r"[\s\S]*OSS[\s\S]*-Report[\s\S]*.xlsx", re.I)
-                if p.search(file_name):
-                    return os.path.join(root, file)
-    except Exception as error:
-        logger.debug("Find report:"+str(error))
-    return ""
-
-
-def convert_excel_to_yaml(oss_report_to_read: str, output_file: str, sheet_names: str = "") -> None:
-    _file_extension = ".yaml"
-    yaml_dict = {}
-
-    if os.path.isfile(oss_report_to_read):
-        try:
-            items = read_oss_report(oss_report_to_read, sheet_names)
-            for item in items:
-                create_yaml_with_ossitem(item, yaml_dict)
-            if yaml_dict:
-                output_file = output_file if output_file.endswith(_file_extension) else output_file + _file_extension
-                success = write_yaml_file(output_file, yaml_dict)
-                if success:
-                    logger.warning(f"Output: {output_file}")
-                else:
-                    logger.error(f"Can't write yaml file : {output_file}")
-                    sys.exit(1)
-        except Exception as error:
-            logger.error(f"Convert yaml: {error}")
-    else:
-        logger.error(f"Can't find a file: {oss_report_to_read}")
-
-
-def write_yaml_file(output_file, json_output):
-    success = True
-    error_msg = ""
-
-    try:
-        with open(output_file, 'w') as f:
-            yaml.dump(json_output, f, sort_keys=False)
-    except Exception as ex:
-        error_msg = str(ex)
-        success = False
-    return success, error_msg