fosslight-util 2.1.20__tar.gz → 2.1.22__tar.gz

{fosslight_util-2.1.20 → fosslight_util-2.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_util
-Version: 2.1.20
+Version: 2.1.22
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util

{fosslight_util-2.1.20 → fosslight_util-2.1.22}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='2.1.20',
+        version='2.1.22',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

{fosslight_util-2.1.20 → fosslight_util-2.1.22}/src/fosslight_util/_get_downloadable_url.py

@@ -13,7 +13,7 @@ import fosslight_util.constant as constant
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
-def extract_name_version_from_link(link):
+def extract_name_version_from_link(link, checkout_version):
     oss_name = ""
     oss_version = ""
     matched = False
@@ -53,8 +53,12 @@ def extract_name_version_from_link(link):
             except Exception as ex:
                 logger.info(f"extract_name_version_from_link {key}:{ex}")
             if oss_name and (not oss_version):
-                if key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
-                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                if checkout_version:
+                    oss_version = checkout_version
+                elif key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
+                    oss_version = get_latest_package_version(link, key, origin_name)
+                if oss_version:
+                    link = get_new_link_with_version(link, key, origin_name, oss_version)
                     logger.info(f'Try to download with the latest version:{link}')
             matched = True
             break
@@ -63,50 +67,56 @@ def extract_name_version_from_link(link):
     return oss_name, oss_version, link, key
 
 
+def get_new_link_with_version(link, pkg_type, oss_name, oss_version):
+    if pkg_type == "pypi":
+        link = f'https://pypi.org/project/{oss_name}/{oss_version}'
+    elif pkg_type == "maven":
+        oss_name = oss_name.replace(':', '/')
+        link = f'https://mvnrepository.com/artifact/{oss_name}/{oss_version}'
+    elif pkg_type == "npm" or pkg_type == "npm2":
+        link = f'https://www.npmjs.com/package/{oss_name}/v/{oss_version}'
+    elif pkg_type == "pub":
+        link = f'https://pub.dev/packages/{oss_name}/versions/{oss_version}'
+    elif pkg_type == "go":
+        link = f'https://pkg.go.dev/{oss_name}@{oss_version}'
+    elif pkg_type == "cargo":
+        link = f'https://crates.io/crates/{oss_name}/{oss_version}'
+    return link
+
+
 def get_latest_package_version(link, pkg_type, oss_name):
     find_version = ''
-    link_with_version = link
 
     try:
         if pkg_type in ['npm', 'npm2']:
             npm_response = requests.get(f"https://registry.npmjs.org/{oss_name}")
             if npm_response.status_code == 200:
                 find_version = npm_response.json().get("dist-tags", {}).get("latest")
-            if find_version:
-                link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
         elif pkg_type == 'pypi':
             find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
-            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
         elif pkg_type == 'maven':
             maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
             if maven_response.status_code == 200:
                 find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
-            oss_name = oss_name.replace(':', '/')
-            if find_version:
-                link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
         elif pkg_type == 'pub':
             pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
             if pub_response.status_code == 200:
                 find_version = pub_response.json().get('latest').get('version')
-            if find_version:
-                link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
         elif pkg_type == 'go':
             go_response = requests.get(f'https://proxy.golang.org/{oss_name}/@latest')
             if go_response.status_code == 200:
                 find_version = go_response.json().get('Version')
-            if find_version:
-                link_with_version = f'https://pkg.go.dev/{oss_name}@{find_version}'
     except Exception as e:
         logger.info(f'Fail to get latest package version({link}:{e})')
-    return find_version, link_with_version
+    return find_version
 
 
-def get_downloadable_url(link):
+def get_downloadable_url(link, checkout_version):
 
     ret = False
     result_link = link
 
-    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link)
+    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link, checkout_version)
     new_link = new_link.replace('http://', '')
     new_link = new_link.replace('https://', '')
 

{fosslight_util-2.1.20 → fosslight_util-2.1.22}/src/fosslight_util/download.py

@@ -27,6 +27,7 @@ import subprocess
 import re
 from typing import Tuple
 import urllib.parse
+import json
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
@@ -96,7 +97,8 @@ def parse_src_link(src_link):
 def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_to: str = "",
                              compressed_only: bool = False, ssh_key: str = "",
                              id: str = "", git_token: str = "",
-                             called_cli: bool = True) -> Tuple[bool, str, str, str]:
+                             called_cli: bool = True,
+                             output: bool = False) -> Tuple[bool, str, str, str]:
     global logger
 
     success = True
@@ -135,7 +137,8 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir,
+                                                                                          compressed_only, checkout_to)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True, compressed_only)
             # Download from rubygems.org
@@ -155,6 +158,17 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
         success = False
         msg = str(error)
 
+    if output:
+        output_result = {
+            "success": success,
+            "message": msg,
+            "oss_name": oss_name,
+            "oss_version": oss_version
+        }
+        output_json = os.path.join(log_dir, "fosslight_download_output.json")
+        with open(output_json, 'w') as f:
+            json.dump(output_result, f, indent=4)
+
     logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})")
     return success, msg, oss_name, oss_version
 
@@ -297,7 +311,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="",
     return success, msg, oss_name, refs_to_checkout
 
 
-def download_wget(link, target_dir, compressed_only):
+def download_wget(link, target_dir, compressed_only, checkout_to):
     success = False
     msg = ""
     oss_name = ""
@@ -314,7 +328,7 @@ def download_wget(link, target_dir, compressed_only):
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link, checkout_to)
         if ret and new_link:
             link = new_link
 
@@ -478,10 +492,17 @@ def main():
     parser.add_argument('-s', '--source', help='Source link to download', type=str, dest='source')
     parser.add_argument('-t', '--target_dir', help='Target directory', type=str, dest='target_dir', default="")
     parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
+    parser.add_argument('-c', '--checkout_to', help='Checkout to  branch or tag', type=str, dest='checkout_to', default="")
+    parser.add_argument('-z', '--compressed_only', help='Unzip only compressed file',
+                        action='store_true', dest='compressed_only', default=False)
+    parser.add_argument('-o', '--output', help='Generate output file', action='store_true', dest='output', default=False)
 
     src_link = ""
     target_dir = os.getcwd()
     log_dir = os.getcwd()
+    checkout_to = ""
+    compressed_only = False
+    output = False
 
     try:
         args = parser.parse_args()
@@ -496,11 +517,19 @@ def main():
         target_dir = args.target_dir
     if args.log_dir:
         log_dir = args.log_dir
+    if args.checkout_to:
+        checkout_to = args.checkout_to
+    if args.compressed_only:
+        compressed_only = args.compressed_only
+    if args.output:
+        output = args.output
 
     if not src_link:
         print_help_msg_download()
     else:
-        cli_download_and_extract(src_link, target_dir, log_dir)
+        cli_download_and_extract(src_link, target_dir, log_dir, checkout_to,
+                                 compressed_only, "", "", "", False,
+                                 output)
 
 
 if __name__ == '__main__':

{fosslight_util-2.1.20 → fosslight_util-2.1.22}/src/fosslight_util/help.py

@@ -34,7 +34,12 @@ _HELP_MESSAGE_DOWNLOAD = """
     Optional:
         -h\t\t      Print help message
         -t\t\t      Output path name
-        -d\t\t      Directory name to save the log file"""
+        -d\t\t      Directory name to save the log file
+        -s\t\t      Source link to download
+        -t\t\t      Directory to download source code
+        -c\t\t      Checkout to branch or tag/ or version
+        -z\t\t      Unzip only compressed file
+        -o\t\t      Generate summary output file with this option"""
 
 
 class PrintHelpMsg():

{fosslight_util-2.1.20 → fosslight_util-2.1.22}/src/fosslight_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.1.20
+Version: 2.1.22
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util