fosslight-util 2.1.21__py3-none-any.whl → 2.1.23__py3-none-any.whl

fosslight_util/_get_downloadable_url.py

@@ -13,7 +13,7 @@ import fosslight_util.constant as constant
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
-def extract_name_version_from_link(link):
+def extract_name_version_from_link(link, checkout_version):
     oss_name = ""
     oss_version = ""
     matched = False
@@ -53,8 +53,12 @@ def extract_name_version_from_link(link):
             except Exception as ex:
                 logger.info(f"extract_name_version_from_link {key}:{ex}")
             if oss_name and (not oss_version):
-                if key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
-                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                if checkout_version:
+                    oss_version = checkout_version
+                elif key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
+                    oss_version = get_latest_package_version(link, key, origin_name)
+                if oss_version:
+                    link = get_new_link_with_version(link, key, origin_name, oss_version)
                     logger.info(f'Try to download with the latest version:{link}')
             matched = True
             break
@@ -63,50 +67,56 @@ def extract_name_version_from_link(link):
     return oss_name, oss_version, link, key
 
 
+def get_new_link_with_version(link, pkg_type, oss_name, oss_version):
+    if pkg_type == "pypi":
+        link = f'https://pypi.org/project/{oss_name}/{oss_version}'
+    elif pkg_type == "maven":
+        oss_name = oss_name.replace(':', '/')
+        link = f'https://mvnrepository.com/artifact/{oss_name}/{oss_version}'
+    elif pkg_type == "npm" or pkg_type == "npm2":
+        link = f'https://www.npmjs.com/package/{oss_name}/v/{oss_version}'
+    elif pkg_type == "pub":
+        link = f'https://pub.dev/packages/{oss_name}/versions/{oss_version}'
+    elif pkg_type == "go":
+        link = f'https://pkg.go.dev/{oss_name}@{oss_version}'
+    elif pkg_type == "cargo":
+        link = f'https://crates.io/crates/{oss_name}/{oss_version}'
+    return link
+
+
 def get_latest_package_version(link, pkg_type, oss_name):
     find_version = ''
-    link_with_version = link
 
     try:
         if pkg_type in ['npm', 'npm2']:
             npm_response = requests.get(f"https://registry.npmjs.org/{oss_name}")
             if npm_response.status_code == 200:
                 find_version = npm_response.json().get("dist-tags", {}).get("latest")
-            if find_version:
-                link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
         elif pkg_type == 'pypi':
             find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
-            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
         elif pkg_type == 'maven':
             maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
             if maven_response.status_code == 200:
                 find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
-            oss_name = oss_name.replace(':', '/')
-            if find_version:
-                link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
         elif pkg_type == 'pub':
             pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
             if pub_response.status_code == 200:
                 find_version = pub_response.json().get('latest').get('version')
-            if find_version:
-                link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
         elif pkg_type == 'go':
             go_response = requests.get(f'https://proxy.golang.org/{oss_name}/@latest')
             if go_response.status_code == 200:
                 find_version = go_response.json().get('Version')
-            if find_version:
-                link_with_version = f'https://pkg.go.dev/{oss_name}@{find_version}'
     except Exception as e:
         logger.info(f'Fail to get latest package version({link}:{e})')
-    return find_version, link_with_version
+    return find_version
 
 
-def get_downloadable_url(link):
+def get_downloadable_url(link, checkout_version):
 
     ret = False
     result_link = link
 
-    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link)
+    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link, checkout_version)
     new_link = new_link.replace('http://', '')
     new_link = new_link.replace('https://', '')
 
@@ -180,6 +190,28 @@ def get_download_location_for_go(link):
     return ret, new_link
 
 
+def get_available_wheel_urls(name, version):
+    try:
+        api_url = f'https://pypi.org/pypi/{name}/{version}/json'
+        response = requests.get(api_url)
+        if response.status_code == 200:
+            data = response.json()
+            wheel_urls = []
+
+            for file_info in data.get('urls', []):
+                if file_info.get('packagetype') == 'bdist_wheel':
+                    wheel_urls.append(file_info.get('url'))
+
+            return wheel_urls
+        else:
+            logger.warning(f'Cannot get PyPI API data for {name}({version})')
+            return []
+
+    except Exception as error:
+        logger.warning(f'Failed to get wheel URLs from PyPI API: {error}')
+        return []
+
+
 def get_download_location_for_pypi(link):
     # get the url for downloading source file: https://docs.pypi.org/api/ Predictable URLs
     ret = False
@@ -192,24 +224,56 @@ def get_download_location_for_pypi(link):
         oss_name = re.sub(r"[-_.]+", "-", oss_name)
         oss_version = dn_loc_re[0][1]
 
+        # 1. Source distribution 시도
         new_link = f'{host}/packages/source/{oss_name[0]}/{oss_name}/{oss_name}-{oss_version}.tar.gz'
         try:
             res = urlopen(new_link)
             if res.getcode() == 200:
                 ret = True
-            else:
-                logger.warning(f'Cannot find the valid link for pypi (url:{new_link}')
+                return ret, new_link
         except Exception:
             oss_name = re.sub(r"[-]+", "_", oss_name)
             new_link = f'{host}/packages/source/{oss_name[0]}/{oss_name}/{oss_name}-{oss_version}.tar.gz'
-            res = urlopen(new_link)
-            if res.getcode() == 200:
-                ret = True
-            else:
-                logger.warning(f'Cannot find the valid link for pypi (url:{new_link}')
+            try:
+                res = urlopen(new_link)
+                if res.getcode() == 200:
+                    ret = True
+                    return ret, new_link
+            except Exception:
+                pass
+
+        # 2. Source distribution이 없으면 wheel 파일들을 시도
+        wheel_urls = get_available_wheel_urls(oss_name, oss_version)
+
+        if wheel_urls:
+            # Pure Python wheel을 우선적으로 찾기
+            for wheel_url in wheel_urls:
+                if 'py3-none-any' in wheel_url or 'py2.py3-none-any' in wheel_url:
+                    try:
+                        res = urlopen(wheel_url)
+                        if res.getcode() == 200:
+                            ret = True
+                            new_link = wheel_url
+                            logger.info(f'Using wheel file : {wheel_url}')
+                            return ret, new_link
+                    except Exception:
+                        continue
+
+            # Pure Python wheel이 없으면 첫 번째 wheel 시도
+            if wheel_urls:
+                try:
+                    res = urlopen(wheel_urls[0])
+                    if res.getcode() == 200:
+                        ret = True
+                        new_link = wheel_urls[0]
+                        logger.info(f'Using wheel file : {wheel_urls[0]}')
+                        return ret, new_link
+                except Exception:
+                    pass
+
     except Exception as error:
         ret = False
-        logger.warning(f'Cannot find the link for pypi (url:{link}({(new_link)})) e:{str(error)}')
+        logger.warning(f'Cannot find the link for pypi (url:{link}({new_link})) e:{str(error)}')
 
     return ret, new_link
 

fosslight_util/download.py

@@ -137,7 +137,8 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir,
+                                                                                          compressed_only, checkout_to)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True, compressed_only)
             # Download from rubygems.org
@@ -310,7 +311,7 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="",
     return success, msg, oss_name, refs_to_checkout
 
 
-def download_wget(link, target_dir, compressed_only):
+def download_wget(link, target_dir, compressed_only, checkout_to):
     success = False
     msg = ""
     oss_name = ""
@@ -327,7 +328,7 @@ def download_wget(link, target_dir, compressed_only):
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link, checkout_to)
         if ret and new_link:
             link = new_link
 
@@ -399,6 +400,8 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True, comp
                 unzip(fname, extract_path)
             elif fname.endswith(".bz2"):
                 decompress_bz2(fname, extract_path)
+            elif fname.endswith(".whl"):
+                unzip(fname, extract_path)
             else:
                 is_compressed_file = False
                 if compressed_only:

fosslight_util/help.py

@@ -34,7 +34,12 @@ _HELP_MESSAGE_DOWNLOAD = """
     Optional:
         -h\t\t      Print help message
         -t\t\t      Output path name
-        -d\t\t      Directory name to save the log file"""
+        -d\t\t      Directory name to save the log file
+        -s\t\t      Source link to download
+        -t\t\t      Directory to download source code
+        -c\t\t      Checkout to branch or tag/ or version
+        -z\t\t      Unzip only compressed file
+        -o\t\t      Generate summary output file with this option"""
 
 
 class PrintHelpMsg():

{fosslight_util-2.1.21.dist-info → fosslight_util-2.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.1.21
+Version: 2.1.23
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util

{fosslight_util-2.1.21.dist-info → fosslight_util-2.1.23.dist-info}/RECORD

@@ -1,12 +1,12 @@
 fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_util/_get_downloadable_url.py,sha256=p6vSYr6BdSwqafqMX3Cl5M45MSE9N02w512Mh5C2Hyo,12355
+fosslight_util/_get_downloadable_url.py,sha256=zCBSVNTEumVxeLyaW5l4HKYKt73NLbC2y7gZoAhuWYA,14618
 fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE4,2916
 fosslight_util/constant.py,sha256=zElnWOzXt020sYiFTiRQn8ZjZyZpL3aPmfAqfQLcxJk,2278
 fosslight_util/correct.py,sha256=1WEAL-9_KhjFPLucPhv0PNN3K7avm0z8mU6sTuSyeHM,3864
 fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
-fosslight_util/download.py,sha256=A-L9zBVUDoP6LQrAaVv3IzpEldOSxE3lBMqNOYTaYVU,18856
+fosslight_util/download.py,sha256=F_cbweXKkr4pSRWRbvsuwK_pMjorneJBkPbTy2bEdJc,19069
 fosslight_util/exclude.py,sha256=fDmBsZJ_F7O9Oh2T-07R03XNbElo1tFaf_z01KfSAqU,2399
-fosslight_util/help.py,sha256=Bmyz-eFP0X0qUfgFPrWiuyUPE0TLQfWjgfHTzJBIInc,2377
+fosslight_util/help.py,sha256=iyWmAaUQSHJtWv5mjFv0f3YoDVlDgEqdsDDEyImEUNc,2646
 fosslight_util/oss_item.py,sha256=8W2HlwqGH3l1iPPdvycrRYKsBSBpqAkqYyYtBVPgMtY,6868
 fosslight_util/output_format.py,sha256=BP23LspxawDZ_a99oWLVKWUQ-G7P5uoUpjEXhkRFKwc,8801
 fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
@@ -24,9 +24,9 @@ fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.1.21.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.1.21.dist-info/METADATA,sha256=BaqzJe4829VFc5HECJui2DBzr8_IhlVEPWNPnUimJa0,6156
-fosslight_util-2.1.21.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-fosslight_util-2.1.21.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
-fosslight_util-2.1.21.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.1.21.dist-info/RECORD,,
+fosslight_util-2.1.23.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.23.dist-info/METADATA,sha256=qcT5R4XpXKm7Zeclyj_1TosxMzzE2H0-FvNvOIGkV_Q,6156
+fosslight_util-2.1.23.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+fosslight_util-2.1.23.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
+fosslight_util-2.1.23.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.23.dist-info/RECORD,,