fosslight-util 2.1.17__py3-none-any.whl → 2.1.19__py3-none-any.whl

fosslight_util/_get_downloadable_url.py

@@ -5,7 +5,6 @@
 import logging
 import re
 import requests
-from npm.bindings import npm_run
 from lastversion import latest
 from bs4 import BeautifulSoup
 from urllib.request import urlopen
@@ -47,6 +46,9 @@ def extract_name_version_from_link(link):
                         origin_name = origin_name[:-1]
                     oss_name = f"go:{origin_name}"
                     oss_version = match.group(2)
+                elif key == "cargo":
+                    oss_name = f"cargo:{origin_name}"
+                    oss_version = match.group(2)
             except Exception as ex:
                 logger.info(f"extract_name_version_from_link {key}:{ex}")
             if oss_name and (not oss_version):
@@ -63,10 +65,11 @@ def get_latest_package_version(link, pkg_type, oss_name):
 
     try:
         if pkg_type in ['npm', 'npm2']:
-            stderr, stdout = npm_run('view', oss_name, 'version')
-            if stdout:
-                find_version = stdout.strip()
-            link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
+            npm_response = requests.get(f"https://registry.npmjs.org/{oss_name}")
+            if npm_response.status_code == 200:
+                find_version = npm_response.json().get("dist-tags", {}).get("latest")
+            if find_version:
+                link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
         elif pkg_type == 'pypi':
             find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
             link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
@@ -75,17 +78,20 @@ def get_latest_package_version(link, pkg_type, oss_name):
             if maven_response.status_code == 200:
                 find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
             oss_name = oss_name.replace(':', '/')
-            link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
+            if find_version:
+                link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
         elif pkg_type == 'pub':
             pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
             if pub_response.status_code == 200:
                 find_version = pub_response.json().get('latest').get('version')
-            link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
+            if find_version:
+                link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
         elif pkg_type == 'go':
             go_response = requests.get(f'https://proxy.golang.org/{oss_name}/@latest')
             if go_response.status_code == 200:
                 find_version = go_response.json().get('Version')
-            link_with_version = f'https://pkg.go.dev/{oss_name}@{find_version}'
+            if find_version:
+                link_with_version = f'https://pkg.go.dev/{oss_name}@{find_version}'
     except Exception as e:
         logger.info(f'Fail to get latest package version({link}:{e})')
     return find_version, link_with_version
@@ -110,8 +116,34 @@ def get_downloadable_url(link):
         ret, result_link = get_download_location_for_pub(new_link)
     elif pkg_type == "go":
         ret, result_link = get_download_location_for_go(new_link)
+    elif pkg_type == "cargo":
+        ret, result_link = get_download_location_for_cargo(new_link)
+    return ret, result_link, oss_name, oss_version, pkg_type
+
+
+def get_download_location_for_cargo(link):
+    # get the url for downloading source file: https://crates.io/api/v1/crates/<name>/<version>/download
+    ret = False
+    new_link = ''
+    host = 'https://crates.io/api/v1/crates'
+
+    try:
+        dn_loc_re = re.findall(r'crates.io\/crates\/([^\/]+)\/?([^\/]*)', link)
+        if dn_loc_re:
+            oss_name = dn_loc_re[0][0]
+            oss_version = dn_loc_re[0][1]
 
-    return ret, result_link, oss_name, oss_version
+            new_link = f'{host}/{oss_name}/{oss_version}/download'
+            res = urlopen(new_link)
+            if res.getcode() == 200:
+                ret = True
+            else:
+                logger.warning(f'Cannot find the valid link for cargo (url:{new_link}')
+    except Exception as error:
+        ret = False
+        logger.warning(f'Cannot find the link for cargo (url:{link}({(new_link)})): {error}')
+
+    return ret, new_link
 
 
 def get_download_location_for_go(link):

fosslight_util/constant.py

@@ -35,6 +35,7 @@ SHEET_NAME_FOR_SCANNER = {
 # pub: https://pub.dev/packages/(package)/versions/(version)
 # Cocoapods : https://cocoapods.org/(package)
 # go : https://pkg.go.dev/(package_name_with_slash)@(version)
+# cargo : https://crates.io/crates/(crate_name)/(version)
 PKG_PATTERN = {
     "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
     "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
@@ -43,5 +44,6 @@ PKG_PATTERN = {
     "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
     "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
     "cocoapods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)',
-    "go": r'https?:\/\/pkg.go.dev\/([^\@]+)\@?v?([^\/]*)'
+    "go": r'https?:\/\/pkg.go.dev\/([^\@]+)\@?v?([^\/]*)',
+    "cargo": r'https?:\/\/crates\.io\/crates\/([^\/]+)\/?([^\/]*)',
 }

fosslight_util/download.py

@@ -314,7 +314,7 @@ def download_wget(link, target_dir, compressed_only):
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link, oss_name, oss_version = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link)
         if ret and new_link:
             link = new_link
 
@@ -323,6 +323,9 @@ def download_wget(link, target_dir, compressed_only):
                 if link.endswith(ext):
                     success = True
                     break
+            if not success:
+                if pkg_type == 'cargo':
+                    success = True
         else:
             success = True
 
@@ -330,7 +333,11 @@ def download_wget(link, target_dir, compressed_only):
             raise Exception('Not supported compression type (link:{0})'.format(link))
 
         logger.info(f"wget: {link}")
-        downloaded_file = wget.download(link, target_dir)
+        if pkg_type == 'cargo':
+            outfile = os.path.join(target_dir, f'{oss_name}.tar.gz')
+            downloaded_file = wget.download(link, out=outfile)
+        else:
+            downloaded_file = wget.download(link, target_dir)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:

{fosslight_util-2.1.17.dist-info → fosslight_util-2.1.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.1.17
+Version: 2.1.19
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -27,7 +27,6 @@ Requires-Dist: python3-wget
 Requires-Dist: beautifulsoup4
 Requires-Dist: jsonmerge
 Requires-Dist: setuptools>=65.5.1
-Requires-Dist: npm
 Requires-Dist: requests
 Requires-Dist: GitPython
 Requires-Dist: numpy; python_version < "3.8"

{fosslight_util-2.1.17.dist-info → fosslight_util-2.1.19.dist-info}/RECORD

@@ -1,10 +1,10 @@
 fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_util/_get_downloadable_url.py,sha256=5QBEM1Ac_T3vhBUPJYSMkaGrI1A_f3VWDptxfVwMiy8,10976
+fosslight_util/_get_downloadable_url.py,sha256=TbzzW-MJky-A610mgw5bpf6xmELyQAwXo9jfubi05d8,12271
 fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE4,2916
-fosslight_util/constant.py,sha256=Ig3ACm9_QirE4389Wt-IfxOqRkVOUjqGnX1B05z2Byo,2151
+fosslight_util/constant.py,sha256=zElnWOzXt020sYiFTiRQn8ZjZyZpL3aPmfAqfQLcxJk,2278
 fosslight_util/correct.py,sha256=1WEAL-9_KhjFPLucPhv0PNN3K7avm0z8mU6sTuSyeHM,3864
 fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
-fosslight_util/download.py,sha256=b55ZDso1nnAwIQwMyAYyxnJcvTYQx8eGVcIg9D1CpCc,17347
+fosslight_util/download.py,sha256=V3EBgXt-xHarJZFrskXHaX4Ij81ZPjj6hzvtmpKu7xE,17642
 fosslight_util/exclude.py,sha256=fDmBsZJ_F7O9Oh2T-07R03XNbElo1tFaf_z01KfSAqU,2399
 fosslight_util/help.py,sha256=Bmyz-eFP0X0qUfgFPrWiuyUPE0TLQfWjgfHTzJBIInc,2377
 fosslight_util/oss_item.py,sha256=8W2HlwqGH3l1iPPdvycrRYKsBSBpqAkqYyYtBVPgMtY,6868
@@ -24,9 +24,9 @@ fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.1.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.1.17.dist-info/METADATA,sha256=dG34Ulv95GV7bhARLITbYUWmEe52VqazekVCRfQ0vqs,6500
-fosslight_util-2.1.17.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-fosslight_util-2.1.17.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-2.1.17.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.1.17.dist-info/RECORD,,
+fosslight_util-2.1.19.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.19.dist-info/METADATA,sha256=rqYqFM4FZEKmhmATRltYAJreokGayqWFTX2tENCgje4,6481
+fosslight_util-2.1.19.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+fosslight_util-2.1.19.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-2.1.19.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.19.dist-info/RECORD,,