fosslight-util 1.4.34__tar.gz → 1.4.36__tar.gz

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight_util
-Version: 1.4.34
+Version: 1.4.36
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/requirements.txt

@@ -10,7 +10,8 @@ python3-wget
 beautifulsoup4
 jsonmerge
 spdx-tools==0.7.0rc0
-npm
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
 numpy; python_version < '3.8'
-numpy>=1.22.2; python_version >= '3.8'
+numpy>=1.22.2; python_version >= '3.8'
+npm
+requests

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='1.4.34',
+        version='1.4.36',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

fosslight_util-1.4.36/src/fosslight_util/_get_downloadable_url.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import re
+import requests
+from npm.bindings import npm_run
+from lastversion import latest
+from bs4 import BeautifulSoup
+from urllib.request import urlopen
+import fosslight_util.constant as constant
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+def extract_name_version_from_link(link):
+    # Github : https://github.com/(owner)/(repo)
+    # npm : https://www.npmjs.com/package/(package)/v/(version)
+    # npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
+    # pypi : https://pypi.org/project/(oss_name)/(version)
+    # pypi2 : https://files.pythonhosted.org/packages/source/(alphabet)/(oss_name)/(oss_name)-(version).tar.gz
+    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)/(version)
+    # pub: https://pub.dev/packages/(package)/versions/(version)
+    # Cocoapods: https://cocoapods.org/(package)
+    pkg_pattern = {
+        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
+        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
+        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)\/?([^\/]*)',
+        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/\@]+)(?:\/v\/)?([^\/]*)',
+        "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
+        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
+        "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
+    }
+    oss_name = ""
+    oss_version = ""
+    if link.startswith("www."):
+        link = link.replace("www.", "https://www.", 1)
+    for key, value in pkg_pattern.items():
+        p = re.compile(value)
+        match = p.match(link)
+        if match:
+            try:
+                origin_name = match.group(1)
+                if (key == "pypi") or (key == "pypi2"):
+                    oss_name = f"pypi:{origin_name}"
+                    oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
+                    oss_version = match.group(2)
+                elif key == "maven":
+                    artifact = match.group(2)
+                    oss_name = f"{origin_name}:{artifact}"
+                    origin_name = oss_name
+                    oss_version = match.group(3)
+                elif key == "npm" or key == "npm2":
+                    oss_name = f"npm:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "pub":
+                    oss_name = f"pub:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "pods":
+                    oss_name = f"cocoapods:{origin_name}"
+            except Exception as ex:
+                logger.info(f"extract_name_version_from_link {key}:{ex}")
+            if oss_name and (not oss_version):
+                if key in ["pypi", "maven", "npm", "npm2", "pub"]:
+                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                    logger.debug(f'Try to download with the latest version:{link}')
+            break
+    return oss_name, oss_version, link, key
+
+
+def get_latest_package_version(link, pkg_type, oss_name):
+    find_version = ''
+    link_with_version = link
+
+    try:
+        if pkg_type in ['npm', 'npm2']:
+            stderr, stdout = npm_run('view', oss_name, 'version')
+            if stdout:
+                find_version = stdout.strip()
+            link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
+        elif pkg_type == 'pypi':
+            find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
+            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
+        elif pkg_type == 'maven':
+            maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
+            if maven_response.status_code == 200:
+                find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
+            oss_name = oss_name.replace(':', '/')
+            link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
+        elif pkg_type == 'pub':
+            pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
+            if pub_response.status_code == 200:
+                find_version = pub_response.json().get('latest').get('version')
+            link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
+    except Exception as e:
+        logger.debug(f'Fail to get latest package version({link}:{e})')
+    return find_version, link_with_version
+
+
+def get_downloadable_url(link):
+
+    ret = False
+    result_link = link
+
+    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link)
+    new_link = new_link.replace('http://', '')
+    new_link = new_link.replace('https://', '')
+
+    if pkg_type == "pypi":
+        ret, result_link = get_download_location_for_pypi(new_link)
+    elif pkg_type == "maven" or new_link.startswith('repo1.maven.org/'):
+        ret, result_link = get_download_location_for_maven(new_link)
+    elif (pkg_type in ["npm", "npm2"]) or new_link.startswith('registry.npmjs.org/'):
+        ret, result_link = get_download_location_for_npm(new_link)
+    elif pkg_type == "pub":
+        ret, result_link = get_download_location_for_pub(new_link)
+
+    return ret, result_link, oss_name, oss_version
+
+
+def get_download_location_for_pypi(link):
+    # get the url for downloading source file in pypi.org/project/(oss_name)/(oss_version)/#files
+    ret = False
+    new_link = ''
+
+    try:
+        dn_loc_re = re.findall(r'pypi.org\/project\/?([^\/]*)\/?([^\/]*)', link)
+        oss_name = dn_loc_re[0][0]
+        oss_version = dn_loc_re[0][1]
+
+        pypi_url = 'https://pypi.org/project/' + oss_name + '/' + oss_version + '/#files'
+
+        content = urlopen(pypi_url).read().decode('utf8')
+        bs_obj = BeautifulSoup(content, 'html.parser')
+
+        card_file_list = bs_obj.findAll('div', {'class': 'card file__card'})
+
+        for card_file in card_file_list:
+            file_code = card_file.find('code').text
+            if file_code == "source":
+                new_link = card_file.find('a').attrs['href']
+                ret = True
+                break
+    except Exception as error:
+        ret = False
+        logger.warning('Cannot find the link for pypi (url:'+link+') '+str(error))
+
+    return ret, new_link
+
+
+def get_download_location_for_maven(link):
+    # get the url for downloading source file in
+    # repo1.maven.org/maven2/(group_id(split to separator '/'))/(artifact_id)/(oss_version)
+    ret = False
+    new_link = ''
+
+    try:
+        if link.startswith('mvnrepository.com/artifact/'):
+            dn_loc_split = link.replace('mvnrepository.com/', '').split('/')
+            group_id = dn_loc_split[1].replace('.', '/')
+            dn_loc = 'https://repo1.maven.org/maven2/' + group_id + '/' + dn_loc_split[2] + '/' + dn_loc_split[3]
+
+        elif link.startswith('repo1.maven.org/maven2/'):
+            dn_loc_split = link.replace('repo1.maven.org/maven2/', '').split('/')
+
+            if link.endswith('.tar.gz') or link.endswith('.jar') or link.endswith('.tar.xz'):
+                new_link = 'https://' + link
+                ret = True
+                return ret, new_link
+            else:
+                dn_loc = 'https://' + link
+        else:
+            raise Exception("not valid url for maven")
+
+        html = urlopen(dn_loc).read().decode('utf8')
+        bs_obj = BeautifulSoup(html, 'html.parser')
+
+        file_name = dn_loc.split('/')[-2] + '-' + dn_loc.split('/')[-1] + '-sources.jar'
+
+        for link in bs_obj.findAll("a"):
+            if link.text == file_name:
+                source_url = link['href']
+                new_link = dn_loc + '/' + source_url
+                break
+            elif link['href'].endswith('sources.jar') or link['href'].endswith('source.jar') or link['href'].endswith('src.jar'):
+                source_url = link['href']
+                new_link = dn_loc + '/' + source_url
+
+        if new_link != '':
+            ret = True
+
+    except Exception as error:
+        ret = False
+        logger.warning('Cannot find the link for maven (url:'+link+') '+str(error))
+
+    return ret, new_link
+
+
+def get_download_location_for_npm(link):
+    # url format : registry.npmjs.org/packagename/-/packagename-version.tgz
+    ret = False
+    new_link = ''
+    oss_version = ""
+    oss_name_npm = ""
+    tar_name = ""
+
+    link = link.replace('%40', '@')
+    if link.startswith('www.npmjs.com/') or link.startswith('registry.npmjs.org/'):
+        try:
+            dn_loc_split = link.split('/')
+            if dn_loc_split[1] == 'package':
+                idx = 2
+            else:
+                idx = 1
+            if dn_loc_split[idx].startswith('@'):
+                oss_name_npm = dn_loc_split[idx]+'/'+dn_loc_split[idx+1]
+                tar_name = dn_loc_split[idx+1]
+                oss_version = dn_loc_split[idx+3]
+            else:
+                oss_name_npm = dn_loc_split[idx]
+                tar_name = oss_name_npm
+                oss_version = dn_loc_split[idx+2]
+
+            tar_name = f'{tar_name}-{oss_version}'
+            new_link = f'https://registry.npmjs.org/{oss_name_npm}/-/{tar_name}.tgz'
+            ret = True
+        except Exception as error:
+            ret = False
+            logger.warning('Cannot find the link for npm (url:'+link+') '+str(error))
+    return ret, new_link
+
+
+def get_download_location_for_pub(link):
+    ret = False
+    new_link = ''
+
+    # url format : https://pub.dev/packages/(oss_name)/versions/(oss_version)
+    # download url format : https://pub.dev/packages/(oss_name)/versions/(oss_version).tar.gz
+    try:
+        if link.startswith('pub.dev/packages'):
+            new_link = f'https://{link}.tar.gz'
+            ret = True
+
+    except Exception as error:
+        ret = False
+        logger.warning('Cannot find the link for pub (url:'+link+') '+str(error))
+
+    return ret, new_link

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/src/fosslight_util/download.py

@@ -24,6 +24,7 @@ import time
 import threading
 import platform
 import subprocess
+import re
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
@@ -115,6 +116,8 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
 
     success = True
     msg = ""
+    oss_name = ""
+    oss_version = ""
     log_file_name = "fosslight_download_" + \
         datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
     logger, log_item = init_log(os.path.join(log_dir, log_file_name))
@@ -135,22 +138,29 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
             is_rubygems = src_info.get("rubygems", False)
 
             # General download (git clone, wget)
-            if (not is_rubygems) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)):
+            success_git, msg, oss_name = download_git_clone(link, target_dir, checkout_to, tag, branch)
+            if (not is_rubygems) and (not success_git):
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True)
             # Download from rubygems.org
             elif is_rubygems and shutil.which("gem"):
                 success = gem_download(link, target_dir, checkout_to)
+        if msg:
+            msg = f'git fail: {msg}'
+            if msg_wget:
+                msg = f'{msg}, wget fail: {msg_wget}'
+            else:
+                msg = f'{msg}, wget success'
     except Exception as error:
         success = False
         msg = str(error)
 
-    logger.info(f"\n* FOSSLight Downloader - Result: {success}\n {msg}")
-    return success, msg
+    logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})")
+    return success, msg, oss_name, oss_version
 
 
 def get_ref_to_checkout(checkout_to, ref_list):
@@ -184,8 +194,19 @@ def decide_checkout(checkout_to="", tag="", branch=""):
     return ref_to_checkout
 
 
+def get_github_ossname(link):
+    oss_name = ""
+    p = re.compile(r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?')
+    match = p.match(link)
+    if match:
+        oss_name = f"{match.group(1)}-{match.group(2)}"
+    return oss_name
+
+
 def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
     ref_to_checkout = decide_checkout(checkout_to, tag, branch)
+    msg = ""
+    oss_name = get_github_ossname(git_url)
 
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -204,7 +225,8 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             del alarm
     except Exception as error:
         logger.warning(f"git clone - failed: {error}")
-        return False
+        msg = str(error)
+        return False, msg, oss_name
     try:
         if ref_to_checkout != "":
             ref_list = [x for x in repo.references]
@@ -213,11 +235,14 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             repo.checkout(ref_to_checkout)
     except Exception as error:
         logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}")
-    return True
+    return True, msg, oss_name
 
 
 def download_wget(link, target_dir, compressed_only):
     success = False
+    msg = ""
+    oss_name = ""
+    oss_version = ""
     downloaded_file = ""
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -228,7 +253,7 @@ def download_wget(link, target_dir, compressed_only):
     try:
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version = get_downloadable_url(link)
         if ret and new_link:
             link = new_link
 
@@ -255,9 +280,10 @@ def download_wget(link, target_dir, compressed_only):
             logger.debug(f"wget - downloaded: {downloaded_file}")
     except Exception as error:
         success = False
+        msg = str(error)
         logger.warning(f"wget - failed: {error}")
 
-    return success, downloaded_file
+    return success, downloaded_file, msg, oss_name, oss_version
 
 
 def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/src/fosslight_util/read_excel.py

@@ -3,6 +3,7 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import logging
+from typing import List, Dict, Any
 import xlrd
 import json
 from fosslight_util.constant import LOGGER_NAME
@@ -12,17 +13,18 @@ from fosslight_util.parsing_yaml import set_value_switch
 logger = logging.getLogger(LOGGER_NAME)
 IDX_CANNOT_FOUND = -1
 PREFIX_BIN = "bin"
+SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
 xlrd.xlsx.ensure_elementtree_imported(False, None)
 xlrd.xlsx.Element_has_iter = True
 
 
-def read_oss_report(excel_file, sheet_names=""):
-    _oss_report_items = []
-    xl_sheets = {}
-    all_sheet_to_read = []
-    not_matched_sheet = []
+def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
+    oss_report_items: List[OssItem] = []
+    xl_sheets: Dict[str, Any] = {}
+    all_sheet_to_read: List[str] = []
+    not_matched_sheet: List[str] = []
     any_sheet_matched = False
-    SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
+
     if sheet_names:
         sheet_name_prefix_match = False
         sheet_name_to_read = sheet_names.split(",")
@@ -113,8 +115,8 @@ def read_oss_report(excel_file, sheet_names=""):
                             else:
                                 valid_row = False if cell_value == "-" else True
                 if valid_row and load_data_cnt > 0:
-                    _oss_report_items.append(item)
+                    oss_report_items.append(item)
 
     except Exception as error:
         logger.error(f"Parsing a OSS Report: {error}")
-    return _oss_report_items
+    return oss_report_items

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/src/fosslight_util/write_excel.py

@@ -129,6 +129,10 @@ def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, ex
                 row_num = 1
                 header_row, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
 
+                if 'Copyright Text' in header_row:
+                    idx = header_row.index('Copyright Text')-1
+                    for item in sheet_content_without_header:
+                        item[idx] = item[idx].replace('\n', ', ')
                 if not separate_sheet:
                     merge_sheet.extend(sheet_content_without_header)
                     if sheet_name == list(sheet_list.keys())[-1]:

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/src/fosslight_util/write_scancodejson.py

@@ -7,12 +7,14 @@ import logging
 import os
 import json
 import fosslight_util.constant as constant
+from fosslight_util.oss_item import OssItem
+from typing import List
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 EMPTY_FILE_PATH = '-'
 
 
-def write_scancodejson(output_dir, output_filename, oss_list):
+def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[OssItem]):
     json_output = {}
     json_output['headers'] = []
     json_output['summary'] = {}

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/src/fosslight_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 1.4.34
+Version: 1.4.36
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics

{fosslight_util-1.4.34 → fosslight_util-1.4.36}/src/fosslight_util.egg-info/requires.txt

@@ -10,8 +10,9 @@ python3-wget
 beautifulsoup4
 jsonmerge
 spdx-tools==0.7.0rc0
-npm
 setuptools>=65.5.1
+npm
+requests
 
 [:python_version < "3.8"]
 numpy

fosslight_util-1.4.34/src/fosslight_util/_get_downloadable_url.py

@@ -1,168 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020 LG Electronics Inc.
-# SPDX-License-Identifier: Apache-2.0
-import logging
-import re
-from bs4 import BeautifulSoup
-from urllib.request import urlopen
-import fosslight_util.constant as constant
-from npm.bindings import npm_run
-
-logger = logging.getLogger(constant.LOGGER_NAME)
-
-
-def get_downloadable_url(link):
-
-    ret = False
-    new_link = ''
-
-    link = link.replace('http://', '')
-    link = link.replace('https://', '')
-
-    if link.startswith('pypi.org/'):
-        ret, new_link = get_download_location_for_pypi(link)
-    elif link.startswith('mvnrepository.com/artifact/') or link.startswith('repo1.maven.org/'):
-        ret, new_link = get_download_location_for_maven(link)
-    elif link.startswith('www.npmjs.com/') or link.startswith('registry.npmjs.org/'):
-        ret, new_link = get_download_location_for_npm(link)
-    elif link.startswith('pub.dev/'):
-        ret, new_link = get_download_location_for_pub(link)
-
-    return ret, new_link
-
-
-def get_download_location_for_pypi(link):
-    # get the url for downloading source file in pypi.org/project/(oss_name)/(oss_version)/#files
-    ret = False
-    new_link = ''
-
-    try:
-        dn_loc_re = re.findall(r'pypi.org\/project\/?([^\/]*)\/?([^\/]*)', link)
-        oss_name = dn_loc_re[0][0]
-        oss_version = dn_loc_re[0][1]
-
-        pypi_url = 'https://pypi.org/project/' + oss_name + '/' + oss_version + '/#files'
-
-        content = urlopen(pypi_url).read().decode('utf8')
-        bs_obj = BeautifulSoup(content, 'html.parser')
-
-        card_file_list = bs_obj.findAll('div', {'class': 'card file__card'})
-
-        for card_file in card_file_list:
-            file_code = card_file.find('code').text
-            if file_code == "source":
-                new_link = card_file.find('a').attrs['href']
-                ret = True
-                break
-    except Exception as error:
-        ret = False
-        logger.warning('Cannot find the link for pypi (url:'+link+') '+str(error))
-
-    return ret, new_link
-
-
-def get_download_location_for_maven(link):
-    # get the url for downloading source file in
-    # repo1.maven.org/maven2/(group_id(split to separator '/'))/(artifact_id)/(oss_version)
-    ret = False
-    new_link = ''
-
-    try:
-        if link.startswith('mvnrepository.com/artifact/'):
-            dn_loc_split = link.replace('mvnrepository.com/', '').split('/')
-            group_id = dn_loc_split[1].replace('.', '/')
-            dn_loc = 'https://repo1.maven.org/maven2/' + group_id + '/' + dn_loc_split[2] + '/' + dn_loc_split[3]
-
-        elif link.startswith('repo1.maven.org/maven2/'):
-            dn_loc_split = link.replace('repo1.maven.org/maven2/', '').split('/')
-
-            if link.endswith('.tar.gz') or link.endswith('.jar') or link.endswith('.tar.xz'):
-                new_link = 'https://' + link
-                ret = True
-                return ret, new_link
-            else:
-                dn_loc = 'https://' + link
-        else:
-            raise Exception("not valid url for maven")
-
-        html = urlopen(dn_loc).read().decode('utf8')
-        bs_obj = BeautifulSoup(html, 'html.parser')
-
-        file_name = dn_loc.split('/')[-2] + '-' + dn_loc.split('/')[-1] + '-sources.jar'
-
-        for link in bs_obj.findAll("a"):
-            if link.text == file_name:
-                source_url = link['href']
-                new_link = dn_loc + '/' + source_url
-                break
-            elif link['href'].endswith('sources.jar') or link['href'].endswith('source.jar') or link['href'].endswith('src.jar'):
-                source_url = link['href']
-                new_link = dn_loc + '/' + source_url
-
-        if new_link != '':
-            ret = True
-
-    except Exception as error:
-        ret = False
-        logger.warning('Cannot find the link for maven (url:'+link+') '+str(error))
-
-    return ret, new_link
-
-
-def get_download_location_for_npm(link):
-    # url format : registry.npmjs.org/packagename/-/packagename-version.tgz
-    ret = False
-    new_link = ''
-    oss_version = ""
-    oss_name_npm = ""
-    tar_name = ""
-
-    if link.startswith('www.npmjs.com/') or link.startswith('registry.npmjs.org/'):
-        try:
-            dn_loc_split = link.split('/')
-            if dn_loc_split[1] == 'package':
-                idx = 2
-            else:
-                idx = 1
-            if dn_loc_split[idx].startswith('@'):
-                oss_name_npm = dn_loc_split[idx]+'/'+dn_loc_split[idx+1]
-                tar_name = dn_loc_split[idx+1]
-                oss_version = dn_loc_split[idx+3]
-            else:
-                oss_name_npm = dn_loc_split[idx]
-                tar_name = oss_name_npm
-                oss_version = dn_loc_split[idx+2]
-        except Exception:
-            pass
-
-        try:
-            if not oss_version:
-                stderr, stdout = npm_run('view', oss_name_npm, 'version')
-                if stdout:
-                    oss_version = stdout.strip()
-            tar_name = f"{tar_name}-{oss_version}"
-            new_link = 'https://registry.npmjs.org/' + oss_name_npm + '/-/' + tar_name + '.tgz'
-            ret = True
-        except Exception as error:
-            ret = False
-            logger.warning('Cannot find the link for npm (url:'+link+') '+str(error))
-    return ret, new_link
-
-
-def get_download_location_for_pub(link):
-    ret = False
-    new_link = ''
-
-    # url format : https://pub.dev/packages/(oss_name)/versions/(oss_version)
-    # download url format : https://pub.dev/packages/(oss_name)/versions/(oss_version).tar.gz
-    try:
-        if link.startswith('pub.dev/packages'):
-            new_link = 'https://{link}.tar.gz'
-            ret = True
-
-    except Exception as error:
-        ret = False
-        logger.warning('Cannot find the link for pub (url:'+link+') '+str(error))
-
-    return ret, new_link