fosslight-util 1.4.34__py3-none-any.whl → 1.4.36__py3-none-any.whl

fosslight_util/_get_downloadable_url.py

@@ -4,32 +4,119 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import re
+import requests
+from npm.bindings import npm_run
+from lastversion import latest
 from bs4 import BeautifulSoup
 from urllib.request import urlopen
 import fosslight_util.constant as constant
-from npm.bindings import npm_run
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 
 
+def extract_name_version_from_link(link):
+    # Github : https://github.com/(owner)/(repo)
+    # npm : https://www.npmjs.com/package/(package)/v/(version)
+    # npm2 : https://www.npmjs.com/package/@(group)/(package)/v/(version)
+    # pypi : https://pypi.org/project/(oss_name)/(version)
+    # pypi2 : https://files.pythonhosted.org/packages/source/(alphabet)/(oss_name)/(oss_name)-(version).tar.gz
+    # Maven: https://mvnrepository.com/artifact/(group)/(artifact)/(version)
+    # pub: https://pub.dev/packages/(package)/versions/(version)
+    # Cocoapods: https://cocoapods.org/(package)
+    pkg_pattern = {
+        "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
+        "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
+        "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)\/?([^\/]*)',
+        "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/\@]+)(?:\/v\/)?([^\/]*)',
+        "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
+        "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',
+        "pods": r'https?:\/\/cocoapods\.org\/pods\/([^\/]+)'
+    }
+    oss_name = ""
+    oss_version = ""
+    if link.startswith("www."):
+        link = link.replace("www.", "https://www.", 1)
+    for key, value in pkg_pattern.items():
+        p = re.compile(value)
+        match = p.match(link)
+        if match:
+            try:
+                origin_name = match.group(1)
+                if (key == "pypi") or (key == "pypi2"):
+                    oss_name = f"pypi:{origin_name}"
+                    oss_name = re.sub(r"[-_.]+", "-", oss_name).lower()
+                    oss_version = match.group(2)
+                elif key == "maven":
+                    artifact = match.group(2)
+                    oss_name = f"{origin_name}:{artifact}"
+                    origin_name = oss_name
+                    oss_version = match.group(3)
+                elif key == "npm" or key == "npm2":
+                    oss_name = f"npm:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "pub":
+                    oss_name = f"pub:{origin_name}"
+                    oss_version = match.group(2)
+                elif key == "pods":
+                    oss_name = f"cocoapods:{origin_name}"
+            except Exception as ex:
+                logger.info(f"extract_name_version_from_link {key}:{ex}")
+            if oss_name and (not oss_version):
+                if key in ["pypi", "maven", "npm", "npm2", "pub"]:
+                    oss_version, link = get_latest_package_version(link, key, origin_name)
+                    logger.debug(f'Try to download with the latest version:{link}')
+            break
+    return oss_name, oss_version, link, key
+
+
+def get_latest_package_version(link, pkg_type, oss_name):
+    find_version = ''
+    link_with_version = link
+
+    try:
+        if pkg_type in ['npm', 'npm2']:
+            stderr, stdout = npm_run('view', oss_name, 'version')
+            if stdout:
+                find_version = stdout.strip()
+            link_with_version = f'https://www.npmjs.com/package/{oss_name}/v/{find_version}'
+        elif pkg_type == 'pypi':
+            find_version = str(latest(oss_name, at='pip', output_format='version', pre_ok=True))
+            link_with_version = f'https://pypi.org/project/{oss_name}/{find_version}'
+        elif pkg_type == 'maven':
+            maven_response = requests.get(f'https://api.deps.dev/v3alpha/systems/maven/packages/{oss_name}')
+            if maven_response.status_code == 200:
+                find_version = maven_response.json().get('versions')[-1].get('versionKey').get('version')
+            oss_name = oss_name.replace(':', '/')
+            link_with_version = f'https://mvnrepository.com/artifact/{oss_name}/{find_version}'
+        elif pkg_type == 'pub':
+            pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
+            if pub_response.status_code == 200:
+                find_version = pub_response.json().get('latest').get('version')
+            link_with_version = f'https://pub.dev/packages/{oss_name}/versions/{find_version}'
+    except Exception as e:
+        logger.debug(f'Fail to get latest package version({link}:{e})')
+    return find_version, link_with_version
+
+
 def get_downloadable_url(link):
 
     ret = False
-    new_link = ''
+    result_link = link
 
-    link = link.replace('http://', '')
-    link = link.replace('https://', '')
+    oss_name, oss_version, new_link, pkg_type = extract_name_version_from_link(link)
+    new_link = new_link.replace('http://', '')
+    new_link = new_link.replace('https://', '')
 
-    if link.startswith('pypi.org/'):
-        ret, new_link = get_download_location_for_pypi(link)
-    elif link.startswith('mvnrepository.com/artifact/') or link.startswith('repo1.maven.org/'):
-        ret, new_link = get_download_location_for_maven(link)
-    elif link.startswith('www.npmjs.com/') or link.startswith('registry.npmjs.org/'):
-        ret, new_link = get_download_location_for_npm(link)
-    elif link.startswith('pub.dev/'):
-        ret, new_link = get_download_location_for_pub(link)
+    if pkg_type == "pypi":
+        ret, result_link = get_download_location_for_pypi(new_link)
+    elif pkg_type == "maven" or new_link.startswith('repo1.maven.org/'):
+        ret, result_link = get_download_location_for_maven(new_link)
+    elif (pkg_type in ["npm", "npm2"]) or new_link.startswith('registry.npmjs.org/'):
+        ret, result_link = get_download_location_for_npm(new_link)
+    elif pkg_type == "pub":
+        ret, result_link = get_download_location_for_pub(new_link)
 
-    return ret, new_link
+    return ret, result_link, oss_name, oss_version
 
 
 def get_download_location_for_pypi(link):
@@ -118,6 +205,7 @@ def get_download_location_for_npm(link):
     oss_name_npm = ""
     tar_name = ""
 
+    link = link.replace('%40', '@')
     if link.startswith('www.npmjs.com/') or link.startswith('registry.npmjs.org/'):
         try:
             dn_loc_split = link.split('/')
@@ -133,16 +221,9 @@ def get_download_location_for_npm(link):
                 oss_name_npm = dn_loc_split[idx]
                 tar_name = oss_name_npm
                 oss_version = dn_loc_split[idx+2]
-        except Exception:
-            pass
 
-        try:
-            if not oss_version:
-                stderr, stdout = npm_run('view', oss_name_npm, 'version')
-                if stdout:
-                    oss_version = stdout.strip()
-            tar_name = f"{tar_name}-{oss_version}"
-            new_link = 'https://registry.npmjs.org/' + oss_name_npm + '/-/' + tar_name + '.tgz'
+            tar_name = f'{tar_name}-{oss_version}'
+            new_link = f'https://registry.npmjs.org/{oss_name_npm}/-/{tar_name}.tgz'
             ret = True
         except Exception as error:
             ret = False
@@ -158,7 +239,7 @@ def get_download_location_for_pub(link):
     # download url format : https://pub.dev/packages/(oss_name)/versions/(oss_version).tar.gz
     try:
         if link.startswith('pub.dev/packages'):
-            new_link = 'https://{link}.tar.gz'
+            new_link = f'https://{link}.tar.gz'
             ret = True
 
     except Exception as error:

fosslight_util/download.py

@@ -24,6 +24,7 @@ import time
 import threading
 import platform
 import subprocess
+import re
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
@@ -115,6 +116,8 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
 
     success = True
     msg = ""
+    oss_name = ""
+    oss_version = ""
     log_file_name = "fosslight_download_" + \
         datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
     logger, log_item = init_log(os.path.join(log_dir, log_file_name))
@@ -135,22 +138,29 @@ def cli_download_and_extract(link, target_dir, log_dir, checkout_to="", compress
             is_rubygems = src_info.get("rubygems", False)
 
             # General download (git clone, wget)
-            if (not is_rubygems) and (not download_git_clone(link, target_dir, checkout_to, tag, branch)):
+            success_git, msg, oss_name = download_git_clone(link, target_dir, checkout_to, tag, branch)
+            if (not is_rubygems) and (not success_git):
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True)
             # Download from rubygems.org
             elif is_rubygems and shutil.which("gem"):
                 success = gem_download(link, target_dir, checkout_to)
+        if msg:
+            msg = f'git fail: {msg}'
+            if msg_wget:
+                msg = f'{msg}, wget fail: {msg_wget}'
+            else:
+                msg = f'{msg}, wget success'
     except Exception as error:
         success = False
         msg = str(error)
 
-    logger.info(f"\n* FOSSLight Downloader - Result: {success}\n {msg}")
-    return success, msg
+    logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})")
+    return success, msg, oss_name, oss_version
 
 
 def get_ref_to_checkout(checkout_to, ref_list):
@@ -184,8 +194,19 @@ def decide_checkout(checkout_to="", tag="", branch=""):
     return ref_to_checkout
 
 
+def get_github_ossname(link):
+    oss_name = ""
+    p = re.compile(r'https?:\/\/github.com\/([^\/]+)\/([^\/\.]+)(\.git)?')
+    match = p.match(link)
+    if match:
+        oss_name = f"{match.group(1)}-{match.group(2)}"
+    return oss_name
+
+
 def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
     ref_to_checkout = decide_checkout(checkout_to, tag, branch)
+    msg = ""
+    oss_name = get_github_ossname(git_url)
 
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -204,7 +225,8 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             del alarm
     except Exception as error:
         logger.warning(f"git clone - failed: {error}")
-        return False
+        msg = str(error)
+        return False, msg, oss_name
     try:
         if ref_to_checkout != "":
             ref_list = [x for x in repo.references]
@@ -213,11 +235,14 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             repo.checkout(ref_to_checkout)
     except Exception as error:
         logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}")
-    return True
+    return True, msg, oss_name
 
 
 def download_wget(link, target_dir, compressed_only):
     success = False
+    msg = ""
+    oss_name = ""
+    oss_version = ""
     downloaded_file = ""
     if platform.system() != "Windows":
         signal.signal(signal.SIGALRM, alarm_handler)
@@ -228,7 +253,7 @@ def download_wget(link, target_dir, compressed_only):
     try:
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version = get_downloadable_url(link)
         if ret and new_link:
             link = new_link
 
@@ -255,9 +280,10 @@ def download_wget(link, target_dir, compressed_only):
             logger.debug(f"wget - downloaded: {downloaded_file}")
     except Exception as error:
         success = False
+        msg = str(error)
         logger.warning(f"wget - failed: {error}")
 
-    return success, downloaded_file
+    return success, downloaded_file, msg, oss_name, oss_version
 
 
 def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):

fosslight_util/read_excel.py

@@ -3,6 +3,7 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import logging
+from typing import List, Dict, Any
 import xlrd
 import json
 from fosslight_util.constant import LOGGER_NAME
@@ -12,17 +13,18 @@ from fosslight_util.parsing_yaml import set_value_switch
 logger = logging.getLogger(LOGGER_NAME)
 IDX_CANNOT_FOUND = -1
 PREFIX_BIN = "bin"
+SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
 xlrd.xlsx.ensure_elementtree_imported(False, None)
 xlrd.xlsx.Element_has_iter = True
 
 
-def read_oss_report(excel_file, sheet_names=""):
-    _oss_report_items = []
-    xl_sheets = {}
-    all_sheet_to_read = []
-    not_matched_sheet = []
+def read_oss_report(excel_file: str, sheet_names: str = "") -> List[OssItem]:
+    oss_report_items: List[OssItem] = []
+    xl_sheets: Dict[str, Any] = {}
+    all_sheet_to_read: List[str] = []
+    not_matched_sheet: List[str] = []
     any_sheet_matched = False
-    SHEET_PREFIX_TO_READ = ["bin", "bom", "src"]
+
     if sheet_names:
         sheet_name_prefix_match = False
         sheet_name_to_read = sheet_names.split(",")
@@ -113,8 +115,8 @@ def read_oss_report(excel_file, sheet_names=""):
                             else:
                                 valid_row = False if cell_value == "-" else True
                 if valid_row and load_data_cnt > 0:
-                    _oss_report_items.append(item)
+                    oss_report_items.append(item)
 
     except Exception as error:
         logger.error(f"Parsing a OSS Report: {error}")
-    return _oss_report_items
+    return oss_report_items

fosslight_util/write_excel.py

@@ -129,6 +129,10 @@ def write_result_to_csv(output_file, sheet_list_origin, separate_sheet=False, ex
                 row_num = 1
                 header_row, sheet_content_without_header = get_header_row(sheet_name, sheet_contents[:], extended_header)
 
+                if 'Copyright Text' in header_row:
+                    idx = header_row.index('Copyright Text')-1
+                    for item in sheet_content_without_header:
+                        item[idx] = item[idx].replace('\n', ', ')
                 if not separate_sheet:
                     merge_sheet.extend(sheet_content_without_header)
                     if sheet_name == list(sheet_list.keys())[-1]:

fosslight_util/write_scancodejson.py

@@ -7,12 +7,14 @@ import logging
 import os
 import json
 import fosslight_util.constant as constant
+from fosslight_util.oss_item import OssItem
+from typing import List
 
 logger = logging.getLogger(constant.LOGGER_NAME)
 EMPTY_FILE_PATH = '-'
 
 
-def write_scancodejson(output_dir, output_filename, oss_list):
+def write_scancodejson(output_dir: str, output_filename: str, oss_list: List[OssItem]):
     json_output = {}
     json_output['headers'] = []
     json_output['summary'] = {}

{fosslight_util-1.4.34.dist-info → fosslight_util-1.4.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 1.4.34
+Version: 1.4.36
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -28,8 +28,9 @@ Requires-Dist: python3-wget
 Requires-Dist: beautifulsoup4
 Requires-Dist: jsonmerge
 Requires-Dist: spdx-tools ==0.7.0rc0
-Requires-Dist: npm
 Requires-Dist: setuptools >=65.5.1
+Requires-Dist: npm
+Requires-Dist: requests
 Requires-Dist: numpy ; python_version < "3.8"
 Requires-Dist: numpy >=1.22.2 ; python_version >= "3.8"
 Requires-Dist: pygit2 ==1.6.1 ; python_version<'3.7'

{fosslight_util-1.4.34.dist-info → fosslight_util-1.4.36.dist-info}/RECORD

@@ -1,30 +1,30 @@
 fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_util/_get_downloadable_url.py,sha256=UggybHxs_MeqxKzOpdS9PuOeKvO3YayhpbwLZPIZDss,5844
+fosslight_util/_get_downloadable_url.py,sha256=Fb_-7S7vTg4HIsdQ_W77WbO1IYTFqGaf0lRRSunbWFI,10167
 fosslight_util/compare_yaml.py,sha256=0bapoyS0yrzkiK70K57E8-3wZ_D6mZAJ-24Eq9TYBlk,2632
 fosslight_util/constant.py,sha256=mDrH3Ahldhs9xT63iC8BmeRLHjN6fewI1H-pxnGvcD8,595
 fosslight_util/convert_excel_to_yaml.py,sha256=7ZsAMMQJIEXrmcl_28nSHvFpGMi1ZiRZYpEfI5O8vP8,2298
 fosslight_util/correct.py,sha256=RQ70zaQ_xuo9Mo0zIcb6pRQwt96bsimQJWkYldE2vbg,5224
-fosslight_util/download.py,sha256=CMT_ZHK_lCQI6_-_6hI90n5fhw_AkxV3elAZl4uilzk,12561
+fosslight_util/download.py,sha256=UVBQ5dtu9BCulUG20A0FBy98AlTmrjRau1Qmb4Ayd5U,13394
 fosslight_util/help.py,sha256=3ej8HhpszfFLRxR99Ob55MwHypjIHce1-YSp5_ENq-A,2113
 fosslight_util/oss_item.py,sha256=Gjw-aI4gZGYrCbAkCL35lXlgA2w2F2plgOUmenQ1Cig,5238
 fosslight_util/output_format.py,sha256=kmlruqjTjrkENsg0sF4B-wAfEUlld_Uc--cGGm5NPs0,3214
 fosslight_util/parsing_yaml.py,sha256=zmetMiZprni_YMxUfj_Nx-nXm0sT31Ck_V0nWFS6ZmE,4332
-fosslight_util/read_excel.py,sha256=XLN4-iCPB_6GoNQwjt4ScMi39_uBfNGdqR73f06j244,5043
+fosslight_util/read_excel.py,sha256=eN2aNkJv3Y2071f6lHfQTHx5IFJe0imxH3TeRO_kodc,5154
 fosslight_util/set_log.py,sha256=8cFGpr4HMk4grHo3a5keR4V1xWBscbjowYPxyrh42ks,3140
 fosslight_util/spdx_licenses.py,sha256=r90hUY4_T-XrHIJHLx1Ox3gWZ3qzdZj9rJFo7AwmkPE,3641
 fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
-fosslight_util/write_excel.py,sha256=DI-Re-__VlkaCC6s4pi9PbEZn5HWrq7NU7ahZkzH41A,9175
+fosslight_util/write_excel.py,sha256=NEwGMs56NkGT5MKxYsearMTuzoD7Xio2tGC-PgTjOHI,9417
 fosslight_util/write_opossum.py,sha256=PGJV5DysNJvIFbzsyGXxh_kRcvZuHAOmLs-WlXP8qMI,11831
-fosslight_util/write_scancodejson.py,sha256=LKvy8C-8YvGRGhdmjIvtu3wD-IUfhS2rg64TJEZE9jI,2189
+fosslight_util/write_scancodejson.py,sha256=CSKjuwbA04nK5ogXklNsCBgGDZXPr805T8KPLgvx71U,2282
 fosslight_util/write_spdx.py,sha256=B_aHv9vScgZI5gHo5Hd56ckNWOHdyAQebRV54bTx9ec,9542
 fosslight_util/write_txt.py,sha256=Ms8E2wbc0U84IYP0dxTAbIfX5PzpQG3xblu8lv1w8J0,574
 fosslight_util/write_yaml.py,sha256=ppBBErAXbXw8jtJ9j7BDeQHHzDlsTRt62XR0GgTcr70,3294
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-1.4.34.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-1.4.34.dist-info/METADATA,sha256=nnfmrSHHm-AxpehOHFY7hOZNvFoSTulea4qNbxQHiBY,6165
-fosslight_util-1.4.34.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-fosslight_util-1.4.34.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-1.4.34.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-1.4.34.dist-info/RECORD,,
+fosslight_util-1.4.36.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-1.4.36.dist-info/METADATA,sha256=POQH9BPStS1HuuZUnuj3AND4EitzjRIgGtJj7ONP7ls,6189
+fosslight_util-1.4.36.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+fosslight_util-1.4.36.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-1.4.36.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-1.4.36.dist-info/RECORD,,

{fosslight_util-1.4.34.dist-info → fosslight_util-1.4.36.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any