fosslight-util 2.1.27__tar.gz → 2.1.29__tar.gz

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: fosslight_util
-Version: 2.1.27
+Version: 2.1.29
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util
@@ -13,6 +13,31 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: XlsxWriter
+Requires-Dist: pandas
+Requires-Dist: openpyxl
+Requires-Dist: progress
+Requires-Dist: PyYAML
+Requires-Dist: lastversion
+Requires-Dist: coloredlogs
+Requires-Dist: beautifulsoup4
+Requires-Dist: jsonmerge
+Requires-Dist: spdx-tools==0.8.*; sys_platform == "linux"
+Requires-Dist: setuptools>=65.5.1
+Requires-Dist: numpy
+Requires-Dist: requests
+Requires-Dist: GitPython
+Requires-Dist: cyclonedx-python-lib==8.5.*; sys_platform == "linux"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: download-url
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 <!--
 Copyright (c) 2021 LG Electronics

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/requirements.txt

@@ -5,7 +5,7 @@ progress
 PyYAML
 lastversion
 coloredlogs
-python3-wget
+
 beautifulsoup4
 jsonmerge
 spdx-tools==0.8.*;sys_platform=="linux"

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/setup.py

@@ -14,7 +14,7 @@ with open('requirements.txt', 'r', 'utf-8') as f:
 if __name__ == "__main__":
     setup(
         name='fosslight_util',
-        version='2.1.27',
+        version='2.1.29',
         package_dir={"": "src"},
         packages=find_packages(where='src'),
         description='FOSSLight Util',

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/src/fosslight_util/_get_downloadable_url.py

@@ -54,76 +54,185 @@ def extract_name_version_from_link(link, checkout_version):
     oss_name = ""
     oss_version = ""
     matched = False
+    direct_maven = False
+
     if link.startswith("www."):
         link = link.replace("www.", "https://www.", 1)
-    for key, value in constant.PKG_PATTERN.items():
-        p = re.compile(value)
-        match = p.match(link)
-        if match:
-            try:
-                origin_name = match.group(1)
-                if (key == "pypi") or (key == "pypi2"):
-                    oss_name = f"pypi:{origin_name}"
-                    oss_name = re.sub(r"[-_.]+", "-", oss_name)
-                    oss_version = match.group(2)
-                elif key == "maven":
-                    artifact = match.group(2)
-                    oss_name = f"{origin_name}:{artifact}"
-                    origin_name = oss_name
-                    oss_version = match.group(3)
-                elif key == "npm" or key == "npm2":
-                    oss_name = f"npm:{origin_name}"
-                    oss_version = match.group(2)
-                elif key == "pub":
-                    oss_name = f"pub:{origin_name}"
-                    oss_version = match.group(2)
-                elif key == "cocoapods":
-                    oss_name = f"cocoapods:{origin_name}"
-                elif key == "go":
-                    if origin_name.endswith('/'):
-                        origin_name = origin_name[:-1]
-                    oss_name = f"go:{origin_name}"
-                    oss_version = match.group(2)
-                elif key == "cargo":
-                    oss_name = f"cargo:{origin_name}"
-                    oss_version = match.group(2)
-            except Exception as ex:
-                logger.info(f"extract_name_version_from_link {key}:{ex}")
-            if oss_name:
-                # Priority: 1) detected oss_version 2) checkout_version 3) latest
-                need_latest = False
-
-                if not oss_version and checkout_version:
-                    oss_version = checkout_version.strip()
-                if key in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
-                    if oss_version:
-                        try:
-                            if not version_exists(key, origin_name, oss_version):
-                                logger.info(f'Version {oss_version} not found for {oss_name}; will attempt latest fallback')
-                                need_latest = True
-                        except Exception as e:
-                            logger.info(f'Version validation failed ({oss_name}:{oss_version}) {e}; will attempt latest fallback')
-                            need_latest = True
-                    else:
-                        need_latest = True
-                    if need_latest:
-                        latest_ver = get_latest_package_version(link, key, origin_name)
-                        if latest_ver:
-                            if oss_version and latest_ver != oss_version:
-                                logger.info(f'Fallback to latest version {latest_ver} (previous invalid: {oss_version})')
-                            elif not oss_version:
-                                logger.info(f'Using latest version {latest_ver} (no version detected)')
-                            oss_version = latest_ver
-                if oss_version:
-                    try:
-                        link = get_new_link_with_version(link, key, origin_name, oss_version)
-                    except Exception as _e:
-                        logger.info(f'Failed to build versioned link for {oss_name}:{oss_version} {_e}')
+
+    if (not matched and (
+        link.startswith('https://repo1.maven.org/maven2/') or
+        link.startswith('https://dl.google.com/android/maven2/')
+    )):
+        parsed = parse_direct_maven_url(link)
+        if parsed:
+            origin_name, parsed_version = parsed
+            oss_name = origin_name  # groupId:artifactId
+            oss_version = parsed_version or ""
             matched = True
-            break
+            direct_maven = True
+            pkg_type = 'maven'
+
+    for direct_key in ["maven_repo1", "maven_google"]:
+        pattern = constant.PKG_PATTERN.get(direct_key)
+        if pattern and re.match(pattern, link):
+            parsed = parse_direct_maven_url(link)
+            if parsed:
+                origin_name, parsed_version = parsed
+                oss_name = origin_name
+                oss_version = parsed_version or ""
+                matched = True
+                direct_maven = True
+                pkg_type = 'maven'
+                break
+
+    if not matched:
+        for key, value in constant.PKG_PATTERN.items():
+            if key in ["maven_repo1", "maven_google"]:
+                continue
+            p = re.compile(value)
+            match = p.match(link)
+            if match:
+                try:
+                    pkg_type = key
+                    origin_name = match.group(1)
+                    if (key == "pypi") or (key == "pypi2"):
+                        oss_name = f"pypi:{origin_name}"
+                        oss_name = re.sub(r"[-_.]+", "-", oss_name)
+                        oss_version = match.group(2)
+                        pkg_type = 'pypi'
+                    elif key == "maven":
+                        artifact = match.group(2)
+                        oss_name = f"{origin_name}:{artifact}"
+                        origin_name = oss_name
+                        oss_version = match.group(3)
+                    elif key == "npm" or key == "npm2":
+                        oss_name = f"npm:{origin_name}"
+                        oss_version = match.group(2)
+                    elif key == "pub":
+                        oss_name = f"pub:{origin_name}"
+                        oss_version = match.group(2)
+                    elif key == "cocoapods":
+                        oss_name = f"cocoapods:{origin_name}"
+                    elif key == "go":
+                        if origin_name.endswith('/'):
+                            origin_name = origin_name[:-1]
+                        oss_name = f"go:{origin_name}"
+                        oss_version = match.group(2)
+                    elif key == "cargo":
+                        oss_name = f"cargo:{origin_name}"
+                        oss_version = match.group(2)
+                except Exception as ex:
+                    logger.info(f"extract_name_version_from_link {key}:{ex}")
+                if oss_name:
+                    matched = True
+                break
+
     if not matched:
-        key = ""
-    return oss_name, oss_version, link, key
+        return "", "", link, ""
+    else:
+        need_latest = False
+        if not oss_version and checkout_version:
+            oss_version = checkout_version.strip()
+        if pkg_type in ["pypi", "maven", "npm", "npm2", "pub", "go"]:
+            if oss_version:
+                try:
+                    if not version_exists(pkg_type, origin_name, oss_version):
+                        logger.info(f'Version {oss_version} not found for {oss_name}; will attempt latest fallback')
+                        need_latest = True
+                except Exception as e:
+                    logger.info(f'Version validation failed ({oss_name}:{oss_version}) {e}; will attempt latest fallback')
+                    need_latest = True
+            else:
+                need_latest = True
+        if need_latest:
+            latest_ver = get_latest_package_version(link, pkg_type, origin_name)
+            if latest_ver:
+                if oss_version and latest_ver != oss_version:
+                    logger.info(f'Fallback to latest version {latest_ver} (previous invalid: {oss_version})')
+                elif not oss_version:
+                    logger.info(f'Using latest version {latest_ver} (no version detected)')
+                oss_version = latest_ver
+
+    try:
+        if oss_version:
+            if pkg_type == 'maven' and direct_maven:
+                # Skip if oss_name malformed
+                if ':' in oss_name:
+                    parts = oss_name.split(':', 1)
+                    group_id, artifact_id = parts[0], parts[1]
+                    group_path = group_id.replace('.', '/')
+                    if (
+                        link.startswith('https://repo1.maven.org/maven2/') or
+                        link.startswith('http://repo1.maven.org/maven2/')
+                    ):
+                        if not re.search(r'/\d[^/]*/*$', link.rstrip('/')):
+                            link = (
+                                f'https://repo1.maven.org/maven2/{group_path}/'
+                                f'{artifact_id}/{oss_version}'
+                            )
+                    elif (
+                        link.startswith('https://dl.google.com/android/maven2/') or
+                        link.startswith('http://dl.google.com/android/maven2/')
+                    ):
+                        if not re.search(r'/\d[^/]*/*$', link.rstrip('/')):
+                            link = (
+                                f'https://dl.google.com/android/maven2/{group_path}/'
+                                f'{artifact_id}/{oss_version}/{artifact_id}-{oss_version}-sources.jar'
+                            )
+                else:
+                    logger.debug(f'Skip maven normalization due to invalid oss_name: {oss_name}')
+            else:
+                link = get_new_link_with_version(link, pkg_type, origin_name, oss_version)
+    except Exception as _e:
+        logger.info(f'Failed to build versioned link for {oss_name or origin_name}:{oss_version} {_e}')
+
+    return oss_name, oss_version, link, pkg_type
+
+
+def parse_direct_maven_url(url):
+    try:
+        clean_url = url.replace('https://', '').replace('http://', '')
+        if clean_url.startswith('repo1.maven.org/maven2/'):
+            base_path = clean_url[len('repo1.maven.org/maven2/'):]
+        elif clean_url.startswith('dl.google.com/android/maven2/'):
+            base_path = clean_url[len('dl.google.com/android/maven2/'):]
+        else:
+            return None
+
+        base_path = base_path.rstrip('/')
+        # Strip file name if ends with known artifact extension.
+        if any(base_path.endswith(ext) for ext in ['.jar', '.pom', '.aar']):
+            base_path = '/'.join(base_path.split('/')[:-1])
+
+        parts = base_path.split('/')
+        if len(parts) < 2:
+            return None
+
+        version = None
+        artifact_id = None
+        if len(parts) >= 3:
+            potential_version = parts[-1]
+            potential_artifact = parts[-2]
+            if re.search(r'\d', potential_version):
+                version = potential_version
+                artifact_id = potential_artifact
+                group_parts = parts[:-2]
+            else:
+                artifact_id = parts[-1]
+                group_parts = parts[:-1]
+        else:
+            artifact_id = parts[-1]
+            group_parts = parts[:-1]
+
+        group_id = '.'.join(group_parts)
+        if not group_id or not artifact_id:
+            return None
+
+        maven_name = f"{group_id}:{artifact_id}"
+        return maven_name, version
+    except Exception as e:
+        logger.debug(f'Failed to parse direct Maven URL {url}: {e}')
+        return None
 
 
 def get_new_link_with_version(link, pkg_type, oss_name, oss_version):
@@ -160,7 +269,45 @@ def get_latest_package_version(link, pkg_type, oss_name):
             if maven_response.status_code == 200:
                 versions = maven_response.json().get('versions', [])
                 if versions:
-                    cand = max(versions, key=lambda v: v.get('publishedAt', ''))
+                    # Some version entries may miss publishedAt; fallback to semantic version ordering.
+                    def sem_key(vstr: str):
+                        # Parse semantic version with optional prerelease label
+                        # Examples: 1.9.0, 1.10.0-alpha, 2.0.0-rc
+                        m = re.match(r'^(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:[-.]([A-Za-z0-9]+))?$', vstr)
+                        if not m:
+                            return (0, 0, 0, 999)
+                        major = int(m.group(1) or 0)
+                        minor = int(m.group(2) or 0)
+                        patch = int(m.group(3) or 0)
+                        label = (m.group(4) or '').lower()
+                        # Assign label weights: stable > rc > beta > alpha
+                        label_weight_map = {
+                            'alpha': -3,
+                            'beta': -2,
+                            'rc': -1
+                        }
+                        weight = label_weight_map.get(label, 0 if label == '' else -4)
+                        return (major, minor, patch, weight)
+
+                    with_pub = [v for v in versions if v.get('publishedAt')]
+                    if with_pub:
+                        cand = max(with_pub, key=lambda v: v.get('publishedAt'))
+                    else:
+                        decorated = []
+                        for v in versions:
+                            vkey = v.get('versionKey', {})
+                            ver = vkey.get('version', '')
+                            if ver:
+                                decorated.append((sem_key(ver), ver, v))
+                        if decorated:
+                            decorated.sort(key=lambda t: t[0])
+                            stable_candidates = [t for t in decorated if t[0][3] == 0]
+                            if stable_candidates:
+                                cand = stable_candidates[-1][2]
+                            else:
+                                cand = decorated[-1][2]
+                        else:
+                            cand = versions[-1]
                     find_version = cand.get('versionKey', {}).get('version', '')
         elif pkg_type == 'pub':
             pub_response = requests.get(f'https://pub.dev/api/packages/{oss_name}')
@@ -188,7 +335,7 @@ def get_downloadable_url(link, checkout_version):
 
     if pkg_type == "pypi":
         ret, result_link = get_download_location_for_pypi(new_link)
-    elif pkg_type == "maven" or new_link.startswith('repo1.maven.org/'):
+    elif pkg_type == "maven" or new_link.startswith('repo1.maven.org/') or new_link.startswith('dl.google.com/android/maven2/'):
         ret, result_link = get_download_location_for_maven(new_link)
     elif (pkg_type in ["npm", "npm2"]) or new_link.startswith('registry.npmjs.org/'):
         ret, result_link = get_download_location_for_npm(new_link)
@@ -352,19 +499,52 @@ def get_download_location_for_maven(link):
 
     try:
         if link.startswith('mvnrepository.com/artifact/'):
-            dn_loc_split = link.replace('mvnrepository.com/', '').split('/')
-            group_id = dn_loc_split[1].replace('.', '/')
-            dn_loc = 'https://repo1.maven.org/maven2/' + group_id + '/' + dn_loc_split[2] + '/' + dn_loc_split[3]
+            parts = link.replace('mvnrepository.com/artifact/', '').split('/')
+            if len(parts) < 2:
+                raise Exception('invalid mvnrepository artifact url')
+            group_raw = parts[0]
+            artifact_id = parts[1]
+            version = parts[2] if len(parts) > 2 and parts[2] else ''
+            group_path = group_raw.replace('.', '/')
+
+            repo_base = f'https://repo1.maven.org/maven2/{group_path}/{artifact_id}'
+            try:
+                urlopen(repo_base)
+                if version:
+                    dn_loc = f'{repo_base}/{version}'
+                else:
+                    new_link = repo_base
+                    ret = True
+                    return ret, new_link
+            except Exception:
+                google_base = f'https://dl.google.com/android/maven2/{group_path}/{artifact_id}'
+                if version:
+                    google_sources = f'{google_base}/{version}/{artifact_id}-{version}-sources.jar'
+                    try:
+                        res_g = urlopen(google_sources)
+                        if res_g.getcode() == 200:
+                            ret = True
+                            return ret, google_sources
+                    except Exception:
+                        pass
+                new_link = google_base
+                ret = True
+                return ret, new_link
 
         elif link.startswith('repo1.maven.org/maven2/'):
-            dn_loc_split = link.replace('repo1.maven.org/maven2/', '').split('/')
-
             if link.endswith('.tar.gz') or link.endswith('.jar') or link.endswith('.tar.xz'):
                 new_link = 'https://' + link
                 ret = True
                 return ret, new_link
             else:
                 dn_loc = 'https://' + link
+        elif link.startswith('dl.google.com/android/maven2/'):
+            if link.endswith('.jar'):
+                new_link = 'https://' + link
+                ret = True
+                return ret, new_link
+            else:
+                dn_loc = 'https://' + link
         else:
             raise Exception("not valid url for maven")
 

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/src/fosslight_util/constant.py

@@ -40,6 +40,8 @@ PKG_PATTERN = {
     "pypi": r'https?:\/\/pypi\.org\/project\/([^\/]+)[\/]?([^\/]*)',
     "pypi2": r'https?:\/\/files\.pythonhosted\.org\/packages\/source\/[\w]\/([^\/]+)\/[\S]+-([^\-]+)\.tar\.gz',
     "maven": r'https?:\/\/mvnrepository\.com\/artifact\/([^\/]+)\/([^\/]+)\/?([^\/]*)',
+    "maven_repo1": r'https?:\/\/repo1\.maven\.org\/maven2\/(.*)',
+    "maven_google": r'https?:\/\/dl\.google\.com\/android\/maven2\/(.*)',
     "npm": r'https?:\/\/www\.npmjs\.com\/package\/([^\/\@]+)(?:\/v\/)?([^\/]*)',
     "npm2": r'https?:\/\/www\.npmjs\.com\/package\/(\@[^\/]+\/[^\/]+)(?:\/v\/)?([^\/]*)',
     "pub": r'https?:\/\/pub\.dev\/packages\/([^\/]+)(?:\/versions\/)?([^\/]*)',

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/src/fosslight_util/download.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import os
 import sys
-import wget
+import requests
 import tarfile
 import zipfile
 import logging
@@ -58,6 +58,22 @@ def alarm_handler(signum, frame):
     raise TimeOutException(f'Timeout ({SIGNAL_TIMEOUT} sec)', 1)
 
 
+def is_downloadable(url):
+    try:
+        h = requests.head(url, allow_redirects=True)
+        header = h.headers
+        content_type = header.get('content-type')
+        if 'text/html' in content_type.lower():
+            return False
+        content_disposition = header.get('content-disposition')
+        if content_disposition and 'attachment' in content_disposition.lower():
+            return True
+        return True
+    except Exception as e:
+        logger.warning(f"is_downloadable - failed: {e}")
+        return False
+
+
 def change_src_link_to_https(src_link):
     src_link = src_link.replace("git://", "https://")
     if src_link.endswith(".git"):
@@ -381,25 +397,30 @@ def download_wget(link, target_dir, compressed_only, checkout_to):
             link = new_link
 
         if compressed_only:
+            # Check if link ends with known compression extensions
             for ext in compression_extension:
                 if link.endswith(ext):
                     success = True
                     break
-            if not success:
-                if pkg_type == 'cargo':
-                    success = True
         else:
-            success = True
+            # If get_downloadable_url found a downloadable file, proceed
+            if ret:
+                success = True
+            else:
+                # No downloadable file found in package repositories, verify link is downloadable
+                if not is_downloadable(link):
+                    raise Exception('Not a downloadable link (link:{0})'.format(link))
+                success = True
 
+        # Fallback: verify link is downloadable for compressed_only case
         if not success:
-            raise Exception('Not supported compression type (link:{0})'.format(link))
+            if is_downloadable(link):
+                success = True
+            else:
+                raise Exception('Not a downloadable link (link:{0})'.format(link))
 
         logger.info(f"wget: {link}")
-        if pkg_type == 'cargo':
-            outfile = os.path.join(target_dir, f'{oss_name}.tar.gz')
-            downloaded_file = wget.download(link, out=outfile)
-        else:
-            downloaded_file = wget.download(link, target_dir)
+        downloaded_file = download_file(link, target_dir)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:
@@ -416,6 +437,49 @@ def download_wget(link, target_dir, compressed_only, checkout_to):
     return success, downloaded_file, msg, oss_name, oss_version
 
 
+def download_file(url, target_dir):
+    local_path = ""
+    try:
+        try:
+            h = requests.head(url, allow_redirects=True)
+            final_url = h.url or url
+            headers = h.headers
+        except Exception:
+            final_url = url
+            headers = {}
+
+        with requests.get(final_url, stream=True, allow_redirects=True) as r:
+            r.raise_for_status()
+
+            filename = ""
+            cd = r.headers.get("Content-Disposition") or headers.get("Content-Disposition")
+            if cd:
+                m_star = re.search(r"filename\*=(?:UTF-8'')?([^;\r\n]+)", cd)
+                if m_star:
+                    filename = urllib.parse.unquote(m_star.group(1).strip('"\''))
+                else:
+                    m = re.search(r"filename=([^;\r\n]+)", cd)
+                    if m:
+                        filename = m.group(1).strip('"\'')
+            if not filename:
+                final_for_name = r.url or final_url
+                filename = os.path.basename(urllib.parse.urlparse(final_for_name).path)
+                if not filename:
+                    filename = "downloaded_file"
+            if os.path.isdir(target_dir):
+                local_path = os.path.join(target_dir, filename)
+            else:
+                local_path = target_dir
+
+            with open(local_path, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+    except Exception as e:
+        logger.warning(f"download_file - failed: {e}")
+        return None
+    return local_path
+
+
 def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):
     logger.debug(f"Extract Dir: {src_dir}")
     try:
@@ -450,6 +514,9 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True, comp
                 decompress_bz2(fname, extract_path)
             elif fname.endswith(".whl"):
                 unzip(fname, extract_path)
+            elif fname.endswith(".crate"):
+                with contextlib.closing(tarfile.open(fname, "r:gz")) as t:
+                    t.extractall(path=extract_path)
             else:
                 is_compressed_file = False
                 if compressed_only:

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/src/fosslight_util.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
-Name: fosslight-util
-Version: 2.1.27
+Metadata-Version: 2.4
+Name: fosslight_util
+Version: 2.1.29
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util
@@ -13,6 +13,31 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: XlsxWriter
+Requires-Dist: pandas
+Requires-Dist: openpyxl
+Requires-Dist: progress
+Requires-Dist: PyYAML
+Requires-Dist: lastversion
+Requires-Dist: coloredlogs
+Requires-Dist: beautifulsoup4
+Requires-Dist: jsonmerge
+Requires-Dist: spdx-tools==0.8.*; sys_platform == "linux"
+Requires-Dist: setuptools>=65.5.1
+Requires-Dist: numpy
+Requires-Dist: requests
+Requires-Dist: GitPython
+Requires-Dist: cyclonedx-python-lib==8.5.*; sys_platform == "linux"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: download-url
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 <!--
 Copyright (c) 2021 LG Electronics

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/src/fosslight_util.egg-info/SOURCES.txt

@@ -34,4 +34,11 @@ src/fosslight_util.egg-info/requires.txt
 src/fosslight_util.egg-info/top_level.txt
 src/fosslight_util/resources/frequentLicenselist.json
 src/fosslight_util/resources/frequent_license_nick_list.json
-src/fosslight_util/resources/licenses.json
+src/fosslight_util/resources/licenses.json
+tests/test_cyclonedx.py
+tests/test_download.py
+tests/test_opossum.py
+tests/test_spdx_licenses.py
+tests/test_text.py
+tests/test_write_output.py
+tests/test_write_yaml.py

{fosslight_util-2.1.27 → fosslight_util-2.1.29}/src/fosslight_util.egg-info/requires.txt

@@ -5,7 +5,6 @@ progress
 PyYAML
 lastversion
 coloredlogs
-python3-wget
 beautifulsoup4
 jsonmerge
 setuptools>=65.5.1

fosslight_util-2.1.29/tests/test_cyclonedx.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from fosslight_util.write_cyclonedx import write_cyclonedx
+from tests import constants
+
+
+def test_cyclonedx(scan_item):
+    # given
+    output_dir = os.path.join(constants.TEST_RESULT_DIR, "cyclonedx")
+    filename_with_dir = os.path.join(output_dir, "FL-TEST_cyclonedx.json")
+
+    # when
+    success, err_msg, _ = write_cyclonedx(filename_with_dir.split('.')[0], '.json', scan_item)
+
+    # then
+    assert success is True
+    assert len(os.listdir(output_dir)) > 0

fosslight_util-2.1.29/tests/test_download.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pytest
+
+from fosslight_util.download import cli_download_and_extract, download_git_clone
+from tests import constants
+
+
+def test_download_from_github():
+    # given
+    git_url = "https://github.com/LGE-OSS/example"
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    log_dir = "test_result/download_log/example"
+
+    # when
+    success, _, _, _ = cli_download_and_extract(git_url, target_dir, log_dir)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+
+
+@pytest.mark.parametrize("git_url",
+                         ["git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;branch=hash-stat2",
+                          "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git;protocol=git;tag=v32"])
+def test_download_from_github_with_branch_or_tag(git_url):
+    # given
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    log_dir = "test_result/download_log/example"
+
+    # when
+    success, _, _, _ = cli_download_and_extract(git_url, target_dir, log_dir)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+
+
+@pytest.mark.parametrize("project_name, project_url",
+                         [("filelock", "https://pypi.org/project/filelock/3.4.1"),
+                          ("dependency", "https://pypi.org/project/fosslight-dependency/3.0.5/"),
+                          ("jackson", "https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.12.2"),
+                          ("pub", "https://pub.dev/packages/file/versions/5.2.1")])
+def test_download_from_wget(project_name, project_url):
+    # given
+    target_dir = os.path.join(constants.TEST_RESULT_DIR,
+                              os.path.join("download", project_name))
+    log_dir = os.path.join(constants.TEST_RESULT_DIR,
+                           os.path.join("download_log" + project_name))
+
+    # when
+    success, _, _, _ = cli_download_and_extract(project_url, target_dir, log_dir)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+
+
+def test_download_git_clone_with_branch():
+    # given
+    git_url = "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git"
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    branch_name = "hash-stat2"
+
+    # when
+    success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", "", branch_name)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+    assert oss_name == ''
+    assert oss_version == branch_name
+
+
+def test_download_git_clone_with_tag():
+    # given
+    git_url = "git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git"
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    tag_name = "v32"
+
+    # when
+    success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", tag_name)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+    assert oss_name == ''
+    assert oss_version == tag_name
+
+
+def test_download_main_branch_when_any_branch_or_tag_not_entered():
+    # given
+    git_url = "https://github.com/LGE-OSS/example"
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    expected_oss_ver = ""
+
+    # when
+    success, _, oss_name, oss_version = download_git_clone(git_url, target_dir)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+    assert oss_name == 'LGE-OSS-example'
+    assert oss_version == expected_oss_ver
+
+
+def test_download_main_branch_when_non_existent_branch_entered():
+    # given
+    git_url = "https://github.com/LGE-OSS/example"
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    branch_name = "non-existent-branch"
+    expected_oss_ver = ""
+
+    # when
+    success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", "", branch_name)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+    assert oss_name == 'LGE-OSS-example'
+    assert oss_version == expected_oss_ver
+
+
+def test_download_main_branch_when_non_existent_tag_entered():
+    # given
+    git_url = "https://github.com/LGE-OSS/example"
+    target_dir = os.path.join(constants.TEST_RESULT_DIR, "download/example")
+    tag_name = "non-existent-tag"
+    expected_oss_ver = ""
+
+    # when
+    success, _, oss_name, oss_version = download_git_clone(git_url, target_dir, "", tag_name)
+
+    # then
+    assert success is True
+    assert len(os.listdir(target_dir)) > 0
+    assert oss_name == 'LGE-OSS-example'
+    assert oss_version == expected_oss_ver

fosslight_util-2.1.29/tests/test_opossum.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from fosslight_util.write_opossum import write_opossum
+from tests import constants
+
+
+def test_opossum(scan_item):
+    # given
+    output_dir = os.path.join(constants.TEST_RESULT_DIR, "opossum")
+    filename_with_dir = os.path.join(output_dir, "FL-TEST_opossum.json")
+
+    # when
+    success, _ = write_opossum(filename_with_dir, scan_item)
+
+    # then
+    assert success is True
+    assert len(os.listdir(output_dir)) > 0

fosslight_util-2.1.29/tests/test_spdx_licenses.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+from fosslight_util.spdx_licenses import get_spdx_licenses_json
+
+
+def test_get_spdx_licenses_json():
+    # when
+    success, _, licenses = get_spdx_licenses_json()
+
+    # then
+    assert success is True
+    assert len(licenses) > 0

fosslight_util-2.1.29/tests/test_text.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from fosslight_util.write_txt import write_txt_file
+
+
+def test_text():
+    # given
+    output_dir = "test_result/txt"
+    file_to_create = os.path.join(output_dir, "test.txt")
+    text_to_write = "Testing - Writing text in a file."
+
+    # when
+    success, _ = write_txt_file(file_to_create, text_to_write)
+    with open(file_to_create, 'r', encoding='utf-8') as result_file:
+        result = result_file.read()
+
+    # then
+    assert success is True
+    assert text_to_write in result

fosslight_util-2.1.29/tests/test_write_output.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+import os
+from copy import deepcopy
+
+import pytest
+
+from fosslight_util.output_format import write_output_file
+
+
+@pytest.mark.parametrize("output_dir, result_file_name, file_extension",
+                         [("test_result/excel_and_csv/excel", "Test_Excel", ".xlsx"),
+                          ("test_result/excel_and_csv/csv", "Test_Csv", ".csv"),
+                          ("test_result/output_format", "FL-TEST_opossum", ".json")])
+def test_write_excel_and_csv(output_dir, result_file_name, file_extension, scan_item):
+    # given
+    output_file_without_extension = os.path.join(output_dir, result_file_name)
+
+    # when
+    success, _, result_file = write_output_file(output_file_without_extension,
+                                                file_extension, deepcopy(scan_item))
+
+    # then
+    assert success is True
+    assert result_file_name + file_extension in result_file

fosslight_util-2.1.29/tests/test_write_yaml.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from fosslight_util.write_yaml import write_yaml
+from tests import constants
+
+
+def test_write_yaml(scan_item):
+    # given
+    output_dir = os.path.join(constants.TEST_RESULT_DIR, "yaml")
+    output_file = os.path.join(output_dir, 'FL-TEST_yaml.yaml')
+
+    # when
+    success, _, output = write_yaml(output_file, scan_item)
+
+    # then
+    assert success is True
+    assert output_file in output