fosslight-util 2.1.28__py3-none-any.whl → 2.1.30__py3-none-any.whl

fosslight_util/_get_downloadable_url.py

@@ -499,13 +499,39 @@ def get_download_location_for_maven(link):
 
     try:
         if link.startswith('mvnrepository.com/artifact/'):
-            dn_loc_split = link.replace('mvnrepository.com/', '').split('/')
-            group_id = dn_loc_split[1].replace('.', '/')
-            dn_loc = 'https://repo1.maven.org/maven2/' + group_id + '/' + dn_loc_split[2] + '/' + dn_loc_split[3]
+            parts = link.replace('mvnrepository.com/artifact/', '').split('/')
+            if len(parts) < 2:
+                raise Exception('invalid mvnrepository artifact url')
+            group_raw = parts[0]
+            artifact_id = parts[1]
+            version = parts[2] if len(parts) > 2 and parts[2] else ''
+            group_path = group_raw.replace('.', '/')
+
+            repo_base = f'https://repo1.maven.org/maven2/{group_path}/{artifact_id}'
+            try:
+                urlopen(repo_base)
+                if version:
+                    dn_loc = f'{repo_base}/{version}'
+                else:
+                    new_link = repo_base
+                    ret = True
+                    return ret, new_link
+            except Exception:
+                google_base = f'https://dl.google.com/android/maven2/{group_path}/{artifact_id}'
+                if version:
+                    google_sources = f'{google_base}/{version}/{artifact_id}-{version}-sources.jar'
+                    try:
+                        res_g = urlopen(google_sources)
+                        if res_g.getcode() == 200:
+                            ret = True
+                            return ret, google_sources
+                    except Exception:
+                        pass
+                new_link = google_base
+                ret = True
+                return ret, new_link
 
         elif link.startswith('repo1.maven.org/maven2/'):
-            dn_loc_split = link.replace('repo1.maven.org/maven2/', '').split('/')
-
             if link.endswith('.tar.gz') or link.endswith('.jar') or link.endswith('.tar.xz'):
                 new_link = 'https://' + link
                 ret = True

fosslight_util/download.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import os
 import sys
-import wget
+import requests
 import tarfile
 import zipfile
 import logging
@@ -58,6 +58,22 @@ def alarm_handler(signum, frame):
     raise TimeOutException(f'Timeout ({SIGNAL_TIMEOUT} sec)', 1)
 
 
+def is_downloadable(url):
+    try:
+        h = requests.head(url, allow_redirects=True)
+        header = h.headers
+        content_type = header.get('content-type')
+        if 'text/html' in content_type.lower():
+            return False
+        content_disposition = header.get('content-disposition')
+        if content_disposition and 'attachment' in content_disposition.lower():
+            return True
+        return True
+    except Exception as e:
+        logger.warning(f"is_downloadable - failed: {e}")
+        return False
+
+
 def change_src_link_to_https(src_link):
     src_link = src_link.replace("git://", "https://")
     if src_link.endswith(".git"):
@@ -381,25 +397,30 @@ def download_wget(link, target_dir, compressed_only, checkout_to):
             link = new_link
 
         if compressed_only:
+            # Check if link ends with known compression extensions
             for ext in compression_extension:
                 if link.endswith(ext):
                     success = True
                     break
-            if not success:
-                if pkg_type == 'cargo':
-                    success = True
         else:
-            success = True
+            # If get_downloadable_url found a downloadable file, proceed
+            if ret:
+                success = True
+            else:
+                # No downloadable file found in package repositories, verify link is downloadable
+                if not is_downloadable(link):
+                    raise Exception('Not a downloadable link (link:{0})'.format(link))
+                success = True
 
+        # Fallback: verify link is downloadable for compressed_only case
         if not success:
-            raise Exception('Not supported compression type (link:{0})'.format(link))
+            if is_downloadable(link):
+                success = True
+            else:
+                raise Exception('Not a downloadable link (link:{0})'.format(link))
 
         logger.info(f"wget: {link}")
-        if pkg_type == 'cargo':
-            outfile = os.path.join(target_dir, f'{oss_name}.tar.gz')
-            downloaded_file = wget.download(link, out=outfile)
-        else:
-            downloaded_file = wget.download(link, target_dir)
+        downloaded_file = download_file(link, target_dir)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:
@@ -416,6 +437,49 @@ def download_wget(link, target_dir, compressed_only, checkout_to):
     return success, downloaded_file, msg, oss_name, oss_version
 
 
+def download_file(url, target_dir):
+    local_path = ""
+    try:
+        try:
+            h = requests.head(url, allow_redirects=True)
+            final_url = h.url or url
+            headers = h.headers
+        except Exception:
+            final_url = url
+            headers = {}
+
+        with requests.get(final_url, stream=True, allow_redirects=True) as r:
+            r.raise_for_status()
+
+            filename = ""
+            cd = r.headers.get("Content-Disposition") or headers.get("Content-Disposition")
+            if cd:
+                m_star = re.search(r"filename\*=(?:UTF-8'')?([^;\r\n]+)", cd)
+                if m_star:
+                    filename = urllib.parse.unquote(m_star.group(1).strip('"\''))
+                else:
+                    m = re.search(r"filename=([^;\r\n]+)", cd)
+                    if m:
+                        filename = m.group(1).strip('"\'')
+            if not filename:
+                final_for_name = r.url or final_url
+                filename = os.path.basename(urllib.parse.urlparse(final_for_name).path)
+                if not filename:
+                    filename = "downloaded_file"
+            if os.path.isdir(target_dir):
+                local_path = os.path.join(target_dir, filename)
+            else:
+                local_path = target_dir
+
+            with open(local_path, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+    except Exception as e:
+        logger.warning(f"download_file - failed: {e}")
+        return None
+    return local_path
+
+
 def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):
     logger.debug(f"Extract Dir: {src_dir}")
     try:
@@ -450,6 +514,9 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True, comp
                 decompress_bz2(fname, extract_path)
             elif fname.endswith(".whl"):
                 unzip(fname, extract_path)
+            elif fname.endswith(".crate"):
+                with contextlib.closing(tarfile.open(fname, "r:gz")) as t:
+                    t.extractall(path=extract_path)
             else:
                 is_compressed_file = False
                 if compressed_only:

fosslight_util/get_pom_license.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import urllib.request
+from urllib.error import URLError, HTTPError
+from defusedxml.ElementTree import fromstring as xml_fromstring
+import fosslight_util.constant as constant
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+def get_license_from_pom(group_id: str = None,
+                         artifact_id: str = None,
+                         version: str = None,
+                         pom_path: str = None,
+                         check_parent: bool = True) -> str:
+
+    def build_urls(g, a, v):
+        group_path = g.replace('.', '/')
+        name = f"{a}-{v}.pom"
+        repo1 = f"https://repo1.maven.org/maven2/{group_path}/{a}/{v}/{name}"
+        google = f"https://dl.google.com/android/maven2/{group_path}/{a}/{v}/{name}"
+        return [repo1, google]
+
+    def fetch_pom(g, a, v):
+        for url in build_urls(g, a, v):
+            try:
+                with urllib.request.urlopen(url) as resp:
+                    return resp.read().decode('utf-8')
+            except (HTTPError, URLError):
+                continue
+        return None
+
+    def extract_licenses(root):
+        licenses_elem = root.find('{*}licenses')
+        if licenses_elem is not None:
+            names = []
+            for lic in licenses_elem.findall('{*}license'):
+                name = lic.findtext('{*}name')
+                if name:
+                    names.append(name.replace(',', ''))
+            if names:
+                return ', '.join(names)
+        return None
+
+    def extract_parent_info(root):
+        parent = root.find('{*}parent')
+        if parent is not None:
+            g = parent.findtext('{*}groupId') or parent.findtext('groupId')
+            a = parent.findtext('{*}artifactId') or parent.findtext('artifactId')
+            v = parent.findtext('{*}version') or parent.findtext('version')
+            if g and a and v:
+                return g, a, v
+        return None, None, None
+
+    visited = set()
+
+    def find_license_in_pom_recursive(g, a, v, check_parent_flag):
+        key = (g, a, v)
+        if key in visited:
+            return ''
+        visited.add(key)
+        content = fetch_pom(g, a, v)
+        if not content:
+            return ''
+        try:
+            root = xml_fromstring(content)
+        except Exception:
+            return ''
+        licenses = extract_licenses(root)
+        if licenses:
+            return licenses
+        if not check_parent_flag:
+            return ''
+        pg, pa, pv = extract_parent_info(root)
+        if pg and pa and pv:
+            return find_license_in_pom_recursive(pg, pa, pv, check_parent_flag)
+        return ''
+
+    try:
+        if pom_path:
+            if not os.path.exists(pom_path):
+                logger.warning(f"POM file not found: {pom_path}")
+                return ''
+            try:
+                with open(pom_path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+                xml_start = content.find('<?xml')
+                if xml_start > 0:
+                    content = content[xml_start:]
+                elif xml_start == -1:
+                    root_start = content.find('<project')
+                    if root_start > 0:
+                        content = content[root_start:]
+                root = xml_fromstring(content)
+            except Exception as e:
+                logger.warning(f"Failed to parse POM file {pom_path}: {e}")
+                return ''
+
+            licenses = extract_licenses(root)
+            if licenses:
+                return licenses
+            if not check_parent:
+                return ''
+            pg, pa, pv = extract_parent_info(root)
+            if pg and pa and pv:
+                return find_license_in_pom_recursive(pg, pa, pv, check_parent)
+
+            logger.debug(f"No license info found in local POM: {pom_path}, Retry with remote fetch.")
+
+        if not (group_id and artifact_id and version):
+            return ''
+        return find_license_in_pom_recursive(group_id, artifact_id, version, check_parent)
+    except Exception as e:
+        logger.warning(f"Error getting license from POM: {e}")
+        return ''

{fosslight_util-2.1.28.dist-info → fosslight_util-2.1.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_util
-Version: 2.1.28
+Version: 2.1.30
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util
@@ -20,7 +20,6 @@ Requires-Dist: progress
 Requires-Dist: PyYAML
 Requires-Dist: lastversion
 Requires-Dist: coloredlogs
-Requires-Dist: python3-wget
 Requires-Dist: beautifulsoup4
 Requires-Dist: jsonmerge
 Requires-Dist: spdx-tools==0.8.*; sys_platform == "linux"

{fosslight_util-2.1.28.dist-info → fosslight_util-2.1.30.dist-info}/RECORD

@@ -1,11 +1,12 @@
 fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_util/_get_downloadable_url.py,sha256=GNfGLJPQVHrU_eYse1flE42AgYUviGNKWZnq9sfUY9c,24473
+fosslight_util/_get_downloadable_url.py,sha256=Komp1ODhqCKbnqGl7T-EDT8KmQuvgOeZ-nkV2bT7E-s,25519
 fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE4,2916
 fosslight_util/constant.py,sha256=3BzJtyxC0o5IFAhYaUW-DeTKkA6f5tDJFJTb0k5ji9Y,2418
 fosslight_util/correct.py,sha256=1WEAL-9_KhjFPLucPhv0PNN3K7avm0z8mU6sTuSyeHM,3864
 fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
-fosslight_util/download.py,sha256=t6-5NAcvCOfmi9TM7O1yp-9X11MiuKd8JdzNEZtEmqQ,20967
+fosslight_util/download.py,sha256=AWwD3FWhF-bMagWINJ-Dg1VMuycXbe8VXX7qQ09YjYs,23565
 fosslight_util/exclude.py,sha256=fDmBsZJ_F7O9Oh2T-07R03XNbElo1tFaf_z01KfSAqU,2399
+fosslight_util/get_pom_license.py,sha256=x4_RHpM91s01j1OUWpKIxUjTGDH6y5AxTNDGkMajs4I,4253
 fosslight_util/help.py,sha256=iyWmAaUQSHJtWv5mjFv0f3YoDVlDgEqdsDDEyImEUNc,2646
 fosslight_util/oss_item.py,sha256=8890JHb5ZoKQWAwN7Fl8badnlYatJtF4MVJz1rdS4yQ,6938
 fosslight_util/output_format.py,sha256=BP23LspxawDZ_a99oWLVKWUQ-G7P5uoUpjEXhkRFKwc,8801
@@ -24,9 +25,9 @@ fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.1.28.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.1.28.dist-info/METADATA,sha256=9aponw8jVAsmq4_4hIgpvIYCyzSfkDcPR7h9a3tg9mo,6367
-fosslight_util-2.1.28.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-fosslight_util-2.1.28.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
-fosslight_util-2.1.28.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.1.28.dist-info/RECORD,,
+fosslight_util-2.1.30.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.30.dist-info/METADATA,sha256=-0EbMo4HMlTn1KyTXOFa8_s4yaWtrhh3QQmmDTroV_w,6339
+fosslight_util-2.1.30.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+fosslight_util-2.1.30.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
+fosslight_util-2.1.30.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.30.dist-info/RECORD,,