fosslight-util 2.0.0__py3-none-any.whl → 2.1.29__py3-none-any.whl

fosslight_util/correct.py

@@ -61,17 +61,15 @@ def correct_with_yaml(correct_filepath, path_to_scan, scan_item):
 
                     yaml_path_exists = True
                     exclude_fileitems.append(idx)
-
-            if not yaml_path_exists:
+            if scanner_name == FOSSLIGHT_SOURCE and not yaml_path_exists:
                 correct_item = copy.deepcopy(yaml_file_item)
                 if os.path.exists(os.path.normpath(yaml_file_item.source_name_or_path)):
                     correct_item.comment = 'Loaded from sbom-info.yaml'
                     correct_fileitems.append(correct_item)
                 else:
-                    if scanner_name == FOSSLIGHT_SOURCE:
-                        correct_item.exclude = True
-                        correct_item.comment = 'Added by sbom-info.yaml'
-                        correct_fileitems.append(correct_item)
+                    correct_item.exclude = True
+                    correct_item.comment = 'Added by sbom-info.yaml'
+                    correct_fileitems.append(correct_item)
         if correct_fileitems:
             scan_item.append_file_items(correct_fileitems, scanner_name)
             find_match = True

fosslight_util/download.py

@@ -4,13 +4,13 @@
 # SPDX-License-Identifier: Apache-2.0
 import os
 import sys
-import wget
+import requests
 import tarfile
 import zipfile
 import logging
 import argparse
 import shutil
-import pygit2 as git
+from git import Repo, GitCommandError, Git
 import bz2
 import contextlib
 from datetime import datetime
@@ -26,9 +26,11 @@ import platform
 import subprocess
 import re
 from typing import Tuple
+import urllib.parse
+import json
 
 logger = logging.getLogger(constant.LOGGER_NAME)
-compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2"}
+compression_extension = {".tar.bz2", ".tar.gz", ".tar.xz", ".tgz", ".tar", ".zip", ".jar", ".bz2", ".whl"}
 prefix_refs = ["refs/remotes/origin/", "refs/tags/"]
 SIGNAL_TIMEOUT = 600
 
@@ -56,6 +58,22 @@ def alarm_handler(signum, frame):
     raise TimeOutException(f'Timeout ({SIGNAL_TIMEOUT} sec)', 1)
 
 
+def is_downloadable(url):
+    try:
+        h = requests.head(url, allow_redirects=True)
+        header = h.headers
+        content_type = header.get('content-type')
+        if 'text/html' in content_type.lower():
+            return False
+        content_disposition = header.get('content-disposition')
+        if content_disposition and 'attachment' in content_disposition.lower():
+            return True
+        return True
+    except Exception as e:
+        logger.warning(f"is_downloadable - failed: {e}")
+        return False
+
+
 def change_src_link_to_https(src_link):
     src_link = src_link.replace("git://", "https://")
     if src_link.endswith(".git"):
@@ -77,7 +95,7 @@ def parse_src_link(src_link):
         if src_link.startswith("git://github.com/"):
             src_link_changed = change_src_link_to_https(src_link_split[0])
         elif src_link.startswith("git@github.com:"):
-            src_link_changed = change_ssh_link_to_https(src_link_split[0])
+            src_link_changed = src_link_split[0]
         else:
             if "rubygems.org" in src_link:
                 src_info["rubygems"] = True
@@ -92,39 +110,11 @@ def parse_src_link(src_link):
     return src_info
 
 
-def main():
-    parser = argparse.ArgumentParser(description='FOSSLight Downloader', prog='fosslight_download', add_help=False)
-    parser.add_argument('-h', '--help', help='Print help message', action='store_true', dest='help')
-    parser.add_argument('-s', '--source', help='Source link to download', type=str, dest='source')
-    parser.add_argument('-t', '--target_dir', help='Target directory', type=str, dest='target_dir', default="")
-    parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
-
-    src_link = ""
-    target_dir = os.getcwd()
-    log_dir = os.getcwd()
-
-    try:
-        args = parser.parse_args()
-    except SystemExit:
-        sys.exit(0)
-
-    if args.help:
-        print_help_msg_download()
-    if args.source:
-        src_link = args.source
-    if args.target_dir:
-        target_dir = args.target_dir
-    if args.log_dir:
-        log_dir = args.log_dir
-
-    if not src_link:
-        print_help_msg_download()
-    else:
-        cli_download_and_extract(src_link, target_dir, log_dir)
-
-
 def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_to: str = "",
-                             compressed_only: bool = False) -> Tuple[bool, str, str, str]:
+                             compressed_only: bool = False, ssh_key: str = "",
+                             id: str = "", git_token: str = "",
+                             called_cli: bool = True,
+                             output: bool = False) -> Tuple[bool, str, str, str]:
     global logger
 
     success = True
@@ -136,6 +126,7 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
         datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
     logger, log_item = init_log(os.path.join(log_dir, log_file_name))
     link = link.strip()
+    is_rubygems = False
 
     try:
         if not link:
@@ -144,6 +135,9 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
         elif os.path.isfile(target_dir):
             success = False
             msg = f"The target directory exists as a file.: {target_dir}"
+        elif os.path.exists(link) or os.path.isdir(link) or os.path.isfile(link):
+            success = False
+            msg = f"You cannot enter a path instead of a link.: {link}"
         else:
             src_info = parse_src_link(link)
             link = src_info.get("url", "")
@@ -152,12 +146,18 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
             is_rubygems = src_info.get("rubygems", False)
 
             # General download (git clone, wget)
-            success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir, checkout_to, tag, branch)
+            success_git, msg, oss_name, oss_version = download_git_clone(link, target_dir,
+                                                                         checkout_to,
+                                                                         tag, branch,
+                                                                         ssh_key, id, git_token,
+                                                                         called_cli)
+            link = change_ssh_link_to_https(link)
             if (not is_rubygems) and (not success_git):
                 if os.path.isfile(target_dir):
                     shutil.rmtree(target_dir)
 
-                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir, compressed_only)
+                success, downloaded_file, msg_wget, oss_name, oss_version = download_wget(link, target_dir,
+                                                                                          compressed_only, checkout_to)
                 if success:
                     success = extract_compressed_file(downloaded_file, target_dir, True, compressed_only)
             # Download from rubygems.org
@@ -177,6 +177,17 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
         success = False
         msg = str(error)
 
+    if output:
+        output_result = {
+            "success": success,
+            "message": msg,
+            "oss_name": oss_name,
+            "oss_version": oss_version
+        }
+        output_json = os.path.join(log_dir, "fosslight_download_output.json")
+        with open(output_json, 'w') as f:
+            json.dump(output_result, f, indent=4)
+
     logger.info(f"\n* FOSSLight Downloader - Result: {success} ({msg})")
     return success, msg, oss_name, oss_version
 
@@ -200,15 +211,60 @@ def get_ref_to_checkout(checkout_to, ref_list):
     return ref_to_checkout
 
 
-def decide_checkout(checkout_to="", tag="", branch=""):
-    if checkout_to:
-        ref_to_checkout = checkout_to
-    else:
-        if branch:
-            ref_to_checkout = branch
-        else:
-            ref_to_checkout = tag
-    return ref_to_checkout
+def get_remote_refs(git_url: str):
+    if not git_url:
+        return {"tags": [], "branches": []}
+    tags = []
+    branches = []
+    try:
+        cp = subprocess.run(["git", "ls-remote", "--tags", "--heads", git_url], capture_output=True, text=True, timeout=30)
+        if cp.returncode == 0:
+            for line in cp.stdout.splitlines():
+                parts = line.split('\t')
+                if len(parts) != 2:
+                    continue
+                ref = parts[1]
+                if ref.startswith('refs/tags/'):
+                    tags.append(ref[len('refs/tags/'):])
+                elif ref.startswith('refs/heads/'):
+                    branches.append(ref[len('refs/heads/'):])
+    except Exception as e:
+        logger.debug(f"get_remote_refs - failed: {e}")
+    return {"tags": tags, "branches": branches}
+
+
+def decide_checkout(checkout_to="", tag="", branch="", git_url=""):
+    base = checkout_to or tag or branch
+    if not base:
+        return ""
+
+    ref_dict = get_remote_refs(git_url)
+    tag_set = set(ref_dict.get("tags", []))
+    branch_set = set(ref_dict.get("branches", []))
+
+    ver_re = re.compile(r'^(?:v\.? ?)?' + re.escape(base) + r'$', re.IGNORECASE)
+
+    # tag: exact -> prefix variant -> endswith
+    if base in tag_set:
+        return base
+    tag_candidates = [c for c in tag_set if ver_re.match(c)]
+    if tag_candidates:
+        return min(tag_candidates, key=lambda x: (len(x), x.lower()))
+    tag_ends = [n for n in tag_set if n.endswith(base)]
+    if tag_ends:
+        return min(tag_ends, key=len)
+
+    # branch: exact -> prefix variant -> endswith
+    if base in branch_set:
+        return base
+    branch_candidates = [c for c in branch_set if ver_re.match(c)]
+    if branch_candidates:
+        return min(branch_candidates, key=lambda x: (len(x), x.lower()))
+    branch_ends = [n for n in branch_set if n.endswith(base)]
+    if branch_ends:
+        return min(branch_ends, key=len)
+
+    return base
 
 
 def get_github_ossname(link):
@@ -229,15 +285,48 @@ def get_github_token(git_url):
     return github_token
 
 
-def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
-    ref_to_checkout = decide_checkout(checkout_to, tag, branch)
-    msg = ""
-    oss_name = get_github_ossname(git_url)
+def download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli=True):
+    success = False
     oss_version = ""
-    github_token = get_github_token(git_url)
-    callbacks = None
-    if github_token != "":
-        callbacks = git.RemoteCallbacks(credentials=git.UserPass("foo", github_token))  # username is not used, so set to dummy
+
+    logger.info(f"Download git url :{git_url}")
+    env = os.environ.copy()
+    if not called_cli:
+        env["GIT_TERMINAL_PROMPT"] = "0"
+    if refs_to_checkout:
+        try:
+            # gitPython uses the branch argument the same whether you check out to a branch or a tag.
+            Repo.clone_from(git_url, target_dir, branch=refs_to_checkout, env=env)
+            if any(Path(target_dir).iterdir()):
+                success = True
+                oss_version = refs_to_checkout
+                logger.info(f"Files found in {target_dir} after clone.")
+            else:
+                logger.info(f"No files found in {target_dir} after clone.")
+                success = False
+        except GitCommandError as error:
+            logger.info(f"Git checkout error:{error}")
+            success = False
+        except Exception as e:
+            logger.info(f"Repo.clone_from error:{e}")
+            success = False
+
+    if not success:
+        Repo.clone_from(git_url, target_dir, env=env)
+        if any(Path(target_dir).iterdir()):
+            success = True
+        else:
+            logger.info(f"No files found in {target_dir} after clone.")
+            success = False
+    return success, oss_version
+
+
+def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch="",
+                       ssh_key="", id="", git_token="", called_cli=True):
+    oss_name = get_github_ossname(git_url)
+    refs_to_checkout = decide_checkout(checkout_to, tag, branch, git_url)
+    msg = ""
+    success = True
 
     try:
         if platform.system() != "Windows":
@@ -248,34 +337,45 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             alarm.start()
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
-        repo = git.clone_repository(git_url, target_dir,
-                                    bare=False, repository=None,
-                                    remote=None, callbacks=callbacks)
-        if platform.system() != "Windows":
-            signal.alarm(0)
+
+        if git_url.startswith("ssh:") and not ssh_key:
+            msg = "Private git needs ssh_key"
+            success = False
         else:
-            del alarm
+            if ssh_key:
+                logger.info(f"Download git with ssh_key:{git_url}")
+                git_ssh_cmd = f'ssh -i {ssh_key}'
+                with Git().custom_environment(GIT_SSH_COMMAND=git_ssh_cmd):
+                    success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli)
+            else:
+                if id and git_token:
+                    try:
+                        m = re.match(r"^(ht|f)tp(s?)\:\/\/", git_url)
+                        protocol = m.group()
+                        if protocol:
+                            encoded_git_token = urllib.parse.quote(git_token, safe='')
+                            encoded_id = urllib.parse.quote(id, safe='')
+                            git_url = git_url.replace(protocol, f"{protocol}{encoded_id}:{encoded_git_token}@")
+                    except Exception as error:
+                        logger.info(f"Failed to insert id, token to git url:{error}")
+                success, oss_version = download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_cli)
+
+            logger.info(f"git checkout: {oss_version}")
+            refs_to_checkout = oss_version
+
+            if platform.system() != "Windows":
+                signal.alarm(0)
+            else:
+                del alarm
     except Exception as error:
+        success = False
         logger.warning(f"git clone - failed: {error}")
         msg = str(error)
-        return False, msg, oss_name, oss_version
-    try:
-        if ref_to_checkout != "":
-            ref_list = [x for x in repo.references]
-            ref_to_checkout = get_ref_to_checkout(ref_to_checkout, ref_list)
-            logger.info(f"git checkout: {ref_to_checkout}")
-            repo.checkout(ref_to_checkout)
 
-            for prefix_ref in prefix_refs:
-                if ref_to_checkout.startswith(prefix_ref):
-                    oss_version = ref_to_checkout[len(prefix_ref):]
-
-    except Exception as error:
-        logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}")
-    return True, msg, oss_name, oss_version
+    return success, msg, oss_name, refs_to_checkout
 
 
-def download_wget(link, target_dir, compressed_only):
+def download_wget(link, target_dir, compressed_only, checkout_to):
     success = False
     msg = ""
     oss_name = ""
@@ -292,23 +392,35 @@ def download_wget(link, target_dir, compressed_only):
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
 
-        ret, new_link, oss_name, oss_version = get_downloadable_url(link)
+        ret, new_link, oss_name, oss_version, pkg_type = get_downloadable_url(link, checkout_to)
         if ret and new_link:
             link = new_link
 
         if compressed_only:
+            # Check if link ends with known compression extensions
             for ext in compression_extension:
                 if link.endswith(ext):
                     success = True
                     break
         else:
-            success = True
+            # If get_downloadable_url found a downloadable file, proceed
+            if ret:
+                success = True
+            else:
+                # No downloadable file found in package repositories, verify link is downloadable
+                if not is_downloadable(link):
+                    raise Exception('Not a downloadable link (link:{0})'.format(link))
+                success = True
 
+        # Fallback: verify link is downloadable for compressed_only case
         if not success:
-            raise Exception('Not supported compression type (link:{0})'.format(link))
+            if is_downloadable(link):
+                success = True
+            else:
+                raise Exception('Not a downloadable link (link:{0})'.format(link))
 
         logger.info(f"wget: {link}")
-        downloaded_file = wget.download(link, target_dir)
+        downloaded_file = download_file(link, target_dir)
         if platform.system() != "Windows":
             signal.alarm(0)
         else:
@@ -325,6 +437,49 @@ def download_wget(link, target_dir, compressed_only):
     return success, downloaded_file, msg, oss_name, oss_version
 
 
+def download_file(url, target_dir):
+    local_path = ""
+    try:
+        try:
+            h = requests.head(url, allow_redirects=True)
+            final_url = h.url or url
+            headers = h.headers
+        except Exception:
+            final_url = url
+            headers = {}
+
+        with requests.get(final_url, stream=True, allow_redirects=True) as r:
+            r.raise_for_status()
+
+            filename = ""
+            cd = r.headers.get("Content-Disposition") or headers.get("Content-Disposition")
+            if cd:
+                m_star = re.search(r"filename\*=(?:UTF-8'')?([^;\r\n]+)", cd)
+                if m_star:
+                    filename = urllib.parse.unquote(m_star.group(1).strip('"\''))
+                else:
+                    m = re.search(r"filename=([^;\r\n]+)", cd)
+                    if m:
+                        filename = m.group(1).strip('"\'')
+            if not filename:
+                final_for_name = r.url or final_url
+                filename = os.path.basename(urllib.parse.urlparse(final_for_name).path)
+                if not filename:
+                    filename = "downloaded_file"
+            if os.path.isdir(target_dir):
+                local_path = os.path.join(target_dir, filename)
+            else:
+                local_path = target_dir
+
+            with open(local_path, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+    except Exception as e:
+        logger.warning(f"download_file - failed: {e}")
+        return None
+    return local_path
+
+
 def extract_compressed_dir(src_dir, target_dir, remove_after_extract=True):
     logger.debug(f"Extract Dir: {src_dir}")
     try:
@@ -357,6 +512,11 @@ def extract_compressed_file(fname, extract_path, remove_after_extract=True, comp
                 unzip(fname, extract_path)
             elif fname.endswith(".bz2"):
                 decompress_bz2(fname, extract_path)
+            elif fname.endswith(".whl"):
+                unzip(fname, extract_path)
+            elif fname.endswith(".crate"):
+                with contextlib.closing(tarfile.open(fname, "r:gz")) as t:
+                    t.extractall(path=extract_path)
             else:
                 is_compressed_file = False
                 if compressed_only:
@@ -443,5 +603,51 @@ def gem_download(link, target_dir, checkout_to):
     return success
 
 
+def main():
+    parser = argparse.ArgumentParser(description='FOSSLight Downloader', prog='fosslight_download', add_help=False)
+    parser.add_argument('-h', '--help', help='Print help message', action='store_true', dest='help')
+    parser.add_argument('-s', '--source', help='Source link to download', type=str, dest='source')
+    parser.add_argument('-t', '--target_dir', help='Target directory', type=str, dest='target_dir', default="")
+    parser.add_argument('-d', '--log_dir', help='Directory to save log file', type=str, dest='log_dir', default="")
+    parser.add_argument('-c', '--checkout_to', help='Checkout to  branch or tag', type=str, dest='checkout_to', default="")
+    parser.add_argument('-z', '--compressed_only', help='Unzip only compressed file',
+                        action='store_true', dest='compressed_only', default=False)
+    parser.add_argument('-o', '--output', help='Generate output file', action='store_true', dest='output', default=False)
+
+    src_link = ""
+    target_dir = os.getcwd()
+    log_dir = os.getcwd()
+    checkout_to = ""
+    compressed_only = False
+    output = False
+
+    try:
+        args = parser.parse_args()
+    except SystemExit:
+        sys.exit(0)
+
+    if args.help:
+        print_help_msg_download()
+    if args.source:
+        src_link = args.source
+    if args.target_dir:
+        target_dir = args.target_dir
+    if args.log_dir:
+        log_dir = args.log_dir
+    if args.checkout_to:
+        checkout_to = args.checkout_to
+    if args.compressed_only:
+        compressed_only = args.compressed_only
+    if args.output:
+        output = args.output
+
+    if not src_link:
+        print_help_msg_download()
+    else:
+        cli_download_and_extract(src_link, target_dir, log_dir, checkout_to,
+                                 compressed_only, "", "", "", False,
+                                 output)
+
+
 if __name__ == '__main__':
     main()

fosslight_util/exclude.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2025 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import fnmatch
+from typing import List
+
+
+def excluding_files(patterns: List[str], path_to_scan: str) -> List[str]:
+    excluded_paths = set()
+
+    # Normalize patterns: e.g., 'sample/', 'sample/*' -> 'sample'
+    # Replace backslash with slash
+    normalized_patterns = []
+    for pattern in patterns:
+        pattern = pattern.replace('\\', '/')
+        if pattern.endswith('/') or pattern.endswith('/*'):
+            pattern = pattern.rstrip('/*')
+        normalized_patterns.append(pattern)
+
+    # Traverse directories
+    for root, dirs, files in os.walk(path_to_scan):
+        remove_dir_list = []
+
+        # (1) Directory matching
+        for d in dirs:
+            dir_name = d
+            dir_path = os.path.relpath(os.path.join(root, d), path_to_scan).replace('\\', '/')
+            matched = False
+
+            for pat in normalized_patterns:
+                # Match directory name
+                if fnmatch.fnmatch(dir_name, pat):
+                    matched = True
+
+                # Match the full relative path
+                if not matched:
+                    if fnmatch.fnmatch(dir_path, pat) or fnmatch.fnmatch(dir_path, pat + "/*"):
+                        matched = True
+
+                # If matched, exclude all files under this directory and stop checking patterns
+                if matched:
+                    sub_root_path = os.path.join(root, d)
+                    for sr, _, sf in os.walk(sub_root_path):
+                        for sub_file in sf:
+                            sub_file_path = os.path.relpath(os.path.join(sr, sub_file), path_to_scan)
+                            excluded_paths.add(sub_file_path.replace('\\', '/'))
+                    remove_dir_list.append(d)
+                    break
+
+        # (1-2) Prune matched directories from further traversal
+        for rd in remove_dir_list:
+            dirs.remove(rd)
+
+        # (2) File matching
+        for f in files:
+            file_path = os.path.relpath(os.path.join(root, f), path_to_scan).replace('\\', '/')
+            for pat in normalized_patterns:
+                if fnmatch.fnmatch(file_path, pat) or fnmatch.fnmatch(file_path, pat + "/*"):
+                    excluded_paths.add(file_path)
+                    break
+
+    return sorted(excluded_paths)

fosslight_util/help.py

@@ -3,7 +3,10 @@
 # Copyright (c) 2021 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
 import sys
-import pkg_resources
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    from importlib_metadata import version, PackageNotFoundError  # Python <3.8
 
 _HELP_MESSAGE_COMMON = """
          _______  _______  _______  _______  ___      ___           __
@@ -31,7 +34,12 @@ _HELP_MESSAGE_DOWNLOAD = """
     Optional:
         -h\t\t      Print help message
         -t\t\t      Output path name
-        -d\t\t      Directory name to save the log file"""
+        -d\t\t      Directory name to save the log file
+        -s\t\t      Source link to download
+        -t\t\t      Directory to download source code
+        -c\t\t      Checkout to branch or tag/ or version
+        -z\t\t      Unzip only compressed file
+        -o\t\t      Generate summary output file with this option"""
 
 
 class PrintHelpMsg():
@@ -50,7 +58,10 @@ class PrintHelpMsg():
 def print_package_version(pkg_name: str, msg: str = "", exitopt: bool = True) -> str:
     if msg == "":
         msg = f"{pkg_name} Version:"
-    cur_version = pkg_resources.get_distribution(pkg_name).version
+    try:
+        cur_version = version(pkg_name)
+    except PackageNotFoundError:
+        cur_version = "unknown"
 
     if exitopt:
         print(f'{msg} {cur_version}')