fosslight-util 2.1.38__py3-none-any.whl → 2.1.40__py3-none-any.whl

fosslight_util/download.py

@@ -10,7 +10,7 @@ import zipfile
 import logging
 import argparse
 import shutil
-from git import Repo, GitCommandError, Git
+from git import Git
 import bz2
 import contextlib
 from datetime import datetime
@@ -217,7 +217,19 @@ def get_remote_refs(git_url: str):
     tags = []
     branches = []
     try:
-        cp = subprocess.run(["git", "ls-remote", "--tags", "--heads", git_url], capture_output=True, text=True, timeout=30)
+        env = os.environ.copy()
+        env["GIT_TERMINAL_PROMPT"] = "0"
+        env["GIT_ASKPASS"] = "echo"
+        env["GIT_CREDENTIAL_HELPER"] = ""
+        if "GIT_SSH_COMMAND" not in env:
+            env["GIT_SSH_COMMAND"] = "ssh -o BatchMode=yes -o StrictHostKeyChecking=no"
+        else:
+            env["GIT_SSH_COMMAND"] = env["GIT_SSH_COMMAND"] + " -o BatchMode=yes"
+        cp = subprocess.run(
+            ["git", "-c", "credential.helper=", "-c", "credential.helper=!",
+             "ls-remote", "--tags", "--heads", git_url],
+            env=env, capture_output=True, text=True, timeout=30,
+            stdin=subprocess.DEVNULL)
         if cp.returncode == 0:
             for line in cp.stdout.splitlines():
                 parts = line.split('\t')
@@ -291,32 +303,84 @@ def download_git_repository(refs_to_checkout, git_url, target_dir, tag, called_c
 
     logger.info(f"Download git url :{git_url}")
     env = os.environ.copy()
-    if not called_cli:
-        env["GIT_TERMINAL_PROMPT"] = "0"
+    env["GIT_TERMINAL_PROMPT"] = "0"
+    if platform.system() == "Windows":
+        env["GIT_ASKPASS"] = "echo"
+    else:
+        env["GIT_ASKPASS"] = "/bin/echo"
+    env["GIT_CREDENTIAL_HELPER"] = ""
+    # Disable credential helper via config
+    if "GIT_CONFIG_COUNT" not in env:
+        env["GIT_CONFIG_COUNT"] = "1"
+        env["GIT_CONFIG_KEY_0"] = "credential.helper"
+        env["GIT_CONFIG_VALUE_0"] = ""
+    if "GIT_SSH_COMMAND" not in env:
+        env["GIT_SSH_COMMAND"] = "ssh -o BatchMode=yes -o StrictHostKeyChecking=no"
+    else:
+        env["GIT_SSH_COMMAND"] = env["GIT_SSH_COMMAND"] + " -o BatchMode=yes"
+
     if refs_to_checkout:
         try:
-            # gitPython uses the branch argument the same whether you check out to a branch or a tag.
-            Repo.clone_from(git_url, target_dir, branch=refs_to_checkout, env=env)
-            if any(Path(target_dir).iterdir()):
-                success = True
-                oss_version = refs_to_checkout
-                logger.info(f"Files found in {target_dir} after clone.")
+            # For tags, we need full history. For branches, shallow clone is possible but
+            # we use full clone to ensure compatibility with all cases
+            # Use subprocess to ensure environment variables are properly passed
+            cmd = ["git", "-c", "credential.helper=", "-c", "credential.helper=!", "clone", git_url, target_dir]
+            result = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=600, stdin=subprocess.DEVNULL)
+            if result.returncode == 0:
+                # Checkout the specific branch or tag
+                checkout_cmd = ["git", "-C", target_dir, "checkout", refs_to_checkout]
+                checkout_result = subprocess.run(
+                    checkout_cmd, env=env, capture_output=True, text=True,
+                    timeout=60, stdin=subprocess.DEVNULL)
+                if checkout_result.returncode == 0:
+                    if any(Path(target_dir).iterdir()):
+                        success = True
+                        oss_version = refs_to_checkout
+                        logger.info(f"Files found in {target_dir} after clone and checkout.")
+                    else:
+                        logger.info(f"No files found in {target_dir} after clone.")
+                        success = False
+                else:
+                    logger.info(f"Git checkout error: {checkout_result.stderr}")
+                    # Clone succeeded but checkout failed (e.g. non-existent ref):
+                    # repo has default branch; treat as success with empty version
+                    if any(Path(target_dir).iterdir()):
+                        success = True
+                        oss_version = ""
+                        logger.info("Checkout failed; keeping default branch.")
             else:
-                logger.info(f"No files found in {target_dir} after clone.")
+                logger.info(f"Git clone error: {result.stderr}")
                 success = False
-        except GitCommandError as error:
-            logger.info(f"Git checkout error:{error}")
+        except subprocess.TimeoutExpired:
+            logger.info("Git clone timeout")
             success = False
         except Exception as e:
-            logger.info(f"Repo.clone_from error:{e}")
+            logger.info(f"Git clone error:{e}")
             success = False
 
     if not success:
-        Repo.clone_from(git_url, target_dir, env=env)
-        if any(Path(target_dir).iterdir()):
-            success = True
-        else:
-            logger.info(f"No files found in {target_dir} after clone.")
+        try:
+            # Use subprocess to ensure environment variables are properly passed
+            # No checkout needed, so shallow clone is sufficient
+            cmd = [
+                "git", "-c", "credential.helper=", "-c", "credential.helper=!",
+                "clone", "--depth", "1", git_url, target_dir
+            ]
+            result = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=600, stdin=subprocess.DEVNULL)
+            if result.returncode == 0:
+                if any(Path(target_dir).iterdir()):
+                    success = True
+                else:
+                    logger.info(f"No files found in {target_dir} after clone.")
+                    success = False
+            else:
+                logger.info(f"Git clone error: {result.stderr}")
+                success = False
+        except subprocess.TimeoutExpired:
+            logger.info("Git clone timeout")
+            success = False
+        except Exception as e:
+            logger.info(f"Git clone error:{e}")
             success = False
     return success, oss_version
 

fosslight_util/exclude.py

@@ -109,7 +109,13 @@ def get_excluded_paths(path_to_scan: str, custom_excluded_paths: list = [], cust
     path_to_exclude_with_dot = []
     excluded_files = set()  # Use set for O(1) operations
     abs_path_to_scan = os.path.abspath(path_to_scan)
-    custom_excluded_normalized = [p.replace('\\', '/') for p in custom_excluded_paths]
+    # Normalize: backslash to slash; trailing /* -> / so directory matching works without special case
+    custom_excluded_normalized = []
+    for p in custom_excluded_paths:
+        p = p.replace('\\', '/')
+        if p.endswith('/*'):
+            p = p[:-2] + '/'
+        custom_excluded_normalized.append(p)
     cnt_file_except_skipped = 0
 
     for root, dirs, files in os.walk(path_to_scan):
@@ -124,6 +130,11 @@ def get_excluded_paths(path_to_scan: str, custom_excluded_paths: list = [], cust
                         path_to_exclude_with_dot.append(rel_path)
                 elif rel_path in custom_excluded_normalized or rel_path + '/' in custom_excluded_normalized:
                     path_to_exclude.append(rel_path)
+                elif any(
+                    fnmatch.fnmatch(rel_path, pattern)
+                    for pattern in custom_excluded_normalized
+                ):
+                    path_to_exclude.append(rel_path)
 
         for file_name in files:
             file_path = os.path.join(root, file_name)
@@ -132,9 +143,11 @@ def get_excluded_paths(path_to_scan: str, custom_excluded_paths: list = [], cust
             except_info_sheet = False
             if not _has_parent_in_exclude_list(rel_path, path_to_exclude):
                 file_ext = os.path.splitext(file_name)[1].lstrip('.').lower()
-                if rel_path in custom_excluded_normalized:
-                    should_exclude = True
-                elif file_name in custom_excluded_normalized:
+                if any(
+                    fnmatch.fnmatch(rel_path, pattern)
+                    or fnmatch.fnmatch(file_name, pattern)
+                    for pattern in custom_excluded_normalized
+                ):
                     should_exclude = True
                 elif file_name.startswith('.'):
                     should_exclude = True

fosslight_util/help.py

@@ -55,23 +55,35 @@ _HELP_MESSAGE_COMMON = f"""
 
 
 _HELP_MESSAGE_DOWNLOAD = """
-    FOSSLight Downloader is a tool to download the package via input URL
-
-    Usage: fosslight_download [option1] <arg1> [options2] <arg2>
-     ex) fosslight_download -s http://github.com/fosslight/fosslight -t output_dir -d log_dir
-
-    Required:
-        -s\t\t      URL of the package to be downloaded
-
-    Optional:
-        -h\t\t      Print help message
-        -t\t\t      Output path name
-        -d\t\t      Directory name to save the log file
-        -s\t\t      Source link to download
-        -t\t\t      Directory to download source code
-        -c\t\t      Checkout to branch or tag/ or version
-        -z\t\t      Unzip only compressed file
-        -o\t\t      Generate summary output file with this option"""
+    📖 Usage
+    ────────────────────────────────────────────────────────────────────
+    fosslight_download [options] <arguments>
+
+    📝 Description
+    ────────────────────────────────────────────────────────────────────
+    FOSSLight Downloader is a tool to download a package or source code from a given URL.
+
+    ⚙️  General Options
+    ────────────────────────────────────────────────────────────────────
+    -s <url>              URL of the package or source to download (required)
+    -t <path>             Output directory to save the downloaded files
+    -d <log_dir>          Directory to save the log file
+    -c <branch/tag>       Checkout to branch, tag, or version after download
+    -z                    Unzip only compressed file
+    -o                    Generate summary output file
+    -h                    Show this help message
+
+    💡 Examples
+    ────────────────────────────────────────────────────────────────────
+    # Download a GitHub repository to output_dir and save log
+    fosslight_download -s https://github.com/fosslight/fosslight -t output_dir -d log_dir
+
+    # Download and checkout to a specific branch
+    fosslight_download -s https://github.com/fosslight/fosslight -t output_dir -c develop
+
+    # Download and unzip a compressed file
+    fosslight_download -s https://example.com/archive.zip -z -t output_dir
+"""
 
 
 class PrintHelpMsg():

fosslight_util/write_excel.py

@@ -9,7 +9,7 @@ import logging
 import os
 import pandas as pd
 from pathlib import Path
-from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER, FOSSLIGHT_BINARY
+from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER, FOSSLIGHT_BINARY, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SOURCE
 from jsonmerge import merge
 
 _HEADER = {'BIN (': ['ID', 'Binary Path', 'Source Code Path',
@@ -121,7 +121,12 @@ def write_result_to_excel(out_file_name, scan_item, extended_header={}, hide_hea
         workbook = xlsxwriter.Workbook(out_file_name)
         write_cover_sheet(workbook, scan_item.cover)
         if scan_item.file_items and len(scan_item.file_items.keys()) > 0:
-            for scanner_name, _ in scan_item.file_items.items():
+            sheet_order = [FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SOURCE, FOSSLIGHT_BINARY]
+            all_scanners = list(scan_item.file_items.keys())
+            priority = {name.lower(): idx for idx, name in enumerate(sheet_order)}
+            sorted_scanner_names = sorted(all_scanners,
+                                          key=lambda x: priority.get(x.lower(), len(priority)))
+            for scanner_name in sorted_scanner_names:
                 sheet_name = ""
                 if scanner_name.lower() in SHEET_NAME_FOR_SCANNER:
                     sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]

{fosslight_util-2.1.38.dist-info → fosslight_util-2.1.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fosslight_util
-Version: 2.1.38
+Version: 2.1.40
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Download-URL: https://github.com/fosslight/fosslight_util
@@ -22,7 +22,7 @@ Requires-Dist: lastversion
 Requires-Dist: coloredlogs
 Requires-Dist: beautifulsoup4
 Requires-Dist: jsonmerge
-Requires-Dist: spdx-tools==0.8.*; sys_platform == "linux"
+Requires-Dist: spdx-tools==0.8.2; sys_platform == "linux"
 Requires-Dist: setuptools>=65.5.1
 Requires-Dist: numpy
 Requires-Dist: requests

{fosslight_util-2.1.38.dist-info → fosslight_util-2.1.40.dist-info}/RECORD

@@ -4,10 +4,10 @@ fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE
 fosslight_util/constant.py,sha256=3BzJtyxC0o5IFAhYaUW-DeTKkA6f5tDJFJTb0k5ji9Y,2418
 fosslight_util/correct.py,sha256=1WEAL-9_KhjFPLucPhv0PNN3K7avm0z8mU6sTuSyeHM,3864
 fosslight_util/cover.py,sha256=yrWwguN5VM7mOllljiDVykPfuLYfI11kcJ6gEs49Bjo,2616
-fosslight_util/download.py,sha256=AWwD3FWhF-bMagWINJ-Dg1VMuycXbe8VXX7qQ09YjYs,23565
-fosslight_util/exclude.py,sha256=a8aEBglvc1Ub5cRtXHqpzbmuyhzgH3O-X1rFDHkOayc,6841
+fosslight_util/download.py,sha256=8um4tTn7j4nBqa7iET4aI6n5r0FCimpN3ko6MhYlxTY,26796
+fosslight_util/exclude.py,sha256=ojQa9xzPla5_S184GTcKdrrk-jWE-KKn4eBQcEA_Huo,7336
 fosslight_util/get_pom_license.py,sha256=vmh2LG1_4U7ts9SSpN3_UAANWbx5GBAVjremwm62OFY,5639
-fosslight_util/help.py,sha256=VomACZeXEMCiT1nxUwPqQAiVdNaMarwHVz9e10qbFc0,4954
+fosslight_util/help.py,sha256=tOKrQz1thNDMfl9nZYGSHJ8gkPjLKBwgGI44AD3kUyI,6143
 fosslight_util/oss_item.py,sha256=8890JHb5ZoKQWAwN7Fl8badnlYatJtF4MVJz1rdS4yQ,6938
 fosslight_util/output_format.py,sha256=BP23LspxawDZ_a99oWLVKWUQ-G7P5uoUpjEXhkRFKwc,8801
 fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
@@ -16,7 +16,7 @@ fosslight_util/set_log.py,sha256=AbcLFLvY9GSOYSN0a110wO5gNcyc8KKnNjl7GxHEW9A,400
 fosslight_util/spdx_licenses.py,sha256=GvMNe_D4v2meapTVwPu2BJXInnTo3_gIzg669eJhUu0,3691
 fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
 fosslight_util/write_cyclonedx.py,sha256=hq817j-0OM89B8jtZKgHgvVa0YEaYHlz_8R5vNpe21I,9662
-fosslight_util/write_excel.py,sha256=QUIMCnmEKJoSpri5RctBcKLvhDShLdZUP_dhHv-sVy8,10165
+fosslight_util/write_excel.py,sha256=lgN1BYXwYRC9qJxsHbvhBrzPfdLPNRt1CZKi4LEF0gQ,10575
 fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
 fosslight_util/write_scancodejson.py,sha256=dMCjTtUnNR5BCL6gBCleDT8bTSAN5Gg2RAfimmkGXUE,2692
 fosslight_util/write_spdx.py,sha256=Ov9jBlfVrkWIymcfAxbupUxDZKfCOZZGOPZ4v-x230M,12108
@@ -25,9 +25,9 @@ fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.1.38.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.1.38.dist-info/METADATA,sha256=8v1p52zrqs998FTozt23F13UkBRPPQofsD7GlHzbPHQ,6365
-fosslight_util-2.1.38.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-fosslight_util-2.1.38.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
-fosslight_util-2.1.38.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.1.38.dist-info/RECORD,,
+fosslight_util-2.1.40.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.40.dist-info/METADATA,sha256=59BAUojlZ3JoIUQVaAIHplVJ-qaQHNkLQx6pb3GdCgA,6365
+fosslight_util-2.1.40.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+fosslight_util-2.1.40.dist-info/entry_points.txt,sha256=0yZggRWNwDaClDG8UmUA10UFG8cVX3Jiy5gG9nW7hJs,68
+fosslight_util-2.1.40.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.40.dist-info/RECORD,,