fosslight-util 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

fosslight_util/download.py

@@ -10,7 +10,7 @@ import zipfile
 import logging
 import argparse
 import shutil
-import pygit2 as git
+from git import Repo, GitCommandError
 import bz2
 import contextlib
 from datetime import datetime
@@ -230,14 +230,10 @@ def get_github_token(git_url):
 
 
 def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
-    ref_to_checkout = decide_checkout(checkout_to, tag, branch)
-    msg = ""
     oss_name = get_github_ossname(git_url)
-    oss_version = ""
-    github_token = get_github_token(git_url)
-    callbacks = None
-    if github_token != "":
-        callbacks = git.RemoteCallbacks(credentials=git.UserPass("foo", github_token))  # username is not used, so set to dummy
+    refs_to_checkout = decide_checkout(checkout_to, tag, branch)
+    clone_default_branch_flag = False
+    msg = ""
 
     try:
         if platform.system() != "Windows":
@@ -248,9 +244,26 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
             alarm.start()
 
         Path(target_dir).mkdir(parents=True, exist_ok=True)
-        repo = git.clone_repository(git_url, target_dir,
-                                    bare=False, repository=None,
-                                    remote=None, callbacks=callbacks)
+        if refs_to_checkout != "":
+            try:
+                # gitPython uses the branch argument the same whether you check out to a branch or a tag.
+                repo = Repo.clone_from(git_url, target_dir, branch=refs_to_checkout)
+            except GitCommandError as error:
+                error_msg = error.args[2].decode("utf-8")
+                if "Remote branch " + refs_to_checkout + " not found in upstream origin" in error_msg:
+                    # clone default branch, when non-existent branch or tag entered
+                    repo = Repo.clone_from(git_url, target_dir)
+                    clone_default_branch_flag = True
+        else:
+            repo = Repo.clone_from(git_url, target_dir)
+            clone_default_branch_flag = True
+
+        if refs_to_checkout != tag or clone_default_branch_flag:
+            oss_version = repo.active_branch.name
+        else:
+            oss_version = repo.git.describe('--tags')
+        logger.info(f"git checkout: {oss_version}")
+
         if platform.system() != "Windows":
             signal.alarm(0)
         else:
@@ -258,20 +271,8 @@ def download_git_clone(git_url, target_dir, checkout_to="", tag="", branch=""):
     except Exception as error:
         logger.warning(f"git clone - failed: {error}")
         msg = str(error)
-        return False, msg, oss_name, oss_version
-    try:
-        if ref_to_checkout != "":
-            ref_list = [x for x in repo.references]
-            ref_to_checkout = get_ref_to_checkout(ref_to_checkout, ref_list)
-            logger.info(f"git checkout: {ref_to_checkout}")
-            repo.checkout(ref_to_checkout)
+        return False, msg, oss_name, refs_to_checkout
 
-            for prefix_ref in prefix_refs:
-                if ref_to_checkout.startswith(prefix_ref):
-                    oss_version = ref_to_checkout[len(prefix_ref):]
-
-    except Exception as error:
-        logger.warning(f"git checkout to {ref_to_checkout} - failed: {error}")
     return True, msg, oss_name, oss_version
 
 

fosslight_util/oss_item.py

@@ -5,6 +5,7 @@
 
 import logging
 import os
+import hashlib
 from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_SCANNER
 from fosslight_util.cover import CoverItem
 from typing import List, Dict
@@ -171,6 +172,22 @@ class FileItem:
         return items
 
 
+def get_checksum_sha1(source_name_or_path) -> str:
+    checksum = CHECKSUM_NULL
+    try:
+        checksum = str(hashlib.sha1(source_name_or_path.encode()).hexdigest())
+    except Exception:
+        try:
+            f = open(source_name_or_path, "rb")
+            byte = f.read()
+            checksum = str(hashlib.sha1(byte).hexdigest())
+            f.close()
+        except Exception as ex:
+            _logger.info(f"(Error) Get_checksum: {ex}")
+
+    return checksum
+
+
 def invalid(cmd):
     _logger.info('[{}] is invalid'.format(cmd))
 

fosslight_util/write_spdx.py

@@ -8,31 +8,36 @@ import uuid
 import logging
 import re
 from pathlib import Path
-from spdx_tools.common.spdx_licensing import spdx_licensing
-from spdx_tools.spdx.model import (
-    Actor,
-    ActorType,
-    Checksum,
-    ChecksumAlgorithm,
-    CreationInfo,
-    Document,
-    File,
-    Package,
-    Relationship,
-    RelationshipType,
-    SpdxNoAssertion,
-    SpdxNone
-)
-from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document
-from spdx_tools.spdx.writer.write_anything import write_file
 from datetime import datetime
 from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick
 from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER,
                                      FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE)
+from fosslight_util.oss_item import CHECKSUM_NULL, get_checksum_sha1
 import traceback
 
 logger = logging.getLogger(LOGGER_NAME)
 
+try:
+    from spdx_tools.common.spdx_licensing import spdx_licensing
+    from spdx_tools.spdx.model import (
+        Actor,
+        ActorType,
+        Checksum,
+        ChecksumAlgorithm,
+        CreationInfo,
+        Document,
+        File,
+        Package,
+        Relationship,
+        RelationshipType,
+        SpdxNoAssertion,
+        SpdxNone
+    )
+    from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document
+    from spdx_tools.spdx.writer.write_anything import write_file
+except Exception:
+    logger.info('No import spdx-tools')
+
 
 def get_license_list_version():
     version = 'N/A'
@@ -81,12 +86,21 @@ def write_spdx(output_file_without_ext, output_extension, scan_item, spdx_versio
                 for file_item in file_items:
                     file = ''  # file의 license, copyright은 oss item에서 append
                     if scanner_name in [FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE]:
+                        if file_item.exclude:
+                            continue
+                        if file_item.checksum == CHECKSUM_NULL:
+                            if os.path.exists(file_item.source_name_or_path):
+                                file_item.checksum = get_checksum_sha1(file_item.source_name_or_path)
+                            if file_item.checksum == CHECKSUM_NULL:
+                                logger.info(f'Failed to get checksum, Skip: {file_item.source_name_or_path}')
+                                continue
                         file_id += 1
                         file = File(name=file_item.source_name_or_path,
                                     spdx_id=f'SPDXRef-File{file_id}',
                                     checksums=[Checksum(ChecksumAlgorithm.SHA1, file_item.checksum)])
                     file_license = []
                     file_copyright = []
+                    file_comment = []
                     for oss_item in file_item.oss_items:
                         oss_licenses = []
                         declared_oss_licenses = []
@@ -100,6 +114,7 @@ def write_spdx(output_file_without_ext, output_extension, scan_item, spdx_versio
                             except Exception:
                                 logger.debug(f'No spdx license name: {oi}')
                                 lic_comment.append(oi)
+                                file_comment.append(oi)
                         if oss_licenses:
                             file_license.extend(oss_licenses)
                         if oss_item.copyright != '':
@@ -157,8 +172,8 @@ def write_spdx(output_file_without_ext, output_extension, scan_item, spdx_versio
                             file.license_info_in_file = file_license
                         if file_copyright:
                             file.copyright_text = '\n'.join(file_copyright)
-                        if lic_comment:
-                            file.license_comment = ' '.join(lic_comment)
+                        if file_comment:
+                            file.license_comment = ' '.join(file_comment)
                         doc.files.append(file)
 
             if len(doc.packages) > 0:

{fosslight_util-2.0.1.dist-info → fosslight_util-2.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.0.1
+Version: 2.1.0
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -26,14 +26,15 @@ Requires-Dist: coloredlogs
 Requires-Dist: python3-wget
 Requires-Dist: beautifulsoup4
 Requires-Dist: jsonmerge
-Requires-Dist: spdx-tools
 Requires-Dist: setuptools>=65.5.1
 Requires-Dist: npm
 Requires-Dist: requests
+Requires-Dist: GitPython
 Requires-Dist: numpy; python_version < "3.8"
 Requires-Dist: numpy>=1.22.2; python_version >= "3.8"
 Requires-Dist: pygit2==1.6.1; python_version < "3.7"
 Requires-Dist: pygit2>=1.10.1; python_version >= "3.7"
+Requires-Dist: spdx-tools>=0.8.2; sys_platform != "win32"
 
 <!--
 Copyright (c) 2021 LG Electronics

{fosslight_util-2.0.1.dist-info → fosslight_util-2.1.0.dist-info}/RECORD

@@ -5,9 +5,9 @@ fosslight_util/constant.py,sha256=Ig3ACm9_QirE4389Wt-IfxOqRkVOUjqGnX1B05z2Byo,21
 fosslight_util/convert_excel_to_yaml.py,sha256=OJ11av4bsoxnVS15aa2aX-X3zGYUZW6M3118TPtHHTc,2323
 fosslight_util/correct.py,sha256=3iUipan8ZX8sbyIIGAPtMkAGvZ4YucjeJwx1K1Bx_z4,3897
 fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
-fosslight_util/download.py,sha256=X-R2RTWwmhx_LSIBZhIxzPTJZ2GwasZnhIsZ5m3hUig,14997
+fosslight_util/download.py,sha256=y2WYHUjAUH5kTTDtPrAop9sD1QDWTiGgFdUbNocIuH0,15070
 fosslight_util/help.py,sha256=M3_XahUkP794US9Q0NS6ujmGvrFFnKBHsTU95Fg1KpA,2181
-fosslight_util/oss_item.py,sha256=lDbBzKDG0diId39Rk-kqtwGtTLpdATDPtkP47FhlkMA,6382
+fosslight_util/oss_item.py,sha256=8W2HlwqGH3l1iPPdvycrRYKsBSBpqAkqYyYtBVPgMtY,6868
 fosslight_util/output_format.py,sha256=je3oVrDDnA160jIkFGpCHlG9Fc4YDlkQGwor2LFSmb0,8173
 fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
 fosslight_util/read_excel.py,sha256=-QvrdxaNqYOpIm1H7ZqIEh5NLvFPymZo6BAOZcQmQug,5263
@@ -17,15 +17,15 @@ fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g
 fosslight_util/write_excel.py,sha256=G0fIslbWoOtWZCJxbBGLCpUKbhmwrrqhI5PHwRw8_44,9931
 fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
 fosslight_util/write_scancodejson.py,sha256=81n7cWNYoyIKE_V4Kx5YtL2CgjMPIjoKdnSU3inkpJY,2163
-fosslight_util/write_spdx.py,sha256=azAaZmkIeFhx9YlV644B59K7SYkLMxQbtAr2mwixwBs,11265
+fosslight_util/write_spdx.py,sha256=Ov9jBlfVrkWIymcfAxbupUxDZKfCOZZGOPZ4v-x230M,12108
 fosslight_util/write_txt.py,sha256=BEFjYBppqk1CITx-fUN4vfvKv0XCs1GXWtc2Iu-etU4,629
 fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,3238
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.0.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.0.1.dist-info/METADATA,sha256=ESvt6K7SJ-ft5Lgt-5eNBzNv9Ss8EeJ4c5aYGB8it6s,6374
-fosslight_util-2.0.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-fosslight_util-2.0.1.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-2.0.1.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.0.1.dist-info/RECORD,,
+fosslight_util-2.1.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.0.dist-info/METADATA,sha256=8Ir3V_wALc0pYZMyeE48wBMFSE1CcOA28-HkPoeKMm4,6431
+fosslight_util-2.1.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+fosslight_util-2.1.0.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-2.1.0.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.0.dist-info/RECORD,,