fosslight-util 2.1.2__py3-none-any.whl → 2.1.4__py3-none-any.whl

fosslight_util/download.py

@@ -106,6 +106,7 @@ def cli_download_and_extract(link: str, target_dir: str, log_dir: str, checkout_
         datetime.now().strftime('%Y%m%d_%H-%M-%S')+".txt"
     logger, log_item = init_log(os.path.join(log_dir, log_file_name))
     link = link.strip()
+    is_rubygems = False
 
     try:
         if not link:

fosslight_util/output_format.py

@@ -8,11 +8,12 @@ from fosslight_util.write_excel import write_result_to_excel, write_result_to_cs
 from fosslight_util.write_opossum import write_opossum
 from fosslight_util.write_yaml import write_yaml
 from fosslight_util.write_spdx import write_spdx
+from fosslight_util.write_cyclonedx import write_cyclonedx
 from typing import Tuple
 
 SUPPORT_FORMAT = {'excel': '.xlsx', 'csv': '.csv', 'opossum': '.json', 'yaml': '.yaml',
                   'spdx-yaml': '.yaml', 'spdx-json': '.json', 'spdx-xml': '.xml',
-                  'spdx-tag': '.tag'}
+                  'spdx-tag': '.tag', 'cyclonedx-json': '.json', 'cyclonedx-xml': '.xml'}
 
 
 def check_output_format(output='', format='', customized_format={}):
@@ -182,12 +183,15 @@ def write_output_file(output_file_without_ext: str, file_extension: str, scan_it
             success, msg = write_opossum(result_file, scan_item)
         elif format == 'yaml':
             success, msg, _ = write_yaml(result_file, scan_item, False)
-        elif format.startswith('spdx'):
+        elif format.startswith('spdx') or format.startswith('cyclonedx'):
             if platform.system() == 'Windows' or platform.system() == 'Darwin':
                 success = False
                 msg = f'{platform.system()} not support spdx format.'
             else:
-                success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version)
+                if format.startswith('spdx'):
+                    success, msg, _ = write_spdx(output_file_without_ext, file_extension, scan_item, spdx_version)
+                elif format.startswith('cyclonedx'):
+                    success, msg, _ = write_cyclonedx(output_file_without_ext, file_extension, scan_item)
     else:
         if file_extension == '.xlsx':
             success, msg = write_result_to_excel(result_file, scan_item, extended_header, hide_header)

fosslight_util/write_cyclonedx.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2024 LG Electronics Inc.
+# Copyright (c) OWASP Foundation.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+import logging
+import re
+import json
+from pathlib import Path
+from datetime import datetime
+from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick
+from fosslight_util.constant import (LOGGER_NAME, FOSSLIGHT_DEPENDENCY, FOSSLIGHT_SCANNER,
+                                     FOSSLIGHT_BINARY, FOSSLIGHT_SOURCE)
+from fosslight_util.oss_item import CHECKSUM_NULL, get_checksum_sha1
+from packageurl import PackageURL
+import traceback
+try:
+    from cyclonedx.builder.this import this_component as cdx_lib_component
+    from cyclonedx.exception import MissingOptionalDependencyException
+    from cyclonedx.factory.license import LicenseFactory
+    from cyclonedx.model import XsUri, ExternalReferenceType
+    from cyclonedx.model.bom import Bom
+    from cyclonedx.model.component import Component, ComponentType, HashAlgorithm, HashType, ExternalReference
+    from cyclonedx.model.contact import OrganizationalEntity
+    from cyclonedx.output import make_outputter, BaseOutput
+    from cyclonedx.output.json import JsonV1Dot6
+    from cyclonedx.schema import OutputFormat, SchemaVersion
+    from cyclonedx.validation import make_schemabased_validator
+    from cyclonedx.validation.json import JsonStrictValidator
+    from cyclonedx.output.json import Json as JsonOutputter
+    from cyclonedx.output.xml import Xml as XmlOutputter
+    from cyclonedx.validation.xml import XmlValidator
+except Exception:
+    logger.info('No import cyclonedx-python-lib')
+logger = logging.getLogger(LOGGER_NAME)
+
+
+def write_cyclonedx(output_file_without_ext, output_extension, scan_item):
+    success = True
+    error_msg = ''
+
+    bom = Bom()
+    if scan_item:
+        try:
+            cover_name = scan_item.cover.get_print_json()["Tool information"].split('(').pop(0).strip()
+            match = re.search(r"(.+) v([0-9.]+)", cover_name)
+            if match:
+                scanner_name = match.group(1)
+            else:
+                scanner_name = FOSSLIGHT_SCANNER
+        except Exception:
+            cover_name = FOSSLIGHT_SCANNER
+            scanner_name = FOSSLIGHT_SCANNER
+
+        lc_factory = LicenseFactory()
+        bom.metadata.tools.components.add(cdx_lib_component())
+        bom.metadata.tools.components.add(Component(name=scanner_name.upper(),
+                                                    type=ComponentType.APPLICATION))
+        comp_id = 0
+        bom.metadata.component = root_component = Component(name='Root Component',
+                                                            type=ComponentType.APPLICATION,
+                                                            bom_ref=str(comp_id))
+        relation_tree = {}
+        bom_ref_packages = []
+
+        output_dir = os.path.dirname(output_file_without_ext)
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        try:
+            root_package = False
+            for scanner_name, file_items in scan_item.file_items.items():
+                for file_item in file_items:
+                    if file_item.exclude:
+                        continue
+                    if scanner_name == FOSSLIGHT_SOURCE:
+                        comp_type = ComponentType.FILE
+                    else:
+                        comp_type = ComponentType.LIBRARY
+
+                    for oss_item in file_item.oss_items:
+                        if oss_item.name == '':
+                            if scanner_name == FOSSLIGHT_DEPENDENCY:
+                                continue
+                            else:
+                                comp_name = file_item.source_name_or_path
+                        else:
+                            comp_name = oss_item.name
+
+                        comp_id += 1
+                        comp = Component(type=comp_type,
+                                         name=comp_name,
+                                         bom_ref=str(comp_id))
+
+                        if oss_item.version != '':
+                            comp.version = oss_item.version
+                        if oss_item.copyright != '':
+                            comp.copyright = oss_item.copyright
+                        if scanner_name == FOSSLIGHT_DEPENDENCY and file_item.purl:
+                            comp.purl = PackageURL.from_string(file_item.purl)
+                        if scanner_name != FOSSLIGHT_DEPENDENCY:
+                            comp.hashes = [HashType(alg=HashAlgorithm.SHA_1, content=file_item.checksum)]
+
+                        if oss_item.download_location != '':
+                            comp.external_references = [ExternalReference(url=XsUri(oss_item.download_location),
+                                                                          type=ExternalReferenceType.WEBSITE)]
+
+                        oss_licenses = []
+                        for ol in oss_item.license:
+                            try:
+                                oss_licenses.append(lc_factory.make_from_string(ol))
+                            except Exception:
+                                logger.info(f'No spdx license name: {oi}')
+                        if oss_licenses:
+                            comp.licenses = oss_licenses
+
+                        root_package = False
+                        if scanner_name == FOSSLIGHT_DEPENDENCY:
+                            if oss_item.comment:
+                                oss_comment = oss_item.comment.split('/')
+                                for oc in oss_comment:
+                                    if oc in ['direct', 'transitive', 'root package']:
+                                        if oc == 'direct':
+                                            bom.register_dependency(root_component, [comp])
+                                        elif oc == 'root package':
+                                            root_package = True
+                                            root_component.name = comp_name
+                                            root_component.type = comp_type
+                                            comp_id -= 1
+                            else:
+                                bom.register_dependency(root_component, [comp])
+                            if len(file_item.depends_on) > 0:
+                                purl = file_item.purl
+                                relation_tree[purl] = []
+                                relation_tree[purl].extend(file_item.depends_on)
+
+                        if not root_package:
+                            bom.components.add(comp)
+
+            if len(bom.components) > 0:
+                for comp_purl in relation_tree:
+                    comp = bom.get_component_by_purl(PackageURL.from_string(comp_purl))
+                    if comp:
+                        dep_comp_list = []
+                        for dep_comp_purl in relation_tree[comp_purl]:
+                            dep_comp = bom.get_component_by_purl(PackageURL.from_string(dep_comp_purl))
+                            if dep_comp:
+                                dep_comp_list.append(dep_comp)
+                        bom.register_dependency(comp, dep_comp_list)
+
+        except Exception as e:
+            success = False
+            error_msg = f'Failed to create CycloneDX document object:{e}, {traceback.format_exc()}'
+    else:
+        success = False
+        error_msg = 'No item to write in output file.'
+
+    result_file = ''
+    if success:
+        result_file = output_file_without_ext + output_extension
+        try:
+            if output_extension == '.json':
+                write_cyclonedx_json(bom, result_file)
+            elif output_extension == '.xml':
+                write_cyclonedx_xml(bom, result_file)
+            else:
+                success = False
+                error_msg = f'Not supported output_extension({output_extension})'
+        except Exception as e:
+            success = False
+            error_msg = f'Failed to write CycloneDX document: {e}'
+            if os.path.exists(result_file):
+                os.remove(result_file)
+
+    return success, error_msg, result_file
+
+
+def write_cyclonedx_json(bom, result_file):
+    success = True
+    try:
+        my_json_outputter: 'JsonOutputter' = JsonV1Dot6(bom)
+        my_json_outputter.output_to_file(result_file)
+        serialized_json = my_json_outputter.output_as_string(indent=2)
+        my_json_validator = JsonStrictValidator(SchemaVersion.V1_6)
+        try:
+            validation_errors = my_json_validator.validate_str(serialized_json)
+            if validation_errors:
+                logger.warning(f'JSON invalid, ValidationError: {repr(validation_errors)}')
+        except MissingOptionalDependencyException as error:
+            logger.debug(f'JSON-validation was skipped due to {error}')
+    except Exception as e:
+        success = False
+    return success
+        
+
+
+def write_cyclonedx_xml(bom, result_file):
+    success = True
+    try:
+        my_xml_outputter: BaseOutput = make_outputter(bom=bom,
+                                                      output_format=OutputFormat.XML,
+                                                      schema_version=SchemaVersion.V1_6)
+        my_xml_outputter.output_to_file(filename=result_file)
+        serialized_xml = my_xml_outputter.output_as_string(indent=2)
+        my_xml_validator = XmlValidator(SchemaVersion.V1_6)
+        try:
+            validation_errors = my_xml_validator.validate_str(serialized_xml)
+            if validation_errors:
+                logger.warning(f'XML invalid, ValidationError: {repr(validation_errors)}')
+        except MissingOptionalDependencyException as error:
+            logger.debug(f'XML-validation was skipped due to {error}')
+    except Exception as e:
+        success = False
+    return success

{fosslight_util-2.1.2.dist-info → fosslight_util-2.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 2.1.2
+Version: 2.1.4
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -35,6 +35,7 @@ Requires-Dist: numpy>=1.22.2; python_version >= "3.8"
 Requires-Dist: pygit2==1.6.1; python_version < "3.7"
 Requires-Dist: pygit2>=1.10.1; python_version >= "3.7"
 Requires-Dist: spdx-tools==0.8.*; sys_platform == "linux"
+Requires-Dist: cyclonedx-python-lib==8.5.*; sys_platform == "linux"
 
 <!--
 Copyright (c) 2021 LG Electronics

{fosslight_util-2.1.2.dist-info → fosslight_util-2.1.4.dist-info}/RECORD

@@ -4,15 +4,16 @@ fosslight_util/compare_yaml.py,sha256=eLqqCLgERxRHN5vsnpQVMXIEU862Lx66mD_y4uMgQE
 fosslight_util/constant.py,sha256=Ig3ACm9_QirE4389Wt-IfxOqRkVOUjqGnX1B05z2Byo,2151
 fosslight_util/correct.py,sha256=3iUipan8ZX8sbyIIGAPtMkAGvZ4YucjeJwx1K1Bx_z4,3897
 fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
-fosslight_util/download.py,sha256=NrAThwiulcWZhV9fBsc8_i7TWLExptmJ_JDnU7TnzuA,16340
+fosslight_util/download.py,sha256=bCKvW76XJTnKMAUW5sJZxg_wBUhiybXovJuL04W4P4c,16364
 fosslight_util/help.py,sha256=M3_XahUkP794US9Q0NS6ujmGvrFFnKBHsTU95Fg1KpA,2181
 fosslight_util/oss_item.py,sha256=8W2HlwqGH3l1iPPdvycrRYKsBSBpqAkqYyYtBVPgMtY,6868
-fosslight_util/output_format.py,sha256=hELTfwo0PqoRtuSnQLbSrPwoX5bjddB50q-aHErs1YE,8324
+fosslight_util/output_format.py,sha256=AdQVzCpar6n3PCDtijLWbJyFAGKQVE7JhLXlHPtITH4,8678
 fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
 fosslight_util/read_excel.py,sha256=-QvrdxaNqYOpIm1H7ZqIEh5NLvFPymZo6BAOZcQmQug,5263
 fosslight_util/set_log.py,sha256=Xpa94AiOyGEK8ucaYkvkAllvlen1Pq_d6UG6kPYBYBc,3780
 fosslight_util/spdx_licenses.py,sha256=GvMNe_D4v2meapTVwPu2BJXInnTo3_gIzg669eJhUu0,3691
 fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
+fosslight_util/write_cyclonedx.py,sha256=V_LFKocErNSwVC8CsLxRcdwlrutVuBSb_3HJ6bO_2J8,9896
 fosslight_util/write_excel.py,sha256=G0fIslbWoOtWZCJxbBGLCpUKbhmwrrqhI5PHwRw8_44,9931
 fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
 fosslight_util/write_scancodejson.py,sha256=81n7cWNYoyIKE_V4Kx5YtL2CgjMPIjoKdnSU3inkpJY,2163
@@ -22,9 +23,9 @@ fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,
 fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
 fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
 fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-2.1.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-2.1.2.dist-info/METADATA,sha256=_ES9pW6X4xAdZEw7uykDuE1h4R_4n4IR7hO7xMxQfqQ,6431
-fosslight_util-2.1.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-fosslight_util-2.1.2.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-2.1.2.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-2.1.2.dist-info/RECORD,,
+fosslight_util-2.1.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.1.4.dist-info/METADATA,sha256=iLIhl1sg0DitSa3xzxABBJf04OUmLwGUX3erMa-CdX8,6499
+fosslight_util-2.1.4.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+fosslight_util-2.1.4.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-2.1.4.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.1.4.dist-info/RECORD,,