fosslight-util 1.4.47__py3-none-any.whl → 2.0.0__py3-none-any.whl

fosslight_util/write_spdx.py

@@ -21,10 +21,10 @@ from spdx.writers import yaml
 from spdx.writers import xml
 from spdx.writers import tagvalue
 from fosslight_util.spdx_licenses import get_spdx_licenses_json, get_license_from_nick
-import fosslight_util.constant as constant
+from fosslight_util.constant import LOGGER_NAME, FOSSLIGHT_DEPENDENCY
 import traceback
 
-logger = logging.getLogger(constant.LOGGER_NAME)
+logger = logging.getLogger(LOGGER_NAME)
 
 
 def get_license_list_version():
@@ -37,11 +37,11 @@ def get_license_list_version():
     return version
 
 
-def write_spdx(output_file_without_ext, output_extension, sheet_list,
+def write_spdx(output_file_without_ext, output_extension, scan_item,
                scanner_name, scanner_version, spdx_version=(2, 3)):
     success = True
     error_msg = ''
-    if sheet_list:
+    if scan_item:
         doc = Document(version=Version(*spdx_version),
                        data_license=License.from_identifier('CC0-1.0'),
                        namespace=f'http://spdx.org/spdxdocs/{scanner_name.lower()}-{uuid.uuid4()}',
@@ -60,44 +60,39 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
         try:
             package_id = 0
             root_package = False
-            for sheet_name, sheet_contents in sheet_list.items():
-                if sheet_name not in constant.supported_sheet_and_scanner.keys():
-                    continue
-                scanner = constant.supported_sheet_and_scanner.get(sheet_name)
-                for oss_item in sheet_contents:
-                    if len(oss_item) < 9:
-                        logger.warning(f"sheet list is too short ({len(oss_item)}): {oss_item}")
-                        continue
+            for scanner_name, _ in scan_item.file_items.items():
+                json_contents = scan_item.get_print_json(scanner_name)
+                for oss_item in json_contents:
                     package_id += 1
                     package = Package(spdx_id=f'SPDXRef-{package_id}')
 
-                    if oss_item[1] != '':
-                        package.name = oss_item[1]  # required
+                    if oss_item.get('name', '') != '':
+                        package.name = oss_item.get('name', '')  # required
                     else:
                         package.name = SPDXNone()
 
-                    if oss_item[2] != '':
-                        package.version = oss_item[2]  # no required
+                    if oss_item.get('version', '') != '':
+                        package.version = oss_item.get('version', '')  # no required
 
-                    if oss_item[4] != '':
-                        package.download_location = oss_item[4]  # required
+                    if oss_item.get('download location', '') != '':
+                        package.download_location = oss_item.get('download location', '')  # required
                     else:
                         package.download_location = SPDXNone()
 
-                    if scanner == constant.FL_DEPENDENCY:
+                    if scanner_name == FOSSLIGHT_DEPENDENCY:
                         package.files_analyzed = False  # If omitted, the default value of true is assumed.
                     else:
                         package.files_analyzed = True
 
-                    if oss_item[5] != '':
-                        package.homepage = oss_item[5]  # no required
+                    if oss_item.get('homepage', '') != '':
+                        package.homepage = oss_item.get('homepage', '')  # no required
 
-                    if oss_item[6] != '':
-                        package.cr_text = oss_item[3]  # required
+                    if oss_item.get('copyright text', '') != '':
+                        package.cr_text = oss_item.get('copyright text', '')  # required
                     else:
                         package.cr_text = SPDXNone()
-                    if oss_item[3] != '':
-                        lic_list = [check_input_license_format(lic.strip()) for lic in oss_item[3].split(',')]
+                    if oss_item.get('license', []) != '':
+                        lic_list = [check_input_license_format(lic.strip()) for lic in oss_item.get('license', [])]
                         first_lic = License.from_identifier(lic_list.pop(0))
                         while lic_list:
                             next_lic = License.from_identifier(lic_list.pop(0))
@@ -109,21 +104,21 @@ def write_spdx(output_file_without_ext, output_extension, sheet_list,
 
                     doc.add_package(package)
 
-                    if scanner == constant.FL_DEPENDENCY:
-                        spdx_id_packages.append([package.name, package.spdx_id])
-                        comment = oss_item[8]
-                        relation_tree[package.name] = {}
-                        relation_tree[package.name]['id'] = package.spdx_id
-                        relation_tree[package.name]['dep'] = []
+                    if scanner_name == FOSSLIGHT_DEPENDENCY:
+                        purl = oss_item.get('package url', '')
+                        spdx_id_packages.append([purl, package.spdx_id])
+                        comment = oss_item.get('comment', '')
+                        relation_tree[purl] = {}
+                        relation_tree[purl]['id'] = package.spdx_id
+                        relation_tree[purl]['dep'] = []
 
                         if 'root package' in comment.split(','):
                             root_package = True
                             relationship = Relationship(f"{doc.spdx_id} DESCRIBES {package.spdx_id}")
                             doc.add_relationship(relationship)
-                        if len(oss_item) > 9:
-                            deps = oss_item[9]
-                            relation_tree[package.name]['dep'].extend([di.strip().split('(')[0] for di in deps.split(',')])
-            if scanner == constant.FL_DEPENDENCY and len(relation_tree) > 0:
+                        deps = oss_item.get('depends on', '')
+                        relation_tree[purl]['dep'].extend([di.strip().split('(')[0] for di in deps])
+            if scanner_name == FOSSLIGHT_DEPENDENCY and len(relation_tree) > 0:
                 for pkg in relation_tree:
                     if len(relation_tree[pkg]['dep']) > 0:
                         pkg_spdx_id = relation_tree[pkg]['id']

fosslight_util/write_txt.py

@@ -4,9 +4,10 @@
 # SPDX-License-Identifier: Apache-2.0
 import os
 from pathlib import Path
+from typing import Tuple
 
 
-def write_txt_file(file_to_create, str_to_write):
+def write_txt_file(file_to_create: str, str_to_write: str) -> Tuple[bool, str]:
     success = True
     error_msg = ""
     try:

fosslight_util/write_yaml.py

@@ -2,60 +2,49 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2022 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
-
 import yaml
 import logging
 import os
-import copy
+import json
 from pathlib import Path
-import fosslight_util.constant as constant
-from fosslight_util.oss_item import OssItem
-from fosslight_util.write_excel import _EMPTY_ITEM_MSG
+from fosslight_util.constant import LOGGER_NAME, SHEET_NAME_FOR_SCANNER
+from typing import Tuple
 
-_logger = logging.getLogger(constant.LOGGER_NAME)
+_logger = logging.getLogger(LOGGER_NAME)
 
 
-def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
+def write_yaml(output_file, scan_item, separate_yaml=False) -> Tuple[bool, str, str]:
     success = True
     error_msg = ""
     output = ""
 
     try:
-        sheet_list = copy.deepcopy(sheet_list_origin)
-        if sheet_list:
-            output_files = []
-            output_dir = os.path.dirname(output_file)
-
-            Path(output_dir).mkdir(parents=True, exist_ok=True)
-            if separate_yaml:
-                filename = os.path.splitext(os.path.basename(output_file))[0]
-                separate_output_file = os.path.join(output_dir, filename)
-
-            merge_sheet = []
-            for sheet_name, sheet_contents in sheet_list.items():
-                if sheet_name not in constant.supported_sheet_and_scanner.keys():
-                    continue
-                scanner_name = constant.supported_sheet_and_scanner[sheet_name]
-                sheet_contents_with_scanner = []
-                for i in sheet_contents:
-                    i.insert(0, scanner_name)
-                    sheet_contents_with_scanner.append(i)
-                if not separate_yaml:
-                    merge_sheet.extend(sheet_contents_with_scanner)
-                else:
-                    output_file = f'{separate_output_file}_{sheet_name}.yaml'
-                    convert_sheet_to_yaml(sheet_contents_with_scanner, output_file)
-                    output_files.append(output_file)
+        output_files = []
+        output_dir = os.path.dirname(output_file)
+
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        if separate_yaml:
+            filename = os.path.splitext(os.path.basename(output_file))[0]
+            separate_output_file = os.path.join(output_dir, filename)
+
+        merge_sheet = []
+        for scanner_name, _ in scan_item.file_items.items():
+            sheet_name = SHEET_NAME_FOR_SCANNER[scanner_name.lower()]
+            json_contents = scan_item.get_print_json(scanner_name)
 
             if not separate_yaml:
-                convert_sheet_to_yaml(merge_sheet, output_file)
+                merge_sheet.extend(json_contents)
+            else:
+                output_file = f'{separate_output_file}_{sheet_name}.yaml'
+                remove_duplicates_and_dump_yaml(json_contents, output_file)
                 output_files.append(output_file)
 
-            if output_files:
-                output = ", ".join(output_files)
-        else:
-            success = False
-            error_msg = _EMPTY_ITEM_MSG
+        if not separate_yaml:
+            remove_duplicates_and_dump_yaml(merge_sheet, output_file)
+            output_files.append(output_file)
+
+        if output_files:
+            output = ", ".join(output_files)
     except Exception as ex:
         error_msg = str(ex)
         success = False
@@ -66,37 +55,37 @@ def write_yaml(output_file, sheet_list_origin, separate_yaml=False):
     return success, error_msg, output
 
 
-def convert_sheet_to_yaml(sheet_contents_with_scanner, output_file):
-    sheet_contents_with_scanner = [list(t) for t in set(tuple(e) for e in sorted(sheet_contents_with_scanner))]
+def remove_duplicates_and_dump_yaml(json_contents, output_file):
+    unique_json_strings = {json.dumps(e, sort_keys=True) for e in json_contents}
+    unique_json_contents = [json.loads(e) for e in unique_json_strings]
 
     yaml_dict = {}
-    for sheet_item in sheet_contents_with_scanner:
-        item = OssItem('')
-        item.set_sheet_item(sheet_item[1:], sheet_item[0])
-        create_yaml_with_ossitem(item, yaml_dict)
+    for uitem in unique_json_contents:
+        create_yaml_with_ossitem(uitem, yaml_dict)
 
     with open(output_file, 'w') as f:
         yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False)
 
 
 def create_yaml_with_ossitem(item, yaml_dict):
-    item_json = item.get_print_json()
-    item_name = item_json.pop("name")
+    item_name = item.pop("name")
 
     if item_name not in yaml_dict.keys():
         yaml_dict[item_name] = []
     merged = False
     for oss_info in yaml_dict[item_name]:
-        if oss_info.get('version', '') == item.version and \
-           oss_info.get('license', []) == item.license and \
-           oss_info.get('copyright text', '') == item.copyright and \
-           oss_info.get('homepage', '') == item.homepage and \
-           oss_info.get('download location', '') == item.download_location and \
-           oss_info.get('exclude', False) == item.exclude:
-            oss_info.get('source path', []).extend(item.source_name_or_path)
+        if oss_info.get('version', '') == item.get('version', '') and \
+           oss_info.get('license', []) == item.get('license', []) and \
+           oss_info.get('copyright text', '') == item.get('copyright text', '') and \
+           oss_info.get('homepage', '') == item.get('homepage', '') and \
+           oss_info.get('download location', '') == item.get('download location', '') and \
+           oss_info.get('exclude', False) == item.get('exclude', False):
+            if isinstance(oss_info.get('source path', []), str):
+                oss_info['source path'] = [oss_info.get('source path', '')]
+            oss_info.get('source path', []).append(item.get('source path', ''))
             oss_info.pop('comment', None)
             merged = True
             break
 
     if not merged:
-        yaml_dict[item_name].append(item_json)
+        yaml_dict[item_name].append(item)

{fosslight_util-1.4.47.dist-info → fosslight_util-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fosslight-util
-Version: 1.4.47
+Version: 2.0.0
 Summary: FOSSLight Util
 Home-page: https://github.com/fosslight/fosslight_util
 Author: LG Electronics
@@ -18,7 +18,6 @@ Classifier: Programming Language :: Python :: 3.11
 Description-Content-Type: text/markdown
 Requires-Dist: XlsxWriter
 Requires-Dist: pandas
-Requires-Dist: xlrd ==1.2.0
 Requires-Dist: openpyxl
 Requires-Dist: progress
 Requires-Dist: PyYAML
@@ -27,14 +26,14 @@ Requires-Dist: coloredlogs
 Requires-Dist: python3-wget
 Requires-Dist: beautifulsoup4
 Requires-Dist: jsonmerge
-Requires-Dist: spdx-tools ==0.7.0rc0
-Requires-Dist: setuptools >=65.5.1
+Requires-Dist: spdx-tools==0.7.0rc0
+Requires-Dist: setuptools>=65.5.1
 Requires-Dist: npm
 Requires-Dist: requests
-Requires-Dist: numpy ; python_version < "3.8"
-Requires-Dist: numpy >=1.22.2 ; python_version >= "3.8"
-Requires-Dist: pygit2 ==1.6.1 ; python_version<'3.7'
-Requires-Dist: pygit2 >=1.10.1 ; python_version>='3.7'
+Requires-Dist: numpy; python_version < "3.8"
+Requires-Dist: numpy>=1.22.2; python_version >= "3.8"
+Requires-Dist: pygit2==1.6.1; python_version < "3.7"
+Requires-Dist: pygit2>=1.10.1; python_version >= "3.7"
 
 <!--
 Copyright (c) 2021 LG Electronics

fosslight_util-2.0.0.dist-info/RECORD

@@ -0,0 +1,31 @@
+fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fosslight_util/_get_downloadable_url.py,sha256=63ZPI4KCpUFgL4oheKm8zvekuCRzpwNkVaLJcA-uA90,9010
+fosslight_util/compare_yaml.py,sha256=znKz8sp6dk9wwBkJl-MQmkKTmL1IKNOrBWZklW7QVtU,2790
+fosslight_util/constant.py,sha256=Ig3ACm9_QirE4389Wt-IfxOqRkVOUjqGnX1B05z2Byo,2151
+fosslight_util/convert_excel_to_yaml.py,sha256=OJ11av4bsoxnVS15aa2aX-X3zGYUZW6M3118TPtHHTc,2323
+fosslight_util/correct.py,sha256=3iUipan8ZX8sbyIIGAPtMkAGvZ4YucjeJwx1K1Bx_z4,3897
+fosslight_util/cover.py,sha256=qqqKzxqFwKimal764FaugRUBcHWdeKt8af6xeK0mH8E,2040
+fosslight_util/download.py,sha256=X-R2RTWwmhx_LSIBZhIxzPTJZ2GwasZnhIsZ5m3hUig,14997
+fosslight_util/help.py,sha256=M3_XahUkP794US9Q0NS6ujmGvrFFnKBHsTU95Fg1KpA,2181
+fosslight_util/oss_item.py,sha256=TlTxBS98_eTOrZ1GifyTgi3lrUHa7edqVbgR4_8rGKw,6324
+fosslight_util/output_format.py,sha256=9O2IdhO3GRmrLvicvED-OXH1jPQpxXcLdwRm_lBo0rI,4983
+fosslight_util/parsing_yaml.py,sha256=2zx_N5lMkXT1dRmfJMpzlrru-y_2F_CkVbGlba6vQpU,5380
+fosslight_util/read_excel.py,sha256=-QvrdxaNqYOpIm1H7ZqIEh5NLvFPymZo6BAOZcQmQug,5263
+fosslight_util/set_log.py,sha256=Xpa94AiOyGEK8ucaYkvkAllvlen1Pq_d6UG6kPYBYBc,3780
+fosslight_util/spdx_licenses.py,sha256=GvMNe_D4v2meapTVwPu2BJXInnTo3_gIzg669eJhUu0,3691
+fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
+fosslight_util/write_excel.py,sha256=G0fIslbWoOtWZCJxbBGLCpUKbhmwrrqhI5PHwRw8_44,9931
+fosslight_util/write_opossum.py,sha256=ltmo6SkugKWdAYupeCqwE4-3lua0GwLpix1XqFC-tT8,11678
+fosslight_util/write_scancodejson.py,sha256=81n7cWNYoyIKE_V4Kx5YtL2CgjMPIjoKdnSU3inkpJY,2163
+fosslight_util/write_spdx.py,sha256=MwR_OO9ua1Uvnk77etJXMHFUqnku2sZuuALdeIp7g4k,9478
+fosslight_util/write_txt.py,sha256=BEFjYBppqk1CITx-fUN4vfvKv0XCs1GXWtc2Iu-etU4,629
+fosslight_util/write_yaml.py,sha256=QlEKoIPQsEaYERfbP53TeKgnllYzhLQWm5wYjnWtVjE,3238
+fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
+fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
+fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
+fosslight_util-2.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+fosslight_util-2.0.0.dist-info/METADATA,sha256=GhApxdXkZnaERDjLepsy2WKvV1sJdt9B4FydaZqlHAk,6384
+fosslight_util-2.0.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+fosslight_util-2.0.0.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
+fosslight_util-2.0.0.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
+fosslight_util-2.0.0.dist-info/RECORD,,

{fosslight_util-1.4.47.dist-info → fosslight_util-2.0.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fosslight_util-1.4.47.dist-info/RECORD

@@ -1,31 +0,0 @@
-fosslight_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fosslight_util/_get_downloadable_url.py,sha256=63ZPI4KCpUFgL4oheKm8zvekuCRzpwNkVaLJcA-uA90,9010
-fosslight_util/compare_yaml.py,sha256=0bapoyS0yrzkiK70K57E8-3wZ_D6mZAJ-24Eq9TYBlk,2632
-fosslight_util/constant.py,sha256=j9uhncoC2Fn4j4ATNsjUoS91nVXhyGzU6xLyWFj941A,1834
-fosslight_util/convert_excel_to_yaml.py,sha256=7ZsAMMQJIEXrmcl_28nSHvFpGMi1ZiRZYpEfI5O8vP8,2298
-fosslight_util/correct.py,sha256=O8c4vIl5Hdc46Ijr63LfcTjIZE5fgqN2iKRadHqluUU,6253
-fosslight_util/cover.py,sha256=GZAeGoQGGCx7OEddErS_Mj6pbMCQrdr5jX-CixU4iAA,1607
-fosslight_util/download.py,sha256=X-R2RTWwmhx_LSIBZhIxzPTJZ2GwasZnhIsZ5m3hUig,14997
-fosslight_util/help.py,sha256=xhAf43fuRLOU3TE7jqaxVTizVn4aANDvHaHYQAdaqYc,2154
-fosslight_util/oss_item.py,sha256=Wc2ka8cZc3D9JiJXl4-Y69A_qcGuyMiyYq053Srq638,7267
-fosslight_util/output_format.py,sha256=r_OuYyQGCsCMkjJ3NYUyl4DJDYEDGZI7AWzz4fXPwSg,4883
-fosslight_util/parsing_yaml.py,sha256=m1kg6sYBJTmLmobsD-fjFxOz2NuIVjUvoEvXxmvBOpU,4561
-fosslight_util/read_excel.py,sha256=3zW2F6Azj4ufsrStBQGArBDCLG-BTJGG54HIqMxlx_A,5392
-fosslight_util/set_log.py,sha256=GsUtbtqmYKgBltHzeaxV3H9gaUWMYQYDJEhr_LdEGoo,3258
-fosslight_util/spdx_licenses.py,sha256=r90hUY4_T-XrHIJHLx1Ox3gWZ3qzdZj9rJFo7AwmkPE,3641
-fosslight_util/timer_thread.py,sha256=5VbZENQPD-N0NUmzEktqGr6Am-e7vxD79K05mmr29g0,433
-fosslight_util/write_excel.py,sha256=na3XB_rIr9ekCugGUMCE-AqrndCbvqhmzhoCO0pZRJk,12298
-fosslight_util/write_opossum.py,sha256=PGJV5DysNJvIFbzsyGXxh_kRcvZuHAOmLs-WlXP8qMI,11831
-fosslight_util/write_scancodejson.py,sha256=CSKjuwbA04nK5ogXklNsCBgGDZXPr805T8KPLgvx71U,2282
-fosslight_util/write_spdx.py,sha256=B_aHv9vScgZI5gHo5Hd56ckNWOHdyAQebRV54bTx9ec,9542
-fosslight_util/write_txt.py,sha256=Ms8E2wbc0U84IYP0dxTAbIfX5PzpQG3xblu8lv1w8J0,574
-fosslight_util/write_yaml.py,sha256=nzuIPdjOAOM4NS5VtU-KFrft41fKEF05-cl0OlEHj7Y,3657
-fosslight_util/resources/frequentLicenselist.json,sha256=GUhzK6tu7ok10fekOnmVmUgIGRC-acGABZKTNKfDyYA,4776157
-fosslight_util/resources/frequent_license_nick_list.json,sha256=ryU2C_6ZxHbz90_sUN9OvI9GXkCMLu7oGcmd9W79YYo,5005
-fosslight_util/resources/licenses.json,sha256=mK55z-bhY7Mjpj2KsO1crKGGL-X3F6MBFQJ0zLlx010,240843
-fosslight_util-1.4.47.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-fosslight_util-1.4.47.dist-info/METADATA,sha256=za9smj8HiWDYAHLX5Oe3zcO1Fag9GxVGtteVGRhs5EY,6418
-fosslight_util-1.4.47.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-fosslight_util-1.4.47.dist-info/entry_points.txt,sha256=bzXX5i7HZ13V8BLKvtu_9KO3ZjtRypH-XszOXT6I3bU,69
-fosslight_util-1.4.47.dist-info/top_level.txt,sha256=2qyYWGLakgBRy4BqoBNt-I5C29tBr_e93e5e1pbuTGA,15
-fosslight_util-1.4.47.dist-info/RECORD,,