fosslight-dependency 3.0.7__py3-none-any.whl → 4.1.30__py3-none-any.whl

fosslight_dependency/package_manager/Cocoapods.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import json
+import yaml
+import re
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+_spec_repos = 'SPEC REPOS'
+_external_sources = 'EXTERNAL SOURCES'
+_dependencies = 'DEPENDENCIES'
+_source_type = ['git', 'http', 'svn', 'hg']
+
+
+class Cocoapods(PackageManager):
+    package_manager_name = const.COCOAPODS
+
+    dn_url = 'https://cocoapods.org/'
+    input_file_name = const.SUPPORT_PACKAE.get(package_manager_name)
+
+    def __init__(self, input_dir, output_dir):
+        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+        self.append_input_package_list_file(self.input_file_name)
+
+    def parse_oss_information(self, f_name):
+        with open(f_name, 'r', encoding='utf8') as input_fp:
+            podfile_yaml = yaml.load(input_fp, Loader=yaml.FullLoader)
+
+        spec_repo_list = []
+        external_source_list = []
+
+        if _spec_repos in podfile_yaml:
+            for spec_item_key in podfile_yaml[_spec_repos]:
+                for spec_item in podfile_yaml[_spec_repos][spec_item_key]:
+                    spec_repo_list.append(spec_item)
+        if _external_sources in podfile_yaml:
+            for external_sources_key in podfile_yaml[_external_sources]:
+                external_source_list.append(external_sources_key)
+                spec_repo_list.append(external_sources_key)
+        if len(spec_repo_list) == 0:
+            logger.error("Cannot find SPEC REPOS or EXTERNAL SOURCES in Podfile.lock.")
+            return ''
+
+        for dep_key in podfile_yaml[_dependencies]:
+            dep_key_re = re.findall(r'(^\S*)', dep_key)
+            self.direct_dep_list.append(dep_key_re[0])
+
+        pod_item_list = {}
+        for pods_i in podfile_yaml['PODS']:
+            if not isinstance(pods_i, str):
+                for key, items in pods_i.items():
+                    k_name, k_ver = get_pods_info(key)
+                    pod_item_list[k_name] = k_ver
+                    self.relation_tree[f'{k_name}({k_ver})'] = []
+                    for item in items:
+                        i_name, _ = get_pods_info(item)
+                        self.relation_tree[f'{k_name}({k_ver})'].append(i_name)
+            else:
+                oss_name, oss_version = get_pods_info(pods_i)
+                pod_item_list[oss_name] = oss_version
+
+        for rel_key in self.relation_tree:
+            try:
+                tmp_item_list = []
+                for ri in self.relation_tree[rel_key]:
+                    ri_version = pod_item_list[ri]
+                    tmp_item_list.append(f'{ri}({ri_version})')
+                self.relation_tree[rel_key] = []
+                self.relation_tree[rel_key].extend(tmp_item_list)
+            except Exception as e:
+                logger.warning(f'Fail to check packages of {rel_key}: {e}')
+                if rel_key in self.relation_tree:
+                    self.relation_tree[rel_key] = []
+
+        purl_dict = {}
+        for pod_oss_name_origin, pod_oss_version in pod_item_list.items():
+            dep_item = DependencyItem()
+            oss_item = OssItem()
+            try:
+                if self.direct_dep and (len(self.direct_dep_list) > 0):
+                    if pod_oss_name_origin in self.direct_dep_list:
+                        oss_item.comment = 'direct'
+                    else:
+                        oss_item.comment = 'transitive'
+                    if f'{pod_oss_name_origin}({pod_oss_version})' in self.relation_tree:
+                        dep_item.depends_on_raw = self.relation_tree[f'{pod_oss_name_origin}({pod_oss_version})']
+
+                oss_item.name = f'{self.package_manager_name}:{pod_oss_name_origin}'
+                pod_oss_name = pod_oss_name_origin
+                oss_item.version = pod_oss_version
+                if '/' in pod_oss_name_origin:
+                    pod_oss_name = pod_oss_name_origin.split('/')[0]
+                if pod_oss_name in external_source_list:
+                    oss_item.name = pod_oss_name_origin
+                    podspec_filename = pod_oss_name + '.podspec.json'
+                    spec_file_path = os.path.join("Pods", "Local Podspecs", podspec_filename)
+                else:
+                    search_oss_name = ""
+                    for alphabet_oss in pod_oss_name:
+                        if not alphabet_oss.isalnum():
+                            search_oss_name += f"\\\\{alphabet_oss}"
+                        else:
+                            search_oss_name += alphabet_oss
+
+                    command = f"pod spec which --regex ^{search_oss_name}$"
+                    spec_which = os.popen(command).readline()
+                    if spec_which.startswith('[!]'):
+                        logger.warning(f"This command({command}) returns an error")
+                        continue
+
+                    file_path = spec_which.rstrip().split(os.path.sep)
+                    if file_path[0] == '':
+                        file_path_without_version = os.path.join(os.sep, *file_path[:-2])
+                    else:
+                        file_path_without_version = os.path.join(*file_path[:-2])
+                    spec_file_path = os.path.join(file_path_without_version, oss_item.version, file_path[-1])
+
+                oss_name, oss_version, oss_item.license, oss_item.download_location, \
+                    oss_item.homepage = self.get_oss_in_podspec(spec_file_path)
+                dep_item.purl = get_url_to_purl(oss_item.homepage, self.package_manager_name, pod_oss_name_origin, oss_version)
+                purl_dict[f'{pod_oss_name_origin}({oss_version})'] = dep_item.purl
+                if pod_oss_name in external_source_list:
+                    oss_item.homepage = ''
+                if oss_name == '':
+                    continue
+                if oss_item.version != oss_version:
+                    logger.warning(f'{pod_oss_name_origin} has different version({oss_item.version})\
+                                   with spec version({oss_version})')
+                dep_item.oss_items.append(oss_item)
+                self.dep_items.append(dep_item)
+            except Exception as e:
+                logger.warning(f"Fail to get {pod_oss_name_origin}:{e}")
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
+
+        return
+
+    def get_oss_in_podspec(self, spec_file_path):
+        oss_name = ''
+        oss_version = ''
+        license_name = ''
+        dn_loc = ''
+        homepage = ''
+        try:
+            with open(spec_file_path, 'r', encoding='utf8') as json_file:
+                json_data = json.load(json_file)
+
+                oss_origin_name = json_data['name']
+                oss_name = f"{self.package_manager_name}:{oss_origin_name}"
+                oss_version = json_data['version']
+                homepage = f"{self.dn_url}pods/{oss_origin_name}"
+
+                if 'license' in json_data:
+                    if not isinstance(json_data['license'], str):
+                        if 'type' in json_data['license']:
+                            license_name = json_data['license']['type']
+                    else:
+                        license_name = json_data['license']
+                else:
+                    license_name = ''
+                license_name = license_name.replace(",", "")
+
+                source_keys = [key for key in json_data['source']]
+                for src_type_i in _source_type:
+                    if src_type_i in source_keys:
+                        dn_loc = json_data['source'][src_type_i]
+                        if dn_loc.endswith('.git'):
+                            dn_loc = dn_loc[:-4]
+        except Exception as e:
+            logger.warning(f"Fail to get oss info in podspec:{e}")
+
+        return oss_name, oss_version, license_name, dn_loc, homepage
+
+    def parse_direct_dependencies(self):
+        self.direct_dep = True
+
+
+def get_pods_info(pods_item):
+    pods_item_re = re.findall(r'(\S*)(?:\s{1}\((.*)\))?', pods_item)
+
+    oss_name = pods_item_re[0][0]
+    oss_version = pods_item_re[0][1]
+
+    return oss_name, oss_version

fosslight_dependency/package_manager/Go.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import subprocess
+import json
+from bs4 import BeautifulSoup
+import urllib.request
+import re
+import shutil
+import time
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+class Go(PackageManager):
+    package_manager_name = const.GO
+
+    input_file_name = ''
+    is_run_plugin = False
+    dn_url = 'https://pkg.go.dev/'
+    tmp_file_name = 'tmp_go_list.json'
+    go_work = 'go.work'
+    tmp_go_work = 'go.work.tmp'
+
+    def __init__(self, input_dir, output_dir):
+        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+        self.input_file_name = ''
+        self.is_run_plugin = False
+
+    def __del__(self):
+        if os.path.isfile(self.tmp_file_name):
+            os.remove(self.tmp_file_name)
+        if os.path.isfile(self.tmp_go_work):
+            shutil.move(self.tmp_go_work, self.go_work)
+
+    def parse_dependency_tree(self, go_deptree_txt):
+        for line in go_deptree_txt.split('\n'):
+            re_result = re.findall(r'(\S+)@v(\S+)\s(\S+)@v(\S+)', line)
+            if len(re_result) > 0 and len(re_result[0]) >= 4:
+                oss_name = re_result[0][0]
+                oss_ver = re_result[0][1]
+                pkg_name = re_result[0][2]
+                pkg_ver = re_result[0][3]
+                if f'{oss_name}({oss_ver})' not in self.relation_tree:
+                    self.relation_tree[f'{oss_name}({oss_ver})'] = []
+                self.relation_tree[f'{oss_name}({oss_ver})'].append(f'{pkg_name}({pkg_ver})')
+
+    def run_plugin(self):
+        ret = True
+
+        if os.path.isfile(self.go_work):
+            shutil.move(self.go_work, self.tmp_go_work)
+
+        logger.info("Execute 'go list -m -mod=mod -json all' to obtain package info.")
+        cmd = f"go list -m -mod=mod -json all > {self.tmp_file_name}"
+
+        ret_cmd = subprocess.call(cmd, shell=True)
+        if ret_cmd != 0:
+            logger.error(f"Failed to make the result: {cmd}")
+            ret = False
+
+        self.append_input_package_list_file(self.tmp_file_name)
+
+        cmd_tree = "go mod graph"
+        ret_cmd_tree = subprocess.check_output(cmd_tree, shell=True, text=True, encoding='utf-8')
+        if ret_cmd_tree != 0:
+            self.parse_dependency_tree(ret_cmd_tree)
+
+        if os.path.isfile(self.tmp_go_work):
+            shutil.move(self.tmp_go_work, self.go_work)
+        return ret
+
+    def parse_oss_information(self, f_name):
+        indirect = 'Indirect'
+        purl_dict = {}
+        json_list = []
+        with open(f_name, 'r', encoding='utf8') as input_fp:
+            json_data_raw = ''
+            for line in input_fp.readlines():
+                json_data_raw += line
+                if line.startswith('}'):
+                    json_list.append(json.loads(json_data_raw))
+                    json_data_raw = ''
+
+        for dep_i in json_list:
+            dep_item = DependencyItem()
+            oss_item = OssItem()
+            try:
+                if 'Main' in dep_i:
+                    if dep_i['Main']:
+                        continue
+                package_path = dep_i['Path']
+                oss_item.name = f"{self.package_manager_name}:{package_path}"
+                oss_origin_version = dep_i['Version']
+                if oss_origin_version.startswith('v'):
+                    oss_item.version = oss_origin_version[1:]
+                else:
+                    oss_item.version = oss_origin_version
+
+                if self.direct_dep:
+                    if indirect in dep_i:
+                        if dep_i[indirect]:
+                            oss_item.comment = 'transitive'
+                        else:
+                            oss_item.comment = 'direct'
+                    else:
+                        oss_item.comment = 'direct'
+
+                if f'{package_path}({oss_item.version})' in self.relation_tree:
+                    dep_item.depends_on_raw = self.relation_tree[f'{package_path}({oss_item.version})']
+
+                dn_loc_set = []
+                tmp_dn_loc = self.dn_url + package_path
+                dep_item.purl = get_url_to_purl(f"{tmp_dn_loc}@{oss_item.version}", self.package_manager_name)
+                purl_dict[f'{package_path}({oss_item.version})'] = dep_item.purl
+
+                if oss_origin_version:
+                    oss_item.download_location = f"{tmp_dn_loc}@{oss_origin_version}"
+                    dn_loc_set.append(oss_item.download_location)
+                dn_loc_set.append(tmp_dn_loc)
+
+                for dn_loc_i in dn_loc_set:
+                    urlopen_success = False
+                    while True:
+                        try:
+                            res = urllib.request.urlopen(dn_loc_i)
+                            if res.getcode() == 200:
+                                urlopen_success = True
+                                if dn_loc_i == tmp_dn_loc:
+                                    if oss_item.version:
+                                        oss_item.comment = (f'Not found {oss_item.download_location}, '
+                                                            'get info from latest version.')
+                                        oss_item.download_location = tmp_dn_loc
+                                break
+                        except urllib.error.HTTPError as e:
+                            if e.code == 429:
+                                logger.info(f"{e} ({dn_loc_i}), Retrying to connect after 20 seconds")
+                                time.sleep(20)
+                                continue
+                            else:
+                                logger.info(f"{e} ({dn_loc_i})")
+                                break
+                        except Exception as e:
+                            logger.warning(f"{e} ({dn_loc_i})")
+                            break
+                    if urlopen_success:
+                        break
+
+                if urlopen_success:
+                    content = res.read().decode('utf8')
+                    bs_obj = BeautifulSoup(content, 'html.parser')
+
+                    license_data = bs_obj.find('a', {'data-test-id': 'UnitHeader-license'})
+                    if license_data:
+                        oss_item.license = license_data.text
+
+                    repository_data = bs_obj.find('div', {'class': 'UnitMeta-repo'})
+                    if repository_data:
+                        oss_item.homepage = repository_data.find('a')['href']
+                    else:
+                        oss_item.homepage = oss_item.download_location
+
+            except Exception as e:
+                logging.warning(f"Fail to parse {package_path} in go mod : {e}")
+                continue
+            dep_item.oss_items.append(oss_item)
+            self.dep_items.append(dep_item)
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
+        return

fosslight_dependency/package_manager/Gradle.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import json
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager
+from fosslight_dependency._package_manager import version_refine, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.get_pom_license import get_license_from_pom
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+class Gradle(PackageManager):
+    package_manager_name = const.GRADLE
+
+    dn_url = 'https://mvnrepository.com/artifact/'
+    input_file_name = os.path.join('build', 'reports', 'license', 'dependency-license.json')
+
+    def __init__(self, input_dir, output_dir, output_custom_dir):
+        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+
+        if output_custom_dir:
+            self.output_custom_dir = output_custom_dir
+            self.input_file_name = os.path.join(output_custom_dir, os.sep.join(self.input_file_name.split(os.sep)[1:]))
+
+        self.append_input_package_list_file(self.input_file_name)
+
+    def parse_oss_information(self, f_name):
+        with open(f_name, 'r', encoding='utf8') as json_file:
+            json_data = json.load(json_file)
+
+        purl_dict = {}
+
+        for d in json_data['dependencies']:
+            dep_item = DependencyItem()
+            oss_item = OssItem()
+            used_filename = False
+            group_id = ""
+            artifact_id = ""
+
+            name = d['name']
+            filename = d['file']
+
+            if name != filename:
+                group_id, artifact_id, oss_ini_version = parse_oss_name_version_in_artifactid(name)
+                oss_name = f"{group_id}:{artifact_id}"
+            else:
+                oss_name, oss_ini_version = parse_oss_name_version_in_filename(filename)
+                used_filename = True
+            oss_item.name = oss_name
+            dep_key = f"{oss_item.name}({oss_ini_version})"
+            if self.total_dep_list:
+                if dep_key not in self.total_dep_list:
+                    continue
+
+            oss_item.version = version_refine(oss_ini_version)
+
+            try:
+                license_names = []
+                for licenses in d['licenses']:
+                    if licenses['name'] != '':
+                        license_names.append(licenses['name'].replace(",", ""))
+                oss_item.license = ', '.join(license_names)
+            except Exception:
+                logger.info("Cannot find the license name")
+            if not oss_item.license:
+                license_names = get_license_from_pom(group_id, artifact_id, oss_ini_version)
+                if license_names:
+                    oss_item.license = license_names
+
+            if used_filename or group_id == "":
+                oss_item.download_location = 'Unknown'
+            else:
+                oss_item.download_location = f"{self.dn_url}{group_id}/{artifact_id}/{oss_ini_version}"
+                oss_item.homepage = f"{self.dn_url}{group_id}/{artifact_id}"
+                dep_item.purl = get_url_to_purl(oss_item.download_location, 'maven')
+                purl_dict[f'{oss_item.name}({oss_ini_version})'] = dep_item.purl
+
+            if self.direct_dep:
+                if len(self.direct_dep_list) > 0:
+                    if dep_key in self.direct_dep_list:
+                        oss_item.comment = 'direct'
+                    else:
+                        oss_item.comment = 'transitive'
+                try:
+                    if dep_key in self.relation_tree:
+                        dep_item.depends_on_raw = self.relation_tree[dep_key]
+                except Exception as e:
+                    logger.error(f"Fail to find oss scope in dependency tree: {e}")
+
+            dep_item.oss_items.append(oss_item)
+            self.dep_items.append(dep_item)
+
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
+        return
+
+
+def parse_oss_name_version_in_filename(name):
+    filename = name.rstrip('.jar')
+    split_name = filename.rpartition('-')
+
+    oss_name = split_name[0]
+    oss_version = split_name[2]
+
+    return oss_name, oss_version
+
+
+def parse_oss_name_version_in_artifactid(name):
+    artifact_comp = name.split(':')
+
+    group_id = artifact_comp[0]
+    artifact_id = artifact_comp[1]
+    oss_version = artifact_comp[2]
+
+    return group_id, artifact_id, oss_version

fosslight_dependency/package_manager/Helm.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2023 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import subprocess
+import yaml
+import shutil
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
+from fosslight_util.download import extract_compressed_dir
+from fosslight_dependency.dependency_item import DependencyItem
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+class Helm(PackageManager):
+    package_manager_name = const.HELM
+    tmp_charts_dir = 'tmp_charts'
+
+    input_file_name = const.SUPPORT_PACKAE.get(package_manager_name)
+
+    def __init__(self, input_dir, output_dir):
+        super().__init__(self.package_manager_name, '', input_dir, output_dir)
+        self.append_input_package_list_file(self.input_file_name)
+
+    def __del__(self):
+        if os.path.exists(self.tmp_charts_dir):
+            shutil.rmtree(self.tmp_charts_dir, ignore_errors=True)
+
+    def run_plugin(self):
+        ret = True
+        charts_dir = 'charts'
+        if os.path.isdir(charts_dir):
+            shutil.copytree(charts_dir, self.tmp_charts_dir)
+        else:
+            logger.info("Execute 'helm dependency build' to obtain package info.")
+            cmd = "helm dependency build"
+
+            ret_cmd = subprocess.call(cmd, shell=True)
+            if ret_cmd != 0:
+                logger.error(f"Failed to build helm dependency: {cmd}")
+                ret = False
+            else:
+                if not os.path.isdir(charts_dir):
+                    logger.warning(f"Cannot create {charts_dir} because of no dependencies in Chart.yaml. "
+                                   f"So you don't need to analyze dependency.")
+                    return True
+                else:
+                    shutil.copytree(charts_dir, self.tmp_charts_dir)
+                    shutil.rmtree(charts_dir, ignore_errors=True)
+        if ret:
+            ret = extract_compressed_dir(self.tmp_charts_dir, self.tmp_charts_dir, False)
+            if not ret:
+                logger.error(f'Fail to extract compressed dir: {self.tmp_charts_dir}')
+            else:
+                logger.warning('Success to extract compressed dir')
+
+        return ret
+
+    def parse_oss_information(self, f_name):
+        dep_item_list = []
+        _dependencies = 'dependencies'
+
+        with open(f_name, 'r', encoding='utf8') as yaml_fp:
+            yaml_f = yaml.safe_load(yaml_fp)
+            if _dependencies in yaml_f:
+                for dep in yaml_f[_dependencies]:
+                    dep_item_list.append(dep['name'])
+        for dep in dep_item_list:
+            try:
+                f_path = os.path.join(self.tmp_charts_dir, dep, f_name)
+                dep_item = DependencyItem()
+                oss_item = OssItem()
+                with open(f_path, 'r', encoding='utf8') as yaml_fp:
+                    yaml_f = yaml.safe_load(yaml_fp)
+                    oss_item.name = f'{self.package_manager_name}:{yaml_f["name"]}'
+                    oss_item.version = yaml_f.get('version', '')
+                    if oss_item.version.startswith('v'):
+                        oss_item.version = oss_item.version[1:]
+
+                    oss_item.homepage = yaml_f.get('home', '')
+                    if yaml_f.get('sources', '') != '':
+                        oss_item.download_location = yaml_f.get('sources', '')[0]
+
+                    dep_item.purl = get_url_to_purl(
+                        oss_item.download_location if oss_item.download_location else oss_item.homepage,
+                        self.package_manager_name
+                    )
+
+                    if yaml_f.get('annotations', '') != '':
+                        oss_item.license = yaml_f['annotations'].get('licenses', '')
+
+                    if self.direct_dep:
+                        oss_item.comment = 'direct'
+
+            except Exception as e:
+                logging.warning(f"Fail to parse chart info {dep}: {e}")
+                continue
+            dep_item.oss_items.append(oss_item)
+            self.dep_items.append(dep_item)
+        return