fosslight-dependency 3.0.7__py3-none-any.whl → 4.1.30__py3-none-any.whl

fosslight_dependency/package_manager/Maven.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import subprocess
+import shutil
+from bs4 import BeautifulSoup as bs
+from defusedxml.ElementTree import parse
+import re
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager
+from fosslight_dependency._package_manager import version_refine, get_url_to_purl, change_file_mode
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.get_pom_license import get_license_from_pom
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+
+
+class Maven(PackageManager):
+    package_manager_name = const.MAVEN
+
+    dn_url = 'https://mvnrepository.com/artifact/'
+    input_file_name = os.path.join('target', 'generated-resources', 'licenses.xml')
+    is_run_plugin = False
+    output_custom_dir = ''
+
+    def __init__(self, input_dir, output_dir, output_custom_dir):
+        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+        self.is_run_plugin = False
+
+        if output_custom_dir:
+            self.output_custom_dir = output_custom_dir
+            self.input_file_name = os.path.join(output_custom_dir, os.sep.join(self.input_file_name.split(os.sep)[1:]))
+
+        self.append_input_package_list_file(self.input_file_name)
+
+    def __del__(self):
+        if self.is_run_plugin:
+            self.clean_run_maven_plugin_output()
+
+    def run_plugin(self):
+        ret = True
+
+        if not os.path.isfile(self.input_file_name):
+            pom_backup = 'pom.xml_backup'
+
+            ret = self.add_plugin_in_pom(pom_backup)
+            if ret:
+                ret_plugin = self.run_maven_plugin()
+                if ret_plugin:
+                    self.is_run_plugin = True
+
+            if os.path.isfile(pom_backup):
+                shutil.move(pom_backup, const.SUPPORT_PACKAE.get(self.package_manager_name))
+        else:
+            self.set_direct_dependencies(False)
+
+        return ret
+
+    def add_plugin_in_pom(self, pom_backup):
+        ret = False
+        xml = 'xml'
+
+        manifest_file = const.SUPPORT_PACKAE.get(self.package_manager_name)
+        if os.path.isfile(manifest_file) != 1:
+            logger.error(f"{manifest_file} is not existed in this directory.")
+            return ret
+
+        try:
+            shutil.move(manifest_file, pom_backup)
+
+            license_maven_plugin = '<plugin>\
+                                            <groupId>org.codehaus.mojo</groupId>\
+                                            <artifactId>license-maven-plugin</artifactId>\
+                                            <version>2.0.0</version>\
+                                            <executions>\
+                                                <execution>\
+                                                    <id>aggregate-download-licenses</id>\
+                                                    <goals>\
+                                                        <goal>aggregate-download-licenses</goal>\
+                                                    </goals>\
+                                                </execution>\
+                                            </executions>\
+                                            <configuration>\
+                                                <excludedScopes>test</excludedScopes>\
+                                            </configuration>\
+                                        </plugin>'
+
+            tmp_plugin = bs(license_maven_plugin, xml)
+
+            license_maven_plugins = f"<plugins>{license_maven_plugin}<plugins>"
+            tmp_plugins = bs(license_maven_plugins, xml)
+
+            with open(pom_backup, 'r', encoding='utf8') as f:
+                f_xml = f.read()
+                f_content = bs(f_xml, xml)
+
+                build = f_content.find('build')
+                if build is not None:
+                    plugins = build.find('plugins')
+                    if plugins is not None:
+                        plugins.append(tmp_plugin.plugin)
+                        ret = True
+                    else:
+                        build.append(tmp_plugins.plugins)
+                        ret = True
+        except Exception as e:
+            ret = False
+            logger.error(f"Failed to add plugin in pom : {e}")
+
+        if ret:
+            with open(manifest_file, "w", encoding='utf8') as f_w:
+                f_w.write(f_content.prettify(formatter="minimal").encode().decode('utf-8'))
+
+        return ret
+
+    def clean_run_maven_plugin_output(self):
+        directory_name = os.path.dirname(self.input_file_name)
+        licenses_path = os.path.join(directory_name, 'licenses')
+        if os.path.isdir(directory_name):
+            if os.path.isdir(licenses_path):
+                shutil.rmtree(licenses_path)
+                os.remove(self.input_file_name)
+
+            if len(os.listdir(directory_name)) == 0:
+                shutil.rmtree(directory_name)
+
+        top_path = self.input_file_name.split(os.sep)[0]
+        if len(os.listdir(top_path)) == 0:
+            shutil.rmtree(top_path)
+
+    def run_maven_plugin(self):
+        ret_plugin = True
+        logger.info('Run maven license scanning plugin with temporary pom.xml')
+        current_mode = ''
+        if os.path.isfile('mvnw') or os.path.isfile('mvnw.cmd'):
+            if self.platform == const.WINDOWS:
+                cmd_mvn = "mvnw.cmd"
+            else:
+                cmd_mvn = "./mvnw"
+            current_mode = change_file_mode(cmd_mvn)
+        else:
+            cmd_mvn = "mvn"
+        cmd = f"{cmd_mvn} license:aggregate-download-licenses"
+
+        ret = subprocess.call(cmd, shell=True)
+        if ret != 0:
+            logger.error(f"Failed to run maven plugin: {cmd}")
+            ret_plugin = False
+
+        if ret_plugin:
+            cmd = f"{cmd_mvn} dependency:tree"
+            try:
+                ret_txt = subprocess.check_output(cmd, text=True, shell=True)
+                if ret_txt is not None:
+                    self.parse_dependency_tree(ret_txt)
+                    self.set_direct_dependencies(True)
+                else:
+                    logger.error(f"Failed to run: {cmd}")
+                    self.set_direct_dependencies(False)
+            except Exception as e:
+                logger.error(f"Failed to run '{cmd}': {e}")
+                self.set_direct_dependencies(False)
+        if current_mode:
+            change_file_mode(cmd_mvn, current_mode)
+        return ret_plugin
+
+    def create_dep_stack(self, dep_line):
+        dep_stack = []
+        cur_flag = ''
+        dep_level = -1
+        dep_level_plus = False
+        for line in dep_line.split('\n'):
+            try:
+                if not re.search(r'[.*INFO.*]', line):
+                    continue
+                if len(line) <= 7:
+                    continue
+                line = line[7:]
+
+                prev_flag = cur_flag
+                prev_dep_level = dep_level
+                dep_level = line.count("|")
+
+                re_result = re.findall(r'([\+|\\]\-)\s([^\:\s]+\:[^\:\s]+)\:(?:[^\:\s]+)\:([^\:\s]+)\:([^\:\s]+)', line)
+                if re_result:
+                    cur_flag = re_result[0][0]
+                    if (prev_flag == '\\-') and (prev_dep_level == dep_level):
+                        dep_level_plus = True
+                    if dep_level_plus and (prev_flag == '\\-') and (prev_dep_level != dep_level):
+                        dep_level_plus = False
+                    if dep_level_plus:
+                        dep_level += 1
+                    if re_result[0][3] == 'test':
+                        continue
+                    dep_name = f'{re_result[0][1]}({re_result[0][2]})'
+                    dep_stack = dep_stack[:dep_level] + [dep_name]
+                    yield dep_stack[:dep_level], dep_name
+                else:
+                    cur_flag = ''
+            except Exception as e:
+                logger.warning(f"Failed to parse dependency tree: {e}")
+
+    def parse_dependency_tree(self, f_name):
+        try:
+            for stack, name in self.create_dep_stack(f_name):
+                if len(stack) == 0:
+                    self.direct_dep_list.append(name)
+                else:
+                    if stack[-1] not in self.relation_tree:
+                        self.relation_tree[stack[-1]] = []
+                    self.relation_tree[stack[-1]].append(name)
+        except Exception as e:
+            logger.warning(f'Fail to parse maven dependency tree:{e}')
+
+    def parse_oss_information(self, f_name):
+        with open(f_name, 'r', encoding='utf8') as input_fp:
+            tree = parse(input_fp)
+
+        root = tree.getroot()
+        dependencies = root.find("dependencies")
+        purl_dict = {}
+
+        for d in dependencies.iter("dependency"):
+            dep_item = DependencyItem()
+            oss_item = OssItem()
+            groupid = d.findtext("groupId")
+            artifactid = d.findtext("artifactId")
+            version = d.findtext("version")
+            oss_item.version = version_refine(version)
+
+            oss_item.name = f"{groupid}:{artifactid}"
+            oss_item.download_location = f"{self.dn_url}{groupid}/{artifactid}/{version}"
+            oss_item.homepage = f"{self.dn_url}{groupid}/{artifactid}"
+            dep_item.purl = get_url_to_purl(oss_item.download_location, self.package_manager_name)
+            purl_dict[f'{oss_item.name}({oss_item.version})'] = dep_item.purl
+
+            licenses = d.find("licenses")
+            if len(licenses):
+                license_names = []
+                for key_license in licenses.iter("license"):
+                    if key_license.findtext("name") is not None:
+                        license_names.append(key_license.findtext("name").replace(",", ""))
+                oss_item.license = ', '.join(license_names)
+            if not oss_item.license:
+                license_names = get_license_from_pom(groupid, artifactid, version)
+                if license_names:
+                    oss_item.license = license_names
+
+            dep_key = f"{oss_item.name}({version})"
+
+            if self.direct_dep:
+                if len(self.direct_dep_list) > 0:
+                    if dep_key in self.direct_dep_list:
+                        oss_item.comment = 'direct'
+                    else:
+                        oss_item.comment = 'transitive'
+                try:
+                    if dep_key in self.relation_tree:
+                        dep_item.depends_on_raw = self.relation_tree[dep_key]
+                except Exception as e:
+                    logger.error(f"Fail to find oss scope in dependency tree: {e}")
+
+            dep_item.oss_items.append(oss_item)
+            self.dep_items.append(dep_item)
+
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
+        return

fosslight_dependency/package_manager/Npm.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+import subprocess
+import json
+import shutil
+import re
+import requests
+import fosslight_util.constant as constant
+import fosslight_dependency.constant as const
+from fosslight_dependency._package_manager import PackageManager, get_url_to_purl
+from fosslight_dependency.dependency_item import DependencyItem, change_dependson_to_purl
+from fosslight_util.oss_item import OssItem
+
+logger = logging.getLogger(constant.LOGGER_NAME)
+node_modules = 'node_modules'
+
+
+class Npm(PackageManager):
+    package_manager_name = const.NPM
+
+    dn_url = 'https://www.npmjs.com/package/'
+    input_file_name = 'tmp_npm_license_output.json'
+    tmp_custom_json = 'custom.json'
+    flag_tmp_node_modules = False
+    _network_available = False
+
+    def __init__(self, input_dir, output_dir):
+        super().__init__(self.package_manager_name, self.dn_url, input_dir, output_dir)
+        self._check_network_available()
+
+    def __del__(self):
+        if os.path.isfile(os.path.join(self.input_dir, self.input_file_name)):
+            os.remove(os.path.join(self.input_dir, self.input_file_name))
+        if self.flag_tmp_node_modules:
+            shutil.rmtree(os.path.join(self.input_dir, node_modules), ignore_errors=True)
+        if os.path.exists(os.path.join(self.input_dir, self.tmp_custom_json)):
+            os.remove(os.path.join(self.input_dir, self.tmp_custom_json))
+
+    def run_plugin(self):
+        ret = self.start_license_checker()
+        return ret
+
+    def start_license_checker(self):
+        ret = True
+        license_checker_cmd = f'license-checker --production --json --out {self.input_file_name}'
+        custom_path_option = ' --customPath '
+        npm_install_cmd = 'npm install --production --ignore-scripts'
+
+        if os.path.isdir(node_modules) != 1:
+            logger.info(f"node_modules directory is not existed. So it executes '{npm_install_cmd}'.")
+            self.flag_tmp_node_modules = True
+            cmd_ret = subprocess.call(npm_install_cmd, shell=True)
+            if cmd_ret != 0:
+                logger.error(f"{npm_install_cmd} failed")
+                return False
+
+        # customized json file for obtaining specific items with license-checker
+        self.make_custom_json(self.tmp_custom_json)
+
+        cmd = license_checker_cmd + custom_path_option + self.tmp_custom_json
+        cmd_ret = subprocess.call(cmd, shell=True)
+        if cmd_ret != 0:
+            logger.error(f"It returns the error: {cmd}")
+            logger.error("Please check if the license-checker is installed.(sudo npm install -g license-checker)")
+            ret = False
+        else:
+            self.append_input_package_list_file(self.input_file_name)
+        if os.path.exists(self.tmp_custom_json):
+            os.remove(self.tmp_custom_json)
+
+        return ret
+
+    def make_custom_json(self, tmp_custom_json):
+        with open(tmp_custom_json, 'w', encoding='utf8') as custom:
+            custom.write(
+                "{\n\t\"name\": \"\",\n\t\"version\": \"\",\n\t\"licenses\": \"\",\n\t\"repository\": \
+                \"\",\n\t\"url\": \"\",\n\t\"copyright\": \"\",\n\t\"licenseText\": \"\"\n}\n".encode().decode("utf-8"))
+
+    def parse_rel_dependencies(self, rel_name, rel_ver, rel_dependencies):
+        _dependencies = 'dependencies'
+        _version = 'version'
+        _peer = 'peerMissing'
+
+        for rel_dep_name in rel_dependencies.keys():
+            # Optional, non-installed dependencies are listed as empty objects
+            if rel_dependencies[rel_dep_name] == {}:
+                continue
+            if _peer in rel_dependencies[rel_dep_name]:
+                if rel_dependencies[rel_dep_name][_peer]:
+                    continue
+            if f'{rel_name}({rel_ver})' not in self.relation_tree:
+                self.relation_tree[f'{rel_name}({rel_ver})'] = []
+            elif f'{rel_dep_name}({rel_dependencies[rel_dep_name][_version]})' in self.relation_tree[f'{rel_name}({rel_ver})']:
+                continue
+            self.relation_tree[f'{rel_name}({rel_ver})'].append(f'{rel_dep_name}({rel_dependencies[rel_dep_name][_version]})')
+            if _dependencies in rel_dependencies[rel_dep_name]:
+                self.parse_rel_dependencies(rel_dep_name, rel_dependencies[rel_dep_name][_version],
+                                            rel_dependencies[rel_dep_name][_dependencies])
+
+    def parse_transitive_relationship(self):
+        _dependencies = 'dependencies'
+        _version = 'version'
+        _name = 'name'
+        ret = True
+        err_msg = ''
+
+        cmd = 'npm ls -a --omit=dev --json -s'
+        result = subprocess.run(cmd, shell=True, capture_output=True, text=True, encoding='utf-8')
+        rel_tree = result.stdout
+        if not rel_tree or rel_tree.strip() == '':
+            logger.error(f"No output for {cmd}, stderr: {result.stderr}")
+            ret = False
+        elif result.returncode > 1:
+            logger.error(f"'{cmd}' failed with exit code({result.returncode}), stderr: {result.stderr}")
+            ret = False
+        if ret:
+            if result.returncode == 1:
+                logger.debug(f"'{cmd}' has warnings: {result.stderr}")
+
+            try:
+                rel_json = json.loads(rel_tree)
+                if len(rel_json) < 1:
+                    ret = False
+                else:
+                    self.package_name = f'{rel_json[_name]}({rel_json.get(_version, "")})'
+                    if _dependencies in rel_json:
+                        self.parse_rel_dependencies(rel_json[_name], rel_json.get(_version, ""), rel_json[_dependencies])
+            except Exception as e:
+                ret = False
+                err_msg = e
+        return ret, err_msg
+
+    def parse_direct_dependencies(self):
+        if not self.direct_dep:
+            return
+        try:
+            if os.path.isfile(const.SUPPORT_PACKAE.get(self.package_manager_name)):
+                ret, err_msg = self.parse_transitive_relationship()
+                if not ret:
+                    self.direct_dep = False
+                    logger.warning(f'It cannot print direct/transitive dependency: {err_msg}')
+            else:
+                logger.info('Direct/transitive support is not possible because the package.json file does not exist.')
+                self.direct_dep = False
+        except Exception as e:
+            logger.warning(f'Cannot print direct/transitive dependency: {e}')
+            self.direct_dep = False
+
+    def parse_oss_information(self, f_name):
+        with open(f_name, 'r', encoding='utf8') as json_file:
+            json_data = json.load(json_file)
+
+        _licenses = 'licenses'
+        _repository = 'repository'
+        _private = 'private'
+
+        keys = [key for key in json_data]
+        purl_dict = {}
+        for i in range(0, len(keys)):
+            dep_item = DependencyItem()
+            oss_item = OssItem()
+            d = json_data.get(keys[i - 1])
+            oss_init_name = d['name']
+            oss_item.name = self.package_manager_name + ":" + oss_init_name
+
+            if d[_licenses]:
+                license_name = d[_licenses]
+            else:
+                license_name = ''
+
+            oss_item.version = d['version']
+            package_path = d['path']
+
+            private_pkg = False
+            if _private in d and d[_private]:
+                private_pkg = True
+
+            oss_item.download_location = ''
+
+            npm_dl_url = f"{self.dn_url}{oss_init_name}/v/{oss_item.version}"
+            npm_home_url = f"{self.dn_url}{oss_init_name}"
+            dep_item.purl = get_url_to_purl(npm_dl_url, self.package_manager_name)
+            purl_dict[f'{oss_init_name}({oss_item.version})'] = dep_item.purl
+
+            repo_url = d[_repository] if d[_repository] else ''
+            if private_pkg:
+                oss_item.homepage = repo_url or ''
+                oss_item.download_location = oss_item.homepage
+                oss_item.comment = 'private'
+            else:
+                npm_url_exists = False
+                if self._network_available is True:
+                    npm_url_exists = self._npm_url_exists(oss_init_name)
+
+                if self._network_available and not npm_url_exists:
+                    oss_item.homepage = repo_url or ""
+                    oss_item.download_location = oss_item.homepage
+                else:
+                    oss_item.homepage = repo_url or npm_home_url
+                    oss_item.download_location = npm_dl_url
+
+            if self.package_name == f'{oss_init_name}({oss_item.version})':
+                oss_item.comment = 'root package'
+            elif self.direct_dep and len(self.relation_tree) > 0:
+                if f'{oss_init_name}({oss_item.version})' in self.relation_tree[self.package_name]:
+                    oss_item.comment = 'direct'
+                else:
+                    oss_item.comment = 'transitive'
+
+                if f'{oss_init_name}({oss_item.version})' in self.relation_tree:
+                    dep_item.depends_on_raw = self.relation_tree[f'{oss_init_name}({oss_item.version})']
+
+            manifest_file_path = os.path.join(package_path, const.SUPPORT_PACKAE.get(self.package_manager_name))
+            multi_license, license_comment, multi_flag = check_multi_license(license_name, manifest_file_path)
+
+            if multi_flag:
+                oss_item.comment = license_comment
+                license_name = multi_license
+            else:
+                license_name = license_name.replace(",", "")
+                license_name = check_unknown_license(license_name, manifest_file_path)
+            oss_item.license = license_name
+
+            dep_item.oss_items.append(oss_item)
+            self.dep_items.append(dep_item)
+
+        if self.direct_dep:
+            self.dep_items = change_dependson_to_purl(purl_dict, self.dep_items)
+        return
+
+    def _check_network_available(self) -> bool:
+        test_url = 'https://registry.npmjs.org/'
+        try:
+            resp = requests.head(test_url, timeout=3, allow_redirects=True)
+            self._network_available = resp.status_code < 400
+        except Exception:
+            self._network_available = False
+        return self._network_available
+
+    def _npm_url_exists(self, package_name: str) -> bool:
+        url = f"https://registry.npmjs.org/{package_name}"
+        try:
+            resp = requests.head(url, timeout=3, allow_redirects=True)
+            if resp.status_code == 405:
+                resp = requests.get(url, timeout=3, allow_redirects=True)
+            return resp.status_code < 400
+        except Exception:
+            return False
+
+
+def check_multi_license(license_name, manifest_file_path):
+    multi_license_list = []
+    multi_license = ''
+    license_comment = ''
+    multi_flag = False
+    try:
+        if isinstance(license_name, list):
+            for i in range(0, len(license_name)):
+                l_i = license_name[i].replace(",", "")
+                multi_license_list.append(check_unknown_license(l_i, manifest_file_path))
+            multi_license = ','.join(multi_license_list)
+            multi_flag = True
+        else:
+            if license_name.startswith('(') and license_name.endswith(')'):
+                license_name = license_name.lstrip('(').rstrip(')')
+                license_comment = license_name
+                multi_license = ','.join(re.split(r'OR|AND', license_name))
+                multi_flag = True
+    except Exception as e:
+        multi_license = license_name
+        logger.warning(f'Fail to parse multi license in npm: {e}')
+
+    return multi_license, license_comment, multi_flag
+
+
+def check_unknown_license(license_name, manifest_file_path):
+    if license_name.endswith('*'):
+        license_name = license_name[:-1]
+
+    if license_name == 'UNKNOWN':
+        try:
+            with open(manifest_file_path, 'r') as f:
+                json_f = json.load(f)
+                for key in json_f.keys():
+                    if key == 'license':
+                        license_name = json_f[key]
+                        break
+        except Exception as e:
+            logging.warning(f"Cannot check unknown license: {e}")
+
+    return license_name