shellfoundry 1.2.25__py3-none-any.whl

shellfoundry/utilities/modifiers/definition/definition_modification.py

@@ -0,0 +1,205 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+from io import open
+
+import ruamel.yaml as yaml
+
+from shellfoundry.exceptions import YmlFieldMissingException
+from shellfoundry.utilities.constants import (
+    TEMPLATE_PROPERTY,
+    TEMPLATE_VERSION,
+    TOSCA_META_LOCATION,
+)
+
+
+class DefinitionModification(object):
+    def __init__(self, shell_path):
+        self.shell_path = shell_path
+        self.entry_definition = os.path.join(
+            self.shell_path, self._find_entry_definition()
+        )
+
+    def edit_definition(self, field, value):
+        """Modify shell-definition.yaml.
+
+        :params field str: field name to modify
+        :params value str: new value to update
+        """
+        self._edit_yaml(self.entry_definition, field, value)
+
+    def edit_tosca_meta(self, field, value):
+        with open(
+            os.path.join(self.shell_path, TOSCA_META_LOCATION), "r", encoding="utf8"
+        ) as tosca_file:
+            is_changed = False
+            tosca_data = []
+            for line in tosca_file:
+                if field in line:
+                    line = re.sub(r":\s+.*", ": {}".format(value), line)
+                    is_changed = True
+                tosca_data.append(line)
+
+        if not is_changed:
+            tosca_data.append("\n{field}: {value}".format(field=field, value=value))
+
+        with open(
+            os.path.join(self.shell_path, TOSCA_META_LOCATION), "w", encoding="utf8"
+        ) as tosca_file:
+            tosca_file.writelines(tosca_data)
+
+    def add_field_to_definition(self, field, value=None, overwrite=False):
+        """Add new field to shell-definition.yaml.
+
+        :params field str: field name to add
+        :params value str: value to add
+        :params overwrite bool: overwrite value if it already exists
+        """
+        try:
+            if overwrite:
+                self.edit_definition(field, value)
+        except YmlFieldMissingException:
+            value = value or self._get_value_from_definition(TEMPLATE_VERSION)
+            yaml_parser = yaml.YAML()
+            loaded = self._load_yaml(
+                yaml_parser=yaml_parser, yaml_file=self.entry_definition
+            )
+
+            section, field_name = field.split("/", 1)
+            loaded[section].update({field_name: value})
+            self._edit_file(
+                yaml_file=self.entry_definition, yaml_parser=yaml_parser, data=loaded
+            )
+
+    def add_properties(self, attribute_names):
+        """Add property to shell-definition.yaml file.
+
+        :params fields tuple/list: sequence of properties name that will be added
+        """
+        results = list(map(self._add_property, attribute_names))
+
+        for item in zip(attribute_names, results):
+            self._comment_attribute(*item)
+
+    def get_artifacts_files(self, artifact_name_list):
+        yaml_parser = yaml.YAML()
+        shell_definition = self._load_yaml(yaml_parser, self.entry_definition)
+
+        for node_type in list(shell_definition["node_types"].values()):
+            if "artifacts" not in node_type:
+                continue
+
+            result = {}
+            for artifact_name, artifact in node_type["artifacts"].items():
+                if artifact_name in artifact_name_list:
+                    result.update({artifact_name: artifact["file"]})
+
+            return result
+
+    def _find_entry_definition(self):
+        with open(
+            os.path.join(self.shell_path, TOSCA_META_LOCATION), "r"
+        ) as tosca_file:
+            entry_definition = dict(
+                list(map(str.strip, str(line).split(":", 1))) for line in tosca_file
+            )["Entry-Definitions"]
+
+            return entry_definition
+
+    def _load_yaml(self, yaml_parser, yaml_file):
+        with open(yaml_file, encoding="utf8") as stream:
+            try:
+                yaml_parser.indent(offset=2)
+                return yaml_parser.load(stream=stream)
+            except yaml.YAMLError as exc:
+                print(exc)  # noqa: T001
+
+    def _edit_yaml(self, yaml_file, field, value):
+        yaml_parser = yaml.YAML()
+        loaded = self._load_yaml(yaml_parser=yaml_parser, yaml_file=yaml_file)
+
+        field_name = field.split("/")[-1]
+        self._get_inner_dict_recursively(loaded, field)[field_name] = value
+
+        self._edit_file(yaml_file=yaml_file, yaml_parser=yaml_parser, data=loaded)
+
+    def _edit_file(self, yaml_file, yaml_parser, data):
+        with open(yaml_file, "wb") as f:
+            yaml_parser.dump(data, stream=f)
+
+    def _get_inner_dict_recursively(self, dic, field):
+        split = field.split("/", 1)
+        i = dic.get(split[0])
+        if not i:
+            raise YmlFieldMissingException("Field does not exists")
+        if not isinstance(i, dict) and len(split) == 1:
+            return dic
+
+        return self._get_inner_dict_recursively(i, split[1])
+
+    def _get_value_from_definition(self, field):
+        yaml_parser = yaml.YAML()
+        loaded = self._load_yaml(yaml_parser, self.entry_definition)
+
+        field_name = field.split("/")[-1]
+        value = self._get_inner_dict_recursively(loaded, field)[field_name]
+        return value
+
+    def _add_property(self, attribute_name):
+        """Add property to shell-definition.yaml file.
+
+        :params fields list: list of properties name that will be added
+        """
+        yaml_parser = yaml.YAML()
+        loaded = self._load_yaml(yaml_parser, self.entry_definition)
+
+        nodes = loaded.get("node_types")
+
+        is_last = False
+        if nodes:
+            for key, value in nodes.items():
+                if key.startswith("vendor."):
+                    properties_data = value.get("properties", {})
+                    if properties_data:
+                        properties_data.update({attribute_name: TEMPLATE_PROPERTY})
+                        is_last = False
+                    else:
+                        value.insert(
+                            1, "properties", {attribute_name: TEMPLATE_PROPERTY}
+                        )
+                        is_last = True
+                    break
+
+            self._edit_file(
+                yaml_file=self.entry_definition, yaml_parser=yaml_parser, data=loaded
+            )
+
+        return is_last
+
+    def _comment_attribute(self, attribute_name, is_last=False):
+        """Comment attribute in shell-definishion.yaml file."""
+        spaces = None
+        need_comment = False
+        lines = []
+        with open(self.entry_definition, "r", encoding="utf8") as f:
+            for line in f:
+                stripped = line.lstrip(" ")
+                if stripped.startswith("{}:".format(attribute_name)):
+                    if is_last:
+                        lines[-1] = "# {}".format(lines[-1])
+                    spaces = len(line) - len(stripped)
+                    need_comment = True
+                    lines.append("# {}".format(line))
+                    continue
+
+                if need_comment and spaces and (len(line) - len(stripped)) > spaces:
+                    lines.append("# {}".format(line))
+                    continue
+
+                need_comment = False
+                lines.append(line)
+
+        with open(self.entry_definition, "w", encoding="utf8") as f:
+            f.writelines(lines)

shellfoundry/utilities/package_builder.py

@@ -0,0 +1,152 @@
+import codecs
+import mimetypes
+import os
+import shutil
+import xml.etree.ElementTree as etree
+
+import click
+
+from shellfoundry.utilities.archive_creator import ArchiveCreator
+from shellfoundry.utilities.shell_datamodel_merger import ShellDataModelMerger
+from shellfoundry.utilities.version_utilities import DriverVersionTimestampBased
+
+
+class PackageBuilder(object):
+    def __init__(self, driver_version_strategy=None):
+        self.driver_version_strategy = (
+            driver_version_strategy or DriverVersionTimestampBased()
+        )
+
+    def build_package(self, path, package_name, driver_name):
+        package_path = os.path.join(path, "package")
+        self._copy_metadata(package_path, path)
+        self._copy_datamodel(package_path, path)
+        self._copy_categories(package_path, path)
+        self._copy_images(package_path, path)
+        self._copy_shellconfig(package_path, path)
+        self._create_driver(package_path, path, driver_name)
+        zip_path = self._zip_package(package_path, path, package_name)
+        shutil.rmtree(path=package_path, ignore_errors=True)
+        click.echo("Shell package was successfully created:")
+        click.echo(zip_path)
+
+    def _copy_metadata(self, package_path, path):
+        src_file_path = os.path.join(path, "datamodel", "metadata.xml")
+        PackageBuilder._copy_file(package_path, src_file_path)
+
+    @staticmethod
+    def _get_file_content_as_string(path):
+        with codecs.open(path, "r", encoding="utf8") as f:
+            text = f.read()
+        return text
+
+    @staticmethod
+    def _save_to_utf_file(content, dest_path):
+        with codecs.open(dest_path, "w", "utf-8-sig") as f:
+            if isinstance(content, bytes):
+                content = content.decode()
+            f.write(content)
+
+    @staticmethod
+    def _save_to_file(content, dest_path):
+        with codecs.open(dest_path, "w") as f:
+            if isinstance(content, bytes):
+                content = content.decode()
+            f.write(content)
+
+    @staticmethod
+    def _copy_datamodel(package_path, path):
+        shell_model_path = os.path.join(path, "datamodel", "shell_model.xml")
+        src_dm_file_path = os.path.join(path, "datamodel", "datamodel.xml")
+        dest_dir_path = os.path.join(package_path, "DataModel")
+
+        if os.path.exists(shell_model_path):
+            shell_model = PackageBuilder._get_file_content_as_string(shell_model_path)
+            dm = PackageBuilder._get_file_content_as_string(src_dm_file_path)
+            merger = ShellDataModelMerger()
+            merged_dm = merger.merge_shell_model(dm, shell_model)
+            if not os.path.exists(dest_dir_path):
+                os.makedirs(dest_dir_path)
+            PackageBuilder._save_to_utf_file(
+                merged_dm, os.path.join(dest_dir_path, "datamodel.xml")
+            )
+
+        else:
+            PackageBuilder._copy_file(dest_dir_path, src_dm_file_path)
+
+    @staticmethod
+    def _is_image(file):
+        file_type, encoding = mimetypes.guess_type(file)
+        return file_type and "image" in file_type
+
+    @staticmethod
+    def _copy_images(package_path, path):
+        dest_dir_path = os.path.join(package_path, "DataModel")
+        datamodel_dir = os.path.join(path, "datamodel")
+        for root, _, files in os.walk(datamodel_dir):
+            images = [
+                dir_file for dir_file in files if PackageBuilder._is_image(dir_file)
+            ]
+            for image in images:
+                PackageBuilder._copy_file(dest_dir_path, os.path.join(root, image))
+
+    @staticmethod
+    def _copy_file(dest_dir_path, src_file_path):
+        if not os.path.exists(dest_dir_path):
+            os.makedirs(dest_dir_path)
+        shutil.copy(src_file_path, dest_dir_path)
+
+    @staticmethod
+    def _copy_shellconfig(package_path, path):
+        src_file_path = os.path.join(path, "datamodel", "shellconfig.xml")
+        if os.path.exists(src_file_path):
+            dest_dir_path = os.path.join(package_path, "Configuration")
+            PackageBuilder._copy_file(dest_dir_path, src_file_path)
+
+    @staticmethod
+    def _copy_categories(package_path, path):
+        src_file_path = os.path.join(path, "categories", "categories.xml")
+        if os.path.exists(src_file_path):
+            dest_dir_path = os.path.join(package_path, "Categories")
+            PackageBuilder._copy_file(dest_dir_path, src_file_path)
+
+    def _create_driver(self, package_path, path, driver_name):
+        dir_to_zip = os.path.join(path, "src")
+        drivermetadata_path = os.path.join(dir_to_zip, "drivermetadata.xml")
+        version = self._update_driver_version(drivermetadata_path)
+        zip_file_path = os.path.join(
+            package_path, "Resource Drivers - Python", driver_name
+        )
+        ArchiveCreator.make_archive(zip_file_path, "zip", dir_to_zip)
+        if version:  # version was replaced
+            self._update_driver_version(drivermetadata_path, version)
+
+    @staticmethod
+    def _parse_xml(xml_string):
+        parser = etree.XMLParser(encoding="utf-8")
+        return etree.fromstring(xml_string, parser)
+
+    def _update_driver_version(self, metadata_path, version=""):
+        if not os.path.isfile(metadata_path):
+            return None
+
+        metadata = self._get_file_content_as_string(metadata_path)
+        metadata_xml = self._parse_xml(metadata)
+        curver = metadata_xml.get("Version")
+
+        if version:
+            metadata_xml.set("Version", version)
+            self._save_to_file(etree.tostring(metadata_xml), metadata_path)
+            return None
+        elif self.driver_version_strategy.supports_version_pattern(curver):
+            newver = self.driver_version_strategy.get_version(curver)
+            metadata_xml.set("Version", newver)
+            self._save_to_file(etree.tostring(metadata_xml), metadata_path)
+            return curver
+        else:
+            return None
+
+    @staticmethod
+    def _zip_package(package_path, path, package_name):
+        zip_file_path = os.path.join(path, "dist", package_name)
+        return ArchiveCreator.make_archive(zip_file_path, "zip", package_path)

shellfoundry/utilities/python_dependencies_packager.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import shutil
+
+try:
+    from pip import main as pip_main
+except Exception:
+    from pip._internal import main as pip_main
+
+
+class PythonDependenciesPackager(object):
+    CS_PYPI_PORT = 8036
+
+    def __init__(self):
+        pass
+
+    def save_offline_dependencies(
+        self, requirements_path, dest_path, cs_server_address=None
+    ):
+
+        if os.path.isdir(dest_path):
+            shutil.rmtree(path=dest_path, ignore_errors=True)
+
+        if not os.path.exists(requirements_path):
+            return
+
+        proxy = os.environ.get("http_proxy")
+        pip_args = ["download"]
+        if proxy:
+            pip_args.append("--proxy")
+            pip_args.append(proxy)
+
+        if cs_server_address:
+            pip_args.append(
+                "--trusted-host={cs_server_address}".format(
+                    cs_server_address=cs_server_address
+                )
+            )
+            pip_args.append(
+                "--extra-index-url=http://{cs_server_address}:{cs_pypi_port}".format(
+                    cs_server_address=cs_server_address, cs_pypi_port=self.CS_PYPI_PORT
+                )
+            )
+
+        pip_args.append(
+            "--requirement={requirements_path}".format(
+                requirements_path=requirements_path
+            )
+        )
+        pip_args.append("--dest={dest_path}".format(dest_path=dest_path))
+        pip_main(pip_args)

shellfoundry/utilities/repository_downloader.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import zipfile
+from abc import ABCMeta, abstractmethod
+from io import open
+
+import requests
+
+from .template_url import construct_template_url
+
+from shellfoundry.exceptions import VersionRequestException
+
+
+class DownloadedRepoExtractor:
+    def __init__(self):
+        pass
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def extract_to_folder(self, repo_link, folder):
+        pass
+
+
+class ZipDownloadedRepoExtractor(DownloadedRepoExtractor):
+    def extract_to_folder(self, repo_link, folder):
+        super(ZipDownloadedRepoExtractor, self).extract_to_folder(repo_link, folder)
+        with zipfile.ZipFile(repo_link, "r") as z:
+            infos = z.infolist()
+            z.extractall(folder)
+        return [info.filename for info in infos]
+
+
+class RepositoryDownloader(object):
+    def __init__(self, repo_extractor=ZipDownloadedRepoExtractor()):
+        self.repo_extractor = repo_extractor
+
+    def download_template(
+        self, target_dir, repo_address, branch, is_need_construct=True
+    ):
+        if is_need_construct:
+            download_url = construct_template_url(repo_address, branch)
+        else:
+            download_url = repo_address
+        archive_path = ""
+        try:
+            archive_path = self.download_file(download_url, target_dir)
+
+            repo_content = self.repo_extractor.extract_to_folder(
+                archive_path, target_dir
+            )
+
+            # The first entry is always the root folder by git zipball convention
+            root_dir = repo_content[0]
+
+            return os.path.join(target_dir, root_dir)
+        finally:
+            if os.path.exists(archive_path):
+                os.remove(archive_path)
+
+    def download_file(self, url, directory):
+        local_filename = os.path.join(directory, url.split("/")[-1])
+        # NOTE the stream=True parameter
+        r = requests.get(url, stream=True)
+        if r.status_code != requests.codes.ok:
+            raise VersionRequestException(
+                "Failed to download zip file from {}".format(url)
+            )
+        with open(local_filename, "wb") as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                if chunk:  # filter out keep-alive new chunks
+                    f.write(chunk)
+                    # f.flush() commented by recommendation from J.F.Sebastian
+        return local_filename

shellfoundry/utilities/shell_config_reader.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import os
+from io import open
+
+import yaml
+
+from shellfoundry.exceptions import ShellYmlMissingException, WrongShellYmlException
+
+VERSION = "version"
+DESCRIPTION = "description"
+EMAIL = "email"
+AUTHOR = "author"
+NAME = "name"
+SHELL = "shell"
+DRIVER_NAME = "driver_name"
+
+
+class ProjectConfig(object):
+    def __init__(self, name, author, email, description, version, driver_name):
+        self.version = version
+        self.description = description
+        self.email = email
+        self.author = author
+        self.name = name
+        self.driver_name = driver_name
+
+
+class ShellConfigReader(object):
+    def read(self):
+        config_path = os.path.join(os.getcwd(), "shell.yml")
+
+        if not os.path.isfile(config_path):
+            raise ShellYmlMissingException("shell.yml is missing")
+
+        with open(config_path, encoding="utf8") as stream:
+            config = yaml.safe_load(stream.read())
+
+        if not config or SHELL not in config:
+            raise WrongShellYmlException("shell section is missing in shell.yml")
+
+        install_config = config[SHELL]
+
+        name = self._get_with_default(install_config, NAME, "")
+        author = self._get_with_default(install_config, AUTHOR, "")
+        email = self._get_with_default(install_config, EMAIL, "")
+        description = self._get_with_default(install_config, DESCRIPTION, "")
+        version = self._get_with_default(install_config, VERSION, "")
+        driver_name = self._get_with_default(install_config, DRIVER_NAME, "")
+
+        return ProjectConfig(name, author, email, description, version, driver_name)
+
+    @staticmethod
+    def _get_with_default(install_config, parameter_name, default_value):
+        return (
+            install_config[parameter_name]
+            if install_config and parameter_name in install_config
+            else default_value
+        )

shellfoundry/utilities/shell_datamodel_merger.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import xml.etree.ElementTree as etree
+
+
+class ShellDataModelMerger:
+    def _parse_xml(self, xml_string):
+        parser = etree.XMLParser(encoding="utf-8")
+        return etree.fromstring(xml_string, parser)
+
+    def merge_shell_model(self, datamodel, shell_model):
+        etree.register_namespace(
+            "",
+            "http://schemas.qualisystems.com/ResourceManagement/DataModelSchema.xsd",  # noqa: E501
+        )
+        datamodel_tree = self._parse_xml(datamodel)
+        shell_tree = self._parse_xml(shell_model)
+
+        shell_family_element = shell_tree.find(".//ShellModel")
+        if shell_family_element is None:
+            raise Exception("Missing ShellModel element in shell_model.xml file")
+
+        family_name = shell_family_element.get("Family")
+
+        family_xpath_expression = ".//{{http://schemas.qualisystems.com/ResourceManagement/DataModelSchema.xsd}}ResourceFamily[@Name='{family_name}']".format(  # noqa: E501
+            family_name=family_name
+        )
+        dm_family_element = datamodel_tree.find(family_xpath_expression)
+
+        if dm_family_element is None:
+            raise Exception("Shell family not found:" + family_name)
+        model_insertion_point = dm_family_element.find(
+            ".//{http://schemas.qualisystems.com/ResourceManagement/DataModelSchema.xsd}Models"  # noqa: E501
+        )
+        models = shell_tree.find(".//ShellModel")
+        model_insertion_point.extend(models)
+
+        attributes = shell_tree.find(".//ShellAttributes")
+        attributes_insertion_point = datamodel_tree.find(
+            ".//{http://schemas.qualisystems.com/ResourceManagement/DataModelSchema.xsd}Attributes"  # noqa: E501
+        )
+        attributes_insertion_point.extend(attributes)
+
+        return etree.tostring(datamodel_tree)

shellfoundry/utilities/shell_package.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import os
+from io import open
+
+import yaml
+
+LAYER_ONE_PREFIX = "CloudshellL1"
+
+
+class ShellPackage(object):
+    def __init__(self, path):
+        self.path = path
+        self.real_shell_name = None
+
+    def get_shell_name(self):
+        """Returns shell name."""
+        head, shell_name = os.path.split(self.path)
+        return shell_name.title().replace("-", "").replace("_", "")
+
+    def get_name_from_definition(self, should_reload=False):
+        """Get shell name from shell-definition.yaml.
+
+        :param bool should_reload: Should reload from
+        :return: template name section from shell-definition.yml or equivalent written in entry definition in tosca.meta  # noqa: E501
+        :rtype: str
+        """
+        # reload the shell name if persisted member is empty or explicit request for reload  # noqa: E501
+        if not self.real_shell_name or should_reload:
+            self._reload_name()
+        return self.real_shell_name
+
+    def is_layer_one(self):
+        """Determines whether a shell is Layer 1."""
+        return bool(LAYER_ONE_PREFIX in self.get_shell_name())
+
+    def is_tosca(self):
+        """Determines whether a shell is a TOSCA based shell."""
+        return os.path.exists(self.get_metadata_path())
+
+    def get_metadata_path(self):
+        """Returns file path of the TOSCA meta file."""
+        return os.path.join(self.path, "TOSCA-Metadata", "TOSCA.meta")
+
+    def _reload_name(self):
+        """Reloads the name from the entry definition in the tosca.meta file."""
+        # fetch entry definition from tosca.meta file
+        with open(self.get_metadata_path()) as stream:
+            s = str(stream.read())
+            entry_definition = dict(
+                list(map(str.strip, line.split(":", 1)))
+                for line in s.splitlines()
+                if line.strip()
+            )["Entry-Definitions"]
+
+        # fetch template name from entry definition file retrieved earlier
+        with open(os.path.join(self.path, entry_definition), encoding="utf8") as stream:
+            definition = yaml.safe_load(stream)
+        self.real_shell_name = definition["metadata"]["template_name"]