ecs_files_composer 0.9.5__py3-none-any.whl

ecs_files_composer/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2022 John Mille<john@compose-x.io>
+
+"""Top-level package for ECS Files Composer."""
+
+__author__ = """John Preston"""
+__email__ = "john@compose-x.io"
+__version__ = "0.9.5"

ecs_files_composer/aws_mgmt.py

@@ -0,0 +1,246 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2022 John Mille<john@compose-x.io>
+
+"""AWS module."""
+
+import re
+
+import boto3
+from boto3.session import Session
+from botocore.exceptions import ClientError
+
+from ecs_files_composer import input
+from ecs_files_composer.common import LOG
+from ecs_files_composer.envsubst import expandvars
+
+
+def create_session_from_creds(tmp_creds: dict, region: str = None):
+    """
+    Function to easily convert the AssumeRole reply into a boto3 session
+    :param tmp_creds:
+    :return:
+    :rtype boto3.session.Session
+    """
+    creds = tmp_creds["Credentials"]
+    params = {
+        "aws_access_key_id": creds["AccessKeyId"],
+        "aws_secret_access_key": creds["SecretAccessKey"],
+        "aws_session_token": creds["SessionToken"],
+    }
+    if region:
+        params["region_name"] = region
+    return boto3.session.Session(**params)
+
+
+def set_session_from_iam_object(iam_config_object, source_session: Session = None):
+    """
+    Function to define the client session based on config input
+
+    :param ecs_files_composer.input.IamOverrideDef iam_config_object:
+    :param source_session:
+    :return: boto session
+    :rtype: boto3.session.Session
+    """
+    if source_session is None:
+        source_session = boto3.Session()
+    if not iam_config_object.access_key_id and not iam_config_object.secret_access_key:
+        params = {
+            "RoleArn": iam_config_object.role_arn,
+            "RoleSessionName": f"{iam_config_object.session_name}@AwsResourceHandlerInit",
+        }
+        if iam_config_object.external_id:
+            params["ExternalId"] = iam_config_object.external_id
+        tmp_creds = source_session.client("sts").assume_role(**params)
+        client_session = create_session_from_creds(
+            tmp_creds, region=iam_config_object.region_name
+        )
+    else:
+        client_session = boto3.session.Session(
+            aws_access_key_id=iam_config_object.access_key_id,
+            aws_secret_access_key=iam_config_object.secret_access_key,
+            aws_session_token=iam_config_object.session_token
+            if iam_config_object.session_token
+            else None,
+        )
+    return client_session
+
+
+class AwsResourceHandler:
+    """
+    Class to handle all AWS related credentials init.
+    """
+
+    def __init__(
+        self,
+        role_arn=None,
+        external_id=None,
+        region=None,
+        iam_config_object=None,
+        client_session_override=None,
+    ):
+        """
+        :param str role_arn:
+        :param str external_id:
+        :param str region:
+        :param ecs_files_composer.input.IamOverrideDef iam_config_object:
+        """
+        self.session = Session()
+        self.client_session = Session()
+        if client_session_override:
+            self.client_session = client_session_override
+        elif not client_session_override and (role_arn or iam_config_object):
+            if role_arn and not iam_config_object:
+                params = {
+                    "RoleArn": role_arn,
+                    "RoleSessionName": "EcsConfigComposer@AwsResourceHandlerInit",
+                }
+                if external_id:
+                    params["ExternalId"] = external_id
+                tmp_creds = self.session.client("sts").assume_role(**params)
+                self.client_session = create_session_from_creds(
+                    tmp_creds, region=region
+                )
+            elif (
+                iam_config_object
+                and hasattr(iam_config_object, "role_arn")
+                and iam_config_object.role_arn
+            ):
+                self.client_session = set_session_from_iam_object(
+                    iam_config_object, self.session
+                )
+
+
+class S3Fetcher(AwsResourceHandler):
+    """
+    Class to handle S3 actions
+    """
+
+    bucket_re = re.compile(r"^s3://(?P<bucket>[a-zA-Z\d\-.]+)/(?P<key>[\S]+)$")
+    compose_x_re = re.compile(r"^(?P<bucket>[a-zA-Z\d\-.]+)::(?P<key>[\S]+)$")
+
+    def __init__(
+        self,
+        role_arn=None,
+        external_id=None,
+        region=None,
+        iam_config_object=None,
+        client_session_override=None,
+    ):
+        super().__init__(
+            role_arn, external_id, region, iam_config_object, client_session_override
+        )
+
+    @property
+    def client(self):
+        return self.client_session.client("s3")
+
+    def get_content(
+        self,
+        s3_uri: str = None,
+        s3_bucket: str = None,
+        s3_key: str = None,
+        composex_uri: str = None,
+    ):
+        """
+        Retrieves a file in a temp dir and returns content
+
+        :return: The Stream Body for the file, allowing to do various things
+        """
+
+        if s3_uri and self.bucket_re.match(s3_uri):
+            s3_bucket = self.bucket_re.match(s3_uri).group("bucket")
+            s3_key = self.bucket_re.match(s3_uri).group("key")
+        elif composex_uri and self.compose_x_re.match(composex_uri):
+            s3_bucket = self.compose_x_re.match(composex_uri).group("bucket")
+            s3_key = self.compose_x_re.match(composex_uri).group("key")
+        try:
+            file_r = self.client.get_object(Bucket=s3_bucket, Key=s3_key)
+            file_content = file_r["Body"]
+            return file_content
+        except self.client.exceptions.NoSuchKey:
+            LOG.error(f"Failed to download the file {s3_key} from bucket {s3_bucket}")
+            raise
+
+
+class SsmFetcher(AwsResourceHandler):
+    """
+    Class to handle SSM actions
+    """
+
+    arn_re = re.compile(
+        r"^arn:aws(?:-[a-z]+)?:ssm:[\S]+:[\d]{12}:parameter(?P<name>/[\S]+)$"
+    )
+
+    def __init__(
+        self,
+        role_arn=None,
+        external_id=None,
+        region=None,
+        iam_config_object=None,
+        client_session_override=None,
+    ):
+        super().__init__(
+            role_arn, external_id, region, iam_config_object, client_session_override
+        )
+        self.client = self.client_session.client("ssm")
+
+    def get_content(self, parameter_name):
+        """
+        Import the Content of a given parameter
+        If the parameter name is a valid ARN, parses and uses the name from ARN
+
+        :param parameter_name:
+        :return:
+        """
+        if self.arn_re.match(parameter_name):
+            parameter_name = self.arn_re.match(parameter_name).group("name")
+        try:
+            parameter = self.client.get_parameter(
+                Name=parameter_name, WithDecryption=True
+            )
+            return parameter["Parameter"]["Value"]
+        except self.client.exceptions:
+            raise
+        except ClientError:
+            raise
+
+
+class SecretFetcher(AwsResourceHandler):
+    """
+    Class to handle Secret Manager actions
+    """
+
+    def __init__(
+        self,
+        role_arn=None,
+        external_id=None,
+        region=None,
+        iam_config_object=None,
+        client_session_override=None,
+    ):
+        super().__init__(
+            role_arn, external_id, region, iam_config_object, client_session_override
+        )
+        self.client = self.client_session.client("secretsmanager")
+
+    def get_content(self, secret):
+        """
+        Import the Content of a given parameter
+
+        :param input.SecretDef secret:
+        :return:
+        """
+        secret_id = expandvars(secret.secret_id)
+        params = {"SecretId": secret_id}
+        LOG.debug(f"Retrieving secretsmanager://{secret_id}")
+        if secret.version_id:
+            params["VersionId"] = secret.version_id
+        if secret.version_stage:
+            params["VersionStage"] = secret.version_stage
+        try:
+            parameter = self.client.get_secret_value(**params)
+            return parameter["SecretString"]
+        except self.client.exceptions:
+            raise
+        except ClientError:
+            raise

ecs_files_composer/certbot_aws_store.py

@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2022 John Mille<john@compose-x.io>
+
+"""
+Manages certbot certificates download from certbot-aws-store
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .input import Model
+
+from os import makedirs, path
+
+from certbot_aws_store.certificate import AcmeCertificate
+
+from ecs_files_composer.common import LOG
+
+
+def handle_certbot_store_certificates(job: Model) -> None:
+    """
+    Pulls certificates from certbot-aws-store to local filesystem
+    If the path does not exist, creates a new directory for download.
+    """
+    if not job.certbot_store:
+        return
+    for _hostname, _definition in job.certbot_store.items():
+        certificate = AcmeCertificate(
+            _hostname,
+            None,
+            table_name=_definition.table_name if _definition.table_name else None,
+            region_name=_definition.table_region_name
+            if _definition.table_region_name
+            else None,
+        )
+        if not path.exists(_definition.storage_path):
+            makedirs(_definition.storage_path, exist_ok=True)
+        try:
+            certificate.pull(_definition.storage_path)
+            LOG.info("Successfully pulled certificates for %s", _hostname)
+        except Exception as error:
+            LOG.exception(error)
+            LOG.error(
+                "Failed to download certificate from certbot-aws-store", _hostname
+            )

ecs_files_composer/certificates_mgmt.py

@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2022 John Mille<john@compose-x.io>
+
+import pathlib
+import socket
+from os import path
+from typing import Any
+
+from OpenSSL import crypto
+
+from ecs_files_composer.files_mgmt import File
+from ecs_files_composer.input import X509CertDef
+
+
+class X509Certificate(X509CertDef):
+    """
+    Class to wrap actions around a new X509 certificate
+    """
+
+    def __init__(self, **data: Any):
+        super().__init__(**data)
+        self.key = None
+        self.cert = None
+        self.key_content = None
+        self.cert_content = None
+        self.cert_file = None
+        self.key_file = None
+        self.cert_file_path = None
+        self.key_file_path = None
+
+    def init_cert_paths(self):
+        self.cert_file_path = path.abspath(f"{self.dir_path}/{self.cert_file_name}")
+        self.key_file_path = path.abspath(f"{self.dir_path}/{self.key_file_name}")
+        print(f"Creating {self.dir_path} folder")
+        dir_path = pathlib.Path(path.abspath(self.dir_path))
+        dir_path.mkdir(parents=True, exist_ok=True)
+
+    def generate_key(self):
+        self.key = crypto.PKey()
+        self.key.generate_key(crypto.TYPE_RSA, 4096)
+
+    def set_common_name(self):
+        if self.common_name is None:
+            self.common_name = socket.gethostname()
+
+    def generate_cert(self):
+        if not self.common_name:
+            self.set_common_name()
+        self.cert = crypto.X509()
+        self.cert.get_subject().C = self.country_name
+        self.cert.get_subject().ST = self.state_or_province_name
+        self.cert.get_subject().L = self.locality_name
+        self.cert.get_subject().O = self.organization_name
+        self.cert.get_subject().OU = self.organization_unit_name
+        self.cert.get_subject().CN = self.common_name
+        self.cert.get_subject().emailAddress = self.email_address
+        self.cert.set_serial_number(0)
+        self.cert.gmtime_adj_notBefore(0)
+        self.cert.gmtime_adj_notAfter(int(self.validity_end_in_seconds))
+        self.cert.set_issuer(self.cert.get_subject())
+        self.cert.set_pubkey(self.key)
+        self.cert.sign(self.key, "sha512")
+
+    def generate_cert_content(self):
+        if not self.key:
+            self.generate_key()
+        if not self.cert:
+            self.generate_cert()
+        self.cert_content = crypto.dump_certificate(
+            crypto.FILETYPE_PEM, self.cert
+        ).decode("utf-8")
+        self.key_content = crypto.dump_privatekey(crypto.FILETYPE_PEM, self.key).decode(
+            "utf-8"
+        )
+
+    def set_cert_files(self):
+        if not self.cert_content or not self.key_content:
+            self.generate_cert_content()
+        self.key_file = File().parse_obj(
+            {
+                "content": self.key_content,
+                "path": self.key_file_path,
+                "mode": "0600",
+                "owner": self.owner,
+                "group": self.group,
+            }
+        )
+        self.cert_file = File().parse_obj(
+            {
+                "content": self.cert_content,
+                "path": self.cert_file_path,
+                "mode": "0600",
+                "owner": self.owner,
+                "group": self.group,
+            }
+        )
+
+
+def process_x509_certs(job):
+    """
+
+    :param ecs_files_composer.input.Model job:
+    :return:
+    """
+    if not hasattr(job.certificates, "x509") or not job.certificates.x509:
+        return
+    for cert_path, cert_def in job.certificates.x509.items():
+        cert_obj = X509Certificate(
+            **(cert_def.dict(by_alias=True)),
+        )
+        cert_obj.dir_path = cert_path
+        cert_obj.init_cert_paths()
+        cert_obj.set_cert_files()
+        job.certificates.x509[cert_path] = cert_obj
+        job.files[cert_obj.cert_file.path] = cert_obj.cert_file
+        job.files[cert_obj.key_file_path] = cert_obj.key_file

ecs_files_composer/cli.py

@@ -0,0 +1,132 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2022 John Mille<john@compose-x.io>
+
+"""Console script for ecs_files_composer."""
+import argparse
+import sys
+from os import environ
+
+from ecs_files_composer.common import LOG
+from ecs_files_composer.ecs_files_composer import init_config, start_jobs
+
+
+def main():
+    """Console script for ecs_files_composer."""
+    parser = argparse.ArgumentParser()
+    options = parser.add_mutually_exclusive_group()
+    options.add_argument(
+        "-f",
+        "--from-file",
+        help="Configuration for execution from a file",
+        type=str,
+        required=False,
+        dest="file_path",
+    )
+    options.add_argument(
+        "-e",
+        "--from-env-var",
+        dest="env_var",
+        required=False,
+        help="Configuration for execution is in an environment variable",
+    )
+    options.add_argument(
+        "--from-ssm",
+        dest="ssm_config",
+        help="Configuration for execution is in an SSM Parameter",
+        required=False,
+    )
+    options.add_argument(
+        "--from-s3",
+        dest="s3_config",
+        required=False,
+        help="Configuration for execution is in an S3",
+    )
+    options.add_argument(
+        "--from-secrets",
+        dest="secret_config",
+        required=False,
+        help="Configuration for execution is in an AWS Secrets Manager",
+    )
+    parser.add_argument(
+        "--role-arn",
+        help="The Role ARN to use for the configuration initialization",
+        required=False,
+    )
+    parser.add_argument(
+        "--decode-base64",
+        help="Whether the source config is in base64 encoded",
+        action="store_true",
+        required=False,
+        default=False,
+    )
+    parser.add_argument(
+        "--context",
+        help="Indicate which context to use (valid: jinja2|plain). Default is jinja2",
+        required=False,
+        default="jinja2",
+    )
+    parser.add_argument(
+        "--override-init-folder",
+        dest="init_folder",
+        required=False,
+        type=str,
+        default="",
+    )
+    parser.add_argument("_", nargs="*")
+    args = parser.parse_args()
+    print("Arguments: " + str(args._))
+    if not (
+        args.env_var or args.ssm_config or args.s3_config or args.file_path
+    ) and environ.get("ECS_CONFIG_CONTENT", None):
+        LOG.info("Using default env variable ECS_CONFIG_CONTENT")
+        config = init_config(
+            env_var="ECS_CONFIG_CONTENT",
+            decode_base64=bool(environ.get("DECODE_BASE64", False)),
+            context=environ.get("context", "jinja2"),
+        )
+    elif args.env_var:
+        config = init_config(
+            env_var=args.env_var,
+            decode_base64=args.decode_base64,
+            context=args.context,
+            override_folder=args.init_folder,
+        )
+    elif args.file_path:
+        config = init_config(
+            file_path=args.file_path,
+            decode_base64=args.decode_base64,
+            context=args.context,
+            override_folder=args.init_folder,
+        )
+    elif args.ssm_config:
+        config = init_config(
+            ssm_parameter=args.ssm_config,
+            decode_base64=args.decode_base64,
+            context=args.context,
+            override_folder=args.init_folder,
+        )
+    elif args.s3_config:
+        config = init_config(
+            s3_config=args.s3_config,
+            decode_base64=args.decode_base64,
+            context=args.context,
+            override_folder=args.init_folder,
+        )
+    elif args.secret_config:
+        config = init_config(
+            secret_config=args.secret_config,
+            decode_base64=args.decode_base64,
+            context=args.context,
+            override_folder=args.init_folder,
+        )
+    else:
+        raise parser.error(
+            "You must specify where the execution configuration comes from or set ECS_CONFIG_CONTENT."
+        )
+
+    start_jobs(config)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())  # pragma: no cover

ecs_files_composer/common.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2022 John Mille<john@compose-x.io>
+
+import logging as logthings
+import sys
+from os import environ
+
+from compose_x_common.compose_x_common import keyisset
+
+
+def setup_logging():
+    """Function to setup logging for ECS ComposeX.
+    In case this is used in a Lambda function, removes the AWS Lambda default log handler
+
+    :returns: the_logger
+    :rtype: Logger
+    """
+    level = environ.get("LOGLEVEL")
+    default_level = True
+    formats = {
+        "INFO": logthings.Formatter(
+            "%(asctime)s [%(levelname)s], %(message)s",
+            "%Y-%m-%d %H:%M:%S",
+        ),
+        "DEBUG": logthings.Formatter(
+            "%(asctime)s [%(levelname)s], %(filename)s.%(lineno)d , %(funcName)s, %(message)s",
+            "%Y-%m-%d %H:%M:%S",
+        ),
+    }
+
+    if level is not None and isinstance(level, str):
+        print("SETTING TO", level.upper())
+        logthings.basicConfig(level=level.upper())
+        default_level = False
+    else:
+        logthings.basicConfig(level="INFO")
+
+    root_logger = logthings.getLogger()
+    for h in root_logger.handlers:
+        root_logger.removeHandler(h)
+    the_logger = logthings.getLogger("EcsComposeX")
+
+    if not the_logger.handlers:
+        if default_level:
+            formatter = formats["INFO"]
+        elif keyisset(level.upper(), formats):
+            formatter = formats[level.upper()]
+        else:
+            formatter = formats["DEBUG"]
+        handler = logthings.StreamHandler(sys.stdout)
+        handler.setFormatter(formatter)
+        the_logger.addHandler(handler)
+
+    return the_logger
+
+
+LOG = setup_logging()