awspub 0.0.1__py3-none-any.whl

awspub/__init__.py

@@ -0,0 +1,3 @@
+from awspub.api import cleanup, create, list, publish, verify
+
+__all__ = ["create", "list", "publish", "cleanup", "verify"]

awspub/api.py

@@ -0,0 +1,165 @@
+import logging
+import pathlib
+from typing import Dict, Iterator, List, Optional, Tuple
+
+from awspub.context import Context
+from awspub.image import Image, _ImageInfo
+from awspub.s3 import S3
+
+logger = logging.getLogger(__name__)
+
+
+def _images_grouped(
+    images: List[Tuple[str, Image, Dict[str, _ImageInfo]]], group: Optional[str]
+) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]]:
+    """
+    Group the given images by name and by group
+
+    :param images: the images
+    :type images: List[Tuple[str, Image, Dict[str, _ImageInfo]]]
+    :param group: a optional group name
+    :type group: Optional[str]
+    :return: the images grouped by name and by group
+    :rtype: Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]
+    """
+    images_by_name: Dict[str, Dict[str, str]] = dict()
+    images_by_group: Dict[str, Dict[str, Dict[str, str]]] = dict()
+    for image_name, image, image_result in images:
+        images_region_id: Dict[str, str] = {key: val.image_id for (key, val) in image_result.items()}
+        images_by_name[image_name] = images_region_id
+        for image_group in image.conf.get("groups", []):
+            if group and image_group != group:
+                continue
+            if not images_by_group.get(image_group):
+                images_by_group[image_group] = {}
+            images_by_group[image_group][image_name] = images_region_id
+    return images_by_name, images_by_group
+
+
+def _images_filtered(context: Context, group: Optional[str]) -> Iterator[Tuple[str, Image]]:
+    """
+    Filter the images from ctx based on the given args
+
+    :param context: the context
+    :type context: a awspub.context.Context instance
+    :param group: a optional group name
+    :type group: Optional[str]
+    """
+    for image_name in context.conf["images"].keys():
+        image = Image(context, image_name)
+        if group:
+            # limit the images to process to the group given on the command line
+            if group not in image.conf.get("groups", []):
+                logger.info(f"skipping image {image_name} because not part of group {group}")
+                continue
+
+        logger.info(f"processing image {image_name} (group: {group})")
+        yield image_name, image
+
+
+def create(
+    config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]
+) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]]:
+    """
+    Create images in the partition of the used account based on
+    the given configuration file and the config mapping
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    :return: the images grouped by name and by group
+    :rtype: Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]
+    """
+
+    ctx = Context(config, config_mapping)
+    s3 = S3(ctx)
+    s3.upload_file(ctx.conf["source"]["path"])
+    images: List[Tuple[str, Image, Dict[str, _ImageInfo]]] = []
+    for image_name, image in _images_filtered(ctx, group):
+        image_result: Dict[str, _ImageInfo] = image.create()
+        images.append((image_name, image, image_result))
+    images_by_name, images_by_group = _images_grouped(images, group)
+    return images_by_name, images_by_group
+
+
+def list(
+    config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]
+) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]]:
+    """
+    List images in the partition of the used account based on
+    the given configuration file and the config mapping
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    :return: the images grouped by name and by group
+    :rtype: Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]
+    """
+    ctx = Context(config, config_mapping)
+    images: List[Tuple[str, Image, Dict[str, _ImageInfo]]] = []
+    for image_name, image in _images_filtered(ctx, group):
+        image_result: Dict[str, _ImageInfo] = image.list()
+        images.append((image_name, image, image_result))
+
+    images_by_name, images_by_group = _images_grouped(images, group)
+    return images_by_name, images_by_group
+
+
+def publish(config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]):
+    """
+    Make available images in the partition of the used account based on
+    the given configuration file public
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    """
+    ctx = Context(config, config_mapping)
+    for image_name, image in _images_filtered(ctx, group):
+        image.publish()
+
+
+def cleanup(config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]):
+    """
+    Cleanup available images in the partition of the used account based on
+    the given configuration file
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    """
+    ctx = Context(config, config_mapping)
+    for image_name, image in _images_filtered(ctx, group):
+        image.cleanup()
+
+
+def verify(config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]) -> Dict[str, Dict]:
+    """
+    Verify available images in the partition of the used account based on
+    the given configuration file.
+    This is EXPERIMENTAL and doesn't work reliable yet!
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    """
+    problems: Dict[str, Dict] = dict()
+    ctx = Context(config, config_mapping)
+    for image_name, image in _images_filtered(ctx, group):
+        problems[image_name] = image.verify()
+    return problems

awspub/cli/__init__.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python3
+
+import argparse
+import json
+import logging
+import pathlib
+import sys
+
+import awspub
+
+logger = logging.getLogger(__name__)
+
+
+def _create(args) -> None:
+    """
+    Create images based on the given configuration and write json
+    data to the given output
+    """
+    images_by_name, images_by_group = awspub.create(args.config, args.config_mapping, args.group)
+    images_json = json.dumps({"images": images_by_name, "images-by-group": images_by_group}, indent=4)
+    args.output.write(images_json)
+
+
+def _list(args) -> None:
+    """
+    List images based on the given configuration and write json
+    data to the given output
+    """
+    images_by_name, images_by_group = awspub.list(args.config, args.config_mapping, args.group)
+    images_json = json.dumps({"images": images_by_name, "images-by-group": images_by_group}, indent=4)
+    args.output.write(images_json)
+
+
+def _verify(args) -> None:
+    """
+    Verify available images against configuration
+    """
+    problems = awspub.verify(args.config, args.config_mapping, args.group)
+    args.output.write((json.dumps({"problems": problems}, indent=4)))
+
+
+def _cleanup(args) -> None:
+    """
+    Cleanup available images
+    """
+    awspub.cleanup(args.config, args.config_mapping, args.group)
+
+
+def _publish(args) -> None:
+    """
+    Make available images public
+    """
+    awspub.publish(args.config, args.config_mapping, args.group)
+
+
+def _parser():
+    parser = argparse.ArgumentParser(description="AWS EC2 publication tool")
+    parser.add_argument("--log-level", choices=["info", "debug"], default="info")
+    parser.add_argument("--log-file", type=pathlib.Path, help="write log to given file instead of stdout")
+    parser.add_argument("--log-console", action=argparse.BooleanOptionalAction, help="write log to stdout")
+    p_sub = parser.add_subparsers(help="sub-command help")
+
+    # create
+    p_create = p_sub.add_parser("create", help="Create images")
+    p_create.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_create.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_create.add_argument("--group", type=str, help="only handles images from given group")
+    p_create.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+    p_create.set_defaults(func=_create)
+
+    # list
+    p_list = p_sub.add_parser("list", help="List images (but don't modify anything)")
+    p_list.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_list.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_list.add_argument("--group", type=str, help="only handles images from given group")
+    p_list.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+    p_list.set_defaults(func=_list)
+
+    # verify
+    p_verify = p_sub.add_parser("verify", help="Verify images")
+    p_verify.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_verify.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_verify.add_argument("--group", type=str, help="only handles images from given group")
+    p_verify.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+
+    p_verify.set_defaults(func=_verify)
+
+    # cleanup
+    p_cleanup = p_sub.add_parser("cleanup", help="Cleanup images")
+    p_cleanup.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_cleanup.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_cleanup.add_argument("--group", type=str, help="only handles images from given group")
+    p_cleanup.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+
+    p_cleanup.set_defaults(func=_cleanup)
+
+    # publish
+    p_publish = p_sub.add_parser("publish", help="Publish images")
+    p_publish.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_publish.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_publish.add_argument("--group", type=str, help="only handles images from given group")
+    p_publish.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+
+    p_publish.set_defaults(func=_publish)
+
+    return parser
+
+
+def main():
+    parser = _parser()
+    args = parser.parse_args()
+    log_formatter = logging.Formatter("%(asctime)s:%(name)s:%(levelname)s:%(message)s")
+    # log level
+    loglevel = logging.INFO
+    if args.log_level == "debug":
+        loglevel = logging.DEBUG
+    root_logger = logging.getLogger()
+    root_logger.setLevel(loglevel)
+    # log file
+    if args.log_file:
+        file_handler = logging.FileHandler(filename=args.log_file)
+        file_handler.setFormatter(log_formatter)
+        root_logger.addHandler(file_handler)
+    # log console
+    if args.log_console:
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(log_formatter)
+        root_logger.addHandler(console_handler)
+    if "func" not in args:
+        sys.exit(parser.print_help())
+    args.func(args)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

awspub/configmodels.py

@@ -0,0 +1,157 @@
+import pathlib
+from typing import Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ConfigS3Model(BaseModel):
+    """
+    S3 configuration.
+    This is required for uploading source files (usually .vmdk) to a bucket so
+    snapshots can be created out of the s3 file
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    bucket_name: str = Field(description="The S3 bucket name")
+
+
+class ConfigSourceModel(BaseModel):
+    """
+    Source configuration.
+    This defines the source (usually .vmdk) that is uploaded
+    to S3 and then used to create EC2 snapshots in different regions.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    path: pathlib.Path = Field(description="Path to a local .vmdk image")
+    architecture: Literal["x86_64", "arm64"] = Field(description="The architecture of the given .vmdk image")
+
+
+class ConfigImageMarketplaceSecurityGroupModel(BaseModel):
+    """
+    Image/AMI Marketplace specific configuration for a security group
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    from_port: int = Field(description="The source port")
+    ip_protocol: Literal["tcp", "udp"] = Field(description="The IP protocol (either 'tcp' or 'udp')")
+    ip_ranges: List[str] = Field(description="IP ranges to allow, in CIDR format (eg. '192.0.2.0/24')")
+    to_port: int = Field(description="The destination port")
+
+
+class ConfigImageMarketplaceModel(BaseModel):
+    """
+    Image/AMI Marketplace specific configuration to request new Marketplace versions
+    See https://docs.aws.amazon.com/marketplace-catalog/latest/api-reference/ami-products.html
+    for further information
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    entity_id: str = Field(description="The entity ID (product ID)")
+    # see https://docs.aws.amazon.com/marketplace/latest/userguide/ami-single-ami-products.html#single-ami-marketplace-ami-access  # noqa:E501
+    access_role_arn: str = Field(
+        description="IAM role Amazon Resource Name (ARN) used by AWS Marketplace to access the provided AMI"
+    )
+    version_title: str = Field(description="The version title. Must be unique across the product")
+    release_notes: str = Field(description="The release notes")
+    user_name: str = Field(description="The login username to access the operating system")
+    scanning_port: int = Field(description="Port to access the operating system (default: 22)", default=22)
+    os_name: str = Field(description="Operating system name displayed to Marketplace buyers")
+    os_version: str = Field(description="Operating system version displayed to Marketplace buyers")
+    usage_instructions: str = Field(
+        description=" Instructions for using the AMI, or a link to more information about the AMI"
+    )
+    recommended_instance_type: str = Field(
+        description="The instance type that is recommended to run the service with the AMI and is the "
+        "default for 1-click installs of your service"
+    )
+    security_groups: Optional[List[ConfigImageMarketplaceSecurityGroupModel]]
+
+
+class ConfigImageSSMParameterModel(BaseModel):
+    """
+    Image/AMI SSM specific configuration to push parameters of type `aws:ec2:image` to the parameter store
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str = Field(
+        description="The fully qualified name of the parameter that you want to add to the system. "
+        "A parameter name must be unique within an Amazon Web Services Region"
+    )
+    description: Optional[str] = Field(
+        description="Information about the parameter that you want to add to the system", default=None
+    )
+    allow_overwrite: Optional[bool] = Field(
+        description="allow to overwrite an existing parameter. Useful for keep a 'latest' parameter (default: false)",
+        default=False,
+    )
+
+
+class ConfigImageModel(BaseModel):
+    """
+    Image/AMI configuration.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    description: Optional[str] = Field(description="Optional image description", default=None)
+    regions: Optional[List[str]] = Field(
+        description="Optional list of regions for this image. If not given, all available regions will be used",
+        default=None,
+    )
+    separate_snapshot: bool = Field(description="Use a separate snapshot for this image?", default=False)
+    billing_products: Optional[List[str]] = Field(description="Optional list of billing codes", default=None)
+    boot_mode: Literal["legacy-bios", "uefi", "uefi-preferred"] = Field(
+        description="The boot mode. For arm64, this needs to be 'uefi'"
+    )
+    root_device_name: Optional[str] = Field(description="The root device name", default="/dev/sda1")
+    root_device_volume_type: Optional[Literal["gp2", "gp3"]] = Field(
+        description="The root device volume type", default="gp3"
+    )
+    root_device_volume_size: Optional[int] = Field(description="The root device volume size (in GB)", default=8)
+    uefi_data: Optional[pathlib.Path] = Field(
+        description="Optional path to a non-volatile UEFI variable store (must be already base64 encoded)", default=None
+    )
+    tpm_support: Optional[Literal["v2.0"]] = Field(
+        description="Optional TPM support. If this is set, 'boot_mode' must be 'uefi'", default=None
+    )
+    imds_support: Optional[Literal["v2.0"]] = Field(description="Optional IMDS support", default=None)
+    share: Optional[List[str]] = Field(
+        description="Optional list of account IDs the image and snapshot will be shared with", default=None
+    )
+    temporary: Optional[bool] = Field(
+        description="Optional boolean field indicates that a image is only temporary", default=False
+    )
+    public: Optional[bool] = Field(
+        description="Optional boolean field indicates if the image should be public", default=False
+    )
+    marketplace: Optional[ConfigImageMarketplaceModel] = Field(
+        description="Optional structure containing Marketplace related configuration for the commercial "
+        "'aws' partition",
+        default=None,
+    )
+    ssm_parameter: Optional[List[ConfigImageSSMParameterModel]] = Field(
+        description="Optional list of SSM parameter paths of type `aws:ec2:image` which will "
+        "be pushed to the parameter store",
+        default=None,
+    )
+    groups: Optional[List[str]] = Field(description="Optional list of groups this image is part of", default=[])
+    tags: Optional[Dict[str, str]] = Field(description="Optional Tags to apply to this image only", default={})
+
+
+class ConfigModel(BaseModel):
+    """
+    The base model for the whole configuration
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    s3: ConfigS3Model
+    source: ConfigSourceModel
+    images: Dict[str, ConfigImageModel]
+    tags: Optional[Dict[str, str]] = Field(description="Optional Tags to apply to all resources", default={})

awspub/context.py

@@ -0,0 +1,108 @@
+import hashlib
+import logging
+import pathlib
+from string import Template
+from typing import Dict
+
+from ruamel.yaml import YAML
+
+from awspub.configmodels import ConfigModel
+
+logger = logging.getLogger(__name__)
+
+
+class Context:
+    """
+    Context holds the used configuration and some
+    automatically calculated values
+    """
+
+    def __init__(self, conf_path: pathlib.Path, conf_template_mapping_path: pathlib.Path):
+        self._conf_path = conf_path
+        self._conf = None
+        self._conf_template_mapping_path = conf_template_mapping_path
+        self._conf_template_mapping = {}
+        yaml = YAML(typ="safe")
+
+        # read the config mapping first
+        if self._conf_template_mapping_path:
+            with open(self._conf_template_mapping_path, "r") as ctm:
+                self._conf_template_mapping = yaml.load(ctm)
+                logger.debug(f"loaded config template mapping for substitution: {self._conf_template_mapping}")
+
+        # read the config itself
+        with open(self._conf_path, "r") as f:
+            template = Template(f.read())
+            # substitute the values in the config with values from the config template mapping
+            ft = template.substitute(**self._conf_template_mapping)
+            y = yaml.load(ft)["awspub"]
+            self._conf = ConfigModel(**y).model_dump()
+            logger.debug(f"config loaded and validated as: {self._conf}")
+
+        # handle relative paths in config files. those are relative to the config file dirname
+        if not self.conf["source"]["path"].is_absolute():
+            self.conf["source"]["path"] = pathlib.Path(self._conf_path).parent / self.conf["source"]["path"]
+
+        for image_name, props in self.conf["images"].items():
+            if props["uefi_data"] and not self.conf["images"][image_name]["uefi_data"].is_absolute():
+                self.conf["images"][image_name]["uefi_data"] = (
+                    pathlib.Path(self._conf_path).parent / self.conf["images"][image_name]["uefi_data"]
+                )
+
+        # calculate the sha256 sum of the source file once
+        self._source_sha256_obj = self._sha256sum(self.conf["source"]["path"])
+        self._source_sha256 = self._source_sha256_obj.hexdigest()
+
+    @property
+    def conf(self):
+        return self._conf
+
+    @property
+    def source_sha256(self):
+        """
+        The sha256 sum hexdigest of the source->path value from the given
+        configuration. This value is used in different places (eg. to automatically
+        upload to S3 with this value as key)
+        """
+        return self._source_sha256
+
+    @property
+    def tags_dict(self) -> Dict[str, str]:
+        """
+        Common tags which will be used for all AWS resources
+        This includes tags defined in the configuration file
+        but doesn't include image group specific tags.
+        Usually the tags() method should be used.
+        """
+        tags = dict()
+        tags["awspub:source:filename"] = self.conf["source"]["path"].name
+        tags["awspub:source:architecture"] = self.conf["source"]["architecture"]
+        tags["awspub:source:sha256"] = self.source_sha256
+        tags.update(self.conf.get("tags", {}))
+        return tags
+
+    @property
+    def tags(self):
+        """
+        Helper to make tags directly usable by the AWS EC2 API
+        which requires a list of dicts with "Key" and "Value" defined.
+        """
+        tags = []
+        for name, value in self.tags_dict.items():
+            tags.append({"Key": name, "Value": value})
+        return tags
+
+    def _sha256sum(self, file_path: pathlib.Path):
+        """
+        Calculate a sha256 sum for a given file
+
+        :param file_path: the path to the local file to upload
+        :type file_path: pathlib.Path
+        :return: a haslib Hash object
+        :rtype: _hashlib.HASH
+        """
+        sha256_hash = hashlib.sha256()
+        with open(file_path.resolve(), "rb") as f:
+            for byte_block in iter(lambda: f.read(4096), b""):
+                sha256_hash.update(byte_block)
+        return sha256_hash

awspub/exceptions.py

@@ -0,0 +1,16 @@
+class MultipleSnapshotsException(Exception):
+    pass
+
+
+class MultipleImportSnapshotTasksException(Exception):
+    pass
+
+
+class MultipleImagesException(Exception):
+    pass
+
+
+class BucketDoesNotExistException(Exception):
+    def __init__(self, bucket_name: str, *args, **kwargs):
+        msg = f"The bucket named '{bucket_name}' does not exist. You will need to create the bucket before proceeding."
+        super().__init__(msg, *args, **kwargs)