awspub 0.0.10__py3-none-any.whl

awspub/__init__.py

@@ -0,0 +1,3 @@
+from awspub.api import cleanup, create, list, publish, verify
+
+__all__ = ["create", "list", "publish", "cleanup", "verify"]

awspub/api.py

@@ -0,0 +1,165 @@
+import logging
+import pathlib
+from typing import Dict, Iterator, List, Optional, Tuple
+
+from awspub.context import Context
+from awspub.image import Image, _ImageInfo
+from awspub.s3 import S3
+
+logger = logging.getLogger(__name__)
+
+
+def _images_grouped(
+    images: List[Tuple[str, Image, Dict[str, _ImageInfo]]], group: Optional[str]
+) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]]:
+    """
+    Group the given images by name and by group
+
+    :param images: the images
+    :type images: List[Tuple[str, Image, Dict[str, _ImageInfo]]]
+    :param group: a optional group name
+    :type group: Optional[str]
+    :return: the images grouped by name and by group
+    :rtype: Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]
+    """
+    images_by_name: Dict[str, Dict[str, str]] = dict()
+    images_by_group: Dict[str, Dict[str, Dict[str, str]]] = dict()
+    for image_name, image, image_result in images:
+        images_region_id: Dict[str, str] = {key: val.image_id for (key, val) in image_result.items()}
+        images_by_name[image_name] = images_region_id
+        for image_group in image.conf.get("groups", []):
+            if group and image_group != group:
+                continue
+            if not images_by_group.get(image_group):
+                images_by_group[image_group] = {}
+            images_by_group[image_group][image_name] = images_region_id
+    return images_by_name, images_by_group
+
+
+def _images_filtered(context: Context, group: Optional[str]) -> Iterator[Tuple[str, Image]]:
+    """
+    Filter the images from ctx based on the given args
+
+    :param context: the context
+    :type context: a awspub.context.Context instance
+    :param group: a optional group name
+    :type group: Optional[str]
+    """
+    for image_name in context.conf["images"].keys():
+        image = Image(context, image_name)
+        if group:
+            # limit the images to process to the group given on the command line
+            if group not in image.conf.get("groups", []):
+                logger.info(f"skipping image {image_name} because not part of group {group}")
+                continue
+
+        logger.info(f"processing image {image_name} (group: {group})")
+        yield image_name, image
+
+
+def create(
+    config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]
+) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]]:
+    """
+    Create images in the partition of the used account based on
+    the given configuration file and the config mapping
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    :return: the images grouped by name and by group
+    :rtype: Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]
+    """
+
+    ctx = Context(config, config_mapping)
+    s3 = S3(ctx)
+    s3.upload_file(ctx.conf["source"]["path"])
+    images: List[Tuple[str, Image, Dict[str, _ImageInfo]]] = []
+    for image_name, image in _images_filtered(ctx, group):
+        image_result: Dict[str, _ImageInfo] = image.create()
+        images.append((image_name, image, image_result))
+    images_by_name, images_by_group = _images_grouped(images, group)
+    return images_by_name, images_by_group
+
+
+def list(
+    config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]
+) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]]:
+    """
+    List images in the partition of the used account based on
+    the given configuration file and the config mapping
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    :return: the images grouped by name and by group
+    :rtype: Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, Dict[str, str]]]
+    """
+    ctx = Context(config, config_mapping)
+    images: List[Tuple[str, Image, Dict[str, _ImageInfo]]] = []
+    for image_name, image in _images_filtered(ctx, group):
+        image_result: Dict[str, _ImageInfo] = image.list()
+        images.append((image_name, image, image_result))
+
+    images_by_name, images_by_group = _images_grouped(images, group)
+    return images_by_name, images_by_group
+
+
+def publish(config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]):
+    """
+    Make available images in the partition of the used account based on
+    the given configuration file public
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    """
+    ctx = Context(config, config_mapping)
+    for image_name, image in _images_filtered(ctx, group):
+        image.publish()
+
+
+def cleanup(config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]):
+    """
+    Cleanup available images in the partition of the used account based on
+    the given configuration file
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    """
+    ctx = Context(config, config_mapping)
+    for image_name, image in _images_filtered(ctx, group):
+        image.cleanup()
+
+
+def verify(config: pathlib.Path, config_mapping: pathlib.Path, group: Optional[str]) -> Dict[str, Dict]:
+    """
+    Verify available images in the partition of the used account based on
+    the given configuration file.
+    This is EXPERIMENTAL and doesn't work reliable yet!
+
+    :param config: the configuration file path
+    :type config: pathlib.Path
+    :param config_mapping: the config template mapping file path
+    :type config_mapping: pathlib.Path
+    :param group: only handles images from given group
+    :type group: Optional[str]
+    """
+    problems: Dict[str, Dict] = dict()
+    ctx = Context(config, config_mapping)
+    for image_name, image in _images_filtered(ctx, group):
+        problems[image_name] = image.verify()
+    return problems

awspub/cli/__init__.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python3
+
+import argparse
+import json
+import logging
+import pathlib
+import sys
+
+import awspub
+
+logger = logging.getLogger(__name__)
+
+
+def _create(args) -> None:
+    """
+    Create images based on the given configuration and write json
+    data to the given output
+    """
+    images_by_name, images_by_group = awspub.create(args.config, args.config_mapping, args.group)
+    images_json = json.dumps({"images": images_by_name, "images-by-group": images_by_group}, indent=4)
+    args.output.write(images_json)
+
+
+def _list(args) -> None:
+    """
+    List images based on the given configuration and write json
+    data to the given output
+    """
+    images_by_name, images_by_group = awspub.list(args.config, args.config_mapping, args.group)
+    images_json = json.dumps({"images": images_by_name, "images-by-group": images_by_group}, indent=4)
+    args.output.write(images_json)
+
+
+def _verify(args) -> None:
+    """
+    Verify available images against configuration
+    """
+    problems = awspub.verify(args.config, args.config_mapping, args.group)
+    args.output.write((json.dumps({"problems": problems}, indent=4)))
+
+
+def _cleanup(args) -> None:
+    """
+    Cleanup available images
+    """
+    awspub.cleanup(args.config, args.config_mapping, args.group)
+
+
+def _publish(args) -> None:
+    """
+    Make available images public
+    """
+    awspub.publish(args.config, args.config_mapping, args.group)
+
+
+def _parser():
+    parser = argparse.ArgumentParser(description="AWS EC2 publication tool")
+    parser.add_argument("--log-level", choices=["info", "debug"], default="info")
+    parser.add_argument("--log-file", type=pathlib.Path, help="write log to given file instead of stdout")
+    parser.add_argument("--log-console", action="store_true", help="write log to stdout")
+    p_sub = parser.add_subparsers(help="sub-command help")
+
+    # create
+    p_create = p_sub.add_parser("create", help="Create images")
+    p_create.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_create.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_create.add_argument("--group", type=str, help="only handles images from given group")
+    p_create.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+    p_create.set_defaults(func=_create)
+
+    # list
+    p_list = p_sub.add_parser("list", help="List images (but don't modify anything)")
+    p_list.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_list.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_list.add_argument("--group", type=str, help="only handles images from given group")
+    p_list.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+    p_list.set_defaults(func=_list)
+
+    # verify
+    p_verify = p_sub.add_parser("verify", help="Verify images")
+    p_verify.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_verify.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_verify.add_argument("--group", type=str, help="only handles images from given group")
+    p_verify.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+
+    p_verify.set_defaults(func=_verify)
+
+    # cleanup
+    p_cleanup = p_sub.add_parser("cleanup", help="Cleanup images")
+    p_cleanup.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_cleanup.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_cleanup.add_argument("--group", type=str, help="only handles images from given group")
+    p_cleanup.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+
+    p_cleanup.set_defaults(func=_cleanup)
+
+    # publish
+    p_publish = p_sub.add_parser("publish", help="Publish images")
+    p_publish.add_argument(
+        "--output", type=argparse.FileType("w+"), help="output file path. defaults to stdout", default=sys.stdout
+    )
+    p_publish.add_argument("--config-mapping", type=pathlib.Path, help="the image config template mapping file path")
+    p_publish.add_argument("--group", type=str, help="only handles images from given group")
+    p_publish.add_argument("config", type=pathlib.Path, help="the image configuration file path")
+
+    p_publish.set_defaults(func=_publish)
+
+    return parser
+
+
+def main():
+    parser = _parser()
+    args = parser.parse_args()
+    log_formatter = logging.Formatter("%(asctime)s:%(name)s:%(levelname)s:%(message)s")
+    # log level
+    loglevel = logging.INFO
+    if args.log_level == "debug":
+        loglevel = logging.DEBUG
+    root_logger = logging.getLogger()
+    root_logger.setLevel(loglevel)
+    # log file
+    if args.log_file:
+        file_handler = logging.FileHandler(filename=args.log_file)
+        file_handler.setFormatter(log_formatter)
+        root_logger.addHandler(file_handler)
+    # log console
+    if args.log_console:
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(log_formatter)
+        root_logger.addHandler(console_handler)
+    if "func" not in args:
+        sys.exit(parser.print_help())
+    args.func(args)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

awspub/common.py

@@ -0,0 +1,64 @@
+import logging
+from typing import List, Tuple
+
+import boto3
+from mypy_boto3_ec2.client import EC2Client
+
+logger = logging.getLogger(__name__)
+
+
+def _split_partition(val: str) -> Tuple[str, str]:
+    """
+    Split a string into partition and resource, separated by a colon. If no partition is given, assume "aws"
+    :param val: the string to split
+    :type val: str
+    :return: the partition and the resource
+    :rtype: Tuple[str, str]
+    """
+    if ":" in val:
+        partition, resource = val.split(":")
+    else:
+        # if no partition is given, assume default commercial partition "aws"
+        partition = "aws"
+        resource = val
+    return partition, resource
+
+
+def _get_regions(region_to_query: str, regions_allowlist: List[str]) -> List[str]:
+    """
+    Get a list of region names querying the `region_to_query` for all regions and
+    then filtering by `regions_allowlist`.
+    If no `regions_allowlist` is given, all queried regions are returned for the
+    current partition.
+    If `regions_allowlist` is given, all regions from that list are returned if
+    the listed region exist in the current partition.
+    Eg. `us-east-1` listed in `regions_allowlist` won't be returned if the current
+    partition is `aws-cn`.
+    :param region_to_query: region name of current partition
+    :type region_to_query: str
+    :praram regions_allowlist: list of regions in config file
+    :type regions_allowlist: List[str]
+    :return: list of regions names
+    :rtype: List[str]
+    """
+
+    # get all available regions
+    ec2client: EC2Client = boto3.client("ec2", region_name=region_to_query)
+    resp = ec2client.describe_regions()
+    ec2_regions_all = [r["RegionName"] for r in resp["Regions"]]
+
+    if regions_allowlist:
+        # filter out regions that are not available in the current partition
+        regions_allowlist_set = set(regions_allowlist)
+        ec2_regions_all_set = set(ec2_regions_all)
+        regions = list(regions_allowlist_set.intersection(ec2_regions_all_set))
+        diff = regions_allowlist_set.difference(ec2_regions_all_set)
+        if diff:
+            logger.warning(
+                f"regions {diff} listed in regions allowlist are not available in the current partition."
+                " Ignoring those."
+            )
+    else:
+        regions = ec2_regions_all
+
+    return regions

awspub/configmodels.py

@@ -0,0 +1,216 @@
+import pathlib
+from enum import Enum
+from typing import Dict, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from awspub.common import _split_partition
+
+
+class ConfigS3Model(BaseModel):
+    """
+    S3 configuration.
+    This is required for uploading source files (usually .vmdk) to a bucket so
+    snapshots can be created out of the s3 file
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    bucket_name: str = Field(description="The S3 bucket name")
+
+
+class ConfigSourceModel(BaseModel):
+    """
+    Source configuration.
+    This defines the source (usually .vmdk) that is uploaded
+    to S3 and then used to create EC2 snapshots in different regions.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    path: pathlib.Path = Field(description="Path to a local .vmdk image")
+    architecture: Literal["x86_64", "arm64"] = Field(description="The architecture of the given .vmdk image")
+
+
+class ConfigImageMarketplaceSecurityGroupModel(BaseModel):
+    """
+    Image/AMI Marketplace specific configuration for a security group
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    from_port: int = Field(description="The source port")
+    ip_protocol: Literal["tcp", "udp"] = Field(description="The IP protocol (either 'tcp' or 'udp')")
+    ip_ranges: List[str] = Field(description="IP ranges to allow, in CIDR format (eg. '192.0.2.0/24')")
+    to_port: int = Field(description="The destination port")
+
+
+class ConfigImageMarketplaceModel(BaseModel):
+    """
+    Image/AMI Marketplace specific configuration to request new Marketplace versions
+    See https://docs.aws.amazon.com/marketplace-catalog/latest/api-reference/ami-products.html
+    for further information
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    entity_id: str = Field(description="The entity ID (product ID)")
+    # see https://docs.aws.amazon.com/marketplace/latest/userguide/ami-single-ami-products.html#single-ami-marketplace-ami-access  # noqa:E501
+    access_role_arn: str = Field(
+        description="IAM role Amazon Resource Name (ARN) used by AWS Marketplace to access the provided AMI"
+    )
+    version_title: str = Field(description="The version title. Must be unique across the product")
+    release_notes: str = Field(description="The release notes")
+    user_name: str = Field(description="The login username to access the operating system")
+    scanning_port: int = Field(description="Port to access the operating system (default: 22)", default=22)
+    os_name: str = Field(description="Operating system name displayed to Marketplace buyers")
+    os_version: str = Field(description="Operating system version displayed to Marketplace buyers")
+    usage_instructions: str = Field(
+        description=" Instructions for using the AMI, or a link to more information about the AMI"
+    )
+    recommended_instance_type: str = Field(
+        description="The instance type that is recommended to run the service with the AMI and is the "
+        "default for 1-click installs of your service"
+    )
+    security_groups: Optional[List[ConfigImageMarketplaceSecurityGroupModel]]
+
+
+class ConfigImageSSMParameterModel(BaseModel):
+    """
+    Image/AMI SSM specific configuration to push parameters of type `aws:ec2:image` to the parameter store
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str = Field(
+        description="The fully qualified name of the parameter that you want to add to the system. "
+        "A parameter name must be unique within an Amazon Web Services Region"
+    )
+    description: Optional[str] = Field(
+        description="Information about the parameter that you want to add to the system", default=None
+    )
+    allow_overwrite: Optional[bool] = Field(
+        description="allow to overwrite an existing parameter. Useful for keep a 'latest' parameter (default: false)",
+        default=False,
+    )
+
+
+class SNSNotificationProtocol(str, Enum):
+    DEFAULT = "default"
+    EMAIL = "email"
+
+
+class ConfigImageSNSNotificationModel(BaseModel):
+    """
+    Image/AMI SNS Notification specific configuration to notify subscribers about new images availability
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    subject: str = Field(description="The subject of SNS Notification", min_length=1, max_length=99)
+    message: Dict[SNSNotificationProtocol, str] = Field(
+        description="The body of the message to be sent to subscribers.",
+        default={SNSNotificationProtocol.DEFAULT: ""},
+    )
+    regions: Optional[List[str]] = Field(
+        description="Optional list of regions for sending notification. If not given, regions where the image "
+        "registered will be used from the currently used parition. If a region doesn't exist in the currently "
+        "used partition, it will be ignored.",
+        default=None,
+    )
+
+    @field_validator("message")
+    def check_message(cls, value):
+        # Check message protocols have default key
+        # Message should contain at least a top-level JSON key of “default”
+        # with a value that is a string
+        if SNSNotificationProtocol.DEFAULT not in value:
+            raise ValueError(f"{SNSNotificationProtocol.DEFAULT.value} key is required to send SNS notification")
+        return value
+
+
+class ConfigImageModel(BaseModel):
+    """
+    Image/AMI configuration.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    description: Optional[str] = Field(description="Optional image description", default=None)
+    regions: Optional[List[str]] = Field(
+        description="Optional list of regions for this image. If not given, all available regions will"
+        "be used from the currently used partition. If a region doesn't exist in the currently used partition,"
+        " it will be ignored.",
+        default=None,
+    )
+    separate_snapshot: bool = Field(description="Use a separate snapshot for this image?", default=False)
+    billing_products: Optional[List[str]] = Field(description="Optional list of billing codes", default=None)
+    boot_mode: Literal["legacy-bios", "uefi", "uefi-preferred"] = Field(
+        description="The boot mode. For arm64, this needs to be 'uefi'"
+    )
+    root_device_name: Optional[str] = Field(description="The root device name", default="/dev/sda1")
+    root_device_volume_type: Optional[Literal["gp2", "gp3"]] = Field(
+        description="The root device volume type", default="gp3"
+    )
+    root_device_volume_size: Optional[int] = Field(description="The root device volume size (in GB)", default=8)
+    uefi_data: Optional[pathlib.Path] = Field(
+        description="Optional path to a non-volatile UEFI variable store (must be already base64 encoded)", default=None
+    )
+    tpm_support: Optional[Literal["v2.0"]] = Field(
+        description="Optional TPM support. If this is set, 'boot_mode' must be 'uefi'", default=None
+    )
+    imds_support: Optional[Literal["v2.0"]] = Field(description="Optional IMDS support", default=None)
+    share: Optional[List[str]] = Field(
+        description="Optional list of account IDs the image and snapshot will be shared with. The account"
+        "ID can be prefixed with the partition and separated by ':'. Eg 'aws-cn:123456789123'",
+        default=None,
+    )
+    temporary: Optional[bool] = Field(
+        description="Optional boolean field indicates that a image is only temporary", default=False
+    )
+    public: Optional[bool] = Field(
+        description="Optional boolean field indicates if the image should be public", default=False
+    )
+    marketplace: Optional[ConfigImageMarketplaceModel] = Field(
+        description="Optional structure containing Marketplace related configuration for the commercial "
+        "'aws' partition",
+        default=None,
+    )
+    ssm_parameter: Optional[List[ConfigImageSSMParameterModel]] = Field(
+        description="Optional list of SSM parameter paths of type `aws:ec2:image` which will "
+        "be pushed to the parameter store",
+        default=None,
+    )
+    groups: Optional[List[str]] = Field(description="Optional list of groups this image is part of", default=[])
+    tags: Optional[Dict[str, str]] = Field(description="Optional Tags to apply to this image only", default={})
+    sns: Optional[List[Dict[str, ConfigImageSNSNotificationModel]]] = Field(
+        description="Optional list of SNS Notification related configuration", default=None
+    )
+
+    @field_validator("share")
+    @classmethod
+    def check_share(cls, v: Optional[List[str]]) -> Optional[List[str]]:
+        """
+        Make sure the account IDs are valid and if given the partition is correct
+        """
+        if v is not None:
+            for val in v:
+                partition, account_id = _split_partition(val)
+                if len(account_id) != 12:
+                    raise ValueError("Account ID must be 12 characters long")
+                if partition not in ["aws", "aws-cn", "aws-us-gov"]:
+                    raise ValueError("Partition must be one of 'aws', 'aws-cn', 'aws-us-gov'")
+        return v
+
+
+class ConfigModel(BaseModel):
+    """
+    The base model for the whole configuration
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    s3: ConfigS3Model
+    source: ConfigSourceModel
+    images: Dict[str, ConfigImageModel]
+    tags: Optional[Dict[str, str]] = Field(description="Optional Tags to apply to all resources", default={})