awspub 0.0.10__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- awspub/__init__.py +3 -0
- awspub/api.py +165 -0
- awspub/cli/__init__.py +146 -0
- awspub/common.py +64 -0
- awspub/configmodels.py +216 -0
- awspub/context.py +108 -0
- awspub/exceptions.py +28 -0
- awspub/image.py +656 -0
- awspub/image_marketplace.py +120 -0
- awspub/s3.py +262 -0
- awspub/snapshot.py +241 -0
- awspub/sns.py +105 -0
- awspub/tests/__init__.py +0 -0
- awspub/tests/fixtures/config-invalid-s3-extra.yaml +12 -0
- awspub/tests/fixtures/config-minimal.yaml +12 -0
- awspub/tests/fixtures/config-valid-nonawspub.yaml +13 -0
- awspub/tests/fixtures/config1.vmdk +0 -0
- awspub/tests/fixtures/config1.yaml +171 -0
- awspub/tests/fixtures/config2-mapping.yaml +2 -0
- awspub/tests/fixtures/config2.yaml +48 -0
- awspub/tests/fixtures/config3-duplicate-keys.yaml +18 -0
- awspub/tests/test_api.py +89 -0
- awspub/tests/test_cli.py +0 -0
- awspub/tests/test_common.py +34 -0
- awspub/tests/test_context.py +88 -0
- awspub/tests/test_image.py +556 -0
- awspub/tests/test_image_marketplace.py +44 -0
- awspub/tests/test_s3.py +74 -0
- awspub/tests/test_snapshot.py +122 -0
- awspub/tests/test_sns.py +189 -0
- awspub-0.0.10.dist-info/LICENSE +675 -0
- awspub-0.0.10.dist-info/METADATA +46 -0
- awspub-0.0.10.dist-info/RECORD +35 -0
- awspub-0.0.10.dist-info/WHEEL +4 -0
- awspub-0.0.10.dist-info/entry_points.txt +3 -0
awspub/image.py
ADDED
@@ -0,0 +1,656 @@
|
|
1
|
+
import hashlib
|
2
|
+
import logging
|
3
|
+
from dataclasses import dataclass
|
4
|
+
from enum import Enum
|
5
|
+
from typing import Any, Dict, List, Optional
|
6
|
+
|
7
|
+
import boto3
|
8
|
+
import botocore.exceptions
|
9
|
+
from mypy_boto3_ec2.client import EC2Client
|
10
|
+
from mypy_boto3_ssm import SSMClient
|
11
|
+
|
12
|
+
from awspub import exceptions
|
13
|
+
from awspub.common import _get_regions, _split_partition
|
14
|
+
from awspub.context import Context
|
15
|
+
from awspub.image_marketplace import ImageMarketplace
|
16
|
+
from awspub.s3 import S3
|
17
|
+
from awspub.snapshot import Snapshot
|
18
|
+
from awspub.sns import SNSNotification
|
19
|
+
|
20
|
+
logger = logging.getLogger(__name__)
|
21
|
+
|
22
|
+
|
23
|
+
@dataclass
|
24
|
+
class _ImageInfo:
|
25
|
+
"""
|
26
|
+
Information about a image from EC2
|
27
|
+
"""
|
28
|
+
|
29
|
+
image_id: str
|
30
|
+
snapshot_id: Optional[str]
|
31
|
+
|
32
|
+
|
33
|
+
class ImageVerificationErrors(str, Enum):
|
34
|
+
"""
|
35
|
+
Possible errors for image verification
|
36
|
+
"""
|
37
|
+
|
38
|
+
NOT_EXIST = "image does not exist"
|
39
|
+
STATE_NOT_AVAILABLE = "image not available"
|
40
|
+
ROOT_DEVICE_TYPE = "root device type mismatch"
|
41
|
+
ROOT_DEVICE_VOLUME_TYPE = "root device volume type mismatch"
|
42
|
+
ROOT_DEVICE_VOLUME_SIZE = "root device volume size mismatch"
|
43
|
+
ROOT_DEVICE_SNAPSHOT_NOT_COMPLETE = "root device snapshot not complete"
|
44
|
+
BOOT_MODE = "boot mode mismatch"
|
45
|
+
TAGS = "tags mismatch"
|
46
|
+
TPM_SUPPORT = "tpm support mismatch"
|
47
|
+
IMDS_SUPPORT = "imds support mismatch"
|
48
|
+
BILLING_PRODUCTS = "billing products mismatch"
|
49
|
+
|
50
|
+
|
51
|
+
class Image:
|
52
|
+
"""
|
53
|
+
Handle EC2 Image/AMI API interaction
|
54
|
+
"""
|
55
|
+
|
56
|
+
def __init__(self, context: Context, image_name: str):
|
57
|
+
self._ctx: Context = context
|
58
|
+
self._image_name: str = image_name
|
59
|
+
self._image_regions: List[str] = []
|
60
|
+
self._image_regions_cached: bool = False
|
61
|
+
|
62
|
+
if self._image_name not in self._ctx.conf["images"].keys():
|
63
|
+
raise ValueError(f"image '{self._image_name}' not found in context configuration")
|
64
|
+
|
65
|
+
self._snapshot: Snapshot = Snapshot(context)
|
66
|
+
self._s3: S3 = S3(context)
|
67
|
+
|
68
|
+
def __repr__(self):
|
69
|
+
return f"<{self.__class__} :'{self.image_name}' (snapshot name: {self.snapshot_name})"
|
70
|
+
|
71
|
+
@property
|
72
|
+
def conf(self) -> Dict[str, Any]:
|
73
|
+
"""
|
74
|
+
The configuration for the current image (based on "image_name") from context
|
75
|
+
"""
|
76
|
+
return self._ctx.conf["images"][self._image_name]
|
77
|
+
|
78
|
+
@property
|
79
|
+
def image_name(self) -> str:
|
80
|
+
"""
|
81
|
+
Get the image name
|
82
|
+
"""
|
83
|
+
return self._image_name
|
84
|
+
|
85
|
+
@property
|
86
|
+
def snapshot_name(self) -> str:
|
87
|
+
"""
|
88
|
+
Get the snapshot name which is a sha256 hexdigest
|
89
|
+
|
90
|
+
The snapshot name is the sha256 hexdigest of the source file given in the source->path
|
91
|
+
configuration option.
|
92
|
+
|
93
|
+
if the "separate_snapshot" config option is set to True, the snapshot name is
|
94
|
+
sha256 hexdigest of the source file given in the source->path conf option and then
|
95
|
+
the sha256 hexdigest of the image-name appended and then the sha256 hexdigest
|
96
|
+
calculated of this concatenated string.
|
97
|
+
|
98
|
+
if the "billing_products" config option is set, the snapshot name is
|
99
|
+
sha256 hexdigest of the source file given in the source->path conf option and then
|
100
|
+
the sha256 hexdigest of each entry in the billing_products appended and then the sha256 hexdigest
|
101
|
+
calculated of this concatenated string.
|
102
|
+
|
103
|
+
Note that both options ("separate_snapshot" and "billing_products") can be combined
|
104
|
+
and the snapshot calculation steps would be combined, too.
|
105
|
+
"""
|
106
|
+
s_name = self._ctx.source_sha256
|
107
|
+
if self.conf["separate_snapshot"] is True:
|
108
|
+
s_name += hashlib.sha256(self.image_name.encode("utf-8")).hexdigest()
|
109
|
+
|
110
|
+
if self.conf["billing_products"]:
|
111
|
+
for bp in self.conf["billing_products"]:
|
112
|
+
s_name += hashlib.sha256(bp.encode("utf-8")).hexdigest()
|
113
|
+
|
114
|
+
# in the separate_snapshot and billing_products had no effect, don't do another sha256 of
|
115
|
+
# the source_sha256 to simplify things
|
116
|
+
if s_name == self._ctx.source_sha256:
|
117
|
+
return s_name
|
118
|
+
|
119
|
+
# do a sha256 of the concatenated string
|
120
|
+
return hashlib.sha256(s_name.encode("utf-8")).hexdigest()
|
121
|
+
|
122
|
+
@property
|
123
|
+
def image_regions(self) -> List[str]:
|
124
|
+
"""
|
125
|
+
Get the image regions.
|
126
|
+
"""
|
127
|
+
if not self._image_regions_cached:
|
128
|
+
regions_configured = self.conf["regions"] if "regions" in self.conf else []
|
129
|
+
self._image_regions = _get_regions(self._s3.bucket_region, regions_configured)
|
130
|
+
self._image_regions_cached = True
|
131
|
+
return self._image_regions
|
132
|
+
|
133
|
+
@property
|
134
|
+
def _tags(self):
|
135
|
+
"""
|
136
|
+
Get the tags for this image (common tags + image specific tags)
|
137
|
+
image specific tags override common tags
|
138
|
+
"""
|
139
|
+
tags = []
|
140
|
+
# the common tags
|
141
|
+
tags_dict = self._ctx.tags_dict
|
142
|
+
# the image specific tags
|
143
|
+
tags_dict.update(self.conf.get("tags", {}))
|
144
|
+
for name, value in tags_dict.items():
|
145
|
+
tags.append({"Key": name, "Value": value})
|
146
|
+
return tags
|
147
|
+
|
148
|
+
def _share_list_filtered(self, share_conf: List[str]) -> List[Dict[str, str]]:
|
149
|
+
"""
|
150
|
+
Get a filtered list of share configurations based on the current partition
|
151
|
+
:param share_conf: the share configuration
|
152
|
+
:type share_conf: List[str]
|
153
|
+
:return: a List of share configurations that is usable by modify_image_attribute()
|
154
|
+
:rtype: List[Dict[str, str]]
|
155
|
+
"""
|
156
|
+
# the current partition
|
157
|
+
partition_current = boto3.client("ec2").meta.partition
|
158
|
+
|
159
|
+
share_list: List[Dict[str, str]] = []
|
160
|
+
for share in share_conf:
|
161
|
+
partition, account_id = _split_partition(share)
|
162
|
+
if partition == partition_current:
|
163
|
+
share_list.append({"UserId": account_id})
|
164
|
+
return share_list
|
165
|
+
|
166
|
+
def _share(self, share_conf: List[str], images: Dict[str, _ImageInfo]):
|
167
|
+
"""
|
168
|
+
Share images with accounts
|
169
|
+
|
170
|
+
:param share_conf: the share configuration containing list
|
171
|
+
:type share_conf: List[str]
|
172
|
+
:param images: a Dict with region names as keys and _ImageInfo objects as values
|
173
|
+
:type images: Dict[str, _ImageInfo]
|
174
|
+
"""
|
175
|
+
share_list = self._share_list_filtered(share_conf)
|
176
|
+
|
177
|
+
if not share_list:
|
178
|
+
logger.info("no valid accounts found for sharing in this partition, skipping")
|
179
|
+
return
|
180
|
+
|
181
|
+
for region, image_info in images.items():
|
182
|
+
ec2client: EC2Client = boto3.client("ec2", region_name=region)
|
183
|
+
# modify image permissions
|
184
|
+
ec2client.modify_image_attribute(
|
185
|
+
Attribute="LaunchPermission",
|
186
|
+
ImageId=image_info.image_id,
|
187
|
+
LaunchPermission={"Add": share_list}, # type: ignore
|
188
|
+
)
|
189
|
+
|
190
|
+
# modify snapshot permissions
|
191
|
+
if image_info.snapshot_id:
|
192
|
+
ec2client.modify_snapshot_attribute(
|
193
|
+
Attribute="createVolumePermission",
|
194
|
+
SnapshotId=image_info.snapshot_id,
|
195
|
+
CreateVolumePermission={"Add": share_list}, # type: ignore
|
196
|
+
)
|
197
|
+
|
198
|
+
logger.info(f"shared images & snapshots with '{share_conf}'")
|
199
|
+
|
200
|
+
def _get_root_device_snapshot_id(self, image):
|
201
|
+
"""
|
202
|
+
Get the root device snapshot id for a given image
|
203
|
+
:param image: a image structure returned by eg. describe_images()["Images"][0]
|
204
|
+
:type image: dict
|
205
|
+
:return: Either None or a snapshot-id
|
206
|
+
:rtype: Optional[str]
|
207
|
+
"""
|
208
|
+
root_device_name = image.get("RootDeviceName")
|
209
|
+
if not root_device_name:
|
210
|
+
logger.debug(f"can not get RootDeviceName for image {image}")
|
211
|
+
return None
|
212
|
+
for bdm in image["BlockDeviceMappings"]:
|
213
|
+
if bdm["DeviceName"] == root_device_name:
|
214
|
+
ebs = bdm.get("Ebs")
|
215
|
+
if not ebs:
|
216
|
+
logger.debug(
|
217
|
+
f"can not get RootDeviceName. root device {root_device_name} doesn't have a Ebs section"
|
218
|
+
)
|
219
|
+
return None
|
220
|
+
logger.debug(f"found Ebs for root device {root_device_name}: {bdm['Ebs']}")
|
221
|
+
return bdm["Ebs"]["SnapshotId"]
|
222
|
+
|
223
|
+
def _get(self, ec2client: EC2Client) -> Optional[_ImageInfo]:
|
224
|
+
"""
|
225
|
+
Get the a _ImageInfo for the current image which contains the ami id and
|
226
|
+
root device snapshot id.
|
227
|
+
This relies on the image name to be unique and will raise a MultipleImagesException
|
228
|
+
if multiple images are found.
|
229
|
+
|
230
|
+
:param ec2client: EC2Client
|
231
|
+
:type ec2client: EC2Client
|
232
|
+
:return: Either None or a _ImageInfo
|
233
|
+
:rtype: Optional[_ImageInfo]
|
234
|
+
"""
|
235
|
+
resp = ec2client.describe_images(
|
236
|
+
Filters=[
|
237
|
+
{"Name": "name", "Values": [self.image_name]},
|
238
|
+
],
|
239
|
+
Owners=["self"],
|
240
|
+
)
|
241
|
+
|
242
|
+
if len(resp.get("Images", [])) == 1:
|
243
|
+
root_device_snapshot_id = self._get_root_device_snapshot_id(resp["Images"][0])
|
244
|
+
return _ImageInfo(resp["Images"][0]["ImageId"], root_device_snapshot_id)
|
245
|
+
elif len(resp.get("Images", [])) == 0:
|
246
|
+
return None
|
247
|
+
else:
|
248
|
+
images = [i["ImageId"] for i in resp.get("Images", [])]
|
249
|
+
raise exceptions.MultipleImagesException(
|
250
|
+
f"Found {len(images)} images ({', '.join(images)}) with "
|
251
|
+
f"name {self.image_name} in region {ec2client.meta.region_name}. There should be only 1."
|
252
|
+
)
|
253
|
+
|
254
|
+
def _put_ssm_parameters(self) -> None:
|
255
|
+
"""
|
256
|
+
Push the configured SSM parameters to the parameter store
|
257
|
+
"""
|
258
|
+
logger.info(f"Pushing SSM parameters for image {self.image_name} in {len(self.image_regions)} regions ...")
|
259
|
+
for region in self.image_regions:
|
260
|
+
ec2client_region: EC2Client = boto3.client("ec2", region_name=region)
|
261
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client_region)
|
262
|
+
|
263
|
+
# image in region not found
|
264
|
+
if not image_info:
|
265
|
+
logger.error(f"image {self.image_name} not available in region {region}. can not push SSM parameter")
|
266
|
+
continue
|
267
|
+
|
268
|
+
ssmclient_region: SSMClient = boto3.client("ssm", region_name=region)
|
269
|
+
# iterate over all defined parameters
|
270
|
+
for parameter in self.conf["ssm_parameter"]:
|
271
|
+
# if overwrite is not allowed, check if the parameter is already there and if so, do nothing
|
272
|
+
if not parameter["allow_overwrite"]:
|
273
|
+
resp = ssmclient_region.get_parameters(Names=[parameter["name"]])
|
274
|
+
if len(resp["Parameters"]) >= 1:
|
275
|
+
# sanity check if the available parameter matches the value we would (but don't) push
|
276
|
+
if resp["Parameters"][0]["Value"] != image_info.image_id:
|
277
|
+
logger.warning(
|
278
|
+
f"SSM parameter {parameter['name']} exists but value does not match "
|
279
|
+
f"(found {resp['Parameters'][0]['Value']}; expected: {image_info.image_id}"
|
280
|
+
)
|
281
|
+
# parameter exists already and overwrite is not allowed so continue
|
282
|
+
continue
|
283
|
+
# push parameter to store
|
284
|
+
ssmclient_region.put_parameter(
|
285
|
+
Name=parameter["name"],
|
286
|
+
Description=parameter.get("description", ""),
|
287
|
+
Value=image_info.image_id,
|
288
|
+
Type="String",
|
289
|
+
Overwrite=parameter["allow_overwrite"],
|
290
|
+
DataType="aws:ec2:image",
|
291
|
+
# TODO: tags can't be used together with overwrite
|
292
|
+
# Tags=self._ctx.tags,
|
293
|
+
)
|
294
|
+
|
295
|
+
logger.info(
|
296
|
+
f"pushed SSM parameter {parameter['name']} with value {image_info.image_id} in region {region}"
|
297
|
+
)
|
298
|
+
|
299
|
+
def _public(self) -> None:
|
300
|
+
"""
|
301
|
+
Make image and underlying root device snapshot public
|
302
|
+
"""
|
303
|
+
logger.info(f"Make image {self.image_name} in {len(self.image_regions)} regions public ...")
|
304
|
+
|
305
|
+
for region in self.image_regions:
|
306
|
+
ec2client_region: EC2Client = boto3.client("ec2", region_name=region)
|
307
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client_region)
|
308
|
+
if image_info:
|
309
|
+
ec2client_region.modify_image_attribute(
|
310
|
+
ImageId=image_info.image_id,
|
311
|
+
LaunchPermission={
|
312
|
+
"Add": [
|
313
|
+
{
|
314
|
+
"Group": "all",
|
315
|
+
},
|
316
|
+
],
|
317
|
+
},
|
318
|
+
)
|
319
|
+
logger.info(f"image {image_info.image_id} in region {region} public now")
|
320
|
+
|
321
|
+
if image_info.snapshot_id:
|
322
|
+
ec2client_region.modify_snapshot_attribute(
|
323
|
+
SnapshotId=image_info.snapshot_id,
|
324
|
+
Attribute="createVolumePermission",
|
325
|
+
GroupNames=[
|
326
|
+
"all",
|
327
|
+
],
|
328
|
+
OperationType="add",
|
329
|
+
)
|
330
|
+
logger.info(
|
331
|
+
f"snapshot {image_info.snapshot_id} ({image_info.image_id}) in region {region} public now"
|
332
|
+
)
|
333
|
+
else:
|
334
|
+
logger.error(
|
335
|
+
f"snapshot for image {self.image_name} ({image_info.image_id}) not available "
|
336
|
+
f"in region {region}. can not make public"
|
337
|
+
)
|
338
|
+
else:
|
339
|
+
logger.error(f"image {self.image_name} not available in region {region}. can not make public")
|
340
|
+
|
341
|
+
def _sns_publish(self) -> None:
|
342
|
+
"""
|
343
|
+
Publish SNS notifiations about newly available images to subscribers
|
344
|
+
"""
|
345
|
+
|
346
|
+
SNSNotification(self._ctx, self.image_name).publish()
|
347
|
+
|
348
|
+
def cleanup(self) -> None:
|
349
|
+
"""
|
350
|
+
Cleanup/delete the temporary images
|
351
|
+
|
352
|
+
If an image is marked as "temporary" in the configuration, do
|
353
|
+
delete that image in all regions.
|
354
|
+
Note: if a temporary image is public, it won't be deleted. A temporary
|
355
|
+
image should never be public
|
356
|
+
Note: the underlying snapshot is currently not deleted. That might change in
|
357
|
+
the future
|
358
|
+
"""
|
359
|
+
if not self.conf["temporary"]:
|
360
|
+
logger.info(f"image {self.image_name} not marked as temporary. no cleanup")
|
361
|
+
return
|
362
|
+
|
363
|
+
# do the cleanup - the image is marked as temporary
|
364
|
+
logger.info(f"Cleanup image {self.image_name} ...")
|
365
|
+
for region in self.image_regions:
|
366
|
+
ec2client_region: EC2Client = boto3.client("ec2", region_name=region)
|
367
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client_region)
|
368
|
+
|
369
|
+
if image_info:
|
370
|
+
resp = ec2client_region.describe_images(
|
371
|
+
Filters=[
|
372
|
+
{"Name": "image-id", "Values": [image_info.image_id]},
|
373
|
+
]
|
374
|
+
)
|
375
|
+
if resp["Images"][0]["Public"] is True:
|
376
|
+
# this shouldn't happen because the image is marked as temporary in the config
|
377
|
+
# so how can it be public?
|
378
|
+
logger.error(
|
379
|
+
f"no cleanup for {self.image_name} in {region} because ({image_info.image_id}) image is public"
|
380
|
+
)
|
381
|
+
else:
|
382
|
+
ec2client_region.deregister_image(ImageId=image_info.image_id)
|
383
|
+
logger.info(f"{self.image_name} in {region} ({image_info.image_id}) deleted")
|
384
|
+
|
385
|
+
def list(self) -> Dict[str, _ImageInfo]:
|
386
|
+
"""
|
387
|
+
Get image based on the available configuration
|
388
|
+
This doesn't change anything - it just tries to get the available image
|
389
|
+
for the different configured regions
|
390
|
+
:return: a Dict with region names as keys and _ImageInfo objects as values
|
391
|
+
:rtype: Dict[str, _ImageInfo]
|
392
|
+
"""
|
393
|
+
images: Dict[str, _ImageInfo] = dict()
|
394
|
+
for region in self.image_regions:
|
395
|
+
ec2client_region: EC2Client = boto3.client("ec2", region_name=region)
|
396
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client_region)
|
397
|
+
if image_info:
|
398
|
+
images[region] = image_info
|
399
|
+
else:
|
400
|
+
logger.warning(f"image {self.image_name} not available in region {region}")
|
401
|
+
return images
|
402
|
+
|
403
|
+
def create(self) -> Dict[str, _ImageInfo]:
|
404
|
+
"""
|
405
|
+
Get or create a image based on the available configuration
|
406
|
+
|
407
|
+
:return: a Dict with region names as keys and _ImageInfo objects as values
|
408
|
+
:rtype: Dict[str, _ImageInfo]
|
409
|
+
"""
|
410
|
+
# this **must** be the region that is used for S3
|
411
|
+
ec2client: EC2Client = boto3.client("ec2", region_name=self._s3.bucket_region)
|
412
|
+
|
413
|
+
# make sure the initial snapshot exists
|
414
|
+
self._snapshot.create(ec2client, self.snapshot_name)
|
415
|
+
|
416
|
+
# make sure the snapshot exist in all required regions
|
417
|
+
snapshot_ids: Dict[str, str] = self._snapshot.copy(
|
418
|
+
self.snapshot_name, self._s3.bucket_region, self.image_regions
|
419
|
+
)
|
420
|
+
|
421
|
+
images: Dict[str, _ImageInfo] = dict()
|
422
|
+
missing_regions: List[str] = []
|
423
|
+
for region in self.image_regions:
|
424
|
+
ec2client_region: EC2Client = boto3.client("ec2", region_name=region)
|
425
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client_region)
|
426
|
+
if image_info:
|
427
|
+
if image_info.snapshot_id != snapshot_ids[region]:
|
428
|
+
logger.warning(
|
429
|
+
f"image with name '{self.image_name}' already exists ({image_info.image_id}) "
|
430
|
+
f"in region {ec2client_region.meta.region_name} but the root device "
|
431
|
+
f"snapshot id is unexpected (got {image_info.snapshot_id} but expected {snapshot_ids[region]})"
|
432
|
+
)
|
433
|
+
else:
|
434
|
+
logger.info(
|
435
|
+
f"image with name '{self.image_name}' already exists ({image_info.image_id}) "
|
436
|
+
f"in region {ec2client_region.meta.region_name}"
|
437
|
+
)
|
438
|
+
images[region] = image_info
|
439
|
+
else:
|
440
|
+
if image := self._register_image(snapshot_ids[region], ec2client_region):
|
441
|
+
images[region] = image
|
442
|
+
else:
|
443
|
+
missing_regions.append(region)
|
444
|
+
# wait for the images
|
445
|
+
logger.info(f"Waiting for {len(images)} images to be ready the regions ...")
|
446
|
+
for region, image_info in images.items():
|
447
|
+
ec2client_region_wait: EC2Client = boto3.client("ec2", region_name=region)
|
448
|
+
logger.info(
|
449
|
+
f"Waiting for {image_info.image_id} in {ec2client_region_wait.meta.region_name} "
|
450
|
+
"to exist/be available ..."
|
451
|
+
)
|
452
|
+
waiter_exists = ec2client_region_wait.get_waiter("image_exists")
|
453
|
+
waiter_exists.wait(ImageIds=[image_info.image_id])
|
454
|
+
waiter_available = ec2client_region_wait.get_waiter("image_available")
|
455
|
+
waiter_available.wait(ImageIds=[image_info.image_id])
|
456
|
+
logger.info(f"{len(images)} images are ready")
|
457
|
+
|
458
|
+
# share
|
459
|
+
if self.conf["share"]:
|
460
|
+
self._share(self.conf["share"], images)
|
461
|
+
|
462
|
+
if missing_regions:
|
463
|
+
logger.error("Failed to publish images to all regions", extra={"missing_regions": missing_regions})
|
464
|
+
raise exceptions.IncompleteImageSetException("Incomplete image set published")
|
465
|
+
|
466
|
+
return images
|
467
|
+
|
468
|
+
def _register_image(self, snapshot_id: str, ec2client: EC2Client) -> Optional[_ImageInfo]:
|
469
|
+
"""
|
470
|
+
Register snapshot_id in region configured for ec2client_region
|
471
|
+
|
472
|
+
:param snapshot_id: snapshot id to use for image registration
|
473
|
+
:type snapshot_id: str
|
474
|
+
:param ec2client: EC2Client for the region to register image to
|
475
|
+
:type ec2client: EC2Client
|
476
|
+
:return: _ImageInfo containing the ImageId SnapshotId pair
|
477
|
+
:rtype: _ImageInfo
|
478
|
+
"""
|
479
|
+
logger.info(f"creating image with name '{self.image_name}' in " f"region {ec2client.meta.region_name} ...")
|
480
|
+
|
481
|
+
register_image_kwargs = dict(
|
482
|
+
Name=self.image_name,
|
483
|
+
Description=self.conf.get("description", ""),
|
484
|
+
Architecture=self._ctx.conf["source"]["architecture"],
|
485
|
+
RootDeviceName=self.conf["root_device_name"],
|
486
|
+
BlockDeviceMappings=[
|
487
|
+
{
|
488
|
+
"Ebs": {
|
489
|
+
"SnapshotId": snapshot_id,
|
490
|
+
"VolumeType": self.conf["root_device_volume_type"],
|
491
|
+
"VolumeSize": self.conf["root_device_volume_size"],
|
492
|
+
},
|
493
|
+
"DeviceName": self.conf["root_device_name"],
|
494
|
+
},
|
495
|
+
# TODO: make those ephemeral block device mappings configurable
|
496
|
+
{"VirtualName": "ephemeral0", "DeviceName": "/dev/sdb"},
|
497
|
+
{"VirtualName": "ephemeral1", "DeviceName": "/dev/sdc"},
|
498
|
+
],
|
499
|
+
EnaSupport=True,
|
500
|
+
SriovNetSupport="simple",
|
501
|
+
VirtualizationType="hvm",
|
502
|
+
BootMode=self.conf["boot_mode"],
|
503
|
+
)
|
504
|
+
|
505
|
+
if self.conf["tpm_support"]:
|
506
|
+
register_image_kwargs["TpmSupport"] = self.conf["tpm_support"]
|
507
|
+
|
508
|
+
if self.conf["imds_support"]:
|
509
|
+
register_image_kwargs["ImdsSupport"] = self.conf["imds_support"]
|
510
|
+
|
511
|
+
if self.conf["uefi_data"]:
|
512
|
+
with open(self.conf["uefi_data"], "r") as f:
|
513
|
+
uefi_data = f.read()
|
514
|
+
register_image_kwargs["UefiData"] = uefi_data
|
515
|
+
|
516
|
+
if self.conf["billing_products"]:
|
517
|
+
register_image_kwargs["BillingProducts"] = self.conf["billing_products"]
|
518
|
+
|
519
|
+
try:
|
520
|
+
resp = ec2client.register_image(**register_image_kwargs)
|
521
|
+
except botocore.exceptions.ClientError as e:
|
522
|
+
if e.response.get("Error", {}).get("Code", None) == "OperationNotPermitted":
|
523
|
+
logger.exception(
|
524
|
+
"Unable to register image",
|
525
|
+
extra={
|
526
|
+
"registration_options": register_image_kwargs,
|
527
|
+
"region": ec2client.meta.region_name,
|
528
|
+
},
|
529
|
+
)
|
530
|
+
return None
|
531
|
+
raise e
|
532
|
+
|
533
|
+
ec2client.create_tags(Resources=[resp["ImageId"]], Tags=self._tags)
|
534
|
+
return _ImageInfo(resp["ImageId"], snapshot_id)
|
535
|
+
|
536
|
+
def publish(self) -> None:
|
537
|
+
"""
|
538
|
+
Handle all publication steps
|
539
|
+
- make image and underlying root device snapshot public if the public flag is set
|
540
|
+
- request a new marketplace version for the image in us-east-1 if the marketplace config is present
|
541
|
+
Note: if the temporary flag is set in the image, this method will do nothing
|
542
|
+
Note: this command doesn't unpublish anything!
|
543
|
+
"""
|
544
|
+
# never publish temporary images
|
545
|
+
if self.conf["temporary"]:
|
546
|
+
logger.warning(f"image {self.image_name} marked as temporary. do not publish")
|
547
|
+
return
|
548
|
+
|
549
|
+
# make snapshot and image public if requested in the image
|
550
|
+
if self.conf["public"]:
|
551
|
+
self._public()
|
552
|
+
else:
|
553
|
+
logger.info(f"image {self.image_name} not marked as public. do not publish")
|
554
|
+
|
555
|
+
# handle SSM parameter store
|
556
|
+
if self.conf["ssm_parameter"]:
|
557
|
+
self._put_ssm_parameters()
|
558
|
+
|
559
|
+
# handle marketplace publication
|
560
|
+
if self.conf["marketplace"]:
|
561
|
+
# the "marketplace" configuration is only valid in the "aws" partition
|
562
|
+
partition = boto3.client("ec2").meta.partition
|
563
|
+
if partition == "aws":
|
564
|
+
logger.info(f"marketplace version request for {self.image_name}")
|
565
|
+
# image needs to be in us-east-1
|
566
|
+
ec2client: EC2Client = boto3.client("ec2", region_name="us-east-1")
|
567
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client)
|
568
|
+
if image_info:
|
569
|
+
im = ImageMarketplace(self._ctx, self.image_name)
|
570
|
+
im.request_new_version(image_info.image_id)
|
571
|
+
else:
|
572
|
+
logger.error(
|
573
|
+
f"can not request marketplace version for {self.image_name} because no image found in us-east-1"
|
574
|
+
)
|
575
|
+
else:
|
576
|
+
logger.info(
|
577
|
+
f"found marketplace config for {self.image_name} and partition 'aws' but "
|
578
|
+
f"currently using partition {partition}. Ignoring marketplace config."
|
579
|
+
)
|
580
|
+
|
581
|
+
# send ssn notification
|
582
|
+
if self.conf["sns"]:
|
583
|
+
self._sns_publish()
|
584
|
+
|
585
|
+
def _verify(self, region: str) -> List[ImageVerificationErrors]:
|
586
|
+
"""
|
587
|
+
Verify (but don't modify or create anything) the image in a single region
|
588
|
+
"""
|
589
|
+
problems: List[ImageVerificationErrors] = []
|
590
|
+
ec2client_region: EC2Client = boto3.client("ec2", region_name=region)
|
591
|
+
image_info: Optional[_ImageInfo] = self._get(ec2client_region)
|
592
|
+
|
593
|
+
if not image_info:
|
594
|
+
problems.append(ImageVerificationErrors.NOT_EXIST)
|
595
|
+
return problems
|
596
|
+
|
597
|
+
image_aws = ec2client_region.describe_images(ImageIds=[image_info.image_id])["Images"][0]
|
598
|
+
|
599
|
+
# verify state
|
600
|
+
if image_aws["State"] != "available":
|
601
|
+
problems.append(ImageVerificationErrors.STATE_NOT_AVAILABLE)
|
602
|
+
|
603
|
+
# verify RootDeviceType
|
604
|
+
if image_aws["RootDeviceType"] != "ebs":
|
605
|
+
problems.append(ImageVerificationErrors.ROOT_DEVICE_TYPE)
|
606
|
+
|
607
|
+
# verify BootMode
|
608
|
+
if image_aws["BootMode"] != self.conf["boot_mode"]:
|
609
|
+
problems.append(ImageVerificationErrors.BOOT_MODE)
|
610
|
+
|
611
|
+
# verify RootDeviceVolumeType, RootDeviceVolumeSize and Snapshot
|
612
|
+
for bdm in image_aws["BlockDeviceMappings"]:
|
613
|
+
if bdm.get("DeviceName") and bdm["DeviceName"] == image_aws["RootDeviceName"]:
|
614
|
+
# here's the root device
|
615
|
+
if bdm["Ebs"]["VolumeType"] != self.conf["root_device_volume_type"]:
|
616
|
+
problems.append(ImageVerificationErrors.ROOT_DEVICE_VOLUME_TYPE)
|
617
|
+
if bdm["Ebs"]["VolumeSize"] != self.conf["root_device_volume_size"]:
|
618
|
+
problems.append(ImageVerificationErrors.ROOT_DEVICE_VOLUME_SIZE)
|
619
|
+
|
620
|
+
# verify snapshot
|
621
|
+
snapshot_aws = ec2client_region.describe_snapshots(SnapshotIds=[bdm["Ebs"]["SnapshotId"]])["Snapshots"][
|
622
|
+
0
|
623
|
+
]
|
624
|
+
if snapshot_aws["State"] != "completed":
|
625
|
+
problems.append(ImageVerificationErrors.ROOT_DEVICE_SNAPSHOT_NOT_COMPLETE)
|
626
|
+
|
627
|
+
# verify tpm support
|
628
|
+
if self.conf["tpm_support"] and image_aws.get("TpmSupport") != self.conf["tpm_support"]:
|
629
|
+
problems.append(ImageVerificationErrors.TPM_SUPPORT)
|
630
|
+
|
631
|
+
# verify imds support
|
632
|
+
if self.conf["imds_support"] and image_aws.get("ImdsSupport") != self.conf["imds_support"]:
|
633
|
+
problems.append(ImageVerificationErrors.IMDS_SUPPORT)
|
634
|
+
|
635
|
+
# billing products
|
636
|
+
if self.conf["billing_products"] and image_aws.get("BillingProducts") != self.conf["billing_products"]:
|
637
|
+
problems.append(ImageVerificationErrors.BILLING_PRODUCTS)
|
638
|
+
|
639
|
+
# verify tags
|
640
|
+
for tag in image_aws["Tags"]:
|
641
|
+
if tag["Key"] == "Name" and tag["Value"] != self.snapshot_name:
|
642
|
+
problems.append(ImageVerificationErrors.TAGS)
|
643
|
+
|
644
|
+
return problems
|
645
|
+
|
646
|
+
def verify(self) -> Dict[str, List[ImageVerificationErrors]]:
|
647
|
+
"""
|
648
|
+
Verify (but don't modify or create anything) that the image configuration
|
649
|
+
matches what is on AWS
|
650
|
+
"""
|
651
|
+
logger.info(f"Verifying image {self.image_name} ...")
|
652
|
+
problems: Dict[str, List[ImageVerificationErrors]] = dict()
|
653
|
+
for region in self.image_regions:
|
654
|
+
problems[region] = self._verify(region)
|
655
|
+
|
656
|
+
return problems
|