cs-image-system-aws-runtime-plugin 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cs_image_system_aws_runtime_plugin-0.1.3/README.md → cs_image_system_aws_runtime_plugin-0.1.5/PKG-INFO +100 -42
- cs_image_system_aws_runtime_plugin-0.1.3/PKG-INFO → cs_image_system_aws_runtime_plugin-0.1.5/README.md +80 -61
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/pyproject.toml +4 -3
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/aws_packer_source.py +1 -1
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/aws_provider_specific_image.py +10 -13
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/aws_runtime_builders.py +14 -7
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/aws_runtime_models.py +4 -8
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/aws_utils.py +66 -82
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/bootstrap.py +107 -5
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/main.py +1 -2
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/.gitignore +0 -0
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/LICENSE +0 -0
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/aws_runtime/__init__.py +0 -0
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/src/cs_image_system/py.typed +0 -0
- {cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/tests/test_aws_provider_specific_image.py +0 -0
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: cs-image-system-aws-runtime-plugin
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: The aws runtime type: Amazon EC2 as a place where images bake and instances run
|
|
5
|
+
Project-URL: Repository, https://github.com/infrastructurebuilder/cs-image-system-3
|
|
6
|
+
Project-URL: Documentation, https://github.com/infrastructurebuilder/cs-image-system-3/blob/develop/docs/OPERATIONS.md
|
|
7
|
+
Project-URL: Package README, https://github.com/infrastructurebuilder/cs-image-system-3/blob/develop/packages/aws-runtime-plugin/README.md
|
|
8
|
+
Author-email: Mykel Alvis <mykelalvis@infrastructurebuilder.org>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Python: >=3.13
|
|
12
|
+
Requires-Dist: boto3>=1.43
|
|
13
|
+
Requires-Dist: botocore>=1.43
|
|
14
|
+
Requires-Dist: cs-image-system-base==0.1.5
|
|
15
|
+
Requires-Dist: cs-image-system-contract==0.1.5
|
|
16
|
+
Requires-Dist: cs-image-system-hashicorp-utils==0.1.5
|
|
17
|
+
Requires-Dist: pydantic>=2.13
|
|
18
|
+
Requires-Dist: python-hcl2>=8.1
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
1
21
|
# cs-image-system-aws-runtime-plugin
|
|
2
22
|
|
|
3
23
|
The `aws` runtime type. A runtime is where images are baked and instances
|
|
@@ -56,9 +76,13 @@ Present in the package but not registered as a service:
|
|
|
56
76
|
cs_image_system.aws_runtime.bootstrap:aws_section`. It owns the questions
|
|
57
77
|
and the terraform for this cloud's one-time initialisation -- the GitHub
|
|
58
78
|
OIDC identity provider, the READ-ONLY and WRITE roles of CI_SETUP.md 3.3
|
|
59
|
-
with their trust and permission documents, and the state
|
|
60
|
-
|
|
61
|
-
|
|
79
|
+
with their trust and permission documents, and the state buckets (the
|
|
80
|
+
default backend's, and since hygiene XX every other declared S3
|
|
81
|
+
backend's that a root is bound to) and the Session Manager instance
|
|
82
|
+
profile when the account lacks them (`tfmodules/bootstrap_aws`). The
|
|
83
|
+
section renders its coverage audit for the generated README: what it
|
|
84
|
+
made, read, adopted, or leaves by hand and why (a backend bound to no
|
|
85
|
+
root is left out and said so). The section is wanted by default where the
|
|
62
86
|
tree declares an AWS runtime or an S3 state backend. Its defaults come
|
|
63
87
|
from the raw `cfg/runtime-builders.yml` and `cfg/state-backends.yml`
|
|
64
88
|
(read as YAML, never loaded; a starter's `REPLACE-ME` values and its
|
|
@@ -287,27 +311,64 @@ included).
|
|
|
287
311
|
|
|
288
312
|
`query_provider_image()` takes one OS-builder runtime entry
|
|
289
313
|
(`OSBuilderBaseImageBuilderSubconfig` in
|
|
290
|
-
[os_builder_runtime_config.py](../base/src/cs_image_system/base/models/os_builder_runtime_config.py))
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
-
|
|
297
|
-
|
|
314
|
+
[os_builder_runtime_config.py](../base/src/cs_image_system/base/models/os_builder_runtime_config.py))
|
|
315
|
+
and asks it for its effective query, `entry.image_query()`: an
|
|
316
|
+
`ImageQuery` in the SYSTEM's vocabulary
|
|
317
|
+
([contract/image.py](../contract/src/cs_image_system/contract/image.py);
|
|
318
|
+
stage 106). That query is the image-source catalog's `aws` block for the
|
|
319
|
+
OS builder's family (the release's layer under the tree's
|
|
320
|
+
`cfg/image-sources*.yml`), with the OS builder's own `query` landing on it
|
|
321
|
+
key by key and the runtime entry's landing on that; the publishers declared
|
|
322
|
+
anywhere (the OS builder's `owners`, this runtime's `default_owners`, the
|
|
323
|
+
entry's `owners`, in that order, deduplicated) replace the catalog's. The
|
|
324
|
+
entry's own `query` mapping is never rewritten (stage 63). The builder then
|
|
325
|
+
translates it with `translate_image_query()` (the table below) and
|
|
298
326
|
`query_image()` in [aws_utils.py](src/cs_image_system/aws_runtime/aws_utils.py)
|
|
299
|
-
runs
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
`
|
|
305
|
-
|
|
306
|
-
`
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
327
|
+
runs the `DescribeImages` request `describe_images_request()` makes from
|
|
328
|
+
it, paging through the results, and returns the newest by `CreationDate`,
|
|
329
|
+
or `None` when nothing matches; with `newest: false` more than one match
|
|
330
|
+
raises `AMIQueryError` as ambiguous. The builder reads the owner straight
|
|
331
|
+
from the AMI record, `ImageOwnerAlias` (for example `amazon`) else
|
|
332
|
+
`OwnerId` (the account id), and returns `(ImageId, owner, raw)`; the base
|
|
333
|
+
registers an `AwsProviderSpecificImage` in the resolved state. A record
|
|
334
|
+
with neither field raises `Could not get owner alias or owner id ...`. No
|
|
335
|
+
match returns `None`, and the base's resolve step then fails the run.
|
|
336
|
+
`validate` has already refused an entry whose query uses a key this
|
|
337
|
+
runtime cannot translate, or has no source at all, and has printed every
|
|
338
|
+
entry's query in both vocabularies (`image source: OS builder '<name>' on
|
|
339
|
+
runtime <rt> (aws): <system query> -> <EC2 query>`).
|
|
340
|
+
|
|
341
|
+
Before stage 106 the entry's `query.filters` were EC2's field names in
|
|
342
|
+
snake case (the system started on AWS and used its filter keys: the
|
|
343
|
+
operator's incidental choice), passed through a key map, with unknown keys
|
|
344
|
+
going to a post-query exact-match filter. The vocabulary is the system's
|
|
345
|
+
now, EC2's spellings kept where they were good names; the old spellings
|
|
346
|
+
still read (`root_device_type`, `virtualization_type`, `owners`, the
|
|
347
|
+
nested `filters:` form), and an unknown key is refused by name instead of
|
|
348
|
+
buried.
|
|
349
|
+
|
|
350
|
+
### Image query translation
|
|
351
|
+
|
|
352
|
+
What `translate_image_query()` does with each key of the system's
|
|
353
|
+
vocabulary (`supported_query_keys()` is the set of rows not marked
|
|
354
|
+
refused; [tests/test_docs_image_sources.py](../../tests/test_docs_image_sources.py)
|
|
355
|
+
holds this table to it). The answer is `{"owners": [...], "filters":
|
|
356
|
+
{<EC2 filter name>: value}, "most_recent": bool}`: what packer's
|
|
357
|
+
`data "amazon-ami"` body and `describe_images` both take.
|
|
358
|
+
|
|
359
|
+
| System key | EC2 filter | Note |
|
|
360
|
+
| --- | --- | --- |
|
|
361
|
+
| `name` | `name` | a glob over the AMI name |
|
|
362
|
+
| `family` | refused | EC2 has no image families; name the image |
|
|
363
|
+
| `version` | refused | the catalog fills `{version}` into `name` |
|
|
364
|
+
| `architecture` | `architecture` | `x86_64` or `arm64`, EC2's own words |
|
|
365
|
+
| `publisher` | `owners` | account ids and the aliases `self`, `amazon`, `aws-marketplace` |
|
|
366
|
+
| `state` | `state` | `available` when unset: the query always forced it |
|
|
367
|
+
| `virtualization` | `virtualization-type` | `hvm` |
|
|
368
|
+
| `root_device` | `root-device-type` | `ebs` |
|
|
369
|
+
| `tags` | `tag:<key>` | one filter per tag |
|
|
370
|
+
| `newest` | `most_recent` | true when unset; false refuses more than one match |
|
|
371
|
+
| `native` | verbatim | each key an EC2 filter name, passed as written |
|
|
311
372
|
|
|
312
373
|
When the entry declares `image_id` (stage 63), the base's resolve step
|
|
313
374
|
does not call `query_provider_image()` at all: it calls
|
|
@@ -315,29 +376,26 @@ does not call `query_provider_image()` at all: it calls
|
|
|
315
376
|
with `ImageIds=[<id>]` through `_query_by_ami_id` in
|
|
316
377
|
[aws_utils.py](src/cs_image_system/aws_runtime/aws_utils.py), with the
|
|
317
378
|
runtime's own client configuration (profile and region). The entry's
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
`InvalidImageID.
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
the query takes the newest match.
|
|
328
|
-
|
|
329
|
-
Stage 63 also removed helpers from
|
|
330
|
-
[aws_utils.py](src/cs_image_system/aws_runtime/aws_utils.py) that nothing
|
|
331
|
-
called: `get_ami_owner`, `get_ami_ssh_user`, `remap_for_aws`,
|
|
332
|
-
`_query_by_name` and `_build_ami_filters`. `_query_by_ami_id` (an exact
|
|
333
|
-
lookup by AMI id) was kept, and since stage 63 it is what
|
|
334
|
-
`query_provider_image_by_id()` calls.
|
|
379
|
+
query is not consulted. The owner is read from the record the same way,
|
|
380
|
+
`ImageOwnerAlias` else `OwnerId` (else `self`), and the result is
|
|
381
|
+
`(ImageId, owner, raw)`. An id AWS answers `InvalidImageID.NotFound` or
|
|
382
|
+
`InvalidImageID.Malformed` for, or one that returns no image, gives
|
|
383
|
+
`None`, and the base stops resolution with `OS builder <name>: image_id
|
|
384
|
+
'<id>' on runtime <rt> is not known to the provider`; any other API
|
|
385
|
+
refusal raises `AMIQueryError`. An AMI id is regional, so a pinned id must
|
|
386
|
+
exist in this runtime's region. Pinning makes the base bake reproducible:
|
|
387
|
+
the same vendor AMI every time, where the query takes the newest match.
|
|
335
388
|
|
|
336
389
|
`AwsProviderSpecificImage.get_query_assets()` gives the packer
|
|
337
390
|
`data "amazon-ami"` body:
|
|
338
391
|
|
|
339
392
|
- resolved: `filters = { image-id = "<ami>" }` plus `owners = ["<owner>"]` when known;
|
|
340
|
-
- deferred (an image a build produces later): `
|
|
393
|
+
- deferred (an image a build produces later): an `ImageQuery` (`name =
|
|
394
|
+
"<pattern>*"`, `root_device = ebs`, `virtualization = hvm`, the
|
|
395
|
+
architecture or `x86_64`, publisher `self`) translated like a vendor
|
|
396
|
+
query and quoted: `owners = ["self"]`, `filters = { name = "<pattern>*",
|
|
397
|
+
root-device-type = "ebs", virtualization-type = "hvm", architecture =
|
|
398
|
+
"<arch>" }`, `most_recent = true`.
|
|
341
399
|
|
|
342
400
|
[tests/test_aws_provider_specific_image.py](tests/test_aws_provider_specific_image.py)
|
|
343
401
|
pins both shapes.
|
|
@@ -1120,7 +1178,7 @@ when the installed version is below it.
|
|
|
1120
1178
|
- [default-os-plugin](../default-os-plugin/README.md): the OS builders whose runtime entries this plugin resolves to AMIs.
|
|
1121
1179
|
- [packer-plugin](../packer-plugin/README.md): the image builder that asks this plugin for the `amazon-ebs` source and the session provisioners.
|
|
1122
1180
|
- [tf-ebs-instance-plugin](../tf-ebs-instance-plugin/README.md): the instance and storage roots that read this model's networking, profile and session profile.
|
|
1123
|
-
- [dummy-plugin](../dummy-plugin/README.md): the extension template
|
|
1181
|
+
- [dummy-plugin](../dummy-plugin/README.md): the extension template (an unregistered copy of its group model sat at the bottom of this package's models module until stage 63).
|
|
1124
1182
|
- Base classes: [runtime.py](../base/src/cs_image_system/base/models/runtime.py), [cloud_builder.py](../base/src/cs_image_system/base/models/cloud_builder.py), [credentials.py](../base/src/cs_image_system/base/models/credentials.py), [provider_specific_image.py](../base/src/cs_image_system/base/models/provider_specific_image.py), [builder_base_runtime.py](../base/src/cs_image_system/base/basic/builder_base_runtime.py), [abstract_version_checker.py](../base/src/cs_image_system/base/basic/abstract_version_checker.py).
|
|
1125
1183
|
- Consumers of the hooks: [state_query.py](../base/src/cs_image_system/base/state_query.py), [retention.py](../base/src/cs_image_system/base/retention.py), [dispose.py](../base/src/cs_image_system/base/commands/dispose.py), [verify_instance.py](../base/src/cs_image_system/base/commands/verify_instance.py), [unmount.py](../base/src/cs_image_system/base/commands/unmount.py), [relabel.py](../base/src/cs_image_system/base/commands/relabel.py), [release.py](../base/src/cs_image_system/base/release.py), [power_state.py](../base/src/cs_image_system/base/power_state.py), [provider_aliases.py](../base/src/cs_image_system/base/provider_aliases.py), [runtime_facts.py](../base/src/cs_image_system/base/commands/runtime_facts.py).
|
|
1126
1184
|
- [docs/OPERATIONS.md](../../docs/OPERATIONS.md) for the credential contract and the operator's cycles; [docs/DESIGN.md](../../docs/DESIGN.md) for the design; [docs/PLUGINS.md](../../docs/PLUGINS.md) for the package index.
|
|
@@ -1,22 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.5
|
|
2
|
-
Name: cs-image-system-aws-runtime-plugin
|
|
3
|
-
Version: 0.1.3
|
|
4
|
-
Summary: The aws runtime type: Amazon EC2 as a place where images bake and instances run
|
|
5
|
-
Project-URL: Repository, https://github.com/infrastructurebuilder/cs-image-system-3
|
|
6
|
-
Project-URL: Documentation, https://github.com/infrastructurebuilder/cs-image-system-3/blob/develop/docs/OPERATIONS.md
|
|
7
|
-
Project-URL: Package README, https://github.com/infrastructurebuilder/cs-image-system-3/blob/develop/packages/aws-runtime-plugin/README.md
|
|
8
|
-
Author-email: Mykel Alvis <mykelalvis@infrastructurebuilder.org>
|
|
9
|
-
License-Expression: Apache-2.0
|
|
10
|
-
License-File: LICENSE
|
|
11
|
-
Requires-Python: >=3.13
|
|
12
|
-
Requires-Dist: boto3>=1.43
|
|
13
|
-
Requires-Dist: botocore>=1.43
|
|
14
|
-
Requires-Dist: cs-image-system-hashicorp-utils==0.1.3
|
|
15
|
-
Requires-Dist: cs-image-system-system==0.1.3
|
|
16
|
-
Requires-Dist: pydantic>=2.13
|
|
17
|
-
Requires-Dist: python-hcl2>=8.1
|
|
18
|
-
Description-Content-Type: text/markdown
|
|
19
|
-
|
|
20
1
|
# cs-image-system-aws-runtime-plugin
|
|
21
2
|
|
|
22
3
|
The `aws` runtime type. A runtime is where images are baked and instances
|
|
@@ -75,9 +56,13 @@ Present in the package but not registered as a service:
|
|
|
75
56
|
cs_image_system.aws_runtime.bootstrap:aws_section`. It owns the questions
|
|
76
57
|
and the terraform for this cloud's one-time initialisation -- the GitHub
|
|
77
58
|
OIDC identity provider, the READ-ONLY and WRITE roles of CI_SETUP.md 3.3
|
|
78
|
-
with their trust and permission documents, and the state
|
|
79
|
-
|
|
80
|
-
|
|
59
|
+
with their trust and permission documents, and the state buckets (the
|
|
60
|
+
default backend's, and since hygiene XX every other declared S3
|
|
61
|
+
backend's that a root is bound to) and the Session Manager instance
|
|
62
|
+
profile when the account lacks them (`tfmodules/bootstrap_aws`). The
|
|
63
|
+
section renders its coverage audit for the generated README: what it
|
|
64
|
+
made, read, adopted, or leaves by hand and why (a backend bound to no
|
|
65
|
+
root is left out and said so). The section is wanted by default where the
|
|
81
66
|
tree declares an AWS runtime or an S3 state backend. Its defaults come
|
|
82
67
|
from the raw `cfg/runtime-builders.yml` and `cfg/state-backends.yml`
|
|
83
68
|
(read as YAML, never loaded; a starter's `REPLACE-ME` values and its
|
|
@@ -306,27 +291,64 @@ included).
|
|
|
306
291
|
|
|
307
292
|
`query_provider_image()` takes one OS-builder runtime entry
|
|
308
293
|
(`OSBuilderBaseImageBuilderSubconfig` in
|
|
309
|
-
[os_builder_runtime_config.py](../base/src/cs_image_system/base/models/os_builder_runtime_config.py))
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
-
|
|
316
|
-
|
|
294
|
+
[os_builder_runtime_config.py](../base/src/cs_image_system/base/models/os_builder_runtime_config.py))
|
|
295
|
+
and asks it for its effective query, `entry.image_query()`: an
|
|
296
|
+
`ImageQuery` in the SYSTEM's vocabulary
|
|
297
|
+
([contract/image.py](../contract/src/cs_image_system/contract/image.py);
|
|
298
|
+
stage 106). That query is the image-source catalog's `aws` block for the
|
|
299
|
+
OS builder's family (the release's layer under the tree's
|
|
300
|
+
`cfg/image-sources*.yml`), with the OS builder's own `query` landing on it
|
|
301
|
+
key by key and the runtime entry's landing on that; the publishers declared
|
|
302
|
+
anywhere (the OS builder's `owners`, this runtime's `default_owners`, the
|
|
303
|
+
entry's `owners`, in that order, deduplicated) replace the catalog's. The
|
|
304
|
+
entry's own `query` mapping is never rewritten (stage 63). The builder then
|
|
305
|
+
translates it with `translate_image_query()` (the table below) and
|
|
317
306
|
`query_image()` in [aws_utils.py](src/cs_image_system/aws_runtime/aws_utils.py)
|
|
318
|
-
runs
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
`
|
|
324
|
-
|
|
325
|
-
`
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
307
|
+
runs the `DescribeImages` request `describe_images_request()` makes from
|
|
308
|
+
it, paging through the results, and returns the newest by `CreationDate`,
|
|
309
|
+
or `None` when nothing matches; with `newest: false` more than one match
|
|
310
|
+
raises `AMIQueryError` as ambiguous. The builder reads the owner straight
|
|
311
|
+
from the AMI record, `ImageOwnerAlias` (for example `amazon`) else
|
|
312
|
+
`OwnerId` (the account id), and returns `(ImageId, owner, raw)`; the base
|
|
313
|
+
registers an `AwsProviderSpecificImage` in the resolved state. A record
|
|
314
|
+
with neither field raises `Could not get owner alias or owner id ...`. No
|
|
315
|
+
match returns `None`, and the base's resolve step then fails the run.
|
|
316
|
+
`validate` has already refused an entry whose query uses a key this
|
|
317
|
+
runtime cannot translate, or has no source at all, and has printed every
|
|
318
|
+
entry's query in both vocabularies (`image source: OS builder '<name>' on
|
|
319
|
+
runtime <rt> (aws): <system query> -> <EC2 query>`).
|
|
320
|
+
|
|
321
|
+
Before stage 106 the entry's `query.filters` were EC2's field names in
|
|
322
|
+
snake case (the system started on AWS and used its filter keys: the
|
|
323
|
+
operator's incidental choice), passed through a key map, with unknown keys
|
|
324
|
+
going to a post-query exact-match filter. The vocabulary is the system's
|
|
325
|
+
now, EC2's spellings kept where they were good names; the old spellings
|
|
326
|
+
still read (`root_device_type`, `virtualization_type`, `owners`, the
|
|
327
|
+
nested `filters:` form), and an unknown key is refused by name instead of
|
|
328
|
+
buried.
|
|
329
|
+
|
|
330
|
+
### Image query translation
|
|
331
|
+
|
|
332
|
+
What `translate_image_query()` does with each key of the system's
|
|
333
|
+
vocabulary (`supported_query_keys()` is the set of rows not marked
|
|
334
|
+
refused; [tests/test_docs_image_sources.py](../../tests/test_docs_image_sources.py)
|
|
335
|
+
holds this table to it). The answer is `{"owners": [...], "filters":
|
|
336
|
+
{<EC2 filter name>: value}, "most_recent": bool}`: what packer's
|
|
337
|
+
`data "amazon-ami"` body and `describe_images` both take.
|
|
338
|
+
|
|
339
|
+
| System key | EC2 filter | Note |
|
|
340
|
+
| --- | --- | --- |
|
|
341
|
+
| `name` | `name` | a glob over the AMI name |
|
|
342
|
+
| `family` | refused | EC2 has no image families; name the image |
|
|
343
|
+
| `version` | refused | the catalog fills `{version}` into `name` |
|
|
344
|
+
| `architecture` | `architecture` | `x86_64` or `arm64`, EC2's own words |
|
|
345
|
+
| `publisher` | `owners` | account ids and the aliases `self`, `amazon`, `aws-marketplace` |
|
|
346
|
+
| `state` | `state` | `available` when unset: the query always forced it |
|
|
347
|
+
| `virtualization` | `virtualization-type` | `hvm` |
|
|
348
|
+
| `root_device` | `root-device-type` | `ebs` |
|
|
349
|
+
| `tags` | `tag:<key>` | one filter per tag |
|
|
350
|
+
| `newest` | `most_recent` | true when unset; false refuses more than one match |
|
|
351
|
+
| `native` | verbatim | each key an EC2 filter name, passed as written |
|
|
330
352
|
|
|
331
353
|
When the entry declares `image_id` (stage 63), the base's resolve step
|
|
332
354
|
does not call `query_provider_image()` at all: it calls
|
|
@@ -334,29 +356,26 @@ does not call `query_provider_image()` at all: it calls
|
|
|
334
356
|
with `ImageIds=[<id>]` through `_query_by_ami_id` in
|
|
335
357
|
[aws_utils.py](src/cs_image_system/aws_runtime/aws_utils.py), with the
|
|
336
358
|
runtime's own client configuration (profile and region). The entry's
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
`InvalidImageID.
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
the query takes the newest match.
|
|
347
|
-
|
|
348
|
-
Stage 63 also removed helpers from
|
|
349
|
-
[aws_utils.py](src/cs_image_system/aws_runtime/aws_utils.py) that nothing
|
|
350
|
-
called: `get_ami_owner`, `get_ami_ssh_user`, `remap_for_aws`,
|
|
351
|
-
`_query_by_name` and `_build_ami_filters`. `_query_by_ami_id` (an exact
|
|
352
|
-
lookup by AMI id) was kept, and since stage 63 it is what
|
|
353
|
-
`query_provider_image_by_id()` calls.
|
|
359
|
+
query is not consulted. The owner is read from the record the same way,
|
|
360
|
+
`ImageOwnerAlias` else `OwnerId` (else `self`), and the result is
|
|
361
|
+
`(ImageId, owner, raw)`. An id AWS answers `InvalidImageID.NotFound` or
|
|
362
|
+
`InvalidImageID.Malformed` for, or one that returns no image, gives
|
|
363
|
+
`None`, and the base stops resolution with `OS builder <name>: image_id
|
|
364
|
+
'<id>' on runtime <rt> is not known to the provider`; any other API
|
|
365
|
+
refusal raises `AMIQueryError`. An AMI id is regional, so a pinned id must
|
|
366
|
+
exist in this runtime's region. Pinning makes the base bake reproducible:
|
|
367
|
+
the same vendor AMI every time, where the query takes the newest match.
|
|
354
368
|
|
|
355
369
|
`AwsProviderSpecificImage.get_query_assets()` gives the packer
|
|
356
370
|
`data "amazon-ami"` body:
|
|
357
371
|
|
|
358
372
|
- resolved: `filters = { image-id = "<ami>" }` plus `owners = ["<owner>"]` when known;
|
|
359
|
-
- deferred (an image a build produces later): `
|
|
373
|
+
- deferred (an image a build produces later): an `ImageQuery` (`name =
|
|
374
|
+
"<pattern>*"`, `root_device = ebs`, `virtualization = hvm`, the
|
|
375
|
+
architecture or `x86_64`, publisher `self`) translated like a vendor
|
|
376
|
+
query and quoted: `owners = ["self"]`, `filters = { name = "<pattern>*",
|
|
377
|
+
root-device-type = "ebs", virtualization-type = "hvm", architecture =
|
|
378
|
+
"<arch>" }`, `most_recent = true`.
|
|
360
379
|
|
|
361
380
|
[tests/test_aws_provider_specific_image.py](tests/test_aws_provider_specific_image.py)
|
|
362
381
|
pins both shapes.
|
|
@@ -1139,7 +1158,7 @@ when the installed version is below it.
|
|
|
1139
1158
|
- [default-os-plugin](../default-os-plugin/README.md): the OS builders whose runtime entries this plugin resolves to AMIs.
|
|
1140
1159
|
- [packer-plugin](../packer-plugin/README.md): the image builder that asks this plugin for the `amazon-ebs` source and the session provisioners.
|
|
1141
1160
|
- [tf-ebs-instance-plugin](../tf-ebs-instance-plugin/README.md): the instance and storage roots that read this model's networking, profile and session profile.
|
|
1142
|
-
- [dummy-plugin](../dummy-plugin/README.md): the extension template
|
|
1161
|
+
- [dummy-plugin](../dummy-plugin/README.md): the extension template (an unregistered copy of its group model sat at the bottom of this package's models module until stage 63).
|
|
1143
1162
|
- Base classes: [runtime.py](../base/src/cs_image_system/base/models/runtime.py), [cloud_builder.py](../base/src/cs_image_system/base/models/cloud_builder.py), [credentials.py](../base/src/cs_image_system/base/models/credentials.py), [provider_specific_image.py](../base/src/cs_image_system/base/models/provider_specific_image.py), [builder_base_runtime.py](../base/src/cs_image_system/base/basic/builder_base_runtime.py), [abstract_version_checker.py](../base/src/cs_image_system/base/basic/abstract_version_checker.py).
|
|
1144
1163
|
- Consumers of the hooks: [state_query.py](../base/src/cs_image_system/base/state_query.py), [retention.py](../base/src/cs_image_system/base/retention.py), [dispose.py](../base/src/cs_image_system/base/commands/dispose.py), [verify_instance.py](../base/src/cs_image_system/base/commands/verify_instance.py), [unmount.py](../base/src/cs_image_system/base/commands/unmount.py), [relabel.py](../base/src/cs_image_system/base/commands/relabel.py), [release.py](../base/src/cs_image_system/base/release.py), [power_state.py](../base/src/cs_image_system/base/power_state.py), [provider_aliases.py](../base/src/cs_image_system/base/provider_aliases.py), [runtime_facts.py](../base/src/cs_image_system/base/commands/runtime_facts.py).
|
|
1145
1164
|
- [docs/OPERATIONS.md](../../docs/OPERATIONS.md) for the credential contract and the operator's cycles; [docs/DESIGN.md](../../docs/DESIGN.md) for the design; [docs/PLUGINS.md](../../docs/PLUGINS.md) for the package index.
|
{cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/pyproject.toml
RENAMED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cs-image-system-aws-runtime-plugin"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.5"
|
|
8
8
|
description = "The aws runtime type: Amazon EC2 as a place where images bake and instances run"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [{ name = "Mykel Alvis", email = "mykelalvis@infrastructurebuilder.org" }]
|
|
@@ -12,8 +12,9 @@ license = "Apache-2.0"
|
|
|
12
12
|
license-files = ["LICENSE"]
|
|
13
13
|
requires-python = ">=3.13"
|
|
14
14
|
dependencies = [
|
|
15
|
-
"cs-image-system-
|
|
16
|
-
"cs-image-system-
|
|
15
|
+
"cs-image-system-contract==0.1.5",
|
|
16
|
+
"cs-image-system-base==0.1.5",
|
|
17
|
+
"cs-image-system-hashicorp-utils==0.1.5",
|
|
17
18
|
"boto3>=1.43",
|
|
18
19
|
"botocore>=1.43",
|
|
19
20
|
"python-hcl2>=8.1",
|
|
@@ -18,7 +18,7 @@ from typing import Any
|
|
|
18
18
|
import hcl2
|
|
19
19
|
from hcl2 import Builder
|
|
20
20
|
|
|
21
|
-
from cs_image_system.
|
|
21
|
+
from cs_image_system.contract.constants import OOPS_DEFAULTS
|
|
22
22
|
from cs_image_system.hashicorp_utils.hashicorp import FO
|
|
23
23
|
|
|
24
24
|
from .aws_runtime_models import AwsCloudBuilderModel, AwsCloudNetworkingModel
|
|
@@ -28,16 +28,13 @@ class AwsProviderSpecificImage(ProviderSpecificImage):
|
|
|
28
28
|
ret["owners"] = [f'"{self.owner}"']
|
|
29
29
|
return ret
|
|
30
30
|
# Deferred: the image is produced by a build in this account; find the
|
|
31
|
-
# most recent image matching the output-name pattern
|
|
32
|
-
#
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
},
|
|
42
|
-
"most_recent": True,
|
|
43
|
-
}
|
|
31
|
+
# most recent image matching the output-name pattern. stage 106: the
|
|
32
|
+
# lookup is an ImageQuery in the system's vocabulary, translated like
|
|
33
|
+
# a vendor query, then quoted for the packer data source.
|
|
34
|
+
from cs_image_system.contract.image import ImageQuery
|
|
35
|
+
from .aws_utils import translate_image_query
|
|
36
|
+
native = translate_image_query(ImageQuery(
|
|
37
|
+
name=f"{self.deferred_name_pattern}*", root_device="ebs", virtualization="hvm",
|
|
38
|
+
architecture=self.architecture if self.architecture else "x86_64", publisher=("self",), newest=True))
|
|
39
|
+
filters = {k: f'"{v}"' for k, v in native["filters"].items() if k != "state"}
|
|
40
|
+
return {"owners": [f'"{o}"' for o in native["owners"]], "filters": filters, "most_recent": True}
|
|
@@ -8,18 +8,19 @@ from typing import Any
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
from cs_image_system.base.basic.abstract_version_checker import AbstractVersionChecker
|
|
11
|
-
from cs_image_system.
|
|
11
|
+
from cs_image_system.contract.assets import AssetSet
|
|
12
12
|
from cs_image_system.base.basic.builder_base_cloud import CloudBuilderBase
|
|
13
|
-
from cs_image_system.
|
|
14
|
-
from cs_image_system.
|
|
13
|
+
from cs_image_system.contract.constants import VCT
|
|
14
|
+
from cs_image_system.contract.lifecycle import ExecutionLifecyclePhase
|
|
15
15
|
from cs_image_system.base.models.os_builder_runtime_config import OSBuilderBaseImageBuilderSubconfig
|
|
16
16
|
from cs_image_system.base.models.provider_specific_image import ProviderSpecificImage
|
|
17
17
|
from cs_image_system.base import power_state
|
|
18
|
-
from cs_image_system.
|
|
18
|
+
from cs_image_system.contract.core import PluginArtifactProtocol
|
|
19
19
|
|
|
20
20
|
from .aws_provider_specific_image import AwsProviderSpecificImage
|
|
21
21
|
from .aws_runtime_models import AWS_CLI, AwsCloudBuilderModel
|
|
22
|
-
from .aws_utils import
|
|
22
|
+
from .aws_utils import AWS_QUERY_KEYS, query_image, translate_image_query
|
|
23
|
+
from cs_image_system.contract.image import ImageQuery
|
|
23
24
|
from cs_image_system.aws_runtime import aws_utils
|
|
24
25
|
|
|
25
26
|
log = logging.getLogger(__name__)
|
|
@@ -43,12 +44,18 @@ class AwsCloudBuilder(CloudBuilderBase[AwsCloudBuilderModel], PluginArtifactProt
|
|
|
43
44
|
def provider_specific_image_class(self) -> type[ProviderSpecificImage]:
|
|
44
45
|
return AwsProviderSpecificImage
|
|
45
46
|
|
|
47
|
+
# --- the image query (stage 106)
|
|
48
|
+
def supported_query_keys(self) -> frozenset[str]:
|
|
49
|
+
return AWS_QUERY_KEYS
|
|
50
|
+
|
|
51
|
+
def translate_image_query(self, query: ImageQuery) -> dict[str, Any]:
|
|
52
|
+
return translate_image_query(query)
|
|
53
|
+
|
|
46
54
|
def query_provider_image(self, os_builder: OSBuilderBaseImageBuilderSubconfig) -> tuple[str, str, Mapping[str,Any]] | None:
|
|
47
55
|
retval: str | None = None
|
|
48
|
-
q
|
|
56
|
+
q = self.translate_image_query(os_builder.image_query()) if os_builder else {}
|
|
49
57
|
kv = query_image(
|
|
50
58
|
query=q,
|
|
51
|
-
post_query_filter=missed,
|
|
52
59
|
session_config=self.model.self_to_aws_client_config(),
|
|
53
60
|
)
|
|
54
61
|
if kv:
|
|
@@ -4,18 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
from dataclasses import field
|
|
6
6
|
from cs_image_system.base.models.credentials import CredentialsBase
|
|
7
|
-
from cs_image_system.
|
|
7
|
+
from cs_image_system.contract.model_config import CSIS_MODEL_CONFIG
|
|
8
8
|
from pydantic.dataclasses import dataclass # stage 23: validation at construction
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
11
|
import logging
|
|
12
12
|
|
|
13
13
|
from cs_image_system.aws_runtime import aws_utils
|
|
14
|
-
from cs_image_system.
|
|
14
|
+
from cs_image_system.contract.fields import fk_field, parent_field
|
|
15
15
|
log = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
from cs_image_system.
|
|
18
|
+
from cs_image_system.contract.constants import DEFAULT, OOPS_DEFAULTS, VCT
|
|
19
19
|
from cs_image_system.base.models.cloud_builder import CloudBuilderModel, CloudNetworkingConfig
|
|
20
20
|
|
|
21
21
|
|
|
@@ -35,11 +35,7 @@ class AwsCloudNetworkingModel(CloudNetworkingConfig):
|
|
|
35
35
|
# ingress to the shared networks' address space).
|
|
36
36
|
ssh_ingress_security_group_ids: list[str] = field(default_factory=list)
|
|
37
37
|
|
|
38
|
-
_model_id: str | None =
|
|
39
|
-
init=False,
|
|
40
|
-
default=None, metadata={
|
|
41
|
-
"description": "The name of the runtime model this object is associated with",
|
|
42
|
-
})
|
|
38
|
+
_model_id: str | None = parent_field(VCT.RUNTIME_BUILDER_MODEL, description="The name of the runtime model this object is associated with")
|
|
43
39
|
|
|
44
40
|
def __post_init__(self) -> None:
|
|
45
41
|
super().__post_init__()
|
|
@@ -6,14 +6,14 @@
|
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
-
import copy
|
|
10
9
|
import logging
|
|
11
|
-
from typing import Any,
|
|
10
|
+
from typing import Any, Mapping, cast
|
|
12
11
|
import boto3 # type: ignore[import-untyped]
|
|
13
12
|
from botocore.client import BaseClient as EC2 # type: ignore[import-untyped]
|
|
14
13
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
15
14
|
|
|
16
|
-
from cs_image_system.
|
|
15
|
+
from cs_image_system.contract.image import ImageQuery
|
|
16
|
+
|
|
17
17
|
|
|
18
18
|
log = logging.getLogger(__name__)
|
|
19
19
|
|
|
@@ -120,81 +120,75 @@ AWS_DI_MAP: dict[str, str] = {
|
|
|
120
120
|
}
|
|
121
121
|
|
|
122
122
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
}
|
|
171
|
-
for _k, _v in v.items()
|
|
172
|
-
]
|
|
173
|
-
else:
|
|
174
|
-
raise ValueError(f"Expected 'filter' value to be a dict, got {type(topv)}")
|
|
175
|
-
ret[topk] = topv
|
|
176
|
-
return (ret, missed)
|
|
123
|
+
#: stage 106: the keys of the system's image-query vocabulary EC2's
|
|
124
|
+
#: DescribeImages can answer, and the filter each becomes.
|
|
125
|
+
AWS_QUERY_KEYS: frozenset[str] = frozenset({"name", "architecture", "publisher", "state", "virtualization",
|
|
126
|
+
"root_device", "tags", "newest", "native"})
|
|
127
|
+
AWS_FILTER_NAMES: dict[str, str] = { # in the order the filters are written out
|
|
128
|
+
"name": "name",
|
|
129
|
+
"root_device": "root-device-type",
|
|
130
|
+
"virtualization": "virtualization-type",
|
|
131
|
+
"architecture": "architecture",
|
|
132
|
+
"state": "state",
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def translate_image_query(query: ImageQuery) -> dict[str, Any]:
|
|
137
|
+
"""The system's query in EC2's words (stage 106): ``{"owners": [...],
|
|
138
|
+
"filters": {<DescribeImages filter name>: value}, "most_recent": bool}``,
|
|
139
|
+
the shape packer's ``amazon-ami`` data source and ``describe_images``
|
|
140
|
+
both take (``describe_images_request`` makes the boto call from it).
|
|
141
|
+
``state`` defaults to ``available``, as the query always has. A key EC2
|
|
142
|
+
cannot answer raises ValueError naming it (``validate`` refuses it
|
|
143
|
+
first); ``native`` keys are EC2 filter names, passed verbatim."""
|
|
144
|
+
unsupported = sorted(k for k in query.keys() if k not in AWS_QUERY_KEYS)
|
|
145
|
+
if unsupported:
|
|
146
|
+
raise ValueError(f"AWS cannot translate image query key(s) {', '.join(unsupported)}")
|
|
147
|
+
filters: dict[str, Any] = {}
|
|
148
|
+
for key, name in AWS_FILTER_NAMES.items():
|
|
149
|
+
value = getattr(query, key)
|
|
150
|
+
if value is not None:
|
|
151
|
+
filters[name] = value
|
|
152
|
+
filters.setdefault("state", "available")
|
|
153
|
+
for k, v in sorted(query.tags.items()):
|
|
154
|
+
filters[f"tag:{k}"] = v
|
|
155
|
+
for k, v in query.native.items():
|
|
156
|
+
filters[str(k)] = v
|
|
157
|
+
return {"owners": list(query.publisher), "filters": filters,
|
|
158
|
+
"most_recent": True if query.newest is None else bool(query.newest)}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def describe_images_request(native: Mapping[str, Any]) -> dict[str, Any]:
|
|
162
|
+
"""A ``describe_images`` request from ``translate_image_query``'s answer."""
|
|
163
|
+
filters = []
|
|
164
|
+
for name, value in dict(native.get("filters") or {}).items():
|
|
165
|
+
if isinstance(value, bool):
|
|
166
|
+
value = str(value).lower()
|
|
167
|
+
filters.append({"Name": str(name), "Values": [str(v) for v in value] if isinstance(value, (list, tuple)) else [str(value)]})
|
|
168
|
+
return {"DryRun": False, "IncludeDisabled": False, "IncludeDeprecated": False,
|
|
169
|
+
"Owners": [str(o) for o in (native.get("owners") or [])], "Filters": filters}
|
|
177
170
|
|
|
178
171
|
|
|
179
172
|
def query_image(
|
|
180
173
|
query: dict[str, Any],
|
|
181
|
-
post_query_filter: dict[str, Any] | None = None,
|
|
182
174
|
session_config: dict[str, Any] | None = None,
|
|
183
175
|
) -> dict[str, Any] | None:
|
|
184
|
-
"""The
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
runtime's boto3 session configuration and is required. A pinned
|
|
193
|
-
image id does not come through here (stage 63 reads the entry's
|
|
194
|
-
``image_id`` through ``query_image_by_id``).
|
|
176
|
+
"""The AMI matching an image query in EC2's words (stage 106:
|
|
177
|
+
``translate_image_query``'s answer, ``owners``/``filters``/``most_recent``;
|
|
178
|
+
the boto-shaped request of before is accepted too). The newest by
|
|
179
|
+
``CreationDate`` is returned, or None when nothing matches; with
|
|
180
|
+
``most_recent`` false, more than one match is refused as ambiguous.
|
|
181
|
+
``session_config`` is the runtime's boto3 session configuration and is
|
|
182
|
+
required. A pinned image id does not come through here (stage 63 reads
|
|
183
|
+
the entry's ``image_id`` through ``query_image_by_id``).
|
|
195
184
|
"""
|
|
196
185
|
if not query:
|
|
197
186
|
raise ValueError("Query dictionary cannot be empty")
|
|
187
|
+
if "Filters" not in query and "Owners" not in query:
|
|
188
|
+
most_recent = query.get("most_recent", True)
|
|
189
|
+
query = describe_images_request(query)
|
|
190
|
+
else:
|
|
191
|
+
most_recent = True
|
|
198
192
|
|
|
199
193
|
try:
|
|
200
194
|
if not session_config:
|
|
@@ -207,22 +201,12 @@ def query_image(
|
|
|
207
201
|
except (BotoCoreError, ClientError) as e:
|
|
208
202
|
raise AMIQueryError(f"Failed to create EC2 client: {e}") from e
|
|
209
203
|
|
|
210
|
-
|
|
211
|
-
# Return most recently created AMI if multiple matches
|
|
212
|
-
ilist = []
|
|
213
|
-
for r in results:
|
|
214
|
-
if post_query_filter:
|
|
215
|
-
match = True
|
|
216
|
-
for k, v in post_query_filter.items():
|
|
217
|
-
if r.get(k) != v:
|
|
218
|
-
match = False
|
|
219
|
-
break
|
|
220
|
-
if not match:
|
|
221
|
-
continue
|
|
222
|
-
ilist.append(r)
|
|
204
|
+
ilist = list(_query_by_filters(ec2_client, query))
|
|
223
205
|
if not ilist:
|
|
224
206
|
return None
|
|
225
207
|
if len(ilist) > 1:
|
|
208
|
+
if not most_recent:
|
|
209
|
+
raise AMIQueryError(f"{len(ilist)} AMIs match and the query says newest: false; narrow it")
|
|
226
210
|
ilist.sort(key=lambda x: x.get("CreationDate", ""), reverse=True)
|
|
227
211
|
|
|
228
212
|
return cast(dict[str, Any], ilist[0])
|
|
@@ -30,8 +30,10 @@ from __future__ import annotations
|
|
|
30
30
|
import json
|
|
31
31
|
from typing import Any
|
|
32
32
|
|
|
33
|
-
from cs_image_system.base.bootstrap.facts import Facts, default_state_backend,
|
|
34
|
-
|
|
33
|
+
from cs_image_system.base.bootstrap.facts import (Facts, bound_backend_names, default_state_backend, other_state_backends,
|
|
34
|
+
real, runtime_of_type)
|
|
35
|
+
from cs_image_system.base.bootstrap.questions import (ADOPTED, BY_HAND, MADE, READ, Answers, Question, Raw, Rendered,
|
|
36
|
+
Secret, Section)
|
|
35
37
|
|
|
36
38
|
MODULE = "bootstrap_aws"
|
|
37
39
|
OIDC_HOST = "token.actions.githubusercontent.com"
|
|
@@ -137,6 +139,55 @@ def _bucket_exists(facts: Facts, answers: Answers) -> bool | None:
|
|
|
137
139
|
("Not Found", "404", "NoSuchBucket"))
|
|
138
140
|
|
|
139
141
|
|
|
142
|
+
def _bucket_named_exists(facts: Facts, answers: Answers, bucket: str) -> bool | None:
|
|
143
|
+
return _exists(_aws(facts, answers, "s3api", "head-bucket", "--bucket", bucket), ("Not Found", "404", "NoSuchBucket"))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def other_s3_buckets(facts: Facts) -> list[tuple[str, str, str, bool]]:
|
|
147
|
+
"""The tree's other S3 backends (hygiene XX item 1): ``(backend name,
|
|
148
|
+
bucket, prefix, bound)`` for each with a real bucket, in declaration
|
|
149
|
+
order; ``bound`` when some root's ``state_configuration`` names it."""
|
|
150
|
+
bound = bound_backend_names(facts)
|
|
151
|
+
out: list[tuple[str, str, str, bool]] = []
|
|
152
|
+
for backend in other_state_backends(facts, "s3"):
|
|
153
|
+
bucket = real(backend.get("bucket"))
|
|
154
|
+
if not bucket:
|
|
155
|
+
continue
|
|
156
|
+
prefix = (real(backend.get("key")) or "").strip("/")
|
|
157
|
+
out.append((str(backend["name"]), bucket, prefix, str(backend["name"]) in bound))
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _other_buckets_default(facts: Facts, answers: Answers) -> str | None:
|
|
162
|
+
"""The other S3 backends' buckets this root should make: every one that
|
|
163
|
+
is bound to a root and that the account says is absent, as
|
|
164
|
+
``bucket=prefix,...``; a bucket bound to no root is left out (the README
|
|
165
|
+
says so); a bucket the account cannot be asked about leaves no default."""
|
|
166
|
+
pairs: list[str] = []
|
|
167
|
+
for _name, bucket, prefix, bound in other_s3_buckets(facts):
|
|
168
|
+
if not bound:
|
|
169
|
+
continue
|
|
170
|
+
exists = _bucket_named_exists(facts, answers, bucket)
|
|
171
|
+
if exists is None:
|
|
172
|
+
return None
|
|
173
|
+
if not exists:
|
|
174
|
+
pairs.append(f"{bucket}={prefix}")
|
|
175
|
+
return ",".join(pairs)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def parse_buckets(text: Any) -> dict[str, str]:
|
|
179
|
+
"""``bucket=prefix,bucket=prefix`` as a mapping (an empty prefix allowed)."""
|
|
180
|
+
out: dict[str, str] = {}
|
|
181
|
+
for part in str(text or "").split(","):
|
|
182
|
+
part = part.strip()
|
|
183
|
+
if not part:
|
|
184
|
+
continue
|
|
185
|
+
bucket, _, prefix = part.partition("=")
|
|
186
|
+
if bucket.strip():
|
|
187
|
+
out[bucket.strip()] = prefix.strip().strip("/")
|
|
188
|
+
return out
|
|
189
|
+
|
|
190
|
+
|
|
140
191
|
def _profile_exists(facts: Facts, answers: Answers) -> bool | None:
|
|
141
192
|
name = answers.get("instance_profile")
|
|
142
193
|
if not name:
|
|
@@ -236,6 +287,14 @@ QUESTIONS: tuple[Question, ...] = (
|
|
|
236
287
|
"here, so a destroy of the root removes it? (a bucket that stood before this root is read, never "
|
|
237
288
|
"managed; one this root makes is managed from then on)", kind="bool",
|
|
238
289
|
default=lambda f, a: not a.get("state_bucket_exists")),
|
|
290
|
+
# hygiene XX item 1: the tree's OTHER S3 backends. A choice, not an
|
|
291
|
+
# observation: the second interview keeps the answer, so a bucket this
|
|
292
|
+
# root made never leaves the plan (the trap item 5 of hygiene XII closed
|
|
293
|
+
# for the default bucket).
|
|
294
|
+
Question("other_state_buckets", "The tree's other S3 backends' buckets this root MAKES when absent and manages "
|
|
295
|
+
"from then on, as bucket=prefix,bucket=prefix (by default every one bound to a root that the account "
|
|
296
|
+
"says is absent; a backend bound to no root is left out)", default=_other_buckets_default,
|
|
297
|
+
help="the account could not be asked about one of the other buckets: no AWS session", may_be_empty=True),
|
|
239
298
|
Question("instance_profile", "The instance profile build and standing instances wear for Session Manager",
|
|
240
299
|
default=_session_profile,
|
|
241
300
|
help="the AWS runtime's `session_instance_profile` in cfg/runtime-builders.yml is unset or a placeholder"),
|
|
@@ -245,12 +304,29 @@ QUESTIONS: tuple[Question, ...] = (
|
|
|
245
304
|
)
|
|
246
305
|
|
|
247
306
|
|
|
248
|
-
def render(answers: Answers) -> Rendered:
|
|
307
|
+
def render(answers: Answers, facts: Facts | None = None) -> Rendered:
|
|
249
308
|
profile = str(answers.get("profile") or "")
|
|
250
309
|
prefix = str(answers["state_prefix"]).strip("/")
|
|
251
310
|
tags = {**parse_tags(str(answers.get("tags") or "")), "ManagedBy": MANAGED_BY}
|
|
252
311
|
bucket_exists = bool(answers["state_bucket_exists"])
|
|
253
312
|
bucket_managed = bool(answers.get("state_bucket_managed")) or not bucket_exists # made here: managed, always
|
|
313
|
+
other_made = parse_buckets(answers.get("other_state_buckets"))
|
|
314
|
+
# the other backends this root does NOT make, with the reason (item 1: a line, never a silence)
|
|
315
|
+
others = other_s3_buckets(facts) if facts is not None else []
|
|
316
|
+
other_rows: list[tuple[str, str, str]] = []
|
|
317
|
+
for name, bucket, _bprefix, bound in others:
|
|
318
|
+
if bucket in other_made:
|
|
319
|
+
other_rows.append((f"the S3 backend `{name}`'s bucket `{bucket}`", MADE,
|
|
320
|
+
"declared, bound to a root and absent at the interview: made here (versioned, encrypted, private) and managed from then on"))
|
|
321
|
+
elif not bound:
|
|
322
|
+
other_rows.append((f"the S3 backend `{name}`'s bucket `{bucket}`", BY_HAND,
|
|
323
|
+
"declared but bound to no root, so nothing would use it: not made; bind a root to it and re-run the interview, or drop the backend"))
|
|
324
|
+
else:
|
|
325
|
+
other_rows.append((f"the S3 backend `{name}`'s bucket `{bucket}`", READ,
|
|
326
|
+
"stood at the interview, or was left out of the answer: read, never managed here"))
|
|
327
|
+
for bucket in other_made:
|
|
328
|
+
if bucket not in {b for _n, b, _p, _b in others}:
|
|
329
|
+
other_rows.append((f"the bucket `{bucket}`", MADE, "named in the answer; no declared S3 backend names it"))
|
|
254
330
|
backend = None
|
|
255
331
|
if bucket_exists:
|
|
256
332
|
settings: dict[str, Any] = {"bucket": answers["state_bucket"], "key": f"{prefix}/bootstrap.tfstate",
|
|
@@ -285,6 +361,7 @@ def render(answers: Answers) -> Rendered:
|
|
|
285
361
|
"instance_profile_exists": "the instance profile already exists (read, not made)",
|
|
286
362
|
}
|
|
287
363
|
variables = tuple((f"aws_{n}", types.get(n, "string"), described[n]) for n in names) + (
|
|
364
|
+
("aws_other_state_buckets", "map(string)", "the tree's other S3 backends' buckets this root makes and manages: bucket -> key prefix"),
|
|
288
365
|
("aws_profile", "string", "the AWS profile this root is applied with (empty: the environment's credentials)"),
|
|
289
366
|
("aws_tags", "map(string)", "tags on everything this root makes"),
|
|
290
367
|
("aws_read_extra_subjects", "list(string)", "other OIDC subjects the READ-ONLY role keeps trusting"),
|
|
@@ -294,6 +371,7 @@ def render(answers: Answers) -> Rendered:
|
|
|
294
371
|
for n in names}
|
|
295
372
|
tfvars["aws_state_bucket_managed"] = bucket_managed
|
|
296
373
|
tfvars["aws_state_prefix"] = prefix
|
|
374
|
+
tfvars["aws_other_state_buckets"] = other_made
|
|
297
375
|
tfvars["aws_profile"] = profile
|
|
298
376
|
tfvars["aws_tags"] = tags
|
|
299
377
|
extras = {k: parse_subjects(answers.get(f"{k}_extra_subjects")) for k in ("read", "write")}
|
|
@@ -308,6 +386,7 @@ def render(answers: Answers) -> Rendered:
|
|
|
308
386
|
if extras[k] else "")
|
|
309
387
|
for k in ("read", "write") if answers.get(f"{k}_role_exists")),
|
|
310
388
|
module_args={**{n: Raw(f"var.aws_{n}") for n in names},
|
|
389
|
+
"other_state_buckets": Raw("var.aws_other_state_buckets"),
|
|
311
390
|
"read_extra_subjects": Raw("var.aws_read_extra_subjects"),
|
|
312
391
|
"write_extra_subjects": Raw("var.aws_write_extra_subjects")},
|
|
313
392
|
variables=variables,
|
|
@@ -317,6 +396,7 @@ def render(answers: Answers) -> Rendered:
|
|
|
317
396
|
("aws_write_role_arn", f"module.{MODULE}.write_role_arn", "the WRITE role's ARN: the AWS_APPLY_ROLE_ARN secret"),
|
|
318
397
|
("aws_oidc_provider_arn", f"module.{MODULE}.oidc_provider_arn", "the GitHub OIDC identity provider"),
|
|
319
398
|
("aws_state_bucket", f"module.{MODULE}.state_bucket", "the state bucket"),
|
|
399
|
+
("aws_other_state_buckets", f"module.{MODULE}.other_state_buckets", "the other S3 backends' buckets made here"),
|
|
320
400
|
("aws_instance_profile_arn", f"module.{MODULE}.instance_profile_arn", "the Session Manager instance profile"),
|
|
321
401
|
),
|
|
322
402
|
required_providers={"aws": {"source": "hashicorp/aws", "version": ">= 5.81"}},
|
|
@@ -349,11 +429,33 @@ def render(answers: Answers) -> Rendered:
|
|
|
349
429
|
"exists: this root's state is bound to it (decision D8); the bucket stood before this root and is not managed here"
|
|
350
430
|
if bucket_exists else
|
|
351
431
|
"is CREATED here (versioned, encrypted, private) and managed from then on: local state for this first apply"),
|
|
432
|
+
*(f"the S3 backend's bucket `{b}` (prefix `{p or '(none)'}`) is CREATED here (versioned, encrypted, private) "
|
|
433
|
+
"and managed from then on" for b, p in sorted(other_made.items())),
|
|
352
434
|
f"the instance profile `{answers['instance_profile']}` "
|
|
353
435
|
+ ("read (it exists)" if answers["instance_profile_exists"] else "created with AmazonSSMManagedInstanceCore"),
|
|
354
436
|
),
|
|
355
437
|
backend=backend,
|
|
356
438
|
root_hcl=imports,
|
|
439
|
+
coverage=(
|
|
440
|
+
("the GitHub OIDC identity provider", READ if answers["oidc_provider_exists"] else MADE,
|
|
441
|
+
"read as a data source when the account has it" if answers["oidc_provider_exists"] else "the account lacked it"),
|
|
442
|
+
*((f"the {k.upper()} role `{answers[f'{k}_role_name']}`", ADOPTED if answers.get(f"{k}_role_exists") else MADE,
|
|
443
|
+
"stood at the interview: an import block, so its trust and policy are maintained from here"
|
|
444
|
+
if answers.get(f"{k}_role_exists") else "the account lacked it") for k in ("read", "write")),
|
|
445
|
+
(f"the default S3 backend's bucket `{answers['state_bucket']}`",
|
|
446
|
+
MADE if bucket_managed else READ,
|
|
447
|
+
"made by an apply of this root and managed from then on" if bucket_managed
|
|
448
|
+
else "stood before this root: read, never managed"),
|
|
449
|
+
*other_rows,
|
|
450
|
+
(f"the Session Manager instance profile `{answers['instance_profile']}`",
|
|
451
|
+
READ if answers["instance_profile_exists"] else MADE,
|
|
452
|
+
"read as a data source when the account has it" if answers["instance_profile_exists"]
|
|
453
|
+
else "the account lacked it: a role with AmazonSSMManagedInstanceCore"),
|
|
454
|
+
("the network: VPC, subnets, security groups, the access gateway", BY_HAND,
|
|
455
|
+
"the team's, by decision (stage 1's network rule): the system never creates or modifies network configuration"),
|
|
456
|
+
("the bake hosts' route to SSM and the package repositories", BY_HAND,
|
|
457
|
+
"part of the network: the team's"),
|
|
458
|
+
),
|
|
357
459
|
)
|
|
358
460
|
|
|
359
461
|
|
|
@@ -363,5 +465,5 @@ def wanted(facts: Facts) -> bool:
|
|
|
363
465
|
|
|
364
466
|
|
|
365
467
|
def aws_section() -> Section:
|
|
366
|
-
return Section(name="aws", title="AWS", questions=QUESTIONS, render=render, wanted=wanted,
|
|
367
|
-
description="the GitHub OIDC provider, the read-only and write roles, the
|
|
468
|
+
return Section(name="aws", title="AWS", questions=QUESTIONS, render=render, wanted=wanted, wants_facts=True,
|
|
469
|
+
description="the GitHub OIDC provider, the read-only and write roles, the state buckets and the instance profile")
|
|
@@ -5,13 +5,12 @@
|
|
|
5
5
|
import logging
|
|
6
6
|
|
|
7
7
|
from cs_image_system.base.basic.abstract_plugin_metadata import AbstractPluginMetadata
|
|
8
|
-
from cs_image_system.base.protocols.plugin_metadata import PluginMetadataProtocol
|
|
9
8
|
from .aws_runtime_builders import AWS_RUNTIME, AwsCLIVersionChecker, AwsCloudBuilder
|
|
10
9
|
from .aws_runtime_models import AwsCloudBuilderModel
|
|
11
10
|
|
|
12
11
|
log = logging.getLogger(__name__)
|
|
13
12
|
|
|
14
|
-
class AwsRuntimePluginMetadata(AbstractPluginMetadata
|
|
13
|
+
class AwsRuntimePluginMetadata(AbstractPluginMetadata):
|
|
15
14
|
def __init__(self) -> None:
|
|
16
15
|
super().__init__( "1", "3.13",
|
|
17
16
|
{AWS_RUNTIME: [
|
{cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/.gitignore
RENAMED
|
File without changes
|
{cs_image_system_aws_runtime_plugin-0.1.3 → cs_image_system_aws_runtime_plugin-0.1.5}/LICENSE
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|