cloud-foundry 0.2.4__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/PKG-INFO +2 -2
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/__init__.py +1 -1
- cloud_foundry-0.4.0/cloud_foundry/migrations/forget_site_objects.py +112 -0
- cloud_foundry-0.4.0/cloud_foundry/pulumi/__init__.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/function.py +80 -2
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/python_function.py +3 -0
- cloud_foundry-0.4.0/cloud_foundry/pulumi/ui_publisher.py +306 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/python_archive_builder.py +49 -19
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/logger.py +13 -4
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/pyproject.toml +20 -2
- cloud_foundry-0.2.4/cloud_foundry/pulumi/ui_publisher.py +0 -137
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/.gitignore +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/LICENSE +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/README.md +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/archive_builder.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/mail_publisher.py +0 -0
- {cloud_foundry-0.2.4/cloud_foundry/pulumi → cloud_foundry-0.4.0/cloud_foundry/migrations}/__init__.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/api_waf.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/cdn.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/cdn_api_origin.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/cdn_site_origin.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/connection_store.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/custom_domain.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/document_repository.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/domain.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/idle_reaper.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/publisher.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/queue.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/rest_api.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/site_bucket.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/topic.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/websocket_api.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/pulumi/websocket_function.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/services/authorization_api/security_lambda.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/services/mail_publisher.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/aws_openapi_editor.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/hash_comparator.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/idle_reaper_client.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/names.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/openapi_editor.py +0 -0
- {cloud_foundry-0.2.4 → cloud_foundry-0.4.0}/cloud_foundry/utils/websocket_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloud-foundry
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: cloud-foundry is a curated set of cloud components that can be used to assemble cloud-based applications.
|
|
5
5
|
Project-URL: Documentation, https://github.com/DanRepik/cloud-foundry
|
|
6
6
|
Project-URL: Source, https://github.com/DanRepik/cloud-foundry
|
|
@@ -17,10 +17,10 @@ Provides-Extra: dev
|
|
|
17
17
|
Requires-Dist: black>=23.0.0; extra == 'dev'
|
|
18
18
|
Requires-Dist: build; extra == 'dev'
|
|
19
19
|
Requires-Dist: docker; extra == 'dev'
|
|
20
|
-
Requires-Dist: fixture-foundry; extra == 'dev'
|
|
21
20
|
Requires-Dist: flake8; extra == 'dev'
|
|
22
21
|
Requires-Dist: flit>=3.9; extra == 'dev'
|
|
23
22
|
Requires-Dist: isort>=5.12.0; extra == 'dev'
|
|
23
|
+
Requires-Dist: moto[s3]>=5.0; extra == 'dev'
|
|
24
24
|
Requires-Dist: mypy>=1.0.0; extra == 'dev'
|
|
25
25
|
Requires-Dist: pre-commit; extra == 'dev'
|
|
26
26
|
Requires-Dist: psycopg2-binary; extra == 'dev'
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""One-time state migration for the single-resource UIPublisher.
|
|
2
|
+
|
|
3
|
+
UIPublisher used to declare one aws.s3.BucketObjectv2 per site file. It now
|
|
4
|
+
publishes the whole build through one resource. Deploying the new version
|
|
5
|
+
straight over old state would be destructive: Pulumi runs the new resource's
|
|
6
|
+
upload first and deletes the no-longer-declared per-file resources last, and
|
|
7
|
+
deleting those removes the very S3 objects that were just uploaded (same
|
|
8
|
+
keys), leaving the site empty.
|
|
9
|
+
|
|
10
|
+
Run this once per stack, right before the first `pulumi up` with the new
|
|
11
|
+
cloud_foundry. It removes the per-file resources from the Pulumi state only;
|
|
12
|
+
the S3 objects stay where they are, and the next deploy takes them over.
|
|
13
|
+
|
|
14
|
+
python -m cloud_foundry.migrations.forget_site_objects --stack dev [--cwd infra/api] [--dry-run]
|
|
15
|
+
|
|
16
|
+
The exported state is saved next to the working directory first, so
|
|
17
|
+
`pulumi stack import --file <backup>` puts it back if needed.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
import time
|
|
27
|
+
|
|
28
|
+
PUBLISHER_TYPE = "cloud_foundry:pulumi:UIPublisher"
|
|
29
|
+
OBJECT_TYPE = "aws:s3/bucketObjectv2:BucketObjectv2"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _urn_type(urn: str) -> str:
|
|
33
|
+
# urn:pulumi:<stack>::<project>::<parent types$...$type>::<name>
|
|
34
|
+
parts = urn.split("::", 3)
|
|
35
|
+
return parts[2].split("$")[-1] if len(parts) == 4 else ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def forget_site_objects(deployment: dict) -> tuple[dict, list[str]]:
|
|
39
|
+
"""Return ``deployment`` without the per-file objects that UIPublisher
|
|
40
|
+
components used to own, and the URNs that were removed."""
|
|
41
|
+
resources = deployment.get("resources") or []
|
|
42
|
+
removed = {
|
|
43
|
+
resource["urn"]
|
|
44
|
+
for resource in resources
|
|
45
|
+
if resource.get("type") == OBJECT_TYPE
|
|
46
|
+
and _urn_type(resource.get("parent", "")) == PUBLISHER_TYPE
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
kept = []
|
|
50
|
+
for resource in resources:
|
|
51
|
+
if resource["urn"] in removed:
|
|
52
|
+
continue
|
|
53
|
+
resource = dict(resource)
|
|
54
|
+
if resource.get("dependencies"):
|
|
55
|
+
resource["dependencies"] = [u for u in resource["dependencies"] if u not in removed]
|
|
56
|
+
if resource.get("propertyDependencies"):
|
|
57
|
+
resource["propertyDependencies"] = {
|
|
58
|
+
name: [u for u in urns if u not in removed]
|
|
59
|
+
for name, urns in resource["propertyDependencies"].items()
|
|
60
|
+
}
|
|
61
|
+
if resource.get("deletedWith") in removed:
|
|
62
|
+
resource.pop("deletedWith")
|
|
63
|
+
kept.append(resource)
|
|
64
|
+
|
|
65
|
+
return {**deployment, "resources": kept}, sorted(removed)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _pulumi(args: list[str], cwd: str) -> None:
|
|
69
|
+
subprocess.run(["pulumi", *args], cwd=cwd, check=True)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def main(argv: list[str] | None = None) -> int:
|
|
73
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
74
|
+
parser.add_argument("--stack", required=True)
|
|
75
|
+
parser.add_argument("--cwd", default=".", help="Pulumi project directory")
|
|
76
|
+
parser.add_argument("--dry-run", action="store_true", help="report only; change nothing")
|
|
77
|
+
args = parser.parse_args(argv)
|
|
78
|
+
|
|
79
|
+
cwd = os.path.abspath(args.cwd)
|
|
80
|
+
stamp = time.strftime("%Y%m%dT%H%M%S")
|
|
81
|
+
backup = os.path.join(cwd, f"{args.stack}-state-backup-{stamp}.json")
|
|
82
|
+
_pulumi(["stack", "export", "--stack", args.stack, "--file", backup], cwd)
|
|
83
|
+
|
|
84
|
+
with open(backup) as f:
|
|
85
|
+
state = json.load(f)
|
|
86
|
+
deployment = state.get("deployment") or {}
|
|
87
|
+
if deployment.get("pending_operations"):
|
|
88
|
+
print(
|
|
89
|
+
"State has pending operations from an interrupted update; resolve those "
|
|
90
|
+
"(pulumi refresh / pulumi cancel) before migrating.",
|
|
91
|
+
file=sys.stderr,
|
|
92
|
+
)
|
|
93
|
+
return 1
|
|
94
|
+
|
|
95
|
+
migrated, removed = forget_site_objects(deployment)
|
|
96
|
+
print(f"{len(removed)} per-file site object(s) to drop from the {args.stack} state (S3 is not touched).")
|
|
97
|
+
if not removed or args.dry_run:
|
|
98
|
+
print(f"State export kept at {backup}")
|
|
99
|
+
return 0
|
|
100
|
+
|
|
101
|
+
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as out:
|
|
102
|
+
json.dump({**state, "deployment": migrated}, out)
|
|
103
|
+
try:
|
|
104
|
+
_pulumi(["stack", "import", "--stack", args.stack, "--file", out.name], cwd)
|
|
105
|
+
finally:
|
|
106
|
+
os.unlink(out.name)
|
|
107
|
+
print(f"Done. Previous state saved to {backup}; restore it with: pulumi stack import --stack {args.stack} --file {backup}")
|
|
108
|
+
return 0
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
sys.exit(main())
|
|
File without changes
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# function.py
|
|
2
|
+
import os
|
|
2
3
|
import platform
|
|
3
4
|
from typing import Any, Optional, TypeAlias, Union
|
|
4
5
|
|
|
@@ -14,6 +15,20 @@ PolicyStatement: TypeAlias = dict[str, Any] | str
|
|
|
14
15
|
PolicyStatements: TypeAlias = list[PolicyStatement]
|
|
15
16
|
PolicyStatementsInput: TypeAlias = pulumi.Input[PolicyStatements]
|
|
16
17
|
|
|
18
|
+
# AWS's documented cap for Lambda's direct/inline code upload (the
|
|
19
|
+
# CreateFunction/UpdateFunctionCode API's request-body limit for a
|
|
20
|
+
# base64-encoded zip passed as `code=`). S3-based upload (code_bucket=)
|
|
21
|
+
# raises this to 250MB unzipped.
|
|
22
|
+
DIRECT_UPLOAD_SIZE_LIMIT = 50 * 1024 * 1024
|
|
23
|
+
|
|
24
|
+
CodeBucketInput = Union[str, pulumi.Output[str], "aws.s3.BucketV2", "aws.s3.Bucket"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _code_bucket_name(bucket: CodeBucketInput) -> pulumi.Input[str]:
|
|
28
|
+
if isinstance(bucket, (aws.s3.BucketV2, aws.s3.Bucket)):
|
|
29
|
+
return bucket.bucket
|
|
30
|
+
return bucket
|
|
31
|
+
|
|
17
32
|
|
|
18
33
|
class Function(pulumi.ComponentResource):
|
|
19
34
|
lambda_: aws.lambda_.Function
|
|
@@ -32,6 +47,7 @@ class Function(pulumi.ComponentResource):
|
|
|
32
47
|
environment: dict[str, Union[str, pulumi.Output[str]]] = None,
|
|
33
48
|
policy_statements: Optional[PolicyStatementsInput] = None,
|
|
34
49
|
vpc_config: Optional[dict] = None,
|
|
50
|
+
code_bucket: Optional[CodeBucketInput] = None,
|
|
35
51
|
opts=None,
|
|
36
52
|
):
|
|
37
53
|
super().__init__("cloud_foundry:lambda:Function", name, {}, opts)
|
|
@@ -47,6 +63,7 @@ class Function(pulumi.ComponentResource):
|
|
|
47
63
|
self.timeout = timeout
|
|
48
64
|
self.policy_statements = policy_statements or []
|
|
49
65
|
self.vpc_config = vpc_config or {}
|
|
66
|
+
self.code_bucket = code_bucket
|
|
50
67
|
self.function_name = f"{pulumi.get_project()}-{pulumi.get_stack()}-{self.name}"
|
|
51
68
|
self.log_group_name = f"/aws/lambda/{self.function_name}"
|
|
52
69
|
# Validate that the environment is a dictionary with string
|
|
@@ -143,9 +160,61 @@ class Function(pulumi.ComponentResource):
|
|
|
143
160
|
|
|
144
161
|
log.info("Environment args type: %s", type(environment_args))
|
|
145
162
|
|
|
163
|
+
code_depends_on = []
|
|
164
|
+
if self.code_bucket is not None:
|
|
165
|
+
# Explicit opt-in: upload via S3 instead of passing the zip
|
|
166
|
+
# inline. Keyed by content hash so unchanged code across
|
|
167
|
+
# deploys reuses the same object instead of re-uploading.
|
|
168
|
+
#
|
|
169
|
+
# hash is required here (unlike the inline path below, which
|
|
170
|
+
# doesn't need it): without a real value, the key degrades to
|
|
171
|
+
# "name/None.zip" -- every version collides on the same S3
|
|
172
|
+
# object, and source_code_hash=None below means Pulumi's own
|
|
173
|
+
# diff sees no change between deploys either, so a later code
|
|
174
|
+
# change wouldn't even trigger a Lambda update.
|
|
175
|
+
if not self.hash:
|
|
176
|
+
raise ValueError(
|
|
177
|
+
f"Function '{self.name}': code_bucket requires hash= "
|
|
178
|
+
f"(used for both the S3 object key and Lambda change "
|
|
179
|
+
f"detection). python_function() always supplies this; "
|
|
180
|
+
f"only the lower-level Function()/function() API can "
|
|
181
|
+
f"omit it."
|
|
182
|
+
)
|
|
183
|
+
bucket_name = _code_bucket_name(self.code_bucket)
|
|
184
|
+
code_object = aws.s3.BucketObjectv2(
|
|
185
|
+
f"{self.name}-code",
|
|
186
|
+
bucket=bucket_name,
|
|
187
|
+
key=f"{self.name}/{self.hash}.zip",
|
|
188
|
+
source=pulumi.FileAsset(self.archive_location),
|
|
189
|
+
opts=pulumi.ResourceOptions(parent=self),
|
|
190
|
+
)
|
|
191
|
+
code_args = {"s3_bucket": bucket_name, "s3_key": code_object.key}
|
|
192
|
+
code_depends_on = [code_object]
|
|
193
|
+
else:
|
|
194
|
+
# Pulumi resource creation is async (the actual CreateFunction
|
|
195
|
+
# call happens in the provider plugin during apply, not
|
|
196
|
+
# synchronously here), so AWS's RequestEntityTooLargeException
|
|
197
|
+
# can't be caught with a plain try/except around the resource
|
|
198
|
+
# declaration below -- check the size ourselves first instead,
|
|
199
|
+
# for the same practical effect: a clear error at plan time
|
|
200
|
+
# instead of a cryptic one after an apply attempt.
|
|
201
|
+
archive_size = os.path.getsize(self.archive_location)
|
|
202
|
+
if archive_size > DIRECT_UPLOAD_SIZE_LIMIT:
|
|
203
|
+
raise ValueError(
|
|
204
|
+
f"Lambda deployment package for '{self.name}' is "
|
|
205
|
+
f"{archive_size / (1024 * 1024):.1f}MB, over AWS's "
|
|
206
|
+
f"~50MB limit for direct/inline upload (the "
|
|
207
|
+
f"CreateFunction API's request-body cap). Pass "
|
|
208
|
+
f"code_bucket=<an aws.s3.BucketV2/Bucket or bucket "
|
|
209
|
+
f"name> to python_function()/Function() to upload it "
|
|
210
|
+
f"via S3 instead, which supports packages up to "
|
|
211
|
+
f"250MB unzipped."
|
|
212
|
+
)
|
|
213
|
+
code_args = {"code": pulumi.FileArchive(self.archive_location)}
|
|
214
|
+
|
|
146
215
|
self.lambda_ = aws.lambda_.Function(
|
|
147
216
|
f"{self.name}-function",
|
|
148
|
-
|
|
217
|
+
**code_args,
|
|
149
218
|
name=self.function_name,
|
|
150
219
|
role=execution_role.arn,
|
|
151
220
|
memory_size=self.memory_size,
|
|
@@ -157,7 +226,8 @@ class Function(pulumi.ComponentResource):
|
|
|
157
226
|
environment=environment_args,
|
|
158
227
|
vpc_config=vpc_config_args,
|
|
159
228
|
opts=pulumi.ResourceOptions(
|
|
160
|
-
depends_on=[execution_role, log_group],
|
|
229
|
+
depends_on=[execution_role, log_group] + code_depends_on,
|
|
230
|
+
parent=self,
|
|
161
231
|
),
|
|
162
232
|
)
|
|
163
233
|
|
|
@@ -351,6 +421,7 @@ def function(
|
|
|
351
421
|
environment: dict[str, str] = None,
|
|
352
422
|
policy_statements: list = None,
|
|
353
423
|
vpc_config: dict = None,
|
|
424
|
+
code_bucket: Optional[CodeBucketInput] = None,
|
|
354
425
|
opts=None,
|
|
355
426
|
) -> Function:
|
|
356
427
|
"""
|
|
@@ -370,6 +441,12 @@ def function(
|
|
|
370
441
|
policy_statements (list): IAM policy statements for the
|
|
371
442
|
Lambda function.
|
|
372
443
|
vpc_config (dict): VPC configuration for the Lambda function.
|
|
444
|
+
code_bucket: An aws.s3.BucketV2/Bucket or bucket name to upload the
|
|
445
|
+
code archive through instead of passing it inline. Required if
|
|
446
|
+
the archive exceeds AWS's ~50MB direct-upload limit -- omitting
|
|
447
|
+
it in that case raises a ValueError explaining this parameter,
|
|
448
|
+
rather than letting the deploy fail on AWS's own cryptic
|
|
449
|
+
RequestEntityTooLargeException.
|
|
373
450
|
opts: Pulumi resource options.
|
|
374
451
|
|
|
375
452
|
Returns:
|
|
@@ -386,6 +463,7 @@ def function(
|
|
|
386
463
|
environment=environment,
|
|
387
464
|
policy_statements=policy_statements,
|
|
388
465
|
vpc_config=vpc_config,
|
|
466
|
+
code_bucket=code_bucket,
|
|
389
467
|
opts=opts,
|
|
390
468
|
)
|
|
391
469
|
|
|
@@ -6,6 +6,7 @@ from cloud_foundry.utils.logger import logger
|
|
|
6
6
|
|
|
7
7
|
from cloud_foundry.python_archive_builder import PythonArchiveBuilder
|
|
8
8
|
from cloud_foundry.pulumi.function import (
|
|
9
|
+
CodeBucketInput,
|
|
9
10
|
Function,
|
|
10
11
|
PolicyStatementsInput,
|
|
11
12
|
default_lambda_architecture,
|
|
@@ -27,6 +28,7 @@ def python_function(
|
|
|
27
28
|
vpc_config: Optional[dict] = None,
|
|
28
29
|
runtime: Optional[str] = None,
|
|
29
30
|
architecture: Optional[str] = None,
|
|
31
|
+
code_bucket: Optional[CodeBucketInput] = None,
|
|
30
32
|
opts=None,
|
|
31
33
|
) -> Function:
|
|
32
34
|
resolved_architecture = architecture or default_lambda_architecture()
|
|
@@ -49,5 +51,6 @@ def python_function(
|
|
|
49
51
|
vpc_config=vpc_config,
|
|
50
52
|
runtime=runtime,
|
|
51
53
|
architectures=[resolved_architecture],
|
|
54
|
+
code_bucket=code_bucket,
|
|
52
55
|
opts=opts,
|
|
53
56
|
)
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import concurrent.futures
|
|
2
|
+
import hashlib
|
|
3
|
+
import os
|
|
4
|
+
from mimetypes import guess_type
|
|
5
|
+
|
|
6
|
+
import pulumi
|
|
7
|
+
from pulumi.dynamic import CreateResult, DiffResult, Resource, ResourceProvider, UpdateResult
|
|
8
|
+
|
|
9
|
+
from cloud_foundry.utils.logger import logger
|
|
10
|
+
|
|
11
|
+
log = logger(__name__)
|
|
12
|
+
|
|
13
|
+
# Fields of a file's manifest entry that decide whether it has to be
|
|
14
|
+
# (re)uploaded. The local path is left out on purpose: the same build checked
|
|
15
|
+
# out somewhere else must not look like a change.
|
|
16
|
+
_OBJECT_FIELDS = ("sha256", "content_type", "cache_control")
|
|
17
|
+
|
|
18
|
+
_UPLOAD_WORKERS = 16
|
|
19
|
+
_DELETE_BATCH = 1000 # DeleteObjects limit
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class UIPublisherArgs:
|
|
23
|
+
def __init__(
|
|
24
|
+
self, name: str, dist_dir: str, project_dir: str = ".", prefix: str = ""
|
|
25
|
+
):
|
|
26
|
+
self.name = name
|
|
27
|
+
self.dist_dir = dist_dir
|
|
28
|
+
self.project_dir = project_dir
|
|
29
|
+
self.prefix = prefix
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _s3_client(connection: dict):
|
|
33
|
+
import boto3
|
|
34
|
+
from botocore.config import Config
|
|
35
|
+
|
|
36
|
+
session = boto3.session.Session(
|
|
37
|
+
profile_name=connection.get("profile") or None,
|
|
38
|
+
aws_access_key_id=connection.get("access_key") or None,
|
|
39
|
+
aws_secret_access_key=connection.get("secret_key") or None,
|
|
40
|
+
aws_session_token=connection.get("token") or None,
|
|
41
|
+
region_name=connection.get("region") or None,
|
|
42
|
+
)
|
|
43
|
+
addressing = "path" if connection.get("path_style") else "auto"
|
|
44
|
+
return session.client(
|
|
45
|
+
"s3",
|
|
46
|
+
endpoint_url=connection.get("endpoint_url") or None,
|
|
47
|
+
config=Config(s3={"addressing_style": addressing}),
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _changed_keys(old_files: dict, new_files: dict) -> list[str]:
|
|
52
|
+
"""Keys that are new or whose content or headers changed."""
|
|
53
|
+
return sorted(
|
|
54
|
+
key
|
|
55
|
+
for key, entry in new_files.items()
|
|
56
|
+
if any(
|
|
57
|
+
(old_files.get(key) or {}).get(field) != entry.get(field)
|
|
58
|
+
for field in _OBJECT_FIELDS
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _upload(client, props: dict, keys: list[str]) -> None:
|
|
64
|
+
def put(key: str) -> None:
|
|
65
|
+
entry = props["files"][key]
|
|
66
|
+
extra = {"CacheControl": entry["cache_control"]}
|
|
67
|
+
if entry.get("content_type"):
|
|
68
|
+
extra["ContentType"] = entry["content_type"]
|
|
69
|
+
with open(os.path.join(props["source_dir"], entry["path"]), "rb") as body:
|
|
70
|
+
client.put_object(Bucket=props["bucket"], Key=key, Body=body, **extra)
|
|
71
|
+
|
|
72
|
+
with concurrent.futures.ThreadPoolExecutor(_UPLOAD_WORKERS) as pool:
|
|
73
|
+
# list() re-raises the first upload error, failing the operation.
|
|
74
|
+
list(pool.map(put, keys))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _delete(client, bucket: str, keys: list[str]) -> None:
|
|
78
|
+
for start in range(0, len(keys), _DELETE_BATCH):
|
|
79
|
+
batch = keys[start : start + _DELETE_BATCH]
|
|
80
|
+
response = client.delete_objects(
|
|
81
|
+
Bucket=bucket,
|
|
82
|
+
Delete={"Objects": [{"Key": key} for key in batch], "Quiet": True},
|
|
83
|
+
)
|
|
84
|
+
if response.get("Errors"):
|
|
85
|
+
raise RuntimeError(f"failed to delete from {bucket}: {response['Errors'][:5]}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _outputs(props: dict) -> dict:
|
|
89
|
+
return {**props, "object_count": len(props["files"])}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class SiteFilesProvider(ResourceProvider):
|
|
93
|
+
"""Keeps a bucket prefix in step with a local build directory as ONE
|
|
94
|
+
Pulumi resource. Each file used to be its own BucketObjectv2, so a site
|
|
95
|
+
rebuild meant hundreds of resource steps -- each followed by a full state
|
|
96
|
+
checkpoint write, which on the DIY S3 backend dominated deploy time."""
|
|
97
|
+
|
|
98
|
+
def diff(self, _id: str, olds: dict, news: dict) -> DiffResult:
|
|
99
|
+
replaces = [name for name in ("bucket", "prefix") if olds.get(name) != news.get(name)]
|
|
100
|
+
old_files, new_files = olds.get("files") or {}, news.get("files") or {}
|
|
101
|
+
changed = bool(_changed_keys(old_files, new_files)) or set(old_files) != set(new_files)
|
|
102
|
+
return DiffResult(
|
|
103
|
+
changes=bool(replaces) or changed,
|
|
104
|
+
replaces=replaces,
|
|
105
|
+
delete_before_replace=False,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def create(self, props: dict) -> CreateResult:
|
|
109
|
+
client = _s3_client(props["connection"])
|
|
110
|
+
_upload(client, props, sorted(props["files"]))
|
|
111
|
+
return CreateResult(
|
|
112
|
+
id_=f"{props['bucket']}/{props['prefix']}", outs=_outputs(props)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def update(self, _id: str, olds: dict, news: dict) -> UpdateResult:
|
|
116
|
+
client = _s3_client(news["connection"])
|
|
117
|
+
old_files = olds.get("files") or {}
|
|
118
|
+
_upload(client, news, _changed_keys(old_files, news["files"]))
|
|
119
|
+
# Uploads first, so a new page never points at an asset that isn't
|
|
120
|
+
# there yet; then drop what the new build no longer has.
|
|
121
|
+
_delete(client, news["bucket"], sorted(set(old_files) - set(news["files"])))
|
|
122
|
+
return UpdateResult(outs=_outputs(news))
|
|
123
|
+
|
|
124
|
+
def delete(self, _id: str, props: dict) -> None:
|
|
125
|
+
# Leaves the bucket empty of our objects, so a bucket without
|
|
126
|
+
# force_destroy (production) can still be deleted after this.
|
|
127
|
+
client = _s3_client(props["connection"])
|
|
128
|
+
try:
|
|
129
|
+
_delete(client, props["bucket"], sorted(props.get("files") or {}))
|
|
130
|
+
except client.exceptions.NoSuchBucket:
|
|
131
|
+
pass
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class SiteFiles(Resource):
|
|
135
|
+
object_count: pulumi.Output[int]
|
|
136
|
+
|
|
137
|
+
def __init__(self, name: str, props: dict, opts: pulumi.ResourceOptions = None):
|
|
138
|
+
super().__init__(
|
|
139
|
+
SiteFilesProvider(),
|
|
140
|
+
name,
|
|
141
|
+
{**props, "object_count": None},
|
|
142
|
+
opts,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _connection() -> dict:
|
|
147
|
+
"""How the sync reaches S3. A dynamic provider talks to AWS through boto3
|
|
148
|
+
rather than the pulumi-aws provider, so the stack's aws: config (region,
|
|
149
|
+
profile, static keys, LocalStack endpoints) is handed over explicitly."""
|
|
150
|
+
config = pulumi.Config("aws")
|
|
151
|
+
endpoints = config.get_object("endpoints") or []
|
|
152
|
+
s3_endpoint = next(
|
|
153
|
+
(entry["s3"] for entry in endpoints if isinstance(entry, dict) and entry.get("s3")),
|
|
154
|
+
None,
|
|
155
|
+
)
|
|
156
|
+
connection = {
|
|
157
|
+
# None falls back to AWS_REGION / the profile, as the aws provider does.
|
|
158
|
+
"region": config.get("region"),
|
|
159
|
+
"profile": config.get("profile"),
|
|
160
|
+
"endpoint_url": s3_endpoint,
|
|
161
|
+
"path_style": (config.get("s3UsePathStyle") or "").lower() == "true",
|
|
162
|
+
}
|
|
163
|
+
access_key, secret_key = config.get("accessKey"), config.get_secret("secretKey")
|
|
164
|
+
if access_key and secret_key is not None:
|
|
165
|
+
connection["access_key"] = access_key
|
|
166
|
+
connection["secret_key"] = secret_key
|
|
167
|
+
token = config.get_secret("token")
|
|
168
|
+
if token is not None:
|
|
169
|
+
connection["token"] = token
|
|
170
|
+
return connection
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class UIPublisher(pulumi.ComponentResource):
|
|
174
|
+
"""
|
|
175
|
+
A Pulumi component to handle the publishing of UI assets to an S3 bucket.
|
|
176
|
+
|
|
177
|
+
All files under the build directory are published by a single resource
|
|
178
|
+
that uploads only new or changed files (by content hash, content type and
|
|
179
|
+
cache policy) and deletes files the build no longer contains.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
def __init__(
|
|
183
|
+
self,
|
|
184
|
+
bucket,
|
|
185
|
+
args: UIPublisherArgs,
|
|
186
|
+
opts: pulumi.ResourceOptions = None,
|
|
187
|
+
):
|
|
188
|
+
"""
|
|
189
|
+
Initialize the UIPublisher component.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
bucket: The S3 bucket resource to upload files to.
|
|
193
|
+
args (UIPublisherArgs): The arguments for the UIPublisher component.
|
|
194
|
+
opts (ResourceOptions): Optional resource options.
|
|
195
|
+
"""
|
|
196
|
+
super().__init__("cloud_foundry:pulumi:UIPublisher", args.name, {}, opts)
|
|
197
|
+
|
|
198
|
+
log.info(f"args: {args.__dict__}")
|
|
199
|
+
self.bucket = bucket
|
|
200
|
+
self.dist_dir = args.dist_dir or os.path.join(args.project_dir, "dist")
|
|
201
|
+
|
|
202
|
+
self.files = self.upload_files(self.dist_dir, bucket, args.prefix)
|
|
203
|
+
|
|
204
|
+
self.register_outputs({})
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def cache_control_for_key(key: str) -> str:
|
|
208
|
+
"""Return a conservative cache policy for a static site object."""
|
|
209
|
+
normalized = key.lower().lstrip("/")
|
|
210
|
+
|
|
211
|
+
if normalized == "_payload.json" or normalized.endswith("/_payload.json"):
|
|
212
|
+
return "public, max-age=31536000, immutable"
|
|
213
|
+
|
|
214
|
+
if normalized.startswith("_nuxt/"):
|
|
215
|
+
return "public, max-age=31536000, immutable"
|
|
216
|
+
|
|
217
|
+
if normalized.endswith(".html"):
|
|
218
|
+
return "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600"
|
|
219
|
+
|
|
220
|
+
if normalized.endswith(
|
|
221
|
+
(
|
|
222
|
+
".js",
|
|
223
|
+
".mjs",
|
|
224
|
+
".css",
|
|
225
|
+
".svg",
|
|
226
|
+
".png",
|
|
227
|
+
".jpg",
|
|
228
|
+
".jpeg",
|
|
229
|
+
".gif",
|
|
230
|
+
".webp",
|
|
231
|
+
".ico",
|
|
232
|
+
".woff",
|
|
233
|
+
".woff2",
|
|
234
|
+
".ttf",
|
|
235
|
+
".otf",
|
|
236
|
+
".eot",
|
|
237
|
+
)
|
|
238
|
+
):
|
|
239
|
+
return "public, max-age=31536000, immutable"
|
|
240
|
+
|
|
241
|
+
if normalized.endswith((".json", ".xml", ".txt", ".webmanifest")):
|
|
242
|
+
return "public, max-age=3600, stale-while-revalidate=300"
|
|
243
|
+
|
|
244
|
+
return "public, max-age=3600"
|
|
245
|
+
|
|
246
|
+
def remap_path_to_s3(self, dir_base: str, key_base: str):
|
|
247
|
+
"""
|
|
248
|
+
Remap local file paths to S3 keys.
|
|
249
|
+
|
|
250
|
+
Args:
|
|
251
|
+
dir_base (str): The base directory containing the files.
|
|
252
|
+
key_base (str): The base key to prepend to the S3 keys.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
list[dict]: A list of dictionaries containing the local file paths and corresponding S3 keys.
|
|
256
|
+
"""
|
|
257
|
+
log.info(f"remap: dir_base: {dir_base}")
|
|
258
|
+
dir_base = os.path.abspath(dir_base)
|
|
259
|
+
return [
|
|
260
|
+
{
|
|
261
|
+
"path": os.path.join(root, file),
|
|
262
|
+
"key": os.path.join(
|
|
263
|
+
key_base, os.path.relpath(os.path.join(root, file), dir_base)
|
|
264
|
+
).replace("\\", "/"),
|
|
265
|
+
}
|
|
266
|
+
for root, _, files in os.walk(dir_base)
|
|
267
|
+
for file in files
|
|
268
|
+
]
|
|
269
|
+
|
|
270
|
+
def build_manifest(self, dir: str, key: str = "") -> dict:
|
|
271
|
+
"""Map each S3 key to its file (relative to ``dir``), content hash,
|
|
272
|
+
content type and cache policy."""
|
|
273
|
+
dir_base = os.path.abspath(dir)
|
|
274
|
+
manifest = {}
|
|
275
|
+
for item in self.remap_path_to_s3(dir_base, key):
|
|
276
|
+
with open(item["path"], "rb") as f:
|
|
277
|
+
sha256 = hashlib.sha256(f.read()).hexdigest()
|
|
278
|
+
content_type, _ = guess_type(item["path"])
|
|
279
|
+
manifest[item["key"]] = {
|
|
280
|
+
"path": os.path.relpath(item["path"], dir_base).replace("\\", "/"),
|
|
281
|
+
"sha256": sha256,
|
|
282
|
+
"content_type": content_type or "",
|
|
283
|
+
"cache_control": self.cache_control_for_key(item["key"]),
|
|
284
|
+
}
|
|
285
|
+
return manifest
|
|
286
|
+
|
|
287
|
+
def upload_files(self, dir: str, bucket, key: str = "") -> SiteFiles:
|
|
288
|
+
"""
|
|
289
|
+
Publish the files in a directory to an S3 bucket as one resource.
|
|
290
|
+
|
|
291
|
+
Args:
|
|
292
|
+
dir (str): The directory containing the files to upload.
|
|
293
|
+
bucket: The S3 bucket resource to upload files to.
|
|
294
|
+
key (str): The prefix to add to the S3 keys (optional).
|
|
295
|
+
"""
|
|
296
|
+
return SiteFiles(
|
|
297
|
+
f"{self._name}-files",
|
|
298
|
+
{
|
|
299
|
+
"bucket": bucket.id,
|
|
300
|
+
"prefix": key,
|
|
301
|
+
"source_dir": os.path.abspath(dir),
|
|
302
|
+
"files": self.build_manifest(dir, key),
|
|
303
|
+
"connection": _connection(),
|
|
304
|
+
},
|
|
305
|
+
opts=pulumi.ResourceOptions(parent=self, depends_on=[bucket]),
|
|
306
|
+
)
|
|
@@ -15,6 +15,18 @@ from urllib.parse import urlparse
|
|
|
15
15
|
|
|
16
16
|
log = logger(__name__)
|
|
17
17
|
|
|
18
|
+
# Bytecode is left out of both the cache hash and the archive. pip compiles
|
|
19
|
+
# with the *build* interpreter, so the .pyc files are tagged for that version
|
|
20
|
+
# (useless to a Lambda runtime on another minor version) and carry the
|
|
21
|
+
# install time in their header -- which changed the hash and the zip bytes on
|
|
22
|
+
# every build and made every deploy re-upload every function.
|
|
23
|
+
_EXCLUDED_DIRS = frozenset({"__pycache__"})
|
|
24
|
+
_EXCLUDED_SUFFIXES = (".pyc", ".pyo")
|
|
25
|
+
|
|
26
|
+
# Fixed zip entry timestamp (the earliest the zip format can store), so an
|
|
27
|
+
# unchanged tree always produces byte-identical archive bytes.
|
|
28
|
+
_ZIP_EPOCH = (1980, 1, 1, 0, 0, 0)
|
|
29
|
+
|
|
18
30
|
|
|
19
31
|
class PythonArchiveBuilder(ArchiveBuilder):
|
|
20
32
|
"""
|
|
@@ -87,13 +99,14 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
87
99
|
hash_comparator.write(self._hash, self._base_dir)
|
|
88
100
|
|
|
89
101
|
def _build_cache_hash(self, hash_comparator: HashComparator) -> str:
|
|
90
|
-
staging_hash = hash_comparator.hash_folder(self._staging)
|
|
91
|
-
libs_hash = hash_comparator.hash_folder(self._libs)
|
|
92
102
|
digest = hashlib.sha256()
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
103
|
+
for folder in (self._staging, self._libs):
|
|
104
|
+
for relative_path, full_path in self._archive_files(folder):
|
|
105
|
+
digest.update(relative_path.encode("utf-8"))
|
|
106
|
+
digest.update(b"\0")
|
|
107
|
+
digest.update(hash_comparator.hash_file(full_path).encode("utf-8"))
|
|
108
|
+
digest.update(b"\0")
|
|
109
|
+
digest.update(b"\0")
|
|
97
110
|
digest.update(self._target_architecture.encode("utf-8"))
|
|
98
111
|
digest.update(b"\0")
|
|
99
112
|
for requirement in self._requirements:
|
|
@@ -119,6 +132,22 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
119
132
|
return ["manylinux2014_aarch64", "manylinux_2_17_aarch64"]
|
|
120
133
|
return ["manylinux2014_x86_64", "manylinux_2_17_x86_64"]
|
|
121
134
|
|
|
135
|
+
@staticmethod
|
|
136
|
+
def _archive_files(folder: str) -> list[tuple[str, str]]:
|
|
137
|
+
"""(relative path, full path) of every file that goes into the
|
|
138
|
+
archive from ``folder``, in a stable order. The cache hash and the
|
|
139
|
+
archive both use this list, so they always agree on the contents."""
|
|
140
|
+
files = []
|
|
141
|
+
for root, dirs, names in os.walk(folder):
|
|
142
|
+
dirs[:] = sorted(d for d in dirs if d not in _EXCLUDED_DIRS)
|
|
143
|
+
for name in sorted(names):
|
|
144
|
+
if name.endswith(_EXCLUDED_SUFFIXES):
|
|
145
|
+
continue
|
|
146
|
+
full_path = os.path.join(root, name)
|
|
147
|
+
relative_path = os.path.relpath(full_path, folder)
|
|
148
|
+
files.append((relative_path.replace(os.sep, "/"), full_path))
|
|
149
|
+
return files
|
|
150
|
+
|
|
122
151
|
def hash(self) -> str:
|
|
123
152
|
"""Return the hash of the current archive."""
|
|
124
153
|
return self._hash
|
|
@@ -148,20 +177,19 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
148
177
|
"""
|
|
149
178
|
log.info(f"building archive: {self.name}")
|
|
150
179
|
try:
|
|
151
|
-
#
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
) as archive:
|
|
180
|
+
# Entries are written in a fixed order with a fixed timestamp and
|
|
181
|
+
# normalized permissions, so the same tree always gives the same
|
|
182
|
+
# bytes -- Pulumi diffs the archive by content.
|
|
183
|
+
with zipfile.ZipFile(self._location, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
156
184
|
# Include both 'staging' and 'libs' folders in the archive
|
|
157
|
-
for folder in
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
185
|
+
for folder in (self._staging, self._libs):
|
|
186
|
+
for relative_path, full_path in self._archive_files(folder):
|
|
187
|
+
info = zipfile.ZipInfo(relative_path, date_time=_ZIP_EPOCH)
|
|
188
|
+
info.compress_type = zipfile.ZIP_DEFLATED
|
|
189
|
+
executable = os.stat(full_path).st_mode & 0o111
|
|
190
|
+
info.external_attr = (0o100755 if executable else 0o100644) << 16
|
|
191
|
+
with open(full_path, "rb") as source:
|
|
192
|
+
archive.writestr(info, source.read())
|
|
165
193
|
|
|
166
194
|
log.info("Archive built successfully")
|
|
167
195
|
except Exception as e:
|
|
@@ -245,6 +273,7 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
245
273
|
"--python-version",
|
|
246
274
|
"3.12",
|
|
247
275
|
"--upgrade",
|
|
276
|
+
"--no-compile",
|
|
248
277
|
"-r",
|
|
249
278
|
requirements_file,
|
|
250
279
|
],
|
|
@@ -283,6 +312,7 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
283
312
|
"--target",
|
|
284
313
|
self._libs,
|
|
285
314
|
"--upgrade",
|
|
315
|
+
"--no-compile",
|
|
286
316
|
"-r",
|
|
287
317
|
requirements_file,
|
|
288
318
|
]
|
|
@@ -3,13 +3,22 @@ import os
|
|
|
3
3
|
|
|
4
4
|
# Configuring the logging module with basic settings, including format and log level,
|
|
5
5
|
# where the log level is obtained from the environment variable LOGGING_LEVEL
|
|
6
|
-
# with a default of
|
|
6
|
+
# with a default of INFO, and force=True to ensure the configuration is applied immediately.
|
|
7
7
|
logging.basicConfig(
|
|
8
8
|
format="%(name)s:%(lineno)s - %(levelname)s - %(message)s",
|
|
9
|
-
level=os.getenv("LOGGING_LEVEL", "
|
|
9
|
+
level=os.getenv("LOGGING_LEVEL", "INFO").upper(),
|
|
10
10
|
force=True,
|
|
11
11
|
)
|
|
12
12
|
|
|
13
|
+
# At DEBUG, botocore logs signed request headers (including
|
|
14
|
+
# x-amz-security-token) and request bodies. Keep the AWS SDK quiet whatever
|
|
15
|
+
# LOGGING_LEVEL says; AWS_SDK_LOGGING_LEVEL turns it back on deliberately.
|
|
16
|
+
AWS_SDK_LOGGERS = ("boto3", "botocore", "s3transfer", "urllib3")
|
|
17
|
+
for _sdk_logger in AWS_SDK_LOGGERS:
|
|
18
|
+
logging.getLogger(_sdk_logger).setLevel(
|
|
19
|
+
os.getenv("AWS_SDK_LOGGING_LEVEL", "WARNING").upper()
|
|
20
|
+
)
|
|
21
|
+
|
|
13
22
|
WARN = logging.WARN
|
|
14
23
|
INFO = logging.INFO
|
|
15
24
|
DEBUG = logging.DEBUG
|
|
@@ -27,8 +36,8 @@ def logger(name=None):
|
|
|
27
36
|
|
|
28
37
|
"""
|
|
29
38
|
# Retrieving the logging level from the environment variable LOGGING_LEVEL
|
|
30
|
-
# with a default of
|
|
31
|
-
loggingLevel = os.getenv("LOGGING_LEVEL", "
|
|
39
|
+
# with a default of INFO, and converting it to uppercase
|
|
40
|
+
loggingLevel = os.getenv("LOGGING_LEVEL", "INFO").upper()
|
|
32
41
|
|
|
33
42
|
# Setting the logging level for the root logger to the obtained logging level
|
|
34
43
|
logging.getLogger().setLevel(loggingLevel)
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
# src/pyproject.toml
|
|
2
2
|
|
|
3
3
|
[build-system]
|
|
4
|
-
|
|
4
|
+
# hatchling>=1.32.0 emits Metadata-Version: 2.5, which PyPI's Warehouse
|
|
5
|
+
# rejects outright with a 400 on every upload regardless of package
|
|
6
|
+
# version -- see https://github.com/pypi/warehouse/issues/19083 (open,
|
|
7
|
+
# unresolved). Pinned below that until Warehouse adds 2.5 support.
|
|
8
|
+
requires = ["hatchling>=1.0,<1.32.0"]
|
|
5
9
|
build-backend = "hatchling.build"
|
|
6
10
|
|
|
7
11
|
[project]
|
|
@@ -45,10 +49,24 @@ dev = [
|
|
|
45
49
|
"requests",
|
|
46
50
|
"python-dotenv",
|
|
47
51
|
"pyjwt",
|
|
48
|
-
"fixture_foundry",
|
|
49
52
|
"psycopg2-binary",
|
|
53
|
+
# In-process S3 for the UIPublisher sync provider's unit tests.
|
|
54
|
+
"moto[s3]>=5.0",
|
|
50
55
|
]
|
|
51
56
|
|
|
57
|
+
# fixture_foundry is NOT listed above on purpose. PyPI's upload API rejects
|
|
58
|
+
# any package whose metadata declares a direct URL/VCS dependency, even one
|
|
59
|
+
# scoped to an optional extra -- so a git-pinned entry here would block
|
|
60
|
+
# every future `cloud-foundry` publish, not just fail quietly. Published
|
|
61
|
+
# fixture_foundry (still 0.2.4 as of this pin) has stale calls into its own
|
|
62
|
+
# context.py API that crash test collection (see DanRepik/fixture_foundry#4,
|
|
63
|
+
# fixed on branch fix/fixtures-stale-teardown-calls but not yet released).
|
|
64
|
+
# Install it separately before running the integration test suite:
|
|
65
|
+
# pip install git+https://github.com/DanRepik/fixture_foundry.git@fix/fixtures-stale-teardown-calls
|
|
66
|
+
# CI does this as its own step -- see .github/workflows/pytest.yml and
|
|
67
|
+
# publish.yml. Once a fixed version is published to PyPI, add a plain
|
|
68
|
+
# "fixture_foundry>=X.Y.Z" back to the dev extra above and drop this note.
|
|
69
|
+
|
|
52
70
|
[tool.pytest.ini_options]
|
|
53
71
|
pythonpath = [".."]
|
|
54
72
|
testpaths = ["tests"]
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
import pulumi
|
|
2
|
-
import pulumi_aws as aws
|
|
3
|
-
import os
|
|
4
|
-
from mimetypes import guess_type
|
|
5
|
-
|
|
6
|
-
from cloud_foundry.utils.logger import logger
|
|
7
|
-
|
|
8
|
-
log = logger(__name__)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class UIPublisherArgs:
|
|
12
|
-
def __init__(
|
|
13
|
-
self, name: str, dist_dir: str, project_dir: str = ".", prefix: str = ""
|
|
14
|
-
):
|
|
15
|
-
self.name = name
|
|
16
|
-
self.dist_dir = dist_dir
|
|
17
|
-
self.project_dir = project_dir
|
|
18
|
-
self.prefix = prefix
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class UIPublisher(pulumi.ComponentResource):
|
|
22
|
-
"""
|
|
23
|
-
A Pulumi component to handle the publishing of UI assets to an S3 bucket.
|
|
24
|
-
|
|
25
|
-
This component takes care of uploading files from a specified directory to an S3 bucket,
|
|
26
|
-
setting the appropriate content type for each file based on its extension.
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
def __init__(
|
|
30
|
-
self,
|
|
31
|
-
bucket,
|
|
32
|
-
args: UIPublisherArgs,
|
|
33
|
-
opts: pulumi.ResourceOptions = None,
|
|
34
|
-
):
|
|
35
|
-
"""
|
|
36
|
-
Initialize the UIPublisher component.
|
|
37
|
-
|
|
38
|
-
Args:
|
|
39
|
-
name (str): The name of the component.
|
|
40
|
-
bucket: The S3 bucket resource to upload files to.
|
|
41
|
-
args (UIPublisherArgs): The arguments for the UIPublisher component.
|
|
42
|
-
opts (ResourceOptions): Optional resource options.
|
|
43
|
-
"""
|
|
44
|
-
super().__init__("cloud_foundry:pulumi:UIPublisher", args.name, {}, opts)
|
|
45
|
-
|
|
46
|
-
log.info(f"args: {args.__dict__}")
|
|
47
|
-
self.bucket = bucket
|
|
48
|
-
self.dist_dir = args.dist_dir or os.path.join(args.project_dir, "dist")
|
|
49
|
-
|
|
50
|
-
self.upload_files(self.dist_dir, bucket, args.prefix)
|
|
51
|
-
|
|
52
|
-
self.register_outputs({})
|
|
53
|
-
|
|
54
|
-
@staticmethod
|
|
55
|
-
def cache_control_for_key(key: str) -> str:
|
|
56
|
-
"""Return a conservative cache policy for a static site object."""
|
|
57
|
-
normalized = key.lower().lstrip("/")
|
|
58
|
-
|
|
59
|
-
if normalized == "_payload.json" or normalized.endswith("/_payload.json"):
|
|
60
|
-
return "public, max-age=31536000, immutable"
|
|
61
|
-
|
|
62
|
-
if normalized.startswith("_nuxt/"):
|
|
63
|
-
return "public, max-age=31536000, immutable"
|
|
64
|
-
|
|
65
|
-
if normalized.endswith(".html"):
|
|
66
|
-
return "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600"
|
|
67
|
-
|
|
68
|
-
if normalized.endswith(
|
|
69
|
-
(
|
|
70
|
-
".js",
|
|
71
|
-
".mjs",
|
|
72
|
-
".css",
|
|
73
|
-
".svg",
|
|
74
|
-
".png",
|
|
75
|
-
".jpg",
|
|
76
|
-
".jpeg",
|
|
77
|
-
".gif",
|
|
78
|
-
".webp",
|
|
79
|
-
".ico",
|
|
80
|
-
".woff",
|
|
81
|
-
".woff2",
|
|
82
|
-
".ttf",
|
|
83
|
-
".otf",
|
|
84
|
-
".eot",
|
|
85
|
-
)
|
|
86
|
-
):
|
|
87
|
-
return "public, max-age=31536000, immutable"
|
|
88
|
-
|
|
89
|
-
if normalized.endswith((".json", ".xml", ".txt", ".webmanifest")):
|
|
90
|
-
return "public, max-age=3600, stale-while-revalidate=300"
|
|
91
|
-
|
|
92
|
-
return "public, max-age=3600"
|
|
93
|
-
|
|
94
|
-
def remap_path_to_s3(self, dir_base: str, key_base: str):
|
|
95
|
-
"""
|
|
96
|
-
Remap local file paths to S3 keys.
|
|
97
|
-
|
|
98
|
-
Args:
|
|
99
|
-
dir_base (str): The base directory containing the files.
|
|
100
|
-
key_base (str): The base key to prepend to the S3 keys.
|
|
101
|
-
|
|
102
|
-
Returns:
|
|
103
|
-
list[dict]: A list of dictionaries containing the local file paths and corresponding S3 keys.
|
|
104
|
-
"""
|
|
105
|
-
log.info(f"remap: dir_base: {dir_base}")
|
|
106
|
-
dir_base = os.path.abspath(dir_base)
|
|
107
|
-
return [
|
|
108
|
-
{
|
|
109
|
-
"path": os.path.join(root, file),
|
|
110
|
-
"key": os.path.join(
|
|
111
|
-
key_base, os.path.relpath(os.path.join(root, file), dir_base)
|
|
112
|
-
).replace("\\", "/"),
|
|
113
|
-
}
|
|
114
|
-
for root, _, files in os.walk(dir_base)
|
|
115
|
-
for file in files
|
|
116
|
-
]
|
|
117
|
-
|
|
118
|
-
def upload_files(self, dir: str, bucket, key: str = ""):
|
|
119
|
-
"""
|
|
120
|
-
Upload files from a directory to an S3 bucket.
|
|
121
|
-
|
|
122
|
-
Args:
|
|
123
|
-
dir (str): The directory containing the files to upload.
|
|
124
|
-
bucket: The S3 bucket resource to upload files to.
|
|
125
|
-
key (str): The prefix to add to the S3 keys (optional).
|
|
126
|
-
"""
|
|
127
|
-
for item in self.remap_path_to_s3(dir, key):
|
|
128
|
-
content_type, _ = guess_type(item["path"])
|
|
129
|
-
aws.s3.BucketObjectv2(
|
|
130
|
-
item["key"],
|
|
131
|
-
bucket=bucket.id,
|
|
132
|
-
key=item["key"],
|
|
133
|
-
source=pulumi.FileAsset(item["path"]),
|
|
134
|
-
content_type=content_type,
|
|
135
|
-
cache_control=self.cache_control_for_key(item["key"]),
|
|
136
|
-
opts=pulumi.ResourceOptions(parent=self, depends_on=[bucket]),
|
|
137
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|