cloud-foundry 0.3.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/PKG-INFO +2 -1
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/__init__.py +1 -1
- cloud_foundry-0.4.2/cloud_foundry/migrations/forget_site_objects.py +141 -0
- cloud_foundry-0.4.2/cloud_foundry/pulumi/__init__.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/cdn_site_origin.py +5 -3
- cloud_foundry-0.4.2/cloud_foundry/pulumi/ui_publisher.py +306 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/python_archive_builder.py +49 -19
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/logger.py +13 -4
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/pyproject.toml +2 -0
- cloud_foundry-0.3.1/cloud_foundry/pulumi/ui_publisher.py +0 -137
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/.gitignore +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/LICENSE +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/README.md +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/archive_builder.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/mail_publisher.py +0 -0
- {cloud_foundry-0.3.1/cloud_foundry/pulumi → cloud_foundry-0.4.2/cloud_foundry/migrations}/__init__.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/api_waf.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/cdn.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/cdn_api_origin.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/connection_store.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/custom_domain.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/document_repository.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/domain.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/function.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/idle_reaper.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/publisher.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/python_function.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/queue.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/rest_api.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/site_bucket.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/topic.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/websocket_api.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/websocket_function.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/services/authorization_api/security_lambda.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/services/mail_publisher.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/aws_openapi_editor.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/hash_comparator.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/idle_reaper_client.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/names.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/openapi_editor.py +0 -0
- {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/websocket_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloud-foundry
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: cloud-foundry is a curated set of cloud components that can be used to assemble cloud-based applications.
|
|
5
5
|
Project-URL: Documentation, https://github.com/DanRepik/cloud-foundry
|
|
6
6
|
Project-URL: Source, https://github.com/DanRepik/cloud-foundry
|
|
@@ -20,6 +20,7 @@ Requires-Dist: docker; extra == 'dev'
|
|
|
20
20
|
Requires-Dist: flake8; extra == 'dev'
|
|
21
21
|
Requires-Dist: flit>=3.9; extra == 'dev'
|
|
22
22
|
Requires-Dist: isort>=5.12.0; extra == 'dev'
|
|
23
|
+
Requires-Dist: moto[s3]>=5.0; extra == 'dev'
|
|
23
24
|
Requires-Dist: mypy>=1.0.0; extra == 'dev'
|
|
24
25
|
Requires-Dist: pre-commit; extra == 'dev'
|
|
25
26
|
Requires-Dist: psycopg2-binary; extra == 'dev'
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""One-time state migration for the single-resource UIPublisher.
|
|
2
|
+
|
|
3
|
+
UIPublisher used to declare one aws.s3.BucketObjectv2 per site file. It now
|
|
4
|
+
publishes the whole build through one resource. Deploying the new version
|
|
5
|
+
straight over old state would be destructive: Pulumi runs the new resource's
|
|
6
|
+
upload first and deletes the no-longer-declared per-file resources last, and
|
|
7
|
+
deleting those removes the very S3 objects that were just uploaded (same
|
|
8
|
+
keys), leaving the site empty.
|
|
9
|
+
|
|
10
|
+
Run this once per stack, right before the first `pulumi up` with the new
|
|
11
|
+
cloud_foundry. It removes the per-file resources from the Pulumi state only;
|
|
12
|
+
the S3 objects stay where they are, and the next deploy takes them over.
|
|
13
|
+
|
|
14
|
+
python -m cloud_foundry.migrations.forget_site_objects --stack dev [--cwd <project dir>] [--dry-run]
|
|
15
|
+
|
|
16
|
+
--cwd is the Pulumi project directory (the one holding Pulumi.yaml),
|
|
17
|
+
relative to where you run the command; it defaults to the current directory.
|
|
18
|
+
|
|
19
|
+
The exported state is saved next to the working directory first, so
|
|
20
|
+
`pulumi stack import --file <backup>` puts it back if needed.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import argparse
|
|
24
|
+
import json
|
|
25
|
+
import os
|
|
26
|
+
import shutil
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import tempfile
|
|
30
|
+
import time
|
|
31
|
+
|
|
32
|
+
PUBLISHER_TYPE = "cloud_foundry:pulumi:UIPublisher"
|
|
33
|
+
OBJECT_TYPE = "aws:s3/bucketObjectv2:BucketObjectv2"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _urn_type(urn: str) -> str:
|
|
37
|
+
# urn:pulumi:<stack>::<project>::<parent types$...$type>::<name>
|
|
38
|
+
parts = urn.split("::", 3)
|
|
39
|
+
return parts[2].split("$")[-1] if len(parts) == 4 else ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def forget_site_objects(deployment: dict) -> tuple[dict, list[str]]:
|
|
43
|
+
"""Return ``deployment`` without the per-file objects that UIPublisher
|
|
44
|
+
components used to own, and the URNs that were removed."""
|
|
45
|
+
resources = deployment.get("resources") or []
|
|
46
|
+
removed = {
|
|
47
|
+
resource["urn"]
|
|
48
|
+
for resource in resources
|
|
49
|
+
if resource.get("type") == OBJECT_TYPE
|
|
50
|
+
and _urn_type(resource.get("parent", "")) == PUBLISHER_TYPE
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
kept = []
|
|
54
|
+
for resource in resources:
|
|
55
|
+
if resource["urn"] in removed:
|
|
56
|
+
continue
|
|
57
|
+
resource = dict(resource)
|
|
58
|
+
if resource.get("dependencies"):
|
|
59
|
+
resource["dependencies"] = [u for u in resource["dependencies"] if u not in removed]
|
|
60
|
+
if resource.get("propertyDependencies"):
|
|
61
|
+
resource["propertyDependencies"] = {
|
|
62
|
+
name: [u for u in urns if u not in removed]
|
|
63
|
+
for name, urns in resource["propertyDependencies"].items()
|
|
64
|
+
}
|
|
65
|
+
if resource.get("deletedWith") in removed:
|
|
66
|
+
resource.pop("deletedWith")
|
|
67
|
+
kept.append(resource)
|
|
68
|
+
|
|
69
|
+
return {**deployment, "resources": kept}, sorted(removed)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _pulumi(args: list[str], cwd: str) -> None:
|
|
73
|
+
subprocess.run(["pulumi", *args], cwd=cwd, check=True)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _project_problem(cwd: str) -> str | None:
|
|
77
|
+
"""Why ``cwd`` can't be used as the Pulumi project directory, if it can't."""
|
|
78
|
+
if not os.path.isdir(cwd):
|
|
79
|
+
return f"{cwd} does not exist"
|
|
80
|
+
if not any(os.path.isfile(os.path.join(cwd, name)) for name in ("Pulumi.yaml", "Pulumi.yml")):
|
|
81
|
+
return f"{cwd} has no Pulumi.yaml"
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def main(argv: list[str] | None = None) -> int:
|
|
86
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
87
|
+
parser.add_argument("--stack", required=True)
|
|
88
|
+
parser.add_argument(
|
|
89
|
+
"--cwd",
|
|
90
|
+
default=".",
|
|
91
|
+
help="Pulumi project directory, relative to where you run this (default: current directory)",
|
|
92
|
+
)
|
|
93
|
+
parser.add_argument("--dry-run", action="store_true", help="report only; change nothing")
|
|
94
|
+
args = parser.parse_args(argv)
|
|
95
|
+
|
|
96
|
+
cwd = os.path.abspath(args.cwd)
|
|
97
|
+
problem = _project_problem(cwd)
|
|
98
|
+
if problem:
|
|
99
|
+
print(
|
|
100
|
+
f"{problem}. --cwd must be the Pulumi project directory, relative to "
|
|
101
|
+
f"where you run this command (it defaults to the current directory). "
|
|
102
|
+
f"Nothing was changed.",
|
|
103
|
+
file=sys.stderr,
|
|
104
|
+
)
|
|
105
|
+
return 2
|
|
106
|
+
if shutil.which("pulumi") is None:
|
|
107
|
+
print("The pulumi CLI is not on PATH. Nothing was changed.", file=sys.stderr)
|
|
108
|
+
return 2
|
|
109
|
+
stamp = time.strftime("%Y%m%dT%H%M%S")
|
|
110
|
+
backup = os.path.join(cwd, f"{args.stack}-state-backup-{stamp}.json")
|
|
111
|
+
_pulumi(["stack", "export", "--stack", args.stack, "--file", backup], cwd)
|
|
112
|
+
|
|
113
|
+
with open(backup) as f:
|
|
114
|
+
state = json.load(f)
|
|
115
|
+
deployment = state.get("deployment") or {}
|
|
116
|
+
if deployment.get("pending_operations"):
|
|
117
|
+
print(
|
|
118
|
+
"State has pending operations from an interrupted update; resolve those "
|
|
119
|
+
"(pulumi refresh / pulumi cancel) before migrating.",
|
|
120
|
+
file=sys.stderr,
|
|
121
|
+
)
|
|
122
|
+
return 1
|
|
123
|
+
|
|
124
|
+
migrated, removed = forget_site_objects(deployment)
|
|
125
|
+
print(f"{len(removed)} per-file site object(s) to drop from the {args.stack} state (S3 is not touched).")
|
|
126
|
+
if not removed or args.dry_run:
|
|
127
|
+
print(f"State export kept at {backup}")
|
|
128
|
+
return 0
|
|
129
|
+
|
|
130
|
+
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as out:
|
|
131
|
+
json.dump({**state, "deployment": migrated}, out)
|
|
132
|
+
try:
|
|
133
|
+
_pulumi(["stack", "import", "--stack", args.stack, "--file", out.name], cwd)
|
|
134
|
+
finally:
|
|
135
|
+
os.unlink(out.name)
|
|
136
|
+
print(f"Done. Previous state saved to {backup}; restore it with: pulumi stack import --stack {args.stack} --file {backup}")
|
|
137
|
+
return 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
if __name__ == "__main__":
|
|
141
|
+
sys.exit(main())
|
|
File without changes
|
|
@@ -119,9 +119,11 @@ class SiteOrigin(pulumi.ComponentResource):
|
|
|
119
119
|
origin_id=self.origin_id,
|
|
120
120
|
origin_access_control_id=origin_access_control.id,
|
|
121
121
|
origin_path=origin_path,
|
|
122
|
-
s3_origin_config
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
# No s3_origin_config: access goes through the origin access
|
|
123
|
+
# control above, and an S3 origin without one gets CloudFront's
|
|
124
|
+
# empty-identity config anyway. Declaring the empty block made
|
|
125
|
+
# every preview show an origins update, because the provider
|
|
126
|
+
# reads that block back as absent.
|
|
125
127
|
)
|
|
126
128
|
|
|
127
129
|
if origin_shield_region:
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import concurrent.futures
|
|
2
|
+
import hashlib
|
|
3
|
+
import os
|
|
4
|
+
from mimetypes import guess_type
|
|
5
|
+
|
|
6
|
+
import pulumi
|
|
7
|
+
from pulumi.dynamic import CreateResult, DiffResult, Resource, ResourceProvider, UpdateResult
|
|
8
|
+
|
|
9
|
+
from cloud_foundry.utils.logger import logger
|
|
10
|
+
|
|
11
|
+
log = logger(__name__)
|
|
12
|
+
|
|
13
|
+
# Fields of a file's manifest entry that decide whether it has to be
|
|
14
|
+
# (re)uploaded. The local path is left out on purpose: the same build checked
|
|
15
|
+
# out somewhere else must not look like a change.
|
|
16
|
+
_OBJECT_FIELDS = ("sha256", "content_type", "cache_control")
|
|
17
|
+
|
|
18
|
+
_UPLOAD_WORKERS = 16
|
|
19
|
+
_DELETE_BATCH = 1000 # DeleteObjects limit
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class UIPublisherArgs:
|
|
23
|
+
def __init__(
|
|
24
|
+
self, name: str, dist_dir: str, project_dir: str = ".", prefix: str = ""
|
|
25
|
+
):
|
|
26
|
+
self.name = name
|
|
27
|
+
self.dist_dir = dist_dir
|
|
28
|
+
self.project_dir = project_dir
|
|
29
|
+
self.prefix = prefix
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _s3_client(connection: dict):
|
|
33
|
+
import boto3
|
|
34
|
+
from botocore.config import Config
|
|
35
|
+
|
|
36
|
+
session = boto3.session.Session(
|
|
37
|
+
profile_name=connection.get("profile") or None,
|
|
38
|
+
aws_access_key_id=connection.get("access_key") or None,
|
|
39
|
+
aws_secret_access_key=connection.get("secret_key") or None,
|
|
40
|
+
aws_session_token=connection.get("token") or None,
|
|
41
|
+
region_name=connection.get("region") or None,
|
|
42
|
+
)
|
|
43
|
+
addressing = "path" if connection.get("path_style") else "auto"
|
|
44
|
+
return session.client(
|
|
45
|
+
"s3",
|
|
46
|
+
endpoint_url=connection.get("endpoint_url") or None,
|
|
47
|
+
config=Config(s3={"addressing_style": addressing}),
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _changed_keys(old_files: dict, new_files: dict) -> list[str]:
|
|
52
|
+
"""Keys that are new or whose content or headers changed."""
|
|
53
|
+
return sorted(
|
|
54
|
+
key
|
|
55
|
+
for key, entry in new_files.items()
|
|
56
|
+
if any(
|
|
57
|
+
(old_files.get(key) or {}).get(field) != entry.get(field)
|
|
58
|
+
for field in _OBJECT_FIELDS
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _upload(client, props: dict, keys: list[str]) -> None:
|
|
64
|
+
def put(key: str) -> None:
|
|
65
|
+
entry = props["files"][key]
|
|
66
|
+
extra = {"CacheControl": entry["cache_control"]}
|
|
67
|
+
if entry.get("content_type"):
|
|
68
|
+
extra["ContentType"] = entry["content_type"]
|
|
69
|
+
with open(os.path.join(props["source_dir"], entry["path"]), "rb") as body:
|
|
70
|
+
client.put_object(Bucket=props["bucket"], Key=key, Body=body, **extra)
|
|
71
|
+
|
|
72
|
+
with concurrent.futures.ThreadPoolExecutor(_UPLOAD_WORKERS) as pool:
|
|
73
|
+
# list() re-raises the first upload error, failing the operation.
|
|
74
|
+
list(pool.map(put, keys))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _delete(client, bucket: str, keys: list[str]) -> None:
|
|
78
|
+
for start in range(0, len(keys), _DELETE_BATCH):
|
|
79
|
+
batch = keys[start : start + _DELETE_BATCH]
|
|
80
|
+
response = client.delete_objects(
|
|
81
|
+
Bucket=bucket,
|
|
82
|
+
Delete={"Objects": [{"Key": key} for key in batch], "Quiet": True},
|
|
83
|
+
)
|
|
84
|
+
if response.get("Errors"):
|
|
85
|
+
raise RuntimeError(f"failed to delete from {bucket}: {response['Errors'][:5]}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _outputs(props: dict) -> dict:
|
|
89
|
+
return {**props, "object_count": len(props["files"])}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class SiteFilesProvider(ResourceProvider):
|
|
93
|
+
"""Keeps a bucket prefix in step with a local build directory as ONE
|
|
94
|
+
Pulumi resource. Each file used to be its own BucketObjectv2, so a site
|
|
95
|
+
rebuild meant hundreds of resource steps -- each followed by a full state
|
|
96
|
+
checkpoint write, which on the DIY S3 backend dominated deploy time."""
|
|
97
|
+
|
|
98
|
+
def diff(self, _id: str, olds: dict, news: dict) -> DiffResult:
|
|
99
|
+
replaces = [name for name in ("bucket", "prefix") if olds.get(name) != news.get(name)]
|
|
100
|
+
old_files, new_files = olds.get("files") or {}, news.get("files") or {}
|
|
101
|
+
changed = bool(_changed_keys(old_files, new_files)) or set(old_files) != set(new_files)
|
|
102
|
+
return DiffResult(
|
|
103
|
+
changes=bool(replaces) or changed,
|
|
104
|
+
replaces=replaces,
|
|
105
|
+
delete_before_replace=False,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def create(self, props: dict) -> CreateResult:
|
|
109
|
+
client = _s3_client(props["connection"])
|
|
110
|
+
_upload(client, props, sorted(props["files"]))
|
|
111
|
+
return CreateResult(
|
|
112
|
+
id_=f"{props['bucket']}/{props['prefix']}", outs=_outputs(props)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def update(self, _id: str, olds: dict, news: dict) -> UpdateResult:
|
|
116
|
+
client = _s3_client(news["connection"])
|
|
117
|
+
old_files = olds.get("files") or {}
|
|
118
|
+
_upload(client, news, _changed_keys(old_files, news["files"]))
|
|
119
|
+
# Uploads first, so a new page never points at an asset that isn't
|
|
120
|
+
# there yet; then drop what the new build no longer has.
|
|
121
|
+
_delete(client, news["bucket"], sorted(set(old_files) - set(news["files"])))
|
|
122
|
+
return UpdateResult(outs=_outputs(news))
|
|
123
|
+
|
|
124
|
+
def delete(self, _id: str, props: dict) -> None:
|
|
125
|
+
# Leaves the bucket empty of our objects, so a bucket without
|
|
126
|
+
# force_destroy (production) can still be deleted after this.
|
|
127
|
+
client = _s3_client(props["connection"])
|
|
128
|
+
try:
|
|
129
|
+
_delete(client, props["bucket"], sorted(props.get("files") or {}))
|
|
130
|
+
except client.exceptions.NoSuchBucket:
|
|
131
|
+
pass
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class SiteFiles(Resource):
|
|
135
|
+
object_count: pulumi.Output[int]
|
|
136
|
+
|
|
137
|
+
def __init__(self, name: str, props: dict, opts: pulumi.ResourceOptions = None):
|
|
138
|
+
super().__init__(
|
|
139
|
+
SiteFilesProvider(),
|
|
140
|
+
name,
|
|
141
|
+
{**props, "object_count": None},
|
|
142
|
+
opts,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _connection() -> dict:
|
|
147
|
+
"""How the sync reaches S3. A dynamic provider talks to AWS through boto3
|
|
148
|
+
rather than the pulumi-aws provider, so the stack's aws: config (region,
|
|
149
|
+
profile, static keys, LocalStack endpoints) is handed over explicitly."""
|
|
150
|
+
config = pulumi.Config("aws")
|
|
151
|
+
endpoints = config.get_object("endpoints") or []
|
|
152
|
+
s3_endpoint = next(
|
|
153
|
+
(entry["s3"] for entry in endpoints if isinstance(entry, dict) and entry.get("s3")),
|
|
154
|
+
None,
|
|
155
|
+
)
|
|
156
|
+
connection = {
|
|
157
|
+
# None falls back to AWS_REGION / the profile, as the aws provider does.
|
|
158
|
+
"region": config.get("region"),
|
|
159
|
+
"profile": config.get("profile"),
|
|
160
|
+
"endpoint_url": s3_endpoint,
|
|
161
|
+
"path_style": (config.get("s3UsePathStyle") or "").lower() == "true",
|
|
162
|
+
}
|
|
163
|
+
access_key, secret_key = config.get("accessKey"), config.get_secret("secretKey")
|
|
164
|
+
if access_key and secret_key is not None:
|
|
165
|
+
connection["access_key"] = access_key
|
|
166
|
+
connection["secret_key"] = secret_key
|
|
167
|
+
token = config.get_secret("token")
|
|
168
|
+
if token is not None:
|
|
169
|
+
connection["token"] = token
|
|
170
|
+
return connection
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class UIPublisher(pulumi.ComponentResource):
|
|
174
|
+
"""
|
|
175
|
+
A Pulumi component to handle the publishing of UI assets to an S3 bucket.
|
|
176
|
+
|
|
177
|
+
All files under the build directory are published by a single resource
|
|
178
|
+
that uploads only new or changed files (by content hash, content type and
|
|
179
|
+
cache policy) and deletes files the build no longer contains.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
def __init__(
|
|
183
|
+
self,
|
|
184
|
+
bucket,
|
|
185
|
+
args: UIPublisherArgs,
|
|
186
|
+
opts: pulumi.ResourceOptions = None,
|
|
187
|
+
):
|
|
188
|
+
"""
|
|
189
|
+
Initialize the UIPublisher component.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
bucket: The S3 bucket resource to upload files to.
|
|
193
|
+
args (UIPublisherArgs): The arguments for the UIPublisher component.
|
|
194
|
+
opts (ResourceOptions): Optional resource options.
|
|
195
|
+
"""
|
|
196
|
+
super().__init__("cloud_foundry:pulumi:UIPublisher", args.name, {}, opts)
|
|
197
|
+
|
|
198
|
+
log.info(f"args: {args.__dict__}")
|
|
199
|
+
self.bucket = bucket
|
|
200
|
+
self.dist_dir = args.dist_dir or os.path.join(args.project_dir, "dist")
|
|
201
|
+
|
|
202
|
+
self.files = self.upload_files(self.dist_dir, bucket, args.prefix)
|
|
203
|
+
|
|
204
|
+
self.register_outputs({})
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def cache_control_for_key(key: str) -> str:
|
|
208
|
+
"""Return a conservative cache policy for a static site object."""
|
|
209
|
+
normalized = key.lower().lstrip("/")
|
|
210
|
+
|
|
211
|
+
if normalized == "_payload.json" or normalized.endswith("/_payload.json"):
|
|
212
|
+
return "public, max-age=31536000, immutable"
|
|
213
|
+
|
|
214
|
+
if normalized.startswith("_nuxt/"):
|
|
215
|
+
return "public, max-age=31536000, immutable"
|
|
216
|
+
|
|
217
|
+
if normalized.endswith(".html"):
|
|
218
|
+
return "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600"
|
|
219
|
+
|
|
220
|
+
if normalized.endswith(
|
|
221
|
+
(
|
|
222
|
+
".js",
|
|
223
|
+
".mjs",
|
|
224
|
+
".css",
|
|
225
|
+
".svg",
|
|
226
|
+
".png",
|
|
227
|
+
".jpg",
|
|
228
|
+
".jpeg",
|
|
229
|
+
".gif",
|
|
230
|
+
".webp",
|
|
231
|
+
".ico",
|
|
232
|
+
".woff",
|
|
233
|
+
".woff2",
|
|
234
|
+
".ttf",
|
|
235
|
+
".otf",
|
|
236
|
+
".eot",
|
|
237
|
+
)
|
|
238
|
+
):
|
|
239
|
+
return "public, max-age=31536000, immutable"
|
|
240
|
+
|
|
241
|
+
if normalized.endswith((".json", ".xml", ".txt", ".webmanifest")):
|
|
242
|
+
return "public, max-age=3600, stale-while-revalidate=300"
|
|
243
|
+
|
|
244
|
+
return "public, max-age=3600"
|
|
245
|
+
|
|
246
|
+
def remap_path_to_s3(self, dir_base: str, key_base: str):
|
|
247
|
+
"""
|
|
248
|
+
Remap local file paths to S3 keys.
|
|
249
|
+
|
|
250
|
+
Args:
|
|
251
|
+
dir_base (str): The base directory containing the files.
|
|
252
|
+
key_base (str): The base key to prepend to the S3 keys.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
list[dict]: A list of dictionaries containing the local file paths and corresponding S3 keys.
|
|
256
|
+
"""
|
|
257
|
+
log.info(f"remap: dir_base: {dir_base}")
|
|
258
|
+
dir_base = os.path.abspath(dir_base)
|
|
259
|
+
return [
|
|
260
|
+
{
|
|
261
|
+
"path": os.path.join(root, file),
|
|
262
|
+
"key": os.path.join(
|
|
263
|
+
key_base, os.path.relpath(os.path.join(root, file), dir_base)
|
|
264
|
+
).replace("\\", "/"),
|
|
265
|
+
}
|
|
266
|
+
for root, _, files in os.walk(dir_base)
|
|
267
|
+
for file in files
|
|
268
|
+
]
|
|
269
|
+
|
|
270
|
+
def build_manifest(self, dir: str, key: str = "") -> dict:
|
|
271
|
+
"""Map each S3 key to its file (relative to ``dir``), content hash,
|
|
272
|
+
content type and cache policy."""
|
|
273
|
+
dir_base = os.path.abspath(dir)
|
|
274
|
+
manifest = {}
|
|
275
|
+
for item in self.remap_path_to_s3(dir_base, key):
|
|
276
|
+
with open(item["path"], "rb") as f:
|
|
277
|
+
sha256 = hashlib.sha256(f.read()).hexdigest()
|
|
278
|
+
content_type, _ = guess_type(item["path"])
|
|
279
|
+
manifest[item["key"]] = {
|
|
280
|
+
"path": os.path.relpath(item["path"], dir_base).replace("\\", "/"),
|
|
281
|
+
"sha256": sha256,
|
|
282
|
+
"content_type": content_type or "",
|
|
283
|
+
"cache_control": self.cache_control_for_key(item["key"]),
|
|
284
|
+
}
|
|
285
|
+
return manifest
|
|
286
|
+
|
|
287
|
+
def upload_files(self, dir: str, bucket, key: str = "") -> SiteFiles:
|
|
288
|
+
"""
|
|
289
|
+
Publish the files in a directory to an S3 bucket as one resource.
|
|
290
|
+
|
|
291
|
+
Args:
|
|
292
|
+
dir (str): The directory containing the files to upload.
|
|
293
|
+
bucket: The S3 bucket resource to upload files to.
|
|
294
|
+
key (str): The prefix to add to the S3 keys (optional).
|
|
295
|
+
"""
|
|
296
|
+
return SiteFiles(
|
|
297
|
+
f"{self._name}-files",
|
|
298
|
+
{
|
|
299
|
+
"bucket": bucket.id,
|
|
300
|
+
"prefix": key,
|
|
301
|
+
"source_dir": os.path.abspath(dir),
|
|
302
|
+
"files": self.build_manifest(dir, key),
|
|
303
|
+
"connection": _connection(),
|
|
304
|
+
},
|
|
305
|
+
opts=pulumi.ResourceOptions(parent=self, depends_on=[bucket]),
|
|
306
|
+
)
|
|
@@ -15,6 +15,18 @@ from urllib.parse import urlparse
|
|
|
15
15
|
|
|
16
16
|
log = logger(__name__)
|
|
17
17
|
|
|
18
|
+
# Bytecode is left out of both the cache hash and the archive. pip compiles
|
|
19
|
+
# with the *build* interpreter, so the .pyc files are tagged for that version
|
|
20
|
+
# (useless to a Lambda runtime on another minor version) and carry the
|
|
21
|
+
# install time in their header -- which changed the hash and the zip bytes on
|
|
22
|
+
# every build and made every deploy re-upload every function.
|
|
23
|
+
_EXCLUDED_DIRS = frozenset({"__pycache__"})
|
|
24
|
+
_EXCLUDED_SUFFIXES = (".pyc", ".pyo")
|
|
25
|
+
|
|
26
|
+
# Fixed zip entry timestamp (the earliest the zip format can store), so an
|
|
27
|
+
# unchanged tree always produces byte-identical archive bytes.
|
|
28
|
+
_ZIP_EPOCH = (1980, 1, 1, 0, 0, 0)
|
|
29
|
+
|
|
18
30
|
|
|
19
31
|
class PythonArchiveBuilder(ArchiveBuilder):
|
|
20
32
|
"""
|
|
@@ -87,13 +99,14 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
87
99
|
hash_comparator.write(self._hash, self._base_dir)
|
|
88
100
|
|
|
89
101
|
def _build_cache_hash(self, hash_comparator: HashComparator) -> str:
|
|
90
|
-
staging_hash = hash_comparator.hash_folder(self._staging)
|
|
91
|
-
libs_hash = hash_comparator.hash_folder(self._libs)
|
|
92
102
|
digest = hashlib.sha256()
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
103
|
+
for folder in (self._staging, self._libs):
|
|
104
|
+
for relative_path, full_path in self._archive_files(folder):
|
|
105
|
+
digest.update(relative_path.encode("utf-8"))
|
|
106
|
+
digest.update(b"\0")
|
|
107
|
+
digest.update(hash_comparator.hash_file(full_path).encode("utf-8"))
|
|
108
|
+
digest.update(b"\0")
|
|
109
|
+
digest.update(b"\0")
|
|
97
110
|
digest.update(self._target_architecture.encode("utf-8"))
|
|
98
111
|
digest.update(b"\0")
|
|
99
112
|
for requirement in self._requirements:
|
|
@@ -119,6 +132,22 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
119
132
|
return ["manylinux2014_aarch64", "manylinux_2_17_aarch64"]
|
|
120
133
|
return ["manylinux2014_x86_64", "manylinux_2_17_x86_64"]
|
|
121
134
|
|
|
135
|
+
@staticmethod
|
|
136
|
+
def _archive_files(folder: str) -> list[tuple[str, str]]:
|
|
137
|
+
"""(relative path, full path) of every file that goes into the
|
|
138
|
+
archive from ``folder``, in a stable order. The cache hash and the
|
|
139
|
+
archive both use this list, so they always agree on the contents."""
|
|
140
|
+
files = []
|
|
141
|
+
for root, dirs, names in os.walk(folder):
|
|
142
|
+
dirs[:] = sorted(d for d in dirs if d not in _EXCLUDED_DIRS)
|
|
143
|
+
for name in sorted(names):
|
|
144
|
+
if name.endswith(_EXCLUDED_SUFFIXES):
|
|
145
|
+
continue
|
|
146
|
+
full_path = os.path.join(root, name)
|
|
147
|
+
relative_path = os.path.relpath(full_path, folder)
|
|
148
|
+
files.append((relative_path.replace(os.sep, "/"), full_path))
|
|
149
|
+
return files
|
|
150
|
+
|
|
122
151
|
def hash(self) -> str:
|
|
123
152
|
"""Return the hash of the current archive."""
|
|
124
153
|
return self._hash
|
|
@@ -148,20 +177,19 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
148
177
|
"""
|
|
149
178
|
log.info(f"building archive: {self.name}")
|
|
150
179
|
try:
|
|
151
|
-
#
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
) as archive:
|
|
180
|
+
# Entries are written in a fixed order with a fixed timestamp and
|
|
181
|
+
# normalized permissions, so the same tree always gives the same
|
|
182
|
+
# bytes -- Pulumi diffs the archive by content.
|
|
183
|
+
with zipfile.ZipFile(self._location, "w", zipfile.ZIP_DEFLATED) as archive:
|
|
156
184
|
# Include both 'staging' and 'libs' folders in the archive
|
|
157
|
-
for folder in
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
185
|
+
for folder in (self._staging, self._libs):
|
|
186
|
+
for relative_path, full_path in self._archive_files(folder):
|
|
187
|
+
info = zipfile.ZipInfo(relative_path, date_time=_ZIP_EPOCH)
|
|
188
|
+
info.compress_type = zipfile.ZIP_DEFLATED
|
|
189
|
+
executable = os.stat(full_path).st_mode & 0o111
|
|
190
|
+
info.external_attr = (0o100755 if executable else 0o100644) << 16
|
|
191
|
+
with open(full_path, "rb") as source:
|
|
192
|
+
archive.writestr(info, source.read())
|
|
165
193
|
|
|
166
194
|
log.info("Archive built successfully")
|
|
167
195
|
except Exception as e:
|
|
@@ -245,6 +273,7 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
245
273
|
"--python-version",
|
|
246
274
|
"3.12",
|
|
247
275
|
"--upgrade",
|
|
276
|
+
"--no-compile",
|
|
248
277
|
"-r",
|
|
249
278
|
requirements_file,
|
|
250
279
|
],
|
|
@@ -283,6 +312,7 @@ class PythonArchiveBuilder(ArchiveBuilder):
|
|
|
283
312
|
"--target",
|
|
284
313
|
self._libs,
|
|
285
314
|
"--upgrade",
|
|
315
|
+
"--no-compile",
|
|
286
316
|
"-r",
|
|
287
317
|
requirements_file,
|
|
288
318
|
]
|
|
@@ -3,13 +3,22 @@ import os
|
|
|
3
3
|
|
|
4
4
|
# Configuring the logging module with basic settings, including format and log level,
|
|
5
5
|
# where the log level is obtained from the environment variable LOGGING_LEVEL
|
|
6
|
-
# with a default of
|
|
6
|
+
# with a default of INFO, and force=True to ensure the configuration is applied immediately.
|
|
7
7
|
logging.basicConfig(
|
|
8
8
|
format="%(name)s:%(lineno)s - %(levelname)s - %(message)s",
|
|
9
|
-
level=os.getenv("LOGGING_LEVEL", "
|
|
9
|
+
level=os.getenv("LOGGING_LEVEL", "INFO").upper(),
|
|
10
10
|
force=True,
|
|
11
11
|
)
|
|
12
12
|
|
|
13
|
+
# At DEBUG, botocore logs signed request headers (including
|
|
14
|
+
# x-amz-security-token) and request bodies. Keep the AWS SDK quiet whatever
|
|
15
|
+
# LOGGING_LEVEL says; AWS_SDK_LOGGING_LEVEL turns it back on deliberately.
|
|
16
|
+
AWS_SDK_LOGGERS = ("boto3", "botocore", "s3transfer", "urllib3")
|
|
17
|
+
for _sdk_logger in AWS_SDK_LOGGERS:
|
|
18
|
+
logging.getLogger(_sdk_logger).setLevel(
|
|
19
|
+
os.getenv("AWS_SDK_LOGGING_LEVEL", "WARNING").upper()
|
|
20
|
+
)
|
|
21
|
+
|
|
13
22
|
WARN = logging.WARN
|
|
14
23
|
INFO = logging.INFO
|
|
15
24
|
DEBUG = logging.DEBUG
|
|
@@ -27,8 +36,8 @@ def logger(name=None):
|
|
|
27
36
|
|
|
28
37
|
"""
|
|
29
38
|
# Retrieving the logging level from the environment variable LOGGING_LEVEL
|
|
30
|
-
# with a default of
|
|
31
|
-
loggingLevel = os.getenv("LOGGING_LEVEL", "
|
|
39
|
+
# with a default of INFO, and converting it to uppercase
|
|
40
|
+
loggingLevel = os.getenv("LOGGING_LEVEL", "INFO").upper()
|
|
32
41
|
|
|
33
42
|
# Setting the logging level for the root logger to the obtained logging level
|
|
34
43
|
logging.getLogger().setLevel(loggingLevel)
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
import pulumi
|
|
2
|
-
import pulumi_aws as aws
|
|
3
|
-
import os
|
|
4
|
-
from mimetypes import guess_type
|
|
5
|
-
|
|
6
|
-
from cloud_foundry.utils.logger import logger
|
|
7
|
-
|
|
8
|
-
log = logger(__name__)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class UIPublisherArgs:
|
|
12
|
-
def __init__(
|
|
13
|
-
self, name: str, dist_dir: str, project_dir: str = ".", prefix: str = ""
|
|
14
|
-
):
|
|
15
|
-
self.name = name
|
|
16
|
-
self.dist_dir = dist_dir
|
|
17
|
-
self.project_dir = project_dir
|
|
18
|
-
self.prefix = prefix
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class UIPublisher(pulumi.ComponentResource):
|
|
22
|
-
"""
|
|
23
|
-
A Pulumi component to handle the publishing of UI assets to an S3 bucket.
|
|
24
|
-
|
|
25
|
-
This component takes care of uploading files from a specified directory to an S3 bucket,
|
|
26
|
-
setting the appropriate content type for each file based on its extension.
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
def __init__(
|
|
30
|
-
self,
|
|
31
|
-
bucket,
|
|
32
|
-
args: UIPublisherArgs,
|
|
33
|
-
opts: pulumi.ResourceOptions = None,
|
|
34
|
-
):
|
|
35
|
-
"""
|
|
36
|
-
Initialize the UIPublisher component.
|
|
37
|
-
|
|
38
|
-
Args:
|
|
39
|
-
name (str): The name of the component.
|
|
40
|
-
bucket: The S3 bucket resource to upload files to.
|
|
41
|
-
args (UIPublisherArgs): The arguments for the UIPublisher component.
|
|
42
|
-
opts (ResourceOptions): Optional resource options.
|
|
43
|
-
"""
|
|
44
|
-
super().__init__("cloud_foundry:pulumi:UIPublisher", args.name, {}, opts)
|
|
45
|
-
|
|
46
|
-
log.info(f"args: {args.__dict__}")
|
|
47
|
-
self.bucket = bucket
|
|
48
|
-
self.dist_dir = args.dist_dir or os.path.join(args.project_dir, "dist")
|
|
49
|
-
|
|
50
|
-
self.upload_files(self.dist_dir, bucket, args.prefix)
|
|
51
|
-
|
|
52
|
-
self.register_outputs({})
|
|
53
|
-
|
|
54
|
-
@staticmethod
|
|
55
|
-
def cache_control_for_key(key: str) -> str:
|
|
56
|
-
"""Return a conservative cache policy for a static site object."""
|
|
57
|
-
normalized = key.lower().lstrip("/")
|
|
58
|
-
|
|
59
|
-
if normalized == "_payload.json" or normalized.endswith("/_payload.json"):
|
|
60
|
-
return "public, max-age=31536000, immutable"
|
|
61
|
-
|
|
62
|
-
if normalized.startswith("_nuxt/"):
|
|
63
|
-
return "public, max-age=31536000, immutable"
|
|
64
|
-
|
|
65
|
-
if normalized.endswith(".html"):
|
|
66
|
-
return "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600"
|
|
67
|
-
|
|
68
|
-
if normalized.endswith(
|
|
69
|
-
(
|
|
70
|
-
".js",
|
|
71
|
-
".mjs",
|
|
72
|
-
".css",
|
|
73
|
-
".svg",
|
|
74
|
-
".png",
|
|
75
|
-
".jpg",
|
|
76
|
-
".jpeg",
|
|
77
|
-
".gif",
|
|
78
|
-
".webp",
|
|
79
|
-
".ico",
|
|
80
|
-
".woff",
|
|
81
|
-
".woff2",
|
|
82
|
-
".ttf",
|
|
83
|
-
".otf",
|
|
84
|
-
".eot",
|
|
85
|
-
)
|
|
86
|
-
):
|
|
87
|
-
return "public, max-age=31536000, immutable"
|
|
88
|
-
|
|
89
|
-
if normalized.endswith((".json", ".xml", ".txt", ".webmanifest")):
|
|
90
|
-
return "public, max-age=3600, stale-while-revalidate=300"
|
|
91
|
-
|
|
92
|
-
return "public, max-age=3600"
|
|
93
|
-
|
|
94
|
-
def remap_path_to_s3(self, dir_base: str, key_base: str):
|
|
95
|
-
"""
|
|
96
|
-
Remap local file paths to S3 keys.
|
|
97
|
-
|
|
98
|
-
Args:
|
|
99
|
-
dir_base (str): The base directory containing the files.
|
|
100
|
-
key_base (str): The base key to prepend to the S3 keys.
|
|
101
|
-
|
|
102
|
-
Returns:
|
|
103
|
-
list[dict]: A list of dictionaries containing the local file paths and corresponding S3 keys.
|
|
104
|
-
"""
|
|
105
|
-
log.info(f"remap: dir_base: {dir_base}")
|
|
106
|
-
dir_base = os.path.abspath(dir_base)
|
|
107
|
-
return [
|
|
108
|
-
{
|
|
109
|
-
"path": os.path.join(root, file),
|
|
110
|
-
"key": os.path.join(
|
|
111
|
-
key_base, os.path.relpath(os.path.join(root, file), dir_base)
|
|
112
|
-
).replace("\\", "/"),
|
|
113
|
-
}
|
|
114
|
-
for root, _, files in os.walk(dir_base)
|
|
115
|
-
for file in files
|
|
116
|
-
]
|
|
117
|
-
|
|
118
|
-
def upload_files(self, dir: str, bucket, key: str = ""):
|
|
119
|
-
"""
|
|
120
|
-
Upload files from a directory to an S3 bucket.
|
|
121
|
-
|
|
122
|
-
Args:
|
|
123
|
-
dir (str): The directory containing the files to upload.
|
|
124
|
-
bucket: The S3 bucket resource to upload files to.
|
|
125
|
-
key (str): The prefix to add to the S3 keys (optional).
|
|
126
|
-
"""
|
|
127
|
-
for item in self.remap_path_to_s3(dir, key):
|
|
128
|
-
content_type, _ = guess_type(item["path"])
|
|
129
|
-
aws.s3.BucketObjectv2(
|
|
130
|
-
item["key"],
|
|
131
|
-
bucket=bucket.id,
|
|
132
|
-
key=item["key"],
|
|
133
|
-
source=pulumi.FileAsset(item["path"]),
|
|
134
|
-
content_type=content_type,
|
|
135
|
-
cache_control=self.cache_control_for_key(item["key"]),
|
|
136
|
-
opts=pulumi.ResourceOptions(parent=self, depends_on=[bucket]),
|
|
137
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|