cloud-foundry 0.3.1__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/PKG-INFO +2 -1
  2. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/__init__.py +1 -1
  3. cloud_foundry-0.4.2/cloud_foundry/migrations/forget_site_objects.py +141 -0
  4. cloud_foundry-0.4.2/cloud_foundry/pulumi/__init__.py +0 -0
  5. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/cdn_site_origin.py +5 -3
  6. cloud_foundry-0.4.2/cloud_foundry/pulumi/ui_publisher.py +306 -0
  7. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/python_archive_builder.py +49 -19
  8. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/logger.py +13 -4
  9. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/pyproject.toml +2 -0
  10. cloud_foundry-0.3.1/cloud_foundry/pulumi/ui_publisher.py +0 -137
  11. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/.gitignore +0 -0
  12. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/LICENSE +0 -0
  13. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/README.md +0 -0
  14. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/archive_builder.py +0 -0
  15. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/mail_publisher.py +0 -0
  16. {cloud_foundry-0.3.1/cloud_foundry/pulumi → cloud_foundry-0.4.2/cloud_foundry/migrations}/__init__.py +0 -0
  17. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/api_waf.py +0 -0
  18. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/cdn.py +0 -0
  19. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/cdn_api_origin.py +0 -0
  20. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/connection_store.py +0 -0
  21. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/custom_domain.py +0 -0
  22. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/document_repository.py +0 -0
  23. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/domain.py +0 -0
  24. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/function.py +0 -0
  25. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/idle_reaper.py +0 -0
  26. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/publisher.py +0 -0
  27. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/python_function.py +0 -0
  28. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/queue.py +0 -0
  29. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/rest_api.py +0 -0
  30. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/site_bucket.py +0 -0
  31. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/topic.py +0 -0
  32. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/websocket_api.py +0 -0
  33. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/pulumi/websocket_function.py +0 -0
  34. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/services/authorization_api/security_lambda.py +0 -0
  35. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/services/mail_publisher.py +0 -0
  36. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/aws_openapi_editor.py +0 -0
  37. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/hash_comparator.py +0 -0
  38. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/idle_reaper_client.py +0 -0
  39. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/names.py +0 -0
  40. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/openapi_editor.py +0 -0
  41. {cloud_foundry-0.3.1 → cloud_foundry-0.4.2}/cloud_foundry/utils/websocket_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cloud-foundry
3
- Version: 0.3.1
3
+ Version: 0.4.2
4
4
  Summary: cloud-foundry is a curated set of cloud components that can be used to assemble cloud-based applications.
5
5
  Project-URL: Documentation, https://github.com/DanRepik/cloud-foundry
6
6
  Project-URL: Source, https://github.com/DanRepik/cloud-foundry
@@ -20,6 +20,7 @@ Requires-Dist: docker; extra == 'dev'
20
20
  Requires-Dist: flake8; extra == 'dev'
21
21
  Requires-Dist: flit>=3.9; extra == 'dev'
22
22
  Requires-Dist: isort>=5.12.0; extra == 'dev'
23
+ Requires-Dist: moto[s3]>=5.0; extra == 'dev'
23
24
  Requires-Dist: mypy>=1.0.0; extra == 'dev'
24
25
  Requires-Dist: pre-commit; extra == 'dev'
25
26
  Requires-Dist: psycopg2-binary; extra == 'dev'
@@ -27,4 +27,4 @@ from cloud_foundry.utils.idle_reaper_client import request_mapping_enable # noq
27
27
  from cloud_foundry.utils.names import resource_id # noqa: F401
28
28
  from cloud_foundry.utils.names import project_slug # noqa: F401
29
29
 
30
- version = "0.3.1"
30
+ version = "0.4.2"
@@ -0,0 +1,141 @@
1
+ """One-time state migration for the single-resource UIPublisher.
2
+
3
+ UIPublisher used to declare one aws.s3.BucketObjectv2 per site file. It now
4
+ publishes the whole build through one resource. Deploying the new version
5
+ straight over old state would be destructive: Pulumi runs the new resource's
6
+ upload first and deletes the no-longer-declared per-file resources last, and
7
+ deleting those removes the very S3 objects that were just uploaded (same
8
+ keys), leaving the site empty.
9
+
10
+ Run this once per stack, right before the first `pulumi up` with the new
11
+ cloud_foundry. It removes the per-file resources from the Pulumi state only;
12
+ the S3 objects stay where they are, and the next deploy takes them over.
13
+
14
+ python -m cloud_foundry.migrations.forget_site_objects --stack dev [--cwd <project dir>] [--dry-run]
15
+
16
+ --cwd is the Pulumi project directory (the one holding Pulumi.yaml),
17
+ relative to where you run the command; it defaults to the current directory.
18
+
19
+ The exported state is saved next to the working directory first, so
20
+ `pulumi stack import --file <backup>` puts it back if needed.
21
+ """
22
+
23
+ import argparse
24
+ import json
25
+ import os
26
+ import shutil
27
+ import subprocess
28
+ import sys
29
+ import tempfile
30
+ import time
31
+
32
+ PUBLISHER_TYPE = "cloud_foundry:pulumi:UIPublisher"
33
+ OBJECT_TYPE = "aws:s3/bucketObjectv2:BucketObjectv2"
34
+
35
+
36
+ def _urn_type(urn: str) -> str:
37
+ # urn:pulumi:<stack>::<project>::<parent types$...$type>::<name>
38
+ parts = urn.split("::", 3)
39
+ return parts[2].split("$")[-1] if len(parts) == 4 else ""
40
+
41
+
42
+ def forget_site_objects(deployment: dict) -> tuple[dict, list[str]]:
43
+ """Return ``deployment`` without the per-file objects that UIPublisher
44
+ components used to own, and the URNs that were removed."""
45
+ resources = deployment.get("resources") or []
46
+ removed = {
47
+ resource["urn"]
48
+ for resource in resources
49
+ if resource.get("type") == OBJECT_TYPE
50
+ and _urn_type(resource.get("parent", "")) == PUBLISHER_TYPE
51
+ }
52
+
53
+ kept = []
54
+ for resource in resources:
55
+ if resource["urn"] in removed:
56
+ continue
57
+ resource = dict(resource)
58
+ if resource.get("dependencies"):
59
+ resource["dependencies"] = [u for u in resource["dependencies"] if u not in removed]
60
+ if resource.get("propertyDependencies"):
61
+ resource["propertyDependencies"] = {
62
+ name: [u for u in urns if u not in removed]
63
+ for name, urns in resource["propertyDependencies"].items()
64
+ }
65
+ if resource.get("deletedWith") in removed:
66
+ resource.pop("deletedWith")
67
+ kept.append(resource)
68
+
69
+ return {**deployment, "resources": kept}, sorted(removed)
70
+
71
+
72
+ def _pulumi(args: list[str], cwd: str) -> None:
73
+ subprocess.run(["pulumi", *args], cwd=cwd, check=True)
74
+
75
+
76
+ def _project_problem(cwd: str) -> str | None:
77
+ """Why ``cwd`` can't be used as the Pulumi project directory, if it can't."""
78
+ if not os.path.isdir(cwd):
79
+ return f"{cwd} does not exist"
80
+ if not any(os.path.isfile(os.path.join(cwd, name)) for name in ("Pulumi.yaml", "Pulumi.yml")):
81
+ return f"{cwd} has no Pulumi.yaml"
82
+ return None
83
+
84
+
85
+ def main(argv: list[str] | None = None) -> int:
86
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
87
+ parser.add_argument("--stack", required=True)
88
+ parser.add_argument(
89
+ "--cwd",
90
+ default=".",
91
+ help="Pulumi project directory, relative to where you run this (default: current directory)",
92
+ )
93
+ parser.add_argument("--dry-run", action="store_true", help="report only; change nothing")
94
+ args = parser.parse_args(argv)
95
+
96
+ cwd = os.path.abspath(args.cwd)
97
+ problem = _project_problem(cwd)
98
+ if problem:
99
+ print(
100
+ f"{problem}. --cwd must be the Pulumi project directory, relative to "
101
+ f"where you run this command (it defaults to the current directory). "
102
+ f"Nothing was changed.",
103
+ file=sys.stderr,
104
+ )
105
+ return 2
106
+ if shutil.which("pulumi") is None:
107
+ print("The pulumi CLI is not on PATH. Nothing was changed.", file=sys.stderr)
108
+ return 2
109
+ stamp = time.strftime("%Y%m%dT%H%M%S")
110
+ backup = os.path.join(cwd, f"{args.stack}-state-backup-{stamp}.json")
111
+ _pulumi(["stack", "export", "--stack", args.stack, "--file", backup], cwd)
112
+
113
+ with open(backup) as f:
114
+ state = json.load(f)
115
+ deployment = state.get("deployment") or {}
116
+ if deployment.get("pending_operations"):
117
+ print(
118
+ "State has pending operations from an interrupted update; resolve those "
119
+ "(pulumi refresh / pulumi cancel) before migrating.",
120
+ file=sys.stderr,
121
+ )
122
+ return 1
123
+
124
+ migrated, removed = forget_site_objects(deployment)
125
+ print(f"{len(removed)} per-file site object(s) to drop from the {args.stack} state (S3 is not touched).")
126
+ if not removed or args.dry_run:
127
+ print(f"State export kept at {backup}")
128
+ return 0
129
+
130
+ with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as out:
131
+ json.dump({**state, "deployment": migrated}, out)
132
+ try:
133
+ _pulumi(["stack", "import", "--stack", args.stack, "--file", out.name], cwd)
134
+ finally:
135
+ os.unlink(out.name)
136
+ print(f"Done. Previous state saved to {backup}; restore it with: pulumi stack import --stack {args.stack} --file {backup}")
137
+ return 0
138
+
139
+
140
+ if __name__ == "__main__":
141
+ sys.exit(main())
File without changes
@@ -119,9 +119,11 @@ class SiteOrigin(pulumi.ComponentResource):
119
119
  origin_id=self.origin_id,
120
120
  origin_access_control_id=origin_access_control.id,
121
121
  origin_path=origin_path,
122
- s3_origin_config=aws.cloudfront.DistributionOriginS3OriginConfigArgs(
123
- origin_access_identity=""
124
- ),
122
+ # No s3_origin_config: access goes through the origin access
123
+ # control above, and an S3 origin without one gets CloudFront's
124
+ # empty-identity config anyway. Declaring the empty block made
125
+ # every preview show an origins update, because the provider
126
+ # reads that block back as absent.
125
127
  )
126
128
 
127
129
  if origin_shield_region:
@@ -0,0 +1,306 @@
1
+ import concurrent.futures
2
+ import hashlib
3
+ import os
4
+ from mimetypes import guess_type
5
+
6
+ import pulumi
7
+ from pulumi.dynamic import CreateResult, DiffResult, Resource, ResourceProvider, UpdateResult
8
+
9
+ from cloud_foundry.utils.logger import logger
10
+
11
+ log = logger(__name__)
12
+
13
+ # Fields of a file's manifest entry that decide whether it has to be
14
+ # (re)uploaded. The local path is left out on purpose: the same build checked
15
+ # out somewhere else must not look like a change.
16
+ _OBJECT_FIELDS = ("sha256", "content_type", "cache_control")
17
+
18
+ _UPLOAD_WORKERS = 16
19
+ _DELETE_BATCH = 1000 # DeleteObjects limit
20
+
21
+
22
+ class UIPublisherArgs:
23
+ def __init__(
24
+ self, name: str, dist_dir: str, project_dir: str = ".", prefix: str = ""
25
+ ):
26
+ self.name = name
27
+ self.dist_dir = dist_dir
28
+ self.project_dir = project_dir
29
+ self.prefix = prefix
30
+
31
+
32
+ def _s3_client(connection: dict):
33
+ import boto3
34
+ from botocore.config import Config
35
+
36
+ session = boto3.session.Session(
37
+ profile_name=connection.get("profile") or None,
38
+ aws_access_key_id=connection.get("access_key") or None,
39
+ aws_secret_access_key=connection.get("secret_key") or None,
40
+ aws_session_token=connection.get("token") or None,
41
+ region_name=connection.get("region") or None,
42
+ )
43
+ addressing = "path" if connection.get("path_style") else "auto"
44
+ return session.client(
45
+ "s3",
46
+ endpoint_url=connection.get("endpoint_url") or None,
47
+ config=Config(s3={"addressing_style": addressing}),
48
+ )
49
+
50
+
51
+ def _changed_keys(old_files: dict, new_files: dict) -> list[str]:
52
+ """Keys that are new or whose content or headers changed."""
53
+ return sorted(
54
+ key
55
+ for key, entry in new_files.items()
56
+ if any(
57
+ (old_files.get(key) or {}).get(field) != entry.get(field)
58
+ for field in _OBJECT_FIELDS
59
+ )
60
+ )
61
+
62
+
63
+ def _upload(client, props: dict, keys: list[str]) -> None:
64
+ def put(key: str) -> None:
65
+ entry = props["files"][key]
66
+ extra = {"CacheControl": entry["cache_control"]}
67
+ if entry.get("content_type"):
68
+ extra["ContentType"] = entry["content_type"]
69
+ with open(os.path.join(props["source_dir"], entry["path"]), "rb") as body:
70
+ client.put_object(Bucket=props["bucket"], Key=key, Body=body, **extra)
71
+
72
+ with concurrent.futures.ThreadPoolExecutor(_UPLOAD_WORKERS) as pool:
73
+ # list() re-raises the first upload error, failing the operation.
74
+ list(pool.map(put, keys))
75
+
76
+
77
+ def _delete(client, bucket: str, keys: list[str]) -> None:
78
+ for start in range(0, len(keys), _DELETE_BATCH):
79
+ batch = keys[start : start + _DELETE_BATCH]
80
+ response = client.delete_objects(
81
+ Bucket=bucket,
82
+ Delete={"Objects": [{"Key": key} for key in batch], "Quiet": True},
83
+ )
84
+ if response.get("Errors"):
85
+ raise RuntimeError(f"failed to delete from {bucket}: {response['Errors'][:5]}")
86
+
87
+
88
+ def _outputs(props: dict) -> dict:
89
+ return {**props, "object_count": len(props["files"])}
90
+
91
+
92
+ class SiteFilesProvider(ResourceProvider):
93
+ """Keeps a bucket prefix in step with a local build directory as ONE
94
+ Pulumi resource. Each file used to be its own BucketObjectv2, so a site
95
+ rebuild meant hundreds of resource steps -- each followed by a full state
96
+ checkpoint write, which on the DIY S3 backend dominated deploy time."""
97
+
98
+ def diff(self, _id: str, olds: dict, news: dict) -> DiffResult:
99
+ replaces = [name for name in ("bucket", "prefix") if olds.get(name) != news.get(name)]
100
+ old_files, new_files = olds.get("files") or {}, news.get("files") or {}
101
+ changed = bool(_changed_keys(old_files, new_files)) or set(old_files) != set(new_files)
102
+ return DiffResult(
103
+ changes=bool(replaces) or changed,
104
+ replaces=replaces,
105
+ delete_before_replace=False,
106
+ )
107
+
108
+ def create(self, props: dict) -> CreateResult:
109
+ client = _s3_client(props["connection"])
110
+ _upload(client, props, sorted(props["files"]))
111
+ return CreateResult(
112
+ id_=f"{props['bucket']}/{props['prefix']}", outs=_outputs(props)
113
+ )
114
+
115
+ def update(self, _id: str, olds: dict, news: dict) -> UpdateResult:
116
+ client = _s3_client(news["connection"])
117
+ old_files = olds.get("files") or {}
118
+ _upload(client, news, _changed_keys(old_files, news["files"]))
119
+ # Uploads first, so a new page never points at an asset that isn't
120
+ # there yet; then drop what the new build no longer has.
121
+ _delete(client, news["bucket"], sorted(set(old_files) - set(news["files"])))
122
+ return UpdateResult(outs=_outputs(news))
123
+
124
+ def delete(self, _id: str, props: dict) -> None:
125
+ # Leaves the bucket empty of our objects, so a bucket without
126
+ # force_destroy (production) can still be deleted after this.
127
+ client = _s3_client(props["connection"])
128
+ try:
129
+ _delete(client, props["bucket"], sorted(props.get("files") or {}))
130
+ except client.exceptions.NoSuchBucket:
131
+ pass
132
+
133
+
134
+ class SiteFiles(Resource):
135
+ object_count: pulumi.Output[int]
136
+
137
+ def __init__(self, name: str, props: dict, opts: pulumi.ResourceOptions = None):
138
+ super().__init__(
139
+ SiteFilesProvider(),
140
+ name,
141
+ {**props, "object_count": None},
142
+ opts,
143
+ )
144
+
145
+
146
+ def _connection() -> dict:
147
+ """How the sync reaches S3. A dynamic provider talks to AWS through boto3
148
+ rather than the pulumi-aws provider, so the stack's aws: config (region,
149
+ profile, static keys, LocalStack endpoints) is handed over explicitly."""
150
+ config = pulumi.Config("aws")
151
+ endpoints = config.get_object("endpoints") or []
152
+ s3_endpoint = next(
153
+ (entry["s3"] for entry in endpoints if isinstance(entry, dict) and entry.get("s3")),
154
+ None,
155
+ )
156
+ connection = {
157
+ # None falls back to AWS_REGION / the profile, as the aws provider does.
158
+ "region": config.get("region"),
159
+ "profile": config.get("profile"),
160
+ "endpoint_url": s3_endpoint,
161
+ "path_style": (config.get("s3UsePathStyle") or "").lower() == "true",
162
+ }
163
+ access_key, secret_key = config.get("accessKey"), config.get_secret("secretKey")
164
+ if access_key and secret_key is not None:
165
+ connection["access_key"] = access_key
166
+ connection["secret_key"] = secret_key
167
+ token = config.get_secret("token")
168
+ if token is not None:
169
+ connection["token"] = token
170
+ return connection
171
+
172
+
173
+ class UIPublisher(pulumi.ComponentResource):
174
+ """
175
+ A Pulumi component to handle the publishing of UI assets to an S3 bucket.
176
+
177
+ All files under the build directory are published by a single resource
178
+ that uploads only new or changed files (by content hash, content type and
179
+ cache policy) and deletes files the build no longer contains.
180
+ """
181
+
182
+ def __init__(
183
+ self,
184
+ bucket,
185
+ args: UIPublisherArgs,
186
+ opts: pulumi.ResourceOptions = None,
187
+ ):
188
+ """
189
+ Initialize the UIPublisher component.
190
+
191
+ Args:
192
+ bucket: The S3 bucket resource to upload files to.
193
+ args (UIPublisherArgs): The arguments for the UIPublisher component.
194
+ opts (ResourceOptions): Optional resource options.
195
+ """
196
+ super().__init__("cloud_foundry:pulumi:UIPublisher", args.name, {}, opts)
197
+
198
+ log.info(f"args: {args.__dict__}")
199
+ self.bucket = bucket
200
+ self.dist_dir = args.dist_dir or os.path.join(args.project_dir, "dist")
201
+
202
+ self.files = self.upload_files(self.dist_dir, bucket, args.prefix)
203
+
204
+ self.register_outputs({})
205
+
206
+ @staticmethod
207
+ def cache_control_for_key(key: str) -> str:
208
+ """Return a conservative cache policy for a static site object."""
209
+ normalized = key.lower().lstrip("/")
210
+
211
+ if normalized == "_payload.json" or normalized.endswith("/_payload.json"):
212
+ return "public, max-age=31536000, immutable"
213
+
214
+ if normalized.startswith("_nuxt/"):
215
+ return "public, max-age=31536000, immutable"
216
+
217
+ if normalized.endswith(".html"):
218
+ return "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600"
219
+
220
+ if normalized.endswith(
221
+ (
222
+ ".js",
223
+ ".mjs",
224
+ ".css",
225
+ ".svg",
226
+ ".png",
227
+ ".jpg",
228
+ ".jpeg",
229
+ ".gif",
230
+ ".webp",
231
+ ".ico",
232
+ ".woff",
233
+ ".woff2",
234
+ ".ttf",
235
+ ".otf",
236
+ ".eot",
237
+ )
238
+ ):
239
+ return "public, max-age=31536000, immutable"
240
+
241
+ if normalized.endswith((".json", ".xml", ".txt", ".webmanifest")):
242
+ return "public, max-age=3600, stale-while-revalidate=300"
243
+
244
+ return "public, max-age=3600"
245
+
246
+ def remap_path_to_s3(self, dir_base: str, key_base: str):
247
+ """
248
+ Remap local file paths to S3 keys.
249
+
250
+ Args:
251
+ dir_base (str): The base directory containing the files.
252
+ key_base (str): The base key to prepend to the S3 keys.
253
+
254
+ Returns:
255
+ list[dict]: A list of dictionaries containing the local file paths and corresponding S3 keys.
256
+ """
257
+ log.info(f"remap: dir_base: {dir_base}")
258
+ dir_base = os.path.abspath(dir_base)
259
+ return [
260
+ {
261
+ "path": os.path.join(root, file),
262
+ "key": os.path.join(
263
+ key_base, os.path.relpath(os.path.join(root, file), dir_base)
264
+ ).replace("\\", "/"),
265
+ }
266
+ for root, _, files in os.walk(dir_base)
267
+ for file in files
268
+ ]
269
+
270
+ def build_manifest(self, dir: str, key: str = "") -> dict:
271
+ """Map each S3 key to its file (relative to ``dir``), content hash,
272
+ content type and cache policy."""
273
+ dir_base = os.path.abspath(dir)
274
+ manifest = {}
275
+ for item in self.remap_path_to_s3(dir_base, key):
276
+ with open(item["path"], "rb") as f:
277
+ sha256 = hashlib.sha256(f.read()).hexdigest()
278
+ content_type, _ = guess_type(item["path"])
279
+ manifest[item["key"]] = {
280
+ "path": os.path.relpath(item["path"], dir_base).replace("\\", "/"),
281
+ "sha256": sha256,
282
+ "content_type": content_type or "",
283
+ "cache_control": self.cache_control_for_key(item["key"]),
284
+ }
285
+ return manifest
286
+
287
+ def upload_files(self, dir: str, bucket, key: str = "") -> SiteFiles:
288
+ """
289
+ Publish the files in a directory to an S3 bucket as one resource.
290
+
291
+ Args:
292
+ dir (str): The directory containing the files to upload.
293
+ bucket: The S3 bucket resource to upload files to.
294
+ key (str): The prefix to add to the S3 keys (optional).
295
+ """
296
+ return SiteFiles(
297
+ f"{self._name}-files",
298
+ {
299
+ "bucket": bucket.id,
300
+ "prefix": key,
301
+ "source_dir": os.path.abspath(dir),
302
+ "files": self.build_manifest(dir, key),
303
+ "connection": _connection(),
304
+ },
305
+ opts=pulumi.ResourceOptions(parent=self, depends_on=[bucket]),
306
+ )
@@ -15,6 +15,18 @@ from urllib.parse import urlparse
15
15
 
16
16
  log = logger(__name__)
17
17
 
18
+ # Bytecode is left out of both the cache hash and the archive. pip compiles
19
+ # with the *build* interpreter, so the .pyc files are tagged for that version
20
+ # (useless to a Lambda runtime on another minor version) and carry the
21
+ # install time in their header -- which changed the hash and the zip bytes on
22
+ # every build and made every deploy re-upload every function.
23
+ _EXCLUDED_DIRS = frozenset({"__pycache__"})
24
+ _EXCLUDED_SUFFIXES = (".pyc", ".pyo")
25
+
26
+ # Fixed zip entry timestamp (the earliest the zip format can store), so an
27
+ # unchanged tree always produces byte-identical archive bytes.
28
+ _ZIP_EPOCH = (1980, 1, 1, 0, 0, 0)
29
+
18
30
 
19
31
  class PythonArchiveBuilder(ArchiveBuilder):
20
32
  """
@@ -87,13 +99,14 @@ class PythonArchiveBuilder(ArchiveBuilder):
87
99
  hash_comparator.write(self._hash, self._base_dir)
88
100
 
89
101
  def _build_cache_hash(self, hash_comparator: HashComparator) -> str:
90
- staging_hash = hash_comparator.hash_folder(self._staging)
91
- libs_hash = hash_comparator.hash_folder(self._libs)
92
102
  digest = hashlib.sha256()
93
- digest.update(staging_hash.encode("utf-8"))
94
- digest.update(b"\0")
95
- digest.update(libs_hash.encode("utf-8"))
96
- digest.update(b"\0")
103
+ for folder in (self._staging, self._libs):
104
+ for relative_path, full_path in self._archive_files(folder):
105
+ digest.update(relative_path.encode("utf-8"))
106
+ digest.update(b"\0")
107
+ digest.update(hash_comparator.hash_file(full_path).encode("utf-8"))
108
+ digest.update(b"\0")
109
+ digest.update(b"\0")
97
110
  digest.update(self._target_architecture.encode("utf-8"))
98
111
  digest.update(b"\0")
99
112
  for requirement in self._requirements:
@@ -119,6 +132,22 @@ class PythonArchiveBuilder(ArchiveBuilder):
119
132
  return ["manylinux2014_aarch64", "manylinux_2_17_aarch64"]
120
133
  return ["manylinux2014_x86_64", "manylinux_2_17_x86_64"]
121
134
 
135
+ @staticmethod
136
+ def _archive_files(folder: str) -> list[tuple[str, str]]:
137
+ """(relative path, full path) of every file that goes into the
138
+ archive from ``folder``, in a stable order. The cache hash and the
139
+ archive both use this list, so they always agree on the contents."""
140
+ files = []
141
+ for root, dirs, names in os.walk(folder):
142
+ dirs[:] = sorted(d for d in dirs if d not in _EXCLUDED_DIRS)
143
+ for name in sorted(names):
144
+ if name.endswith(_EXCLUDED_SUFFIXES):
145
+ continue
146
+ full_path = os.path.join(root, name)
147
+ relative_path = os.path.relpath(full_path, folder)
148
+ files.append((relative_path.replace(os.sep, "/"), full_path))
149
+ return files
150
+
122
151
  def hash(self) -> str:
123
152
  """Return the hash of the current archive."""
124
153
  return self._hash
@@ -148,20 +177,19 @@ class PythonArchiveBuilder(ArchiveBuilder):
148
177
  """
149
178
  log.info(f"building archive: {self.name}")
150
179
  try:
151
- # Create the archive file
152
- archive_name = self._location.replace(".zip", "")
153
- with zipfile.ZipFile(
154
- f"{archive_name}.zip", "w", zipfile.ZIP_DEFLATED
155
- ) as archive:
180
+ # Entries are written in a fixed order with a fixed timestamp and
181
+ # normalized permissions, so the same tree always gives the same
182
+ # bytes -- Pulumi diffs the archive by content.
183
+ with zipfile.ZipFile(self._location, "w", zipfile.ZIP_DEFLATED) as archive:
156
184
  # Include both 'staging' and 'libs' folders in the archive
157
- for folder in ["staging", "libs"]:
158
- folder_path = os.path.join(self._base_dir, folder)
159
- if os.path.exists(folder_path):
160
- for root, _, files in os.walk(folder_path):
161
- for file in files:
162
- full_path = os.path.join(root, file)
163
- relative_path = os.path.relpath(full_path, folder_path)
164
- archive.write(full_path, relative_path)
185
+ for folder in (self._staging, self._libs):
186
+ for relative_path, full_path in self._archive_files(folder):
187
+ info = zipfile.ZipInfo(relative_path, date_time=_ZIP_EPOCH)
188
+ info.compress_type = zipfile.ZIP_DEFLATED
189
+ executable = os.stat(full_path).st_mode & 0o111
190
+ info.external_attr = (0o100755 if executable else 0o100644) << 16
191
+ with open(full_path, "rb") as source:
192
+ archive.writestr(info, source.read())
165
193
 
166
194
  log.info("Archive built successfully")
167
195
  except Exception as e:
@@ -245,6 +273,7 @@ class PythonArchiveBuilder(ArchiveBuilder):
245
273
  "--python-version",
246
274
  "3.12",
247
275
  "--upgrade",
276
+ "--no-compile",
248
277
  "-r",
249
278
  requirements_file,
250
279
  ],
@@ -283,6 +312,7 @@ class PythonArchiveBuilder(ArchiveBuilder):
283
312
  "--target",
284
313
  self._libs,
285
314
  "--upgrade",
315
+ "--no-compile",
286
316
  "-r",
287
317
  requirements_file,
288
318
  ]
@@ -3,13 +3,22 @@ import os
3
3
 
4
4
  # Configuring the logging module with basic settings, including format and log level,
5
5
  # where the log level is obtained from the environment variable LOGGING_LEVEL
6
- # with a default of DEBUG, and force=True to ensure the configuration is applied immediately.
6
+ # with a default of INFO, and force=True to ensure the configuration is applied immediately.
7
7
  logging.basicConfig(
8
8
  format="%(name)s:%(lineno)s - %(levelname)s - %(message)s",
9
- level=os.getenv("LOGGING_LEVEL", "DEBUG").upper(),
9
+ level=os.getenv("LOGGING_LEVEL", "INFO").upper(),
10
10
  force=True,
11
11
  )
12
12
 
13
+ # At DEBUG, botocore logs signed request headers (including
14
+ # x-amz-security-token) and request bodies. Keep the AWS SDK quiet whatever
15
+ # LOGGING_LEVEL says; AWS_SDK_LOGGING_LEVEL turns it back on deliberately.
16
+ AWS_SDK_LOGGERS = ("boto3", "botocore", "s3transfer", "urllib3")
17
+ for _sdk_logger in AWS_SDK_LOGGERS:
18
+ logging.getLogger(_sdk_logger).setLevel(
19
+ os.getenv("AWS_SDK_LOGGING_LEVEL", "WARNING").upper()
20
+ )
21
+
13
22
  WARN = logging.WARN
14
23
  INFO = logging.INFO
15
24
  DEBUG = logging.DEBUG
@@ -27,8 +36,8 @@ def logger(name=None):
27
36
 
28
37
  """
29
38
  # Retrieving the logging level from the environment variable LOGGING_LEVEL
30
- # with a default of DEBUG, and converting it to uppercase
31
- loggingLevel = os.getenv("LOGGING_LEVEL", "DEBUG").upper()
39
+ # with a default of INFO, and converting it to uppercase
40
+ loggingLevel = os.getenv("LOGGING_LEVEL", "INFO").upper()
32
41
 
33
42
  # Setting the logging level for the root logger to the obtained logging level
34
43
  logging.getLogger().setLevel(loggingLevel)
@@ -50,6 +50,8 @@ dev = [
50
50
  "python-dotenv",
51
51
  "pyjwt",
52
52
  "psycopg2-binary",
53
+ # In-process S3 for the UIPublisher sync provider's unit tests.
54
+ "moto[s3]>=5.0",
53
55
  ]
54
56
 
55
57
  # fixture_foundry is NOT listed above on purpose. PyPI's upload API rejects
@@ -1,137 +0,0 @@
1
- import pulumi
2
- import pulumi_aws as aws
3
- import os
4
- from mimetypes import guess_type
5
-
6
- from cloud_foundry.utils.logger import logger
7
-
8
- log = logger(__name__)
9
-
10
-
11
- class UIPublisherArgs:
12
- def __init__(
13
- self, name: str, dist_dir: str, project_dir: str = ".", prefix: str = ""
14
- ):
15
- self.name = name
16
- self.dist_dir = dist_dir
17
- self.project_dir = project_dir
18
- self.prefix = prefix
19
-
20
-
21
- class UIPublisher(pulumi.ComponentResource):
22
- """
23
- A Pulumi component to handle the publishing of UI assets to an S3 bucket.
24
-
25
- This component takes care of uploading files from a specified directory to an S3 bucket,
26
- setting the appropriate content type for each file based on its extension.
27
- """
28
-
29
- def __init__(
30
- self,
31
- bucket,
32
- args: UIPublisherArgs,
33
- opts: pulumi.ResourceOptions = None,
34
- ):
35
- """
36
- Initialize the UIPublisher component.
37
-
38
- Args:
39
- name (str): The name of the component.
40
- bucket: The S3 bucket resource to upload files to.
41
- args (UIPublisherArgs): The arguments for the UIPublisher component.
42
- opts (ResourceOptions): Optional resource options.
43
- """
44
- super().__init__("cloud_foundry:pulumi:UIPublisher", args.name, {}, opts)
45
-
46
- log.info(f"args: {args.__dict__}")
47
- self.bucket = bucket
48
- self.dist_dir = args.dist_dir or os.path.join(args.project_dir, "dist")
49
-
50
- self.upload_files(self.dist_dir, bucket, args.prefix)
51
-
52
- self.register_outputs({})
53
-
54
- @staticmethod
55
- def cache_control_for_key(key: str) -> str:
56
- """Return a conservative cache policy for a static site object."""
57
- normalized = key.lower().lstrip("/")
58
-
59
- if normalized == "_payload.json" or normalized.endswith("/_payload.json"):
60
- return "public, max-age=31536000, immutable"
61
-
62
- if normalized.startswith("_nuxt/"):
63
- return "public, max-age=31536000, immutable"
64
-
65
- if normalized.endswith(".html"):
66
- return "public, max-age=300, s-maxage=86400, stale-while-revalidate=3600"
67
-
68
- if normalized.endswith(
69
- (
70
- ".js",
71
- ".mjs",
72
- ".css",
73
- ".svg",
74
- ".png",
75
- ".jpg",
76
- ".jpeg",
77
- ".gif",
78
- ".webp",
79
- ".ico",
80
- ".woff",
81
- ".woff2",
82
- ".ttf",
83
- ".otf",
84
- ".eot",
85
- )
86
- ):
87
- return "public, max-age=31536000, immutable"
88
-
89
- if normalized.endswith((".json", ".xml", ".txt", ".webmanifest")):
90
- return "public, max-age=3600, stale-while-revalidate=300"
91
-
92
- return "public, max-age=3600"
93
-
94
- def remap_path_to_s3(self, dir_base: str, key_base: str):
95
- """
96
- Remap local file paths to S3 keys.
97
-
98
- Args:
99
- dir_base (str): The base directory containing the files.
100
- key_base (str): The base key to prepend to the S3 keys.
101
-
102
- Returns:
103
- list[dict]: A list of dictionaries containing the local file paths and corresponding S3 keys.
104
- """
105
- log.info(f"remap: dir_base: {dir_base}")
106
- dir_base = os.path.abspath(dir_base)
107
- return [
108
- {
109
- "path": os.path.join(root, file),
110
- "key": os.path.join(
111
- key_base, os.path.relpath(os.path.join(root, file), dir_base)
112
- ).replace("\\", "/"),
113
- }
114
- for root, _, files in os.walk(dir_base)
115
- for file in files
116
- ]
117
-
118
- def upload_files(self, dir: str, bucket, key: str = ""):
119
- """
120
- Upload files from a directory to an S3 bucket.
121
-
122
- Args:
123
- dir (str): The directory containing the files to upload.
124
- bucket: The S3 bucket resource to upload files to.
125
- key (str): The prefix to add to the S3 keys (optional).
126
- """
127
- for item in self.remap_path_to_s3(dir, key):
128
- content_type, _ = guess_type(item["path"])
129
- aws.s3.BucketObjectv2(
130
- item["key"],
131
- bucket=bucket.id,
132
- key=item["key"],
133
- source=pulumi.FileAsset(item["path"]),
134
- content_type=content_type,
135
- cache_control=self.cache_control_for_key(item["key"]),
136
- opts=pulumi.ResourceOptions(parent=self, depends_on=[bucket]),
137
- )
File without changes
File without changes
File without changes