cloud-foundry 0.4.2__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/PKG-INFO +38 -1
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/README.md +36 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/__init__.py +9 -1
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/idle_reaper.py +7 -5
- cloud_foundry-0.5.1/cloud_foundry/pulumi/postgres_schema.py +259 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/queue.py +35 -1
- cloud_foundry-0.5.1/cloud_foundry/pulumi/schedule.py +155 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/topic.py +32 -42
- cloud_foundry-0.5.1/cloud_foundry/pulumi/topic_subscription.py +115 -0
- cloud_foundry-0.5.1/cloud_foundry/services/postgres_schema_bootstrap.py +274 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/pyproject.toml +1 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/.gitignore +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/LICENSE +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/archive_builder.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/mail_publisher.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/migrations/__init__.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/migrations/forget_site_objects.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/__init__.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/api_waf.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/cdn.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/cdn_api_origin.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/cdn_site_origin.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/connection_store.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/custom_domain.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/document_repository.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/domain.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/function.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/publisher.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/python_function.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/rest_api.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/site_bucket.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/ui_publisher.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/websocket_api.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/pulumi/websocket_function.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/python_archive_builder.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/services/authorization_api/security_lambda.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/services/mail_publisher.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/aws_openapi_editor.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/hash_comparator.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/idle_reaper_client.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/logger.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/names.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/openapi_editor.py +0 -0
- {cloud_foundry-0.4.2 → cloud_foundry-0.5.1}/cloud_foundry/utils/websocket_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloud-foundry
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: cloud-foundry is a curated set of cloud components that can be used to assemble cloud-based applications.
|
|
5
5
|
Project-URL: Documentation, https://github.com/DanRepik/cloud-foundry
|
|
6
6
|
Project-URL: Source, https://github.com/DanRepik/cloud-foundry
|
|
@@ -13,6 +13,7 @@ Requires-Python: >=3.9
|
|
|
13
13
|
Requires-Dist: boto3
|
|
14
14
|
Requires-Dist: pulumi
|
|
15
15
|
Requires-Dist: pulumi-aws
|
|
16
|
+
Requires-Dist: pulumi-random
|
|
16
17
|
Provides-Extra: dev
|
|
17
18
|
Requires-Dist: black>=23.0.0; extra == 'dev'
|
|
18
19
|
Requires-Dist: build; extra == 'dev'
|
|
@@ -876,3 +877,39 @@ cdn_instance = CDN(
|
|
|
876
877
|
#### **Conclusion**
|
|
877
878
|
|
|
878
879
|
The `cdn` function simplifies the deployment of a Content Delivery Network using AWS CloudFront, Route 53, and ACM. It supports static sites, APIs, custom domains, and advanced features like logging and geo-restrictions. This function is ideal for building scalable, secure, and performant web applications.
|
|
880
|
+
|
|
881
|
+
## `subscribe_queue_to_topic`, `schedule` and `postgres_schema`
|
|
882
|
+
|
|
883
|
+
Added in 0.5.0.
|
|
884
|
+
|
|
885
|
+
- **`subscribe_queue_to_topic(name, topic_arn=..., queue=..., filter_policy=None, filter_policy_scope=None, raw_message_delivery=False)`** subscribes a `Queue` to an SNS topic, including one imported by ARN from another stack, with an optional filter policy (`filter_policy_scope="MessageBody"` filters on fields of a JSON body). `Queue.subscribe_to_topic(topic_arn, ...)` is the same thing from the queue's side, and `Topic.subscribe` now takes the same filter arguments. The queue policy it creates is the queue's only policy, so give each topic its own queue.
|
|
886
|
+
- **`schedule(name, schedule_expression=..., function=... | target_arn=..., input=None, enabled=True)`** creates an EventBridge rule with one target. For a function it also grants EventBridge the invoke permission; for any other target, pass `target_arn` (and `role_arn` if the target needs one) and grant access yourself.
|
|
887
|
+
- **`postgres_schema(name, cluster_arn=..., master_secret_arn=..., database=..., role_name=..., schema_sql_path=..., schemas=[...], extensions=[{"name": "vector", "schema": "..."}], member_of=[...], search_path=[...])`** gives an application its own login role and schema on a shared Aurora cluster. It creates the role's credentials secret and a bootstrap Lambda that, over the RDS Data API and with no VPC attachment, creates or updates the role as the master user, creates the schemas (granting the role USAGE and CREATE) and extensions, grants the `member_of` roles, sets the search path, and then applies the script as the role in one transaction. Pulumi re-runs it when the script, the configuration or the password changes. The script and configuration are bundled into the function, never read from the invocation event.
|
|
888
|
+
|
|
889
|
+
```python
|
|
890
|
+
from cloud_foundry import postgres_schema, python_function, queue, schedule
|
|
891
|
+
|
|
892
|
+
db = postgres_schema(
|
|
893
|
+
"insight-db",
|
|
894
|
+
cluster_arn=shared_cluster_arn,
|
|
895
|
+
master_secret_arn=shared_master_secret_arn,
|
|
896
|
+
database="candidate_evidence",
|
|
897
|
+
role_name="insight_app",
|
|
898
|
+
schemas=["insight"],
|
|
899
|
+
extensions=[{"name": "vector", "schema": "insight"}],
|
|
900
|
+
member_of=["cep_lens_reader"],
|
|
901
|
+
search_path=["insight", "cep", "public"],
|
|
902
|
+
schema_sql_path="resources/insight_schema.sql",
|
|
903
|
+
)
|
|
904
|
+
|
|
905
|
+
inbox = queue("index-requests", visibility_timeout=900)
|
|
906
|
+
inbox.subscribe_to_topic(
|
|
907
|
+
cep_events_topic_arn,
|
|
908
|
+
filter_policy={"operation": ["artifact_version_created"]},
|
|
909
|
+
filter_policy_scope="MessageBody",
|
|
910
|
+
)
|
|
911
|
+
worker = python_function("index-worker", sources={"app.py": "file://worker.py"})
|
|
912
|
+
inbox.subscribe(worker, batch_size=1, maximum_concurrency=4)
|
|
913
|
+
schedule("index-reconcile", schedule_expression="rate(1 hour)",
|
|
914
|
+
function=worker, input={"operation": "index_reconcile"})
|
|
915
|
+
```
|
|
@@ -841,3 +841,39 @@ cdn_instance = CDN(
|
|
|
841
841
|
#### **Conclusion**
|
|
842
842
|
|
|
843
843
|
The `cdn` function simplifies the deployment of a Content Delivery Network using AWS CloudFront, Route 53, and ACM. It supports static sites, APIs, custom domains, and advanced features like logging and geo-restrictions. This function is ideal for building scalable, secure, and performant web applications.
|
|
844
|
+
|
|
845
|
+
## `subscribe_queue_to_topic`, `schedule` and `postgres_schema`
|
|
846
|
+
|
|
847
|
+
Added in 0.5.0.
|
|
848
|
+
|
|
849
|
+
- **`subscribe_queue_to_topic(name, topic_arn=..., queue=..., filter_policy=None, filter_policy_scope=None, raw_message_delivery=False)`** subscribes a `Queue` to an SNS topic, including one imported by ARN from another stack, with an optional filter policy (`filter_policy_scope="MessageBody"` filters on fields of a JSON body). `Queue.subscribe_to_topic(topic_arn, ...)` is the same thing from the queue's side, and `Topic.subscribe` now takes the same filter arguments. The queue policy it creates is the queue's only policy, so give each topic its own queue.
|
|
850
|
+
- **`schedule(name, schedule_expression=..., function=... | target_arn=..., input=None, enabled=True)`** creates an EventBridge rule with one target. For a function it also grants EventBridge the invoke permission; for any other target, pass `target_arn` (and `role_arn` if the target needs one) and grant access yourself.
|
|
851
|
+
- **`postgres_schema(name, cluster_arn=..., master_secret_arn=..., database=..., role_name=..., schema_sql_path=..., schemas=[...], extensions=[{"name": "vector", "schema": "..."}], member_of=[...], search_path=[...])`** gives an application its own login role and schema on a shared Aurora cluster. It creates the role's credentials secret and a bootstrap Lambda that, over the RDS Data API and with no VPC attachment, creates or updates the role as the master user, creates the schemas (granting the role USAGE and CREATE) and extensions, grants the `member_of` roles, sets the search path, and then applies the script as the role in one transaction. Pulumi re-runs it when the script, the configuration or the password changes. The script and configuration are bundled into the function, never read from the invocation event.
|
|
852
|
+
|
|
853
|
+
```python
|
|
854
|
+
from cloud_foundry import postgres_schema, python_function, queue, schedule
|
|
855
|
+
|
|
856
|
+
db = postgres_schema(
|
|
857
|
+
"insight-db",
|
|
858
|
+
cluster_arn=shared_cluster_arn,
|
|
859
|
+
master_secret_arn=shared_master_secret_arn,
|
|
860
|
+
database="candidate_evidence",
|
|
861
|
+
role_name="insight_app",
|
|
862
|
+
schemas=["insight"],
|
|
863
|
+
extensions=[{"name": "vector", "schema": "insight"}],
|
|
864
|
+
member_of=["cep_lens_reader"],
|
|
865
|
+
search_path=["insight", "cep", "public"],
|
|
866
|
+
schema_sql_path="resources/insight_schema.sql",
|
|
867
|
+
)
|
|
868
|
+
|
|
869
|
+
inbox = queue("index-requests", visibility_timeout=900)
|
|
870
|
+
inbox.subscribe_to_topic(
|
|
871
|
+
cep_events_topic_arn,
|
|
872
|
+
filter_policy={"operation": ["artifact_version_created"]},
|
|
873
|
+
filter_policy_scope="MessageBody",
|
|
874
|
+
)
|
|
875
|
+
worker = python_function("index-worker", sources={"app.py": "file://worker.py"})
|
|
876
|
+
inbox.subscribe(worker, batch_size=1, maximum_concurrency=4)
|
|
877
|
+
schedule("index-reconcile", schedule_expression="rate(1 hour)",
|
|
878
|
+
function=worker, input={"operation": "index_reconcile"})
|
|
879
|
+
```
|
|
@@ -21,10 +21,18 @@ from cloud_foundry.pulumi.domain import domain # noqa: F401
|
|
|
21
21
|
|
|
22
22
|
from cloud_foundry.pulumi.queue import queue # noqa: F401
|
|
23
23
|
from cloud_foundry.pulumi.topic import topic # noqa: F401
|
|
24
|
+
from cloud_foundry.pulumi.topic_subscription import ( # noqa: F401
|
|
25
|
+
subscribe_queue_to_topic,
|
|
26
|
+
)
|
|
27
|
+
from cloud_foundry.pulumi.schedule import schedule, Schedule # noqa: F401
|
|
28
|
+
from cloud_foundry.pulumi.postgres_schema import ( # noqa: F401
|
|
29
|
+
postgres_schema,
|
|
30
|
+
PostgresSchema,
|
|
31
|
+
)
|
|
24
32
|
from cloud_foundry.pulumi.idle_reaper import attach_idle_reaper # noqa: F401
|
|
25
33
|
from cloud_foundry.utils.idle_reaper_client import request_mapping_enable # noqa: F401
|
|
26
34
|
|
|
27
35
|
from cloud_foundry.utils.names import resource_id # noqa: F401
|
|
28
36
|
from cloud_foundry.utils.names import project_slug # noqa: F401
|
|
29
37
|
|
|
30
|
-
version = "0.
|
|
38
|
+
version = "0.5.1"
|
|
@@ -32,7 +32,7 @@ import json
|
|
|
32
32
|
from typing import Optional
|
|
33
33
|
|
|
34
34
|
import pulumi_aws as aws
|
|
35
|
-
from pulumi import Output, ResourceOptions
|
|
35
|
+
from pulumi import Input, Output, ResourceOptions
|
|
36
36
|
|
|
37
37
|
from cloud_foundry.utils.names import account_id, region
|
|
38
38
|
|
|
@@ -57,8 +57,8 @@ def attach_idle_reaper(
|
|
|
57
57
|
queue,
|
|
58
58
|
event_source_mapping: aws.lambda_.EventSourceMapping,
|
|
59
59
|
*,
|
|
60
|
-
reaper_function_name: str,
|
|
61
|
-
reaper_role_name: str,
|
|
60
|
+
reaper_function_name: Input[str],
|
|
61
|
+
reaper_role_name: Input[str],
|
|
62
62
|
resource_prefix: str,
|
|
63
63
|
interval_minutes: int = 10,
|
|
64
64
|
event_bus_name: str = "default",
|
|
@@ -128,8 +128,10 @@ def attach_idle_reaper(
|
|
|
128
128
|
opts=opts,
|
|
129
129
|
)
|
|
130
130
|
|
|
131
|
-
|
|
132
|
-
|
|
131
|
+
# Either a plain str or an Output (e.g. a StackReference output from
|
|
132
|
+
# another project), so build the ARN inside apply().
|
|
133
|
+
reaper_arn = Output.from_input(reaper_function_name).apply(
|
|
134
|
+
lambda name: f"arn:aws:lambda:{region()}:{account_id()}:function:{name}"
|
|
133
135
|
)
|
|
134
136
|
|
|
135
137
|
aws.cloudwatch.EventTarget(
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Own a Postgres schema on a shared Aurora cluster: login role, secret,
|
|
2
|
+
schemas, extensions and a bundled schema script, applied over the RDS Data
|
|
3
|
+
API on every change.
|
|
4
|
+
|
|
5
|
+
Creates a credentials secret for the role, a bootstrap Lambda (no VPC; see
|
|
6
|
+
``cloud_foundry/services/postgres_schema_bootstrap.py``) and an invocation
|
|
7
|
+
that re-runs it when the script, the configuration or the password changes.
|
|
8
|
+
|
|
9
|
+
Example:
|
|
10
|
+
>>> from cloud_foundry import postgres_schema
|
|
11
|
+
>>> insight_db = postgres_schema(
|
|
12
|
+
... "insight-db",
|
|
13
|
+
... cluster_arn=shared_cluster_arn,
|
|
14
|
+
... master_secret_arn=shared_secret_arn,
|
|
15
|
+
... database="candidate_evidence",
|
|
16
|
+
... role_name="insight_app",
|
|
17
|
+
... schemas=["insight"],
|
|
18
|
+
... extensions=[{"name": "vector", "schema": "insight"}],
|
|
19
|
+
... member_of=["cep_lens_reader"],
|
|
20
|
+
... search_path=["insight", "cep", "public"],
|
|
21
|
+
... schema_sql_path="infra/index/resources/insight_schema.sql",
|
|
22
|
+
... )
|
|
23
|
+
>>> insight_db.secret_arn # credentials for the role
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import hashlib
|
|
27
|
+
import json
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Optional
|
|
30
|
+
|
|
31
|
+
import pulumi
|
|
32
|
+
import pulumi_aws as aws
|
|
33
|
+
import pulumi_random as random
|
|
34
|
+
from pulumi import ComponentResource, Input, Output, ResourceOptions
|
|
35
|
+
|
|
36
|
+
from cloud_foundry.pulumi.python_function import python_function
|
|
37
|
+
from cloud_foundry.utils.names import resource_id
|
|
38
|
+
|
|
39
|
+
_SERVICE = "file://" + str(
|
|
40
|
+
Path(__file__).resolve().parent.parent / "services" / "postgres_schema_bootstrap.py"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class PostgresSchema(ComponentResource):
|
|
45
|
+
"""A login role and its schema, kept in step with a script.
|
|
46
|
+
|
|
47
|
+
Attributes:
|
|
48
|
+
secret (aws.secretsmanager.Secret): The role's credentials
|
|
49
|
+
(``username``, ``password``, ``engine``, ``database`` and, when
|
|
50
|
+
given, ``host`` and ``port``).
|
|
51
|
+
function: The bootstrap Lambda.
|
|
52
|
+
invocation (aws.lambda_.Invocation): The latest bootstrap run; its
|
|
53
|
+
``result`` is the handler's JSON summary.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
name: str,
|
|
59
|
+
*,
|
|
60
|
+
cluster_arn: Input[str],
|
|
61
|
+
master_secret_arn: Input[str],
|
|
62
|
+
database: Input[str],
|
|
63
|
+
role_name: str,
|
|
64
|
+
schema_sql_path: Optional[str] = None,
|
|
65
|
+
schemas: Optional[list[str]] = None,
|
|
66
|
+
extensions: Optional[list[dict]] = None,
|
|
67
|
+
member_of: Optional[list[str]] = None,
|
|
68
|
+
search_path: Optional[list[str]] = None,
|
|
69
|
+
create_database: bool = False,
|
|
70
|
+
maintenance_database: str = "postgres",
|
|
71
|
+
host: Optional[Input[str]] = None,
|
|
72
|
+
port: Optional[Input[int]] = None,
|
|
73
|
+
opts: Optional[ResourceOptions] = None,
|
|
74
|
+
):
|
|
75
|
+
super().__init__("cloud_foundry:postgres:PostgresSchema", name, {}, opts)
|
|
76
|
+
child = ResourceOptions(parent=self)
|
|
77
|
+
rid = resource_id(name)
|
|
78
|
+
|
|
79
|
+
password = random.RandomPassword(
|
|
80
|
+
f"{name}-password", length=32, special=False, opts=child
|
|
81
|
+
)
|
|
82
|
+
self.secret = aws.secretsmanager.Secret(
|
|
83
|
+
f"{name}-secret",
|
|
84
|
+
name=f"{rid}-credentials",
|
|
85
|
+
description=f"Postgres credentials for role {role_name}",
|
|
86
|
+
recovery_window_in_days=0,
|
|
87
|
+
opts=child,
|
|
88
|
+
)
|
|
89
|
+
secret_version = aws.secretsmanager.SecretVersion(
|
|
90
|
+
f"{name}-secret-version",
|
|
91
|
+
secret_id=self.secret.id,
|
|
92
|
+
secret_string=Output.all(
|
|
93
|
+
password=password.result, database=database, host=host, port=port
|
|
94
|
+
).apply(
|
|
95
|
+
lambda v: json.dumps(
|
|
96
|
+
{
|
|
97
|
+
key: value
|
|
98
|
+
for key, value in {
|
|
99
|
+
"username": role_name,
|
|
100
|
+
"password": v["password"],
|
|
101
|
+
"engine": "postgres",
|
|
102
|
+
"database": v["database"],
|
|
103
|
+
"host": v["host"],
|
|
104
|
+
"port": v["port"],
|
|
105
|
+
}.items()
|
|
106
|
+
if value is not None
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
),
|
|
110
|
+
opts=child,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Static structure only; ARNs and the database name may be Outputs,
|
|
114
|
+
# so they travel in the environment instead.
|
|
115
|
+
config = {
|
|
116
|
+
"role_name": role_name,
|
|
117
|
+
"schemas": schemas or [],
|
|
118
|
+
"extensions": extensions or [],
|
|
119
|
+
"member_of": member_of or [],
|
|
120
|
+
"search_path": search_path or [],
|
|
121
|
+
"create_database": create_database,
|
|
122
|
+
}
|
|
123
|
+
config_json = json.dumps(config, sort_keys=True)
|
|
124
|
+
schema_sql = (
|
|
125
|
+
Path(schema_sql_path).read_text(encoding="utf-8") if schema_sql_path else ""
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
sources = {
|
|
129
|
+
"app.py": _SERVICE,
|
|
130
|
+
"config.json": config_json,
|
|
131
|
+
}
|
|
132
|
+
if schema_sql_path:
|
|
133
|
+
sources["schema.sql"] = "file://" + str(Path(schema_sql_path).resolve())
|
|
134
|
+
|
|
135
|
+
self.function = python_function(
|
|
136
|
+
f"{name}-bootstrap",
|
|
137
|
+
sources=sources,
|
|
138
|
+
# Bundled rather than the runtime's boto3, which can predate
|
|
139
|
+
# rds-data's DatabaseResumingException (auto-pause).
|
|
140
|
+
requirements=["boto3"],
|
|
141
|
+
timeout=300,
|
|
142
|
+
environment={
|
|
143
|
+
"CLUSTER_ARN": cluster_arn,
|
|
144
|
+
"MASTER_SECRET_ARN": master_secret_arn,
|
|
145
|
+
"ROLE_SECRET_ARN": self.secret.arn,
|
|
146
|
+
"DATABASE": database,
|
|
147
|
+
"MAINTENANCE_DATABASE": maintenance_database,
|
|
148
|
+
},
|
|
149
|
+
policy_statements=[
|
|
150
|
+
{
|
|
151
|
+
"Effect": "Allow",
|
|
152
|
+
"Actions": [
|
|
153
|
+
"rds-data:ExecuteStatement",
|
|
154
|
+
"rds-data:BeginTransaction",
|
|
155
|
+
"rds-data:CommitTransaction",
|
|
156
|
+
"rds-data:RollbackTransaction",
|
|
157
|
+
],
|
|
158
|
+
"Resources": [cluster_arn],
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
"Effect": "Allow",
|
|
162
|
+
"Actions": ["secretsmanager:GetSecretValue"],
|
|
163
|
+
"Resources": [master_secret_arn, self.secret.arn],
|
|
164
|
+
},
|
|
165
|
+
],
|
|
166
|
+
opts=child,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
digest = hashlib.sha256((config_json + "\n" + schema_sql).encode()).hexdigest()
|
|
170
|
+
self.invocation = aws.lambda_.Invocation(
|
|
171
|
+
f"{name}-invoke",
|
|
172
|
+
# The Lambda resource's name Output, not Function.function_name
|
|
173
|
+
# (a plain string): only the Output makes Pulumi wait for the
|
|
174
|
+
# function to exist before invoking it.
|
|
175
|
+
function_name=self.function.lambda_.name,
|
|
176
|
+
# The handler ignores the event; input and triggers only make
|
|
177
|
+
# Pulumi re-invoke when the script, configuration or password
|
|
178
|
+
# changes.
|
|
179
|
+
input=json.dumps({"digest": digest}),
|
|
180
|
+
triggers={
|
|
181
|
+
"digest": digest,
|
|
182
|
+
"secret_version": secret_version.version_id,
|
|
183
|
+
},
|
|
184
|
+
opts=ResourceOptions(parent=self, depends_on=[secret_version]),
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
self.register_outputs(
|
|
188
|
+
{"secret_arn": self.secret.arn, "result": self.invocation.result}
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def secret_arn(self) -> Output[str]:
|
|
193
|
+
return self.secret.arn
|
|
194
|
+
|
|
195
|
+
@property
|
|
196
|
+
def result(self) -> Output[str]:
|
|
197
|
+
return self.invocation.result
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def postgres_schema(
|
|
201
|
+
name: str,
|
|
202
|
+
*,
|
|
203
|
+
cluster_arn: Input[str],
|
|
204
|
+
master_secret_arn: Input[str],
|
|
205
|
+
database: Input[str],
|
|
206
|
+
role_name: str,
|
|
207
|
+
schema_sql_path: Optional[str] = None,
|
|
208
|
+
schemas: Optional[list[str]] = None,
|
|
209
|
+
extensions: Optional[list[dict]] = None,
|
|
210
|
+
member_of: Optional[list[str]] = None,
|
|
211
|
+
search_path: Optional[list[str]] = None,
|
|
212
|
+
create_database: bool = False,
|
|
213
|
+
maintenance_database: str = "postgres",
|
|
214
|
+
host: Optional[Input[str]] = None,
|
|
215
|
+
port: Optional[Input[int]] = None,
|
|
216
|
+
opts: Optional[ResourceOptions] = None,
|
|
217
|
+
) -> PostgresSchema:
|
|
218
|
+
"""Create a PostgresSchema component.
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
name: Logical name; the secret is ``<resource_id(name)>-credentials``.
|
|
222
|
+
cluster_arn: Aurora cluster with the Data API enabled.
|
|
223
|
+
master_secret_arn: Secret of a user that can create roles, schemas
|
|
224
|
+
and extensions and grant ``member_of`` roles.
|
|
225
|
+
database: Target database.
|
|
226
|
+
role_name: Login role to create or update; uses ``schemas`` and
|
|
227
|
+
applies the script.
|
|
228
|
+
schema_sql_path: Script applied as ``role_name`` in one transaction.
|
|
229
|
+
Write it to be re-runnable (``IF NOT EXISTS``, ``CREATE OR
|
|
230
|
+
REPLACE``).
|
|
231
|
+
schemas: Schemas created by the master user, with USAGE and CREATE
|
|
232
|
+
granted to ``role_name`` (which owns what its script creates).
|
|
233
|
+
extensions: ``[{"name": ..., "schema": ...}]``, created by the master
|
|
234
|
+
user (extensions need it).
|
|
235
|
+
member_of: Existing roles granted to ``role_name``.
|
|
236
|
+
search_path: Set on the role in ``database``.
|
|
237
|
+
create_database: Create ``database`` if it does not exist.
|
|
238
|
+
maintenance_database: Database the master user connects to for role
|
|
239
|
+
and database creation.
|
|
240
|
+
host, port: Included in the secret for clients that connect directly.
|
|
241
|
+
opts: Pulumi resource options.
|
|
242
|
+
"""
|
|
243
|
+
return PostgresSchema(
|
|
244
|
+
name,
|
|
245
|
+
cluster_arn=cluster_arn,
|
|
246
|
+
master_secret_arn=master_secret_arn,
|
|
247
|
+
database=database,
|
|
248
|
+
role_name=role_name,
|
|
249
|
+
schema_sql_path=schema_sql_path,
|
|
250
|
+
schemas=schemas,
|
|
251
|
+
extensions=extensions,
|
|
252
|
+
member_of=member_of,
|
|
253
|
+
search_path=search_path,
|
|
254
|
+
create_database=create_database,
|
|
255
|
+
maintenance_database=maintenance_database,
|
|
256
|
+
host=host,
|
|
257
|
+
port=port,
|
|
258
|
+
opts=opts,
|
|
259
|
+
)
|
|
@@ -165,7 +165,7 @@ class Queue(ComponentResource):
|
|
|
165
165
|
if isinstance(function, Function):
|
|
166
166
|
function_name = function.function_name
|
|
167
167
|
elif isinstance(function, aws.lambda_.Function):
|
|
168
|
-
function_name = function.
|
|
168
|
+
function_name = function.name
|
|
169
169
|
else:
|
|
170
170
|
function_name = function
|
|
171
171
|
|
|
@@ -201,6 +201,40 @@ class Queue(ComponentResource):
|
|
|
201
201
|
**mapping_args,
|
|
202
202
|
)
|
|
203
203
|
|
|
204
|
+
def subscribe_to_topic(
|
|
205
|
+
self,
|
|
206
|
+
topic_arn,
|
|
207
|
+
filter_policy: Optional[dict] = None,
|
|
208
|
+
filter_policy_scope: Optional[str] = None,
|
|
209
|
+
raw_message_delivery: bool = False,
|
|
210
|
+
) -> aws.sns.TopicSubscription:
|
|
211
|
+
"""Subscribe this queue to an SNS topic by ARN, such as one exported
|
|
212
|
+
by another stack.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
topic_arn: ARN of the topic (str or Output).
|
|
216
|
+
filter_policy: SNS filter policy. None delivers every message.
|
|
217
|
+
filter_policy_scope: "MessageAttributes" (the SNS default) or
|
|
218
|
+
"MessageBody".
|
|
219
|
+
raw_message_delivery: Deliver without the SNS envelope.
|
|
220
|
+
|
|
221
|
+
Returns:
|
|
222
|
+
aws.sns.TopicSubscription: The created subscription.
|
|
223
|
+
"""
|
|
224
|
+
from cloud_foundry.pulumi.topic_subscription import (
|
|
225
|
+
subscribe_queue_to_topic,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
return subscribe_queue_to_topic(
|
|
229
|
+
f"{self.name}-topic",
|
|
230
|
+
topic_arn=topic_arn,
|
|
231
|
+
queue=self,
|
|
232
|
+
filter_policy=filter_policy,
|
|
233
|
+
filter_policy_scope=filter_policy_scope,
|
|
234
|
+
raw_message_delivery=raw_message_delivery,
|
|
235
|
+
opts=ResourceOptions(parent=self),
|
|
236
|
+
)
|
|
237
|
+
|
|
204
238
|
|
|
205
239
|
def queue(
|
|
206
240
|
name: str,
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Run a Lambda function on a schedule (EventBridge rule, target, permission).
|
|
2
|
+
|
|
3
|
+
Example:
|
|
4
|
+
>>> from cloud_foundry import python_function, schedule
|
|
5
|
+
>>> worker = python_function("index-worker", sources={...})
|
|
6
|
+
>>> schedule(
|
|
7
|
+
... "index-reconcile",
|
|
8
|
+
... schedule_expression="rate(1 hour)",
|
|
9
|
+
... function=worker,
|
|
10
|
+
... input={"operation": "index_reconcile"},
|
|
11
|
+
... )
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from typing import Any, Optional, Union
|
|
16
|
+
|
|
17
|
+
import pulumi_aws as aws
|
|
18
|
+
from pulumi import ComponentResource, Input, Output, ResourceOptions
|
|
19
|
+
|
|
20
|
+
from cloud_foundry.pulumi.function import Function
|
|
21
|
+
from cloud_foundry.utils.names import resource_id
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _function_name(function) -> Input[str]:
|
|
25
|
+
# Outputs of the Lambda resource, not Function.function_name (a plain
|
|
26
|
+
# string), so the permission waits for the function to exist.
|
|
27
|
+
if isinstance(function, Function):
|
|
28
|
+
return function.lambda_.name
|
|
29
|
+
if isinstance(function, aws.lambda_.Function):
|
|
30
|
+
return function.name
|
|
31
|
+
return function
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _function_arn(function) -> Input[str]:
|
|
35
|
+
if isinstance(function, Function):
|
|
36
|
+
return function.lambda_.arn
|
|
37
|
+
if isinstance(function, aws.lambda_.Function):
|
|
38
|
+
return function.arn
|
|
39
|
+
raise TypeError(
|
|
40
|
+
"schedule() needs a cloud_foundry Function or aws.lambda_.Function; "
|
|
41
|
+
"pass target_arn for any other target"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Schedule(ComponentResource):
|
|
46
|
+
"""An EventBridge rule on a schedule with one target.
|
|
47
|
+
|
|
48
|
+
Targets a Lambda function (the invoke permission is created here), or any
|
|
49
|
+
other EventBridge target by ``target_arn`` (the caller grants EventBridge
|
|
50
|
+
whatever that target needs, and may pass ``role_arn``).
|
|
51
|
+
|
|
52
|
+
Attributes:
|
|
53
|
+
rule (aws.cloudwatch.EventRule): The scheduled rule.
|
|
54
|
+
target (aws.cloudwatch.EventTarget): Its single target.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(
|
|
58
|
+
self,
|
|
59
|
+
name: str,
|
|
60
|
+
*,
|
|
61
|
+
schedule_expression: str,
|
|
62
|
+
function: Union[Function, aws.lambda_.Function, None] = None,
|
|
63
|
+
target_arn: Optional[Input[str]] = None,
|
|
64
|
+
role_arn: Optional[Input[str]] = None,
|
|
65
|
+
input: Optional[Union[dict[str, Any], Output]] = None,
|
|
66
|
+
enabled: bool = True,
|
|
67
|
+
description: Optional[str] = None,
|
|
68
|
+
opts: Optional[ResourceOptions] = None,
|
|
69
|
+
):
|
|
70
|
+
super().__init__("cloud_foundry:schedule:Schedule", name, {}, opts)
|
|
71
|
+
if (function is None) == (target_arn is None):
|
|
72
|
+
raise ValueError("schedule() takes exactly one of function or target_arn")
|
|
73
|
+
|
|
74
|
+
rid = resource_id(name)
|
|
75
|
+
child = ResourceOptions(parent=self)
|
|
76
|
+
|
|
77
|
+
# EventRule names are limited to 64 characters.
|
|
78
|
+
self.rule = aws.cloudwatch.EventRule(
|
|
79
|
+
f"{name}-rule",
|
|
80
|
+
name=rid[:64],
|
|
81
|
+
schedule_expression=schedule_expression,
|
|
82
|
+
state="ENABLED" if enabled else "DISABLED",
|
|
83
|
+
description=description,
|
|
84
|
+
opts=child,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
target_args: dict[str, Any] = {
|
|
88
|
+
"rule": self.rule.name,
|
|
89
|
+
"target_id": rid[:64],
|
|
90
|
+
"arn": _function_arn(function) if function is not None else target_arn,
|
|
91
|
+
}
|
|
92
|
+
if input is not None:
|
|
93
|
+
target_args["input"] = Output.from_input(input).apply(json.dumps)
|
|
94
|
+
if role_arn is not None:
|
|
95
|
+
target_args["role_arn"] = role_arn
|
|
96
|
+
self.target = aws.cloudwatch.EventTarget(
|
|
97
|
+
f"{name}-target", **target_args, opts=child
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if function is not None:
|
|
101
|
+
aws.lambda_.Permission(
|
|
102
|
+
f"{name}-invoke",
|
|
103
|
+
statement_id=f"{rid}-schedule"[:100],
|
|
104
|
+
action="lambda:InvokeFunction",
|
|
105
|
+
function=_function_name(function),
|
|
106
|
+
principal="events.amazonaws.com",
|
|
107
|
+
source_arn=self.rule.arn,
|
|
108
|
+
opts=child,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
self.register_outputs({"rule_arn": self.rule.arn})
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def rule_arn(self) -> Output[str]:
|
|
115
|
+
return self.rule.arn
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def schedule(
|
|
119
|
+
name: str,
|
|
120
|
+
*,
|
|
121
|
+
schedule_expression: str,
|
|
122
|
+
function: Union[Function, aws.lambda_.Function, None] = None,
|
|
123
|
+
target_arn: Optional[Input[str]] = None,
|
|
124
|
+
role_arn: Optional[Input[str]] = None,
|
|
125
|
+
input: Optional[Union[dict[str, Any], Output]] = None,
|
|
126
|
+
enabled: bool = True,
|
|
127
|
+
description: Optional[str] = None,
|
|
128
|
+
opts: Optional[ResourceOptions] = None,
|
|
129
|
+
) -> Schedule:
|
|
130
|
+
"""Create a Schedule component.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
name: Logical name; the rule is named ``resource_id(name)``.
|
|
134
|
+
schedule_expression: ``rate(...)`` or ``cron(...)``.
|
|
135
|
+
function: A Lambda function to invoke. Exactly one of ``function``
|
|
136
|
+
and ``target_arn``.
|
|
137
|
+
target_arn: Any other EventBridge target.
|
|
138
|
+
role_arn: Role EventBridge assumes for ``target_arn`` targets that
|
|
139
|
+
need one.
|
|
140
|
+
input: JSON-serializable event sent to the target.
|
|
141
|
+
enabled: Create the rule disabled when False.
|
|
142
|
+
description: Rule description.
|
|
143
|
+
opts: Pulumi resource options.
|
|
144
|
+
"""
|
|
145
|
+
return Schedule(
|
|
146
|
+
name,
|
|
147
|
+
schedule_expression=schedule_expression,
|
|
148
|
+
function=function,
|
|
149
|
+
target_arn=target_arn,
|
|
150
|
+
role_arn=role_arn,
|
|
151
|
+
input=input,
|
|
152
|
+
enabled=enabled,
|
|
153
|
+
description=description,
|
|
154
|
+
opts=opts,
|
|
155
|
+
)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"""SNS Topic component for Pub/Sub messaging."""
|
|
2
2
|
|
|
3
|
-
import json
|
|
4
3
|
from typing import Optional
|
|
5
4
|
from pulumi import ComponentResource, Output, ResourceOptions
|
|
6
5
|
import pulumi_aws as aws
|
|
7
6
|
from cloud_foundry.utils.names import resource_id
|
|
8
7
|
from .queue import Queue
|
|
8
|
+
from .topic_subscription import subscribe_queue_to_topic
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class TopicArgs:
|
|
@@ -60,7 +60,14 @@ class Topic(ComponentResource):
|
|
|
60
60
|
if args.subscriptions:
|
|
61
61
|
for subscription in args.subscriptions:
|
|
62
62
|
if subscription.get("queue"):
|
|
63
|
-
self.subscribe(
|
|
63
|
+
self.subscribe(
|
|
64
|
+
subscription["queue"],
|
|
65
|
+
filter_policy=subscription.get("filter_policy"),
|
|
66
|
+
filter_policy_scope=subscription.get("filter_policy_scope"),
|
|
67
|
+
raw_message_delivery=subscription.get(
|
|
68
|
+
"raw_message_delivery", False
|
|
69
|
+
),
|
|
70
|
+
)
|
|
64
71
|
|
|
65
72
|
# Register outputs to signal component completion
|
|
66
73
|
self.register_outputs(
|
|
@@ -86,51 +93,34 @@ class Topic(ComponentResource):
|
|
|
86
93
|
"""
|
|
87
94
|
return self.topic.name
|
|
88
95
|
|
|
89
|
-
def subscribe(
|
|
96
|
+
def subscribe(
|
|
97
|
+
self,
|
|
98
|
+
queue: Queue,
|
|
99
|
+
opts: ResourceOptions = None,
|
|
100
|
+
filter_policy: Optional[dict] = None,
|
|
101
|
+
filter_policy_scope: Optional[str] = None,
|
|
102
|
+
raw_message_delivery: bool = False,
|
|
103
|
+
) -> None:
|
|
90
104
|
"""Subscribe an SQS queue to this SNS topic.
|
|
91
105
|
Args:
|
|
92
106
|
queue (Queue): The SQS queue to subscribe.
|
|
93
107
|
opts (ResourceOptions, optional): Pulumi resource options.
|
|
108
|
+
filter_policy (dict, optional): SNS filter policy. None delivers
|
|
109
|
+
every message.
|
|
110
|
+
filter_policy_scope (str, optional): "MessageAttributes" (the
|
|
111
|
+
SNS default) or "MessageBody".
|
|
112
|
+
raw_message_delivery (bool): Deliver without the SNS envelope.
|
|
94
113
|
"""
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
lambda args: json.dumps(
|
|
106
|
-
{
|
|
107
|
-
"Version": "2012-10-17",
|
|
108
|
-
"Statement": [
|
|
109
|
-
{
|
|
110
|
-
"Effect": "Allow",
|
|
111
|
-
"Principal": {"Service": "sns.amazonaws.com"},
|
|
112
|
-
"Action": "sqs:SendMessage",
|
|
113
|
-
"Resource": args["queue_arn"],
|
|
114
|
-
"Condition": {
|
|
115
|
-
"ArnEquals": {"aws:SourceArn": args["topic_arn"]}
|
|
116
|
-
},
|
|
117
|
-
}
|
|
118
|
-
],
|
|
119
|
-
}
|
|
120
|
-
)
|
|
121
|
-
),
|
|
122
|
-
opts=resource_opts,
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
# Subscribe queue to topic (depends on policy being created first)
|
|
126
|
-
aws.sns.TopicSubscription(
|
|
127
|
-
f"{resource_id(name)}-sub",
|
|
128
|
-
topic=self.topic.arn,
|
|
129
|
-
protocol="sqs",
|
|
130
|
-
endpoint=queue.arn,
|
|
131
|
-
opts=ResourceOptions.merge(
|
|
132
|
-
resource_opts, ResourceOptions(depends_on=[queue_policy])
|
|
133
|
-
),
|
|
114
|
+
subscribe_queue_to_topic(
|
|
115
|
+
f"{self.name}-{queue.name}",
|
|
116
|
+
topic_arn=self.topic.arn,
|
|
117
|
+
queue=queue,
|
|
118
|
+
filter_policy=filter_policy,
|
|
119
|
+
filter_policy_scope=filter_policy_scope,
|
|
120
|
+
raw_message_delivery=raw_message_delivery,
|
|
121
|
+
# Parented to the topic, as before, so existing stacks keep
|
|
122
|
+
# their resource URNs.
|
|
123
|
+
opts=ResourceOptions.merge(ResourceOptions(parent=self), opts),
|
|
134
124
|
)
|
|
135
125
|
|
|
136
126
|
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Subscribe an SQS queue to an SNS topic, optionally with a filter policy.
|
|
2
|
+
|
|
3
|
+
Works with a topic this program creates (``Topic.subscribe`` delegates here)
|
|
4
|
+
and with one imported by ARN from another stack, which is the common case
|
|
5
|
+
when a consumer stack listens to a producer's event topic.
|
|
6
|
+
|
|
7
|
+
Example:
|
|
8
|
+
>>> from cloud_foundry import queue, subscribe_queue_to_topic
|
|
9
|
+
>>> inbox = queue("index-requests", visibility_timeout=900)
|
|
10
|
+
>>> subscribe_queue_to_topic(
|
|
11
|
+
... "index-requests-events",
|
|
12
|
+
... topic_arn=foundation.get_output("ingestion_events_topic_arn"),
|
|
13
|
+
... queue=inbox,
|
|
14
|
+
... filter_policy={"operation": ["artifact_version_created"]},
|
|
15
|
+
... filter_policy_scope="MessageBody",
|
|
16
|
+
... )
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
from typing import Any, Optional, Union
|
|
21
|
+
|
|
22
|
+
import pulumi_aws as aws
|
|
23
|
+
from pulumi import Input, Output, ResourceOptions
|
|
24
|
+
|
|
25
|
+
from cloud_foundry.utils.names import resource_id
|
|
26
|
+
|
|
27
|
+
FilterPolicy = Union[dict[str, Any], Output]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _policy_json(filter_policy: Optional[FilterPolicy]):
|
|
31
|
+
if filter_policy is None:
|
|
32
|
+
return None
|
|
33
|
+
return Output.from_input(filter_policy).apply(
|
|
34
|
+
lambda policy: json.dumps(policy, sort_keys=True)
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def subscribe_queue_to_topic(
|
|
39
|
+
name: str,
|
|
40
|
+
*,
|
|
41
|
+
topic_arn: Input[str],
|
|
42
|
+
queue,
|
|
43
|
+
filter_policy: Optional[FilterPolicy] = None,
|
|
44
|
+
filter_policy_scope: Optional[str] = None,
|
|
45
|
+
raw_message_delivery: bool = False,
|
|
46
|
+
opts: Optional[ResourceOptions] = None,
|
|
47
|
+
) -> aws.sns.TopicSubscription:
|
|
48
|
+
"""Allow the topic to send to the queue, and subscribe the queue.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
name: Logical name; the queue policy and subscription are named
|
|
52
|
+
``<resource_id(name)>-policy`` and ``<resource_id(name)>-sub``.
|
|
53
|
+
topic_arn: ARN of the topic, created here or imported.
|
|
54
|
+
queue: A cloud_foundry ``Queue`` (anything exposing ``.arn`` and
|
|
55
|
+
``.url``).
|
|
56
|
+
filter_policy: SNS filter policy, as a dict. ``None`` delivers
|
|
57
|
+
every message.
|
|
58
|
+
filter_policy_scope: ``"MessageAttributes"`` (SNS's default) or
|
|
59
|
+
``"MessageBody"`` to filter on top-level fields of a JSON body.
|
|
60
|
+
raw_message_delivery: Deliver the published message as is, without
|
|
61
|
+
the SNS envelope.
|
|
62
|
+
opts: Pulumi options applied to both resources.
|
|
63
|
+
|
|
64
|
+
Note:
|
|
65
|
+
The queue policy created here is the queue's only policy. A queue
|
|
66
|
+
subscribed to two topics this way would have each policy overwrite
|
|
67
|
+
the other; give each topic its own queue.
|
|
68
|
+
"""
|
|
69
|
+
resource_opts = opts or ResourceOptions()
|
|
70
|
+
rid = resource_id(name)
|
|
71
|
+
|
|
72
|
+
queue_policy = aws.sqs.QueuePolicy(
|
|
73
|
+
f"{rid}-policy",
|
|
74
|
+
queue_url=queue.url,
|
|
75
|
+
policy=Output.all(queue_arn=queue.arn, topic_arn=topic_arn).apply(
|
|
76
|
+
lambda args: json.dumps(
|
|
77
|
+
{
|
|
78
|
+
"Version": "2012-10-17",
|
|
79
|
+
"Statement": [
|
|
80
|
+
{
|
|
81
|
+
"Effect": "Allow",
|
|
82
|
+
"Principal": {"Service": "sns.amazonaws.com"},
|
|
83
|
+
"Action": "sqs:SendMessage",
|
|
84
|
+
"Resource": args["queue_arn"],
|
|
85
|
+
"Condition": {
|
|
86
|
+
"ArnEquals": {"aws:SourceArn": args["topic_arn"]}
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
],
|
|
90
|
+
}
|
|
91
|
+
)
|
|
92
|
+
),
|
|
93
|
+
opts=resource_opts,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
subscription_args: dict[str, Any] = {
|
|
97
|
+
"topic": topic_arn,
|
|
98
|
+
"protocol": "sqs",
|
|
99
|
+
"endpoint": queue.arn,
|
|
100
|
+
}
|
|
101
|
+
policy_json = _policy_json(filter_policy)
|
|
102
|
+
if policy_json is not None:
|
|
103
|
+
subscription_args["filter_policy"] = policy_json
|
|
104
|
+
if filter_policy_scope:
|
|
105
|
+
subscription_args["filter_policy_scope"] = filter_policy_scope
|
|
106
|
+
if raw_message_delivery:
|
|
107
|
+
subscription_args["raw_message_delivery"] = True
|
|
108
|
+
|
|
109
|
+
return aws.sns.TopicSubscription(
|
|
110
|
+
f"{rid}-sub",
|
|
111
|
+
**subscription_args,
|
|
112
|
+
opts=ResourceOptions.merge(
|
|
113
|
+
resource_opts, ResourceOptions(depends_on=[queue_policy])
|
|
114
|
+
),
|
|
115
|
+
)
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Bootstrap a Postgres login role and schema over the RDS Data API.
|
|
2
|
+
|
|
3
|
+
Deployed by ``cloud_foundry.postgres_schema``. Everything it does is fixed at
|
|
4
|
+
deploy time: ``config.json`` (role, schemas, extensions, role memberships,
|
|
5
|
+
search path) and ``schema.sql`` are bundled into the package, and the cluster,
|
|
6
|
+
database and secret ARNs come from the function's environment
|
|
7
|
+
(``CLUSTER_ARN``, ``MASTER_SECRET_ARN``, ``ROLE_SECRET_ARN``, ``DATABASE``,
|
|
8
|
+
``MAINTENANCE_DATABASE``). The invocation event is ignored, because with master credentials in play an
|
|
9
|
+
event-supplied script would let any principal that can invoke the function
|
|
10
|
+
run arbitrary SQL.
|
|
11
|
+
|
|
12
|
+
Steps, each idempotent:
|
|
13
|
+
|
|
14
|
+
1. As the master user, in ``maintenance_database``: create or update the
|
|
15
|
+
login role with the password from its own secret, and create the target
|
|
16
|
+
database if ``create_database`` is set.
|
|
17
|
+
2. As the master user, in the target database: grant CONNECT, create each
|
|
18
|
+
schema and grant the role USAGE and CREATE on it, create each extension in its schema, grant the
|
|
19
|
+
listed roles to the role, and set its search_path.
|
|
20
|
+
3. As the role itself: apply ``schema.sql`` in one transaction.
|
|
21
|
+
|
|
22
|
+
The function needs no VPC attachment and no password in its environment.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
import time
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any, Callable, Optional
|
|
33
|
+
|
|
34
|
+
HERE = Path(__file__).parent
|
|
35
|
+
CONFIG_PATH = HERE / "config.json"
|
|
36
|
+
SCHEMA_SQL_PATH = HERE / "schema.sql"
|
|
37
|
+
|
|
38
|
+
IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
39
|
+
|
|
40
|
+
# Aurora Serverless auto-pauses; the first calls after a pause raise
|
|
41
|
+
# DatabaseResumingException until it is back.
|
|
42
|
+
RESUME_RETRY_BUDGET_SECONDS = 120
|
|
43
|
+
RESUME_RETRY_INTERVAL_SECONDS = 5
|
|
44
|
+
|
|
45
|
+
_rds_data = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def rds_data():
|
|
49
|
+
global _rds_data
|
|
50
|
+
if _rds_data is None:
|
|
51
|
+
import boto3
|
|
52
|
+
from botocore.config import Config
|
|
53
|
+
|
|
54
|
+
_rds_data = boto3.client(
|
|
55
|
+
"rds-data", config=Config(read_timeout=70, retries={"max_attempts": 1})
|
|
56
|
+
)
|
|
57
|
+
return _rds_data
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_TOKEN = re.compile(
|
|
61
|
+
r"""
|
|
62
|
+
(?P<dollar>\$(?P<tag>[A-Za-z_][A-Za-z_0-9]*)?\$)
|
|
63
|
+
| (?P<quote>'(?:[^']|'')*')
|
|
64
|
+
| (?P<ident>"(?:[^"]|"")*")
|
|
65
|
+
| (?P<line_comment>--[^\n]*)
|
|
66
|
+
| (?P<block_comment>/\*.*?\*/)
|
|
67
|
+
| (?P<semicolon>;)
|
|
68
|
+
""",
|
|
69
|
+
re.VERBOSE | re.DOTALL,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def split_statements(sql: str) -> list[str]:
|
|
74
|
+
"""Split a script on top-level semicolons (the Data API runs one
|
|
75
|
+
statement per call). Semicolons in literals, quoted identifiers, comments
|
|
76
|
+
and dollar-quoted bodies are not statement ends. Leading comments are
|
|
77
|
+
stripped and comment-only fragments dropped."""
|
|
78
|
+
statements: list[str] = []
|
|
79
|
+
start = 0
|
|
80
|
+
position = 0
|
|
81
|
+
while True:
|
|
82
|
+
match = _TOKEN.search(sql, position)
|
|
83
|
+
if not match:
|
|
84
|
+
break
|
|
85
|
+
if match.group("dollar"):
|
|
86
|
+
closing = sql.find(match.group("dollar"), match.end())
|
|
87
|
+
if closing == -1:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
"Unterminated dollar-quoted body starting at offset "
|
|
90
|
+
f"{match.start()}"
|
|
91
|
+
)
|
|
92
|
+
position = closing + len(match.group("dollar"))
|
|
93
|
+
continue
|
|
94
|
+
if match.group("semicolon"):
|
|
95
|
+
statements.append(sql[start : match.start()])
|
|
96
|
+
start = match.end()
|
|
97
|
+
position = match.end()
|
|
98
|
+
statements.append(sql[start:])
|
|
99
|
+
|
|
100
|
+
leading = re.compile(r"\A(?:\s+|--[^\n]*|/\*.*?\*/)*", re.DOTALL)
|
|
101
|
+
stripped = (leading.sub("", statement).strip() for statement in statements)
|
|
102
|
+
return [statement for statement in stripped if statement]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def identifier(value: str) -> str:
|
|
106
|
+
if not IDENTIFIER.match(value or ""):
|
|
107
|
+
raise ValueError(f"unsafe SQL identifier: {value!r}")
|
|
108
|
+
return f'"{value}"'
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def literal(value: str) -> str:
|
|
112
|
+
# standard_conforming_strings is on, so doubling quotes is sufficient.
|
|
113
|
+
return "'" + value.replace("'", "''") + "'"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _call(method: Callable, **kwargs) -> dict:
|
|
117
|
+
deadline = time.monotonic() + RESUME_RETRY_BUDGET_SECONDS
|
|
118
|
+
while True:
|
|
119
|
+
try:
|
|
120
|
+
return method(**kwargs)
|
|
121
|
+
except rds_data().exceptions.DatabaseResumingException:
|
|
122
|
+
if time.monotonic() >= deadline:
|
|
123
|
+
raise
|
|
124
|
+
time.sleep(RESUME_RETRY_INTERVAL_SECONDS)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _session(cluster_arn: str, secret_arn: str, database: str):
|
|
128
|
+
target = {"resourceArn": cluster_arn, "secretArn": secret_arn, "database": database}
|
|
129
|
+
|
|
130
|
+
def run(sql: str, parameters: Optional[list] = None) -> dict:
|
|
131
|
+
kwargs = dict(target, sql=sql)
|
|
132
|
+
if parameters:
|
|
133
|
+
kwargs["parameters"] = parameters
|
|
134
|
+
return _call(rds_data().execute_statement, **kwargs)
|
|
135
|
+
|
|
136
|
+
def exists(sql: str, name: str) -> bool:
|
|
137
|
+
params = [{"name": "n", "value": {"stringValue": name}}]
|
|
138
|
+
return bool(run(sql, params).get("records"))
|
|
139
|
+
|
|
140
|
+
return run, exists
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _secret_password(secret_arn: str) -> str:
|
|
144
|
+
import boto3
|
|
145
|
+
|
|
146
|
+
value = boto3.client("secretsmanager").get_secret_value(SecretId=secret_arn)
|
|
147
|
+
return json.loads(value["SecretString"])["password"]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def master_statements(config: dict[str, Any]) -> list[str]:
|
|
151
|
+
"""The statements step 2 runs as the master user in the target database.
|
|
152
|
+
Identifiers are validated and quoted (DDL takes no bind parameters)."""
|
|
153
|
+
role = identifier(config["role_name"])
|
|
154
|
+
database = identifier(config["database"])
|
|
155
|
+
statements = [f"GRANT CONNECT ON DATABASE {database} TO {role}"]
|
|
156
|
+
for schema in config.get("schemas", []):
|
|
157
|
+
# Created and owned by the master user, with USAGE and CREATE for the
|
|
158
|
+
# role, which then owns what its script creates. AUTHORIZATION role
|
|
159
|
+
# would need the master user to be able to SET ROLE to it, which an
|
|
160
|
+
# Aurora master (rds_superuser, not a superuser) cannot by default
|
|
161
|
+
# on PostgreSQL 16.
|
|
162
|
+
statements.append(f"CREATE SCHEMA IF NOT EXISTS {identifier(schema)}")
|
|
163
|
+
statements.append(
|
|
164
|
+
f"GRANT USAGE, CREATE ON SCHEMA {identifier(schema)} TO {role}"
|
|
165
|
+
)
|
|
166
|
+
for extension in config.get("extensions", []):
|
|
167
|
+
statement = f"CREATE EXTENSION IF NOT EXISTS {identifier(extension['name'])}"
|
|
168
|
+
if extension.get("schema"):
|
|
169
|
+
statement += f" WITH SCHEMA {identifier(extension['schema'])}"
|
|
170
|
+
statements.append(statement)
|
|
171
|
+
for member_of in config.get("member_of", []):
|
|
172
|
+
statements.append(f"GRANT {identifier(member_of)} TO {role}")
|
|
173
|
+
search_path = config.get("search_path") or []
|
|
174
|
+
if search_path:
|
|
175
|
+
path = ", ".join(identifier(s) for s in search_path)
|
|
176
|
+
# The Data API cannot pass connection options, so search_path is
|
|
177
|
+
# set on the role instead of per connection.
|
|
178
|
+
statements.append(
|
|
179
|
+
f"ALTER ROLE {role} IN DATABASE {database} SET search_path = {path}"
|
|
180
|
+
)
|
|
181
|
+
return statements
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def ensure_role(config: dict[str, Any], password: str) -> None:
|
|
185
|
+
run, exists = _session(
|
|
186
|
+
config["cluster_arn"],
|
|
187
|
+
config["master_secret_arn"],
|
|
188
|
+
config.get("maintenance_database") or "postgres",
|
|
189
|
+
)
|
|
190
|
+
role = config["role_name"]
|
|
191
|
+
verb = (
|
|
192
|
+
"ALTER"
|
|
193
|
+
if exists("SELECT 1 FROM pg_roles WHERE rolname = :n", role)
|
|
194
|
+
else "CREATE"
|
|
195
|
+
)
|
|
196
|
+
run(f"{verb} ROLE {identifier(role)} WITH LOGIN PASSWORD {literal(password)}")
|
|
197
|
+
if config.get("create_database") and not exists(
|
|
198
|
+
"SELECT 1 FROM pg_database WHERE datname = :n", config["database"]
|
|
199
|
+
):
|
|
200
|
+
# CREATE DATABASE cannot run inside a transaction; each Data API call
|
|
201
|
+
# without a transactionId autocommits.
|
|
202
|
+
run(f"CREATE DATABASE {identifier(config['database'])}")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def prepare_database(config: dict[str, Any]) -> list[str]:
|
|
206
|
+
run, _ = _session(
|
|
207
|
+
config["cluster_arn"], config["master_secret_arn"], config["database"]
|
|
208
|
+
)
|
|
209
|
+
statements = master_statements(config)
|
|
210
|
+
for statement in statements:
|
|
211
|
+
run(statement)
|
|
212
|
+
return statements
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def apply_schema(config: dict[str, Any], schema_sql: str) -> int:
|
|
216
|
+
statements = split_statements(schema_sql)
|
|
217
|
+
if not statements:
|
|
218
|
+
return 0
|
|
219
|
+
target = {
|
|
220
|
+
"resourceArn": config["cluster_arn"],
|
|
221
|
+
"secretArn": config["role_secret_arn"],
|
|
222
|
+
}
|
|
223
|
+
client = rds_data()
|
|
224
|
+
transaction_id = _call(
|
|
225
|
+
client.begin_transaction, database=config["database"], **target
|
|
226
|
+
)["transactionId"]
|
|
227
|
+
try:
|
|
228
|
+
for statement in statements:
|
|
229
|
+
_call(
|
|
230
|
+
client.execute_statement,
|
|
231
|
+
database=config["database"],
|
|
232
|
+
sql=statement,
|
|
233
|
+
transactionId=transaction_id,
|
|
234
|
+
**target,
|
|
235
|
+
)
|
|
236
|
+
except Exception:
|
|
237
|
+
client.rollback_transaction(transactionId=transaction_id, **target)
|
|
238
|
+
raise
|
|
239
|
+
client.commit_transaction(transactionId=transaction_id, **target)
|
|
240
|
+
return len(statements)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
_ENVIRONMENT = {
|
|
244
|
+
"cluster_arn": "CLUSTER_ARN",
|
|
245
|
+
"master_secret_arn": "MASTER_SECRET_ARN",
|
|
246
|
+
"role_secret_arn": "ROLE_SECRET_ARN",
|
|
247
|
+
"database": "DATABASE",
|
|
248
|
+
"maintenance_database": "MAINTENANCE_DATABASE",
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def load_config() -> dict[str, Any]:
|
|
253
|
+
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
254
|
+
for key, variable in _ENVIRONMENT.items():
|
|
255
|
+
if os.environ.get(variable):
|
|
256
|
+
config[key] = os.environ[variable]
|
|
257
|
+
return config
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def handler(_event: Any, _context: Any) -> dict[str, Any]:
|
|
261
|
+
config = load_config()
|
|
262
|
+
ensure_role(config, _secret_password(config["role_secret_arn"]))
|
|
263
|
+
prepared = prepare_database(config)
|
|
264
|
+
schema_sql = (
|
|
265
|
+
SCHEMA_SQL_PATH.read_text(encoding="utf-8") if SCHEMA_SQL_PATH.exists() else ""
|
|
266
|
+
)
|
|
267
|
+
applied = apply_schema(config, schema_sql)
|
|
268
|
+
return {
|
|
269
|
+
"ok": True,
|
|
270
|
+
"database": config["database"],
|
|
271
|
+
"role": config["role_name"],
|
|
272
|
+
"master_statements": len(prepared),
|
|
273
|
+
"schema_statements": applied,
|
|
274
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|