eoapi-cdk 8.2.2__tar.gz → 8.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eoapi_cdk-8.2.2/src/eoapi_cdk.egg-info → eoapi_cdk-8.3.0}/PKG-INFO +1 -1
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/setup.py +2 -2
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk/__init__.py +461 -80
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk/_jsii/__init__.py +1 -1
- eoapi_cdk-8.3.0/src/eoapi_cdk/_jsii/eoapi-cdk@8.3.0.jsii.tgz +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0/src/eoapi_cdk.egg-info}/PKG-INFO +1 -1
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk.egg-info/SOURCES.txt +1 -1
- eoapi_cdk-8.2.2/src/eoapi_cdk/_jsii/eoapi-cdk@8.2.2.jsii.tgz +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/LICENSE +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/MANIFEST.in +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/README.md +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/pyproject.toml +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/setup.cfg +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk/py.typed +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk.egg-info/dependency_links.txt +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk.egg-info/requires.txt +0 -0
- {eoapi_cdk-8.2.2 → eoapi_cdk-8.3.0}/src/eoapi_cdk.egg-info/top_level.txt +0 -0
|
@@ -5,7 +5,7 @@ kwargs = json.loads(
|
|
|
5
5
|
"""
|
|
6
6
|
{
|
|
7
7
|
"name": "eoapi-cdk",
|
|
8
|
-
"version": "8.
|
|
8
|
+
"version": "8.3.0",
|
|
9
9
|
"description": "A set of constructs deploying pgSTAC with CDK",
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"url": "https://github.com/developmentseed/eoapi-cdk.git",
|
|
@@ -26,7 +26,7 @@ kwargs = json.loads(
|
|
|
26
26
|
],
|
|
27
27
|
"package_data": {
|
|
28
28
|
"eoapi_cdk._jsii": [
|
|
29
|
-
"eoapi-cdk@8.
|
|
29
|
+
"eoapi-cdk@8.3.0.jsii.tgz"
|
|
30
30
|
],
|
|
31
31
|
"eoapi_cdk": [
|
|
32
32
|
"py.typed"
|
|
@@ -2828,27 +2828,27 @@ class StacIngestorProps:
|
|
|
2828
2828
|
)
|
|
2829
2829
|
|
|
2830
2830
|
|
|
2831
|
-
class
|
|
2831
|
+
class StacLoader(
|
|
2832
2832
|
_constructs_77d1e7e8.Construct,
|
|
2833
2833
|
metaclass=jsii.JSIIMeta,
|
|
2834
|
-
jsii_type="eoapi-cdk.
|
|
2834
|
+
jsii_type="eoapi-cdk.StacLoader",
|
|
2835
2835
|
):
|
|
2836
|
-
'''(experimental) AWS CDK Construct for STAC
|
|
2836
|
+
'''(experimental) AWS CDK Construct for STAC Object Loading Infrastructure.
|
|
2837
2837
|
|
|
2838
|
-
The
|
|
2839
|
-
STAC (SpatioTemporal Asset Catalog)
|
|
2838
|
+
The StacLoader creates a serverless, event-driven system for loading
|
|
2839
|
+
STAC (SpatioTemporal Asset Catalog) objects into a PostgreSQL database with
|
|
2840
2840
|
the pgstac extension. This construct supports multiple ingestion pathways
|
|
2841
|
-
for flexible STAC
|
|
2841
|
+
for flexible STAC object loading.
|
|
2842
2842
|
|
|
2843
2843
|
|
|
2844
2844
|
Architecture Overview
|
|
2845
2845
|
|
|
2846
2846
|
This construct creates the following AWS resources:
|
|
2847
2847
|
|
|
2848
|
-
- **SNS Topic**: Entry point for STAC
|
|
2848
|
+
- **SNS Topic**: Entry point for STAC objects and S3 event notifications
|
|
2849
2849
|
- **SQS Queue**: Buffers and batches messages before processing (60-second visibility timeout)
|
|
2850
2850
|
- **Dead Letter Queue**: Captures failed loading attempts after 5 retries
|
|
2851
|
-
- **Lambda Function**: Python function that processes batches and inserts
|
|
2851
|
+
- **Lambda Function**: Python function that processes batches and inserts objects into pgstac
|
|
2852
2852
|
|
|
2853
2853
|
|
|
2854
2854
|
|
|
@@ -2857,20 +2857,20 @@ class StacItemLoader(
|
|
|
2857
2857
|
The loader supports two primary data ingestion patterns:
|
|
2858
2858
|
|
|
2859
2859
|
|
|
2860
|
-
Direct STAC
|
|
2860
|
+
Direct STAC Object Publishing
|
|
2861
2861
|
|
|
2862
|
-
1. STAC
|
|
2863
|
-
2. The SQS queue collects messages and batches them (up to {batchSize}
|
|
2864
|
-
3. The Lambda function receives batches, validates
|
|
2862
|
+
1. STAC objects (JSON) are published directly to the SNS topic in message bodies
|
|
2863
|
+
2. The SQS queue collects messages and batches them (up to {batchSize} objects or 1 minute window)
|
|
2864
|
+
3. The Lambda function receives batches, validates objects, and inserts into pgstac
|
|
2865
2865
|
|
|
2866
2866
|
|
|
2867
2867
|
|
|
2868
2868
|
S3 Event-Driven Loading
|
|
2869
2869
|
|
|
2870
2870
|
1. An S3 bucket is configured to send notifications to the SNS topic when json files are created
|
|
2871
|
-
2. STAC
|
|
2872
|
-
3. S3 event notifications are sent to the SNS topic when
|
|
2873
|
-
4. The Lambda function receives S3 events in the SQS message batch, fetches
|
|
2871
|
+
2. STAC objects are uploaded to S3 buckets as JSON/GeoJSON files
|
|
2872
|
+
3. S3 event notifications are sent to the SNS topic when objects are uploaded
|
|
2873
|
+
4. The Lambda function receives S3 events in the SQS message batch, fetches objects from S3, and loads into pgstac
|
|
2874
2874
|
|
|
2875
2875
|
|
|
2876
2876
|
|
|
@@ -2883,7 +2883,7 @@ class StacItemLoader(
|
|
|
2883
2883
|
triggers after ``maxBatchingWindow`` minutes (default: 1 minute)
|
|
2884
2884
|
- **Trigger Condition**: Lambda executes when EITHER condition is met first
|
|
2885
2885
|
- **Concurrency**: Limited to ``maxConcurrency`` concurrent executions to prevent database overload
|
|
2886
|
-
- **Partial Failures**: Uses ``reportBatchItemFailures`` to retry only failed
|
|
2886
|
+
- **Partial Failures**: Uses ``reportBatchItemFailures`` to retry only failed objects
|
|
2887
2887
|
|
|
2888
2888
|
This approach balances throughput (larger batches = fewer database connections)
|
|
2889
2889
|
with latency (time-based triggers prevent indefinite waiting).
|
|
@@ -2893,14 +2893,14 @@ class StacItemLoader(
|
|
|
2893
2893
|
|
|
2894
2894
|
Failed messages are sent to the dead letter queue after 5 processing attempts.
|
|
2895
2895
|
**Important**: This construct provides NO automated handling of dead letter queue
|
|
2896
|
-
messages - monitoring, inspection, and reprocessing of failed
|
|
2896
|
+
messages - monitoring, inspection, and reprocessing of failed objects is the
|
|
2897
2897
|
responsibility of the implementing application.
|
|
2898
2898
|
|
|
2899
2899
|
Consider implementing:
|
|
2900
2900
|
|
|
2901
2901
|
- CloudWatch alarms on dead letter queue depth
|
|
2902
2902
|
- Manual or automated reprocessing workflows
|
|
2903
|
-
- Logging and alerting for failed
|
|
2903
|
+
- Logging and alerting for failed objects
|
|
2904
2904
|
- Regular cleanup of old dead letter messages (14-day retention)
|
|
2905
2905
|
|
|
2906
2906
|
|
|
@@ -2919,7 +2919,7 @@ class StacItemLoader(
|
|
|
2919
2919
|
|
|
2920
2920
|
Before using this construct, ensure:
|
|
2921
2921
|
|
|
2922
|
-
- The pgstac database has collections loaded (
|
|
2922
|
+
- The pgstac database has collections loaded (objects require existing collection IDs)
|
|
2923
2923
|
- Database credentials are stored in AWS Secrets Manager
|
|
2924
2924
|
- The pgstac extension is properly installed and configured
|
|
2925
2925
|
|
|
@@ -2934,25 +2934,38 @@ class StacItemLoader(
|
|
|
2934
2934
|
pgstacVersion: '0.9.5'
|
|
2935
2935
|
});
|
|
2936
2936
|
|
|
2937
|
-
// Create
|
|
2938
|
-
const loader = new
|
|
2937
|
+
// Create Object loader
|
|
2938
|
+
const loader = new StacLoader(this, 'StacLoader', {
|
|
2939
2939
|
pgstacDb: database,
|
|
2940
|
-
batchSize: 1000, // Process up to 1000
|
|
2940
|
+
batchSize: 1000, // Process up to 1000 objects per batch
|
|
2941
2941
|
maxBatchingWindowMinutes: 1, // Wait max 1 minute to fill batch
|
|
2942
2942
|
lambdaTimeoutSeconds: 300 // Allow up to 300 seconds for database operations
|
|
2943
2943
|
});
|
|
2944
2944
|
|
|
2945
|
-
// The topic ARN can be used by other services to publish
|
|
2945
|
+
// The topic ARN can be used by other services to publish objects
|
|
2946
2946
|
new CfnOutput(this, 'LoaderTopicArn', {
|
|
2947
2947
|
value: loader.topic.topicArn
|
|
2948
2948
|
});
|
|
2949
2949
|
|
|
2950
2950
|
|
|
2951
|
-
Direct
|
|
2951
|
+
Direct Object Publishing
|
|
2952
|
+
|
|
2953
|
+
External services can publish STAC objects directly to the topic::
|
|
2952
2954
|
|
|
2953
|
-
|
|
2955
|
+
aws sns publish --topic-arn $STAC_LOAD_TOPIC --message '{
|
|
2956
|
+
"id": "example-collection",
|
|
2957
|
+
"type": "Collection",
|
|
2958
|
+
"title": "Example Collection",
|
|
2959
|
+
"description": "An example collection",
|
|
2960
|
+
"license": "proprietary",
|
|
2961
|
+
"extent": {
|
|
2962
|
+
"spatial": {"bbox": [[-180, -90, 180, 90]]},
|
|
2963
|
+
"temporal": {"interval": [[null, null]]},
|
|
2964
|
+
},
|
|
2965
|
+
"stac_version": "1.1.0",
|
|
2966
|
+
}'
|
|
2954
2967
|
|
|
2955
|
-
aws sns publish --topic-arn $
|
|
2968
|
+
aws sns publish --topic-arn $STAC_LOAD_TOPIC --message '{
|
|
2956
2969
|
"type": "Feature",
|
|
2957
2970
|
"stac_version": "1.0.0",
|
|
2958
2971
|
"id": "example-item",
|
|
@@ -2965,7 +2978,7 @@ class StacItemLoader(
|
|
|
2965
2978
|
S3 Event Configuration
|
|
2966
2979
|
|
|
2967
2980
|
To enable S3 event-driven loading, configure S3 bucket notifications to send
|
|
2968
|
-
events to the SNS topic when STAC
|
|
2981
|
+
events to the SNS topic when STAC objects (.json or .geojson files) are uploaded::
|
|
2969
2982
|
|
|
2970
2983
|
// Configure S3 bucket to send notifications to the loader topic
|
|
2971
2984
|
bucket.addEventNotification(
|
|
@@ -2980,19 +2993,19 @@ class StacItemLoader(
|
|
|
2980
2993
|
{ suffix: '.geojson' }
|
|
2981
2994
|
);
|
|
2982
2995
|
|
|
2983
|
-
When STAC
|
|
2996
|
+
When STAC objects are uploaded to the configured S3 bucket, the loader will:
|
|
2984
2997
|
|
|
2985
2998
|
1. Receive S3 event notifications via SNS
|
|
2986
|
-
2. Fetch the STAC
|
|
2987
|
-
3. Validate and load the
|
|
2999
|
+
2. Fetch the STAC JSON from S3
|
|
3000
|
+
3. Validate and load the objects into the pgstac database
|
|
2988
3001
|
|
|
2989
3002
|
|
|
2990
3003
|
|
|
2991
3004
|
Monitoring and Troubleshooting
|
|
2992
3005
|
|
|
2993
3006
|
- Monitor Lambda logs: ``/aws/lambda/{FunctionName}``
|
|
2994
|
-
- **Dead Letter Queue**: Check for failed
|
|
2995
|
-
- Use batch
|
|
3007
|
+
- **Dead Letter Queue**: Check for failed objects - **no automated handling provided**
|
|
3008
|
+
- Use batch objects failure reporting for partial batch processing
|
|
2996
3009
|
- CloudWatch metrics available for queue depth and Lambda performance
|
|
2997
3010
|
|
|
2998
3011
|
|
|
@@ -3036,25 +3049,25 @@ class StacItemLoader(
|
|
|
3036
3049
|
'''
|
|
3037
3050
|
:param scope: -
|
|
3038
3051
|
:param id: -
|
|
3039
|
-
:param pgstac_db: (experimental) The PgSTAC database instance to load
|
|
3040
|
-
:param batch_size: (experimental) SQS batch size for lambda event source. This determines the maximum number of STAC
|
|
3052
|
+
:param pgstac_db: (experimental) The PgSTAC database instance to load data into. This database must have the pgstac extension installed and be properly configured with collections before objects can be loaded. The loader will use AWS Secrets Manager to securely access database credentials.
|
|
3053
|
+
:param batch_size: (experimental) SQS batch size for lambda event source. This determines the maximum number of STAC objects that will be processed together in a single lambda invocation. Larger batch sizes improve database insertion efficiency but require more memory and longer processing time. **Batching Behavior**: SQS will wait to accumulate up to this many messages before triggering the Lambda, OR until the maxBatchingWindow timeout is reached, whichever comes first. This creates an efficient balance between throughput and latency. Default: 500
|
|
3041
3054
|
:param environment: (experimental) Additional environment variables for the lambda function. These will be merged with the default environment variables including PGSTAC_SECRET_ARN. Use this for custom configuration or debugging flags. If you want to enable the option to upload a boilerplate collection record in the event that the collection record does not yet exist for an item that is set to be loaded, set the variable ``"CREATE_COLLECTIONS_IF_MISSING": "TRUE"``.
|
|
3042
3055
|
:param lambda_function_options: (experimental) Can be used to override the default lambda function properties. Default: - defined in the construct.
|
|
3043
3056
|
:param lambda_runtime: (experimental) The lambda runtime to use for the item loading function. The function is implemented in Python and uses pypgstac for database operations. Ensure the runtime version is compatible with the pgstac version specified in the database configuration. Default: lambda.Runtime.PYTHON_3_11
|
|
3044
|
-
:param lambda_timeout_seconds: (experimental) The timeout for the item load lambda in seconds. This should accommodate the time needed to process up to ``batchSize``
|
|
3045
|
-
:param max_batching_window_minutes: (experimental) Maximum batching window in minutes. Even if the batch size isn't reached, the lambda will be triggered after this time period to ensure timely processing of
|
|
3046
|
-
:param max_concurrency: (experimental) Maximum concurrent executions for the
|
|
3047
|
-
:param memory_size: (experimental) Memory size for the lambda function in MB. Higher memory allocation may improve performance when processing large batches of STAC
|
|
3057
|
+
:param lambda_timeout_seconds: (experimental) The timeout for the item load lambda in seconds. This should accommodate the time needed to process up to ``batchSize`` objects and perform database insertions. The SQS visibility timeout will be set to this value plus 10 seconds. Default: 300
|
|
3058
|
+
:param max_batching_window_minutes: (experimental) Maximum batching window in minutes. Even if the batch size isn't reached, the lambda will be triggered after this time period to ensure timely processing of objects. This prevents objects from waiting indefinitely in low-volume scenarios. **Important**: This timeout works in conjunction with batchSize - SQS will trigger the Lambda when EITHER the batch size is reached OR this time window expires, ensuring objects are processed in a timely manner regardless of volume. Default: 1
|
|
3059
|
+
:param max_concurrency: (experimental) Maximum concurrent executions for the StacLoader Lambda function. This limit will be applied to the Lambda function and will control how many concurrent batches will be released from the SQS queue. Default: 2
|
|
3060
|
+
:param memory_size: (experimental) Memory size for the lambda function in MB. Higher memory allocation may improve performance when processing large batches of STAC objects, especially for memory-intensive database operations. Default: 1024
|
|
3048
3061
|
:param subnet_selection: (experimental) Subnet into which the lambda should be deployed.
|
|
3049
3062
|
:param vpc: (experimental) VPC into which the lambda should be deployed.
|
|
3050
3063
|
|
|
3051
3064
|
:stability: experimental
|
|
3052
3065
|
'''
|
|
3053
3066
|
if __debug__:
|
|
3054
|
-
type_hints = typing.get_type_hints(
|
|
3067
|
+
type_hints = typing.get_type_hints(_typecheckingstub__07e7381e15a2bb6d7b4a36ff70efaaf4a44e5bcce99a053eb5c48cb9dfca728d)
|
|
3055
3068
|
check_type(argname="argument scope", value=scope, expected_type=type_hints["scope"])
|
|
3056
3069
|
check_type(argname="argument id", value=id, expected_type=type_hints["id"])
|
|
3057
|
-
props =
|
|
3070
|
+
props = StacLoaderProps(
|
|
3058
3071
|
pgstac_db=pgstac_db,
|
|
3059
3072
|
batch_size=batch_size,
|
|
3060
3073
|
environment=environment,
|
|
@@ -3073,7 +3086,7 @@ class StacItemLoader(
|
|
|
3073
3086
|
@builtins.property
|
|
3074
3087
|
@jsii.member(jsii_name="deadLetterQueue")
|
|
3075
3088
|
def dead_letter_queue(self) -> _aws_cdk_aws_sqs_ceddda9d.Queue:
|
|
3076
|
-
'''(experimental) Dead letter queue for failed
|
|
3089
|
+
'''(experimental) Dead letter queue for failed objects loading attempts.
|
|
3077
3090
|
|
|
3078
3091
|
Messages that fail processing after 5 attempts are sent here
|
|
3079
3092
|
for inspection and potential replay. Retains messages for 14 days
|
|
@@ -3095,13 +3108,13 @@ class StacItemLoader(
|
|
|
3095
3108
|
@builtins.property
|
|
3096
3109
|
@jsii.member(jsii_name="lambdaFunction")
|
|
3097
3110
|
def lambda_function(self) -> _aws_cdk_aws_lambda_ceddda9d.Function:
|
|
3098
|
-
'''(experimental) The Lambda function that loads STAC
|
|
3111
|
+
'''(experimental) The Lambda function that loads STAC objects into the pgstac database.
|
|
3099
3112
|
|
|
3100
3113
|
This Python function receives batches of messages from SQS and processes
|
|
3101
3114
|
them based on their type:
|
|
3102
3115
|
|
|
3103
|
-
- Direct STAC
|
|
3104
|
-
- S3 events: Fetches STAC
|
|
3116
|
+
- Direct STAC objects: Validates and loads directly into pgstac
|
|
3117
|
+
- S3 events: Fetches STAC JSON from S3, validates, and loads into pgstac
|
|
3105
3118
|
|
|
3106
3119
|
The function connects to PostgreSQL using credentials from Secrets Manager
|
|
3107
3120
|
and uses pypgstac for efficient database operations.
|
|
@@ -3115,7 +3128,7 @@ class StacItemLoader(
|
|
|
3115
3128
|
def queue(self) -> _aws_cdk_aws_sqs_ceddda9d.Queue:
|
|
3116
3129
|
'''(experimental) The SQS queue that buffers messages before processing.
|
|
3117
3130
|
|
|
3118
|
-
This queue collects both direct STAC
|
|
3131
|
+
This queue collects both direct STAC objects from SNS and S3 event
|
|
3119
3132
|
notifications, batching them for efficient database operations.
|
|
3120
3133
|
Configured with a visibility timeout that accommodates Lambda
|
|
3121
3134
|
processing time plus buffer.
|
|
@@ -3127,12 +3140,12 @@ class StacItemLoader(
|
|
|
3127
3140
|
@builtins.property
|
|
3128
3141
|
@jsii.member(jsii_name="topic")
|
|
3129
3142
|
def topic(self) -> _aws_cdk_aws_sns_ceddda9d.Topic:
|
|
3130
|
-
'''(experimental) The SNS topic that receives STAC
|
|
3143
|
+
'''(experimental) The SNS topic that receives STAC objects and S3 event notifications for loading.
|
|
3131
3144
|
|
|
3132
3145
|
This topic serves as the entry point for two types of events:
|
|
3133
3146
|
|
|
3134
|
-
1. Direct STAC
|
|
3135
|
-
2. S3 event notifications when STAC
|
|
3147
|
+
1. Direct STAC JSON documents published by external services
|
|
3148
|
+
2. S3 event notifications when STAC objects are uploaded to configured buckets
|
|
3136
3149
|
|
|
3137
3150
|
The topic fans out to the SQS queue for batched processing.
|
|
3138
3151
|
|
|
@@ -3142,7 +3155,7 @@ class StacItemLoader(
|
|
|
3142
3155
|
|
|
3143
3156
|
|
|
3144
3157
|
@jsii.data_type(
|
|
3145
|
-
jsii_type="eoapi-cdk.
|
|
3158
|
+
jsii_type="eoapi-cdk.StacLoaderProps",
|
|
3146
3159
|
jsii_struct_bases=[],
|
|
3147
3160
|
name_mapping={
|
|
3148
3161
|
"pgstac_db": "pgstacDb",
|
|
@@ -3158,7 +3171,7 @@ class StacItemLoader(
|
|
|
3158
3171
|
"vpc": "vpc",
|
|
3159
3172
|
},
|
|
3160
3173
|
)
|
|
3161
|
-
class
|
|
3174
|
+
class StacLoaderProps:
|
|
3162
3175
|
def __init__(
|
|
3163
3176
|
self,
|
|
3164
3177
|
*,
|
|
@@ -3174,26 +3187,26 @@ class StacItemLoaderProps:
|
|
|
3174
3187
|
subnet_selection: typing.Optional[typing.Union[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection, typing.Dict[builtins.str, typing.Any]]] = None,
|
|
3175
3188
|
vpc: typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc] = None,
|
|
3176
3189
|
) -> None:
|
|
3177
|
-
'''(experimental) Configuration properties for the
|
|
3190
|
+
'''(experimental) Configuration properties for the StacLoader construct.
|
|
3178
3191
|
|
|
3179
|
-
The
|
|
3180
|
-
that loads STAC items into a pgstac database. This construct creates
|
|
3181
|
-
the infrastructure for receiving STAC
|
|
3192
|
+
The StacLoader is part of a two-phase serverless STAC ingestion pipeline
|
|
3193
|
+
that loads STAC collections and items into a pgstac database. This construct creates
|
|
3194
|
+
the infrastructure for receiving STAC objects from multiple sources:
|
|
3182
3195
|
|
|
3183
3196
|
1. SNS messages containing STAC metadata (direct ingestion)
|
|
3184
|
-
2. S3 event notifications for STAC
|
|
3197
|
+
2. S3 event notifications for STAC objects uploaded to S3 buckets
|
|
3185
3198
|
|
|
3186
|
-
|
|
3199
|
+
Objects from both sources are batched and inserted into PostgreSQL with the pgstac extension.
|
|
3187
3200
|
|
|
3188
|
-
:param pgstac_db: (experimental) The PgSTAC database instance to load
|
|
3189
|
-
:param batch_size: (experimental) SQS batch size for lambda event source. This determines the maximum number of STAC
|
|
3201
|
+
:param pgstac_db: (experimental) The PgSTAC database instance to load data into. This database must have the pgstac extension installed and be properly configured with collections before objects can be loaded. The loader will use AWS Secrets Manager to securely access database credentials.
|
|
3202
|
+
:param batch_size: (experimental) SQS batch size for lambda event source. This determines the maximum number of STAC objects that will be processed together in a single lambda invocation. Larger batch sizes improve database insertion efficiency but require more memory and longer processing time. **Batching Behavior**: SQS will wait to accumulate up to this many messages before triggering the Lambda, OR until the maxBatchingWindow timeout is reached, whichever comes first. This creates an efficient balance between throughput and latency. Default: 500
|
|
3190
3203
|
:param environment: (experimental) Additional environment variables for the lambda function. These will be merged with the default environment variables including PGSTAC_SECRET_ARN. Use this for custom configuration or debugging flags. If you want to enable the option to upload a boilerplate collection record in the event that the collection record does not yet exist for an item that is set to be loaded, set the variable ``"CREATE_COLLECTIONS_IF_MISSING": "TRUE"``.
|
|
3191
3204
|
:param lambda_function_options: (experimental) Can be used to override the default lambda function properties. Default: - defined in the construct.
|
|
3192
3205
|
:param lambda_runtime: (experimental) The lambda runtime to use for the item loading function. The function is implemented in Python and uses pypgstac for database operations. Ensure the runtime version is compatible with the pgstac version specified in the database configuration. Default: lambda.Runtime.PYTHON_3_11
|
|
3193
|
-
:param lambda_timeout_seconds: (experimental) The timeout for the item load lambda in seconds. This should accommodate the time needed to process up to ``batchSize``
|
|
3194
|
-
:param max_batching_window_minutes: (experimental) Maximum batching window in minutes. Even if the batch size isn't reached, the lambda will be triggered after this time period to ensure timely processing of
|
|
3195
|
-
:param max_concurrency: (experimental) Maximum concurrent executions for the
|
|
3196
|
-
:param memory_size: (experimental) Memory size for the lambda function in MB. Higher memory allocation may improve performance when processing large batches of STAC
|
|
3206
|
+
:param lambda_timeout_seconds: (experimental) The timeout for the item load lambda in seconds. This should accommodate the time needed to process up to ``batchSize`` objects and perform database insertions. The SQS visibility timeout will be set to this value plus 10 seconds. Default: 300
|
|
3207
|
+
:param max_batching_window_minutes: (experimental) Maximum batching window in minutes. Even if the batch size isn't reached, the lambda will be triggered after this time period to ensure timely processing of objects. This prevents objects from waiting indefinitely in low-volume scenarios. **Important**: This timeout works in conjunction with batchSize - SQS will trigger the Lambda when EITHER the batch size is reached OR this time window expires, ensuring objects are processed in a timely manner regardless of volume. Default: 1
|
|
3208
|
+
:param max_concurrency: (experimental) Maximum concurrent executions for the StacLoader Lambda function. This limit will be applied to the Lambda function and will control how many concurrent batches will be released from the SQS queue. Default: 2
|
|
3209
|
+
:param memory_size: (experimental) Memory size for the lambda function in MB. Higher memory allocation may improve performance when processing large batches of STAC objects, especially for memory-intensive database operations. Default: 1024
|
|
3197
3210
|
:param subnet_selection: (experimental) Subnet into which the lambda should be deployed.
|
|
3198
3211
|
:param vpc: (experimental) VPC into which the lambda should be deployed.
|
|
3199
3212
|
|
|
@@ -3201,7 +3214,7 @@ class StacItemLoaderProps:
|
|
|
3201
3214
|
|
|
3202
3215
|
Example::
|
|
3203
3216
|
|
|
3204
|
-
const loader = new
|
|
3217
|
+
const loader = new StacLoader(this, 'StacLoader', {
|
|
3205
3218
|
pgstacDb: database,
|
|
3206
3219
|
batchSize: 1000,
|
|
3207
3220
|
maxBatchingWindowMinutes: 1,
|
|
@@ -3211,7 +3224,7 @@ class StacItemLoaderProps:
|
|
|
3211
3224
|
if isinstance(subnet_selection, dict):
|
|
3212
3225
|
subnet_selection = _aws_cdk_aws_ec2_ceddda9d.SubnetSelection(**subnet_selection)
|
|
3213
3226
|
if __debug__:
|
|
3214
|
-
type_hints = typing.get_type_hints(
|
|
3227
|
+
type_hints = typing.get_type_hints(_typecheckingstub__f4ba6b725ddd9c1870cc4cb313a955d81ffb068eda41ee786161dacf35a03192)
|
|
3215
3228
|
check_type(argname="argument pgstac_db", value=pgstac_db, expected_type=type_hints["pgstac_db"])
|
|
3216
3229
|
check_type(argname="argument batch_size", value=batch_size, expected_type=type_hints["batch_size"])
|
|
3217
3230
|
check_type(argname="argument environment", value=environment, expected_type=type_hints["environment"])
|
|
@@ -3249,10 +3262,10 @@ class StacItemLoaderProps:
|
|
|
3249
3262
|
|
|
3250
3263
|
@builtins.property
|
|
3251
3264
|
def pgstac_db(self) -> PgStacDatabase:
|
|
3252
|
-
'''(experimental) The PgSTAC database instance to load
|
|
3265
|
+
'''(experimental) The PgSTAC database instance to load data into.
|
|
3253
3266
|
|
|
3254
3267
|
This database must have the pgstac extension installed and be properly
|
|
3255
|
-
configured with collections before
|
|
3268
|
+
configured with collections before objects can be loaded. The loader will
|
|
3256
3269
|
use AWS Secrets Manager to securely access database credentials.
|
|
3257
3270
|
|
|
3258
3271
|
:stability: experimental
|
|
@@ -3265,7 +3278,7 @@ class StacItemLoaderProps:
|
|
|
3265
3278
|
def batch_size(self) -> typing.Optional[jsii.Number]:
|
|
3266
3279
|
'''(experimental) SQS batch size for lambda event source.
|
|
3267
3280
|
|
|
3268
|
-
This determines the maximum number of STAC
|
|
3281
|
+
This determines the maximum number of STAC objects that will be
|
|
3269
3282
|
processed together in a single lambda invocation. Larger batch
|
|
3270
3283
|
sizes improve database insertion efficiency but require more
|
|
3271
3284
|
memory and longer processing time.
|
|
@@ -3331,7 +3344,7 @@ class StacItemLoaderProps:
|
|
|
3331
3344
|
'''(experimental) The timeout for the item load lambda in seconds.
|
|
3332
3345
|
|
|
3333
3346
|
This should accommodate the time needed to process up to ``batchSize``
|
|
3334
|
-
|
|
3347
|
+
objects and perform database insertions. The SQS visibility timeout
|
|
3335
3348
|
will be set to this value plus 10 seconds.
|
|
3336
3349
|
|
|
3337
3350
|
:default: 300
|
|
@@ -3346,12 +3359,12 @@ class StacItemLoaderProps:
|
|
|
3346
3359
|
'''(experimental) Maximum batching window in minutes.
|
|
3347
3360
|
|
|
3348
3361
|
Even if the batch size isn't reached, the lambda will be triggered
|
|
3349
|
-
after this time period to ensure timely processing of
|
|
3350
|
-
This prevents
|
|
3362
|
+
after this time period to ensure timely processing of objects.
|
|
3363
|
+
This prevents objects from waiting indefinitely in low-volume scenarios.
|
|
3351
3364
|
|
|
3352
3365
|
**Important**: This timeout works in conjunction with batchSize - SQS
|
|
3353
3366
|
will trigger the Lambda when EITHER the batch size is reached OR this
|
|
3354
|
-
time window expires, ensuring
|
|
3367
|
+
time window expires, ensuring objects are processed in a timely manner
|
|
3355
3368
|
regardless of volume.
|
|
3356
3369
|
|
|
3357
3370
|
:default: 1
|
|
@@ -3363,7 +3376,7 @@ class StacItemLoaderProps:
|
|
|
3363
3376
|
|
|
3364
3377
|
@builtins.property
|
|
3365
3378
|
def max_concurrency(self) -> typing.Optional[jsii.Number]:
|
|
3366
|
-
'''(experimental) Maximum concurrent executions for the
|
|
3379
|
+
'''(experimental) Maximum concurrent executions for the StacLoader Lambda function.
|
|
3367
3380
|
|
|
3368
3381
|
This limit will be applied to the Lambda function and will control how
|
|
3369
3382
|
many concurrent batches will be released from the SQS queue.
|
|
@@ -3380,7 +3393,7 @@ class StacItemLoaderProps:
|
|
|
3380
3393
|
'''(experimental) Memory size for the lambda function in MB.
|
|
3381
3394
|
|
|
3382
3395
|
Higher memory allocation may improve performance when processing
|
|
3383
|
-
large batches of STAC
|
|
3396
|
+
large batches of STAC objects, especially for memory-intensive
|
|
3384
3397
|
database operations.
|
|
3385
3398
|
|
|
3386
3399
|
:default: 1024
|
|
@@ -3417,7 +3430,7 @@ class StacItemLoaderProps:
|
|
|
3417
3430
|
return not (rhs == self)
|
|
3418
3431
|
|
|
3419
3432
|
def __repr__(self) -> str:
|
|
3420
|
-
return "
|
|
3433
|
+
return "StacLoaderProps(%s)" % ", ".join(
|
|
3421
3434
|
k + "=" + repr(v) for k, v in self._values.items()
|
|
3422
3435
|
)
|
|
3423
3436
|
|
|
@@ -3489,7 +3502,7 @@ class StactoolsItemGenerator(
|
|
|
3489
3502
|
Example::
|
|
3490
3503
|
|
|
3491
3504
|
// Create item loader first (or get existing topic ARN)
|
|
3492
|
-
const loader = new
|
|
3505
|
+
const loader = new StacLoader(this, 'ItemLoader', {
|
|
3493
3506
|
pgstacDb: database
|
|
3494
3507
|
});
|
|
3495
3508
|
|
|
@@ -3574,7 +3587,7 @@ class StactoolsItemGenerator(
|
|
|
3574
3587
|
'''
|
|
3575
3588
|
:param scope: -
|
|
3576
3589
|
:param id: -
|
|
3577
|
-
:param item_load_topic_arn: (experimental) ARN of the SNS topic to publish generated items to. This is typically the topic from a
|
|
3590
|
+
:param item_load_topic_arn: (experimental) ARN of the SNS topic to publish generated items to. This is typically the topic from a StacLoader construct. Generated STAC items will be published here for downstream processing and database insertion.
|
|
3578
3591
|
:param batch_size: (experimental) SQS batch size for lambda event source. This determines how many generation requests are processed together in a single lambda invocation. Unlike the loader, generation typically processes items individually, so smaller batch sizes are common. Default: 10
|
|
3579
3592
|
:param environment: (experimental) Additional environment variables for the lambda function. These will be merged with default environment variables including ITEM_LOAD_TOPIC_ARN and LOG_LEVEL. Use this for custom configuration or to pass credentials for external data sources.
|
|
3580
3593
|
:param lambda_function_options: (experimental) Can be used to override the default lambda function properties. Default: - defined in the construct.
|
|
@@ -3696,7 +3709,7 @@ class StactoolsItemGeneratorProps:
|
|
|
3696
3709
|
infrastructure for the first phase of the pipeline - processing metadata
|
|
3697
3710
|
about assets and transforming them into standardized STAC items.
|
|
3698
3711
|
|
|
3699
|
-
:param item_load_topic_arn: (experimental) ARN of the SNS topic to publish generated items to. This is typically the topic from a
|
|
3712
|
+
:param item_load_topic_arn: (experimental) ARN of the SNS topic to publish generated items to. This is typically the topic from a StacLoader construct. Generated STAC items will be published here for downstream processing and database insertion.
|
|
3700
3713
|
:param batch_size: (experimental) SQS batch size for lambda event source. This determines how many generation requests are processed together in a single lambda invocation. Unlike the loader, generation typically processes items individually, so smaller batch sizes are common. Default: 10
|
|
3701
3714
|
:param environment: (experimental) Additional environment variables for the lambda function. These will be merged with default environment variables including ITEM_LOAD_TOPIC_ARN and LOG_LEVEL. Use this for custom configuration or to pass credentials for external data sources.
|
|
3702
3715
|
:param lambda_function_options: (experimental) Can be used to override the default lambda function properties. Default: - defined in the construct.
|
|
@@ -3758,7 +3771,7 @@ class StactoolsItemGeneratorProps:
|
|
|
3758
3771
|
def item_load_topic_arn(self) -> builtins.str:
|
|
3759
3772
|
'''(experimental) ARN of the SNS topic to publish generated items to.
|
|
3760
3773
|
|
|
3761
|
-
This is typically the topic from a
|
|
3774
|
+
This is typically the topic from a StacLoader construct.
|
|
3762
3775
|
Generated STAC items will be published here for downstream
|
|
3763
3776
|
processing and database insertion.
|
|
3764
3777
|
|
|
@@ -4385,6 +4398,336 @@ class TitilerPgstacApiLambda(
|
|
|
4385
4398
|
jsii.set(self, "titilerPgstacLambdaFunction", value) # pyright: ignore[reportArgumentType]
|
|
4386
4399
|
|
|
4387
4400
|
|
|
4401
|
+
class StacItemLoader(
|
|
4402
|
+
StacLoader,
|
|
4403
|
+
metaclass=jsii.JSIIMeta,
|
|
4404
|
+
jsii_type="eoapi-cdk.StacItemLoader",
|
|
4405
|
+
):
|
|
4406
|
+
'''
|
|
4407
|
+
:deprecated: Use StacLoader instead. StacItemLoader will be removed in a future version.
|
|
4408
|
+
|
|
4409
|
+
:stability: deprecated
|
|
4410
|
+
'''
|
|
4411
|
+
|
|
4412
|
+
def __init__(
|
|
4413
|
+
self,
|
|
4414
|
+
scope: _constructs_77d1e7e8.Construct,
|
|
4415
|
+
id: builtins.str,
|
|
4416
|
+
*,
|
|
4417
|
+
pgstac_db: PgStacDatabase,
|
|
4418
|
+
batch_size: typing.Optional[jsii.Number] = None,
|
|
4419
|
+
environment: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None,
|
|
4420
|
+
lambda_function_options: typing.Any = None,
|
|
4421
|
+
lambda_runtime: typing.Optional[_aws_cdk_aws_lambda_ceddda9d.Runtime] = None,
|
|
4422
|
+
lambda_timeout_seconds: typing.Optional[jsii.Number] = None,
|
|
4423
|
+
max_batching_window_minutes: typing.Optional[jsii.Number] = None,
|
|
4424
|
+
max_concurrency: typing.Optional[jsii.Number] = None,
|
|
4425
|
+
memory_size: typing.Optional[jsii.Number] = None,
|
|
4426
|
+
subnet_selection: typing.Optional[typing.Union[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection, typing.Dict[builtins.str, typing.Any]]] = None,
|
|
4427
|
+
vpc: typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc] = None,
|
|
4428
|
+
) -> None:
|
|
4429
|
+
'''
|
|
4430
|
+
:param scope: -
|
|
4431
|
+
:param id: -
|
|
4432
|
+
:param pgstac_db: (experimental) The PgSTAC database instance to load data into. This database must have the pgstac extension installed and be properly configured with collections before objects can be loaded. The loader will use AWS Secrets Manager to securely access database credentials.
|
|
4433
|
+
:param batch_size: (experimental) SQS batch size for lambda event source. This determines the maximum number of STAC objects that will be processed together in a single lambda invocation. Larger batch sizes improve database insertion efficiency but require more memory and longer processing time. **Batching Behavior**: SQS will wait to accumulate up to this many messages before triggering the Lambda, OR until the maxBatchingWindow timeout is reached, whichever comes first. This creates an efficient balance between throughput and latency. Default: 500
|
|
4434
|
+
:param environment: (experimental) Additional environment variables for the lambda function. These will be merged with the default environment variables including PGSTAC_SECRET_ARN. Use this for custom configuration or debugging flags. If you want to enable the option to upload a boilerplate collection record in the event that the collection record does not yet exist for an item that is set to be loaded, set the variable ``"CREATE_COLLECTIONS_IF_MISSING": "TRUE"``.
|
|
4435
|
+
:param lambda_function_options: (experimental) Can be used to override the default lambda function properties. Default: - defined in the construct.
|
|
4436
|
+
:param lambda_runtime: (experimental) The lambda runtime to use for the item loading function. The function is implemented in Python and uses pypgstac for database operations. Ensure the runtime version is compatible with the pgstac version specified in the database configuration. Default: lambda.Runtime.PYTHON_3_11
|
|
4437
|
+
:param lambda_timeout_seconds: (experimental) The timeout for the item load lambda in seconds. This should accommodate the time needed to process up to ``batchSize`` objects and perform database insertions. The SQS visibility timeout will be set to this value plus 10 seconds. Default: 300
|
|
4438
|
+
:param max_batching_window_minutes: (experimental) Maximum batching window in minutes. Even if the batch size isn't reached, the lambda will be triggered after this time period to ensure timely processing of objects. This prevents objects from waiting indefinitely in low-volume scenarios. **Important**: This timeout works in conjunction with batchSize - SQS will trigger the Lambda when EITHER the batch size is reached OR this time window expires, ensuring objects are processed in a timely manner regardless of volume. Default: 1
|
|
4439
|
+
:param max_concurrency: (experimental) Maximum concurrent executions for the StacLoader Lambda function. This limit will be applied to the Lambda function and will control how many concurrent batches will be released from the SQS queue. Default: 2
|
|
4440
|
+
:param memory_size: (experimental) Memory size for the lambda function in MB. Higher memory allocation may improve performance when processing large batches of STAC objects, especially for memory-intensive database operations. Default: 1024
|
|
4441
|
+
:param subnet_selection: (experimental) Subnet into which the lambda should be deployed.
|
|
4442
|
+
:param vpc: (experimental) VPC into which the lambda should be deployed.
|
|
4443
|
+
|
|
4444
|
+
:stability: deprecated
|
|
4445
|
+
'''
|
|
4446
|
+
if __debug__:
|
|
4447
|
+
type_hints = typing.get_type_hints(_typecheckingstub__3993e1684b5582edef005457882c4cfe3c53062e1f47dc584d35a9cfa71c31e0)
|
|
4448
|
+
check_type(argname="argument scope", value=scope, expected_type=type_hints["scope"])
|
|
4449
|
+
check_type(argname="argument id", value=id, expected_type=type_hints["id"])
|
|
4450
|
+
props = StacLoaderProps(
|
|
4451
|
+
pgstac_db=pgstac_db,
|
|
4452
|
+
batch_size=batch_size,
|
|
4453
|
+
environment=environment,
|
|
4454
|
+
lambda_function_options=lambda_function_options,
|
|
4455
|
+
lambda_runtime=lambda_runtime,
|
|
4456
|
+
lambda_timeout_seconds=lambda_timeout_seconds,
|
|
4457
|
+
max_batching_window_minutes=max_batching_window_minutes,
|
|
4458
|
+
max_concurrency=max_concurrency,
|
|
4459
|
+
memory_size=memory_size,
|
|
4460
|
+
subnet_selection=subnet_selection,
|
|
4461
|
+
vpc=vpc,
|
|
4462
|
+
)
|
|
4463
|
+
|
|
4464
|
+
jsii.create(self.__class__, self, [scope, id, props])
|
|
4465
|
+
|
|
4466
|
+
|
|
4467
|
+
@jsii.data_type(
|
|
4468
|
+
jsii_type="eoapi-cdk.StacItemLoaderProps",
|
|
4469
|
+
jsii_struct_bases=[StacLoaderProps],
|
|
4470
|
+
name_mapping={
|
|
4471
|
+
"pgstac_db": "pgstacDb",
|
|
4472
|
+
"batch_size": "batchSize",
|
|
4473
|
+
"environment": "environment",
|
|
4474
|
+
"lambda_function_options": "lambdaFunctionOptions",
|
|
4475
|
+
"lambda_runtime": "lambdaRuntime",
|
|
4476
|
+
"lambda_timeout_seconds": "lambdaTimeoutSeconds",
|
|
4477
|
+
"max_batching_window_minutes": "maxBatchingWindowMinutes",
|
|
4478
|
+
"max_concurrency": "maxConcurrency",
|
|
4479
|
+
"memory_size": "memorySize",
|
|
4480
|
+
"subnet_selection": "subnetSelection",
|
|
4481
|
+
"vpc": "vpc",
|
|
4482
|
+
},
|
|
4483
|
+
)
|
|
4484
|
+
class StacItemLoaderProps(StacLoaderProps):
|
|
4485
|
+
def __init__(
|
|
4486
|
+
self,
|
|
4487
|
+
*,
|
|
4488
|
+
pgstac_db: PgStacDatabase,
|
|
4489
|
+
batch_size: typing.Optional[jsii.Number] = None,
|
|
4490
|
+
environment: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None,
|
|
4491
|
+
lambda_function_options: typing.Any = None,
|
|
4492
|
+
lambda_runtime: typing.Optional[_aws_cdk_aws_lambda_ceddda9d.Runtime] = None,
|
|
4493
|
+
lambda_timeout_seconds: typing.Optional[jsii.Number] = None,
|
|
4494
|
+
max_batching_window_minutes: typing.Optional[jsii.Number] = None,
|
|
4495
|
+
max_concurrency: typing.Optional[jsii.Number] = None,
|
|
4496
|
+
memory_size: typing.Optional[jsii.Number] = None,
|
|
4497
|
+
subnet_selection: typing.Optional[typing.Union[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection, typing.Dict[builtins.str, typing.Any]]] = None,
|
|
4498
|
+
vpc: typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc] = None,
|
|
4499
|
+
) -> None:
|
|
4500
|
+
'''
|
|
4501
|
+
:param pgstac_db: (experimental) The PgSTAC database instance to load data into. This database must have the pgstac extension installed and be properly configured with collections before objects can be loaded. The loader will use AWS Secrets Manager to securely access database credentials.
|
|
4502
|
+
:param batch_size: (experimental) SQS batch size for lambda event source. This determines the maximum number of STAC objects that will be processed together in a single lambda invocation. Larger batch sizes improve database insertion efficiency but require more memory and longer processing time. **Batching Behavior**: SQS will wait to accumulate up to this many messages before triggering the Lambda, OR until the maxBatchingWindow timeout is reached, whichever comes first. This creates an efficient balance between throughput and latency. Default: 500
|
|
4503
|
+
:param environment: (experimental) Additional environment variables for the lambda function. These will be merged with the default environment variables including PGSTAC_SECRET_ARN. Use this for custom configuration or debugging flags. If you want to enable the option to upload a boilerplate collection record in the event that the collection record does not yet exist for an item that is set to be loaded, set the variable ``"CREATE_COLLECTIONS_IF_MISSING": "TRUE"``.
|
|
4504
|
+
:param lambda_function_options: (experimental) Can be used to override the default lambda function properties. Default: - defined in the construct.
|
|
4505
|
+
:param lambda_runtime: (experimental) The lambda runtime to use for the item loading function. The function is implemented in Python and uses pypgstac for database operations. Ensure the runtime version is compatible with the pgstac version specified in the database configuration. Default: lambda.Runtime.PYTHON_3_11
|
|
4506
|
+
:param lambda_timeout_seconds: (experimental) The timeout for the item load lambda in seconds. This should accommodate the time needed to process up to ``batchSize`` objects and perform database insertions. The SQS visibility timeout will be set to this value plus 10 seconds. Default: 300
|
|
4507
|
+
:param max_batching_window_minutes: (experimental) Maximum batching window in minutes. Even if the batch size isn't reached, the lambda will be triggered after this time period to ensure timely processing of objects. This prevents objects from waiting indefinitely in low-volume scenarios. **Important**: This timeout works in conjunction with batchSize - SQS will trigger the Lambda when EITHER the batch size is reached OR this time window expires, ensuring objects are processed in a timely manner regardless of volume. Default: 1
|
|
4508
|
+
:param max_concurrency: (experimental) Maximum concurrent executions for the StacLoader Lambda function. This limit will be applied to the Lambda function and will control how many concurrent batches will be released from the SQS queue. Default: 2
|
|
4509
|
+
:param memory_size: (experimental) Memory size for the lambda function in MB. Higher memory allocation may improve performance when processing large batches of STAC objects, especially for memory-intensive database operations. Default: 1024
|
|
4510
|
+
:param subnet_selection: (experimental) Subnet into which the lambda should be deployed.
|
|
4511
|
+
:param vpc: (experimental) VPC into which the lambda should be deployed.
|
|
4512
|
+
|
|
4513
|
+
:deprecated: Use StacLoaderProps instead. StacItemLoaderProps will be removed in a future version.
|
|
4514
|
+
|
|
4515
|
+
:stability: deprecated
|
|
4516
|
+
'''
|
|
4517
|
+
if isinstance(subnet_selection, dict):
|
|
4518
|
+
subnet_selection = _aws_cdk_aws_ec2_ceddda9d.SubnetSelection(**subnet_selection)
|
|
4519
|
+
if __debug__:
|
|
4520
|
+
type_hints = typing.get_type_hints(_typecheckingstub__4bbe16b8ca579680ab8c75903813675de5c4f4e8877137194262298af5f8093d)
|
|
4521
|
+
check_type(argname="argument pgstac_db", value=pgstac_db, expected_type=type_hints["pgstac_db"])
|
|
4522
|
+
check_type(argname="argument batch_size", value=batch_size, expected_type=type_hints["batch_size"])
|
|
4523
|
+
check_type(argname="argument environment", value=environment, expected_type=type_hints["environment"])
|
|
4524
|
+
check_type(argname="argument lambda_function_options", value=lambda_function_options, expected_type=type_hints["lambda_function_options"])
|
|
4525
|
+
check_type(argname="argument lambda_runtime", value=lambda_runtime, expected_type=type_hints["lambda_runtime"])
|
|
4526
|
+
check_type(argname="argument lambda_timeout_seconds", value=lambda_timeout_seconds, expected_type=type_hints["lambda_timeout_seconds"])
|
|
4527
|
+
check_type(argname="argument max_batching_window_minutes", value=max_batching_window_minutes, expected_type=type_hints["max_batching_window_minutes"])
|
|
4528
|
+
check_type(argname="argument max_concurrency", value=max_concurrency, expected_type=type_hints["max_concurrency"])
|
|
4529
|
+
check_type(argname="argument memory_size", value=memory_size, expected_type=type_hints["memory_size"])
|
|
4530
|
+
check_type(argname="argument subnet_selection", value=subnet_selection, expected_type=type_hints["subnet_selection"])
|
|
4531
|
+
check_type(argname="argument vpc", value=vpc, expected_type=type_hints["vpc"])
|
|
4532
|
+
self._values: typing.Dict[builtins.str, typing.Any] = {
|
|
4533
|
+
"pgstac_db": pgstac_db,
|
|
4534
|
+
}
|
|
4535
|
+
if batch_size is not None:
|
|
4536
|
+
self._values["batch_size"] = batch_size
|
|
4537
|
+
if environment is not None:
|
|
4538
|
+
self._values["environment"] = environment
|
|
4539
|
+
if lambda_function_options is not None:
|
|
4540
|
+
self._values["lambda_function_options"] = lambda_function_options
|
|
4541
|
+
if lambda_runtime is not None:
|
|
4542
|
+
self._values["lambda_runtime"] = lambda_runtime
|
|
4543
|
+
if lambda_timeout_seconds is not None:
|
|
4544
|
+
self._values["lambda_timeout_seconds"] = lambda_timeout_seconds
|
|
4545
|
+
if max_batching_window_minutes is not None:
|
|
4546
|
+
self._values["max_batching_window_minutes"] = max_batching_window_minutes
|
|
4547
|
+
if max_concurrency is not None:
|
|
4548
|
+
self._values["max_concurrency"] = max_concurrency
|
|
4549
|
+
if memory_size is not None:
|
|
4550
|
+
self._values["memory_size"] = memory_size
|
|
4551
|
+
if subnet_selection is not None:
|
|
4552
|
+
self._values["subnet_selection"] = subnet_selection
|
|
4553
|
+
if vpc is not None:
|
|
4554
|
+
self._values["vpc"] = vpc
|
|
4555
|
+
|
|
4556
|
+
@builtins.property
|
|
4557
|
+
def pgstac_db(self) -> PgStacDatabase:
|
|
4558
|
+
'''(experimental) The PgSTAC database instance to load data into.
|
|
4559
|
+
|
|
4560
|
+
This database must have the pgstac extension installed and be properly
|
|
4561
|
+
configured with collections before objects can be loaded. The loader will
|
|
4562
|
+
use AWS Secrets Manager to securely access database credentials.
|
|
4563
|
+
|
|
4564
|
+
:stability: experimental
|
|
4565
|
+
'''
|
|
4566
|
+
result = self._values.get("pgstac_db")
|
|
4567
|
+
assert result is not None, "Required property 'pgstac_db' is missing"
|
|
4568
|
+
return typing.cast(PgStacDatabase, result)
|
|
4569
|
+
|
|
4570
|
+
@builtins.property
|
|
4571
|
+
def batch_size(self) -> typing.Optional[jsii.Number]:
|
|
4572
|
+
'''(experimental) SQS batch size for lambda event source.
|
|
4573
|
+
|
|
4574
|
+
This determines the maximum number of STAC objects that will be
|
|
4575
|
+
processed together in a single lambda invocation. Larger batch
|
|
4576
|
+
sizes improve database insertion efficiency but require more
|
|
4577
|
+
memory and longer processing time.
|
|
4578
|
+
|
|
4579
|
+
**Batching Behavior**: SQS will wait to accumulate up to this many
|
|
4580
|
+
messages before triggering the Lambda, OR until the maxBatchingWindow
|
|
4581
|
+
timeout is reached, whichever comes first. This creates an efficient
|
|
4582
|
+
balance between throughput and latency.
|
|
4583
|
+
|
|
4584
|
+
:default: 500
|
|
4585
|
+
|
|
4586
|
+
:stability: experimental
|
|
4587
|
+
'''
|
|
4588
|
+
result = self._values.get("batch_size")
|
|
4589
|
+
return typing.cast(typing.Optional[jsii.Number], result)
|
|
4590
|
+
|
|
4591
|
+
@builtins.property
|
|
4592
|
+
def environment(
|
|
4593
|
+
self,
|
|
4594
|
+
) -> typing.Optional[typing.Mapping[builtins.str, builtins.str]]:
|
|
4595
|
+
'''(experimental) Additional environment variables for the lambda function.
|
|
4596
|
+
|
|
4597
|
+
These will be merged with the default environment variables including
|
|
4598
|
+
PGSTAC_SECRET_ARN. Use this for custom configuration or debugging flags.
|
|
4599
|
+
|
|
4600
|
+
If you want to enable the option to upload a boilerplate collection record
|
|
4601
|
+
in the event that the collection record does not yet exist for an item that
|
|
4602
|
+
is set to be loaded, set the variable ``"CREATE_COLLECTIONS_IF_MISSING": "TRUE"``.
|
|
4603
|
+
|
|
4604
|
+
:stability: experimental
|
|
4605
|
+
'''
|
|
4606
|
+
result = self._values.get("environment")
|
|
4607
|
+
return typing.cast(typing.Optional[typing.Mapping[builtins.str, builtins.str]], result)
|
|
4608
|
+
|
|
4609
|
+
@builtins.property
|
|
4610
|
+
def lambda_function_options(self) -> typing.Any:
|
|
4611
|
+
'''(experimental) Can be used to override the default lambda function properties.
|
|
4612
|
+
|
|
4613
|
+
:default: - defined in the construct.
|
|
4614
|
+
|
|
4615
|
+
:stability: experimental
|
|
4616
|
+
'''
|
|
4617
|
+
result = self._values.get("lambda_function_options")
|
|
4618
|
+
return typing.cast(typing.Any, result)
|
|
4619
|
+
|
|
4620
|
+
@builtins.property
|
|
4621
|
+
def lambda_runtime(self) -> typing.Optional[_aws_cdk_aws_lambda_ceddda9d.Runtime]:
|
|
4622
|
+
'''(experimental) The lambda runtime to use for the item loading function.
|
|
4623
|
+
|
|
4624
|
+
The function is implemented in Python and uses pypgstac for database
|
|
4625
|
+
operations. Ensure the runtime version is compatible with the pgstac
|
|
4626
|
+
version specified in the database configuration.
|
|
4627
|
+
|
|
4628
|
+
:default: lambda.Runtime.PYTHON_3_11
|
|
4629
|
+
|
|
4630
|
+
:stability: experimental
|
|
4631
|
+
'''
|
|
4632
|
+
result = self._values.get("lambda_runtime")
|
|
4633
|
+
return typing.cast(typing.Optional[_aws_cdk_aws_lambda_ceddda9d.Runtime], result)
|
|
4634
|
+
|
|
4635
|
+
@builtins.property
|
|
4636
|
+
def lambda_timeout_seconds(self) -> typing.Optional[jsii.Number]:
|
|
4637
|
+
'''(experimental) The timeout for the item load lambda in seconds.
|
|
4638
|
+
|
|
4639
|
+
This should accommodate the time needed to process up to ``batchSize``
|
|
4640
|
+
objects and perform database insertions. The SQS visibility timeout
|
|
4641
|
+
will be set to this value plus 10 seconds.
|
|
4642
|
+
|
|
4643
|
+
:default: 300
|
|
4644
|
+
|
|
4645
|
+
:stability: experimental
|
|
4646
|
+
'''
|
|
4647
|
+
result = self._values.get("lambda_timeout_seconds")
|
|
4648
|
+
return typing.cast(typing.Optional[jsii.Number], result)
|
|
4649
|
+
|
|
4650
|
+
@builtins.property
|
|
4651
|
+
def max_batching_window_minutes(self) -> typing.Optional[jsii.Number]:
|
|
4652
|
+
'''(experimental) Maximum batching window in minutes.
|
|
4653
|
+
|
|
4654
|
+
Even if the batch size isn't reached, the lambda will be triggered
|
|
4655
|
+
after this time period to ensure timely processing of objects.
|
|
4656
|
+
This prevents objects from waiting indefinitely in low-volume scenarios.
|
|
4657
|
+
|
|
4658
|
+
**Important**: This timeout works in conjunction with batchSize - SQS
|
|
4659
|
+
will trigger the Lambda when EITHER the batch size is reached OR this
|
|
4660
|
+
time window expires, ensuring objects are processed in a timely manner
|
|
4661
|
+
regardless of volume.
|
|
4662
|
+
|
|
4663
|
+
:default: 1
|
|
4664
|
+
|
|
4665
|
+
:stability: experimental
|
|
4666
|
+
'''
|
|
4667
|
+
result = self._values.get("max_batching_window_minutes")
|
|
4668
|
+
return typing.cast(typing.Optional[jsii.Number], result)
|
|
4669
|
+
|
|
4670
|
+
@builtins.property
|
|
4671
|
+
def max_concurrency(self) -> typing.Optional[jsii.Number]:
|
|
4672
|
+
'''(experimental) Maximum concurrent executions for the StacLoader Lambda function.
|
|
4673
|
+
|
|
4674
|
+
This limit will be applied to the Lambda function and will control how
|
|
4675
|
+
many concurrent batches will be released from the SQS queue.
|
|
4676
|
+
|
|
4677
|
+
:default: 2
|
|
4678
|
+
|
|
4679
|
+
:stability: experimental
|
|
4680
|
+
'''
|
|
4681
|
+
result = self._values.get("max_concurrency")
|
|
4682
|
+
return typing.cast(typing.Optional[jsii.Number], result)
|
|
4683
|
+
|
|
4684
|
+
@builtins.property
|
|
4685
|
+
def memory_size(self) -> typing.Optional[jsii.Number]:
|
|
4686
|
+
'''(experimental) Memory size for the lambda function in MB.
|
|
4687
|
+
|
|
4688
|
+
Higher memory allocation may improve performance when processing
|
|
4689
|
+
large batches of STAC objects, especially for memory-intensive
|
|
4690
|
+
database operations.
|
|
4691
|
+
|
|
4692
|
+
:default: 1024
|
|
4693
|
+
|
|
4694
|
+
:stability: experimental
|
|
4695
|
+
'''
|
|
4696
|
+
result = self._values.get("memory_size")
|
|
4697
|
+
return typing.cast(typing.Optional[jsii.Number], result)
|
|
4698
|
+
|
|
4699
|
+
@builtins.property
|
|
4700
|
+
def subnet_selection(
|
|
4701
|
+
self,
|
|
4702
|
+
) -> typing.Optional[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection]:
|
|
4703
|
+
'''(experimental) Subnet into which the lambda should be deployed.
|
|
4704
|
+
|
|
4705
|
+
:stability: experimental
|
|
4706
|
+
'''
|
|
4707
|
+
result = self._values.get("subnet_selection")
|
|
4708
|
+
return typing.cast(typing.Optional[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection], result)
|
|
4709
|
+
|
|
4710
|
+
@builtins.property
|
|
4711
|
+
def vpc(self) -> typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc]:
|
|
4712
|
+
'''(experimental) VPC into which the lambda should be deployed.
|
|
4713
|
+
|
|
4714
|
+
:stability: experimental
|
|
4715
|
+
'''
|
|
4716
|
+
result = self._values.get("vpc")
|
|
4717
|
+
return typing.cast(typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc], result)
|
|
4718
|
+
|
|
4719
|
+
def __eq__(self, rhs: typing.Any) -> builtins.bool:
|
|
4720
|
+
return isinstance(rhs, self.__class__) and rhs._values == self._values
|
|
4721
|
+
|
|
4722
|
+
def __ne__(self, rhs: typing.Any) -> builtins.bool:
|
|
4723
|
+
return not (rhs == self)
|
|
4724
|
+
|
|
4725
|
+
def __repr__(self) -> str:
|
|
4726
|
+
return "StacItemLoaderProps(%s)" % ", ".join(
|
|
4727
|
+
k + "=" + repr(v) for k, v in self._values.items()
|
|
4728
|
+
)
|
|
4729
|
+
|
|
4730
|
+
|
|
4388
4731
|
__all__ = [
|
|
4389
4732
|
"BastionHost",
|
|
4390
4733
|
"BastionHostProps",
|
|
@@ -4399,6 +4742,8 @@ __all__ = [
|
|
|
4399
4742
|
"StacIngestorProps",
|
|
4400
4743
|
"StacItemLoader",
|
|
4401
4744
|
"StacItemLoaderProps",
|
|
4745
|
+
"StacLoader",
|
|
4746
|
+
"StacLoaderProps",
|
|
4402
4747
|
"StactoolsItemGenerator",
|
|
4403
4748
|
"StactoolsItemGeneratorProps",
|
|
4404
4749
|
"TiPgApiLambda",
|
|
@@ -4743,7 +5088,7 @@ def _typecheckingstub__e3a57d9ce730427c9464d66460707367708eb0bc9a9f69fbe2eb0cdbf
|
|
|
4743
5088
|
"""Type checking stubs"""
|
|
4744
5089
|
pass
|
|
4745
5090
|
|
|
4746
|
-
def
|
|
5091
|
+
def _typecheckingstub__07e7381e15a2bb6d7b4a36ff70efaaf4a44e5bcce99a053eb5c48cb9dfca728d(
|
|
4747
5092
|
scope: _constructs_77d1e7e8.Construct,
|
|
4748
5093
|
id: builtins.str,
|
|
4749
5094
|
*,
|
|
@@ -4762,7 +5107,7 @@ def _typecheckingstub__3993e1684b5582edef005457882c4cfe3c53062e1f47dc584d35a9cfa
|
|
|
4762
5107
|
"""Type checking stubs"""
|
|
4763
5108
|
pass
|
|
4764
5109
|
|
|
4765
|
-
def
|
|
5110
|
+
def _typecheckingstub__f4ba6b725ddd9c1870cc4cb313a955d81ffb068eda41ee786161dacf35a03192(
|
|
4766
5111
|
*,
|
|
4767
5112
|
pgstac_db: PgStacDatabase,
|
|
4768
5113
|
batch_size: typing.Optional[jsii.Number] = None,
|
|
@@ -4882,3 +5227,39 @@ def _typecheckingstub__1ae429080881692eda11326adbfaae8dacaec10c61b4af3a038e97d46
|
|
|
4882
5227
|
) -> None:
|
|
4883
5228
|
"""Type checking stubs"""
|
|
4884
5229
|
pass
|
|
5230
|
+
|
|
5231
|
+
def _typecheckingstub__3993e1684b5582edef005457882c4cfe3c53062e1f47dc584d35a9cfa71c31e0(
|
|
5232
|
+
scope: _constructs_77d1e7e8.Construct,
|
|
5233
|
+
id: builtins.str,
|
|
5234
|
+
*,
|
|
5235
|
+
pgstac_db: PgStacDatabase,
|
|
5236
|
+
batch_size: typing.Optional[jsii.Number] = None,
|
|
5237
|
+
environment: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None,
|
|
5238
|
+
lambda_function_options: typing.Any = None,
|
|
5239
|
+
lambda_runtime: typing.Optional[_aws_cdk_aws_lambda_ceddda9d.Runtime] = None,
|
|
5240
|
+
lambda_timeout_seconds: typing.Optional[jsii.Number] = None,
|
|
5241
|
+
max_batching_window_minutes: typing.Optional[jsii.Number] = None,
|
|
5242
|
+
max_concurrency: typing.Optional[jsii.Number] = None,
|
|
5243
|
+
memory_size: typing.Optional[jsii.Number] = None,
|
|
5244
|
+
subnet_selection: typing.Optional[typing.Union[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection, typing.Dict[builtins.str, typing.Any]]] = None,
|
|
5245
|
+
vpc: typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc] = None,
|
|
5246
|
+
) -> None:
|
|
5247
|
+
"""Type checking stubs"""
|
|
5248
|
+
pass
|
|
5249
|
+
|
|
5250
|
+
def _typecheckingstub__4bbe16b8ca579680ab8c75903813675de5c4f4e8877137194262298af5f8093d(
|
|
5251
|
+
*,
|
|
5252
|
+
pgstac_db: PgStacDatabase,
|
|
5253
|
+
batch_size: typing.Optional[jsii.Number] = None,
|
|
5254
|
+
environment: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None,
|
|
5255
|
+
lambda_function_options: typing.Any = None,
|
|
5256
|
+
lambda_runtime: typing.Optional[_aws_cdk_aws_lambda_ceddda9d.Runtime] = None,
|
|
5257
|
+
lambda_timeout_seconds: typing.Optional[jsii.Number] = None,
|
|
5258
|
+
max_batching_window_minutes: typing.Optional[jsii.Number] = None,
|
|
5259
|
+
max_concurrency: typing.Optional[jsii.Number] = None,
|
|
5260
|
+
memory_size: typing.Optional[jsii.Number] = None,
|
|
5261
|
+
subnet_selection: typing.Optional[typing.Union[_aws_cdk_aws_ec2_ceddda9d.SubnetSelection, typing.Dict[builtins.str, typing.Any]]] = None,
|
|
5262
|
+
vpc: typing.Optional[_aws_cdk_aws_ec2_ceddda9d.IVpc] = None,
|
|
5263
|
+
) -> None:
|
|
5264
|
+
"""Type checking stubs"""
|
|
5265
|
+
pass
|
|
Binary file
|
|
Binary file
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|