konokenj.cdk-api-mcp-server 0.49.0__py3-none-any.whl → 0.50.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of konokenj.cdk-api-mcp-server might be problematic. Click here for more details.
- cdk_api_mcp_server/__about__.py +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/aws-bedrock-agentcore-alpha/README.md +796 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/aws-eks-v2-alpha/README.md +116 -29
- cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/aws-msk-alpha/README.md +8 -8
- cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/mixins-preview/README.md +16 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigateway/integ.api-with-authorizer-and-proxy.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigateway/integ.lambda-api.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-authorizers/integ.iam.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-authorizers/integ.lambda.ts +2 -2
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-authorizers/integ.user-pool.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.add-subroute-integration.ts +7 -4
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.http-proxy.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.lambda-proxy.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.lambda.ts +4 -4
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-appsync/integ.graphql-lambda-permission.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-appsync/integ.js-resolver.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudformation/integ.core-custom-resources-node-18.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudformation/integ.core-custom-resources-service-timeout.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudfront-origins/integ.origin-response-completion-timeout.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudtrail/integ.cloudtrail-data-events-only.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecr/README.md +40 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecr/integ.tag-mutability-exclusion.ts +30 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs/README.md +3 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs/integ.managedinstances-capacity-provider.ts +5 -3
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs/integ.managedinstances-no-default-capacity-provider.ts +107 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs-patterns/integ.alb-fargate-service-public-private-switch.ts +45 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-elasticloadbalancingv2/integ.alb-lambda-multi-value-headers.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-elasticloadbalancingv2/integ.alb.oidc.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-elasticloadbalancingv2-actions/integ.cognito.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/README.md +96 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.record-format-conversion-schema.ts +154 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.record-format-conversion.ts +178 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.binary-payload.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.logging-config.ts +8 -8
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.params-and-secrets.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.runtime-management.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.runtime.inlinecode.ts +0 -7
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda-nodejs/integ.dependencies-pnpm.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda-nodejs/integ.function-exclude-smithy-models.ts +2 -2
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda-nodejs/integ.nodejs.build.images.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-logs/integ.subscriptionfilter.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-rds/integ.cluster-data-api-to-imported-cluster.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-rds/integ.cluster-data-api.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-s3-deployment/integ.bucket-deployment-data.ts +4 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-stepfunctions-tasks/README.md +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-stepfunctions-tasks/integ.call-aws-service-cross-region-lambda.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-stepfunctions-tasks/integ.invoke-json-path.ts +102 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-synthetics/integ.canary.ts +2 -0
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/custom-resources/integ.aws-custom-resource.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/custom-resources/integ.custom-resource-config-lambda-node-runtime.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/custom-resources/integ.invoke-function-payload.ts +1 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/cx-api/FEATURE_FLAGS.md +43 -1
- cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/cx-api/README.md +32 -1
- {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/METADATA +2 -2
- {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/RECORD +58 -50
- {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/WHEEL +0 -0
- {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/entry_points.txt +0 -0
- {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/licenses/LICENSE.txt +0 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import * as ec2 from 'aws-cdk-lib/aws-ec2';
|
|
2
|
+
import * as iam from 'aws-cdk-lib/aws-iam';
|
|
3
|
+
import * as cdk from 'aws-cdk-lib';
|
|
4
|
+
import * as ecs from 'aws-cdk-lib/aws-ecs';
|
|
5
|
+
import * as integ from '@aws-cdk/integ-tests-alpha';
|
|
6
|
+
|
|
7
|
+
const app = new cdk.App({
|
|
8
|
+
postCliContext: {
|
|
9
|
+
'@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm': true,
|
|
10
|
+
'@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature': false,
|
|
11
|
+
'@aws-cdk/aws-ecs:disableEcsImdsBlocking': false,
|
|
12
|
+
},
|
|
13
|
+
});
|
|
14
|
+
const stack = new cdk.Stack(app, 'integ-managedinstances-no-default-capacity-provider');
|
|
15
|
+
|
|
16
|
+
const vpc = new ec2.Vpc(stack, 'Vpc', { maxAzs: 2, restrictDefaultSecurityGroup: false });
|
|
17
|
+
const cluster = new ecs.Cluster(stack, 'ManagedInstancesCluster', {
|
|
18
|
+
vpc,
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
// Create IAM roles required for FMI following Omakase specifications
|
|
22
|
+
const infrastructureRole = new iam.Role(stack, 'InfrastructureRole', {
|
|
23
|
+
roleName: 'InfrastructureRole',
|
|
24
|
+
assumedBy: new iam.ServicePrincipal('ecs.amazonaws.com'),
|
|
25
|
+
managedPolicies: [
|
|
26
|
+
iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonECSInfrastructureRolePolicyForManagedInstances'),
|
|
27
|
+
],
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const instanceRole = new iam.Role(stack, 'InstanceRole', {
|
|
31
|
+
roleName: 'InstanceRole',
|
|
32
|
+
assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
|
|
33
|
+
managedPolicies: [
|
|
34
|
+
iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonECSInstanceRolePolicyForManagedInstances'),
|
|
35
|
+
],
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
infrastructureRole.grantPassRole(instanceRole);
|
|
39
|
+
|
|
40
|
+
const instanceProfile = new iam.InstanceProfile(stack, 'InstanceProfile', {
|
|
41
|
+
instanceProfileName: 'InstanceProfile',
|
|
42
|
+
role: instanceRole,
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// Create a security group for FMI instances
|
|
46
|
+
const fmiSecurityGroup = new ec2.SecurityGroup(stack, 'ManagedInstancesSecurityGroup', {
|
|
47
|
+
vpc,
|
|
48
|
+
description: 'Security group for ManagedInstances capacity provider instances',
|
|
49
|
+
allowAllOutbound: true,
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// Create MI Capacity Provider
|
|
53
|
+
const miCapacityProvider = new ecs.ManagedInstancesCapacityProvider(stack, 'ManagedInstancesCapacityProvider', {
|
|
54
|
+
infrastructureRole: infrastructureRole,
|
|
55
|
+
ec2InstanceProfile: instanceProfile,
|
|
56
|
+
subnets: vpc.privateSubnets,
|
|
57
|
+
securityGroups: [fmiSecurityGroup],
|
|
58
|
+
propagateTags: ecs.PropagateManagedInstancesTags.CAPACITY_PROVIDER,
|
|
59
|
+
instanceRequirements: {
|
|
60
|
+
vCpuCountMin: 1,
|
|
61
|
+
memoryMin: cdk.Size.gibibytes(2),
|
|
62
|
+
cpuManufacturers: [ec2.CpuManufacturer.INTEL],
|
|
63
|
+
acceleratorManufacturers: [ec2.AcceleratorManufacturer.NVIDIA],
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// Add FMI capacity provider to cluster
|
|
68
|
+
cluster.addManagedInstancesCapacityProvider(miCapacityProvider);
|
|
69
|
+
|
|
70
|
+
// Create a task definition compatible with Managed Instances and Fargate
|
|
71
|
+
const taskDefinition = new ecs.TaskDefinition(stack, 'TaskDef', {
|
|
72
|
+
compatibility: ecs.Compatibility.FARGATE_AND_MANAGED_INSTANCES,
|
|
73
|
+
cpu: '256',
|
|
74
|
+
memoryMiB: '512',
|
|
75
|
+
networkMode: ecs.NetworkMode.AWS_VPC,
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
taskDefinition.addContainer('web', {
|
|
79
|
+
image: ecs.ContainerImage.fromRegistry('public.ecr.aws/docker/library/httpd:2.4'),
|
|
80
|
+
memoryLimitMiB: 512,
|
|
81
|
+
portMappings: [
|
|
82
|
+
{
|
|
83
|
+
containerPort: 80,
|
|
84
|
+
protocol: ecs.Protocol.TCP,
|
|
85
|
+
},
|
|
86
|
+
],
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
// Create a service using the MI capacity provider
|
|
90
|
+
new ecs.FargateService(stack, 'ManagedInstancesService', {
|
|
91
|
+
cluster,
|
|
92
|
+
taskDefinition,
|
|
93
|
+
capacityProviderStrategies: [
|
|
94
|
+
{
|
|
95
|
+
capacityProvider: miCapacityProvider.capacityProviderName,
|
|
96
|
+
weight: 1,
|
|
97
|
+
},
|
|
98
|
+
],
|
|
99
|
+
desiredCount: 1,
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
new integ.IntegTest(app, 'ManagedInstancesCapacityProviders', {
|
|
103
|
+
testCases: [stack],
|
|
104
|
+
regions: ['us-west-2'],
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
app.synth();
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import * as ec2 from 'aws-cdk-lib/aws-ec2';
|
|
2
|
+
import * as ecs from 'aws-cdk-lib/aws-ecs';
|
|
3
|
+
import * as cdk from 'aws-cdk-lib';
|
|
4
|
+
import * as integ from '@aws-cdk/integ-tests-alpha';
|
|
5
|
+
import * as ecsPatterns from 'aws-cdk-lib/aws-ecs-patterns';
|
|
6
|
+
|
|
7
|
+
const app = new cdk.App();
|
|
8
|
+
const stack = new cdk.Stack(app, 'aws-ecs-integ-alb-fargate-public-private-switch');
|
|
9
|
+
|
|
10
|
+
const vpc = new ec2.Vpc(stack, 'Vpc', { maxAzs: 2, restrictDefaultSecurityGroup: false });
|
|
11
|
+
const cluster = new ecs.Cluster(stack, 'FargateCluster', { vpc });
|
|
12
|
+
|
|
13
|
+
// Test private load balancer (the problematic case from the issue)
|
|
14
|
+
new ecsPatterns.ApplicationLoadBalancedFargateService(stack, 'PrivateALBFargateService', {
|
|
15
|
+
cluster,
|
|
16
|
+
memoryLimitMiB: 1024,
|
|
17
|
+
cpu: 512,
|
|
18
|
+
publicLoadBalancer: false, // This should create ECSPrivate target group
|
|
19
|
+
taskImageOptions: {
|
|
20
|
+
image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'),
|
|
21
|
+
},
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
// Test public load balancer for comparison
|
|
25
|
+
new ecsPatterns.ApplicationLoadBalancedFargateService(stack, 'PublicALBFargateService', {
|
|
26
|
+
cluster,
|
|
27
|
+
memoryLimitMiB: 1024,
|
|
28
|
+
cpu: 512,
|
|
29
|
+
publicLoadBalancer: true, // This should create ECS target group
|
|
30
|
+
taskImageOptions: {
|
|
31
|
+
image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'),
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
new integ.IntegTest(app, 'ALBFargatePublicPrivateSwitchTest', {
|
|
36
|
+
testCases: [stack],
|
|
37
|
+
allowDestroy: [
|
|
38
|
+
'PrivateALBFargateServiceLB3F43693F',
|
|
39
|
+
'PrivateALBFargateServiceLBPublicListenerECSPrivateGroup81AA5B8B',
|
|
40
|
+
'PublicALBFargateServiceLBBDD839E7',
|
|
41
|
+
'PublicALBFargateServiceLBPublicListenerECSGroupD991EA00',
|
|
42
|
+
],
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
app.synth();
|
|
@@ -19,7 +19,7 @@ const vpc = new ec2.Vpc(stack, 'VPC', {
|
|
|
19
19
|
});
|
|
20
20
|
|
|
21
21
|
const fn = new lambda.Function(stack, 'Function', {
|
|
22
|
-
runtime: lambda.Runtime.
|
|
22
|
+
runtime: lambda.Runtime.NODEJS_20_X,
|
|
23
23
|
handler: 'index.handler',
|
|
24
24
|
code: lambda.Code.fromInline(`
|
|
25
25
|
exports.handler = async (event) => {
|
|
@@ -186,7 +186,7 @@ const signinFunction = new lambda.Function(testCase, 'Signin', {
|
|
|
186
186
|
functionName: 'cdk-integ-alb-oidc-signin-handler',
|
|
187
187
|
code: lambda.Code.fromAsset('alb-oidc-signin-handler', { exclude: ['*.ts'] }),
|
|
188
188
|
handler: 'index.handler',
|
|
189
|
-
runtime: lambda.Runtime.
|
|
189
|
+
runtime: lambda.Runtime.NODEJS_20_X,
|
|
190
190
|
environment: {
|
|
191
191
|
TEST_USERNAME: testUser.username,
|
|
192
192
|
TEST_PASSWORD: testUser.password,
|
|
@@ -206,7 +206,7 @@ const signinFunction = new lambda.Function(testCase, 'Signin', {
|
|
|
206
206
|
functionName: 'cdk-integ-alb-cognito-signin-handler',
|
|
207
207
|
code: lambda.Code.fromAsset('alb-cognito-signin-handler', { exclude: ['*.ts'] }),
|
|
208
208
|
handler: 'index.handler',
|
|
209
|
-
runtime: lambda.Runtime.
|
|
209
|
+
runtime: lambda.Runtime.NODEJS_20_X,
|
|
210
210
|
environment: {
|
|
211
211
|
TEST_USERNAME: testUser.username,
|
|
212
212
|
TEST_PASSWORD: testUser.password,
|
|
@@ -125,6 +125,102 @@ const s3Destination = new firehose.S3Bucket(bucket, {
|
|
|
125
125
|
});
|
|
126
126
|
```
|
|
127
127
|
|
|
128
|
+
## Data Format Conversion
|
|
129
|
+
|
|
130
|
+
Data format conversion allows automatic conversion of inputs from JSON to either Parquet or ORC.
|
|
131
|
+
Converting JSON records to columnar formats like Parquet or ORC can help speed up analytical querying while also increasing compression efficiency.
|
|
132
|
+
When data format conversion is specified, it automatically enables Snappy compression on the output.
|
|
133
|
+
|
|
134
|
+
Only S3 Destinations support data format conversion.
|
|
135
|
+
|
|
136
|
+
An example of defining an S3 destination configured with data format conversion:
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
declare const bucket: s3.Bucket;
|
|
140
|
+
declare const schemaGlueTable: glue.CfnTable;
|
|
141
|
+
const s3Destination = new firehose.S3Bucket(bucket, {
|
|
142
|
+
dataFormatConversion: {
|
|
143
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaGlueTable),
|
|
144
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
145
|
+
outputFormat: firehose.OutputFormat.PARQUET,
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
When data format conversion is enabled, the Delivery Stream's buffering size must be at least 64 MiB.
|
|
151
|
+
Additionally, the default buffering size is changed from 5 MiB to 128 MiB. This mirrors the Cloudformation behavior.
|
|
152
|
+
|
|
153
|
+
You can only parse JSON and transform it into either Parquet or ORC:
|
|
154
|
+
- to read JSON using OpenX parser, choose `InputFormat.OPENX_JSON`.
|
|
155
|
+
- to read JSON using Hive parser, choose `InputFormat.HIVE_JSON`.
|
|
156
|
+
- to transform into Parquet, choose `OutputFormat.PARQUET`.
|
|
157
|
+
- to transform into ORC, choose `OutputFormat.ORC`.
|
|
158
|
+
|
|
159
|
+
The following subsections explain how to specify advanced configuration options for each input and output format if the defaults are not desirable
|
|
160
|
+
|
|
161
|
+
### Input Format: OpenX JSON
|
|
162
|
+
|
|
163
|
+
Example creation of custom OpenX JSON InputFormat:
|
|
164
|
+
|
|
165
|
+
```ts
|
|
166
|
+
const inputFormat = new firehose.OpenXJsonInputFormat({
|
|
167
|
+
lowercaseColumnNames: false,
|
|
168
|
+
columnToJsonKeyMappings: {"ts": "timestamp"},
|
|
169
|
+
convertDotsInJsonKeysToUnderscores: true,
|
|
170
|
+
})
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### Input Format: Hive JSON
|
|
174
|
+
|
|
175
|
+
Example creation of custom Hive JSON InputFormat:
|
|
176
|
+
|
|
177
|
+
```ts
|
|
178
|
+
const inputFormat = new firehose.HiveJsonInputFormat({
|
|
179
|
+
timestampParsers: [
|
|
180
|
+
firehose.TimestampParser.fromFormatString('yyyy-MM-dd'),
|
|
181
|
+
firehose.TimestampParser.EPOCH_MILLIS,
|
|
182
|
+
]
|
|
183
|
+
})
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Hive JSON allows you to specify custom timestamp formats to parse. The syntax of the format string is Joda Time.
|
|
187
|
+
|
|
188
|
+
To parse timestamps formatted as milliseconds since epoch, use the convenience constant `TimestampParser.EPOCH_MILLIS`.
|
|
189
|
+
|
|
190
|
+
### Output Format: Parquet
|
|
191
|
+
|
|
192
|
+
Example of a custom Parquet OutputFormat, with all values changed from the defaults.
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
const outputFormat = new firehose.ParquetOutputFormat({
|
|
196
|
+
blockSize: Size.mebibytes(512),
|
|
197
|
+
compression: firehose.ParquetCompression.UNCOMPRESSED,
|
|
198
|
+
enableDictionaryCompression: true,
|
|
199
|
+
maxPadding: Size.bytes(10),
|
|
200
|
+
pageSize: Size.mebibytes(2),
|
|
201
|
+
writerVersion: firehose.ParquetWriterVersion.V2,
|
|
202
|
+
})
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Output Format: ORC
|
|
206
|
+
|
|
207
|
+
Example creation of custom ORC OutputFormat, with all values changed from the defaults.
|
|
208
|
+
|
|
209
|
+
```ts
|
|
210
|
+
const outputFormat = new firehose.OrcOutputFormat({
|
|
211
|
+
formatVersion: firehose.OrcFormatVersion.V0_11,
|
|
212
|
+
blockSize: Size.mebibytes(256),
|
|
213
|
+
compression: firehose.OrcCompression.NONE,
|
|
214
|
+
bloomFilterColumns: ['columnA'],
|
|
215
|
+
bloomFilterFalsePositiveProbability: 0.1,
|
|
216
|
+
dictionaryKeyThreshold: 0.7,
|
|
217
|
+
enablePadding: true,
|
|
218
|
+
paddingTolerance: 0.2,
|
|
219
|
+
rowIndexStride: 9000,
|
|
220
|
+
stripeSize: Size.mebibytes(32),
|
|
221
|
+
})
|
|
222
|
+
```
|
|
223
|
+
|
|
128
224
|
## Server-side Encryption
|
|
129
225
|
|
|
130
226
|
Enabling server-side encryption (SSE) requires Amazon Data Firehose to encrypt all data
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
|
|
2
|
+
import * as s3 from 'aws-cdk-lib/aws-s3';
|
|
3
|
+
import * as glue from 'aws-cdk-lib/aws-glue';
|
|
4
|
+
import * as cdk from 'aws-cdk-lib';
|
|
5
|
+
import * as integ from '@aws-cdk/integ-tests-alpha';
|
|
6
|
+
import { Construct } from 'constructs';
|
|
7
|
+
|
|
8
|
+
const app = new cdk.App();
|
|
9
|
+
|
|
10
|
+
const SCHEMA_COLUMNS = [
|
|
11
|
+
{
|
|
12
|
+
name: 'column_a',
|
|
13
|
+
type: 'string',
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
name: 'column_b',
|
|
17
|
+
type: 'string',
|
|
18
|
+
},
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
class TestStack extends cdk.Stack {
|
|
22
|
+
public readonly deliveryStreamsToTest: firehose.IDeliveryStream[];
|
|
23
|
+
public readonly bucket: s3.Bucket;
|
|
24
|
+
constructor(scope: Construct, id: string, props?: cdk.StackProps) {
|
|
25
|
+
super(scope, id, props);
|
|
26
|
+
|
|
27
|
+
this.bucket = new s3.Bucket(this, 'Bucket', {
|
|
28
|
+
removalPolicy: cdk.RemovalPolicy.DESTROY,
|
|
29
|
+
autoDeleteObjects: true,
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const database = new glue.CfnDatabase(this, 'Database', {
|
|
33
|
+
databaseInput: {
|
|
34
|
+
description: 'My database',
|
|
35
|
+
},
|
|
36
|
+
catalogId: this.account,
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const registry = new glue.CfnRegistry(this, 'SchemaRegistry', {
|
|
40
|
+
name: 'my_schema_registry',
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const inlineSchemaTable = this.createTableWithInlineSchema(database);
|
|
44
|
+
const inlineSchemaDeliveryStream = this.createDeliveryStreamWithDataFormatConversion('InlineSchema', {
|
|
45
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(inlineSchemaTable),
|
|
46
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
47
|
+
outputFormat: firehose.OutputFormat.PARQUET,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const registrySchemaTable = this.createTableWithRegistrySchema(database, registry);
|
|
51
|
+
const registrySchemaDeliveryStream = this.createDeliveryStreamWithDataFormatConversion('RegistrySchema', {
|
|
52
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(registrySchemaTable),
|
|
53
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
54
|
+
outputFormat: firehose.OutputFormat.PARQUET,
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
this.deliveryStreamsToTest = [
|
|
58
|
+
inlineSchemaDeliveryStream,
|
|
59
|
+
registrySchemaDeliveryStream,
|
|
60
|
+
];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private createTableWithInlineSchema(database: glue.CfnDatabase): glue.CfnTable {
|
|
64
|
+
return new glue.CfnTable(this, 'InlineSchemaTable', {
|
|
65
|
+
catalogId: database.catalogId,
|
|
66
|
+
databaseName: database.ref,
|
|
67
|
+
tableInput: {
|
|
68
|
+
storageDescriptor: {
|
|
69
|
+
columns: SCHEMA_COLUMNS,
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private createTableWithRegistrySchema(database: glue.CfnDatabase, registry: glue.CfnRegistry): glue.CfnTable {
|
|
76
|
+
const schemaDefinition = JSON.stringify({
|
|
77
|
+
type: 'record',
|
|
78
|
+
name: 'MyRecord',
|
|
79
|
+
fields: SCHEMA_COLUMNS,
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
const schema = new glue.CfnSchema(this, 'Schema', {
|
|
83
|
+
registry: {
|
|
84
|
+
arn: registry.attrArn,
|
|
85
|
+
},
|
|
86
|
+
compatibility: 'NONE',
|
|
87
|
+
dataFormat: 'AVRO',
|
|
88
|
+
name: 'my_schema',
|
|
89
|
+
schemaDefinition: schemaDefinition,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
return new glue.CfnTable(this, 'RegistrySchemaTable', {
|
|
93
|
+
catalogId: database.catalogId,
|
|
94
|
+
databaseName: database.ref,
|
|
95
|
+
tableInput: {
|
|
96
|
+
storageDescriptor: {
|
|
97
|
+
schemaReference: {
|
|
98
|
+
schemaVersionId: schema.attrInitialSchemaVersionId,
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
private createDeliveryStreamWithDataFormatConversion(
|
|
106
|
+
id: string,
|
|
107
|
+
dataFormatConversion: firehose.DataFormatConversionProps,
|
|
108
|
+
): firehose.DeliveryStream {
|
|
109
|
+
return new firehose.DeliveryStream(this, id, {
|
|
110
|
+
destination: new firehose.S3Bucket(this.bucket, {
|
|
111
|
+
dataOutputPrefix: `success/${id}/`,
|
|
112
|
+
errorOutputPrefix: `error/${id}/`,
|
|
113
|
+
bufferingInterval: cdk.Duration.seconds(0),
|
|
114
|
+
dataFormatConversion: dataFormatConversion,
|
|
115
|
+
}),
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const stack = new TestStack(app, 'RecordFormatConversionSchema');
|
|
121
|
+
const testCase = new integ.IntegTest(app, 'RecordFormatConversionSchemaTest', {
|
|
122
|
+
testCases: [stack],
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const assertions = testCase.assertions;
|
|
126
|
+
|
|
127
|
+
// Test each delivery stream with the same input, and verify that each writes the output to the success prefix
|
|
128
|
+
// Relies on waiting timeout to tell if record format conversion failed.
|
|
129
|
+
stack.deliveryStreamsToTest.forEach(deliveryStream => {
|
|
130
|
+
const putDataCall = assertions.awsApiCall('Firehose', 'putRecord', {
|
|
131
|
+
DeliveryStreamName: deliveryStream.deliveryStreamName,
|
|
132
|
+
Record: {
|
|
133
|
+
Data: JSON.stringify({
|
|
134
|
+
Column_A: 'foo',
|
|
135
|
+
Column_B: 'bar',
|
|
136
|
+
}),
|
|
137
|
+
},
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const waitForResultCall = assertions.awsApiCall('S3', 'listObjectsV2', {
|
|
141
|
+
Bucket: stack.bucket.bucketName,
|
|
142
|
+
Prefix: `success/${deliveryStream.node.id}/`,
|
|
143
|
+
}).expect(integ.ExpectedResult.objectLike({
|
|
144
|
+
KeyCount: 1,
|
|
145
|
+
})).waitForAssertions({
|
|
146
|
+
interval: cdk.Duration.seconds(5),
|
|
147
|
+
totalTimeout: cdk.Duration.minutes(2),
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
const api = waitForResultCall as integ.AwsApiCall;
|
|
151
|
+
api.waiterProvider?.addPolicyStatementFromSdkCall('s3', 'ListBucket', [stack.bucket.bucketArn]);
|
|
152
|
+
api.waiterProvider?.addPolicyStatementFromSdkCall('s3', 'GetObject', [stack.bucket.arnForObjects('*')]);
|
|
153
|
+
putDataCall.next(waitForResultCall);
|
|
154
|
+
});
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
|
|
2
|
+
import * as s3 from 'aws-cdk-lib/aws-s3';
|
|
3
|
+
import * as glue from 'aws-cdk-lib/aws-glue';
|
|
4
|
+
import * as cdk from 'aws-cdk-lib';
|
|
5
|
+
import * as integ from '@aws-cdk/integ-tests-alpha';
|
|
6
|
+
import { Construct } from 'constructs';
|
|
7
|
+
|
|
8
|
+
const app = new cdk.App();
|
|
9
|
+
|
|
10
|
+
const SCHEMA_COLUMNS = [
|
|
11
|
+
{
|
|
12
|
+
name: 'column_a',
|
|
13
|
+
type: 'string',
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
name: 'column_b',
|
|
17
|
+
type: 'string',
|
|
18
|
+
},
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
class TestStack extends cdk.Stack {
|
|
22
|
+
public readonly bucket: s3.Bucket;
|
|
23
|
+
constructor(scope: Construct, id: string, props?: cdk.StackProps) {
|
|
24
|
+
super(scope, id, props);
|
|
25
|
+
|
|
26
|
+
this.bucket = new s3.Bucket(this, 'Bucket', {
|
|
27
|
+
removalPolicy: cdk.RemovalPolicy.DESTROY,
|
|
28
|
+
autoDeleteObjects: true,
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
const database = new glue.CfnDatabase(this, 'Database', {
|
|
32
|
+
databaseInput: {
|
|
33
|
+
description: 'My database',
|
|
34
|
+
},
|
|
35
|
+
catalogId: this.account,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const schemaTable = this.createTableWithInlineSchema(database);
|
|
39
|
+
|
|
40
|
+
// default hive json input with default orc output
|
|
41
|
+
this.createDeliveryStreamWithDataFormatConversion('DefaultHiveJsonOrc', {
|
|
42
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
43
|
+
inputFormat: firehose.InputFormat.HIVE_JSON,
|
|
44
|
+
outputFormat: firehose.OutputFormat.ORC,
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
// default openx json input with default parquet output
|
|
48
|
+
this.createDeliveryStreamWithDataFormatConversion('DefaultOpenXJsonParquet', {
|
|
49
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
50
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
51
|
+
outputFormat: firehose.OutputFormat.PARQUET,
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
// custom hive json input
|
|
55
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomHiveJson', {
|
|
56
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
57
|
+
inputFormat: new firehose.HiveJsonInputFormat({
|
|
58
|
+
timestampParsers: [
|
|
59
|
+
firehose.TimestampParser.EPOCH_MILLIS,
|
|
60
|
+
firehose.TimestampParser.fromFormatString('yyyy-MM-dd'),
|
|
61
|
+
],
|
|
62
|
+
}),
|
|
63
|
+
outputFormat: firehose.OutputFormat.ORC,
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// custom openx json input
|
|
67
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomOpenXJson', {
|
|
68
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
69
|
+
inputFormat: new firehose.OpenXJsonInputFormat({
|
|
70
|
+
lowercaseColumnNames: false,
|
|
71
|
+
columnToJsonKeyMappings: { column_yay: 'Column_A' },
|
|
72
|
+
convertDotsInJsonKeysToUnderscores: true,
|
|
73
|
+
}),
|
|
74
|
+
outputFormat: firehose.OutputFormat.PARQUET,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
// custom orc output
|
|
78
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomOrc', {
|
|
79
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
80
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
81
|
+
outputFormat: new firehose.OrcOutputFormat({
|
|
82
|
+
blockSize: cdk.Size.mebibytes(256),
|
|
83
|
+
bloomFilterColumns: ['column_a'],
|
|
84
|
+
bloomFilterFalsePositiveProbability: 0.5,
|
|
85
|
+
compression: firehose.OrcCompression.NONE,
|
|
86
|
+
dictionaryKeyThreshold: 0.3,
|
|
87
|
+
formatVersion: firehose.OrcFormatVersion.V0_11,
|
|
88
|
+
enablePadding: true,
|
|
89
|
+
paddingTolerance: 0.4,
|
|
90
|
+
rowIndexStride: 5000,
|
|
91
|
+
stripeSize: cdk.Size.mebibytes(32),
|
|
92
|
+
}),
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// ORC ZLIB compression
|
|
96
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomOrcZlib', {
|
|
97
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
98
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
99
|
+
outputFormat: new firehose.OrcOutputFormat({
|
|
100
|
+
compression: firehose.OrcCompression.ZLIB,
|
|
101
|
+
formatVersion: firehose.OrcFormatVersion.V0_12,
|
|
102
|
+
}),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// ORC SNAPPY compression
|
|
106
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomOrcSnappy', {
|
|
107
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
108
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
109
|
+
outputFormat: new firehose.OrcOutputFormat({
|
|
110
|
+
compression: firehose.OrcCompression.SNAPPY,
|
|
111
|
+
}),
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// custom parquet output format
|
|
115
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomParquet', {
|
|
116
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
117
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
118
|
+
outputFormat: new firehose.ParquetOutputFormat({
|
|
119
|
+
blockSize: cdk.Size.mebibytes(128),
|
|
120
|
+
pageSize: cdk.Size.mebibytes(2),
|
|
121
|
+
compression: firehose.ParquetCompression.UNCOMPRESSED,
|
|
122
|
+
writerVersion: firehose.ParquetWriterVersion.V2,
|
|
123
|
+
enableDictionaryCompression: true,
|
|
124
|
+
maxPadding: cdk.Size.bytes(100),
|
|
125
|
+
}),
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
// Parquet GZIP compression
|
|
129
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomParquetGzip', {
|
|
130
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
131
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
132
|
+
outputFormat: new firehose.ParquetOutputFormat({
|
|
133
|
+
compression: firehose.ParquetCompression.GZIP,
|
|
134
|
+
writerVersion: firehose.ParquetWriterVersion.V1,
|
|
135
|
+
}),
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
// Parquet SNAPPY compression
|
|
139
|
+
this.createDeliveryStreamWithDataFormatConversion('CustomParquetSnappy', {
|
|
140
|
+
schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
|
|
141
|
+
inputFormat: firehose.InputFormat.OPENX_JSON,
|
|
142
|
+
outputFormat: new firehose.ParquetOutputFormat({
|
|
143
|
+
compression: firehose.ParquetCompression.SNAPPY,
|
|
144
|
+
}),
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private createTableWithInlineSchema(database: glue.CfnDatabase): glue.CfnTable {
|
|
149
|
+
return new glue.CfnTable(this, 'InlineSchemaTable', {
|
|
150
|
+
catalogId: database.catalogId,
|
|
151
|
+
databaseName: database.ref,
|
|
152
|
+
tableInput: {
|
|
153
|
+
storageDescriptor: {
|
|
154
|
+
columns: SCHEMA_COLUMNS,
|
|
155
|
+
},
|
|
156
|
+
},
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
private createDeliveryStreamWithDataFormatConversion(
|
|
161
|
+
id: string,
|
|
162
|
+
dataFormatConversion: firehose.DataFormatConversionProps,
|
|
163
|
+
): firehose.DeliveryStream {
|
|
164
|
+
return new firehose.DeliveryStream(this, id, {
|
|
165
|
+
destination: new firehose.S3Bucket(this.bucket, {
|
|
166
|
+
dataOutputPrefix: `success/${id}/`,
|
|
167
|
+
errorOutputPrefix: `error/${id}/`,
|
|
168
|
+
bufferingInterval: cdk.Duration.seconds(0),
|
|
169
|
+
dataFormatConversion: dataFormatConversion,
|
|
170
|
+
}),
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const stack = new TestStack(app, 'RecordFormatConversion');
|
|
176
|
+
new integ.IntegTest(app, 'RecordFormatConversionTest', {
|
|
177
|
+
testCases: [stack],
|
|
178
|
+
});
|
cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.binary-payload.ts
CHANGED
|
@@ -10,7 +10,7 @@ const app = new App({
|
|
|
10
10
|
const stack = new Stack(app, 'IntegBinaryPayload');
|
|
11
11
|
|
|
12
12
|
const fn = new Function(stack, 'fn', {
|
|
13
|
-
runtime: Runtime.
|
|
13
|
+
runtime: Runtime.NODEJS_20_X,
|
|
14
14
|
handler: 'index.handler',
|
|
15
15
|
code: Code.fromInline(`
|
|
16
16
|
exports.handler = async (event) => {
|