konokenj.cdk-api-mcp-server 0.49.0__py3-none-any.whl → 0.50.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of konokenj.cdk-api-mcp-server might be problematic. Click here for more details.

Files changed (58) hide show
  1. cdk_api_mcp_server/__about__.py +1 -1
  2. cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/aws-bedrock-agentcore-alpha/README.md +796 -0
  3. cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/aws-eks-v2-alpha/README.md +116 -29
  4. cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/aws-msk-alpha/README.md +8 -8
  5. cdk_api_mcp_server/resources/aws-cdk/constructs/@aws-cdk/mixins-preview/README.md +16 -0
  6. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigateway/integ.api-with-authorizer-and-proxy.ts +1 -1
  7. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigateway/integ.lambda-api.ts +1 -1
  8. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-authorizers/integ.iam.ts +1 -1
  9. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-authorizers/integ.lambda.ts +2 -2
  10. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-authorizers/integ.user-pool.ts +1 -1
  11. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.add-subroute-integration.ts +7 -4
  12. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.http-proxy.ts +1 -1
  13. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.lambda-proxy.ts +1 -1
  14. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-apigatewayv2-integrations/integ.lambda.ts +4 -4
  15. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-appsync/integ.graphql-lambda-permission.ts +1 -1
  16. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-appsync/integ.js-resolver.ts +1 -1
  17. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudformation/integ.core-custom-resources-node-18.ts +1 -1
  18. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudformation/integ.core-custom-resources-service-timeout.ts +1 -1
  19. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudfront-origins/integ.origin-response-completion-timeout.ts +1 -1
  20. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-cloudtrail/integ.cloudtrail-data-events-only.ts +1 -1
  21. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecr/README.md +40 -1
  22. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecr/integ.tag-mutability-exclusion.ts +30 -0
  23. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs/README.md +3 -0
  24. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs/integ.managedinstances-capacity-provider.ts +5 -3
  25. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs/integ.managedinstances-no-default-capacity-provider.ts +107 -0
  26. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-ecs-patterns/integ.alb-fargate-service-public-private-switch.ts +45 -0
  27. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-elasticloadbalancingv2/integ.alb-lambda-multi-value-headers.ts +1 -1
  28. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-elasticloadbalancingv2/integ.alb.oidc.ts +1 -1
  29. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-elasticloadbalancingv2-actions/integ.cognito.ts +1 -1
  30. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/README.md +96 -0
  31. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.record-format-conversion-schema.ts +154 -0
  32. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.record-format-conversion.ts +178 -0
  33. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.binary-payload.ts +1 -1
  34. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.logging-config.ts +8 -8
  35. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.params-and-secrets.ts +1 -1
  36. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.runtime-management.ts +1 -1
  37. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda/integ.runtime.inlinecode.ts +0 -7
  38. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda-nodejs/integ.dependencies-pnpm.ts +1 -1
  39. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda-nodejs/integ.function-exclude-smithy-models.ts +2 -2
  40. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-lambda-nodejs/integ.nodejs.build.images.ts +1 -1
  41. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-logs/integ.subscriptionfilter.ts +1 -1
  42. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-rds/integ.cluster-data-api-to-imported-cluster.ts +1 -1
  43. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-rds/integ.cluster-data-api.ts +1 -1
  44. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-s3-deployment/integ.bucket-deployment-data.ts +4 -0
  45. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-stepfunctions-tasks/README.md +1 -1
  46. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-stepfunctions-tasks/integ.call-aws-service-cross-region-lambda.ts +1 -1
  47. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-stepfunctions-tasks/integ.invoke-json-path.ts +102 -0
  48. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-synthetics/integ.canary.ts +2 -0
  49. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/custom-resources/integ.aws-custom-resource.ts +1 -1
  50. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/custom-resources/integ.custom-resource-config-lambda-node-runtime.ts +1 -1
  51. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/custom-resources/integ.invoke-function-payload.ts +1 -1
  52. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/cx-api/FEATURE_FLAGS.md +43 -1
  53. cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/cx-api/README.md +32 -1
  54. {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/METADATA +2 -2
  55. {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/RECORD +58 -50
  56. {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/WHEEL +0 -0
  57. {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/entry_points.txt +0 -0
  58. {konokenj_cdk_api_mcp_server-0.49.0.dist-info → konokenj_cdk_api_mcp_server-0.50.0.dist-info}/licenses/LICENSE.txt +0 -0
@@ -0,0 +1,107 @@
1
+ import * as ec2 from 'aws-cdk-lib/aws-ec2';
2
+ import * as iam from 'aws-cdk-lib/aws-iam';
3
+ import * as cdk from 'aws-cdk-lib';
4
+ import * as ecs from 'aws-cdk-lib/aws-ecs';
5
+ import * as integ from '@aws-cdk/integ-tests-alpha';
6
+
7
+ const app = new cdk.App({
8
+ postCliContext: {
9
+ '@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm': true,
10
+ '@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature': false,
11
+ '@aws-cdk/aws-ecs:disableEcsImdsBlocking': false,
12
+ },
13
+ });
14
+ const stack = new cdk.Stack(app, 'integ-managedinstances-no-default-capacity-provider');
15
+
16
+ const vpc = new ec2.Vpc(stack, 'Vpc', { maxAzs: 2, restrictDefaultSecurityGroup: false });
17
+ const cluster = new ecs.Cluster(stack, 'ManagedInstancesCluster', {
18
+ vpc,
19
+ });
20
+
21
+ // Create IAM roles required for FMI following Omakase specifications
22
+ const infrastructureRole = new iam.Role(stack, 'InfrastructureRole', {
23
+ roleName: 'InfrastructureRole',
24
+ assumedBy: new iam.ServicePrincipal('ecs.amazonaws.com'),
25
+ managedPolicies: [
26
+ iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonECSInfrastructureRolePolicyForManagedInstances'),
27
+ ],
28
+ });
29
+
30
+ const instanceRole = new iam.Role(stack, 'InstanceRole', {
31
+ roleName: 'InstanceRole',
32
+ assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
33
+ managedPolicies: [
34
+ iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonECSInstanceRolePolicyForManagedInstances'),
35
+ ],
36
+ });
37
+
38
+ infrastructureRole.grantPassRole(instanceRole);
39
+
40
+ const instanceProfile = new iam.InstanceProfile(stack, 'InstanceProfile', {
41
+ instanceProfileName: 'InstanceProfile',
42
+ role: instanceRole,
43
+ });
44
+
45
+ // Create a security group for FMI instances
46
+ const fmiSecurityGroup = new ec2.SecurityGroup(stack, 'ManagedInstancesSecurityGroup', {
47
+ vpc,
48
+ description: 'Security group for ManagedInstances capacity provider instances',
49
+ allowAllOutbound: true,
50
+ });
51
+
52
+ // Create MI Capacity Provider
53
+ const miCapacityProvider = new ecs.ManagedInstancesCapacityProvider(stack, 'ManagedInstancesCapacityProvider', {
54
+ infrastructureRole: infrastructureRole,
55
+ ec2InstanceProfile: instanceProfile,
56
+ subnets: vpc.privateSubnets,
57
+ securityGroups: [fmiSecurityGroup],
58
+ propagateTags: ecs.PropagateManagedInstancesTags.CAPACITY_PROVIDER,
59
+ instanceRequirements: {
60
+ vCpuCountMin: 1,
61
+ memoryMin: cdk.Size.gibibytes(2),
62
+ cpuManufacturers: [ec2.CpuManufacturer.INTEL],
63
+ acceleratorManufacturers: [ec2.AcceleratorManufacturer.NVIDIA],
64
+ },
65
+ });
66
+
67
+ // Add FMI capacity provider to cluster
68
+ cluster.addManagedInstancesCapacityProvider(miCapacityProvider);
69
+
70
+ // Create a task definition compatible with Managed Instances and Fargate
71
+ const taskDefinition = new ecs.TaskDefinition(stack, 'TaskDef', {
72
+ compatibility: ecs.Compatibility.FARGATE_AND_MANAGED_INSTANCES,
73
+ cpu: '256',
74
+ memoryMiB: '512',
75
+ networkMode: ecs.NetworkMode.AWS_VPC,
76
+ });
77
+
78
+ taskDefinition.addContainer('web', {
79
+ image: ecs.ContainerImage.fromRegistry('public.ecr.aws/docker/library/httpd:2.4'),
80
+ memoryLimitMiB: 512,
81
+ portMappings: [
82
+ {
83
+ containerPort: 80,
84
+ protocol: ecs.Protocol.TCP,
85
+ },
86
+ ],
87
+ });
88
+
89
+ // Create a service using the MI capacity provider
90
+ new ecs.FargateService(stack, 'ManagedInstancesService', {
91
+ cluster,
92
+ taskDefinition,
93
+ capacityProviderStrategies: [
94
+ {
95
+ capacityProvider: miCapacityProvider.capacityProviderName,
96
+ weight: 1,
97
+ },
98
+ ],
99
+ desiredCount: 1,
100
+ });
101
+
102
+ new integ.IntegTest(app, 'ManagedInstancesCapacityProviders', {
103
+ testCases: [stack],
104
+ regions: ['us-west-2'],
105
+ });
106
+
107
+ app.synth();
@@ -0,0 +1,45 @@
1
+ import * as ec2 from 'aws-cdk-lib/aws-ec2';
2
+ import * as ecs from 'aws-cdk-lib/aws-ecs';
3
+ import * as cdk from 'aws-cdk-lib';
4
+ import * as integ from '@aws-cdk/integ-tests-alpha';
5
+ import * as ecsPatterns from 'aws-cdk-lib/aws-ecs-patterns';
6
+
7
+ const app = new cdk.App();
8
+ const stack = new cdk.Stack(app, 'aws-ecs-integ-alb-fargate-public-private-switch');
9
+
10
+ const vpc = new ec2.Vpc(stack, 'Vpc', { maxAzs: 2, restrictDefaultSecurityGroup: false });
11
+ const cluster = new ecs.Cluster(stack, 'FargateCluster', { vpc });
12
+
13
+ // Test private load balancer (the problematic case from the issue)
14
+ new ecsPatterns.ApplicationLoadBalancedFargateService(stack, 'PrivateALBFargateService', {
15
+ cluster,
16
+ memoryLimitMiB: 1024,
17
+ cpu: 512,
18
+ publicLoadBalancer: false, // This should create ECSPrivate target group
19
+ taskImageOptions: {
20
+ image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'),
21
+ },
22
+ });
23
+
24
+ // Test public load balancer for comparison
25
+ new ecsPatterns.ApplicationLoadBalancedFargateService(stack, 'PublicALBFargateService', {
26
+ cluster,
27
+ memoryLimitMiB: 1024,
28
+ cpu: 512,
29
+ publicLoadBalancer: true, // This should create ECS target group
30
+ taskImageOptions: {
31
+ image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'),
32
+ },
33
+ });
34
+
35
+ new integ.IntegTest(app, 'ALBFargatePublicPrivateSwitchTest', {
36
+ testCases: [stack],
37
+ allowDestroy: [
38
+ 'PrivateALBFargateServiceLB3F43693F',
39
+ 'PrivateALBFargateServiceLBPublicListenerECSPrivateGroup81AA5B8B',
40
+ 'PublicALBFargateServiceLBBDD839E7',
41
+ 'PublicALBFargateServiceLBPublicListenerECSGroupD991EA00',
42
+ ],
43
+ });
44
+
45
+ app.synth();
@@ -19,7 +19,7 @@ const vpc = new ec2.Vpc(stack, 'VPC', {
19
19
  });
20
20
 
21
21
  const fn = new lambda.Function(stack, 'Function', {
22
- runtime: lambda.Runtime.NODEJS_18_X,
22
+ runtime: lambda.Runtime.NODEJS_20_X,
23
23
  handler: 'index.handler',
24
24
  code: lambda.Code.fromInline(`
25
25
  exports.handler = async (event) => {
@@ -186,7 +186,7 @@ const signinFunction = new lambda.Function(testCase, 'Signin', {
186
186
  functionName: 'cdk-integ-alb-oidc-signin-handler',
187
187
  code: lambda.Code.fromAsset('alb-oidc-signin-handler', { exclude: ['*.ts'] }),
188
188
  handler: 'index.handler',
189
- runtime: lambda.Runtime.NODEJS_18_X,
189
+ runtime: lambda.Runtime.NODEJS_20_X,
190
190
  environment: {
191
191
  TEST_USERNAME: testUser.username,
192
192
  TEST_PASSWORD: testUser.password,
@@ -206,7 +206,7 @@ const signinFunction = new lambda.Function(testCase, 'Signin', {
206
206
  functionName: 'cdk-integ-alb-cognito-signin-handler',
207
207
  code: lambda.Code.fromAsset('alb-cognito-signin-handler', { exclude: ['*.ts'] }),
208
208
  handler: 'index.handler',
209
- runtime: lambda.Runtime.NODEJS_18_X,
209
+ runtime: lambda.Runtime.NODEJS_20_X,
210
210
  environment: {
211
211
  TEST_USERNAME: testUser.username,
212
212
  TEST_PASSWORD: testUser.password,
@@ -125,6 +125,102 @@ const s3Destination = new firehose.S3Bucket(bucket, {
125
125
  });
126
126
  ```
127
127
 
128
+ ## Data Format Conversion
129
+
130
+ Data format conversion allows automatic conversion of inputs from JSON to either Parquet or ORC.
131
+ Converting JSON records to columnar formats like Parquet or ORC can help speed up analytical querying while also increasing compression efficiency.
132
+ When data format conversion is specified, it automatically enables Snappy compression on the output.
133
+
134
+ Only S3 Destinations support data format conversion.
135
+
136
+ An example of defining an S3 destination configured with data format conversion:
137
+
138
+ ```ts
139
+ declare const bucket: s3.Bucket;
140
+ declare const schemaGlueTable: glue.CfnTable;
141
+ const s3Destination = new firehose.S3Bucket(bucket, {
142
+ dataFormatConversion: {
143
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaGlueTable),
144
+ inputFormat: firehose.InputFormat.OPENX_JSON,
145
+ outputFormat: firehose.OutputFormat.PARQUET,
146
+ }
147
+ });
148
+ ```
149
+
150
+ When data format conversion is enabled, the Delivery Stream's buffering size must be at least 64 MiB.
151
+ Additionally, the default buffering size is changed from 5 MiB to 128 MiB. This mirrors the Cloudformation behavior.
152
+
153
+ You can only parse JSON and transform it into either Parquet or ORC:
154
+ - to read JSON using OpenX parser, choose `InputFormat.OPENX_JSON`.
155
+ - to read JSON using Hive parser, choose `InputFormat.HIVE_JSON`.
156
+ - to transform into Parquet, choose `OutputFormat.PARQUET`.
157
+ - to transform into ORC, choose `OutputFormat.ORC`.
158
+
159
+ The following subsections explain how to specify advanced configuration options for each input and output format if the defaults are not desirable
160
+
161
+ ### Input Format: OpenX JSON
162
+
163
+ Example creation of custom OpenX JSON InputFormat:
164
+
165
+ ```ts
166
+ const inputFormat = new firehose.OpenXJsonInputFormat({
167
+ lowercaseColumnNames: false,
168
+ columnToJsonKeyMappings: {"ts": "timestamp"},
169
+ convertDotsInJsonKeysToUnderscores: true,
170
+ })
171
+ ```
172
+
173
+ ### Input Format: Hive JSON
174
+
175
+ Example creation of custom Hive JSON InputFormat:
176
+
177
+ ```ts
178
+ const inputFormat = new firehose.HiveJsonInputFormat({
179
+ timestampParsers: [
180
+ firehose.TimestampParser.fromFormatString('yyyy-MM-dd'),
181
+ firehose.TimestampParser.EPOCH_MILLIS,
182
+ ]
183
+ })
184
+ ```
185
+
186
+ Hive JSON allows you to specify custom timestamp formats to parse. The syntax of the format string is Joda Time.
187
+
188
+ To parse timestamps formatted as milliseconds since epoch, use the convenience constant `TimestampParser.EPOCH_MILLIS`.
189
+
190
+ ### Output Format: Parquet
191
+
192
+ Example of a custom Parquet OutputFormat, with all values changed from the defaults.
193
+
194
+ ```ts
195
+ const outputFormat = new firehose.ParquetOutputFormat({
196
+ blockSize: Size.mebibytes(512),
197
+ compression: firehose.ParquetCompression.UNCOMPRESSED,
198
+ enableDictionaryCompression: true,
199
+ maxPadding: Size.bytes(10),
200
+ pageSize: Size.mebibytes(2),
201
+ writerVersion: firehose.ParquetWriterVersion.V2,
202
+ })
203
+ ```
204
+
205
+ ### Output Format: ORC
206
+
207
+ Example creation of custom ORC OutputFormat, with all values changed from the defaults.
208
+
209
+ ```ts
210
+ const outputFormat = new firehose.OrcOutputFormat({
211
+ formatVersion: firehose.OrcFormatVersion.V0_11,
212
+ blockSize: Size.mebibytes(256),
213
+ compression: firehose.OrcCompression.NONE,
214
+ bloomFilterColumns: ['columnA'],
215
+ bloomFilterFalsePositiveProbability: 0.1,
216
+ dictionaryKeyThreshold: 0.7,
217
+ enablePadding: true,
218
+ paddingTolerance: 0.2,
219
+ rowIndexStride: 9000,
220
+ stripeSize: Size.mebibytes(32),
221
+ })
222
+ ```
223
+
128
224
  ## Server-side Encryption
129
225
 
130
226
  Enabling server-side encryption (SSE) requires Amazon Data Firehose to encrypt all data
@@ -0,0 +1,154 @@
1
+ import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
2
+ import * as s3 from 'aws-cdk-lib/aws-s3';
3
+ import * as glue from 'aws-cdk-lib/aws-glue';
4
+ import * as cdk from 'aws-cdk-lib';
5
+ import * as integ from '@aws-cdk/integ-tests-alpha';
6
+ import { Construct } from 'constructs';
7
+
8
+ const app = new cdk.App();
9
+
10
+ const SCHEMA_COLUMNS = [
11
+ {
12
+ name: 'column_a',
13
+ type: 'string',
14
+ },
15
+ {
16
+ name: 'column_b',
17
+ type: 'string',
18
+ },
19
+ ];
20
+
21
+ class TestStack extends cdk.Stack {
22
+ public readonly deliveryStreamsToTest: firehose.IDeliveryStream[];
23
+ public readonly bucket: s3.Bucket;
24
+ constructor(scope: Construct, id: string, props?: cdk.StackProps) {
25
+ super(scope, id, props);
26
+
27
+ this.bucket = new s3.Bucket(this, 'Bucket', {
28
+ removalPolicy: cdk.RemovalPolicy.DESTROY,
29
+ autoDeleteObjects: true,
30
+ });
31
+
32
+ const database = new glue.CfnDatabase(this, 'Database', {
33
+ databaseInput: {
34
+ description: 'My database',
35
+ },
36
+ catalogId: this.account,
37
+ });
38
+
39
+ const registry = new glue.CfnRegistry(this, 'SchemaRegistry', {
40
+ name: 'my_schema_registry',
41
+ });
42
+
43
+ const inlineSchemaTable = this.createTableWithInlineSchema(database);
44
+ const inlineSchemaDeliveryStream = this.createDeliveryStreamWithDataFormatConversion('InlineSchema', {
45
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(inlineSchemaTable),
46
+ inputFormat: firehose.InputFormat.OPENX_JSON,
47
+ outputFormat: firehose.OutputFormat.PARQUET,
48
+ });
49
+
50
+ const registrySchemaTable = this.createTableWithRegistrySchema(database, registry);
51
+ const registrySchemaDeliveryStream = this.createDeliveryStreamWithDataFormatConversion('RegistrySchema', {
52
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(registrySchemaTable),
53
+ inputFormat: firehose.InputFormat.OPENX_JSON,
54
+ outputFormat: firehose.OutputFormat.PARQUET,
55
+ });
56
+
57
+ this.deliveryStreamsToTest = [
58
+ inlineSchemaDeliveryStream,
59
+ registrySchemaDeliveryStream,
60
+ ];
61
+ }
62
+
63
+ private createTableWithInlineSchema(database: glue.CfnDatabase): glue.CfnTable {
64
+ return new glue.CfnTable(this, 'InlineSchemaTable', {
65
+ catalogId: database.catalogId,
66
+ databaseName: database.ref,
67
+ tableInput: {
68
+ storageDescriptor: {
69
+ columns: SCHEMA_COLUMNS,
70
+ },
71
+ },
72
+ });
73
+ }
74
+
75
+ private createTableWithRegistrySchema(database: glue.CfnDatabase, registry: glue.CfnRegistry): glue.CfnTable {
76
+ const schemaDefinition = JSON.stringify({
77
+ type: 'record',
78
+ name: 'MyRecord',
79
+ fields: SCHEMA_COLUMNS,
80
+ });
81
+
82
+ const schema = new glue.CfnSchema(this, 'Schema', {
83
+ registry: {
84
+ arn: registry.attrArn,
85
+ },
86
+ compatibility: 'NONE',
87
+ dataFormat: 'AVRO',
88
+ name: 'my_schema',
89
+ schemaDefinition: schemaDefinition,
90
+ });
91
+
92
+ return new glue.CfnTable(this, 'RegistrySchemaTable', {
93
+ catalogId: database.catalogId,
94
+ databaseName: database.ref,
95
+ tableInput: {
96
+ storageDescriptor: {
97
+ schemaReference: {
98
+ schemaVersionId: schema.attrInitialSchemaVersionId,
99
+ },
100
+ },
101
+ },
102
+ });
103
+ }
104
+
105
+ private createDeliveryStreamWithDataFormatConversion(
106
+ id: string,
107
+ dataFormatConversion: firehose.DataFormatConversionProps,
108
+ ): firehose.DeliveryStream {
109
+ return new firehose.DeliveryStream(this, id, {
110
+ destination: new firehose.S3Bucket(this.bucket, {
111
+ dataOutputPrefix: `success/${id}/`,
112
+ errorOutputPrefix: `error/${id}/`,
113
+ bufferingInterval: cdk.Duration.seconds(0),
114
+ dataFormatConversion: dataFormatConversion,
115
+ }),
116
+ });
117
+ }
118
+ }
119
+
120
+ const stack = new TestStack(app, 'RecordFormatConversionSchema');
121
+ const testCase = new integ.IntegTest(app, 'RecordFormatConversionSchemaTest', {
122
+ testCases: [stack],
123
+ });
124
+
125
+ const assertions = testCase.assertions;
126
+
127
+ // Test each delivery stream with the same input, and verify that each writes the output to the success prefix
128
+ // Relies on waiting timeout to tell if record format conversion failed.
129
+ stack.deliveryStreamsToTest.forEach(deliveryStream => {
130
+ const putDataCall = assertions.awsApiCall('Firehose', 'putRecord', {
131
+ DeliveryStreamName: deliveryStream.deliveryStreamName,
132
+ Record: {
133
+ Data: JSON.stringify({
134
+ Column_A: 'foo',
135
+ Column_B: 'bar',
136
+ }),
137
+ },
138
+ });
139
+
140
+ const waitForResultCall = assertions.awsApiCall('S3', 'listObjectsV2', {
141
+ Bucket: stack.bucket.bucketName,
142
+ Prefix: `success/${deliveryStream.node.id}/`,
143
+ }).expect(integ.ExpectedResult.objectLike({
144
+ KeyCount: 1,
145
+ })).waitForAssertions({
146
+ interval: cdk.Duration.seconds(5),
147
+ totalTimeout: cdk.Duration.minutes(2),
148
+ });
149
+
150
+ const api = waitForResultCall as integ.AwsApiCall;
151
+ api.waiterProvider?.addPolicyStatementFromSdkCall('s3', 'ListBucket', [stack.bucket.bucketArn]);
152
+ api.waiterProvider?.addPolicyStatementFromSdkCall('s3', 'GetObject', [stack.bucket.arnForObjects('*')]);
153
+ putDataCall.next(waitForResultCall);
154
+ });
@@ -0,0 +1,178 @@
1
+ import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
2
+ import * as s3 from 'aws-cdk-lib/aws-s3';
3
+ import * as glue from 'aws-cdk-lib/aws-glue';
4
+ import * as cdk from 'aws-cdk-lib';
5
+ import * as integ from '@aws-cdk/integ-tests-alpha';
6
+ import { Construct } from 'constructs';
7
+
8
+ const app = new cdk.App();
9
+
10
+ const SCHEMA_COLUMNS = [
11
+ {
12
+ name: 'column_a',
13
+ type: 'string',
14
+ },
15
+ {
16
+ name: 'column_b',
17
+ type: 'string',
18
+ },
19
+ ];
20
+
21
+ class TestStack extends cdk.Stack {
22
+ public readonly bucket: s3.Bucket;
23
+ constructor(scope: Construct, id: string, props?: cdk.StackProps) {
24
+ super(scope, id, props);
25
+
26
+ this.bucket = new s3.Bucket(this, 'Bucket', {
27
+ removalPolicy: cdk.RemovalPolicy.DESTROY,
28
+ autoDeleteObjects: true,
29
+ });
30
+
31
+ const database = new glue.CfnDatabase(this, 'Database', {
32
+ databaseInput: {
33
+ description: 'My database',
34
+ },
35
+ catalogId: this.account,
36
+ });
37
+
38
+ const schemaTable = this.createTableWithInlineSchema(database);
39
+
40
+ // default hive json input with default orc output
41
+ this.createDeliveryStreamWithDataFormatConversion('DefaultHiveJsonOrc', {
42
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
43
+ inputFormat: firehose.InputFormat.HIVE_JSON,
44
+ outputFormat: firehose.OutputFormat.ORC,
45
+ });
46
+
47
+ // default openx json input with default parquet output
48
+ this.createDeliveryStreamWithDataFormatConversion('DefaultOpenXJsonParquet', {
49
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
50
+ inputFormat: firehose.InputFormat.OPENX_JSON,
51
+ outputFormat: firehose.OutputFormat.PARQUET,
52
+ });
53
+
54
+ // custom hive json input
55
+ this.createDeliveryStreamWithDataFormatConversion('CustomHiveJson', {
56
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
57
+ inputFormat: new firehose.HiveJsonInputFormat({
58
+ timestampParsers: [
59
+ firehose.TimestampParser.EPOCH_MILLIS,
60
+ firehose.TimestampParser.fromFormatString('yyyy-MM-dd'),
61
+ ],
62
+ }),
63
+ outputFormat: firehose.OutputFormat.ORC,
64
+ });
65
+
66
+ // custom openx json input
67
+ this.createDeliveryStreamWithDataFormatConversion('CustomOpenXJson', {
68
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
69
+ inputFormat: new firehose.OpenXJsonInputFormat({
70
+ lowercaseColumnNames: false,
71
+ columnToJsonKeyMappings: { column_yay: 'Column_A' },
72
+ convertDotsInJsonKeysToUnderscores: true,
73
+ }),
74
+ outputFormat: firehose.OutputFormat.PARQUET,
75
+ });
76
+
77
+ // custom orc output
78
+ this.createDeliveryStreamWithDataFormatConversion('CustomOrc', {
79
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
80
+ inputFormat: firehose.InputFormat.OPENX_JSON,
81
+ outputFormat: new firehose.OrcOutputFormat({
82
+ blockSize: cdk.Size.mebibytes(256),
83
+ bloomFilterColumns: ['column_a'],
84
+ bloomFilterFalsePositiveProbability: 0.5,
85
+ compression: firehose.OrcCompression.NONE,
86
+ dictionaryKeyThreshold: 0.3,
87
+ formatVersion: firehose.OrcFormatVersion.V0_11,
88
+ enablePadding: true,
89
+ paddingTolerance: 0.4,
90
+ rowIndexStride: 5000,
91
+ stripeSize: cdk.Size.mebibytes(32),
92
+ }),
93
+ });
94
+
95
+ // ORC ZLIB compression
96
+ this.createDeliveryStreamWithDataFormatConversion('CustomOrcZlib', {
97
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
98
+ inputFormat: firehose.InputFormat.OPENX_JSON,
99
+ outputFormat: new firehose.OrcOutputFormat({
100
+ compression: firehose.OrcCompression.ZLIB,
101
+ formatVersion: firehose.OrcFormatVersion.V0_12,
102
+ }),
103
+ });
104
+
105
+ // ORC SNAPPY compression
106
+ this.createDeliveryStreamWithDataFormatConversion('CustomOrcSnappy', {
107
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
108
+ inputFormat: firehose.InputFormat.OPENX_JSON,
109
+ outputFormat: new firehose.OrcOutputFormat({
110
+ compression: firehose.OrcCompression.SNAPPY,
111
+ }),
112
+ });
113
+
114
+ // custom parquet output format
115
+ this.createDeliveryStreamWithDataFormatConversion('CustomParquet', {
116
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
117
+ inputFormat: firehose.InputFormat.OPENX_JSON,
118
+ outputFormat: new firehose.ParquetOutputFormat({
119
+ blockSize: cdk.Size.mebibytes(128),
120
+ pageSize: cdk.Size.mebibytes(2),
121
+ compression: firehose.ParquetCompression.UNCOMPRESSED,
122
+ writerVersion: firehose.ParquetWriterVersion.V2,
123
+ enableDictionaryCompression: true,
124
+ maxPadding: cdk.Size.bytes(100),
125
+ }),
126
+ });
127
+
128
+ // Parquet GZIP compression
129
+ this.createDeliveryStreamWithDataFormatConversion('CustomParquetGzip', {
130
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
131
+ inputFormat: firehose.InputFormat.OPENX_JSON,
132
+ outputFormat: new firehose.ParquetOutputFormat({
133
+ compression: firehose.ParquetCompression.GZIP,
134
+ writerVersion: firehose.ParquetWriterVersion.V1,
135
+ }),
136
+ });
137
+
138
+ // Parquet SNAPPY compression
139
+ this.createDeliveryStreamWithDataFormatConversion('CustomParquetSnappy', {
140
+ schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
141
+ inputFormat: firehose.InputFormat.OPENX_JSON,
142
+ outputFormat: new firehose.ParquetOutputFormat({
143
+ compression: firehose.ParquetCompression.SNAPPY,
144
+ }),
145
+ });
146
+ }
147
+
148
+ private createTableWithInlineSchema(database: glue.CfnDatabase): glue.CfnTable {
149
+ return new glue.CfnTable(this, 'InlineSchemaTable', {
150
+ catalogId: database.catalogId,
151
+ databaseName: database.ref,
152
+ tableInput: {
153
+ storageDescriptor: {
154
+ columns: SCHEMA_COLUMNS,
155
+ },
156
+ },
157
+ });
158
+ }
159
+
160
+ private createDeliveryStreamWithDataFormatConversion(
161
+ id: string,
162
+ dataFormatConversion: firehose.DataFormatConversionProps,
163
+ ): firehose.DeliveryStream {
164
+ return new firehose.DeliveryStream(this, id, {
165
+ destination: new firehose.S3Bucket(this.bucket, {
166
+ dataOutputPrefix: `success/${id}/`,
167
+ errorOutputPrefix: `error/${id}/`,
168
+ bufferingInterval: cdk.Duration.seconds(0),
169
+ dataFormatConversion: dataFormatConversion,
170
+ }),
171
+ });
172
+ }
173
+ }
174
+
175
+ const stack = new TestStack(app, 'RecordFormatConversion');
176
+ new integ.IntegTest(app, 'RecordFormatConversionTest', {
177
+ testCases: [stack],
178
+ });
@@ -10,7 +10,7 @@ const app = new App({
10
10
  const stack = new Stack(app, 'IntegBinaryPayload');
11
11
 
12
12
  const fn = new Function(stack, 'fn', {
13
- runtime: Runtime.NODEJS_18_X,
13
+ runtime: Runtime.NODEJS_20_X,
14
14
  handler: 'index.handler',
15
15
  code: Code.fromInline(`
16
16
  exports.handler = async (event) => {