konokenj.cdk-api-mcp-server 0.31.0__py3-none-any.whl → 0.57.0__py3-none-any.whl

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesis/README.md

@@ -15,6 +15,7 @@ intake and aggregation.
     - [Write Permissions](#write-permissions)
     - [Custom Permissions](#custom-permissions)
   - [Metrics](#metrics)
+    - [Shard-level Metrics](#shard-level-metrics)
 - [Stream Consumers](#stream-consumers)
   - [Read Permissions](#read-permissions-1)
 - [Resource Policy](#resource-policy)
@@ -191,6 +192,47 @@ stream.metricGetRecordsSuccess();
 stream.metricGetRecordsSuccess({ statistic: 'Maximum' });
 ```
 
+#### Shard-level Metrics
+
+You can enable enhanced shard-level metrics for your Kinesis stream to get detailed monitoring of individual shards. Shard-level metrics provide more granular insights into the performance and health of your stream.
+
+```ts
+const stream = new kinesis.Stream(this, 'MyStream', {
+  shardLevelMetrics: [kinesis.ShardLevelMetrics.ALL],
+});
+```
+
+You can also specify individual metrics that you want to monitor:
+
+```ts
+const stream = new kinesis.Stream(this, 'MyStream', {
+  shardLevelMetrics: [
+    kinesis.ShardLevelMetrics.INCOMING_BYTES,
+    kinesis.ShardLevelMetrics.INCOMING_RECORDS,
+    kinesis.ShardLevelMetrics.ITERATOR_AGE_MILLISECONDS,
+    kinesis.ShardLevelMetrics.OUTGOING_BYTES,
+    kinesis.ShardLevelMetrics.OUTGOING_RECORDS,
+    kinesis.ShardLevelMetrics.READ_PROVISIONED_THROUGHPUT_EXCEEDED,
+    kinesis.ShardLevelMetrics.WRITE_PROVISIONED_THROUGHPUT_EXCEEDED,
+  ],
+});
+```
+
+Available shard-level metrics include:
+
+- `INCOMING_BYTES` - The number of bytes successfully put to the shard
+- `INCOMING_RECORDS` - The number of records successfully put to the shard
+- `ITERATOR_AGE_MILLISECONDS` - The age of the last record in all GetRecords calls made against a shard
+- `OUTGOING_BYTES` - The number of bytes retrieved from the shard
+- `OUTGOING_RECORDS` - The number of records retrieved from the shard
+- `READ_PROVISIONED_THROUGHPUT_EXCEEDED` - The number of GetRecords calls throttled for the shard
+- `WRITE_PROVISIONED_THROUGHPUT_EXCEEDED` - The number of records rejected due to throttling for the shard
+- `ALL` - All available metrics
+
+Note: You cannot specify `ALL` together with other individual metrics. If you want all metrics, use `ALL` alone.
+
+For more information about shard-level metrics, see [Monitoring the Amazon Kinesis Data Streams Service with Amazon CloudWatch](https://docs.aws.amazon.com/streams/latest/dev/monitoring-with-cloudwatch.html#kinesis-metrics-shard).
+
 ## Stream Consumers
 
 Creating stream consumers allow consumers to receive data from the stream using enhanced fan-out at a rate of up to 2 MiB per second for every shard.

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesis/integ.stream-shard-level-monitoring.ts

@@ -0,0 +1,47 @@
+import { App, Stack, RemovalPolicies } from 'aws-cdk-lib';
+import * as kinesis from 'aws-cdk-lib/aws-kinesis';
+import { ExpectedResult, IntegTest } from '@aws-cdk/integ-tests-alpha';
+
+const app = new App();
+const stack = new Stack(app, 'kinesis-stream-shard-level-monitoring-stack');
+
+const explicitStream = new kinesis.Stream(stack, 'ExplicitStream', {
+  shardLevelMetrics: [
+    kinesis.ShardLevelMetrics.INCOMING_BYTES,
+    kinesis.ShardLevelMetrics.INCOMING_RECORDS,
+    kinesis.ShardLevelMetrics.ITERATOR_AGE_MILLISECONDS,
+    kinesis.ShardLevelMetrics.OUTGOING_BYTES,
+    kinesis.ShardLevelMetrics.OUTGOING_RECORDS,
+    kinesis.ShardLevelMetrics.WRITE_PROVISIONED_THROUGHPUT_EXCEEDED,
+    kinesis.ShardLevelMetrics.READ_PROVISIONED_THROUGHPUT_EXCEEDED,
+  ],
+});
+
+const allStream = new kinesis.Stream(stack, 'AllStream', {
+  shardLevelMetrics: [kinesis.ShardLevelMetrics.ALL],
+});
+
+RemovalPolicies.of(stack).destroy();
+
+const integ = new IntegTest(app, 'integ-kinesis-stream-consumer', {
+  testCases: [stack],
+});
+
+const streams = [explicitStream, allStream];
+streams.forEach((stream) => {
+  integ.assertions.awsApiCall('Kinesis', 'describeStream', {
+    StreamName: stream.streamName,
+  }).expect(ExpectedResult.objectLike({
+    StreamDescription: {
+      ShardLevelMetrics: [
+        'IncomingBytes',
+        'IncomingRecords',
+        'IteratorAgeMilliseconds',
+        'OutgoingBytes',
+        'OutgoingRecords',
+        'WriteProvisionedThroughputExceeded',
+        'ReadProvisionedThroughputExceeded',
+      ],
+    },
+  }));
+});

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/README.md

@@ -102,10 +102,13 @@ will be used for files successfully delivered to S3. `errorOutputPrefix` will be
 failed records before writing them to S3.
 
 ```ts
+import { TimeZone } from 'aws-cdk-lib';
 declare const bucket: s3.Bucket;
 const s3Destination = new firehose.S3Bucket(bucket, {
   dataOutputPrefix: 'myFirehose/DeliveredYear=!{timestamp:yyyy}/anyMonth/rand=!{firehose:random-string}',
   errorOutputPrefix: 'myFirehoseFailures/!{firehose:error-output-type}/!{timestamp:yyyy}/anyMonth/!{timestamp:dd}',
+  // The time zone of timestamps (default UTC)
+  timeZone: TimeZone.ASIA_TOKYO,
 });
 ```
 
@@ -122,6 +125,102 @@ const s3Destination = new firehose.S3Bucket(bucket, {
 });
 ```
 
+## Data Format Conversion
+
+Data format conversion allows automatic conversion of inputs from JSON to either Parquet or ORC.
+Converting JSON records to columnar formats like Parquet or ORC can help speed up analytical querying while also increasing compression efficiency.
+When data format conversion is specified, it automatically enables Snappy compression on the output.
+
+Only S3 Destinations support data format conversion.
+
+An example of defining an S3 destination configured with data format conversion:
+
+```ts
+declare const bucket: s3.Bucket;
+declare const schemaGlueTable: glue.CfnTable;
+const s3Destination = new firehose.S3Bucket(bucket, {
+  dataFormatConversion: {
+    schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaGlueTable),
+    inputFormat: firehose.InputFormat.OPENX_JSON,
+    outputFormat: firehose.OutputFormat.PARQUET,
+  }
+});
+```
+
+When data format conversion is enabled, the Delivery Stream's buffering size must be at least 64 MiB. 
+Additionally, the default buffering size is changed from 5 MiB to 128 MiB. This mirrors the Cloudformation behavior.
+
+You can only parse JSON and transform it into either Parquet or ORC:
+- to read JSON using OpenX parser, choose `InputFormat.OPENX_JSON`.
+- to read JSON using Hive parser, choose `InputFormat.HIVE_JSON`.
+- to transform into Parquet, choose `OutputFormat.PARQUET`.
+- to transform into ORC, choose `OutputFormat.ORC`.
+
+The following subsections explain how to specify advanced configuration options for each input and output format if the defaults are not desirable
+
+### Input Format: OpenX JSON
+
+Example creation of custom OpenX JSON InputFormat:
+
+```ts
+const inputFormat = new firehose.OpenXJsonInputFormat({
+  lowercaseColumnNames: false,
+  columnToJsonKeyMappings: {"ts": "timestamp"},
+  convertDotsInJsonKeysToUnderscores: true,
+})
+```
+
+### Input Format: Hive JSON
+
+Example creation of custom Hive JSON InputFormat:
+
+```ts
+const inputFormat = new firehose.HiveJsonInputFormat({
+  timestampParsers: [
+    firehose.TimestampParser.fromFormatString('yyyy-MM-dd'),
+    firehose.TimestampParser.EPOCH_MILLIS,
+  ]
+})
+```
+
+Hive JSON allows you to specify custom timestamp formats to parse. The syntax of the format string is Joda Time.
+
+To parse timestamps formatted as milliseconds since epoch, use the convenience constant `TimestampParser.EPOCH_MILLIS`.
+
+### Output Format: Parquet
+
+Example of a custom Parquet OutputFormat, with all values changed from the defaults.
+
+```ts
+const outputFormat = new firehose.ParquetOutputFormat({
+  blockSize: Size.mebibytes(512),
+  compression: firehose.ParquetCompression.UNCOMPRESSED,
+  enableDictionaryCompression: true,
+  maxPadding: Size.bytes(10),
+  pageSize: Size.mebibytes(2),
+  writerVersion: firehose.ParquetWriterVersion.V2,
+})
+```
+
+### Output Format: ORC
+
+Example creation of custom ORC OutputFormat, with all values changed from the defaults.
+
+```ts
+const outputFormat = new firehose.OrcOutputFormat({
+  formatVersion: firehose.OrcFormatVersion.V0_11,
+  blockSize: Size.mebibytes(256),
+  compression: firehose.OrcCompression.NONE,
+  bloomFilterColumns: ['columnA'],
+  bloomFilterFalsePositiveProbability: 0.1,
+  dictionaryKeyThreshold: 0.7,
+  enablePadding: true,
+  paddingTolerance: 0.2,
+  rowIndexStride: 9000,
+  stripeSize: Size.mebibytes(32),
+})
+```
+
 ## Server-side Encryption
 
 Enabling server-side encryption (SSE) requires Amazon Data Firehose to encrypt all data
@@ -384,8 +483,11 @@ Data can be transformed before being delivered to destinations. There are two ty
 data processing for delivery streams: record transformation with AWS Lambda, and record
 format conversion using a schema stored in an AWS Glue table. If both types of data
 processing are configured, then the Lambda transformation is performed first. By default,
-no data processing occurs. This construct library currently only supports data
-transformation with AWS Lambda. See [#15501](https://github.com/aws/aws-cdk/issues/15501)
+no data processing occurs.
+
+This construct library currently only supports data
+transformation with AWS Lambda and some built-in data processors.
+See [#15501](https://github.com/aws/aws-cdk/issues/15501)
 to track the status of adding support for record format conversion.
 
 ### Data transformation with AWS Lambda
@@ -421,7 +523,7 @@ const lambdaProcessor = new firehose.LambdaFunctionProcessor(lambdaFunction, {
 });
 declare const bucket: s3.Bucket;
 const s3Destination = new firehose.S3Bucket(bucket, {
-  processor: lambdaProcessor,
+  processors: [lambdaProcessor],
 });
 new firehose.DeliveryStream(this, 'Delivery Stream', {
   destination: s3Destination,
@@ -433,6 +535,60 @@ new firehose.DeliveryStream(this, 'Delivery Stream', {
 See: [Data Transformation](https://docs.aws.amazon.com/firehose/latest/dev/data-transformation.html)
 in the *Amazon Data Firehose Developer Guide*.
 
+### Add a new line delimiter when delivering data to Amazon S3
+
+You can specify the `AppendDelimiterToRecordProcessor` built-in processor to add a new line delimiter between records in objects that are delivered to Amazon S3. This can be helpful for parsing objects in Amazon S3.
+For details, see [Use Amazon S3 bucket prefix to deliver data](https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning-s3bucketprefix.html).
+
+```ts
+declare const bucket: s3.Bucket;
+const s3Destination = new firehose.S3Bucket(bucket, {
+  processors: [
+    new firehose.AppendDelimiterToRecordProcessor(),
+  ],
+});
+new firehose.DeliveryStream(this, 'Delivery Stream', {
+  destination: s3Destination,
+});
+```
+
+### Decompress and extract message of CloudWatch Logs
+
+CloudWatch Logs events are sent to Firehose in compressed gzip format. If you want to deliver decompressed log events to Firehose destinations, you can use the `DecompressionProcessor` to automatically decompress CloudWatch Logs.
+For details, see [Send CloudWatch Logs to Firehose](https://docs.aws.amazon.com/firehose/latest/dev/writing-with-cloudwatch-logs.html).
+
+You may also needed to specify `AppendDelimiterToRecordProcessor`
+because decompressed log events record has no trailing newline.
+
+```ts
+declare const bucket: s3.Bucket;
+const s3Destination = new firehose.S3Bucket(bucket, {
+  processors: [
+    new firehose.DecompressionProcessor(),
+    new firehose.AppendDelimiterToRecordProcessor(),
+  ],
+});
+new firehose.DeliveryStream(this, 'Delivery Stream', {
+  destination: s3Destination,
+});
+```
+
+When you enable decompression, you have the option to also enable message extraction. When using message extraction, Firehose filters out all metadata, such as owner, loggroup, logstream, and others from the decompressed CloudWatch Logs records and delivers only the content inside the message fields.
+
+```ts
+declare const bucket: s3.Bucket;
+const s3Destination = new firehose.S3Bucket(bucket, {
+  processors: [
+    new firehose.DecompressionProcessor(),
+    new firehose.CloudWatchLogProcessor({ dataMessageExtraction: true }),
+  ],
+});
+new firehose.DeliveryStream(this, 'Delivery Stream', {
+  destination: s3Destination,
+});
+```
+
+
 ## Specifying an IAM role
 
 The DeliveryStream class automatically creates IAM service roles with all the minimum

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.cloudwatch-logs-processors.ts

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+import * as path from 'path';
+import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
+import * as lambdanodejs from 'aws-cdk-lib/aws-lambda-nodejs';
+import * as s3 from 'aws-cdk-lib/aws-s3';
+import * as cdk from 'aws-cdk-lib';
+import { IntegTest } from '@aws-cdk/integ-tests-alpha';
+
+const app = new cdk.App();
+
+const stack = new cdk.Stack(app, 'firehose-delivery-stream-cloudwatch-logs-processors');
+
+const bucket = new s3.Bucket(stack, 'DestinationBucket', {
+  removalPolicy: cdk.RemovalPolicy.DESTROY,
+  autoDeleteObjects: true,
+});
+
+const dataProcessorFunction = new lambdanodejs.NodejsFunction(stack, 'DataProcessorFunction', {
+  entry: path.join(__dirname, 'lambda-data-processor.js'),
+  timeout: cdk.Duration.minutes(1),
+});
+
+new firehose.DeliveryStream(stack, 'DecompressCloudWatchLogsEntry', {
+  destination: new firehose.S3Bucket(bucket, {
+    processors: [
+      new firehose.DecompressionProcessor(),
+      new firehose.AppendDelimiterToRecordProcessor(),
+      new firehose.LambdaFunctionProcessor(dataProcessorFunction),
+    ],
+  }),
+});
+
+new firehose.DeliveryStream(stack, 'ExtractCloudWatchLogsEntry', {
+  destination: new firehose.S3Bucket(bucket, {
+    processors: [
+      new firehose.DecompressionProcessor(),
+      new firehose.CloudWatchLogProcessor({ dataMessageExtraction: true }),
+      new firehose.LambdaFunctionProcessor(dataProcessorFunction),
+    ],
+  }),
+});
+
+new IntegTest(app, 'integ-tests', {
+  testCases: [stack],
+});

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.record-format-conversion-schema.ts

@@ -0,0 +1,154 @@
+import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
+import * as s3 from 'aws-cdk-lib/aws-s3';
+import * as glue from 'aws-cdk-lib/aws-glue';
+import * as cdk from 'aws-cdk-lib';
+import * as integ from '@aws-cdk/integ-tests-alpha';
+import { Construct } from 'constructs';
+
+const app = new cdk.App();
+
+const SCHEMA_COLUMNS = [
+  {
+    name: 'column_a',
+    type: 'string',
+  },
+  {
+    name: 'column_b',
+    type: 'string',
+  },
+];
+
+class TestStack extends cdk.Stack {
+  public readonly deliveryStreamsToTest: firehose.IDeliveryStream[];
+  public readonly bucket: s3.Bucket;
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    this.bucket = new s3.Bucket(this, 'Bucket', {
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      autoDeleteObjects: true,
+    });
+
+    const database = new glue.CfnDatabase(this, 'Database', {
+      databaseInput: {
+        description: 'My database',
+      },
+      catalogId: this.account,
+    });
+
+    const registry = new glue.CfnRegistry(this, 'SchemaRegistry', {
+      name: 'my_schema_registry',
+    });
+
+    const inlineSchemaTable = this.createTableWithInlineSchema(database);
+    const inlineSchemaDeliveryStream = this.createDeliveryStreamWithDataFormatConversion('InlineSchema', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(inlineSchemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: firehose.OutputFormat.PARQUET,
+    });
+
+    const registrySchemaTable = this.createTableWithRegistrySchema(database, registry);
+    const registrySchemaDeliveryStream = this.createDeliveryStreamWithDataFormatConversion('RegistrySchema', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(registrySchemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: firehose.OutputFormat.PARQUET,
+    });
+
+    this.deliveryStreamsToTest = [
+      inlineSchemaDeliveryStream,
+      registrySchemaDeliveryStream,
+    ];
+  }
+
+  private createTableWithInlineSchema(database: glue.CfnDatabase): glue.CfnTable {
+    return new glue.CfnTable(this, 'InlineSchemaTable', {
+      catalogId: database.catalogId,
+      databaseName: database.ref,
+      tableInput: {
+        storageDescriptor: {
+          columns: SCHEMA_COLUMNS,
+        },
+      },
+    });
+  }
+
+  private createTableWithRegistrySchema(database: glue.CfnDatabase, registry: glue.CfnRegistry): glue.CfnTable {
+    const schemaDefinition = JSON.stringify({
+      type: 'record',
+      name: 'MyRecord',
+      fields: SCHEMA_COLUMNS,
+    });
+
+    const schema = new glue.CfnSchema(this, 'Schema', {
+      registry: {
+        arn: registry.attrArn,
+      },
+      compatibility: 'NONE',
+      dataFormat: 'AVRO',
+      name: 'my_schema',
+      schemaDefinition: schemaDefinition,
+    });
+
+    return new glue.CfnTable(this, 'RegistrySchemaTable', {
+      catalogId: database.catalogId,
+      databaseName: database.ref,
+      tableInput: {
+        storageDescriptor: {
+          schemaReference: {
+            schemaVersionId: schema.attrInitialSchemaVersionId,
+          },
+        },
+      },
+    });
+  }
+
+  private createDeliveryStreamWithDataFormatConversion(
+    id: string,
+    dataFormatConversion: firehose.DataFormatConversionProps,
+  ): firehose.DeliveryStream {
+    return new firehose.DeliveryStream(this, id, {
+      destination: new firehose.S3Bucket(this.bucket, {
+        dataOutputPrefix: `success/${id}/`,
+        errorOutputPrefix: `error/${id}/`,
+        bufferingInterval: cdk.Duration.seconds(0),
+        dataFormatConversion: dataFormatConversion,
+      }),
+    });
+  }
+}
+
+const stack = new TestStack(app, 'RecordFormatConversionSchema');
+const testCase = new integ.IntegTest(app, 'RecordFormatConversionSchemaTest', {
+  testCases: [stack],
+});
+
+const assertions = testCase.assertions;
+
+// Test each delivery stream with the same input, and verify that each writes the output to the success prefix
+// Relies on waiting timeout to tell if record format conversion failed.
+stack.deliveryStreamsToTest.forEach(deliveryStream => {
+  const putDataCall = assertions.awsApiCall('Firehose', 'putRecord', {
+    DeliveryStreamName: deliveryStream.deliveryStreamName,
+    Record: {
+      Data: JSON.stringify({
+        Column_A: 'foo',
+        Column_B: 'bar',
+      }),
+    },
+  });
+
+  const waitForResultCall = assertions.awsApiCall('S3', 'listObjectsV2', {
+    Bucket: stack.bucket.bucketName,
+    Prefix: `success/${deliveryStream.node.id}/`,
+  }).expect(integ.ExpectedResult.objectLike({
+    KeyCount: 1,
+  })).waitForAssertions({
+    interval: cdk.Duration.seconds(5),
+    totalTimeout: cdk.Duration.minutes(2),
+  });
+
+  const api = waitForResultCall as integ.AwsApiCall;
+  api.waiterProvider?.addPolicyStatementFromSdkCall('s3', 'ListBucket', [stack.bucket.bucketArn]);
+  api.waiterProvider?.addPolicyStatementFromSdkCall('s3', 'GetObject', [stack.bucket.arnForObjects('*')]);
+  putDataCall.next(waitForResultCall);
+});

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.record-format-conversion.ts

@@ -0,0 +1,178 @@
+import * as firehose from 'aws-cdk-lib/aws-kinesisfirehose';
+import * as s3 from 'aws-cdk-lib/aws-s3';
+import * as glue from 'aws-cdk-lib/aws-glue';
+import * as cdk from 'aws-cdk-lib';
+import * as integ from '@aws-cdk/integ-tests-alpha';
+import { Construct } from 'constructs';
+
+const app = new cdk.App();
+
+const SCHEMA_COLUMNS = [
+  {
+    name: 'column_a',
+    type: 'string',
+  },
+  {
+    name: 'column_b',
+    type: 'string',
+  },
+];
+
+class TestStack extends cdk.Stack {
+  public readonly bucket: s3.Bucket;
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    this.bucket = new s3.Bucket(this, 'Bucket', {
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      autoDeleteObjects: true,
+    });
+
+    const database = new glue.CfnDatabase(this, 'Database', {
+      databaseInput: {
+        description: 'My database',
+      },
+      catalogId: this.account,
+    });
+
+    const schemaTable = this.createTableWithInlineSchema(database);
+
+    // default hive json input with default orc output
+    this.createDeliveryStreamWithDataFormatConversion('DefaultHiveJsonOrc', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.HIVE_JSON,
+      outputFormat: firehose.OutputFormat.ORC,
+    });
+
+    // default openx json input with default parquet output
+    this.createDeliveryStreamWithDataFormatConversion('DefaultOpenXJsonParquet', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: firehose.OutputFormat.PARQUET,
+    });
+
+    // custom hive json input
+    this.createDeliveryStreamWithDataFormatConversion('CustomHiveJson', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: new firehose.HiveJsonInputFormat({
+        timestampParsers: [
+          firehose.TimestampParser.EPOCH_MILLIS,
+          firehose.TimestampParser.fromFormatString('yyyy-MM-dd'),
+        ],
+      }),
+      outputFormat: firehose.OutputFormat.ORC,
+    });
+
+    // custom openx json input
+    this.createDeliveryStreamWithDataFormatConversion('CustomOpenXJson', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: new firehose.OpenXJsonInputFormat({
+        lowercaseColumnNames: false,
+        columnToJsonKeyMappings: { column_yay: 'Column_A' },
+        convertDotsInJsonKeysToUnderscores: true,
+      }),
+      outputFormat: firehose.OutputFormat.PARQUET,
+    });
+
+    // custom orc output
+    this.createDeliveryStreamWithDataFormatConversion('CustomOrc', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: new firehose.OrcOutputFormat({
+        blockSize: cdk.Size.mebibytes(256),
+        bloomFilterColumns: ['column_a'],
+        bloomFilterFalsePositiveProbability: 0.5,
+        compression: firehose.OrcCompression.NONE,
+        dictionaryKeyThreshold: 0.3,
+        formatVersion: firehose.OrcFormatVersion.V0_11,
+        enablePadding: true,
+        paddingTolerance: 0.4,
+        rowIndexStride: 5000,
+        stripeSize: cdk.Size.mebibytes(32),
+      }),
+    });
+
+    // ORC ZLIB compression
+    this.createDeliveryStreamWithDataFormatConversion('CustomOrcZlib', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: new firehose.OrcOutputFormat({
+        compression: firehose.OrcCompression.ZLIB,
+        formatVersion: firehose.OrcFormatVersion.V0_12,
+      }),
+    });
+
+    // ORC SNAPPY compression
+    this.createDeliveryStreamWithDataFormatConversion('CustomOrcSnappy', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: new firehose.OrcOutputFormat({
+        compression: firehose.OrcCompression.SNAPPY,
+      }),
+    });
+
+    // custom parquet output format
+    this.createDeliveryStreamWithDataFormatConversion('CustomParquet', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: new firehose.ParquetOutputFormat({
+        blockSize: cdk.Size.mebibytes(128),
+        pageSize: cdk.Size.mebibytes(2),
+        compression: firehose.ParquetCompression.UNCOMPRESSED,
+        writerVersion: firehose.ParquetWriterVersion.V2,
+        enableDictionaryCompression: true,
+        maxPadding: cdk.Size.bytes(100),
+      }),
+    });
+
+    // Parquet GZIP compression
+    this.createDeliveryStreamWithDataFormatConversion('CustomParquetGzip', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: new firehose.ParquetOutputFormat({
+        compression: firehose.ParquetCompression.GZIP,
+        writerVersion: firehose.ParquetWriterVersion.V1,
+      }),
+    });
+
+    // Parquet SNAPPY compression
+    this.createDeliveryStreamWithDataFormatConversion('CustomParquetSnappy', {
+      schemaConfiguration: firehose.SchemaConfiguration.fromCfnTable(schemaTable),
+      inputFormat: firehose.InputFormat.OPENX_JSON,
+      outputFormat: new firehose.ParquetOutputFormat({
+        compression: firehose.ParquetCompression.SNAPPY,
+      }),
+    });
+  }
+
+  private createTableWithInlineSchema(database: glue.CfnDatabase): glue.CfnTable {
+    return new glue.CfnTable(this, 'InlineSchemaTable', {
+      catalogId: database.catalogId,
+      databaseName: database.ref,
+      tableInput: {
+        storageDescriptor: {
+          columns: SCHEMA_COLUMNS,
+        },
+      },
+    });
+  }
+
+  private createDeliveryStreamWithDataFormatConversion(
+    id: string,
+    dataFormatConversion: firehose.DataFormatConversionProps,
+  ): firehose.DeliveryStream {
+    return new firehose.DeliveryStream(this, id, {
+      destination: new firehose.S3Bucket(this.bucket, {
+        dataOutputPrefix: `success/${id}/`,
+        errorOutputPrefix: `error/${id}/`,
+        bufferingInterval: cdk.Duration.seconds(0),
+        dataFormatConversion: dataFormatConversion,
+      }),
+    });
+  }
+}
+
+const stack = new TestStack(app, 'RecordFormatConversion');
+new integ.IntegTest(app, 'RecordFormatConversionTest', {
+  testCases: [stack],
+});

cdk_api_mcp_server/resources/aws-cdk/constructs/aws-cdk-lib/aws-kinesisfirehose/integ.s3-bucket.lit.ts

@@ -56,6 +56,7 @@ const deliveryStream = new firehose.DeliveryStream(stack, 'DeliveryStream', {
     dataOutputPrefix: 'regularPrefix',
     errorOutputPrefix: 'errorPrefix',
     fileExtension: '.log.gz',
+    timeZone: cdk.TimeZone.ASIA_TOKYO,
     bufferingInterval: cdk.Duration.seconds(60),
     bufferingSize: cdk.Size.mebibytes(1),
     encryptionKey: key,