presidium 0.15.30 → 0.15.31

package/KinesisStream.js

@@ -48,6 +48,7 @@ const KinesisStream = function (options) {
   this.shardUpdatePeriod = options.shardUpdatePeriod ?? 15000
   this.getRecordsInterval = options.getRecordsInterval ?? 1000
   this.shardIteratorType = options.shardIteratorType ?? 'LATEST'
+  this.shardCount = options.shardCount ?? 1
   this.timestamp = options.timestamp
   this.kinesis = new Kinesis(omit(['name'])(options))
   this.cancelToken = new Promise((_, reject) => (this.canceller = reject))
@@ -65,7 +66,7 @@ const KinesisStream = function (options) {
   }).catch(async () => {
     await this.kinesis.client.createStream({
       StreamName: this.name,
-      ShardCount: 1,
+      ShardCount: this.shardCount,
     }).promise()
     await this.kinesis.client.waitFor('streamExists', {
       StreamName: this.name
@@ -92,7 +93,7 @@ KinesisStream.prototype.delete = function deleteStream() {
  * @synopsis
  * ```coffeescript [specscript]
  * KinesisStream(options).putRecord(
- *   data string|binary,
+ *   data string|Buffer,
  *   options {
  *     partitionKey: string, // input to aws hash function to determine which shard
  *     explicitHashKey: string, // skips aws hash function to determine which shard
@@ -118,6 +119,57 @@ KinesisStream.prototype.putRecord = async function putRecord(data, options = {})
   }).promise()
 }
 
+/**
+ * @name KinesisStream.prototype.putRecords
+ *
+ * @synopsis
+ * ```coffeescript [specscript]
+ * KinesisStream(options).putRecords(records Array<{
+ *   data: string|Buffer,
+ *   partitionKey: string, // input to aws hash function to determine which shard
+ *   explicitHashKey: string, // skips aws hash function to determine which shard
+ * }>) -> Promise<{
+ *   FailedRecordCount: number, // number of unsuccessfully processed records
+ *   Records: Array<{
+ *     SequenceNumber: string,
+ *     ShardId: string,
+ *     ErrorCode?: 'ProvisionedThroughputExceededException'|'InternalFailure',
+ *     ErrorMessage?: string,
+ *   }>,
+ *   EncryptionType: 'NONE'|'KMS',
+ * }>
+ * ```
+ *
+ * @description
+ * Limits/Quotas:
+ *  * 500 records max per request
+ *  * 1 MB per record max
+ *  * 5 MB per request max
+ *  * 1000 records per second per shard
+ *  * 1 MB per second per shard
+ *
+ * https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Kinesis.html#putRecords-property
+ */
+KinesisStream.prototype.putRecords = async function putRecords(records) {
+  return this.kinesis.client.putRecords({
+    StreamName: this.name,
+    Records: records.map(({ data, partitionKey, explicitHashKey }) => ({
+      Data: data,
+      PartitionKey: partitionKey ?? data.slice(0, 255),
+      ...explicitHashKey && { ExplicitHashKey: explicitHashKey },
+    }))
+  }).promise().then(tap(response => {
+    if (response.Records.some(has('ErrorCode'))) {
+      const errors = response.Records.filter(has('ErrorCode')).map(Record => {
+        const error = new Error(Record.ErrorMessage)
+        error.code = Record.ErrorCode
+        return error
+      })
+      throw new AggregateError(errors, 'Some records failed to process')
+    }
+  }))
+}
+
 // () => ()
 KinesisStream.prototype.close = function close() {
   this.closed = true

package/KinesisStream.test.js

@@ -29,6 +29,71 @@ const test = new Test('KinesisStream', KinesisStream)
   this.streams.push(myStream)
 })
 
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'TRIM_HORIZON',
+  getRecordsLimit: 1,
+  listShardsLimit: 1,
+  shardCount: 2,
+}, async function (myStream) {
+  await myStream.ready
+  await myStream.putRecord('hey', { partitionKey: 'a' })
+  await myStream.putRecord('ho', { partitionKey: 'a' })
+  await myStream.putRecord('hi', { partitionKey: 'a' })
+  const first3 = await asyncIterableTake(3)(myStream)
+  const first3Again = await asyncIterableTake(3)(myStream)
+  assert.deepEqual(first3, first3Again)
+  this.streams.push(myStream)
+})
+
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'TRIM_HORIZON',
+}, async function (myStream) {
+  await myStream.ready
+  await myStream.putRecords([
+    { data: 'hey' },
+    { data: 'ho', partitionKey: 'ho' },
+    { data: 'hi', explicitHashKey: '127' },
+  ])
+  const first3 = await asyncIterableTake(3)(myStream)
+  const first3Again = await asyncIterableTake(3)(myStream)
+  assert.deepEqual(first3, first3Again)
+  this.streams.push(myStream)
+})
+
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'TRIM_HORIZON',
+}, async function (myStream) {
+  myStream.kinesis.client.putRecords = () => ({
+    promise: async () => ({
+      Records: [
+        {
+          ErrorCode: 'ProvisionedThroughputExceededException',
+          ErrorMessage: 'Some message with accountId, stream name, and shard ID',
+        },
+        {
+          ErrorCode: 'ProvisionedThroughputExceededException',
+          ErrorMessage: 'Some message with accountId, stream name, and shard ID',
+        },
+      ],
+    }),
+  })
+  await myStream.ready
+  await assert.rejects(
+    () => myStream.putRecords([{ data: 'hey' }]),
+    new AggregateError([
+      new Error('Some message with accountId, stream name, and shard ID'),
+      new Error('Some message with accountId, stream name, and shard ID'),
+    ], 'Some records failed to process')
+  )
+  this.streams.push(myStream)
+})
+
 .case({
   name: 'my-stream',
   endpoint: 'http://localhost:4567',
@@ -86,8 +151,11 @@ const test = new Test('KinesisStream', KinesisStream)
   await new Promise(resolve => setTimeout(resolve, 1000))
 })
 
-.after(async function() {
-  await map(stream => stream.delete())(this.streams)
+.after(async function () {
+  await map(async function cleanup(stream) {
+    stream.close()
+    await stream.delete()
+  })(this.streams)
 })
 
 if (process.argv[1] == __filename) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "presidium",
-  "version": "0.15.30",
+  "version": "0.15.31",
   "description": "A library for creating web services",
   "author": "Richard Tong",
   "license": "MIT",