presidium 0.15.27 → 0.15.31

package/DynamoStream.js

@@ -144,20 +144,20 @@ DynamoStream.prototype.getRecords = async function* getRecords(
     */
 
   }).promise().then(get('ShardIterator'))
+
   let records = await this.client.getRecords({
     ShardIterator: startingShardIterator,
     Limit: this.getRecordsLimit
   }).promise()
-
   if (records.Records.length > 0) {
     yield* records.Records.map(assign({
       table: always(this.table),
       shardId: always(Shard.ShardId),
     }))
   }
+  await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
 
   while (!this.closed && records.NextShardIterator != null) {
-    await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
     records = await this.client.getRecords({
       ShardIterator: records.NextShardIterator,
       Limit: this.getRecordsLimit
@@ -168,6 +168,7 @@ DynamoStream.prototype.getRecords = async function* getRecords(
         shardId: always(Shard.ShardId),
       }))
     }
+    await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
   }
 }
 

package/KinesisStream.js

@@ -29,11 +29,6 @@ const {
  *   endpoint: string,
  *   shardIteratorType: 'AT_SEQUENCE_NUMBER'|'AFTER_SEQUENCE_NUMBER'|'TRIM_HORIZON'|'LATEST'|'AT_TIMESTAMP',
  *   timestamp: Date|string|number, // find events at date (requires shardIteratorType 'AT_TIMESTAMP')
- *   startingSequenceNumber: string, // find events at data record (requires shardIteratorType 'AT_SEQUENCE_NUMBER' or 'AFTER_SEQUENCE_NUMBER')
- *   shardFilterType: 'AFTER_SHARD_ID'|'AT_TRIM_HORIZON'|'FROM_TRIM_HORIZON'|'AT_LATEST'|'AT_TIMESTAMP'|'FROM_TIMESTAMP',
- *   shardFilterShardId: string,
- *   shardFilterTimestamp: Date|string|number,
- *   streamCreationTimestamp: Date|string|number, // distinguishes streams of same name e.g. after deleting
  * }) -> KinesisStream
  * ```
  *
@@ -53,13 +48,8 @@ const KinesisStream = function (options) {
   this.shardUpdatePeriod = options.shardUpdatePeriod ?? 15000
   this.getRecordsInterval = options.getRecordsInterval ?? 1000
   this.shardIteratorType = options.shardIteratorType ?? 'LATEST'
-  this.shardIteratorTimestamp = options.shardIteratorTimestamp
-  this.shardFilterType = options.shardFilterType
-  this.shardFilterShardId = options.shardFilterShardId
-  this.shardFilterTimestamp = options.shardFilterTimestamp
-  this.streamCreationTimestamp = options.streamCreationTimestamp
+  this.shardCount = options.shardCount ?? 1
   this.timestamp = options.timestamp
-  this.startingSequenceNumber = options.startingSequenceNumber
   this.kinesis = new Kinesis(omit(['name'])(options))
   this.cancelToken = new Promise((_, reject) => (this.canceller = reject))
 
@@ -76,7 +66,7 @@ const KinesisStream = function (options) {
   }).catch(async () => {
     await this.kinesis.client.createStream({
       StreamName: this.name,
-      ShardCount: 1,
+      ShardCount: this.shardCount,
     }).promise()
     await this.kinesis.client.waitFor('streamExists', {
       StreamName: this.name
@@ -103,7 +93,7 @@ KinesisStream.prototype.delete = function deleteStream() {
  * @synopsis
  * ```coffeescript [specscript]
  * KinesisStream(options).putRecord(
- *   data string|binary,
+ *   data string|Buffer,
  *   options {
  *     partitionKey: string, // input to aws hash function to determine which shard
  *     explicitHashKey: string, // skips aws hash function to determine which shard
@@ -129,6 +119,57 @@ KinesisStream.prototype.putRecord = async function putRecord(data, options = {})
   }).promise()
 }
 
+/**
+ * @name KinesisStream.prototype.putRecords
+ *
+ * @synopsis
+ * ```coffeescript [specscript]
+ * KinesisStream(options).putRecords(records Array<{
+ *   data: string|Buffer,
+ *   partitionKey: string, // input to aws hash function to determine which shard
+ *   explicitHashKey: string, // skips aws hash function to determine which shard
+ * }>) -> Promise<{
+ *   FailedRecordCount: number, // number of unsuccessfully processed records
+ *   Records: Array<{
+ *     SequenceNumber: string,
+ *     ShardId: string,
+ *     ErrorCode?: 'ProvisionedThroughputExceededException'|'InternalFailure',
+ *     ErrorMessage?: string,
+ *   }>,
+ *   EncryptionType: 'NONE'|'KMS',
+ * }>
+ * ```
+ *
+ * @description
+ * Limits/Quotas:
+ *  * 500 records max per request
+ *  * 1 MB per record max
+ *  * 5 MB per request max
+ *  * 1000 records per second per shard
+ *  * 1 MB per second per shard
+ *
+ * https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Kinesis.html#putRecords-property
+ */
+KinesisStream.prototype.putRecords = async function putRecords(records) {
+  return this.kinesis.client.putRecords({
+    StreamName: this.name,
+    Records: records.map(({ data, partitionKey, explicitHashKey }) => ({
+      Data: data,
+      PartitionKey: partitionKey ?? data.slice(0, 255),
+      ...explicitHashKey && { ExplicitHashKey: explicitHashKey },
+    }))
+  }).promise().then(tap(response => {
+    if (response.Records.some(has('ErrorCode'))) {
+      const errors = response.Records.filter(has('ErrorCode')).map(Record => {
+        const error = new Error(Record.ErrorMessage)
+        error.code = Record.ErrorCode
+        return error
+      })
+      throw new AggregateError(errors, 'Some records failed to process')
+    }
+  }))
+}
+
 // () => ()
 KinesisStream.prototype.close = function close() {
   this.closed = true
@@ -163,9 +204,7 @@ KinesisStream.prototype.getRecords = async function* getRecords(Shard) {
     ShardId: Shard.ShardId,
     StreamName: this.name,
     ShardIteratorType: this.shardIteratorType,
-    ...this.shardIteratorTimestamp && {
-      Timestamp: this.shardIteratorTimestamp,
-    },
+    ...this.timestamp == null ? {} : { Timestamp: this.timestamp },
   }).promise().then(get('ShardIterator'))
 
   let records = await this.kinesis.client.getRecords({
@@ -175,9 +214,9 @@ KinesisStream.prototype.getRecords = async function* getRecords(Shard) {
   if (records.Records.length > 0) {
     yield* records.Records
   }
+  await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
 
   while (!this.closed && records.NextShardIterator != null) {
-    await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
     records = await this.kinesis.client.getRecords({
       ShardIterator: records.NextShardIterator,
       Limit: this.getRecordsLimit,
@@ -185,6 +224,7 @@ KinesisStream.prototype.getRecords = async function* getRecords(Shard) {
     if (records.Records.length > 0) {
       yield* records.Records
     }
+    await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
   }
 }
 

package/KinesisStream.test.js

@@ -7,9 +7,11 @@ const map = require('rubico/map')
 const thunkify = require('rubico/thunkify')
 
 const test = new Test('KinesisStream', KinesisStream)
+
 .before(function () {
   this.streams = []
 })
+
 .case({
   name: 'my-stream',
   endpoint: 'http://localhost:4567',
@@ -21,7 +23,87 @@ const test = new Test('KinesisStream', KinesisStream)
   await myStream.putRecord('hey')
   await myStream.putRecord('ho', { partitionKey: 'ho' })
   await myStream.putRecord('hi', { explicitHashKey: '127' })
+  const first3 = await asyncIterableTake(3)(myStream)
+  const first3Again = await asyncIterableTake(3)(myStream)
+  assert.deepEqual(first3, first3Again)
+  this.streams.push(myStream)
+})
+
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'TRIM_HORIZON',
+  getRecordsLimit: 1,
+  listShardsLimit: 1,
+  shardCount: 2,
+}, async function (myStream) {
+  await myStream.ready
+  await myStream.putRecord('hey', { partitionKey: 'a' })
+  await myStream.putRecord('ho', { partitionKey: 'a' })
+  await myStream.putRecord('hi', { partitionKey: 'a' })
+  const first3 = await asyncIterableTake(3)(myStream)
+  const first3Again = await asyncIterableTake(3)(myStream)
+  assert.deepEqual(first3, first3Again)
+  this.streams.push(myStream)
+})
 
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'TRIM_HORIZON',
+}, async function (myStream) {
+  await myStream.ready
+  await myStream.putRecords([
+    { data: 'hey' },
+    { data: 'ho', partitionKey: 'ho' },
+    { data: 'hi', explicitHashKey: '127' },
+  ])
+  const first3 = await asyncIterableTake(3)(myStream)
+  const first3Again = await asyncIterableTake(3)(myStream)
+  assert.deepEqual(first3, first3Again)
+  this.streams.push(myStream)
+})
+
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'TRIM_HORIZON',
+}, async function (myStream) {
+  myStream.kinesis.client.putRecords = () => ({
+    promise: async () => ({
+      Records: [
+        {
+          ErrorCode: 'ProvisionedThroughputExceededException',
+          ErrorMessage: 'Some message with accountId, stream name, and shard ID',
+        },
+        {
+          ErrorCode: 'ProvisionedThroughputExceededException',
+          ErrorMessage: 'Some message with accountId, stream name, and shard ID',
+        },
+      ],
+    }),
+  })
+  await myStream.ready
+  await assert.rejects(
+    () => myStream.putRecords([{ data: 'hey' }]),
+    new AggregateError([
+      new Error('Some message with accountId, stream name, and shard ID'),
+      new Error('Some message with accountId, stream name, and shard ID'),
+    ], 'Some records failed to process')
+  )
+  this.streams.push(myStream)
+})
+
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardIteratorType: 'AT_TIMESTAMP',
+  timestamp: new Date(Date.now() - 5000),
+}, async function (myStream) {
+  await myStream.ready
+  await myStream.putRecord('hey')
+  await myStream.putRecord('ho', { partitionKey: 'ho' })
+  await myStream.putRecord('hi', { explicitHashKey: '127' })
   const first3 = await asyncIterableTake(3)(myStream)
   const first3Again = await asyncIterableTake(3)(myStream)
   assert.deepEqual(first3, first3Again)
@@ -69,8 +151,11 @@ const test = new Test('KinesisStream', KinesisStream)
   await new Promise(resolve => setTimeout(resolve, 1000))
 })
 
-.after(async function() {
-  await map(stream => stream.delete())(this.streams)
+.after(async function () {
+  await map(async function cleanup(stream) {
+    stream.close()
+    await stream.delete()
+  })(this.streams)
 })
 
 if (process.argv[1] == __filename) {

package/TranscribeStream.js

@@ -177,6 +177,10 @@ TranscribeStream.prototype.sendAudioChunk = function (chunk) {
   this.websocket.send(bytes)
 }
 
+TranscribeStream.prototype.close = function () {
+  this.websocket.close()
+}
+
 /**
  * @name marshalHeaders
  *

package/TranscribeStream.test.js

@@ -53,7 +53,7 @@ const test = new Test('TranscribeStream', async function () {
       wav.fromMuLaw()
       testTranscribeStream.sendAudioChunk(Buffer.from(wav.data.samples))
     } else if (event.event == 'stop') {
-      testTranscribeStream.websocket.close()
+      testTranscribeStream.close()
     }
   })
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "presidium",
-  "version": "0.15.27",
+  "version": "0.15.31",
   "description": "A library for creating web services",
   "author": "Richard Tong",
   "license": "MIT",