presidium 0.15.18 → 0.15.23

package/DynamoStream.js

@@ -58,9 +58,8 @@ const DynamoStream = function (options) {
   this.getRecordsLimit = options.getRecordsLimit ?? 1000
   this.getRecordsInterval = options.getRecordsInterval ?? 1000
   this.shardIteratorType = options.shardIteratorType ?? 'LATEST'
-  this.shardUpdatePeriod = options.shardUpdatePeriod ?? 30000
+  this.shardUpdatePeriod = options.shardUpdatePeriod ?? 15000
   this.listStreamsLimit = options.listStreamsLimit ?? 100
-  this.debug = options.debug ?? false
   this.client = new DynamoDBStreams({
     apiVersion: '2012-08-10',
     accessKeyId: 'id',
@@ -90,14 +89,18 @@ DynamoStream.prototype.getStreams = async function* getStreams() {
     Limit: this.listStreamsLimit,
     TableName: this.table
   }).promise()
-  yield* streams.Streams
+  if (streams.Streams.length > 0) {
+    yield* streams.Streams
+  }
   while (!this.closed && streams.LastEvaluatedStreamArn != null) {
     streams = await this.client.listStreams({
       Limit: this.listStreamsLimit,
       TableName: this.table,
       ExclusiveStartStreamArn: streams.LastEvaluatedStreamArn,
     }).promise()
-    yield* streams.Streams
+    if (streams.Streams.length > 0) {
+      yield* streams.Streams
+    }
   }
 }
 
@@ -109,16 +112,18 @@ DynamoStream.prototype.getShards = async function* getShards(
     StreamArn: Stream.StreamArn,
     Limit: 100,
   }).promise().then(get('StreamDescription'))
-  yield* shards.Shards.map(
-    (Shard, ShardNumber) => ({ ...Shard, Stream, ShardNumber }))
+  if (shards.Shards.length > 0) {
+    yield* shards.Shards.map(assign({ Stream: always(Stream) }))
+  }
   while (!this.closed && shards.LastEvaluatedShardId != null) {
     shards = await this.client.describeStream({
       StreamArn: Stream.StreamArn,
       Limit: 100,
       ExclusiveStartShardId: shards.LastEvaluatedShardId,
     }).promise().then(get('StreamDescription'))
-    yield* shards.Shards.map(
-      (Shard, ShardNumber) => ({ ...Shard, Stream, ShardNumber }))
+    if (shards.Shards.length > 0) {
+      yield* shards.Shards.map(assign({ Stream: always(Stream) }))
+    }
   }
 }
 
@@ -144,13 +149,23 @@ DynamoStream.prototype.getRecords = async function* getRecords(
     Limit: this.getRecordsLimit
   }).promise()
 
-  yield* records.Records
+  if (records.Records.length > 0) {
+    yield* records.Records.map(assign({
+      table: always(this.table),
+      shardId: always(Shard.ShardId),
+    }))
+  }
   while (!this.closed && records.NextShardIterator != null) {
     records = await this.client.getRecords({
       ShardIterator: records.NextShardIterator,
       Limit: this.getRecordsLimit
     }).promise()
-    yield* records.Records
+    if (records.Records.length > 0) {
+      yield* records.Records.map(assign({
+        table: always(this.table),
+        shardId: always(Shard.ShardId),
+      }))
+    }
     await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
   }
 }
@@ -166,21 +181,21 @@ DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
     })),
     transform(map(identity), []),
   ])()
-  let muxAsyncIterator = Mux.race(shards.map(Shard => this.getRecords(Shard)))
-  let iterationPromise = muxAsyncIterator.next()
-  let shardUpdatePromise = new Promise(resolve => setTimeout(
-    thunkify(resolve, SymbolUpdateShards), this.shardUpdatePeriod))
-
-  if (this.debug) {
-    console.log('Starting shards:', shards.map(get('ShardId')))
-  }
+  let muxAsyncIterator = Mux.race([
+    ...shards.map(Shard => this.getRecords(Shard)),
+    (async function* UpdateShardsGenerator() {
+      while (true) {
+        await new Promise(resolve => {
+          setTimeout(resolve, this.shardUpdatePeriod)
+        })
+        yield SymbolUpdateShards
+      }
+    }).call(this),
+  ])
 
   while (!this.closed) {
-    const iteration = await Promise.race([
-      shardUpdatePromise,
-      iterationPromise,
-    ])
-    if (iteration == SymbolUpdateShards) {
+    const { value, done } = await muxAsyncIterator.next()
+    if (value == SymbolUpdateShards) {
       const latestShards = await pipe([
         always(this.getStreams()),
         flatMap(Stream => this.getShards(Stream)),
@@ -197,23 +212,17 @@ DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
         })),
       ])()
 
-      if (this.debug) {
-        console.log('Latest shards:', latestShards.map(get('ShardId')))
-      }
-
       shards = latestShards
-      muxAsyncIterator = newShards.length == 0 ? muxAsyncIterator : Mux.race([
-        ...newShards.map(Shard => this.getRecords(Shard)),
-        muxAsyncIterator,
-      ])
-      shardUpdatePromise = new Promise(resolve => setTimeout(
-        thunkify(resolve, SymbolUpdateShards), this.shardUpdatePeriod))
-    } else if (iteration.done) {
+      if (newShards.length > 0) {
+        muxAsyncIterator = Mux.race([
+          ...newShards.map(Shard => this.getRecords(Shard)),
+          muxAsyncIterator,
+        ])
+      }
+    } else if (done) {
       await new Promise(resolve => setTimeout(resolve, 1000))
-      iterationPromise = muxAsyncIterator.next()
     } else {
-      yield iteration.value
-      iterationPromise = muxAsyncIterator.next()
+      yield value
     }
   }
 }

package/KinesisStream.js

@@ -50,7 +50,7 @@ const KinesisStream = function (options) {
   this.name = options.name
   this.listShardsLimit = options.listShardsLimit ?? 1000
   this.getRecordsLimit = options.getRecordsLimit ?? 1000
-  this.shardUpdateInterval = options.shardUpdateInterval ?? 10000
+  this.shardUpdatePeriod = options.shardUpdatePeriod ?? 15000
   this.getRecordsInterval = options.getRecordsInterval ?? 1000
   this.shardIteratorType = options.shardIteratorType ?? 'LATEST'
   this.shardIteratorTimestamp = options.shardIteratorTimestamp
@@ -183,43 +183,44 @@ const SymbolUpdateShards = Symbol('UpdateShards')
 
 KinesisStream.prototype[Symbol.asyncIterator] = async function* generateRecords() {
   let shards = await transform(map(identity), [])(this.listShards())
-  let muxAsyncIterator = Mux.race(shards.map(Shard => this.getRecords(Shard)))
-  let iterationPromise = muxAsyncIterator.next()
-  let shardUpdatePromise = new Promise(resolve => setTimeout(
-    thunkify(resolve, SymbolUpdateShards),
-    this.shardUpdateInterval,
-  ))
+  let muxAsyncIterator = Mux.race([
+    ...shards.map(Shard => this.getRecords(Shard)),
+    (async function* UpdateShardsGenerator() {
+      while (true) {
+        await new Promise(resolve => {
+          setTimeout(resolve, this.shardUpdatePeriod)
+        })
+        yield SymbolUpdateShards
+      }
+    }).call(this),
+  ])
 
   while (!this.closed) {
-    const iteration = await Promise.race([
-      shardUpdatePromise,
-      iterationPromise,
-    ])
-    if (iteration == SymbolUpdateShards) {
+    const { value, done } = await muxAsyncIterator.next()
+    if (value == SymbolUpdateShards) {
       const latestShards = await transform(map(identity), [])(this.listShards())
-      const newShards = differenceWith(
-        (ShardA, ShardB) => ShardA.ShardId == ShardB.ShardId,
-        latestShards,
-      )(shards)
-      const closedShards = differenceWith(
-        (ShardA, ShardB) => ShardA.ShardId == ShardB.ShardId,
-        shards,
-      )(latestShards)
+      const newShards = pipe([
+        always(shards),
+        differenceWith(
+          (ShardA, ShardB) => ShardA.ShardId == ShardB.ShardId,
+          latestShards,
+        ),
+        map(assign({
+          ShardIteratorType: always('TRIM_HORIZON'),
+        })),
+      ])()
 
-      closedShards.forEach(Shard => (Shard.closed = true))
       shards = latestShards
-      muxAsyncIterator = newShards.length == 0 ? muxAsyncIterator : Mux.race([
-        ...newShards.map(Shard => this.getRecords(Shard)),
-        muxAsyncIterator,
-      ])
-      shardUpdatePromise = new Promise(resolve => setTimeout(
-        thunkify(resolve, SymbolUpdateShards), this.shardUpdateInterval))
-    } else if (iteration.done) {
+      if (newShards.length > 0) {
+        muxAsyncIterator = Mux.race([
+          ...newShards.map(Shard => this.getRecords(Shard)),
+          muxAsyncIterator,
+        ])
+      }
+    } else if (done) {
       await new Promise(resolve => setTimeout(resolve, 1000))
-      iterationPromise = muxAsyncIterator.next()
     } else {
-      yield iteration.value
-      iterationPromise = muxAsyncIterator.next()
+      yield value
     }
   }
 }

package/KinesisStream.test.js

@@ -27,6 +27,7 @@ const test = new Test('KinesisStream', KinesisStream)
   assert.deepEqual(first3, first3Again)
   this.streams.push(myStream)
 })
+
 .case({
   name: 'my-stream',
   endpoint: 'http://localhost:4567',
@@ -46,6 +47,26 @@ const test = new Test('KinesisStream', KinesisStream)
   myStream.close()
   this.streams.push(myStream)
 })
+
+.case({
+  name: 'my-stream',
+  endpoint: 'http://localhost:4567',
+  shardUpdatePeriod: 500,
+}, async function (myStream) {
+  await myStream.ready
+
+  // there shouldn't be any more records, so this should hang
+  const latestRecordPromise = asyncIterableTake(1)(myStream)
+  const raceResult = await Promise.race([
+    latestRecordPromise,
+    new Promise(resolve => setTimeout(thunkify(resolve, 'hey'), 3000))
+  ])
+  assert.equal(raceResult, 'hey')
+  // wait a second for shard update
+  await new Promise(resolve => setTimeout(thunkify(resolve, 'hey'), 1000))
+  myStream.close()
+})
+
 .after(async function() {
   await map(stream => stream.delete())(this.streams)
 })

package/{TranscribeStreaming.js → TranscribeStream.js}

@@ -29,11 +29,11 @@ const CHECKSUM_LENGTH = 4
 const MINIMUM_MESSAGE_LENGTH = PRELUDE_LENGTH + CHECKSUM_LENGTH * 2
 
 /**
- * @name TranscribeStreaming
+ * @name TranscribeStream
  *
  * @synopsis
  * ```coffeescript [specscript]
- * const myTranscribeStream = new TranscribeStreaming(options {
+ * const myTranscribeStream = new TranscribeStream(options {
  *   accessKeyId: string,
  *   secretAccessKey: string,
  *   region: string,
@@ -44,6 +44,24 @@ const MINIMUM_MESSAGE_LENGTH = PRELUDE_LENGTH + CHECKSUM_LENGTH * 2
  *   sessionId?: string,
  *   vocabularyName?: string,
  * })
+ *
+ * myTranscribeStream.on('transcription', transcriptionHandler (transcription {
+ *   Alternatives: Array<{
+ *     Items: Array<{
+ *       Confidence?: number,
+ *       Content: string,
+ *       EndTime: number, // seconds
+ *       StartTime: number, // seconds
+ *       Type: 'pronunciation'|'punctuation',
+ *       VocabularyFilterMatch: boolean,
+ *     }>,
+ *     Transcript: string,
+ *   }>,
+ *   EndTime: number, // seconds
+ *   IsPartial: boolean,
+ *   ResultId: string, // uuid
+ *   StartTime: number, // seconds
+ * })=><>)
  * ```
  *
  * @description
@@ -59,7 +77,7 @@ const MINIMUM_MESSAGE_LENGTH = PRELUDE_LENGTH + CHECKSUM_LENGTH * 2
  *
  * `vocabularyName` - The name of the vocabulary to use when processing the transcription job, if any.
  */
-const TranscribeStreaming = function (options) {
+const TranscribeStream = function (options) {
   const {
     accessKeyId,
     secretAccessKey,
@@ -109,10 +127,10 @@ const TranscribeStreaming = function (options) {
   return this
 }
 
-TranscribeStreaming.prototype = EventEmitter.prototype
+TranscribeStream.prototype = EventEmitter.prototype
 
 /**
- * @name TranscribeStreaming.prototype.sendAudioChunk
+ * @name TranscribeStream.prototype.sendAudioChunk
  *
  * @synopsis
  * ```coffeescript [specscript]
@@ -125,7 +143,7 @@ TranscribeStreaming.prototype = EventEmitter.prototype
  * https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html
  * https://github.com/aws-samples/amazon-transcribe-comprehend-medical-twilio/blob/main/lib/transcribe-service.js
  */
-TranscribeStreaming.prototype.sendAudioChunk = function (chunk) {
+TranscribeStream.prototype.sendAudioChunk = function (chunk) {
   const headersBytes = marshalHeaders({
     ':message-type': {
       type: 'string',
@@ -273,4 +291,4 @@ const unmarshalHeaders = function (headersView) {
   return headers
 }
 
-module.exports = TranscribeStreaming
+module.exports = TranscribeStream

package/{TranscribeStreaming.test.js → TranscribeStream.test.js}

@@ -4,7 +4,7 @@ const WaveFile = require('wavefile').WaveFile
 const Test = require('thunk-test')
 const assert = require('assert')
 const rubico = require('rubico')
-const TranscribeStreaming = require('./TranscribeStreaming')
+const TranscribeStream = require('./TranscribeStream')
 const AwsCredentials = require('./internal/AwsCredentials')
 
 const {
@@ -18,7 +18,7 @@ const {
   curry, __,
 } = rubico
 
-const test = new Test('TranscribeStreaming', async function () {
+const test = new Test('TranscribeStream', async function () {
   const awsCreds = await AwsCredentials('default').catch(error => {
     if (error.code == 'ENOENT') {
       const accessKeyId = process.env.AWS_ACCESS_KEY_ID
@@ -32,13 +32,13 @@ const test = new Test('TranscribeStreaming', async function () {
   })
   awsCreds.region = 'us-east-1' // only valid region for transcribe
 
-  const testTranscribeStreaming = new TranscribeStreaming({
+  const testTranscribeStream = new TranscribeStream({
     languageCode: 'en-US',
     mediaEncoding: 'pcm',
     sampleRate: 8000,
     ...awsCreds,
   })
-  await testTranscribeStreaming.ready
+  await testTranscribeStream.ready
 
   const mediaStreamFixtureAwsKeynote =
     fs.createReadStream('./media-stream-fixture-aws-keynote.txt')
@@ -51,14 +51,14 @@ const test = new Test('TranscribeStreaming', async function () {
       const wav = new WaveFile()
       wav.fromScratch(1, 8000, '8', Buffer.from(event.media.payload, 'base64'))
       wav.fromMuLaw()
-      testTranscribeStreaming.sendAudioChunk(Buffer.from(wav.data.samples))
+      testTranscribeStream.sendAudioChunk(Buffer.from(wav.data.samples))
     } else if (event.event == 'stop') {
-      testTranscribeStreaming.websocket.close()
+      testTranscribeStream.websocket.close()
     }
   })
 
   const testTranscription = await new Promise(resolve => {
-    testTranscribeStreaming.on('transcription', transcription => {
+    testTranscribeStream.on('transcription', transcription => {
       resolve(transcription.Alternatives[0].Transcript)
     })
   })

package/index.js

@@ -31,24 +31,43 @@ const S3Bucket = require('./S3Bucket')
 // const Redshift = require('./Redshift')
 // const Gremlin = require('./Gremlin')
 const Redis = require('./Redis')
+const TranscribeStream = require('./TranscribeStream')
 
 const Presidium = {
-  Http, HttpServer,
-  WebSocket, WebSocketServer,
-  Dynamo, DynamoTable, DynamoIndex, DynamoStream,
-  Docker, DockerImage, DockerContainer, DockerSwarm, DockerService,
-  // ElasticTranscoder, ElasticTranscoderPipeline,
-  // KinesisAnalytics, KinesisAnalyticsStream,
-  // KinesisVideo, KinesisVideoStream,
-  Kinesis, KinesisStream,
-  Lambda, LambdaFunction,
-  Mongo, MongoCollection,
+  Http,
+  HttpServer,
+  WebSocket,
+  WebSocketServer,
+  Dynamo,
+  DynamoTable,
+  DynamoIndex,
+  DynamoStream,
+  Docker,
+  DockerImage,
+  DockerContainer,
+  DockerSwarm,
+  DockerService,
+  // ElasticTranscoder,
+  // ElasticTranscoderPipeline,
+  // KinesisAnalytics,
+  // KinesisAnalyticsStream,
+  // KinesisVideo,
+  // KinesisVideoStream,
+  Kinesis,
+  KinesisStream,
+  Lambda,
+  LambdaFunction,
+  Mongo,
+  MongoCollection,
   ElasticsearchIndex,
-  // SNS, SNSTopic,
-  S3, S3Bucket,
+  // SNS,
+  // SNSTopic,
+  S3,
+  S3Bucket,
   // Redshift,
   // Gremlin,
   Redis,
+  TranscribeStream,
 }
 
 module.exports = Presidium

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "presidium",
-  "version": "0.15.18",
+  "version": "0.15.23",
   "description": "A library for creating web services",
   "author": "Richard Tong",
   "license": "MIT",