presidium 0.15.16 → 0.15.20

package/DynamoStream.js

@@ -58,7 +58,7 @@ const DynamoStream = function (options) {
   this.getRecordsLimit = options.getRecordsLimit ?? 1000
   this.getRecordsInterval = options.getRecordsInterval ?? 1000
   this.shardIteratorType = options.shardIteratorType ?? 'LATEST'
-  this.shardUpdatePeriod = options.shardUpdatePeriod ?? 30000
+  this.shardUpdatePeriod = options.shardUpdatePeriod ?? 15000
   this.listStreamsLimit = options.listStreamsLimit ?? 100
   this.client = new DynamoDBStreams({
     apiVersion: '2012-08-10',
@@ -89,14 +89,18 @@ DynamoStream.prototype.getStreams = async function* getStreams() {
     Limit: this.listStreamsLimit,
     TableName: this.table
   }).promise()
-  yield* streams.Streams
+  if (streams.Streams.length > 0) {
+    yield* streams.Streams
+  }
   while (!this.closed && streams.LastEvaluatedStreamArn != null) {
     streams = await this.client.listStreams({
       Limit: this.listStreamsLimit,
       TableName: this.table,
       ExclusiveStartStreamArn: streams.LastEvaluatedStreamArn,
     }).promise()
-    yield* streams.Streams
+    if (streams.Streams.length > 0) {
+      yield* streams.Streams
+    }
   }
 }
 
@@ -108,16 +112,18 @@ DynamoStream.prototype.getShards = async function* getShards(
     StreamArn: Stream.StreamArn,
     Limit: 100,
   }).promise().then(get('StreamDescription'))
-  yield* shards.Shards.map(
-    (Shard, ShardNumber) => ({ ...Shard, Stream, ShardNumber }))
+  if (shards.Shards.length > 0) {
+    yield* shards.Shards.map(assign({ Stream: always(Stream) }))
+  }
   while (!this.closed && shards.LastEvaluatedShardId != null) {
     shards = await this.client.describeStream({
       StreamArn: Stream.StreamArn,
       Limit: 100,
       ExclusiveStartShardId: shards.LastEvaluatedShardId,
     }).promise().then(get('StreamDescription'))
-    yield* shards.Shards.map(
-      (Shard, ShardNumber) => ({ ...Shard, Stream, ShardNumber }))
+    if (shards.Shards.length > 0) {
+      yield* shards.Shards.map(assign({ Stream: always(Stream) }))
+    }
   }
 }
 
@@ -129,25 +135,37 @@ DynamoStream.prototype.getRecords = async function* getRecords(
     ShardId: Shard.ShardId,
     StreamArn: Shard.Stream.StreamArn,
     ShardIteratorType: Shard.ShardIteratorType,
+
+    /*
     ...(
-      this.shardIteratorType == 'AFTER_SEQUENCE_NUMBER'
-      || this.shardIteratorType == 'AT_SEQUENCE_NUMBER'
-    ) ? {
-      SequenceNumber: Shard.SequenceNumberRange.StartingSequenceNumber,
-    } : {},
+      Shard.ShardIteratorType == 'AFTER_SEQUENCE_NUMBER'
+      || Shard.ShardIteratorType == 'AT_SEQUENCE_NUMBER'
+    ) ? { SequenceNumber: Shard.SequenceNumber } : {},
+    */
+
   }).promise().then(get('ShardIterator'))
   let records = await this.client.getRecords({
     ShardIterator: startingShardIterator,
     Limit: this.getRecordsLimit
   }).promise()
 
-  yield* records.Records
+  if (records.Records.length > 0) {
+    yield* records.Records.map(assign({
+      table: always(this.table),
+      shardId: always(Shard.ShardId),
+    }))
+  }
   while (!this.closed && records.NextShardIterator != null) {
     records = await this.client.getRecords({
       ShardIterator: records.NextShardIterator,
       Limit: this.getRecordsLimit
     }).promise()
-    yield* records.Records
+    if (records.Records.length > 0) {
+      yield* records.Records.map(assign({
+        table: always(this.table),
+        shardId: always(Shard.ShardId),
+      }))
+    }
     await new Promise(resolve => setTimeout(resolve, this.getRecordsInterval))
   }
 }
@@ -156,46 +174,55 @@ const SymbolUpdateShards = Symbol('UpdateShards')
 
 DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
   let shards = await pipe([
-    transform(map(identity), []),
+    always(this.getStreams()),
     flatMap(Stream => this.getShards(Stream)),
-    map(assign({ ShardIteratorType: always(this.shardIteratorType) })),
-  ])(this.getStreams())
-  let muxAsyncIterator = Mux.race(shards.map(Shard => this.getRecords(Shard)))
-  let iterationPromise = muxAsyncIterator.next()
-  let shardUpdatePromise = new Promise(resolve => setTimeout(
-    thunkify(resolve, SymbolUpdateShards), this.shardUpdatePeriod))
+    map(assign({
+      ShardIteratorType: always(this.shardIteratorType),
+    })),
+    transform(map(identity), []),
+  ])()
+  let muxAsyncIterator = Mux.race([
+    ...shards.map(Shard => this.getRecords(Shard)),
+    (async function* UpdateShardsGenerator() {
+      while (true) {
+        await new Promise(resolve => {
+          setTimeout(resolve, this.shardUpdatePeriod)
+        })
+        yield SymbolUpdateShards
+      }
+    }).call(this),
+  ])
 
   while (!this.closed) {
-    const iteration = await Promise.race([
-      shardUpdatePromise,
-      iterationPromise,
-    ])
-    if (iteration == SymbolUpdateShards) {
+    const { value, done } = await muxAsyncIterator.next()
+    if (value == SymbolUpdateShards) {
       const latestShards = await pipe([
-        transform(map(identity), []),
+        always(this.getStreams()),
         flatMap(Stream => this.getShards(Stream)),
-      ])(this.getStreams())
+        transform(map(identity), []),
+      ])()
       const newShards = pipe([
+        always(shards),
         differenceWith(
           (ShardA, ShardB) => ShardA.ShardId == ShardB.ShardId,
           latestShards,
         ),
-        map(assign({ ShardIteratorType: always('TRIM_HORIZON') })),
-      ])(shards)
+        map(assign({
+          ShardIteratorType: always('TRIM_HORIZON'),
+        })),
+      ])()
 
       shards = latestShards
-      muxAsyncIterator = newShards.length == 0 ? muxAsyncIterator : Mux.race([
-        ...newShards.map(Shard => this.getRecords(Shard)),
-        muxAsyncIterator,
-      ])
-      shardUpdatePromise = new Promise(resolve => setTimeout(
-        thunkify(resolve, SymbolUpdateShards), this.shardUpdatePeriod))
-    } else if (iteration.done) {
+      if (newShards.length > 0) {
+        muxAsyncIterator = Mux.race([
+          ...newShards.map(Shard => this.getRecords(Shard)),
+          muxAsyncIterator,
+        ])
+      }
+    } else if (done) {
       await new Promise(resolve => setTimeout(resolve, 1000))
-      iterationPromise = muxAsyncIterator.next()
     } else {
-      yield iteration.value
-      iterationPromise = muxAsyncIterator.next()
+      yield value
     }
   }
 }

package/DynamoStream.test.js

@@ -223,52 +223,6 @@ const test = Test('DynamoStream', DynamoStream)
   myStream.close()
 })
 
-.case({
-  table: 'my-table',
-  endpoint: 'http://localhost:8000',
-  listStreamsLimit: 1,
-  shardIteratorType: 'AFTER_SEQUENCE_NUMBER',
-  debug: true,
-}, async function (myStream) {
-  await myStream.ready
-
-  const table = this.table
-  await table.putItem({
-    id: '1',
-    status: 'waitlist',
-    createTime: 1000,
-    name: 'George',
-  })
-  await table.putItem({
-    id: '2',
-    status: 'waitlist',
-    createTime: 1001,
-    name: 'geo',
-  })
-  await table.putItem({
-    id: '3',
-    status: 'waitlist',
-    createTime: 1002,
-    name: 'john',
-  })
-  await table.putItem({
-    id: '4',
-    status: 'approved',
-    createTime: 1003,
-    name: 'sally',
-  })
-  await table.putItem({
-    id: '5',
-    status: 'approved',
-    createTime: 1004,
-    name: 'sally',
-  })
-
-  const first5 = await asyncIterableTake(5)(myStream)
-  assert.strictEqual(first5.length, 5)
-  myStream.close()
-})
-
 .case({
   table: 'my-table',
   endpoint: 'http://localhost:8000',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "presidium",
-  "version": "0.15.16",
+  "version": "0.15.20",
   "description": "A library for creating web services",
   "author": "Richard Tong",
   "license": "MIT",