presidium 0.15.12 → 0.15.18

package/.eslintrc.js

@@ -193,7 +193,7 @@ module.exports = {
                 'allowForLoopAfterthoughts': true
             }
         ],
-        'no-process-env': 'error',
+        'no-process-env': 'off',
         'no-process-exit': 'error',
         'no-promise-executor-return': 'off',
         'no-proto': 'error',

package/.github/workflows/nodejs.yml

@@ -38,3 +38,6 @@ jobs:
       with:
         node-version: ${{ matrix.node-version }}
     - run: npm i && npx nyc --reporter=lcovonly npm test && npx codecov --token=3586c688-30cd-46be-a6c4-79a6e0f7fe80 --file=coverage/lcov.info && npm run lint
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

package/DynamoStream.js

@@ -60,6 +60,7 @@ const DynamoStream = function (options) {
   this.shardIteratorType = options.shardIteratorType ?? 'LATEST'
   this.shardUpdatePeriod = options.shardUpdatePeriod ?? 30000
   this.listStreamsLimit = options.listStreamsLimit ?? 100
+  this.debug = options.debug ?? false
   this.client = new DynamoDBStreams({
     apiVersion: '2012-08-10',
     accessKeyId: 'id',
@@ -129,12 +130,14 @@ DynamoStream.prototype.getRecords = async function* getRecords(
     ShardId: Shard.ShardId,
     StreamArn: Shard.Stream.StreamArn,
     ShardIteratorType: Shard.ShardIteratorType,
+
+    /*
     ...(
-      this.shardIteratorType == 'AFTER_SEQUENCE_NUMBER'
-      || this.shardIteratorType == 'AT_SEQUENCE_NUMBER'
-    ) ? {
-      SequenceNumber: Shard.SequenceNumberRange.StartingSequenceNumber,
-    } : {},
+      Shard.ShardIteratorType == 'AFTER_SEQUENCE_NUMBER'
+      || Shard.ShardIteratorType == 'AT_SEQUENCE_NUMBER'
+    ) ? { SequenceNumber: Shard.SequenceNumber } : {},
+    */
+
   }).promise().then(get('ShardIterator'))
   let records = await this.client.getRecords({
     ShardIterator: startingShardIterator,
@@ -156,15 +159,22 @@ const SymbolUpdateShards = Symbol('UpdateShards')
 
 DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
   let shards = await pipe([
-    transform(map(identity), []),
+    always(this.getStreams()),
     flatMap(Stream => this.getShards(Stream)),
-    map(assign({ ShardIteratorType: always(this.shardIteratorType) })),
-  ])(this.getStreams())
+    map(assign({
+      ShardIteratorType: always(this.shardIteratorType),
+    })),
+    transform(map(identity), []),
+  ])()
   let muxAsyncIterator = Mux.race(shards.map(Shard => this.getRecords(Shard)))
   let iterationPromise = muxAsyncIterator.next()
   let shardUpdatePromise = new Promise(resolve => setTimeout(
     thunkify(resolve, SymbolUpdateShards), this.shardUpdatePeriod))
 
+  if (this.debug) {
+    console.log('Starting shards:', shards.map(get('ShardId')))
+  }
+
   while (!this.closed) {
     const iteration = await Promise.race([
       shardUpdatePromise,
@@ -172,16 +182,24 @@ DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
     ])
     if (iteration == SymbolUpdateShards) {
       const latestShards = await pipe([
-        transform(map(identity), []),
+        always(this.getStreams()),
         flatMap(Stream => this.getShards(Stream)),
-      ])(this.getStreams())
+        transform(map(identity), []),
+      ])()
       const newShards = pipe([
+        always(shards),
         differenceWith(
           (ShardA, ShardB) => ShardA.ShardId == ShardB.ShardId,
           latestShards,
         ),
-        map(assign({ ShardIteratorType: always('TRIM_HORIZON') })),
-      ])(shards)
+        map(assign({
+          ShardIteratorType: always('TRIM_HORIZON'),
+        })),
+      ])()
+
+      if (this.debug) {
+        console.log('Latest shards:', latestShards.map(get('ShardId')))
+      }
 
       shards = latestShards
       muxAsyncIterator = newShards.length == 0 ? muxAsyncIterator : Mux.race([

package/DynamoStream.test.js

@@ -226,52 +226,7 @@ const test = Test('DynamoStream', DynamoStream)
 .case({
   table: 'my-table',
   endpoint: 'http://localhost:8000',
-  listStreamsLimit: 1,
-  shardIteratorType: 'AFTER_SEQUENCE_NUMBER',
-  debug: true,
-}, async function (myStream) {
-  await myStream.ready
-
-  const table = this.table
-  await table.putItem({
-    id: '1',
-    status: 'waitlist',
-    createTime: 1000,
-    name: 'George',
-  })
-  await table.putItem({
-    id: '2',
-    status: 'waitlist',
-    createTime: 1001,
-    name: 'geo',
-  })
-  await table.putItem({
-    id: '3',
-    status: 'waitlist',
-    createTime: 1002,
-    name: 'john',
-  })
-  await table.putItem({
-    id: '4',
-    status: 'approved',
-    createTime: 1003,
-    name: 'sally',
-  })
-  await table.putItem({
-    id: '5',
-    status: 'approved',
-    createTime: 1004,
-    name: 'sally',
-  })
-
-  const first5 = await asyncIterableTake(5)(myStream)
-  assert.strictEqual(first5.length, 5)
-  myStream.close()
-})
-
-.case({
-  table: 'my-table',
-  endpoint: 'http://localhost:8000',
+  shardUpdatePeriod: 500,
 }, async function (myStream) {
   await myStream.ready
 
@@ -282,6 +237,8 @@ const test = Test('DynamoStream', DynamoStream)
     new Promise(resolve => setTimeout(thunkify(resolve, 'hey'), 3000))
   ])
   assert.equal(raceResult, 'hey')
+  // wait a second for shard update
+  await new Promise(resolve => setTimeout(thunkify(resolve, 'hey'), 1000))
   myStream.close()
 })
 

package/DynamoTable.js

@@ -144,7 +144,10 @@ DynamoTable.prototype.putItem = async function dynamoTablePutItem(item, options)
     TableName: this.name,
     Item: map(Dynamo.AttributeValue)(item),
     ...options,
-  }).promise()
+  }).promise().catch(error => {
+    error.tableName = this.name
+    throw error
+  })
 }
 
 /**
@@ -162,9 +165,14 @@ DynamoTable.prototype.getItem = async function dynamoTableGetItem(key) {
     Key: map(Dynamo.AttributeValue)(key),
   }).promise().then(result => {
     if (result.Item == null) {
-      throw new Error(`Item not found for ${stringifyJSON(key)}`)
+      const error = new Error(`Item not found for ${stringifyJSON(key)}`)
+      error.tableName = this.name
+      throw error
     }
     return result
+  }).catch(error => {
+    error.tableName = this.name
+    throw error
   })
 }
 
@@ -223,7 +231,10 @@ DynamoTable.prototype.updateItem = async function dynamoTableUpdateItem(
       ([key, value]) => [`:${hashJSON(value)}`, Dynamo.AttributeValue(value)],
     )(updates),
     ...options,
-  }).promise()
+  }).promise().catch(error => {
+    error.tableName = this.name
+    throw error
+  })
 }
 
 /**
@@ -263,7 +274,10 @@ DynamoTable.prototype.incrementItem = async function incrementItem(
       ([key, value]) => [`:${hashJSON(value)}`, Dynamo.AttributeValue(value)],
     )(incrementUpdates),
     ...options,
-  }).promise()
+  }).promise().catch(error => {
+    error.tableName = this.name
+    throw error
+  })
 }
 
 /**
@@ -287,7 +301,10 @@ DynamoTable.prototype.deleteItem = async function dynamoTableDeleteItem(key, opt
     TableName: this.name,
     Key: map(Dynamo.AttributeValue)(key),
     ...options,
-  }).promise()
+  }).promise().catch(error => {
+    error.tableName = this.name
+    throw error
+  })
 }
 
 /**
@@ -298,6 +315,7 @@ DynamoTable.prototype.deleteItem = async function dynamoTableDeleteItem(key, opt
  * DynamoTable(options).scan(options {
  *   limit: number,
  *   exclusiveStartKey: Object<string=>DynamoAttributeValue>
+ *   forceTableName?: string, // a test parameter
  * }) -> Promise<{
  *   Items: Array<Object<string=>DynamoAttributeValue>>
  *   Count: number, // number of Items
@@ -307,14 +325,18 @@ DynamoTable.prototype.deleteItem = async function dynamoTableDeleteItem(key, opt
  * ```
  */
 DynamoTable.prototype.scan = async function scan(options = {}) {
+  const { forceTableName } = options
   await this.ready
   return this.client.scan({
-    TableName: this.name,
+    TableName: forceTableName ?? this.name,
     Limit: options.limit ?? 100,
     ...options.exclusiveStartKey && {
       ExclusiveStartKey: options.exclusiveStartKey,
     },
-  }).promise()
+  }).promise().catch(error => {
+    error.tableName = this.name
+    throw error
+  })
 }
 
 module.exports = DynamoTable

package/DynamoTable.test.js

@@ -16,13 +16,37 @@ const test = new Test('DynamoTable', DynamoTable)
     endpoint: 'http://localhost:8000/',
     key: [{ id: 'string' }],
   }, async function (testTable) {
+    await testTable.ready
+    // if we created another instance of testTable it shouldn't have to create now
+    await new DynamoTable({
+      name: 'test-tablename',
+      endpoint: 'http://localhost:8000/',
+      key: [{ id: 'string' }],
+    }).ready
+
     // .case('http://localhost:8000/', 'test-tablename', async function (testTable) {
     await testTable.putItem({ id: '1', name: 'george' })
     await testTable.putItem({ id: '2', name: 'henry' })
     await testTable.putItem({ id: '3', name: 'jude' })
+    assert.rejects(
+      testTable.putItem({ somekey: 'hey' }),
+      {
+        name: 'ValidationException',
+        message: 'One of the required keys was not given a value',
+        tableName: 'test-tablename',
+      },
+    )
     assert.deepEqual(
       await testTable.getItem({ id: '1' }),
       { Item: map(Dynamo.AttributeValue)({ id: '1', name: 'george' }) })
+    assert.rejects(
+      testTable.getItem({ id: 'not-exists' }),
+      {
+        name: 'Error',
+        message: 'Item not found for {"id":"not-exists"}',
+        tableName: 'test-tablename',
+      },
+    )
     assert.deepEqual(
       await testTable.putItem({ id: '1', name: 'george' }, {
         ReturnValues: 'ALL_OLD',
@@ -51,6 +75,17 @@ const test = new Test('DynamoTable', DynamoTable)
       },
     )
 
+    assert.rejects(
+      testTable.updateItem({ id: 'not-exists' }, { a: 1 }, {
+        ConditionExpression: 'attribute_exists(id)',
+      }),
+      {
+        name: 'ConditionalCheckFailedException',
+        message: 'The conditional request failed',
+        tableName: 'test-tablename',
+      }
+    )
+
     assert.deepEqual(
       await testTable.getItem({ id: '1' }),
       {
@@ -99,20 +134,51 @@ const test = new Test('DynamoTable', DynamoTable)
       ),
       {
         name: 'ValidationException',
-        message: 'An operand in the update expression has an incorrect data type'
+        message: 'An operand in the update expression has an incorrect data type',
+        tableName: 'test-tablename',
       },
     )
 
+    assert.rejects(
+      testTable.incrementItem({ id: 'not-exists' }, { a: 1 }, {
+        ConditionExpression: 'attribute_exists(id)',
+      }),
+      {
+        name: 'ConditionalCheckFailedException',
+        message: 'The conditional request failed',
+        tableName: 'test-tablename',
+      }
+    )
+
     {
       const scanResult1 = await testTable.scan({ limit: 1 })
       const scanResult2 = await testTable.scan({ limit: 2, exclusiveStartKey: scanResult1.LastEvaluatedKey })
-      const scanResult3 = await testTable.scan({ limit: 2, exclusiveStartKey: scanResult2.LastEvaluatedKey })
+      const scanResult3 = await testTable.scan({ exclusiveStartKey: scanResult2.LastEvaluatedKey })
+      const bareScanResult = await testTable.scan()
       assert.strictEqual(scanResult1.Items.length, 1)
       assert.strictEqual(scanResult2.Items.length, 2)
       assert.strictEqual(scanResult3.Items.length, 0)
+      assert.strictEqual(bareScanResult.Items.length, 3)
+
+      assert.rejects(
+        testTable.scan({ forceTableName: 'nonexistent-table-name' }),
+        {
+          name: 'ResourceNotFoundException',
+          message: 'Cannot do operations on a non-existent table',
+          tableName: 'test-tablename',
+        },
+      )
     }
 
     await testTable.deleteItem({ id: '1' })
+    assert.rejects(
+      testTable.deleteItem({ somekey: 'a' }),
+      {
+        name: 'ValidationException',
+        message: 'One of the required keys was not given a value',
+        tableName: 'test-tablename',
+      }
+    )
     const shouldReject = testTable.getItem({ id: '1' })
     assert.rejects(
       () => shouldReject,

package/TranscribeStreaming.js

@@ -0,0 +1,276 @@
+const EventEmitter = require('events')
+const rubico = require('rubico')
+const WebSocket = require('./WebSocket')
+const sha256 = require('./internal/sha256')
+const AwsPresignedUrlV4 = require('./internal/AwsPresignedUrlV4')
+const Crc32 = require('./internal/Crc32')
+
+const {
+  pipe, tap,
+  switchCase, tryCatch,
+  fork, assign, get, pick, omit,
+  map, filter, reduce, transform, flatMap,
+  and, or, not, any, all,
+  eq, gt, lt, gte, lte,
+  thunkify, always,
+  curry, __,
+} = rubico
+
+// All prelude components are unsigned, 32-bit integers
+const PRELUDE_MEMBER_LENGTH = 4
+
+// The prelude consists of two components
+const PRELUDE_LENGTH = PRELUDE_MEMBER_LENGTH * 2
+
+// Checksums are always CRC32 hashes.
+const CHECKSUM_LENGTH = 4
+
+// Messages must include a full prelude, a prelude checksum, and a message checksum
+const MINIMUM_MESSAGE_LENGTH = PRELUDE_LENGTH + CHECKSUM_LENGTH * 2
+
+/**
+ * @name TranscribeStreaming
+ *
+ * @synopsis
+ * ```coffeescript [specscript]
+ * const myTranscribeStream = new TranscribeStreaming(options {
+ *   accessKeyId: string,
+ *   secretAccessKey: string,
+ *   region: string,
+ *   endpoint: string,
+ *   languageCode: string,
+ *   mediaEncoding: string,
+ *   sampleRate: number,
+ *   sessionId?: string,
+ *   vocabularyName?: string,
+ * })
+ * ```
+ *
+ * @description
+ * https://docs.aws.amazon.com/TranscribeStreaming/latest/dg/websocket.html
+ *
+ * `languageCode` - `en-AU`, `en-GB`, `en-US`, `es-US`, `fr-CA`, `fr-FR`, `de-DE`, `ja-JP`, `ko-KR`, `pt-BR`, `zh-CN` or `it-IT`.
+ *
+ * `mediaEncoding` - `pcm`, `ogg-opus`, or `flac`.
+ *
+ * `sampleRate` - The sample rate of the input audio in Hertz. We suggest that you use 8,000 Hz for low-quality audio and 16,000 Hz (or higher) for high-quality audio. The sample rate must match the sample rate in the audio file.
+ *
+ * `sessionId` - id for the transcription session. If you don't provide a session ID, Amazon Transcribe generates one for you and returns it in the response.
+ *
+ * `vocabularyName` - The name of the vocabulary to use when processing the transcription job, if any.
+ */
+const TranscribeStreaming = function (options) {
+  const {
+    accessKeyId,
+    secretAccessKey,
+    region,
+    languageCode,
+    mediaEncoding,
+    sampleRate,
+    sessionId,
+    vocabularyName,
+  } = options
+
+  const url = AwsPresignedUrlV4({
+    accessKeyId,
+    secretAccessKey,
+    region,
+    method: 'GET',
+    endpoint: `transcribestreaming.${region}.amazonaws.com:8443`,
+    protocol: 'wss',
+    canonicalUri: '/stream-transcription-websocket',
+    serviceName: 'transcribe',
+    payloadHash: sha256(''),
+    expires: 300,
+    queryParams: {
+      'language-code': languageCode,
+      'media-encoding': mediaEncoding,
+      'sample-rate': sampleRate,
+      ...sessionId == null ? {} : { 'session-id': sessionId },
+      ...vocabularyName == null ? {} : { 'vocabulary-name': vocabularyName },
+    },
+  })
+
+  this.websocket = new WebSocket(url)
+  this.ready = new Promise(resolve => {
+    this.websocket.on('open', resolve)
+  })
+  this.websocket.on('message', chunk => {
+    const { headers, body } = unmarshalMessage(chunk)
+    if (body.Transcript.Results.length > 0) {
+      if (body.Transcript.Results[0].IsPartial) {
+        this.emit('partialTranscription', body.Transcript.Results[0])
+      } else {
+        this.emit('transcription', body.Transcript.Results[0])
+      }
+    }
+  })
+
+  return this
+}
+
+TranscribeStreaming.prototype = EventEmitter.prototype
+
+/**
+ * @name TranscribeStreaming.prototype.sendAudioChunk
+ *
+ * @synopsis
+ * ```coffeescript [specscript]
+ * myTranscribeStream.sendAudioChunk(
+ *   chunk Buffer, // chunk is binary and assumed to be properly encoded in the specified mediaEncoding
+ * ) -> undefined
+ * ```
+ *
+ * @description
+ * https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html
+ * https://github.com/aws-samples/amazon-transcribe-comprehend-medical-twilio/blob/main/lib/transcribe-service.js
+ */
+TranscribeStreaming.prototype.sendAudioChunk = function (chunk) {
+  const headersBytes = marshalHeaders({
+    ':message-type': {
+      type: 'string',
+      value: 'event',
+    },
+    ':event-type': {
+      type: 'string',
+      value: 'AudioEvent',
+    },
+    ':content-type': {
+      type: 'string',
+      value: 'application/octet-stream',
+    },
+    // hey: { type: 'string', value: 'yo' },
+  })
+  const length = headersBytes.byteLength + chunk.byteLength + 16
+  const bytes = new Uint8Array(length)
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  const checksum = new Crc32()
+
+  view.setUint32(0, length, false)
+  view.setUint32(4, headersBytes.byteLength, false)
+  view.setUint32(8, checksum.update(bytes.subarray(0, 8)).digest(), false)
+  bytes.set(headersBytes, 12)
+  bytes.set(chunk, headersBytes.byteLength + 12)
+  view.setUint32(
+    length - 4,
+    checksum.update(bytes.subarray(8, length - 4)).digest(),
+    false
+  )
+  this.websocket.send(bytes)
+}
+
+/**
+ * @name marshalHeaders
+ *
+ * @synopsis
+ * ```coffeescript [specscript]
+ * marshalHeaders(headers Object<
+ *   [headerName string]: {
+ *     type: 'string',
+ *     value: string,
+ *   }
+ * >) -> headersBytes Uint8Array
+ * ```
+ */
+const marshalHeaders = function (headers) {
+  const chunks = []
+  for (const headerName in headers) {
+    const nameBytes = Buffer.from(headerName, 'utf8')
+    const header = headers[headerName]
+    chunks.push(Buffer.from([nameBytes.byteLength]))
+    chunks.push(nameBytes)
+    if (header.type == 'string') {
+      chunks.push(marshalStringHeaderValue(header.value))
+    } else {
+      throw new Error(`Unrecognized header type ${header.type}`)
+    }
+  }
+  const headersBytes = new Uint8Array(
+    chunks.reduce((total, bytes) => total + bytes.byteLength, 0),
+  )
+  let index = 0
+  for (const chunk of chunks) {
+    headersBytes.set(chunk, index)
+    index += chunk.byteLength
+  }
+  return headersBytes
+}
+
+/**
+ * @synopsis
+ * ```coffeescript [specscript]
+ * marshalStringHeaderValue(value string) -> bytes Uint8Array
+ * ```
+ */
+const marshalStringHeaderValue = function (value) {
+  const buffer = Buffer.from(value, 'utf8')
+  const view = new DataView(new ArrayBuffer(3 + buffer.byteLength))
+  view.setUint8(0, 7) // string value type
+  view.setUint16(1, buffer.byteLength, false)
+  const result = new Uint8Array(view.buffer)
+  result.set(buffer, 3)
+  return result
+}
+
+/**
+ * @synopsis
+ * ```coffeescript [specscript]
+ * unmarshalMessage(chunk ArrayBuffer) -> message {
+ *   headers: DataView,
+ *   body: Uint8Array,
+ * }
+ * ```
+ */
+const unmarshalMessage = function (chunk) {
+  const { buffer, byteOffset, byteLength } = chunk
+  const view = new DataView(buffer, byteOffset, byteLength)
+  const messageLength = view.getUint32(0, false)
+  const headerLength = view.getUint32(PRELUDE_MEMBER_LENGTH, false)
+  return {
+    headers: unmarshalHeaders(new DataView(
+      buffer,
+      byteOffset + PRELUDE_LENGTH + CHECKSUM_LENGTH,
+      headerLength
+    )),
+    body: JSON.parse(String.fromCharCode.apply(null, new Uint8Array(
+      buffer,
+      byteOffset + PRELUDE_LENGTH + CHECKSUM_LENGTH + headerLength,
+      messageLength - headerLength - (
+        PRELUDE_LENGTH + CHECKSUM_LENGTH + CHECKSUM_LENGTH
+      )
+    ))),
+  }
+}
+
+/**
+ * @synopsis
+ * ```coffeescript [specscript]
+ * unmarshalHeaders(headersView DataView) -> headers Object
+ * ```
+ */
+const unmarshalHeaders = function (headersView) {
+  const headers = {}
+  let index = 0
+  while (index < headersView.byteLength) {
+    const nameLength = headersView.getUint8(index)
+    index += 1
+    const name = String.fromCharCode.apply(null, new Uint8Array(
+      headersView.buffer,
+      headersView.byteOffset + index,
+      nameLength,
+    ))
+    index += nameLength
+    index += 1 // byte for header type, assumed to be all strings for now
+    const stringLength = headersView.getUint16(index, false)
+    index += 2
+    headers[name] = String.fromCharCode.apply(null, new Uint8Array(
+      headersView.buffer,
+      headersView.byteOffset + index,
+      stringLength,
+    ))
+    index += stringLength
+  }
+  return headers
+}
+
+module.exports = TranscribeStreaming