presidium 0.15.10 → 0.15.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +1 -1
- package/.github/workflows/nodejs.yml +3 -0
- package/DynamoStream.js +10 -9
- package/DynamoStream.test.js +3 -0
- package/DynamoTable.js +29 -7
- package/DynamoTable.test.js +68 -2
- package/TranscribeStreaming.js +276 -0
- package/TranscribeStreaming.test.js +122 -0
- package/internal/AmzCanonicalHeaders.js +18 -0
- package/internal/AmzDate.js +14 -0
- package/internal/AmzSignature.js +35 -0
- package/internal/AmzSignedHeaders.js +13 -0
- package/internal/AwsCredentials.js +32 -0
- package/internal/AwsPresignedUrlV4.js +111 -0
- package/internal/Crc32.js +93 -0
- package/internal/sha256.js +1 -1
- package/media-stream-fixture-aws-keynote.txt +366 -0
- package/package.json +3 -2
package/.eslintrc.js
CHANGED
|
@@ -38,3 +38,6 @@ jobs:
|
|
|
38
38
|
with:
|
|
39
39
|
node-version: ${{ matrix.node-version }}
|
|
40
40
|
- run: npm i && npx nyc --reporter=lcovonly npm test && npx codecov --token=3586c688-30cd-46be-a6c4-79a6e0f7fe80 --file=coverage/lcov.info && npm run lint
|
|
41
|
+
env:
|
|
42
|
+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
43
|
+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
package/DynamoStream.js
CHANGED
|
@@ -128,7 +128,7 @@ DynamoStream.prototype.getRecords = async function* getRecords(
|
|
|
128
128
|
const startingShardIterator = await this.client.getShardIterator({
|
|
129
129
|
ShardId: Shard.ShardId,
|
|
130
130
|
StreamArn: Shard.Stream.StreamArn,
|
|
131
|
-
ShardIteratorType:
|
|
131
|
+
ShardIteratorType: Shard.ShardIteratorType,
|
|
132
132
|
...(
|
|
133
133
|
this.shardIteratorType == 'AFTER_SEQUENCE_NUMBER'
|
|
134
134
|
|| this.shardIteratorType == 'AT_SEQUENCE_NUMBER'
|
|
@@ -142,7 +142,7 @@ DynamoStream.prototype.getRecords = async function* getRecords(
|
|
|
142
142
|
}).promise()
|
|
143
143
|
|
|
144
144
|
yield* records.Records
|
|
145
|
-
while (!this.closed &&
|
|
145
|
+
while (!this.closed && records.NextShardIterator != null) {
|
|
146
146
|
records = await this.client.getRecords({
|
|
147
147
|
ShardIterator: records.NextShardIterator,
|
|
148
148
|
Limit: this.getRecordsLimit
|
|
@@ -158,6 +158,7 @@ DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
|
|
|
158
158
|
let shards = await pipe([
|
|
159
159
|
transform(map(identity), []),
|
|
160
160
|
flatMap(Stream => this.getShards(Stream)),
|
|
161
|
+
map(assign({ ShardIteratorType: always(this.shardIteratorType) })),
|
|
161
162
|
])(this.getStreams())
|
|
162
163
|
let muxAsyncIterator = Mux.race(shards.map(Shard => this.getRecords(Shard)))
|
|
163
164
|
let iterationPromise = muxAsyncIterator.next()
|
|
@@ -174,14 +175,14 @@ DynamoStream.prototype[Symbol.asyncIterator] = async function* () {
|
|
|
174
175
|
transform(map(identity), []),
|
|
175
176
|
flatMap(Stream => this.getShards(Stream)),
|
|
176
177
|
])(this.getStreams())
|
|
177
|
-
const newShards =
|
|
178
|
-
(
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
178
|
+
const newShards = pipe([
|
|
179
|
+
differenceWith(
|
|
180
|
+
(ShardA, ShardB) => ShardA.ShardId == ShardB.ShardId,
|
|
181
|
+
latestShards,
|
|
182
|
+
),
|
|
183
|
+
map(assign({ ShardIteratorType: always('TRIM_HORIZON') })),
|
|
184
|
+
])(shards)
|
|
183
185
|
|
|
184
|
-
closedShards.forEach(Shard => (Shard.closed = true))
|
|
185
186
|
shards = latestShards
|
|
186
187
|
muxAsyncIterator = newShards.length == 0 ? muxAsyncIterator : Mux.race([
|
|
187
188
|
...newShards.map(Shard => this.getRecords(Shard)),
|
package/DynamoStream.test.js
CHANGED
|
@@ -272,6 +272,7 @@ const test = Test('DynamoStream', DynamoStream)
|
|
|
272
272
|
.case({
|
|
273
273
|
table: 'my-table',
|
|
274
274
|
endpoint: 'http://localhost:8000',
|
|
275
|
+
shardUpdatePeriod: 500,
|
|
275
276
|
}, async function (myStream) {
|
|
276
277
|
await myStream.ready
|
|
277
278
|
|
|
@@ -282,6 +283,8 @@ const test = Test('DynamoStream', DynamoStream)
|
|
|
282
283
|
new Promise(resolve => setTimeout(thunkify(resolve, 'hey'), 3000))
|
|
283
284
|
])
|
|
284
285
|
assert.equal(raceResult, 'hey')
|
|
286
|
+
// wait a second for shard update
|
|
287
|
+
await new Promise(resolve => setTimeout(thunkify(resolve, 'hey'), 1000))
|
|
285
288
|
myStream.close()
|
|
286
289
|
})
|
|
287
290
|
|
package/DynamoTable.js
CHANGED
|
@@ -144,7 +144,10 @@ DynamoTable.prototype.putItem = async function dynamoTablePutItem(item, options)
|
|
|
144
144
|
TableName: this.name,
|
|
145
145
|
Item: map(Dynamo.AttributeValue)(item),
|
|
146
146
|
...options,
|
|
147
|
-
}).promise()
|
|
147
|
+
}).promise().catch(error => {
|
|
148
|
+
error.tableName = this.name
|
|
149
|
+
throw error
|
|
150
|
+
})
|
|
148
151
|
}
|
|
149
152
|
|
|
150
153
|
/**
|
|
@@ -162,9 +165,14 @@ DynamoTable.prototype.getItem = async function dynamoTableGetItem(key) {
|
|
|
162
165
|
Key: map(Dynamo.AttributeValue)(key),
|
|
163
166
|
}).promise().then(result => {
|
|
164
167
|
if (result.Item == null) {
|
|
165
|
-
|
|
168
|
+
const error = new Error(`Item not found for ${stringifyJSON(key)}`)
|
|
169
|
+
error.tableName = this.name
|
|
170
|
+
throw error
|
|
166
171
|
}
|
|
167
172
|
return result
|
|
173
|
+
}).catch(error => {
|
|
174
|
+
error.tableName = this.name
|
|
175
|
+
throw error
|
|
168
176
|
})
|
|
169
177
|
}
|
|
170
178
|
|
|
@@ -223,7 +231,10 @@ DynamoTable.prototype.updateItem = async function dynamoTableUpdateItem(
|
|
|
223
231
|
([key, value]) => [`:${hashJSON(value)}`, Dynamo.AttributeValue(value)],
|
|
224
232
|
)(updates),
|
|
225
233
|
...options,
|
|
226
|
-
}).promise()
|
|
234
|
+
}).promise().catch(error => {
|
|
235
|
+
error.tableName = this.name
|
|
236
|
+
throw error
|
|
237
|
+
})
|
|
227
238
|
}
|
|
228
239
|
|
|
229
240
|
/**
|
|
@@ -263,7 +274,10 @@ DynamoTable.prototype.incrementItem = async function incrementItem(
|
|
|
263
274
|
([key, value]) => [`:${hashJSON(value)}`, Dynamo.AttributeValue(value)],
|
|
264
275
|
)(incrementUpdates),
|
|
265
276
|
...options,
|
|
266
|
-
}).promise()
|
|
277
|
+
}).promise().catch(error => {
|
|
278
|
+
error.tableName = this.name
|
|
279
|
+
throw error
|
|
280
|
+
})
|
|
267
281
|
}
|
|
268
282
|
|
|
269
283
|
/**
|
|
@@ -287,7 +301,10 @@ DynamoTable.prototype.deleteItem = async function dynamoTableDeleteItem(key, opt
|
|
|
287
301
|
TableName: this.name,
|
|
288
302
|
Key: map(Dynamo.AttributeValue)(key),
|
|
289
303
|
...options,
|
|
290
|
-
}).promise()
|
|
304
|
+
}).promise().catch(error => {
|
|
305
|
+
error.tableName = this.name
|
|
306
|
+
throw error
|
|
307
|
+
})
|
|
291
308
|
}
|
|
292
309
|
|
|
293
310
|
/**
|
|
@@ -298,6 +315,7 @@ DynamoTable.prototype.deleteItem = async function dynamoTableDeleteItem(key, opt
|
|
|
298
315
|
* DynamoTable(options).scan(options {
|
|
299
316
|
* limit: number,
|
|
300
317
|
* exclusiveStartKey: Object<string=>DynamoAttributeValue>
|
|
318
|
+
* forceTableName?: string, // a test parameter
|
|
301
319
|
* }) -> Promise<{
|
|
302
320
|
* Items: Array<Object<string=>DynamoAttributeValue>>
|
|
303
321
|
* Count: number, // number of Items
|
|
@@ -307,14 +325,18 @@ DynamoTable.prototype.deleteItem = async function dynamoTableDeleteItem(key, opt
|
|
|
307
325
|
* ```
|
|
308
326
|
*/
|
|
309
327
|
DynamoTable.prototype.scan = async function scan(options = {}) {
|
|
328
|
+
const { forceTableName } = options
|
|
310
329
|
await this.ready
|
|
311
330
|
return this.client.scan({
|
|
312
|
-
TableName: this.name,
|
|
331
|
+
TableName: forceTableName ?? this.name,
|
|
313
332
|
Limit: options.limit ?? 100,
|
|
314
333
|
...options.exclusiveStartKey && {
|
|
315
334
|
ExclusiveStartKey: options.exclusiveStartKey,
|
|
316
335
|
},
|
|
317
|
-
}).promise()
|
|
336
|
+
}).promise().catch(error => {
|
|
337
|
+
error.tableName = this.name
|
|
338
|
+
throw error
|
|
339
|
+
})
|
|
318
340
|
}
|
|
319
341
|
|
|
320
342
|
module.exports = DynamoTable
|
package/DynamoTable.test.js
CHANGED
|
@@ -16,13 +16,37 @@ const test = new Test('DynamoTable', DynamoTable)
|
|
|
16
16
|
endpoint: 'http://localhost:8000/',
|
|
17
17
|
key: [{ id: 'string' }],
|
|
18
18
|
}, async function (testTable) {
|
|
19
|
+
await testTable.ready
|
|
20
|
+
// if we created another instance of testTable it shouldn't have to create now
|
|
21
|
+
await new DynamoTable({
|
|
22
|
+
name: 'test-tablename',
|
|
23
|
+
endpoint: 'http://localhost:8000/',
|
|
24
|
+
key: [{ id: 'string' }],
|
|
25
|
+
}).ready
|
|
26
|
+
|
|
19
27
|
// .case('http://localhost:8000/', 'test-tablename', async function (testTable) {
|
|
20
28
|
await testTable.putItem({ id: '1', name: 'george' })
|
|
21
29
|
await testTable.putItem({ id: '2', name: 'henry' })
|
|
22
30
|
await testTable.putItem({ id: '3', name: 'jude' })
|
|
31
|
+
assert.rejects(
|
|
32
|
+
testTable.putItem({ somekey: 'hey' }),
|
|
33
|
+
{
|
|
34
|
+
name: 'ValidationException',
|
|
35
|
+
message: 'One of the required keys was not given a value',
|
|
36
|
+
tableName: 'test-tablename',
|
|
37
|
+
},
|
|
38
|
+
)
|
|
23
39
|
assert.deepEqual(
|
|
24
40
|
await testTable.getItem({ id: '1' }),
|
|
25
41
|
{ Item: map(Dynamo.AttributeValue)({ id: '1', name: 'george' }) })
|
|
42
|
+
assert.rejects(
|
|
43
|
+
testTable.getItem({ id: 'not-exists' }),
|
|
44
|
+
{
|
|
45
|
+
name: 'Error',
|
|
46
|
+
message: 'Item not found for {"id":"not-exists"}',
|
|
47
|
+
tableName: 'test-tablename',
|
|
48
|
+
},
|
|
49
|
+
)
|
|
26
50
|
assert.deepEqual(
|
|
27
51
|
await testTable.putItem({ id: '1', name: 'george' }, {
|
|
28
52
|
ReturnValues: 'ALL_OLD',
|
|
@@ -51,6 +75,17 @@ const test = new Test('DynamoTable', DynamoTable)
|
|
|
51
75
|
},
|
|
52
76
|
)
|
|
53
77
|
|
|
78
|
+
assert.rejects(
|
|
79
|
+
testTable.updateItem({ id: 'not-exists' }, { a: 1 }, {
|
|
80
|
+
ConditionExpression: 'attribute_exists(id)',
|
|
81
|
+
}),
|
|
82
|
+
{
|
|
83
|
+
name: 'ConditionalCheckFailedException',
|
|
84
|
+
message: 'The conditional request failed',
|
|
85
|
+
tableName: 'test-tablename',
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
|
|
54
89
|
assert.deepEqual(
|
|
55
90
|
await testTable.getItem({ id: '1' }),
|
|
56
91
|
{
|
|
@@ -99,20 +134,51 @@ const test = new Test('DynamoTable', DynamoTable)
|
|
|
99
134
|
),
|
|
100
135
|
{
|
|
101
136
|
name: 'ValidationException',
|
|
102
|
-
message: 'An operand in the update expression has an incorrect data type'
|
|
137
|
+
message: 'An operand in the update expression has an incorrect data type',
|
|
138
|
+
tableName: 'test-tablename',
|
|
103
139
|
},
|
|
104
140
|
)
|
|
105
141
|
|
|
142
|
+
assert.rejects(
|
|
143
|
+
testTable.incrementItem({ id: 'not-exists' }, { a: 1 }, {
|
|
144
|
+
ConditionExpression: 'attribute_exists(id)',
|
|
145
|
+
}),
|
|
146
|
+
{
|
|
147
|
+
name: 'ConditionalCheckFailedException',
|
|
148
|
+
message: 'The conditional request failed',
|
|
149
|
+
tableName: 'test-tablename',
|
|
150
|
+
}
|
|
151
|
+
)
|
|
152
|
+
|
|
106
153
|
{
|
|
107
154
|
const scanResult1 = await testTable.scan({ limit: 1 })
|
|
108
155
|
const scanResult2 = await testTable.scan({ limit: 2, exclusiveStartKey: scanResult1.LastEvaluatedKey })
|
|
109
|
-
const scanResult3 = await testTable.scan({
|
|
156
|
+
const scanResult3 = await testTable.scan({ exclusiveStartKey: scanResult2.LastEvaluatedKey })
|
|
157
|
+
const bareScanResult = await testTable.scan()
|
|
110
158
|
assert.strictEqual(scanResult1.Items.length, 1)
|
|
111
159
|
assert.strictEqual(scanResult2.Items.length, 2)
|
|
112
160
|
assert.strictEqual(scanResult3.Items.length, 0)
|
|
161
|
+
assert.strictEqual(bareScanResult.Items.length, 3)
|
|
162
|
+
|
|
163
|
+
assert.rejects(
|
|
164
|
+
testTable.scan({ forceTableName: 'nonexistent-table-name' }),
|
|
165
|
+
{
|
|
166
|
+
name: 'ResourceNotFoundException',
|
|
167
|
+
message: 'Cannot do operations on a non-existent table',
|
|
168
|
+
tableName: 'test-tablename',
|
|
169
|
+
},
|
|
170
|
+
)
|
|
113
171
|
}
|
|
114
172
|
|
|
115
173
|
await testTable.deleteItem({ id: '1' })
|
|
174
|
+
assert.rejects(
|
|
175
|
+
testTable.deleteItem({ somekey: 'a' }),
|
|
176
|
+
{
|
|
177
|
+
name: 'ValidationException',
|
|
178
|
+
message: 'One of the required keys was not given a value',
|
|
179
|
+
tableName: 'test-tablename',
|
|
180
|
+
}
|
|
181
|
+
)
|
|
116
182
|
const shouldReject = testTable.getItem({ id: '1' })
|
|
117
183
|
assert.rejects(
|
|
118
184
|
() => shouldReject,
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
const EventEmitter = require('events')
|
|
2
|
+
const rubico = require('rubico')
|
|
3
|
+
const WebSocket = require('./WebSocket')
|
|
4
|
+
const sha256 = require('./internal/sha256')
|
|
5
|
+
const AwsPresignedUrlV4 = require('./internal/AwsPresignedUrlV4')
|
|
6
|
+
const Crc32 = require('./internal/Crc32')
|
|
7
|
+
|
|
8
|
+
const {
|
|
9
|
+
pipe, tap,
|
|
10
|
+
switchCase, tryCatch,
|
|
11
|
+
fork, assign, get, pick, omit,
|
|
12
|
+
map, filter, reduce, transform, flatMap,
|
|
13
|
+
and, or, not, any, all,
|
|
14
|
+
eq, gt, lt, gte, lte,
|
|
15
|
+
thunkify, always,
|
|
16
|
+
curry, __,
|
|
17
|
+
} = rubico
|
|
18
|
+
|
|
19
|
+
// All prelude components are unsigned, 32-bit integers
|
|
20
|
+
const PRELUDE_MEMBER_LENGTH = 4
|
|
21
|
+
|
|
22
|
+
// The prelude consists of two components
|
|
23
|
+
const PRELUDE_LENGTH = PRELUDE_MEMBER_LENGTH * 2
|
|
24
|
+
|
|
25
|
+
// Checksums are always CRC32 hashes.
|
|
26
|
+
const CHECKSUM_LENGTH = 4
|
|
27
|
+
|
|
28
|
+
// Messages must include a full prelude, a prelude checksum, and a message checksum
|
|
29
|
+
const MINIMUM_MESSAGE_LENGTH = PRELUDE_LENGTH + CHECKSUM_LENGTH * 2
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @name TranscribeStreaming
|
|
33
|
+
*
|
|
34
|
+
* @synopsis
|
|
35
|
+
* ```coffeescript [specscript]
|
|
36
|
+
* const myTranscribeStream = new TranscribeStreaming(options {
|
|
37
|
+
* accessKeyId: string,
|
|
38
|
+
* secretAccessKey: string,
|
|
39
|
+
* region: string,
|
|
40
|
+
* endpoint: string,
|
|
41
|
+
* languageCode: string,
|
|
42
|
+
* mediaEncoding: string,
|
|
43
|
+
* sampleRate: number,
|
|
44
|
+
* sessionId?: string,
|
|
45
|
+
* vocabularyName?: string,
|
|
46
|
+
* })
|
|
47
|
+
* ```
|
|
48
|
+
*
|
|
49
|
+
* @description
|
|
50
|
+
* https://docs.aws.amazon.com/TranscribeStreaming/latest/dg/websocket.html
|
|
51
|
+
*
|
|
52
|
+
* `languageCode` - `en-AU`, `en-GB`, `en-US`, `es-US`, `fr-CA`, `fr-FR`, `de-DE`, `ja-JP`, `ko-KR`, `pt-BR`, `zh-CN` or `it-IT`.
|
|
53
|
+
*
|
|
54
|
+
* `mediaEncoding` - `pcm`, `ogg-opus`, or `flac`.
|
|
55
|
+
*
|
|
56
|
+
* `sampleRate` - The sample rate of the input audio in Hertz. We suggest that you use 8,000 Hz for low-quality audio and 16,000 Hz (or higher) for high-quality audio. The sample rate must match the sample rate in the audio file.
|
|
57
|
+
*
|
|
58
|
+
* `sessionId` - id for the transcription session. If you don't provide a session ID, Amazon Transcribe generates one for you and returns it in the response.
|
|
59
|
+
*
|
|
60
|
+
* `vocabularyName` - The name of the vocabulary to use when processing the transcription job, if any.
|
|
61
|
+
*/
|
|
62
|
+
const TranscribeStreaming = function (options) {
|
|
63
|
+
const {
|
|
64
|
+
accessKeyId,
|
|
65
|
+
secretAccessKey,
|
|
66
|
+
region,
|
|
67
|
+
languageCode,
|
|
68
|
+
mediaEncoding,
|
|
69
|
+
sampleRate,
|
|
70
|
+
sessionId,
|
|
71
|
+
vocabularyName,
|
|
72
|
+
} = options
|
|
73
|
+
|
|
74
|
+
const url = AwsPresignedUrlV4({
|
|
75
|
+
accessKeyId,
|
|
76
|
+
secretAccessKey,
|
|
77
|
+
region,
|
|
78
|
+
method: 'GET',
|
|
79
|
+
endpoint: `transcribestreaming.${region}.amazonaws.com:8443`,
|
|
80
|
+
protocol: 'wss',
|
|
81
|
+
canonicalUri: '/stream-transcription-websocket',
|
|
82
|
+
serviceName: 'transcribe',
|
|
83
|
+
payloadHash: sha256(''),
|
|
84
|
+
expires: 300,
|
|
85
|
+
queryParams: {
|
|
86
|
+
'language-code': languageCode,
|
|
87
|
+
'media-encoding': mediaEncoding,
|
|
88
|
+
'sample-rate': sampleRate,
|
|
89
|
+
...sessionId == null ? {} : { 'session-id': sessionId },
|
|
90
|
+
...vocabularyName == null ? {} : { 'vocabulary-name': vocabularyName },
|
|
91
|
+
},
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
this.websocket = new WebSocket(url)
|
|
95
|
+
this.ready = new Promise(resolve => {
|
|
96
|
+
this.websocket.on('open', resolve)
|
|
97
|
+
})
|
|
98
|
+
this.websocket.on('message', chunk => {
|
|
99
|
+
const { headers, body } = unmarshalMessage(chunk)
|
|
100
|
+
if (body.Transcript.Results.length > 0) {
|
|
101
|
+
if (body.Transcript.Results[0].IsPartial) {
|
|
102
|
+
this.emit('partialTranscription', body.Transcript.Results[0])
|
|
103
|
+
} else {
|
|
104
|
+
this.emit('transcription', body.Transcript.Results[0])
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
return this
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
TranscribeStreaming.prototype = EventEmitter.prototype
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* @name TranscribeStreaming.prototype.sendAudioChunk
|
|
116
|
+
*
|
|
117
|
+
* @synopsis
|
|
118
|
+
* ```coffeescript [specscript]
|
|
119
|
+
* myTranscribeStream.sendAudioChunk(
|
|
120
|
+
* chunk Buffer, // chunk is binary and assumed to be properly encoded in the specified mediaEncoding
|
|
121
|
+
* ) -> undefined
|
|
122
|
+
* ```
|
|
123
|
+
*
|
|
124
|
+
* @description
|
|
125
|
+
* https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html
|
|
126
|
+
* https://github.com/aws-samples/amazon-transcribe-comprehend-medical-twilio/blob/main/lib/transcribe-service.js
|
|
127
|
+
*/
|
|
128
|
+
TranscribeStreaming.prototype.sendAudioChunk = function (chunk) {
|
|
129
|
+
const headersBytes = marshalHeaders({
|
|
130
|
+
':message-type': {
|
|
131
|
+
type: 'string',
|
|
132
|
+
value: 'event',
|
|
133
|
+
},
|
|
134
|
+
':event-type': {
|
|
135
|
+
type: 'string',
|
|
136
|
+
value: 'AudioEvent',
|
|
137
|
+
},
|
|
138
|
+
':content-type': {
|
|
139
|
+
type: 'string',
|
|
140
|
+
value: 'application/octet-stream',
|
|
141
|
+
},
|
|
142
|
+
// hey: { type: 'string', value: 'yo' },
|
|
143
|
+
})
|
|
144
|
+
const length = headersBytes.byteLength + chunk.byteLength + 16
|
|
145
|
+
const bytes = new Uint8Array(length)
|
|
146
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
147
|
+
const checksum = new Crc32()
|
|
148
|
+
|
|
149
|
+
view.setUint32(0, length, false)
|
|
150
|
+
view.setUint32(4, headersBytes.byteLength, false)
|
|
151
|
+
view.setUint32(8, checksum.update(bytes.subarray(0, 8)).digest(), false)
|
|
152
|
+
bytes.set(headersBytes, 12)
|
|
153
|
+
bytes.set(chunk, headersBytes.byteLength + 12)
|
|
154
|
+
view.setUint32(
|
|
155
|
+
length - 4,
|
|
156
|
+
checksum.update(bytes.subarray(8, length - 4)).digest(),
|
|
157
|
+
false
|
|
158
|
+
)
|
|
159
|
+
this.websocket.send(bytes)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @name marshalHeaders
|
|
164
|
+
*
|
|
165
|
+
* @synopsis
|
|
166
|
+
* ```coffeescript [specscript]
|
|
167
|
+
* marshalHeaders(headers Object<
|
|
168
|
+
* [headerName string]: {
|
|
169
|
+
* type: 'string',
|
|
170
|
+
* value: string,
|
|
171
|
+
* }
|
|
172
|
+
* >) -> headersBytes Uint8Array
|
|
173
|
+
* ```
|
|
174
|
+
*/
|
|
175
|
+
const marshalHeaders = function (headers) {
|
|
176
|
+
const chunks = []
|
|
177
|
+
for (const headerName in headers) {
|
|
178
|
+
const nameBytes = Buffer.from(headerName, 'utf8')
|
|
179
|
+
const header = headers[headerName]
|
|
180
|
+
chunks.push(Buffer.from([nameBytes.byteLength]))
|
|
181
|
+
chunks.push(nameBytes)
|
|
182
|
+
if (header.type == 'string') {
|
|
183
|
+
chunks.push(marshalStringHeaderValue(header.value))
|
|
184
|
+
} else {
|
|
185
|
+
throw new Error(`Unrecognized header type ${header.type}`)
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
const headersBytes = new Uint8Array(
|
|
189
|
+
chunks.reduce((total, bytes) => total + bytes.byteLength, 0),
|
|
190
|
+
)
|
|
191
|
+
let index = 0
|
|
192
|
+
for (const chunk of chunks) {
|
|
193
|
+
headersBytes.set(chunk, index)
|
|
194
|
+
index += chunk.byteLength
|
|
195
|
+
}
|
|
196
|
+
return headersBytes
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* @synopsis
|
|
201
|
+
* ```coffeescript [specscript]
|
|
202
|
+
* marshalStringHeaderValue(value string) -> bytes Uint8Array
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
const marshalStringHeaderValue = function (value) {
|
|
206
|
+
const buffer = Buffer.from(value, 'utf8')
|
|
207
|
+
const view = new DataView(new ArrayBuffer(3 + buffer.byteLength))
|
|
208
|
+
view.setUint8(0, 7) // string value type
|
|
209
|
+
view.setUint16(1, buffer.byteLength, false)
|
|
210
|
+
const result = new Uint8Array(view.buffer)
|
|
211
|
+
result.set(buffer, 3)
|
|
212
|
+
return result
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* @synopsis
|
|
217
|
+
* ```coffeescript [specscript]
|
|
218
|
+
* unmarshalMessage(chunk ArrayBuffer) -> message {
|
|
219
|
+
* headers: DataView,
|
|
220
|
+
* body: Uint8Array,
|
|
221
|
+
* }
|
|
222
|
+
* ```
|
|
223
|
+
*/
|
|
224
|
+
const unmarshalMessage = function (chunk) {
|
|
225
|
+
const { buffer, byteOffset, byteLength } = chunk
|
|
226
|
+
const view = new DataView(buffer, byteOffset, byteLength)
|
|
227
|
+
const messageLength = view.getUint32(0, false)
|
|
228
|
+
const headerLength = view.getUint32(PRELUDE_MEMBER_LENGTH, false)
|
|
229
|
+
return {
|
|
230
|
+
headers: unmarshalHeaders(new DataView(
|
|
231
|
+
buffer,
|
|
232
|
+
byteOffset + PRELUDE_LENGTH + CHECKSUM_LENGTH,
|
|
233
|
+
headerLength
|
|
234
|
+
)),
|
|
235
|
+
body: JSON.parse(String.fromCharCode.apply(null, new Uint8Array(
|
|
236
|
+
buffer,
|
|
237
|
+
byteOffset + PRELUDE_LENGTH + CHECKSUM_LENGTH + headerLength,
|
|
238
|
+
messageLength - headerLength - (
|
|
239
|
+
PRELUDE_LENGTH + CHECKSUM_LENGTH + CHECKSUM_LENGTH
|
|
240
|
+
)
|
|
241
|
+
))),
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* @synopsis
|
|
247
|
+
* ```coffeescript [specscript]
|
|
248
|
+
* unmarshalHeaders(headersView DataView) -> headers Object
|
|
249
|
+
* ```
|
|
250
|
+
*/
|
|
251
|
+
const unmarshalHeaders = function (headersView) {
|
|
252
|
+
const headers = {}
|
|
253
|
+
let index = 0
|
|
254
|
+
while (index < headersView.byteLength) {
|
|
255
|
+
const nameLength = headersView.getUint8(index)
|
|
256
|
+
index += 1
|
|
257
|
+
const name = String.fromCharCode.apply(null, new Uint8Array(
|
|
258
|
+
headersView.buffer,
|
|
259
|
+
headersView.byteOffset + index,
|
|
260
|
+
nameLength,
|
|
261
|
+
))
|
|
262
|
+
index += nameLength
|
|
263
|
+
index += 1 // byte for header type, assumed to be all strings for now
|
|
264
|
+
const stringLength = headersView.getUint16(index, false)
|
|
265
|
+
index += 2
|
|
266
|
+
headers[name] = String.fromCharCode.apply(null, new Uint8Array(
|
|
267
|
+
headersView.buffer,
|
|
268
|
+
headersView.byteOffset + index,
|
|
269
|
+
stringLength,
|
|
270
|
+
))
|
|
271
|
+
index += stringLength
|
|
272
|
+
}
|
|
273
|
+
return headers
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
module.exports = TranscribeStreaming
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const readline = require('readline')
|
|
3
|
+
const WaveFile = require('wavefile').WaveFile
|
|
4
|
+
const Test = require('thunk-test')
|
|
5
|
+
const assert = require('assert')
|
|
6
|
+
const rubico = require('rubico')
|
|
7
|
+
const TranscribeStreaming = require('./TranscribeStreaming')
|
|
8
|
+
const AwsCredentials = require('./internal/AwsCredentials')
|
|
9
|
+
|
|
10
|
+
const {
|
|
11
|
+
pipe, tap,
|
|
12
|
+
switchCase, tryCatch,
|
|
13
|
+
fork, assign, get, set, pick, omit,
|
|
14
|
+
map, filter, reduce, transform, flatMap,
|
|
15
|
+
and, or, not, any, all,
|
|
16
|
+
eq, gt, lt, gte, lte,
|
|
17
|
+
thunkify, always,
|
|
18
|
+
curry, __,
|
|
19
|
+
} = rubico
|
|
20
|
+
|
|
21
|
+
const test = new Test('TranscribeStreaming', async function () {
|
|
22
|
+
const awsCreds = await AwsCredentials('default').catch(error => {
|
|
23
|
+
if (error.code == 'ENOENT') {
|
|
24
|
+
const accessKeyId = process.env.AWS_ACCESS_KEY_ID
|
|
25
|
+
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY
|
|
26
|
+
if (accessKeyId == null || secretAccessKey == null) {
|
|
27
|
+
throw new Error('No AWS credential file or environment variables')
|
|
28
|
+
}
|
|
29
|
+
return { accessKeyId, secretAccessKey }
|
|
30
|
+
}
|
|
31
|
+
throw error
|
|
32
|
+
})
|
|
33
|
+
awsCreds.region = 'us-east-1' // only valid region for transcribe
|
|
34
|
+
|
|
35
|
+
const testTranscribeStreaming = new TranscribeStreaming({
|
|
36
|
+
languageCode: 'en-US',
|
|
37
|
+
mediaEncoding: 'pcm',
|
|
38
|
+
sampleRate: 8000,
|
|
39
|
+
...awsCreds,
|
|
40
|
+
})
|
|
41
|
+
await testTranscribeStreaming.ready
|
|
42
|
+
|
|
43
|
+
const mediaStreamFixtureAwsKeynote =
|
|
44
|
+
fs.createReadStream('./media-stream-fixture-aws-keynote.txt')
|
|
45
|
+
const rl = readline.createInterface({
|
|
46
|
+
input: mediaStreamFixtureAwsKeynote,
|
|
47
|
+
})
|
|
48
|
+
rl.on('line', line => {
|
|
49
|
+
const event = JSON.parse(line)
|
|
50
|
+
if (event.event == 'media') {
|
|
51
|
+
const wav = new WaveFile()
|
|
52
|
+
wav.fromScratch(1, 8000, '8', Buffer.from(event.media.payload, 'base64'))
|
|
53
|
+
wav.fromMuLaw()
|
|
54
|
+
testTranscribeStreaming.sendAudioChunk(Buffer.from(wav.data.samples))
|
|
55
|
+
} else if (event.event == 'stop') {
|
|
56
|
+
testTranscribeStreaming.websocket.close()
|
|
57
|
+
}
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
const testTranscription = await new Promise(resolve => {
|
|
61
|
+
testTranscribeStreaming.on('transcription', transcription => {
|
|
62
|
+
resolve(transcription.Alternatives[0].Transcript)
|
|
63
|
+
})
|
|
64
|
+
})
|
|
65
|
+
assert.equal(testTranscription, 'Hello, world.')
|
|
66
|
+
|
|
67
|
+
/*
|
|
68
|
+
// fill media-stream-fixture-aws-keynote.txt
|
|
69
|
+
const Twilio = require('@claimyr_hq/twilio/Twilio')
|
|
70
|
+
const ngrok = require('ngrok')
|
|
71
|
+
const WebSocketServer = require('./WebSocketServer')
|
|
72
|
+
const testPort = 7538
|
|
73
|
+
const mediaStreamFixtureAwsKeynote =
|
|
74
|
+
fs.createWriteStream('./media-stream-fixture-aws-keynote.txt', { flags: 'a' })
|
|
75
|
+
new WebSocketServer(socket => {
|
|
76
|
+
socket.on('message', chunk => {
|
|
77
|
+
const message = JSON.parse(chunk.toString('utf8'))
|
|
78
|
+
mediaStreamFixtureAwsKeynote.write(chunk.toString('utf8'))
|
|
79
|
+
mediaStreamFixtureAwsKeynote.write('\n')
|
|
80
|
+
console.log('Got message:')
|
|
81
|
+
console.log(message)
|
|
82
|
+
})
|
|
83
|
+
socket.on('error', error => {
|
|
84
|
+
console.error(error)
|
|
85
|
+
})
|
|
86
|
+
}).listen(testPort, () => {
|
|
87
|
+
console.log('test websocket server listening on port', testPort)
|
|
88
|
+
})
|
|
89
|
+
const exposedUrl = await ngrok.connect(testPort)
|
|
90
|
+
console.log('exposing', testPort, 'at', exposedUrl)
|
|
91
|
+
|
|
92
|
+
const twilio = new Twilio({
|
|
93
|
+
accountSid: 'redacted', // Toll Free accountSid
|
|
94
|
+
authToken: 'redacted', // Toll Free authToken
|
|
95
|
+
})
|
|
96
|
+
await twilio.createCall({
|
|
97
|
+
from: '+18557821665', // test tollfree numbers makes real calls
|
|
98
|
+
to: '+18586883603',
|
|
99
|
+
twiml: `
|
|
100
|
+
<Response>
|
|
101
|
+
<Start>
|
|
102
|
+
<Stream url="${exposedUrl.replace(/^https/, 'wss')}" />
|
|
103
|
+
</Start>
|
|
104
|
+
<Pause length="84600" />
|
|
105
|
+
</Response>
|
|
106
|
+
`.trim(),
|
|
107
|
+
}).then(async response => {
|
|
108
|
+
if (response.ok) {
|
|
109
|
+
console.log(await response.json())
|
|
110
|
+
} else {
|
|
111
|
+
console.error(await response.text())
|
|
112
|
+
}
|
|
113
|
+
})
|
|
114
|
+
*/
|
|
115
|
+
|
|
116
|
+
}).case()
|
|
117
|
+
|
|
118
|
+
if (process.argv[1] == __filename) {
|
|
119
|
+
test()
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
module.exports = test
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @name AmzCanonicalHeaders
|
|
3
|
+
*
|
|
4
|
+
* @synopsis
|
|
5
|
+
* ```coffeescript [specscript]
|
|
6
|
+
* AmzCanonicalHeaders(headers Object) -> canonicalHeaders string
|
|
7
|
+
* ```
|
|
8
|
+
*/
|
|
9
|
+
const AmzCanonicalHeaders = function (headers) {
|
|
10
|
+
let canonicalHeaders = ''
|
|
11
|
+
for (const key in headers) {
|
|
12
|
+
const value = headers[key]
|
|
13
|
+
canonicalHeaders += `${key.toLowerCase()}:${value}\n`
|
|
14
|
+
}
|
|
15
|
+
return canonicalHeaders
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
module.exports = AmzCanonicalHeaders
|