qdone 2.0.6-alpha → 2.0.8-alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/commonjs/src/enqueue.js +21 -21
  2. package/package.json +1 -1
  3. package/src/enqueue.js +19 -19
  4. package/npm-shrinkwrap.json +0 -16001
package/commonjs/src/enqueue.js CHANGED
@@ -328,7 +328,7 @@ function sendMessageBatch(qrl, messages, opt) {
328
328
  if (opt.fifo) {
329
329
  params.Entries = params.Entries.map(function (message) { return Object.assign({
330
330
  MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
331
- MessageDeduplicationId: uuidFunction()
331
+ MessageDeduplicationId: opt.deduplicationId || uuidFunction()
332
332
  }, message); });
333
333
  }
334
334
  if (opt.delay) {
@@ -400,14 +400,13 @@ function sendMessageBatch(qrl, messages, opt) {
400
400
  });
401
401
  }
402
402
  exports.sendMessageBatch = sendMessageBatch;
403
- var messages = {};
404
403
  var requestCount = 0;
405
404
  //
406
405
  // Flushes the internal message buffer for qrl.
407
406
  // If the message is too large, batch is retried with half the messages.
408
407
  // Returns number of messages flushed.
409
408
  //
410
- function flushMessages(qrl, opt) {
409
+ function flushMessages(qrl, opt, sendBuffer) {
411
410
  return __awaiter(this, void 0, void 0, function () {
412
411
  function whileNotEmpty() {
413
412
  return __awaiter(this, void 0, void 0, function () {
@@ -415,18 +414,18 @@ function flushMessages(qrl, opt) {
415
414
  return __generator(this, function (_a) {
416
415
  switch (_a.label) {
417
416
  case 0:
418
- if (!(messages[qrl] && messages[qrl].length))
417
+ if (!(sendBuffer[qrl] && sendBuffer[qrl].length))
419
418
  return [2 /*return*/, numFlushed
420
419
  // Construct batch until full
421
420
  ];
422
421
  batch = [];
423
- nextSize = JSON.stringify(messages[qrl][0]).length;
422
+ nextSize = JSON.stringify(sendBuffer[qrl][0]).length;
424
423
  totalSize = 0;
425
- while ((totalSize + nextSize) < 262144 && messages[qrl].length && batch.length < 10) {
426
- batch.push(messages[qrl].shift());
424
+ while ((totalSize + nextSize) < 262144 && sendBuffer[qrl].length && batch.length < 10) {
425
+ batch.push(sendBuffer[qrl].shift());
427
426
  totalSize += nextSize;
428
- if (messages[qrl].length)
429
- nextSize = JSON.stringify(messages[qrl][0]).length;
427
+ if (sendBuffer[qrl].length)
428
+ nextSize = JSON.stringify(sendBuffer[qrl][0]).length;
430
429
  else
431
430
  nextSize = 0;
432
431
  }
@@ -468,17 +467,16 @@ exports.flushMessages = flushMessages;
468
467
  // Automaticaly flushes if queue has >= 10 messages.
469
468
  // Returns number of messages flushed.
470
469
  //
471
- var messageIndex = 0;
472
- function addMessage(qrl, command, opt) {
470
+ function addMessage(qrl, command, messageIndex, opt, sendBuffer) {
473
471
  return __awaiter(this, void 0, void 0, function () {
474
472
  var message;
475
473
  return __generator(this, function (_a) {
476
- message = formatMessage(command, messageIndex++);
477
- messages[qrl] = messages[qrl] || [];
478
- messages[qrl].push(message);
479
- debug({ location: 'addMessage', messages: messages });
480
- if (messages[qrl].length >= 10) {
481
- return [2 /*return*/, flushMessages(qrl, opt)];
474
+ message = formatMessage(command, messageIndex);
475
+ sendBuffer[qrl] = sendBuffer[qrl] || [];
476
+ sendBuffer[qrl].push(message);
477
+ debug({ location: 'addMessage', sendBuffer: sendBuffer });
478
+ if (sendBuffer[qrl].length >= 10) {
479
+ return [2 /*return*/, flushMessages(qrl, opt, sendBuffer)];
482
480
  }
483
481
  return [2 /*return*/, 0];
484
482
  });
@@ -512,7 +510,7 @@ exports.enqueue = enqueue;
512
510
  //
513
511
  function enqueueBatch(pairs, options) {
514
512
  return __awaiter(this, void 0, void 0, function () {
515
- var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, initialFlushTotal, _a, normalizedPairs_1, _b, qname, command, qrl, _c, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
513
+ var opt, normalizedPairs, uniqueQnames, createPromises, _i, uniqueQnames_1, qname, sendBuffer, messageIndex, initialFlushTotal, _a, normalizedPairs_1, _b, qname, command, qrl, _c, extraFlushPromises, qrl, extraFlushCounts, extraFlushTotal, totalFlushed;
516
514
  return __generator(this, function (_d) {
517
515
  switch (_d.label) {
518
516
  case 0:
@@ -540,6 +538,8 @@ function enqueueBatch(pairs, options) {
540
538
  // After we've prefetched, all qrls are in cache
541
539
  // so go back through the list of pairs and fire off messages
542
540
  requestCount = 0;
541
+ sendBuffer = {};
542
+ messageIndex = 0;
543
543
  initialFlushTotal = 0;
544
544
  _a = 0, normalizedPairs_1 = normalizedPairs;
545
545
  _d.label = 2;
@@ -550,7 +550,7 @@ function enqueueBatch(pairs, options) {
550
550
  case 3:
551
551
  qrl = _d.sent();
552
552
  _c = initialFlushTotal;
553
- return [4 /*yield*/, addMessage(qrl, command, opt)];
553
+ return [4 /*yield*/, addMessage(qrl, command, messageIndex++, opt, sendBuffer)];
554
554
  case 4:
555
555
  initialFlushTotal = _c + _d.sent();
556
556
  _d.label = 5;
@@ -559,8 +559,8 @@ function enqueueBatch(pairs, options) {
559
559
  return [3 /*break*/, 2];
560
560
  case 6:
561
561
  extraFlushPromises = [];
562
- for (qrl in messages) {
563
- extraFlushPromises.push(flushMessages(qrl, opt));
562
+ for (qrl in sendBuffer) {
563
+ extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer));
564
564
  }
565
565
  return [4 /*yield*/, Promise.all(extraFlushPromises)];
566
566
  case 7:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "qdone",
3
- "version": "2.0.6-alpha",
3
+ "version": "2.0.8-alpha",
4
4
  "description": "Language agnostic job queue for SQS",
5
5
  "type": "module",
6
6
  "main": "./index.js",
package/src/enqueue.js CHANGED
@@ -209,7 +209,7 @@ export async function sendMessageBatch (qrl, messages, opt) {
209
209
  params.Entries = params.Entries.map(
210
210
  message => Object.assign({
211
211
  MessageGroupId: opt.groupIdPerMessage ? uuidFunction() : opt.groupId,
212
- MessageDeduplicationId: uuidFunction()
212
+ MessageDeduplicationId: opt.deduplicationId || uuidFunction()
213
213
  }, message)
214
214
  )
215
215
  }
@@ -267,7 +267,6 @@ export async function sendMessageBatch (qrl, messages, opt) {
267
267
  return backoff.run(send, shouldRetry)
268
268
  }
269
269
 
270
- const messages = {}
271
270
  let requestCount = 0
272
271
 
273
272
  //
@@ -275,20 +274,20 @@ let requestCount = 0
275
274
  // If the message is too large, batch is retried with half the messages.
276
275
  // Returns number of messages flushed.
277
276
  //
278
- export async function flushMessages (qrl, opt) {
277
+ export async function flushMessages (qrl, opt, sendBuffer) {
279
278
  debug('flushMessages', qrl)
280
279
  // Flush until empty
281
280
  let numFlushed = 0
282
281
  async function whileNotEmpty () {
283
- if (!(messages[qrl] && messages[qrl].length)) return numFlushed
282
+ if (!(sendBuffer[qrl] && sendBuffer[qrl].length)) return numFlushed
284
283
  // Construct batch until full
285
284
  const batch = []
286
- let nextSize = JSON.stringify(messages[qrl][0]).length
285
+ let nextSize = JSON.stringify(sendBuffer[qrl][0]).length
287
286
  let totalSize = 0
288
- while ((totalSize + nextSize) < 262144 && messages[qrl].length && batch.length < 10) {
289
- batch.push(messages[qrl].shift())
287
+ while ((totalSize + nextSize) < 262144 && sendBuffer[qrl].length && batch.length < 10) {
288
+ batch.push(sendBuffer[qrl].shift())
290
289
  totalSize += nextSize
291
- if (messages[qrl].length) nextSize = JSON.stringify(messages[qrl][0]).length
290
+ if (sendBuffer[qrl].length) nextSize = JSON.stringify(sendBuffer[qrl][0]).length
292
291
  else nextSize = 0
293
292
  }
294
293
 
@@ -321,14 +320,13 @@ export async function flushMessages (qrl, opt) {
321
320
  // Automaticaly flushes if queue has >= 10 messages.
322
321
  // Returns number of messages flushed.
323
322
  //
324
- let messageIndex = 0
325
- export async function addMessage (qrl, command, opt) {
326
- const message = formatMessage(command, messageIndex++)
327
- messages[qrl] = messages[qrl] || []
328
- messages[qrl].push(message)
329
- debug({ location: 'addMessage', messages })
330
- if (messages[qrl].length >= 10) {
331
- return flushMessages(qrl, opt)
323
+ export async function addMessage (qrl, command, messageIndex, opt, sendBuffer) {
324
+ const message = formatMessage(command, messageIndex)
325
+ sendBuffer[qrl] = sendBuffer[qrl] || []
326
+ sendBuffer[qrl].push(message)
327
+ debug({ location: 'addMessage', sendBuffer })
328
+ if (sendBuffer[qrl].length >= 10) {
329
+ return flushMessages(qrl, opt, sendBuffer)
332
330
  }
333
331
  return 0
334
332
  }
@@ -370,16 +368,18 @@ export async function enqueueBatch (pairs, options) {
370
368
  // After we've prefetched, all qrls are in cache
371
369
  // so go back through the list of pairs and fire off messages
372
370
  requestCount = 0
371
+ const sendBuffer = {}
372
+ let messageIndex = 0
373
373
  let initialFlushTotal = 0
374
374
  for (const { qname, command } of normalizedPairs) {
375
375
  const qrl = await getOrCreateQueue(qname, opt)
376
- initialFlushTotal += await addMessage(qrl, command, opt)
376
+ initialFlushTotal += await addMessage(qrl, command, messageIndex++, opt, sendBuffer)
377
377
  }
378
378
 
379
379
  // And flush any remaining messages
380
380
  const extraFlushPromises = []
381
- for (const qrl in messages) {
382
- extraFlushPromises.push(flushMessages(qrl, opt))
381
+ for (const qrl in sendBuffer) {
382
+ extraFlushPromises.push(flushMessages(qrl, opt, sendBuffer))
383
383
  }
384
384
  const extraFlushCounts = await Promise.all(extraFlushPromises)
385
385
  const extraFlushTotal = extraFlushCounts.reduce((a, b) => a + b, 0)