qdone 2.0.36-alpha → 2.0.38-alpha

package/commonjs/src/dedup.js

@@ -176,9 +176,6 @@ async function dedupShouldEnqueueMulti(messages, opt) {
     }
     const responses = await incrPipeline.exec();
     debug({ dedupShouldEnqueueMulti: { messages, responses } });
-    // Figure out dedup period
-    const minDedupPeriod = 6 * 60;
-    const dedupPeriod = Math.min(opt.dedupPeriod, minDedupPeriod);
     // Interpret responses and expire keys for races we won
     const expirePipeline = (0, cache_js_1.getCacheClient)(opt).pipeline();
     const statsPipeline = opt.dedupStats ? (0, cache_js_1.getCacheClient)(opt).pipeline() : undefined;
@@ -212,6 +209,7 @@ exports.dedupShouldEnqueueMulti = dedupShouldEnqueueMulti;
  * @returns {Number} 1 if a cache key was deleted, otherwise 0
  */
 async function dedupSuccessfullyProcessed(message, opt) {
+    debug({ dedupSuccessfullyProcessed: { message, opt } });
     const client = (0, cache_js_1.getCacheClient)(opt);
     const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
     if (dedupId) {

package/commonjs/src/defaults.js

@@ -57,7 +57,7 @@ function validateInteger(opt, name) {
     return parsed;
 }
 function validateMessageOptions(messageOptions) {
-    const validKeys = ['deduplicaitonId', 'groupId'];
+    const validKeys = ['deduplicationId', 'groupId'];
     if (typeof messageOptions === 'object' &&
         !Array.isArray(messageOptions) &&
         messageOptions !== null) {

package/commonjs/src/enqueue.js

@@ -274,7 +274,9 @@ async function sendMessageBatch(qrl, messages, opt) {
     debug({ params });
     // See which messages we even have to send
     if (opt.externalDedup) {
-        params.Entries = await (0, dedup_js_1.dedupShouldEnqueueMulti)(params.Entries, opt);
+        const promises = params.Entries.map(async (m) => ({ m, shouldEnqueue: await (0, dedup_js_1.dedupShouldEnqueue)(m, opt) }));
+        const results = await Promise.all(promises);
+        params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue).map(({ m }) => m);
         if (!params.Entries.length)
             return;
     }

package/commonjs/src/scheduler/jobExecutor.js

@@ -216,10 +216,10 @@ class JobExecutor {
                     else if (!this.opt.disableLog) {
                         console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }));
                     }
+                    // Mark batch as processed for dedup
+                    await Promise.all(result.Successful.map(e => (0, dedup_js_1.dedupSuccessfullyProcessed)(this.jobsByMessageId[e.Id], this.opt)));
                 }
                 debug('DeleteMessageBatch returned', result);
-                // Mark batch as processed for dedup
-                await (0, dedup_js_1.dedupSuccessfullyProcessedMulti)(entries.map(e => this.jobsByMessageId[e.Id]), this.opt);
                 // TODO Sentry
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.36-alpha",
+  "version": "2.0.38-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/dedup.js

@@ -112,9 +112,9 @@ export async function statMaintenance (opt) {
   const now = new Date().getTime()
 
   // Grab a batch of expired keys
-  debug({ statMaintenance: { aboutToGo: true, expirationSet }})
+  debug({ statMaintenance: { aboutToGo: true, expirationSet } })
   const expiredStats = await client.zrange(expirationSet, '-inf', now, 'BYSCORE')
-  debug({ statMaintenance: { expiredStats }})
+  debug({ statMaintenance: { expiredStats } })
 
   // And remove them from indexes, main storage
   if (expiredStats.length) {
@@ -122,7 +122,7 @@ export async function statMaintenance (opt) {
       .zrem(expirationSet, expiredStats)
       .zrem(duplicateSet, expiredStats)
       .exec()
-    debug({ statMaintenance: { result }})
+    debug({ statMaintenance: { result } })
   }
 }
 
@@ -159,7 +159,7 @@ export async function dedupShouldEnqueue (message, opt) {
  * @returns {Array[Object]} an array of messages that can be safely enqueued. Could be empty.
  */
 export async function dedupShouldEnqueueMulti (messages, opt) {
-  debug({ dedupShouldEnqueueMulti: { messages, opt }})
+  debug({ dedupShouldEnqueueMulti: { messages, opt } })
   const expireAt = new Date().getTime() + opt.dedupPeriod
   // Increment all
   const incrPipeline = getCacheClient(opt).pipeline()
@@ -171,10 +171,6 @@ export async function dedupShouldEnqueueMulti (messages, opt) {
   const responses = await incrPipeline.exec()
   debug({ dedupShouldEnqueueMulti: { messages, responses } })
 
-  // Figure out dedup period
-  const minDedupPeriod = 6 * 60
-  const dedupPeriod = Math.min(opt.dedupPeriod, minDedupPeriod)
-
   // Interpret responses and expire keys for races we won
   const expirePipeline = getCacheClient(opt).pipeline()
   const statsPipeline = opt.dedupStats ? getCacheClient(opt).pipeline() : undefined
@@ -206,6 +202,7 @@ export async function dedupShouldEnqueueMulti (messages, opt) {
  * @returns {Number} 1 if a cache key was deleted, otherwise 0
  */
 export async function dedupSuccessfullyProcessed (message, opt) {
+  debug({ dedupSuccessfullyProcessed: { message, opt } })
   const client = getCacheClient(opt)
   const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
   if (dedupId) {
@@ -230,7 +227,7 @@ export async function dedupSuccessfullyProcessed (message, opt) {
  * @returns {Number} number of deleted keys
  */
 export async function dedupSuccessfullyProcessedMulti (messages, opt) {
-  debug({ messages, dedupSuccessfullyProcessedMulti: { messages, opt }})
+  debug({ messages, dedupSuccessfullyProcessedMulti: { messages, opt } })
   const cacheKeys = []
   for (const message of messages) {
     const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
@@ -239,7 +236,7 @@ export async function dedupSuccessfullyProcessedMulti (messages, opt) {
       cacheKeys.push(cacheKey)
     }
   }
-  debug({ dedupSuccessfullyProcessedMulti: { cacheKeys }})
+  debug({ dedupSuccessfullyProcessedMulti: { cacheKeys } })
   if (cacheKeys.length) {
     const numDeleted = await getCacheClient(opt).del(cacheKeys)
     // const numDeleted = results.map(([, val]) => val).reduce((a, b) => a + b, 0)

package/src/defaults.js

@@ -60,7 +60,7 @@ function validateInteger (opt, name) {
 }
 
 export function validateMessageOptions (messageOptions) {
-  const validKeys = ['deduplicaitonId', 'groupId']
+  const validKeys = ['deduplicationId', 'groupId']
   if (typeof messageOptions === 'object' &&
       !Array.isArray(messageOptions) &&
       messageOptions !== null) {

package/src/enqueue.js

@@ -23,7 +23,6 @@ import { getSQSClient } from './sqs.js'
 import {
   addDedupParamsToMessage,
   dedupShouldEnqueue,
-  dedupShouldEnqueueMulti,
   dedupSuccessfullyProcessed
 } from './dedup.js'
 import { getOptionsWithDefaults, validateMessageOptions } from './defaults.js'
@@ -274,7 +273,9 @@ export async function sendMessageBatch (qrl, messages, opt) {
 
   // See which messages we even have to send
   if (opt.externalDedup) {
-    params.Entries = await dedupShouldEnqueueMulti(params.Entries, opt)
+    const promises = params.Entries.map(async m => ({ m, shouldEnqueue: await dedupShouldEnqueue(m, opt) }))
+    const results = await Promise.all(promises)
+    params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue).map(({ m }) => m)
     if (!params.Entries.length) return
   }
 

package/src/scheduler/jobExecutor.js

@@ -8,7 +8,7 @@ import { ChangeMessageVisibilityBatchCommand, DeleteMessageBatchCommand } from '
 import chalk from 'chalk'
 import Debug from 'debug'
 
-import { dedupSuccessfullyProcessedMulti } from '../dedup.js'
+import { dedupSuccessfullyProcessed } from '../dedup.js'
 import { getSQSClient } from '../sqs.js'
 
 const debug = Debug('qdone:jobExecutor')
@@ -220,11 +220,16 @@ export class JobExecutor {
           } else if (!this.opt.disableLog) {
             console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }))
           }
+
+          // Mark batch as processed for dedup
+          await Promise.all(
+            result.Successful.map(
+              e => dedupSuccessfullyProcessed(this.jobsByMessageId[e.Id], this.opt)
+            )
+          )
         }
         debug('DeleteMessageBatch returned', result)
 
-        // Mark batch as processed for dedup
-        await dedupSuccessfullyProcessedMulti(entries.map(e => this.jobsByMessageId[e.Id]), this.opt)
 
         // TODO Sentry
       }