qdone 2.0.37-alpha → 2.0.39-alpha

package/commonjs/src/dedup.js

@@ -209,6 +209,7 @@ exports.dedupShouldEnqueueMulti = dedupShouldEnqueueMulti;
  * @returns {Number} 1 if a cache key was deleted, otherwise 0
  */
 async function dedupSuccessfullyProcessed(message, opt) {
+    debug({ dedupSuccessfullyProcessed: { message, opt } });
     const client = (0, cache_js_1.getCacheClient)(opt);
     const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
     if (dedupId) {

package/commonjs/src/enqueue.js

@@ -276,9 +276,9 @@ async function sendMessageBatch(qrl, messages, opt) {
     if (opt.externalDedup) {
         const promises = params.Entries.map(async (m) => ({ m, shouldEnqueue: await (0, dedup_js_1.dedupShouldEnqueue)(m, opt) }));
         const results = await Promise.all(promises);
-        params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue);
+        params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue).map(({ m }) => m);
         if (!params.Entries.length)
-            return;
+            return { Failed: [], Successful: [] };
     }
     // Send them
     const client = (0, sqs_js_1.getSQSClient)();

package/commonjs/src/scheduler/jobExecutor.js

@@ -216,10 +216,10 @@ class JobExecutor {
                     else if (!this.opt.disableLog) {
                         console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }));
                     }
+                    // Mark batch as processed for dedup
+                    await Promise.all(result.Successful.map(e => (0, dedup_js_1.dedupSuccessfullyProcessed)(this.jobsByMessageId[e.Id].message, this.opt)));
                 }
                 debug('DeleteMessageBatch returned', result);
-                // Mark batch as processed for dedup
-                await Promise.all(entries.map(e => (0, dedup_js_1.dedupSuccessfullyProcessed)(this.jobsByMessageId[e.Id], this.opt)));
                 // TODO Sentry
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.37-alpha",
+  "version": "2.0.39-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/dedup.js

@@ -202,6 +202,7 @@ export async function dedupShouldEnqueueMulti (messages, opt) {
  * @returns {Number} 1 if a cache key was deleted, otherwise 0
  */
 export async function dedupSuccessfullyProcessed (message, opt) {
+  debug({ dedupSuccessfullyProcessed: { message, opt } })
   const client = getCacheClient(opt)
   const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
   if (dedupId) {

package/src/enqueue.js

@@ -275,8 +275,8 @@ export async function sendMessageBatch (qrl, messages, opt) {
   if (opt.externalDedup) {
     const promises = params.Entries.map(async m => ({ m, shouldEnqueue: await dedupShouldEnqueue(m, opt) }))
     const results = await Promise.all(promises)
-    params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue)
-    if (!params.Entries.length) return
+    params.Entries = results.filter(({ shouldEnqueue }) => shouldEnqueue).map(({ m }) => m)
+    if (!params.Entries.length) return { Failed: [], Successful: [] }
   }
 
   // Send them

package/src/scheduler/jobExecutor.js

@@ -220,11 +220,16 @@ export class JobExecutor {
           } else if (!this.opt.disableLog) {
             console.log(JSON.stringify({ event: 'DELETE_MESSAGES', timestamp: start, count, qrl }))
           }
+
+          // Mark batch as processed for dedup
+          await Promise.all(
+            result.Successful.map(
+              e => dedupSuccessfullyProcessed(this.jobsByMessageId[e.Id].message, this.opt)
+            )
+          )
         }
         debug('DeleteMessageBatch returned', result)
 
-        // Mark batch as processed for dedup
-        await Promise.all(entries.map(e => dedupSuccessfullyProcessed(this.jobsByMessageId[e.Id], this.opt)))
 
         // TODO Sentry
       }