qdone 2.0.48-alpha → 2.0.50-alpha

package/commonjs/src/defaults.js

@@ -48,7 +48,8 @@ exports.defaults = Object.freeze({
     delete: false,
     unpair: false,
     // Check
-    create: false
+    create: false,
+    overwrite: false
 });
 function validateInteger(opt, name) {
     const parsed = parseInt(opt[name], 10);
@@ -87,50 +88,50 @@ function getOptionsWithDefaults(options) {
     const dlq = options.dlq || !!(options['dlq-suffix'] || options['dlq-after'] || options['dlq-name'] || options.dlqSuffix || options.dlqAfter || options.dlqName);
     const opt = {
         // Shared
-        prefix: options.prefix === '' ? options.prefix : (options.prefix || exports.defaults.prefix),
-        failSuffix: options.failSuffix || options['fail-suffix'] || exports.defaults.failSuffix,
-        region: options.region || process.env.AWS_REGION || exports.defaults.region,
-        quiet: options.quiet || exports.defaults.quiet,
-        verbose: options.verbose || exports.defaults.verbose,
-        fifo: options.fifo || exports.defaults.fifo,
-        sentryDsn: options.sentryDsn || options['sentry-dsn'],
-        disableLog: options.disableLog || options['disable-log'] || exports.defaults.disableLog,
-        includeFailed: options.includeFailed || options['include-failed'] || exports.defaults.includeFailed,
-        includeDead: options.includeDead || options['include-dead'] || exports.defaults.includeDead,
-        externalDedup: options.externalDedup || options['external-dedup'] || exports.defaults.externalDedup,
-        dedupPeriod: options.dedupPeriod || options['dedup-period'] || exports.defaults.dedupPeriod,
-        dedupStats: options.dedupStats || options['dedup-stats'] || exports.defaults.dedupStats,
+        prefix: options.prefix === '' ? options.prefix : (options.prefix || process.env.QDONE_PREFIX || exports.defaults.prefix),
+        failSuffix: options.failSuffix || options['fail-suffix'] || process.env.QDONE_FAIL_SUFFIX || exports.defaults.failSuffix,
+        region: options.region || process.env.QDONE_REGION || process.env.AWS_REGION || exports.defaults.region,
+        quiet: options.quiet || process.env.QDONE_QUIET === 'true' || exports.defaults.quiet,
+        verbose: options.verbose || process.env.QDONE_VERBOSE === 'true' || exports.defaults.verbose,
+        fifo: options.fifo || process.env.QDONE_FIFO === 'true' || exports.defaults.fifo,
+        sentryDsn: options.sentryDsn || options['sentry-dsn'] || process.env.QDONE_SENTRY_DSN,
+        disableLog: options.disableLog || options['disable-log'] || process.env.QDONE_DISABLE_LOG === 'true' || exports.defaults.disableLog,
+        includeFailed: options.includeFailed || options['include-failed'] || process.env.QDONE_INCLUDE_FAILED === 'true' || exports.defaults.includeFailed,
+        includeDead: options.includeDead || options['include-dead'] || process.env.QDONE_INCLUDE_DEAD === 'true' || exports.defaults.includeDead,
+        externalDedup: options.externalDedup || options['external-dedup'] || process.env.QDONE_EXTERNAL_DEDUP === 'true' || exports.defaults.externalDedup,
+        dedupPeriod: options.dedupPeriod || options['dedup-period'] || process.env.QDONE_DEDUP_PERIOD || exports.defaults.dedupPeriod,
+        dedupStats: options.dedupStats || options['dedup-stats'] || process.env.QDONE_DEDUP_STATS === 'true' || exports.defaults.dedupStats,
         // Cache
-        cacheUri: options.cacheUri || options['cache-uri'] || exports.defaults.cacheUri,
-        cachePrefix: options.cachePrefix || options['cache-prefix'] || exports.defaults.cachePrefix,
-        cacheTtlSeconds: options.cacheTtlSeconds || options['cache-ttl-seconds'] || exports.defaults.cacheTtlSeconds,
+        cacheUri: options.cacheUri || options['cache-uri'] || process.env.QDONE_CACHE_URI || exports.defaults.cacheUri,
+        cachePrefix: options.cachePrefix || options['cache-prefix'] || process.env.QDONE_CACHE_PREFIX || exports.defaults.cachePrefix,
+        cacheTtlSeconds: options.cacheTtlSeconds || options['cache-ttl-seconds'] || process.env.QDONE_CACHE_TTL_SECONDS || exports.defaults.cacheTtlSeconds,
         // Enqueue
-        groupId: options.groupId || options['group-id'] || exports.defaults.groupId,
+        groupId: options.groupId || options['group-id'] || process.env.QDONE_GROUP_ID || exports.defaults.groupId,
         groupIdPerMessage: false,
-        deduplicationId: options.deduplicationId || options['deduplication-id'] || exports.defaults.deduplicationId,
-        dedupIdPerMessage: options.dedupIdPerMessage || options['dedup-id-per-message'] || exports.defaults.dedupIdPerMessage,
-        messageRetentionPeriod: options.messageRetentionPeriod || options['message-retention-period'] || exports.defaults.messageRetentionPeriod,
-        delay: options.delay || exports.defaults.delay,
-        sendRetries: options['send-retries'] || exports.defaults.sendRetries,
-        failDelay: options.failDelay || options['fail-delay'] || exports.defaults.failDelay,
+        deduplicationId: options.deduplicationId || options['deduplication-id'] || process.env.QDONE_DEDUPLICATION_ID || exports.defaults.deduplicationId,
+        dedupIdPerMessage: options.dedupIdPerMessage || options['dedup-id-per-message'] || process.env.QDONE_DEDUP_ID_PER_MESSAGE === 'true' || exports.defaults.dedupIdPerMessage,
+        messageRetentionPeriod: options.messageRetentionPeriod || options['message-retention-period'] || process.env.QDONE_MESSAGE_RETENTION_PERIOD || exports.defaults.messageRetentionPeriod,
+        delay: options.delay || process.env.QDONE_DELAY || exports.defaults.delay,
+        sendRetries: options.sendRetries || options['send-retries'] || process.env.QDONE_SEND_RETRIES || exports.defaults.sendRetries,
+        failDelay: options.failDelay || options['fail-delay'] || process.env.QDONE_FAIL_DELAY || exports.defaults.failDelay,
         dlq: dlq === false ? false : exports.defaults.dlq,
-        dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || exports.defaults.dlqSuffix,
-        dlqAfter: options.dlqAfter || options['dlq-after'] || exports.defaults.dlqAfter,
+        dlqSuffix: options.dlqSuffix || options['dlq-suffix'] || process.env.QDONE_DLQ_SUFFIX || exports.defaults.dlqSuffix,
+        dlqAfter: options.dlqAfter || options['dlq-after'] || process.env.QDONE_DLQ_AFTER || exports.defaults.dlqAfter,
         tags: options.tags || undefined,
         // Worker
-        waitTime: options.waitTime || options['wait-time'] || exports.defaults.waitTime,
-        killAfter: options.killAfter || options['kill-after'] || exports.defaults.killAfter,
-        archive: options.archive || exports.defaults.archive,
-        activeOnly: options.activeOnly || options['active-only'] || exports.defaults.activeOnly,
-        maxConcurrentJobs: options.maxConcurrentJobs || exports.defaults.maxConcurrentJobs,
-        maxMemoryPercent: options.maxMemoryPercent || exports.defaults.maxMemoryPercent,
+        waitTime: options.waitTime || options['wait-time'] || process.env.QDONE_WAIT_TIME || exports.defaults.waitTime,
+        killAfter: options.killAfter || options['kill-after'] || process.env.QDONE_KILL_AFTER || exports.defaults.killAfter,
+        archive: options.archive || process.env.QDONE_ARCHIVE === 'true' || exports.defaults.archive,
+        activeOnly: options.activeOnly || options['active-only'] || process.env.QDONE_ACTIVE_ONLY === 'true' || exports.defaults.activeOnly,
+        maxConcurrentJobs: options.maxConcurrentJobs || process.env.QDONE_MAX_CONCURRENT_JOBS || exports.defaults.maxConcurrentJobs,
+        maxMemoryPercent: options.maxMemoryPercent || process.env.QDONE_MAX_MEMORY_PERCENT || exports.defaults.maxMemoryPercent,
         // Idle Queues
-        idleFor: options.idleFor || options['idle-for'] || exports.defaults.idleFor,
-        delete: options.delete || exports.defaults.delete,
-        unpair: options.delete || exports.defaults.unpair,
+        idleFor: options.idleFor || options['idle-for'] || process.env.QDONE_IDLE_FOR || exports.defaults.idleFor,
+        delete: options.delete || process.env.QDONE_DELETE === 'true' || exports.defaults.delete,
+        unpair: options.unpair || process.env.QDONE_UNPAIR === 'true' || exports.defaults.unpair,
         // Check
-        create: options.create || exports.defaults.create,
-        overwrite: options.overwrite || exports.defaults.overwrite
+        create: options.create || process.env.QDONE_CREATE === 'true' || exports.defaults.create,
+        overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || exports.defaults.overwrite
     };
     // Setting this env here means we don't have to in AWS SDK constructors
     process.env.AWS_REGION = opt.region;

package/commonjs/src/idleQueues.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.idleQueues = exports.processQueuePair = exports.processQueue = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
+exports.idleQueues = exports.processQueueSet = exports.processQueue = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
 /**
  * Implementation of checks and caching of checks to determine if queues are idle.
  */
@@ -39,6 +39,7 @@ const metricNames = [
  * Actual SQS call, used in conjunction with cache.
  */
 async function _cheapIdleCheck(qname, qrl, opt) {
+    debug('_cheapIdleCheck', qname, qrl);
     try {
         const client = (0, sqs_js_1.getSQSClient)();
         const cmd = new client_sqs_1.GetQueueAttributesCommand({ AttributeNames: exports.attributeNames, QueueUrl: qrl });
@@ -48,12 +49,14 @@ async function _cheapIdleCheck(qname, qrl, opt) {
         result.queue = qname.slice(opt.prefix.length);
         // We are idle if all the messages attributes are zero
         result.idle = exports.attributeNames.filter(k => result[k] === '0').length === exports.attributeNames.length;
+        result.exists = true;
+        debug({ result, SQS: 1 });
         return { result, SQS: 1 };
     }
     catch (e) {
+        debug({ _cheapIdleCheck: e });
         if (e instanceof client_sqs_1.QueueDoesNotExist) {
-            // Count deleted queues as idle
-            return { result: { idle: true }, SQS: 1 };
+            return { result: { idle: undefined, exists: false }, SQS: 1 };
         }
         else {
             throw e;
@@ -66,6 +69,7 @@ exports._cheapIdleCheck = _cheapIdleCheck;
  * at this immediate moment.
  */
 async function cheapIdleCheck(qname, qrl, opt) {
+    debug('cheapIdleCheck', qname, qrl);
     // Just call the API if we don't have a cache
     if (!opt.cacheUri)
         return _cheapIdleCheck(qname, qrl, opt);
@@ -129,18 +133,19 @@ async function checkIdle(qname, qrl, opt) {
     debug('checkIdle', qname, qrl);
     const { result: cheapResult, SQS } = await cheapIdleCheck(qname, qrl, opt);
     debug('cheapResult', cheapResult);
-    // Short circuit further calls if cheap result shows data
-    if (cheapResult.idle === false) {
+    // Short circuit further calls if cheap result is conclusive
+    if (cheapResult.idle === false || cheapResult.exists === false) {
         return {
             queue: qname.slice(opt.prefix.length),
             cheap: cheapResult,
-            idle: false,
+            idle: cheapResult.idle,
+            exists: cheapResult.exists,
             apiCalls: { SQS, CloudWatch: 0 }
         };
     }
     // If we get here, there's nothing in the queue at the moment,
     // so we have to check metrics one at a time
-    const apiCalls = { SQS: 1, CloudWatch: 0 };
+    const apiCalls = { SQS, CloudWatch: 0 };
     const results = [];
     let idle = true;
     for (const metricName of metricNames) {
@@ -159,7 +164,8 @@ async function checkIdle(qname, qrl, opt) {
         queue: qname.slice(opt.prefix.length),
         cheap: cheapResult,
         apiCalls,
-        idle
+        idle,
+        exists: true
     }, ...results // merge in results from CloudWatch
     );
     debug('checkIdle stats', stats);
@@ -211,90 +217,99 @@ async function processQueue(qname, qrl, opt) {
 }
 exports.processQueue = processQueue;
 /**
- * Processes a queue and its fail queue, treating them as a unit.
+ * Processes a queue and its fail and delete queue, treating them as a unit.
  */
-async function processQueuePair(qname, qrl, opt) {
+async function processQueueSet(qname, qrl, opt) {
     const isFifo = qname.endsWith('.fifo');
     const normalizeOptions = Object.assign({}, opt, { fifo: isFifo });
+    // Generate DLQ name/url
+    const dqname = (0, qrlCache_js_1.normalizeDLQName)(qname, normalizeOptions);
+    const dqrl = (0, qrlCache_js_1.normalizeDLQName)(dqname, normalizeOptions);
     // Generate fail queue name/url
     const fqname = (0, qrlCache_js_1.normalizeFailQueueName)(qname, normalizeOptions);
     const fqrl = (0, qrlCache_js_1.normalizeFailQueueName)(fqname, normalizeOptions);
+    debug({ qname, qrl, dqname, dqrl, fqname, fqrl });
     // Idle check
-    const result = await checkIdle(qname, qrl, opt);
-    debug('result', result);
-    // Queue is active
-    const active = !result.idle;
-    if (active) {
-        if (opt.verbose)
-            console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-        return result;
-    }
+    const qresult = await checkIdle(qname, qrl, opt);
+    const fqresult = await checkIdle(fqname, fqrl, opt);
+    const dqresult = await checkIdle(dqname, dqrl, opt);
+    debug({ qresult, fqresult, dqresult });
+    // Start building return value
+    const result = Object.assign({
+        queue: qname,
+        idle: (qresult.idle &&
+            (!fqresult.exists || fqresult.idle) &&
+            (!fqresult.exists || dqresult.idle)),
+        apiCalls: {
+            SQS: qresult.apiCalls.SQS + fqresult.apiCalls.SQS + dqresult.apiCalls.SQS,
+            CloudWatch: qresult.apiCalls.CloudWatch + fqresult.apiCalls.CloudWatch + dqresult.apiCalls.CloudWatch
+        }
+    });
     // Queue is idle
-    if (opt.verbose)
+    if (qresult.idle && opt.verbose)
         console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-    // Check fail queue
-    try {
-        const fresult = await checkIdle(fqname, fqrl, opt);
-        debug('fresult', fresult);
-        const idleCheckResult = Object.assign(result, { idle: result.idle && fresult.idle, failq: fresult }, {
-            apiCalls: {
-                SQS: result.apiCalls.SQS + fresult.apiCalls.SQS,
-                CloudWatch: result.apiCalls.CloudWatch + fresult.apiCalls.CloudWatch
+    if (fqresult.idle && opt.verbose)
+        console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
+    if (dqresult.idle && opt.verbose)
+        console.error(chalk_1.default.blue('Queue ') + dqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
+    // Delete if all are idle
+    const canDelete = ((qresult.idle || qresult.exists === false) &&
+        (fqresult.idle || fqresult.exists === false) &&
+        (dqresult.idle || dqresult.exists === false));
+    debug({ canDelete });
+    if (opt.delete && canDelete) {
+        // Normal
+        const qdresult = await (async () => {
+            debug({ qresult });
+            try {
+                if (qresult.idle)
+                    return deleteQueue(qname, qrl, opt);
             }
-        });
-        // Queue is active
-        const factive = !fresult.idle;
-        if (factive) {
-            if (opt.verbose)
-                console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-            return idleCheckResult;
+            catch (e) {
+                if (!(e instanceof client_sqs_1.QueueDoesNotExist))
+                    throw e;
+            }
+        })();
+        if (qdresult) {
+            result.apiCalls.SQS += qdresult.apiCalls.SQS;
         }
-        // Queue is idle
-        if (opt.verbose)
-            console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-        // Trigger a delete if the user wants it
-        if (!opt.delete)
-            return idleCheckResult;
-        const [dresult, dfresult] = await Promise.all([
-            deleteQueue(qname, qrl, opt),
-            deleteQueue(fqname, fqrl, opt)
-        ]);
-        return Object.assign(idleCheckResult, {
-            apiCalls: {
-                // Sum the SQS calls across all four
-                SQS: [result, fresult, dresult, dfresult]
-                    .map(r => r.apiCalls.SQS)
-                    .reduce((a, b) => a + b, 0),
-                // Sum the CloudWatch calls across all four
-                CloudWatch: [result, fresult, dresult, dfresult]
-                    .map(r => r.apiCalls.CloudWatch)
-                    .reduce((a, b) => a + b, 0)
+        debug({ qdresult });
+        // Fail
+        const fqdresult = await (async () => {
+            debug({ fqresult });
+            try {
+                if (fqresult.idle)
+                    return deleteQueue(fqname, fqrl, opt);
             }
-        });
-    }
-    catch (e) {
-        // Handle the case where the fail queue has been deleted or was never
-        // created for some reason
-        if (!(e instanceof client_sqs_1.QueueDoesNotExist))
-            throw e;
-        // Fail queue doesn't exist if we get here
-        if (opt.verbose)
-            console.error(chalk_1.default.blue('Queue ') + fqname.slice(opt.prefix.length) + chalk_1.default.blue(' does not exist.'));
-        // Handle delete
-        if (!opt.delete)
-            return result;
-        const deleteResult = await deleteQueue(qname, qrl, opt);
-        const resultIncludingDelete = Object.assign(result, {
-            deleted: deleteResult.deleted,
-            apiCalls: {
-                SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
-                CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
+            catch (e) {
+                if (!(e instanceof client_sqs_1.QueueDoesNotExist))
+                    throw e;
             }
-        });
-        return resultIncludingDelete;
+        })();
+        if (fqdresult) {
+            result.apiCalls.SQS += fqdresult.apiCalls.SQS;
+        }
+        debug({ fqdresult });
+        // Dead
+        const dqdresult = await (async () => {
+            debug({ dqresult });
+            try {
+                if (dqresult.idle)
+                    return deleteQueue(dqname, dqrl, opt);
+            }
+            catch (e) {
+                if (!(e instanceof client_sqs_1.QueueDoesNotExist))
+                    throw e;
+            }
+        })();
+        if (dqdresult) {
+            result.apiCalls.SQS += dqdresult.apiCalls.SQS;
+        }
+        debug({ dqdresult });
     }
+    return result;
 }
-exports.processQueuePair = processQueuePair;
+exports.processQueueSet = processQueueSet;
 //
 // Resolve queues for listening loop listen
 //
@@ -315,7 +330,11 @@ async function idleQueues(queues, options) {
         const sufFifo = opt.failSuffix + qrlCache_js_1.fifoSuffix;
         const isFail = entry.qname.endsWith(suf);
         const isFifoFail = entry.qname.endsWith(sufFifo);
-        return opt.includeFailed ? true : (!isFail && !isFifoFail);
+        const sufDead = opt.dlqSuffix;
+        const sufFifoDead = opt.dlqSuffix + qrlCache_js_1.fifoSuffix;
+        const isDead = entry.qname.endsWith(sufDead);
+        const isFifoDead = entry.qname.endsWith(sufFifoDead);
+        return opt.includeFailed ? true : (!isFail && !isFifoFail && !isDead && !isFifoDead);
     });
     // But only if we have queues to remove
     if (filteredEntries.length) {
@@ -327,7 +346,7 @@ async function idleQueues(queues, options) {
         // Check each queue in parallel
         if (opt.unpair)
             return Promise.all(filteredEntries.map(e => processQueue(e.qname, e.qrl, opt)));
-        return Promise.all(filteredEntries.map(e => processQueuePair(e.qname, e.qrl, opt)));
+        return Promise.all(filteredEntries.map(e => processQueueSet(e.qname, e.qrl, opt)));
     }
     // Otherwise, let caller know
     return 'noQueues';