qdone 2.0.51-alpha → 2.0.52-alpha

package/README.md

@@ -561,13 +561,8 @@ If a queue name ends with the * (wildcard) character, worker will listen on all
 
     -o, --idle-for number   Minutes of inactivity after which a queue is considered
                             idle. [default: 60]
-    --delete                Delete the queue if it is idle. The fail queue also must be
-                            idle unless you use --unpair.
-    --unpair                Treat queues and their fail queues as independent. By default
-                            they are treated as a unit.
-    --include-failed        When using '*' do not ignore fail queues. This option only
-                            applies if you use --unpair. Otherwise, queues and fail queues
-                            are treated as a unit.
+    --delete                Delete the queue if it is idle along with its corresponding
+                            fail queue and dlq.
     --prefix string         Prefix to place at the front of each SQS queue name [default: qdone_]
     --fail-suffix string    Suffix to append to each queue to generate fail queue name [default: _failed]
     --region string         AWS region for Queues [default: us-east-1]

package/commonjs/src/defaults.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.validateMessageOptions = exports.validateInteger = exports.defaults = void 0;
+exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.validateMessageOptions = exports.validateQueueName = exports.validateInteger = exports.defaults = void 0;
 /**
  * Default options for qdone. Accepts a command line options object and
  * returns nicely-named options.
@@ -46,7 +46,6 @@ exports.defaults = Object.freeze({
     // Idle Queues
     idleFor: 60,
     delete: false,
-    unpair: false,
     // Check
     create: false,
     overwrite: false
@@ -58,6 +57,14 @@ function validateInteger(opt, name) {
     return parsed;
 }
 exports.validateInteger = validateInteger;
+function validateQueueName(opt, name) {
+    if (typeof name !== 'string')
+        throw new Error(`${name} must be a string.`);
+    if (!name.match(/^[a-z0-9-_]+$/i))
+        throw new Error(`${name} can contain only numbers, letters, hypens and underscores.`);
+    return name;
+}
+exports.validateQueueName = validateQueueName;
 function validateMessageOptions(messageOptions) {
     const validKeys = ['deduplicationId', 'groupId'];
     if (typeof messageOptions === 'object' &&
@@ -128,7 +135,6 @@ function getOptionsWithDefaults(options) {
         // Idle Queues
         idleFor: options.idleFor || options['idle-for'] || process.env.QDONE_IDLE_FOR || exports.defaults.idleFor,
         delete: options.delete || process.env.QDONE_DELETE === 'true' || exports.defaults.delete,
-        unpair: options.unpair || process.env.QDONE_UNPAIR === 'true' || exports.defaults.unpair,
         // Check
         create: options.create || process.env.QDONE_CREATE === 'true' || exports.defaults.create,
         overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || exports.defaults.overwrite
@@ -148,6 +154,10 @@ function getOptionsWithDefaults(options) {
     opt.maxConcurrentJobs = validateInteger(opt, 'maxConcurrentJobs');
     opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent');
     opt.idleFor = validateInteger(opt, 'idleFor');
+    validateQueueName(opt, 'region');
+    validateQueueName(opt, 'prefix');
+    validateQueueName(opt, 'failSuffix');
+    validateQueueName(opt, 'dlqSuffix');
     // Validate dedup args
     if (opt.externalDedup && !opt.cacheUri)
         throw new Error('--external-dedup requires the --cache-uri argument');

package/commonjs/src/idleQueues.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.idleQueues = exports.processQueueSet = exports.processQueue = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
+exports.idleQueues = exports.stripSuffixes = exports.processQueueSet = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
 /**
  * Implementation of checks and caching of checks to determine if queues are idle.
  */
@@ -187,35 +187,6 @@ async function deleteQueue(qname, qrl, opt) {
     };
 }
 exports.deleteQueue = deleteQueue;
-/**
- * Processes a single queue, checking for idle, deleting if applicable.
- */
-async function processQueue(qname, qrl, opt) {
-    const result = await checkIdle(qname, qrl, opt);
-    debug(qname, result);
-    // Queue is active
-    if (!result.idle) {
-        // Notify and return
-        if (opt.verbose)
-            console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'active' + chalk_1.default.blue(' in the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-        return result;
-    }
-    // Queue is idle
-    if (opt.verbose)
-        console.error(chalk_1.default.blue('Queue ') + qname.slice(opt.prefix.length) + chalk_1.default.blue(' has been ') + 'idle' + chalk_1.default.blue(' for the last ') + opt.idleFor + chalk_1.default.blue(' minutes.'));
-    if (opt.delete) {
-        const deleteResult = await deleteQueue(qname, qrl, opt);
-        const resultIncludingDelete = Object.assign(result, {
-            deleted: deleteResult.deleted,
-            apiCalls: {
-                SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
-                CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
-            }
-        });
-        return resultIncludingDelete;
-    }
-}
-exports.processQueue = processQueue;
 /**
  * Processes a queue and its fail and delete queue, treating them as a unit.
  */
@@ -311,6 +282,14 @@ async function processQueueSet(qname, qrl, opt) {
 }
 exports.processQueueSet = processQueueSet;
 //
+// Strips failed and dlq suffix from a queue name or URL
+//
+function stripSuffixes(queueName, opt) {
+    const suffixFinder = new RegExp(`(${opt.dlqSuffix}|${opt.failSuffix}){1}(|${qrlCache_js_1.fifoSuffix})$`);
+    return queueName.replace(suffixFinder, '$2');
+}
+exports.stripSuffixes = stripSuffixes;
+//
 // Resolve queues for listening loop listen
 //
 async function idleQueues(queues, options) {
@@ -324,17 +303,18 @@ async function idleQueues(queues, options) {
         console.error(chalk_1.default.blue('  done'));
         console.error();
     }
-    // Filter out any queue ending in suffix unless --include-failed is set
+    // Filter out failed and dead queues, but if we have an orphaned fail or
+    // dead queue, keep the original parent queue name so that orphans can be
+    // deleted.
+    const queueNames = new Set();
     const filteredEntries = entries.filter(entry => {
-        const suf = opt.failSuffix;
-        const sufFifo = opt.failSuffix + qrlCache_js_1.fifoSuffix;
-        const isFail = entry.qname.endsWith(suf);
-        const isFifoFail = entry.qname.endsWith(sufFifo);
-        const sufDead = opt.dlqSuffix;
-        const sufFifoDead = opt.dlqSuffix + qrlCache_js_1.fifoSuffix;
-        const isDead = entry.qname.endsWith(sufDead);
-        const isFifoDead = entry.qname.endsWith(sufFifoDead);
-        return opt.includeFailed ? true : (!isFail && !isFifoFail && !isDead && !isFifoDead);
+        const stripped = stripSuffixes(entry.qname, opt);
+        if (queueNames.has(stripped))
+            return false;
+        queueNames.add(stripped);
+        entry.qname = stripped;
+        entry.qrl = stripSuffixes(entry.qrl, opt);
+        return true;
     });
     // But only if we have queues to remove
     if (filteredEntries.length) {
@@ -344,8 +324,6 @@ async function idleQueues(queues, options) {
             console.error();
         }
         // Check each queue in parallel
-        if (opt.unpair)
-            return Promise.all(filteredEntries.map(e => processQueue(e.qname, e.qrl, opt)));
         return Promise.all(filteredEntries.map(e => processQueueSet(e.qname, e.qrl, opt)));
     }
     // Otherwise, let caller know

package/commonjs/src/monitor.js

@@ -17,9 +17,11 @@ const debug = (0, debug_1.default)('qdone:monitor');
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
 async function monitor(queue, save, options) {
+    if (queue.endsWith('.fifo'))
+        options.fifo = true;
     const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
     const queueName = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
-    debug({ opt, queueName });
+    debug({ options, opt, queue, queueName });
     const data = await getAggregateData(queueName);
     console.log(data);
     if (save) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.0.51-alpha",
+  "version": "2.0.52-alpha",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/cli.js

@@ -440,9 +440,7 @@ export async function worker (argv, testHook) {
 export async function idleQueues (argv, testHook) {
   const optionDefinitions = [
     { name: 'idle-for', alias: 'o', type: Number, defaultValue: defaults.idleFor, description: `Minutes of inactivity after which a queue is considered idle. [default: ${defaults.idleFor}]` },
-    { name: 'delete', type: Boolean, description: 'Delete the queue if it is idle. The fail queue also must be idle unless you use --unpair.' },
-    { name: 'unpair', type: Boolean, description: 'Treat queues and their fail queues as independent. By default they are treated as a unit.' },
-    { name: 'include-failed', type: Boolean, description: 'When using \'*\' do not ignore fail queues. This option only applies if you use --unpair. Otherwise, queues and fail queues are treated as a unit.' }
+    { name: 'delete', type: Boolean, description: 'Delete the queue if it is idle along with its corresponding fail queue and dlq.' }
   ].concat(globalOptionDefinitions)
 
   const usageSections = [

package/src/defaults.js

@@ -47,7 +47,6 @@ export const defaults = Object.freeze({
   // Idle Queues
   idleFor: 60,
   delete: false,
-  unpair: false,
 
   // Check
   create: false,
@@ -60,6 +59,12 @@ export function validateInteger (opt, name) {
   return parsed
 }
 
+export function validateQueueName (opt, name) {
+  if (typeof name !== 'string') throw new Error(`${name} must be a string.`)
+  if (!name.match(/^[a-z0-9-_]+$/i)) throw new Error(`${name} can contain only numbers, letters, hypens and underscores.`)
+  return name
+}
+
 export function validateMessageOptions (messageOptions) {
   const validKeys = ['deduplicationId', 'groupId']
   if (typeof messageOptions === 'object' &&
@@ -134,7 +139,6 @@ export function getOptionsWithDefaults (options) {
     // Idle Queues
     idleFor: options.idleFor || options['idle-for'] || process.env.QDONE_IDLE_FOR || defaults.idleFor,
     delete: options.delete || process.env.QDONE_DELETE === 'true' || defaults.delete,
-    unpair: options.unpair || process.env.QDONE_UNPAIR === 'true' || defaults.unpair,
 
     // Check
     create: options.create || process.env.QDONE_CREATE === 'true' || defaults.create,
@@ -158,6 +162,11 @@ export function getOptionsWithDefaults (options) {
   opt.maxMemoryPercent = validateInteger(opt, 'maxMemoryPercent')
   opt.idleFor = validateInteger(opt, 'idleFor')
 
+  validateQueueName(opt, 'region')
+  validateQueueName(opt, 'prefix')
+  validateQueueName(opt, 'failSuffix')
+  validateQueueName(opt, 'dlqSuffix')
+
   // Validate dedup args
   if (opt.externalDedup && !opt.cacheUri) throw new Error('--external-dedup requires the --cache-uri argument')
   if (opt.externalDedup && (opt.dedupPeriod < 1)) throw new Error('--external-dedup of redis requires a --dedup-period > 1 second')

package/src/idleQueues.js

@@ -186,35 +186,6 @@ export async function deleteQueue (qname, qrl, opt) {
   }
 }
 
-/**
- * Processes a single queue, checking for idle, deleting if applicable.
- */
-export async function processQueue (qname, qrl, opt) {
-  const result = await checkIdle(qname, qrl, opt)
-  debug(qname, result)
-
-  // Queue is active
-  if (!result.idle) {
-    // Notify and return
-    if (opt.verbose) console.error(chalk.blue('Queue ') + qname.slice(opt.prefix.length) + chalk.blue(' has been ') + 'active' + chalk.blue(' in the last ') + opt.idleFor + chalk.blue(' minutes.'))
-    return result
-  }
-
-  // Queue is idle
-  if (opt.verbose) console.error(chalk.blue('Queue ') + qname.slice(opt.prefix.length) + chalk.blue(' has been ') + 'idle' + chalk.blue(' for the last ') + opt.idleFor + chalk.blue(' minutes.'))
-  if (opt.delete) {
-    const deleteResult = await deleteQueue(qname, qrl, opt)
-    const resultIncludingDelete = Object.assign(result, {
-      deleted: deleteResult.deleted,
-      apiCalls: {
-        SQS: result.apiCalls.SQS + deleteResult.apiCalls.SQS,
-        CloudWatch: result.apiCalls.CloudWatch + deleteResult.apiCalls.CloudWatch
-      }
-    })
-    return resultIncludingDelete
-  }
-}
-
 /**
  * Processes a queue and its fail and delete queue, treating them as a unit.
  */
@@ -308,6 +279,14 @@ export async function processQueueSet (qname, qrl, opt) {
   return result
 }
 
+//
+// Strips failed and dlq suffix from a queue name or URL
+//
+export function stripSuffixes (queueName, opt) {
+  const suffixFinder = new RegExp(`(${opt.dlqSuffix}|${opt.failSuffix}){1}(|${fifoSuffix})$`)
+  return queueName.replace(suffixFinder, '$2')
+}
+
 //
 // Resolve queues for listening loop listen
 //
@@ -322,17 +301,17 @@ export async function idleQueues (queues, options) {
     console.error()
   }
 
-  // Filter out any queue ending in suffix unless --include-failed is set
+  // Filter out failed and dead queues, but if we have an orphaned fail or
+  // dead queue, keep the original parent queue name so that orphans can be
+  // deleted.
+  const queueNames = new Set()
   const filteredEntries = entries.filter(entry => {
-    const suf = opt.failSuffix
-    const sufFifo = opt.failSuffix + fifoSuffix
-    const isFail = entry.qname.endsWith(suf)
-    const isFifoFail = entry.qname.endsWith(sufFifo)
-    const sufDead = opt.dlqSuffix
-    const sufFifoDead = opt.dlqSuffix + fifoSuffix
-    const isDead = entry.qname.endsWith(sufDead)
-    const isFifoDead = entry.qname.endsWith(sufFifoDead)
-    return opt.includeFailed ? true : (!isFail && !isFifoFail && !isDead && !isFifoDead)
+    const stripped = stripSuffixes(entry.qname, opt)
+    if (queueNames.has(stripped)) return false
+    queueNames.add(stripped)
+    entry.qname = stripped
+    entry.qrl = stripSuffixes(entry.qrl, opt)
+    return true
   })
 
   // But only if we have queues to remove
@@ -345,7 +324,6 @@ export async function idleQueues (queues, options) {
       console.error()
     }
     // Check each queue in parallel
-    if (opt.unpair) return Promise.all(filteredEntries.map(e => processQueue(e.qname, e.qrl, opt)))
     return Promise.all(filteredEntries.map(e => processQueueSet(e.qname, e.qrl, opt)))
   }
 

package/src/monitor.js

@@ -13,9 +13,10 @@ const debug = Debug('qdone:monitor')
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
 export async function monitor (queue, save, options) {
+  if (queue.endsWith('.fifo')) options.fifo = true
   const opt = getOptionsWithDefaults(options)
   const queueName = normalizeQueueName(queue, opt)
-  debug({ opt, queueName })
+  debug({ options, opt, queue, queueName })
   const data = await getAggregateData(queueName)
   console.log(data)
   if (save) {