qdone 2.1.1 → 2.2.1

package/commonjs/src/cache.js

@@ -3,7 +3,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setCache = exports.getCache = exports.shutdownCache = exports.getCacheClient = void 0;
+exports.getCacheClient = getCacheClient;
+exports.shutdownCache = shutdownCache;
+exports.getCache = getCache;
+exports.setCache = setCache;
 const ioredis_1 = __importDefault(require("ioredis"));
 const url_1 = require("url");
 const debug_1 = __importDefault(require("debug"));
@@ -38,13 +41,11 @@ function getCacheClient(opt) {
         throw new UsageError('Caching requires the --cache-uri option');
     }
 }
-exports.getCacheClient = getCacheClient;
 function shutdownCache() {
     if (client)
         client.quit();
     client = undefined;
 }
-exports.shutdownCache = shutdownCache;
 /**
  * Returns a promise for the item. Resolves to false if cache is empty, object
  * if it is found.
@@ -57,7 +58,6 @@ async function getCache(key, opt) {
     debug({ action: 'getCache got', cacheKey, result });
     return result ? JSON.parse(result) : undefined;
 }
-exports.getCache = getCache;
 /**
  * Returns a promise for the status. Encodes object as JSON
  */
@@ -68,5 +68,4 @@ async function setCache(key, value, opt) {
     debug({ action: 'setCache', cacheKey, value });
     return client.setex(cacheKey, opt.cacheTtlSeconds, encoded);
 }
-exports.setCache = setCache;
 debug('loaded');

package/commonjs/src/cloudWatch.js

@@ -6,7 +6,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.putAggregateData = exports.setCloudWatchClient = exports.getCloudWatchClient = void 0;
+exports.getCloudWatchClient = getCloudWatchClient;
+exports.setCloudWatchClient = setCloudWatchClient;
+exports.putAggregateData = putAggregateData;
 const client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
 const debug_1 = __importDefault(require("debug"));
 const debug = (0, debug_1.default)('qdone:cloudWatch');
@@ -20,14 +22,12 @@ function getCloudWatchClient() {
     client = new client_cloudwatch_1.CloudWatchClient();
     return client;
 }
-exports.getCloudWatchClient = getCloudWatchClient;
 /**
  * Utility function to set the client explicitly, used in testing.
  */
 function setCloudWatchClient(explicitClient) {
     client = explicitClient;
 }
-exports.setCloudWatchClient = setCloudWatchClient;
 /**
  * Takes data in the form returned by getAggregageData() and pushes it to
  * CloudWatch metrics under the given queueName.
@@ -90,6 +90,16 @@ async function putAggregateData(total, timestamp) {
                 Timestamp: now,
                 Value: total.ApproximateNumberOfMessagesNotVisible || 0,
                 Unit: 'Count'
+            },
+            {
+                MetricName: 'ApproximateAgeOfOldestMessage',
+                Dimensions: [{
+                        Name: 'queueName',
+                        Value: total.queueName
+                    }],
+                Timestamp: now,
+                Value: total.ApproximateAgeOfOldestMessage || 0,
+                Unit: 'Seconds'
             }
         ]
     };
@@ -98,5 +108,4 @@ async function putAggregateData(total, timestamp) {
     const response = await client.send(command);
     debug({ response });
 }
-exports.putAggregateData = putAggregateData;
 debug('loaded');

package/commonjs/src/consumer.js

@@ -6,7 +6,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.processMessages = exports.getMessages = exports.requestShutdown = void 0;
+exports.requestShutdown = requestShutdown;
+exports.getMessages = getMessages;
+exports.processMessages = processMessages;
 const os_1 = require("os");
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const chalk_1 = __importDefault(require("chalk"));
@@ -30,7 +32,6 @@ async function requestShutdown() {
     }
     debug('requestShutdown done');
 }
-exports.requestShutdown = requestShutdown;
 async function getMessages(qrl, opt, maxMessages) {
     const params = {
         AttributeNames: ['All'],
@@ -44,7 +45,6 @@ async function getMessages(qrl, opt, maxMessages) {
     // debug('ReceiveMessage response', response)
     return response.Messages || [];
 }
-exports.getMessages = getMessages;
 //
 // Consumer
 //
@@ -170,4 +170,3 @@ async function processMessages(queues, callback, options) {
     }
     debug('after all');
 }
-exports.processMessages = processMessages;

package/commonjs/src/dedup.js

@@ -3,7 +3,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.dedupSuccessfullyProcessedMulti = exports.dedupSuccessfullyProcessed = exports.dedupShouldEnqueueMulti = exports.dedupShouldEnqueue = exports.statMaintenance = exports.updateStats = exports.addDedupParamsToMessage = exports.getCacheKey = exports.getDeduplicationId = void 0;
+exports.getDeduplicationId = getDeduplicationId;
+exports.getCacheKey = getCacheKey;
+exports.addDedupParamsToMessage = addDedupParamsToMessage;
+exports.updateStats = updateStats;
+exports.statMaintenance = statMaintenance;
+exports.dedupShouldEnqueue = dedupShouldEnqueue;
+exports.dedupShouldEnqueueMulti = dedupShouldEnqueueMulti;
+exports.dedupSuccessfullyProcessed = dedupSuccessfullyProcessed;
+exports.dedupSuccessfullyProcessedMulti = dedupSuccessfullyProcessedMulti;
 const crypto_1 = require("crypto");
 const uuid_1 = require("uuid");
 const cache_js_1 = require("./cache.js");
@@ -30,7 +38,6 @@ function getDeduplicationId(dedupContent, opt) {
     const id = `sha1:{${hash}}:body:${truncated}`;
     return id;
 }
-exports.getDeduplicationId = getDeduplicationId;
 /**
  * Returns the cache key given a deduplication id.
  * @param {String} dedupId - a deduplication id returned from getDeduplicationId
@@ -42,7 +49,6 @@ function getCacheKey(dedupId, opt) {
     debug({ getCacheKey: { cacheKey } });
     return cacheKey;
 }
-exports.getCacheKey = getCacheKey;
 /**
  * Modifies a message (parameters to SendMessageCommand) to add the parameters
  * for whatever deduplication options the caller has set.
@@ -88,7 +94,6 @@ function addDedupParamsToMessage(message, opt, messageOptions) {
     }
     return message;
 }
-exports.addDedupParamsToMessage = addDedupParamsToMessage;
 /**
  * Updates statistics in redis, of which there are two:
  * 1. duplicateSet - a set who's members are cache keys and scores are the number of duplicate
@@ -113,7 +118,6 @@ async function updateStats(cacheKey, duplicates, expireAt, opt, pipeline) {
             await pipeline.exec();
     }
 }
-exports.updateStats = updateStats;
 /**
  * Removes expired items from stats.
  */
@@ -135,7 +139,6 @@ async function statMaintenance(opt) {
         debug({ statMaintenance: { result } });
     }
 }
-exports.statMaintenance = statMaintenance;
 /**
  * Determines whether we should enqueue this message or whether it is a duplicate.
  * Returns true if enqueuing the message would not result in a duplicate.
@@ -160,7 +163,6 @@ async function dedupShouldEnqueue(message, opt) {
     }
     return false;
 }
-exports.dedupShouldEnqueue = dedupShouldEnqueue;
 /**
  * Determines which messages we should enqueue, returning only those that
  * would not be duplicates.
@@ -203,7 +205,6 @@ async function dedupShouldEnqueueMulti(messages, opt) {
         await statsPipeline.exec();
     return messagesToEnqueue;
 }
-exports.dedupShouldEnqueueMulti = dedupShouldEnqueueMulti;
 /**
  * Marks a message as processed so that subsequent calls to dedupShouldEnqueue
  * and dedupShouldEnqueueMulti will allow a message to be enqueued again
@@ -229,7 +230,6 @@ async function dedupSuccessfullyProcessed(message, opt) {
     }
     return 0;
 }
-exports.dedupSuccessfullyProcessed = dedupSuccessfullyProcessed;
 /**
  * Marks an array of messages as processed so that subsequent calls to
  * dedupShouldEnqueue and dedupShouldEnqueueMulti will allow a message to be
@@ -263,4 +263,3 @@ async function dedupSuccessfullyProcessedMulti(messages, opt) {
     }
     return 0;
 }
-exports.dedupSuccessfullyProcessedMulti = dedupSuccessfullyProcessedMulti;

package/commonjs/src/defaults.js

@@ -1,6 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setupVerbose = exports.setupAWS = exports.getOptionsWithDefaults = exports.validateMessageOptions = exports.validateQueueName = exports.validateInteger = exports.defaults = void 0;
+exports.defaults = void 0;
+exports.validateInteger = validateInteger;
+exports.validateQueueName = validateQueueName;
+exports.validateMessageOptions = validateMessageOptions;
+exports.getOptionsWithDefaults = getOptionsWithDefaults;
+exports.setupAWS = setupAWS;
+exports.setupVerbose = setupVerbose;
 /**
  * Default options for qdone. Accepts a command line options object and
  * returns nicely-named options.
@@ -56,7 +62,6 @@ function validateInteger(opt, name) {
         throw new Error(`${name} needs to be an integer.`);
     return parsed;
 }
-exports.validateInteger = validateInteger;
 function validateQueueName(opt, name) {
     if (typeof name !== 'string')
         throw new Error(`${name} must be a string.`);
@@ -64,9 +69,8 @@ function validateQueueName(opt, name) {
         throw new Error(`${name} can contain only numbers, letters, hypens and underscores.`);
     return name;
 }
-exports.validateQueueName = validateQueueName;
 function validateMessageOptions(messageOptions) {
-    const validKeys = ['deduplicationId', 'groupId'];
+    const validKeys = ['deduplicationId', 'groupId', 'delay'];
     if (typeof messageOptions === 'object' &&
         !Array.isArray(messageOptions) &&
         messageOptions !== null) {
@@ -78,7 +82,6 @@ function validateMessageOptions(messageOptions) {
     }
     return {};
 }
-exports.validateMessageOptions = validateMessageOptions;
 /**
  * This function should be called by each exposed API entry point on the
  * options passed in from the caller. It supports options named in camelCase
@@ -137,7 +140,9 @@ function getOptionsWithDefaults(options) {
         delete: options.delete || process.env.QDONE_DELETE === 'true' || exports.defaults.delete,
         // Check
         create: options.create || process.env.QDONE_CREATE === 'true' || exports.defaults.create,
-        overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || exports.defaults.overwrite
+        overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || exports.defaults.overwrite,
+        // Dependency injection
+        Redis: options.Redis
     };
     // Setting this env here means we don't have to in AWS SDK constructors
     process.env.AWS_REGION = opt.region;
@@ -167,16 +172,13 @@ function getOptionsWithDefaults(options) {
         throw new Error('Use either --deduplication-id or --dedup-id-per-message but not both');
     return opt;
 }
-exports.getOptionsWithDefaults = getOptionsWithDefaults;
 function setupAWS(options) {
     const opt = getOptionsWithDefaults(options);
     process.env.AWS_REGION = opt.region;
 }
-exports.setupAWS = setupAWS;
 function setupVerbose(options) {
     const verbose = options.verbose || (process.stderr.isTTY && !options.quiet);
     const quiet = options.quiet || (!process.stderr.isTTY && !options.verbose);
     options.verbose = verbose;
     options.quiet = quiet;
 }
-exports.setupVerbose = setupVerbose;

package/commonjs/src/enqueue.js

@@ -3,7 +3,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.enqueueBatch = exports.enqueue = exports.addMessage = exports.flushMessages = exports.sendMessageBatch = exports.sendMessage = exports.formatMessage = exports.getQueueAttributes = exports.getOrCreateQueue = exports.getQueueParams = exports.getOrCreateFailQueue = exports.getFailParams = exports.getOrCreateDLQ = exports.getDLQParams = void 0;
+exports.getDLQParams = getDLQParams;
+exports.getOrCreateDLQ = getOrCreateDLQ;
+exports.getFailParams = getFailParams;
+exports.getOrCreateFailQueue = getOrCreateFailQueue;
+exports.getQueueParams = getQueueParams;
+exports.getOrCreateQueue = getOrCreateQueue;
+exports.getQueueAttributes = getQueueAttributes;
+exports.formatMessage = formatMessage;
+exports.sendMessage = sendMessage;
+exports.sendMessageBatch = sendMessageBatch;
+exports.flushMessages = flushMessages;
+exports.addMessage = addMessage;
+exports.enqueue = enqueue;
+exports.enqueueBatch = enqueueBatch;
 const node_1 = require("@sentry/node");
 const uuid_1 = require("uuid");
 const chalk_1 = __importDefault(require("chalk"));
@@ -27,7 +40,6 @@ function getDLQParams(queue, opt) {
         params.Attributes.FifoQueue = 'true';
     return { dqname, params };
 }
-exports.getDLQParams = getDLQParams;
 async function getOrCreateDLQ(queue, opt) {
     debug('getOrCreateDLQ(', queue, ')');
     const { dqname, params } = getDLQParams(queue, opt);
@@ -51,7 +63,6 @@ async function getOrCreateDLQ(queue, opt) {
         return dqrl;
     }
 }
-exports.getOrCreateDLQ = getOrCreateDLQ;
 /**
  * Returns the parameters needed for creating a failed queue. If DLQ options
  * are set, it makes an API call to get this DLQ's ARN.
@@ -81,7 +92,6 @@ async function getFailParams(queue, opt) {
         params.Attributes.FifoQueue = 'true';
     return params;
 }
-exports.getFailParams = getFailParams;
 /**
  * Returns the qrl for the failed queue for the given queue. Creates the queue
  * if it does not exist.
@@ -124,7 +134,6 @@ async function getOrCreateFailQueue(queue, opt, doesNotExist) {
         return fqrl;
     }
 }
-exports.getOrCreateFailQueue = getOrCreateFailQueue;
 /**
  * Returns the parameters needed for creating a queue. If fail options
  * are set, it makes an API call to get the fail queue's ARN.
@@ -150,7 +159,6 @@ async function getQueueParams(queue, opt) {
         params.Attributes.FifoQueue = 'true';
     return params;
 }
-exports.getQueueParams = getQueueParams;
 /**
  * Returns a qrl for a queue that either exists or does not
  */
@@ -190,7 +198,6 @@ async function getOrCreateQueue(queue, opt) {
         return qrl;
     }
 }
-exports.getOrCreateQueue = getOrCreateQueue;
 async function getQueueAttributes(qrl) {
     debug('getQueueAttributes(', qrl, ')');
     const client = (0, sqs_js_1.getSQSClient)();
@@ -201,7 +208,6 @@ async function getQueueAttributes(qrl) {
     debug('GetQueueAttributes returned', data);
     return data;
 }
-exports.getQueueAttributes = getQueueAttributes;
 function formatMessage(body, id, opt, messageOptions) {
     const message = { MessageBody: body };
     if (typeof id !== 'undefined')
@@ -216,7 +222,6 @@ function formatMessage(body, id, opt, messageOptions) {
         message.DelaySeconds = messageOptions.delay;
     return message;
 }
-exports.formatMessage = formatMessage;
 // Retry happens within the context of the send functions
 const retryableExceptions = [
     client_sqs_1.RequestThrottled,
@@ -272,7 +277,6 @@ async function sendMessage(qrl, queue, command, opt, messageOptions) {
     debug({ sendMessageResult: result });
     return result;
 }
-exports.sendMessage = sendMessage;
 async function sendMessageBatch(qrl, queue, messages, opt) {
     debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), { MessageBody: e.MessageBody.slice(0, 10) + '...' })), ')');
     const params = { Entries: messages, QueueUrl: qrl };
@@ -350,7 +354,6 @@ async function sendMessageBatch(qrl, queue, messages, opt) {
     };
     return backoff.run(send, shouldRetry);
 }
-exports.sendMessageBatch = sendMessageBatch;
 let requestCount = 0;
 //
 // Flushes the internal message buffer for qrl.
@@ -421,7 +424,6 @@ async function flushMessages(qrl, queue, opt, sendBuffer) {
     }
     return whileNotEmpty();
 }
-exports.flushMessages = flushMessages;
 //
 // Adds a message to the inernal message buffer for the given qrl.
 // Automaticaly flushes if queue has >= 10 messages.
@@ -438,7 +440,6 @@ async function addMessage(qrl, queue, command, messageIndex, opt, sendBuffer, me
     }
     return { numFlushed: 0, results: [] };
 }
-exports.addMessage = addMessage;
 //
 // Enqueue a single command
 // Returns a promise for the SQS API response.
@@ -458,7 +459,6 @@ async function enqueue(queue, command, options) {
         throw e;
     }
 }
-exports.enqueue = enqueue;
 //
 // Enqueue many commands formatted as an array of {queue: ..., command: ...} pairs.
 // Returns a promise for the total number of messages enqueued.
@@ -517,5 +517,4 @@ async function enqueueBatch(pairs, options) {
         throw e;
     }
 }
-exports.enqueueBatch = enqueueBatch;
 debug('loaded');

package/commonjs/src/idleQueues.js

@@ -3,7 +3,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.idleQueues = exports.stripSuffixes = exports.processQueueSet = exports.deleteQueue = exports.checkIdle = exports.getMetric = exports.cheapIdleCheck = exports._cheapIdleCheck = exports.attributeNames = void 0;
+exports.attributeNames = void 0;
+exports._cheapIdleCheck = _cheapIdleCheck;
+exports.cheapIdleCheck = cheapIdleCheck;
+exports.getMetric = getMetric;
+exports.checkIdle = checkIdle;
+exports.deleteQueue = deleteQueue;
+exports.processQueueSet = processQueueSet;
+exports.stripSuffixes = stripSuffixes;
+exports.idleQueues = idleQueues;
 /**
  * Implementation of checks and caching of checks to determine if queues are idle.
  */
@@ -63,7 +71,6 @@ async function _cheapIdleCheck(qname, qrl, opt) {
         }
     }
 }
-exports._cheapIdleCheck = _cheapIdleCheck;
 /**
  * Gets queue attributes from the SQS api and assesses whether queue is idle
  * at this immediate moment.
@@ -91,7 +98,6 @@ async function cheapIdleCheck(qname, qrl, opt) {
         return { result, SQS };
     }
 }
-exports.cheapIdleCheck = cheapIdleCheck;
 /**
  * Gets a single metric from the CloudWatch api.
  */
@@ -116,7 +122,6 @@ async function getMetric(qname, qrl, metricName, opt) {
         [metricName]: data.Datapoints.map(d => d.Sum).reduce((a, b) => a + b, 0)
     };
 }
-exports.getMetric = getMetric;
 /**
  * Checks if a single queue is idle. First queries the cheap ($0.40/1M) SQS API and
  * continues to the expensive ($10/1M) CloudWatch API, checking to make sure there
@@ -171,7 +176,6 @@ async function checkIdle(qname, qrl, opt) {
     debug('checkIdle stats', stats);
     return stats;
 }
-exports.checkIdle = checkIdle;
 /**
  * Just deletes a queue.
  */
@@ -186,7 +190,6 @@ async function deleteQueue(qname, qrl, opt) {
         apiCalls: { SQS: 1, CloudWatch: 0 }
     };
 }
-exports.deleteQueue = deleteQueue;
 /**
  * Processes a queue and its fail and delete queue, treating them as a unit.
  */
@@ -280,7 +283,6 @@ async function processQueueSet(qname, qrl, opt) {
     }
     return result;
 }
-exports.processQueueSet = processQueueSet;
 //
 // Strips failed and dlq suffix from a queue name or URL
 //
@@ -288,7 +290,6 @@ function stripSuffixes(queueName, opt) {
     const suffixFinder = new RegExp(`(${opt.dlqSuffix}|${opt.failSuffix}){1}(|${qrlCache_js_1.fifoSuffix})$`);
     return queueName.replace(suffixFinder, '$2');
 }
-exports.stripSuffixes = stripSuffixes;
 //
 // Resolve queues for listening loop listen
 //
@@ -329,5 +330,4 @@ async function idleQueues(queues, options) {
     // Otherwise, let caller know
     return 'noQueues';
 }
-exports.idleQueues = idleQueues;
 debug('loaded');

package/commonjs/src/monitor.js

@@ -6,9 +6,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAggregateData = exports.interpretWildcard = exports.monitor = void 0;
+exports.monitor = monitor;
+exports.interpretWildcard = interpretWildcard;
+exports.getQueueAge = getQueueAge;
+exports.getAggregateData = getAggregateData;
 const sqs_js_1 = require("./sqs.js");
 const cloudWatch_js_1 = require("./cloudWatch.js");
+const client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
 const defaults_js_1 = require("./defaults.js");
 const qrlCache_js_1 = require("./qrlCache.js");
 const debug_1 = __importDefault(require("debug"));
@@ -32,7 +36,6 @@ async function monitor(queue, save, options) {
             process.stderr.write('done\n');
     }
 }
-exports.monitor = monitor;
 /**
  * Splits a queue name with a single wildcard into prefix and suffix regex.
  */
@@ -44,13 +47,43 @@ function interpretWildcard(queueName) {
     // debug({ prefix, suffix, safeSuffix, suffixRegex })
     return { prefix, suffix, safeSuffix, suffixRegex };
 }
-exports.interpretWildcard = interpretWildcard;
+/**
+ * Gets ApproximateAgeOfOldestMessage for a single queue from CloudWatch.
+ * This metric is not available via the SQS GetQueueAttributes API.
+ */
+async function getQueueAge(queueName) {
+    const now = new Date();
+    const params = {
+        StartTime: new Date(now.getTime() - 1000 * 60 * 5),
+        EndTime: now,
+        MetricName: 'ApproximateAgeOfOldestMessage',
+        Namespace: 'AWS/SQS',
+        Period: 300,
+        Dimensions: [{ Name: 'QueueName', Value: queueName }],
+        Statistics: ['Maximum']
+    };
+    const client = (0, cloudWatch_js_1.getCloudWatchClient)();
+    const cmd = new client_cloudwatch_1.GetMetricStatisticsCommand(params);
+    try {
+        const data = await client.send(cmd);
+        debug('getQueueAge', queueName, data);
+        if (!data.Datapoints || data.Datapoints.length === 0)
+            return 0;
+        return Math.max(...data.Datapoints.map(d => d.Maximum));
+    }
+    catch (e) {
+        debug('getQueueAge error', queueName, e);
+        return 0;
+    }
+}
 /**
  * Aggregates inmportant attributes across queues and reports a summary.
- * Attributes:
+ * Attributes (from SQS GetQueueAttributes):
  *  - ApproximateNumberOfMessages: Sum
  *  - ApproximateNumberOfMessagesDelayed: Sum
  *  - ApproximateNumberOfMessagesNotVisible: Sum
+ * Metrics (from CloudWatch):
+ *  - ApproximateAgeOfOldestMessage: Max
  */
 async function getAggregateData(queueName) {
     const { prefix, suffixRegex } = interpretWildcard(queueName);
@@ -70,11 +103,14 @@ async function getAggregateData(queueName) {
             }
         }
     }
+    // Fetch ApproximateAgeOfOldestMessage from CloudWatch (not available via SQS API)
+    // Only query queues with messages to minimize CloudWatch API costs
+    const ageResults = await Promise.all([...total.contributingQueueNames].map(queue => getQueueAge(queue)));
+    total.ApproximateAgeOfOldestMessage = Math.max(0, ...ageResults);
     // debug({ total })
     // convert set to array
     total.contributingQueueNames = [...total.contributingQueueNames.values()];
     total.queueName = queueName;
     return total;
 }
-exports.getAggregateData = getAggregateData;
 debug('loaded');

package/commonjs/src/qrlCache.js

@@ -7,7 +7,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getQnameUrlPairs = exports.getMatchingQueues = exports.ingestQRLs = exports.qrlCacheInvalidate = exports.qrlCacheSet = exports.qrlCacheGet = exports.qrlCacheClear = exports.normalizeDLQName = exports.normalizeFailQueueName = exports.normalizeQueueName = exports.fifoSuffix = void 0;
+exports.fifoSuffix = void 0;
+exports.normalizeQueueName = normalizeQueueName;
+exports.normalizeFailQueueName = normalizeFailQueueName;
+exports.normalizeDLQName = normalizeDLQName;
+exports.qrlCacheClear = qrlCacheClear;
+exports.qrlCacheGet = qrlCacheGet;
+exports.qrlCacheSet = qrlCacheSet;
+exports.qrlCacheInvalidate = qrlCacheInvalidate;
+exports.ingestQRLs = ingestQRLs;
+exports.getMatchingQueues = getMatchingQueues;
+exports.getQnameUrlPairs = getQnameUrlPairs;
 const url_1 = require("url");
 const path_1 = require("path");
 const client_sqs_1 = require("@aws-sdk/client-sqs");
@@ -27,7 +37,6 @@ function normalizeQueueName(qname, opt) {
     const base = hasFifo ? qname.slice(0, -exports.fifoSuffix.length) : qname;
     return (needsPrefix ? opt.prefix : '') + base + (needsFifo ? exports.fifoSuffix : '');
 }
-exports.normalizeQueueName = normalizeQueueName;
 //
 // Normalizes fail queue name, appending both --fail-suffix and .fifo depending on opt
 //
@@ -38,7 +47,6 @@ function normalizeFailQueueName(fqname, opt) {
     const needsFail = !base.endsWith(opt.failSuffix);
     return base + (needsFail ? opt.failSuffix : '') + (opt.fifo ? exports.fifoSuffix : '');
 }
-exports.normalizeFailQueueName = normalizeFailQueueName;
 //
 // Normalizes dlq queue name, appending both --dlq-suffix and .fifo depending on opt
 //
@@ -48,14 +56,12 @@ function normalizeDLQName(dqname, opt) {
     const needsDead = !base.endsWith(opt.dlqSuffix);
     return base + (needsDead ? opt.dlqSuffix : '') + (opt.fifo ? exports.fifoSuffix : '');
 }
-exports.normalizeDLQName = normalizeDLQName;
 //
 // Clear cache
 //
 function qrlCacheClear() {
     qcache.clear();
 }
-exports.qrlCacheClear = qrlCacheClear;
 //
 // Get QRL (Queue URL)
 // Returns a promise for the queue name
@@ -80,7 +86,6 @@ async function qrlCacheGet(qname) {
     // debug('qcache', Object.keys(qcache), 'get', qname, ' => ', qcache[qname])
     return qrl;
 }
-exports.qrlCacheGet = qrlCacheGet;
 //
 // Set QRL (Queue URL)
 // Immediately updates the cache
@@ -90,7 +95,6 @@ function qrlCacheSet(qname, qrl) {
         qcache.set(qname, qrl);
     // debug('qcache', Object.keys(qcache), 'set', qname, ' => ', qcache[qname])
 }
-exports.qrlCacheSet = qrlCacheSet;
 //
 // Invalidate cache for given qname
 //
@@ -98,7 +102,6 @@ function qrlCacheInvalidate(qname) {
     // debug('qcache', Object.keys(qcache), 'delete', qname, ' (was ', qcache[qname], ')')
     qcache.delete(qname);
 }
-exports.qrlCacheInvalidate = qrlCacheInvalidate;
 //
 // Ingets multiple QRLs
 // Extracts queue names from an array of QRLs and immediately updates the cache.
@@ -114,7 +117,6 @@ function ingestQRLs(qrls) {
     }
     return pairs;
 }
-exports.ingestQRLs = ingestQRLs;
 /**
  * Returns qrls for queues matching the given prefix and regex.
  */
@@ -133,7 +135,6 @@ async function getMatchingQueues(prefix, nextToken) {
         (qrls || []).push(...await getMatchingQueues(prefix, keepGoing));
     return qrls || [];
 }
-exports.getMatchingQueues = getMatchingQueues;
 //
 // Resolves into a flattened aray of {qname: ..., qrl: ...} objects.
 //
@@ -168,5 +169,4 @@ async function getQnameUrlPairs(qnames, opt) {
     const results = await Promise.all(promises);
     return ([].concat.apply([], results)).filter(r => r);
 }
-exports.getQnameUrlPairs = getQnameUrlPairs;
 debug('loaded');

package/commonjs/src/scheduler/jobExecutor.js

@@ -311,7 +311,15 @@ class JobExecutor {
             this.stats.runningJobs++;
             this.stats.waitingJobs--;
             const queue = job.qname.slice(this.opt.prefix.length);
-            const result = await job.callback(queue, job.payload);
+            const attributes = {
+                queueName: job.qname,
+                messageId: job.message.MessageId || '',
+                receiveCount: job.message.Attributes?.ApproximateReceiveCount || '1',
+                sentTimestamp: job.message.Attributes?.SentTimestamp || '',
+                firstReceiveTimestamp: job.message.Attributes?.ApproximateFirstReceiveTimestamp || '',
+                messageGroupId: job.message.Attributes?.MessageGroupId || ''
+            };
+            const result = await job.callback(queue, job.payload, attributes);
             debug('executeJob callback finished', { payload: job.payload, result });
             if (this.opt.verbose) {
                 console.error(chalk_1.default.green('SUCCESS'), job.payload);
@@ -369,7 +377,7 @@ class JobExecutor {
         for (const [job, i] of jobs.map((job, i) => [job, i])) {
             // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
             const nextJob = jobs[i + 1];
-            const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId;
+            const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.MessageGroupId === nextJob.message?.Attributes?.MessageGroupId;
             // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
             // Execute serial or parallel
             if (nextJobIsSerial)

package/commonjs/src/sqs.js

@@ -6,7 +6,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getQueueAttributes = exports.getMatchingQueues = exports.setSQSClient = exports.getSQSClient = void 0;
+exports.getSQSClient = getSQSClient;
+exports.setSQSClient = setSQSClient;
+exports.getMatchingQueues = getMatchingQueues;
+exports.getQueueAttributes = getQueueAttributes;
 const client_sqs_1 = require("@aws-sdk/client-sqs");
 const path_1 = require("path");
 const debug_1 = __importDefault(require("debug"));
@@ -21,14 +24,12 @@ function getSQSClient() {
     client = new client_sqs_1.SQSClient();
     return client;
 }
-exports.getSQSClient = getSQSClient;
 /**
  * Utility function to set the client explicitly, used in testing.
  */
 function setSQSClient(explicitClient) {
     client = explicitClient;
 }
-exports.setSQSClient = setSQSClient;
 /**
  * Returns qrls for queues matching the given prefix and regex.
  */
@@ -48,7 +49,6 @@ async function getMatchingQueues(prefix, regex) {
     }
     return processQueues();
 }
-exports.getMatchingQueues = getMatchingQueues;
 /**
  * Gets attributes on every queue in parallel.
  */
@@ -93,5 +93,4 @@ async function getQueueAttributes(qrls) {
     }
     return Promise.all(promises);
 }
-exports.getQueueAttributes = getQueueAttributes;
 debug('loaded');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.1.1",
+  "version": "2.2.1",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",
@@ -14,15 +14,13 @@
   "dependencies": {
     "@aws-sdk/client-cloudwatch": "3.465.0",
     "@aws-sdk/client-sqs": "3.465.0",
-    "@sentry/node": "^7.87.0",
+    "@sentry/node": "^7.120.4",
     "chalk": "^4.1.2",
     "command-line-args": "^5.2.1",
     "command-line-commands": "^3.0.2",
-    "command-line-usage": "^7.0.1",
-    "debug": "^4.3.4",
-    "ioredis": "^5.3.2",
-    "ioredis-mock": "^8.9.0",
-    "standard": "^17.1.0",
+    "command-line-usage": "^7.0.3",
+    "debug": "^4.4.3",
+    "ioredis": "^5.9.3",
     "tree-kill": "^1.2.2",
     "uuid": "^9.0.1"
   },
@@ -36,10 +34,12 @@
     }
   },
   "devDependencies": {
-    "aws-sdk-client-mock": "^3.0.0",
-    "aws-sdk-client-mock-jest": "^3.0.0",
+    "aws-sdk-client-mock": "^3.1.0",
+    "aws-sdk-client-mock-jest": "^3.1.0",
+    "ioredis-mock": "^8.13.1",
     "jest": "^29.7.0",
-    "typescript": "^5.3.3"
+    "standard": "^17.1.2",
+    "typescript": "^5.9.3"
   },
   "preferGlobal": true,
   "engines": {

package/src/cloudWatch.js

@@ -85,6 +85,16 @@ export async function putAggregateData (total, timestamp) {
         Timestamp: now,
         Value: total.ApproximateNumberOfMessagesNotVisible || 0,
         Unit: 'Count'
+      },
+      {
+        MetricName: 'ApproximateAgeOfOldestMessage',
+        Dimensions: [{
+          Name: 'queueName',
+          Value: total.queueName
+        }],
+        Timestamp: now,
+        Value: total.ApproximateAgeOfOldestMessage || 0,
+        Unit: 'Seconds'
       }
     ]
   }

package/src/defaults.js

@@ -66,7 +66,7 @@ export function validateQueueName (opt, name) {
 }
 
 export function validateMessageOptions (messageOptions) {
-  const validKeys = ['deduplicationId', 'groupId']
+  const validKeys = ['deduplicationId', 'groupId', 'delay']
   if (typeof messageOptions === 'object' &&
       !Array.isArray(messageOptions) &&
       messageOptions !== null) {
@@ -142,7 +142,10 @@ export function getOptionsWithDefaults (options) {
 
     // Check
     create: options.create || process.env.QDONE_CREATE === 'true' || defaults.create,
-    overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || defaults.overwrite
+    overwrite: options.overwrite || process.env.QDONE_OVERWRITE === 'true' || defaults.overwrite,
+
+    // Dependency injection
+    Redis: options.Redis
   }
 
   // Setting this env here means we don't have to in AWS SDK constructors

package/src/idleQueues.js

@@ -7,7 +7,7 @@ import { getCloudWatchClient } from './cloudWatch.js'
 import { getOptionsWithDefaults } from './defaults.js'
 import { GetQueueAttributesCommand, DeleteQueueCommand, QueueDoesNotExist } from '@aws-sdk/client-sqs'
 import { GetMetricStatisticsCommand } from '@aws-sdk/client-cloudwatch'
-import { normalizeFailQueueName, normalizeDLQName, getQnameUrlPairs, fifoSuffix, qrlCacheSet } from './qrlCache.js'
+import { normalizeFailQueueName, normalizeDLQName, getQnameUrlPairs, fifoSuffix } from './qrlCache.js'
 import { getCache, setCache } from './cache.js'
 // const AWS = require('aws-sdk')
 

package/src/monitor.js

@@ -3,7 +3,8 @@
  */
 
 import { getMatchingQueues, getQueueAttributes } from './sqs.js'
-import { putAggregateData } from './cloudWatch.js'
+import { putAggregateData, getCloudWatchClient } from './cloudWatch.js'
+import { GetMetricStatisticsCommand } from '@aws-sdk/client-cloudwatch'
 import { getOptionsWithDefaults } from './defaults.js'
 import { normalizeQueueName } from './qrlCache.js'
 import Debug from 'debug'
@@ -38,12 +39,42 @@ export function interpretWildcard (queueName) {
   return { prefix, suffix, safeSuffix, suffixRegex }
 }
 
+/**
+ * Gets ApproximateAgeOfOldestMessage for a single queue from CloudWatch.
+ * This metric is not available via the SQS GetQueueAttributes API.
+ */
+export async function getQueueAge (queueName) {
+  const now = new Date()
+  const params = {
+    StartTime: new Date(now.getTime() - 1000 * 60 * 5),
+    EndTime: now,
+    MetricName: 'ApproximateAgeOfOldestMessage',
+    Namespace: 'AWS/SQS',
+    Period: 300,
+    Dimensions: [{ Name: 'QueueName', Value: queueName }],
+    Statistics: ['Maximum']
+  }
+  const client = getCloudWatchClient()
+  const cmd = new GetMetricStatisticsCommand(params)
+  try {
+    const data = await client.send(cmd)
+    debug('getQueueAge', queueName, data)
+    if (!data.Datapoints || data.Datapoints.length === 0) return 0
+    return Math.max(...data.Datapoints.map(d => d.Maximum))
+  } catch (e) {
+    debug('getQueueAge error', queueName, e)
+    return 0
+  }
+}
+
 /**
  * Aggregates inmportant attributes across queues and reports a summary.
- * Attributes:
+ * Attributes (from SQS GetQueueAttributes):
  *  - ApproximateNumberOfMessages: Sum
  *  - ApproximateNumberOfMessagesDelayed: Sum
  *  - ApproximateNumberOfMessagesNotVisible: Sum
+ * Metrics (from CloudWatch):
+ *  - ApproximateAgeOfOldestMessage: Max
  */
 export async function getAggregateData (queueName) {
   const { prefix, suffixRegex } = interpretWildcard(queueName)
@@ -63,6 +94,14 @@ export async function getAggregateData (queueName) {
       }
     }
   }
+
+  // Fetch ApproximateAgeOfOldestMessage from CloudWatch (not available via SQS API)
+  // Only query queues with messages to minimize CloudWatch API costs
+  const ageResults = await Promise.all(
+    [...total.contributingQueueNames].map(queue => getQueueAge(queue))
+  )
+  total.ApproximateAgeOfOldestMessage = Math.max(0, ...ageResults)
+
   // debug({ total })
   // convert set to array
   total.contributingQueueNames = [...total.contributingQueueNames.values()]

package/src/scheduler/jobExecutor.js

@@ -322,7 +322,15 @@ export class JobExecutor {
       this.stats.runningJobs++
       this.stats.waitingJobs--
       const queue = job.qname.slice(this.opt.prefix.length)
-      const result = await job.callback(queue, job.payload)
+      const attributes = {
+        queueName: job.qname,
+        messageId: job.message.MessageId || '',
+        receiveCount: job.message.Attributes?.ApproximateReceiveCount || '1',
+        sentTimestamp: job.message.Attributes?.SentTimestamp || '',
+        firstReceiveTimestamp: job.message.Attributes?.ApproximateFirstReceiveTimestamp || '',
+        messageGroupId: job.message.Attributes?.MessageGroupId || ''
+      }
+      const result = await job.callback(queue, job.payload, attributes)
       debug('executeJob callback finished', { payload: job.payload, result })
       if (this.opt.verbose) {
         console.error(chalk.green('SUCCESS'), job.payload)
@@ -381,7 +389,7 @@ export class JobExecutor {
     for (const [job, i] of jobs.map((job, i) => [job, i])) {
       // Figure out if the next job needs to happen in serial, otherwise we can parallel execute
       const nextJob = jobs[i + 1]
-      const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.GroupId === nextJob.message?.Attributes?.GroupId
+      const nextJobIsSerial = isFifo && nextJob && job.message?.Attributes?.MessageGroupId === nextJob.message?.Attributes?.MessageGroupId
 
       // console.log({ i, nextJobAtt: nextJob?.message?.Attributes, nextJobIsSerial })
       // Execute serial or parallel

package/src/worker.js

@@ -104,10 +104,21 @@ export async function executeJob (job, qname, qrl, opt) {
   const treeKiller = setTimeout(killTree, opt.killAfter * 1000)
   debug({ treeKiller: opt.killAfter * 1000, date: Date.now() })
 
+  // Build environment with SQS message attributes for child process
+  const env = {
+    ...process.env,
+    QDONE_QUEUE_NAME: qname,
+    SQS_MESSAGE_ID: job.MessageId || '',
+    SQS_RECEIVE_COUNT: job.Attributes?.ApproximateReceiveCount || '1',
+    SQS_SENT_TIMESTAMP: job.Attributes?.SentTimestamp || '',
+    SQS_FIRST_RECEIVE_TIMESTAMP: job.Attributes?.ApproximateFirstReceiveTimestamp || '',
+    SQS_MESSAGE_GROUP_ID: job.Attributes?.MessageGroupId || ''
+  }
+
   try {
     // Success path for job execution
     const { stdout, stderr } = await new Promise(function (resolve, reject) {
-      child = exec(cmd, function (err, stdout, stderr) {
+      child = exec(cmd, { env }, function (err, stdout, stderr) {
         if (err) {
           err.stdout = stdout
           err.stderr = stderr