qdone 2.2.2 → 2.2.4

package/commonjs/src/dedup.js

@@ -125,7 +125,7 @@ async function statMaintenance(opt) {
     const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet';
     const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet';
     const client = (0, cache_js_1.getCacheClient)(opt);
-    const now = new Date().getTime();
+    const now = Math.floor(Date.now() / 1000);
     // Grab a batch of expired keys
     debug({ statMaintenance: { aboutToGo: true, expirationSet } });
     const expiredStats = await client.zrange(expirationSet, '-inf', now, 'BYSCORE');
@@ -150,7 +150,7 @@ async function dedupShouldEnqueue(message, opt) {
     const client = (0, cache_js_1.getCacheClient)(opt);
     const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue;
     const cacheKey = getCacheKey(dedupId, opt);
-    const expireAt = new Date().getTime() + opt.dedupPeriod;
+    const expireAt = Math.floor(Date.now() / 1000) + opt.dedupPeriod;
     const copies = await client.incr(cacheKey);
     debug({ action: 'shouldEnqueue', cacheKey, copies });
     if (copies === 1) {
@@ -172,7 +172,7 @@ async function dedupShouldEnqueue(message, opt) {
  */
 async function dedupShouldEnqueueMulti(messages, opt) {
     debug({ dedupShouldEnqueueMulti: { messages, opt } });
-    const expireAt = new Date().getTime() + opt.dedupPeriod;
+    const expireAt = Math.floor(Date.now() / 1000) + opt.dedupPeriod;
     // Increment all
     const incrPipeline = (0, cache_js_1.getCacheClient)(opt).pipeline();
     for (const message of messages) {

package/commonjs/src/monitor.js

@@ -26,7 +26,7 @@ async function monitor(queue, save, options) {
     const opt = (0, defaults_js_1.getOptionsWithDefaults)(options);
     const queueName = (0, qrlCache_js_1.normalizeQueueName)(queue, opt);
     debug({ options, opt, queue, queueName });
-    const data = await getAggregateData(queueName);
+    const data = await getAggregateData(queueName, opt);
     console.log(data);
     if (save) {
         if (opt.verbose)
@@ -85,7 +85,7 @@ async function getQueueAge(queueName) {
  * Metrics (from CloudWatch):
  *  - ApproximateAgeOfOldestMessage: Max
  */
-async function getAggregateData(queueName) {
+async function getAggregateData(queueName, opt) {
     const { prefix, suffixRegex } = interpretWildcard(queueName);
     const qrls = await (0, sqs_js_1.getMatchingQueues)(prefix, suffixRegex);
     // debug({ qrls })
@@ -105,7 +105,19 @@ async function getAggregateData(queueName) {
     }
     // Fetch ApproximateAgeOfOldestMessage from CloudWatch (not available via SQS API)
     // Only query queues with messages to minimize CloudWatch API costs
-    const ageResults = await Promise.all([...total.contributingQueueNames].map(queue => getQueueAge(queue)));
+    // Filter out dead and failed queues for age calculation only — their messages
+    // age indefinitely by design, polluting the active age metric.
+    // But if the pattern itself targets dead/failed queues, don't filter them out.
+    const failSuffix = (opt && opt.failSuffix) || defaults_js_1.defaults.failSuffix;
+    const dlqSuffix = (opt && opt.dlqSuffix) || defaults_js_1.defaults.dlqSuffix;
+    const strippedPattern = queueName.replace(/\.fifo$/, '');
+    const patternTargetsDeadFailed = strippedPattern.endsWith(failSuffix) || strippedPattern.endsWith(dlqSuffix);
+    const esc = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const deadFailedRegex = new RegExp(`(${esc(failSuffix)}|${esc(dlqSuffix)})(\\.fifo)?$`);
+    const activeQueueNames = patternTargetsDeadFailed
+        ? [...total.contributingQueueNames]
+        : [...total.contributingQueueNames].filter(q => !deadFailedRegex.test(q));
+    const ageResults = await Promise.all(activeQueueNames.map(queue => getQueueAge(queue)));
     total.ApproximateAgeOfOldestMessage = Math.max(0, ...ageResults);
     // debug({ total })
     // convert set to array

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qdone",
-  "version": "2.2.2",
+  "version": "2.2.4",
   "description": "A distributed scheduler for SQS",
   "type": "module",
   "main": "./index.js",

package/src/cli.js

@@ -197,10 +197,11 @@ export async function check (argv, testHook) {
 }
 
 const monitorOptionDefinitions = [
-  { name: 'save', alias: 's', type: Boolean, description: 'Saves data to CloudWatch' }
+  { name: 'save', alias: 's', type: Boolean, description: 'Saves data to CloudWatch' },
+  { name: 'dlq-suffix', type: String, description: `Suffix to append to each queue to generate DLQ name [default: ${defaults.dlqSuffix}]` }
 ]
 
-export async function monitor (argv) {
+export async function monitor (argv, testHook) {
   const optionDefinitions = [].concat(monitorOptionDefinitions, globalOptionDefinitions)
   const usageSections = [
     { content: 'usage: qdone monitor <queuePattern> ', raw: true },
@@ -234,7 +235,8 @@ export async function monitor (argv) {
 
   // Load module after AWS global load
   setupAWS(options)
-  const { monitor } = await import('./monitor.js')
+  const { monitor: monitorOriginal } = await import('./monitor.js')
+  const monitor = testHook || monitorOriginal
   return monitor(queue, options.save, options)
 }
 

package/src/dedup.js

@@ -112,7 +112,7 @@ export async function statMaintenance (opt) {
   const duplicateSet = opt.cachePrefix + 'dedup-stats:duplicateSet'
   const expirationSet = opt.cachePrefix + 'dedup-stats:expirationSet'
   const client = getCacheClient(opt)
-  const now = new Date().getTime()
+  const now = Math.floor(Date.now() / 1000)
 
   // Grab a batch of expired keys
   debug({ statMaintenance: { aboutToGo: true, expirationSet } })
@@ -140,7 +140,7 @@ export async function dedupShouldEnqueue (message, opt) {
   const client = getCacheClient(opt)
   const dedupId = message?.MessageAttributes?.QdoneDeduplicationId?.StringValue
   const cacheKey = getCacheKey(dedupId, opt)
-  const expireAt = new Date().getTime() + opt.dedupPeriod
+  const expireAt = Math.floor(Date.now() / 1000) + opt.dedupPeriod
   const copies = await client.incr(cacheKey)
   debug({ action: 'shouldEnqueue', cacheKey, copies })
   if (copies === 1) {
@@ -163,7 +163,7 @@ export async function dedupShouldEnqueue (message, opt) {
  */
 export async function dedupShouldEnqueueMulti (messages, opt) {
   debug({ dedupShouldEnqueueMulti: { messages, opt } })
-  const expireAt = new Date().getTime() + opt.dedupPeriod
+  const expireAt = Math.floor(Date.now() / 1000) + opt.dedupPeriod
   // Increment all
   const incrPipeline = getCacheClient(opt).pipeline()
   for (const message of messages) {

package/src/monitor.js

@@ -5,7 +5,7 @@
 import { getMatchingQueues, getQueueAttributes } from './sqs.js'
 import { putAggregateData, getCloudWatchClient } from './cloudWatch.js'
 import { GetMetricStatisticsCommand } from '@aws-sdk/client-cloudwatch'
-import { getOptionsWithDefaults } from './defaults.js'
+import { getOptionsWithDefaults, defaults } from './defaults.js'
 import { normalizeQueueName } from './qrlCache.js'
 import Debug from 'debug'
 const debug = Debug('qdone:monitor')
@@ -18,7 +18,7 @@ export async function monitor (queue, save, options) {
   const opt = getOptionsWithDefaults(options)
   const queueName = normalizeQueueName(queue, opt)
   debug({ options, opt, queue, queueName })
-  const data = await getAggregateData(queueName)
+  const data = await getAggregateData(queueName, opt)
   console.log(data)
   if (save) {
     if (opt.verbose) process.stderr.write('Saving to CloudWatch...')
@@ -76,7 +76,7 @@ export async function getQueueAge (queueName) {
  * Metrics (from CloudWatch):
  *  - ApproximateAgeOfOldestMessage: Max
  */
-export async function getAggregateData (queueName) {
+export async function getAggregateData (queueName, opt) {
   const { prefix, suffixRegex } = interpretWildcard(queueName)
   const qrls = await getMatchingQueues(prefix, suffixRegex)
   // debug({ qrls })
@@ -97,8 +97,21 @@ export async function getAggregateData (queueName) {
 
   // Fetch ApproximateAgeOfOldestMessage from CloudWatch (not available via SQS API)
   // Only query queues with messages to minimize CloudWatch API costs
+  // Filter out dead and failed queues for age calculation only — their messages
+  // age indefinitely by design, polluting the active age metric.
+  // But if the pattern itself targets dead/failed queues, don't filter them out.
+  const failSuffix = (opt && opt.failSuffix) || defaults.failSuffix
+  const dlqSuffix = (opt && opt.dlqSuffix) || defaults.dlqSuffix
+  const strippedPattern = queueName.replace(/\.fifo$/, '')
+  const patternTargetsDeadFailed = strippedPattern.endsWith(failSuffix) || strippedPattern.endsWith(dlqSuffix)
+  const esc = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  const deadFailedRegex = new RegExp(`(${esc(failSuffix)}|${esc(dlqSuffix)})(\\.fifo)?$`)
+  const activeQueueNames = patternTargetsDeadFailed
+    ? [...total.contributingQueueNames]
+    : [...total.contributingQueueNames].filter(q => !deadFailedRegex.test(q))
+
   const ageResults = await Promise.all(
-    [...total.contributingQueueNames].map(queue => getQueueAge(queue))
+    activeQueueNames.map(queue => getQueueAge(queue))
   )
   total.ApproximateAgeOfOldestMessage = Math.max(0, ...ageResults)