screwdriver-queue-service 2.0.41 → 2.0.43

package/config/custom-environment-variables.yaml

@@ -219,6 +219,8 @@ ecosystem:
         max_size_mb: CACHE_MAX_SIZE_MB
 
 queue:
+  # redis or redisCluster(beta)
+  connectionType: REDIS_TYPE 
   redisConnection:
     host: REDIS_HOST
     port: REDIS_PORT
@@ -226,6 +228,14 @@ queue:
       password: REDIS_PASSWORD
       tls: REDIS_TLS_ENABLED
     database: REDIS_DB_NUMBER
+  redisClusterConnection:
+    hosts:
+      __name: REDIS_CLUSTER_HOSTS
+      __format: json
+    options:
+      password: REDIS_PASSWORD
+      tls: REDIS_TLS_ENABLED
+    slotsRefreshTimeout: REDIS_CLUSTER_SLOTS_REFRESH_TIMEOUT
   prefix: REDIS_QUEUE_PREFIX
 
 plugins:
@@ -292,4 +302,4 @@ kafka:
   # AWS region 
   region: AWS_REGION
   # Flag to use Short Region Name like use2,usw2
-  shortRegion : USE_SHORT_REGION_NAME
+  shortRegion : USE_SHORT_REGION_NAME

package/config/default.yaml

@@ -149,6 +149,8 @@ ecosystem:
     max_size_mb: 0
 
 queue:
+  # redis or redisCluster(beta)
+  connectionType: redis 
   redisConnection:
     host: REDIS_HOST
     port: REDIS_PORT
@@ -156,7 +158,13 @@ queue:
       password: a-secure-password
       tls: false
     database: 0
-  prefix: ''
+  redisClusterConnection:
+    hosts: []
+    options:
+      password: a-secure-password
+      tls: false
+    slotsRefreshTimeout: 1000
+  prefix: ""
 
 plugins:
   blockedBy:

package/config/redis.js

@@ -3,18 +3,41 @@
 const config = require('config');
 
 const queueConfig = config.get('queue');
-const redisConfig = queueConfig.redisConnection;
+const connectionType = queueConfig.connectionType;
+
+if (!connectionType || (connectionType !== 'redis' && connectionType !== 'redisCluster')) {
+    throw new Error(
+        `connectionType '${connectionType}' is not supported, use 'redis' or 'redisCluster' for the queue.connectionType setting`
+    );
+}
+
+const redisConfig = queueConfig[`${connectionType}Connection`];
 const connectionDetails = {
-    pkg: 'ioredis',
-    host: redisConfig.host,
-    options: {
+    redisOptions: {
         password: redisConfig.options && redisConfig.options.password,
         tls: redisConfig.options ? redisConfig.options.tls : false
-    },
-    port: redisConfig.port,
-    database: redisConfig.database
+    }
 };
 
+let queueNamespace;
+
+// for redisCluster config
+if (connectionType === 'redisCluster') {
+    connectionDetails.redisClusterHosts = redisConfig.hosts;
+    connectionDetails.slotsRefreshTimeout = parseInt(redisConfig.slotsRefreshTimeout, 10);
+    // NOTE: node-resque has an issue  in multi-key operation for Redis Cluster
+    // https://github.com/actionhero/node-resque/issues/786
+    // so we have to set the namespace option with a hash tag so that the resque's keys are set in the same slots in Redis Cluster
+    // https://redis.io/docs/manual/scaling/#redis-cluster-data-sharding
+    queueNamespace = 'resque:{screwdriver-resque}';
+} else {
+    // for non-cluster redis config
+    connectionDetails.redisOptions.host = redisConfig.host;
+    connectionDetails.redisOptions.port = redisConfig.port;
+    connectionDetails.redisOptions.database = redisConfig.database;
+    queueNamespace = 'resque';
+}
+
 const queuePrefix = queueConfig.prefix || '';
 
 const runningJobsPrefix = `${queuePrefix}running_job_`;
@@ -22,6 +45,7 @@ const waitingJobsPrefix = `${queuePrefix}waiting_job_`;
 
 module.exports = {
     connectionDetails,
+    queueNamespace,
     queuePrefix,
     runningJobsPrefix,
     waitingJobsPrefix

package/lib/queue.js

@@ -6,6 +6,8 @@ const Breaker = fuses.breaker;
 const FuseBox = fuses.box;
 const logger = require('screwdriver-logger');
 const redis = require('../plugins/redis');
+const { queueNamespace } = require('../config/redis');
+const resqueConnection = { redis, namespace: queueNamespace };
 
 module.exports = class ExecutorQueue {
     /**
@@ -37,14 +39,10 @@ module.exports = class ExecutorQueue {
         this.unzipQueue = `${this.prefix}unzip`;
         this.webhookQueue = `${this.prefix}webhooks`;
 
-        const redisConnection = { ...config.redisConnection, pkg: 'ioredis' };
-
-        this.redisConnection = redisConnection;
-
         this.redis = redis;
 
         // eslint-disable-next-line new-cap
-        this.queue = new Resque.Queue({ connection: redisConnection });
+        this.queue = new Resque.Queue({ connection: resqueConnection });
         this.queue.on('error', error => logger.info(`Resque queue error >> ${error}`));
         this.queueBreaker = new Breaker((funcName, ...args) => {
             const callback = args.pop();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.41",
+  "version": "2.0.43",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -20,11 +20,11 @@
     "config": "^1.31.0",
     "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
-    "ioredis": "^5.0.4",
+    "ioredis": "^5.2.3",
     "joi": "^17.5.0",
     "js-yaml": "^3.14.1",
     "jsonwebtoken": "^8.5.1",
-    "node-resque": "^5.5.3",
+    "node-resque": "^9.2.0",
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
     "screwdriver-aws-producer-service": "^1.3.0",

package/plugins/queue/scheduler.js

@@ -4,10 +4,11 @@ const logger = require('screwdriver-logger');
 const configSchema = require('screwdriver-data-schema').config;
 const TOKEN_CONFIG_SCHEMA = configSchema.tokenConfig;
 const { merge, reach } = require('@hapi/hoek');
-const Resque = require('node-resque');
+const { MultiWorker, Scheduler, Plugins } = require('node-resque');
 const cron = require('./utils/cron');
 const helper = require('../helper');
 const { timeOutOfWindows } = require('./utils/freezeWindows');
+const { queueNamespace } = require('../../config/redis');
 const DEFAULT_BUILD_TIMEOUT = 90;
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
@@ -455,9 +456,9 @@ async function start(executor, config) {
 async function init(executor) {
     if (executor.multiWorker) return 'Scheduler running';
 
-    const { redisConnection } = executor;
+    const resqueConnection = { redis: executor.redis, namespace: queueNamespace };
     const retryOptions = {
-        plugins: ['Retry'],
+        plugins: [Plugins.Retry],
         pluginOptions: {
             Retry: {
                 retryLimit: RETRY_LIMIT,
@@ -512,9 +513,9 @@ async function init(executor) {
         }
     };
 
-    executor.multiWorker = new Resque.MultiWorker(
+    executor.multiWorker = new MultiWorker(
         {
-            connection: redisConnection,
+            connection: resqueConnection,
             queues: [executor.periodicBuildQueue, executor.frozenBuildQueue],
             minTaskProcessors: 1,
             maxTaskProcessors: 10,
@@ -525,7 +526,7 @@ async function init(executor) {
         jobs
     );
 
-    executor.scheduler = new Resque.Scheduler({ connection: redisConnection });
+    executor.scheduler = new Scheduler({ connection: resqueConnection });
 
     executor.multiWorker.on('start', workerId => logger.info(`worker[${workerId}] started`));
     executor.multiWorker.on('end', workerId => logger.info(`worker[${workerId}] ended`));
@@ -538,22 +539,19 @@ async function init(executor) {
     executor.multiWorker.on('reEnqueue', (workerId, queue, job, plugin) =>
         logger.info(`worker[${workerId}] reEnqueue job (${plugin}) ${queue} ${JSON.stringify(job)}`)
     );
-    executor.multiWorker.on('success', (workerId, queue, job, result) =>
-        logger.info(`worker[${workerId}] job success ${queue} ${JSON.stringify(job)} >> ${result}`)
+    executor.multiWorker.on('success', (workerId, queue, job, result, duration) =>
+        logger.info(`worker[${workerId}] job success ${queue} ${JSON.stringify(job)} >> ${result} (${duration}ms)`)
     );
-    executor.multiWorker.on('failure', (workerId, queue, job, failure) =>
-        logger.info(`worker[${workerId}] job failure ${queue} ${JSON.stringify(job)} >> ${failure}`)
+    executor.multiWorker.on('failure', (workerId, queue, job, failure, duration) =>
+        logger.info(`worker[${workerId}] job failure ${queue} ${JSON.stringify(job)} >> ${failure} (${duration}ms)`)
     );
     executor.multiWorker.on('error', (workerId, queue, job, error) =>
         logger.error(`worker[${workerId}] error ${queue} ${JSON.stringify(job)} >> ${error}`)
     );
 
-    // multiWorker emitters
-    executor.multiWorker.on('internalError', error => logger.error(error));
-
     executor.scheduler.on('start', () => logger.info('scheduler started'));
     executor.scheduler.on('end', () => logger.info('scheduler ended'));
-    executor.scheduler.on('master', state => logger.info(`scheduler became master ${state}`));
+    executor.scheduler.on('leader', () => logger.info(`scheduler became leader`));
     executor.scheduler.on('error', error => logger.info(`scheduler error >> ${error}`));
     executor.scheduler.on('workingTimestamp', timestamp => logger.info(`scheduler working timestamp ${timestamp}`));
     executor.scheduler.on('cleanStuckWorker', (workerName, errorPayload, delta) =>

package/plugins/redis.js

@@ -4,12 +4,17 @@ const Redis = require('ioredis');
 const logger = require('screwdriver-logger');
 const { connectionDetails } = require('../config/redis');
 
-const redis = new Redis({
-    port: connectionDetails.port,
-    host: connectionDetails.host,
-    password: connectionDetails.options.password,
-    tls: connectionDetails.options.tls
-});
+let redis;
+
+if (connectionDetails.redisClusterHosts) {
+    redis = new Redis.Cluster(connectionDetails.redisClusterHosts, {
+        redisOptions: connectionDetails.redisOptions,
+        slotsRefreshTimeout: connectionDetails.slotsRefreshTimeout,
+        clusterRetryStrategy: () => 100
+    });
+} else {
+    redis = new Redis(connectionDetails.redisOptions);
+}
 
 redis.on('connecting', () => {
     logger.info('Connecting to Redis.');

package/plugins/worker/lib/jobs.js

@@ -6,6 +6,7 @@ const hoek = require('@hapi/hoek');
 const ExecutorRouter = require('screwdriver-executor-router');
 const logger = require('screwdriver-logger');
 const AWSProducer = require('screwdriver-aws-producer-service');
+const { Plugins } = require('node-resque');
 const helper = require('../../helper');
 const { BlockedBy } = require('./BlockedBy');
 const { Filter } = require('./Filter');
@@ -346,7 +347,7 @@ async function sendWebhook(configs) {
 
 module.exports = {
     start: {
-        plugins: [Filter, 'Retry', BlockedBy],
+        plugins: [Filter, Plugins.Retry, BlockedBy],
         pluginOptions: {
             Retry: retryOptions,
             BlockedBy: blockedByOptions
@@ -354,21 +355,21 @@ module.exports = {
         perform: start
     },
     stop: {
-        plugins: [Filter, 'Retry'], // stop shouldn't use blockedBy
+        plugins: [Filter, Plugins.Retry], // stop shouldn't use blockedBy
         pluginOptions: {
             Retry: retryOptions
         },
         perform: stop
     },
     clear: {
-        plugins: [CacheFilter, 'Retry'],
+        plugins: [CacheFilter, Plugins.Retry],
         pluginOptions: {
             Retry: retryOptions
         },
         perform: clear
     },
     sendWebhook: {
-        plugins: ['Retry'],
+        plugins: [Plugins.Retry],
         pluginOptions: {
             Retry: retryOptions
         },

package/plugins/worker/worker.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const NodeResque = require('node-resque');
+const { MultiWorker, Scheduler } = require('node-resque');
 const config = require('config');
 const logger = require('screwdriver-logger');
 const Redlock = require('redlock');
@@ -8,7 +8,7 @@ const jobs = require('./lib/jobs');
 const timeout = require('./lib/timeout');
 const helper = require('../helper');
 const workerConfig = config.get('worker');
-const { connectionDetails, queuePrefix } = require('../../config/redis');
+const { queueNamespace, queuePrefix } = require('../../config/redis');
 const redis = require('../redis');
 // https://github.com/mike-marcacci/node-redlock
 const redlock = new Redlock([redis], {
@@ -17,6 +17,7 @@ const redlock = new Redlock([redis], {
     retryDelay: 200, // time in ms
     retryJitter: 200 // time in ms
 });
+const resqueConnection = { redis, namespace: queueNamespace };
 
 /**
  * Shutdown both worker and scheduler and then exit the process
@@ -38,9 +39,9 @@ async function shutDownAll(worker, scheduler) {
     }
 }
 
-const multiWorker = new NodeResque.MultiWorker(
+const multiWorker = new MultiWorker(
     {
-        connection: connectionDetails,
+        connection: resqueConnection,
         queues: [`${queuePrefix}builds`, `${queuePrefix}cache`, `${queuePrefix}webhooks`],
         minTaskProcessors: workerConfig.minTaskProcessors,
         maxTaskProcessors: workerConfig.maxTaskProcessors,
@@ -50,7 +51,7 @@ const multiWorker = new NodeResque.MultiWorker(
     jobs
 );
 
-const scheduler = new NodeResque.Scheduler({ connection: connectionDetails });
+const scheduler = new Scheduler({ connection: resqueConnection });
 
 /**
  * Start worker & scheduler
@@ -81,10 +82,14 @@ async function invoke() {
                 )}`
             )
         );
-        multiWorker.on('success', (workerId, queue, job, result) =>
-            logger.info(`queueWorker->worker[${workerId}] ${job} success ${queue} ${JSON.stringify(job)} >> ${result}`)
+        multiWorker.on('success', (workerId, queue, job, result, duration) =>
+            logger.info(
+                `queueWorker->worker[${workerId}] ${job} success ${queue} ${JSON.stringify(
+                    job
+                )} >> ${result} (${duration}ms)`
+            )
         );
-        multiWorker.on('failure', (workerId, queue, job, failure) =>
+        multiWorker.on('failure', (workerId, queue, job, failure, duration) =>
             helper
                 .updateBuildStatus({
                     redisInstance: redis,
@@ -96,14 +101,14 @@ async function invoke() {
                     logger.info(
                         `queueWorker->worker[${workerId}] ${JSON.stringify(job)} ` +
                             `failure ${queue} ` +
-                            `${JSON.stringify(job)} >> successfully update build status: ${failure}`
+                            `${JSON.stringify(job)} >> successfully update build status: ${failure} (${duration}ms)`
                     );
                 })
                 .catch(err => {
                     logger.error(
                         `queueWorker->worker[${workerId}] ${job} failure ` +
                             `${queue} ${JSON.stringify(job)} ` +
-                            `>> ${failure} ${err}`
+                            `>> ${failure} (${duration}ms) ${err}`
                     );
                 })
         );
@@ -113,7 +118,6 @@ async function invoke() {
         multiWorker.on('pause', workerId => logger.info(`queueWorker->worker[${workerId}] paused`));
 
         // multiWorker emitters
-        multiWorker.on('internalError', error => logger.error(error));
         multiWorker.on('multiWorkerAction', (verb, delay) =>
             logger.info(`queueWorker->*** checked for worker status: ${verb} (event loop delay: ${delay}ms)`)
         );
@@ -121,7 +125,7 @@ async function invoke() {
         scheduler.on('start', () => logger.info('queueWorker->scheduler started'));
         scheduler.on('end', () => logger.info('queueWorker->scheduler ended'));
         scheduler.on('poll', () => logger.info('queueWorker->scheduler polling'));
-        scheduler.on('master', state => logger.info(`queueWorker->scheduler became master ${state}`));
+        scheduler.on('leader', () => logger.info(`queueWorker->scheduler became leader`));
         scheduler.on('error', error => logger.info(`queueWorker->scheduler error >> ${error}`));
         scheduler.on('workingTimestamp', timestamp =>
             logger.info(`queueWorker->scheduler working timestamp ${timestamp}`)