screwdriver-queue-service 2.0.34 → 2.0.37

package/lib/queue.js

@@ -1,11 +1,11 @@
 'use strict';
 
-const Redis = require('ioredis');
 const Resque = require('node-resque');
 const fuses = require('circuit-fuses');
 const Breaker = fuses.breaker;
 const FuseBox = fuses.box;
 const logger = require('screwdriver-logger');
+const redis = require('../plugins/redis');
 
 module.exports = class ExecutorQueue {
     /**
@@ -41,10 +41,11 @@ module.exports = class ExecutorQueue {
 
         this.redisConnection = redisConnection;
 
-        this.redis = new Redis(redisConnection.port, redisConnection.host, redisConnection.options);
+        this.redis = redis;
 
         // eslint-disable-next-line new-cap
         this.queue = new Resque.Queue({ connection: redisConnection });
+        this.queue.on('error', error => logger.info(`Resque queue error >> ${error}`));
         this.queueBreaker = new Breaker((funcName, ...args) => {
             const callback = args.pop();
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.34",
+  "version": "2.0.37",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -20,7 +20,7 @@
     "config": "^1.31.0",
     "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
-    "ioredis": "^3.2.2",
+    "ioredis": "^5.0.4",
     "joi": "^17.5.0",
     "js-yaml": "^3.14.1",
     "jsonwebtoken": "^8.5.1",

package/plugins/helper.js

@@ -76,7 +76,7 @@ function formatOptions(method, url, token, json, retryStrategyFn) {
         });
     }
 
-    logger.info(`${options.method} ${options.uri}`);
+    logger.info(`${options.method} ${options.url}`);
 
     return options;
 }

package/plugins/queue/scheduler.js

@@ -166,15 +166,6 @@ async function startPeriodic(executor, config) {
 
     if (triggerBuild) {
         config.causeMessage = 'Started by periodic build scheduler';
-
-        // Even if post event failed for this event after retry, we should still enqueue the next event
-        try {
-            await postBuildEvent(executor, config);
-        } catch (err) {
-            logger.error(
-                `periodic builds: failed to post build event for job ${job.id} in pipeline ${pipeline.id}: ${err}`
-            );
-        }
     }
 
     if (buildCron && job.state === 'ENABLED' && !job.archived) {
@@ -214,18 +205,28 @@ async function startPeriodic(executor, config) {
                 logger.error(`failed to enqueue for job ${job.id}: ${err}`);
             }
         }
-        if (!shouldRetry) {
-            logger.info(`successfully added to queue for job ${job.id}`);
 
-            return Promise.resolve();
+        if (shouldRetry) {
+            try {
+                await executor.queue.enqueueAt(next, executor.periodicBuildQueue, 'startDelayed', [{ jobId: job.id }]);
+            } catch (err) {
+                logger.error(`failed to add to delayed queue for job ${job.id}: ${err}`);
+            }
+        } else {
+            logger.info(`successfully added to queue for job ${job.id}`);
         }
+    }
+    logger.info(`added to delayed queue for job ${job.id}`);
+
+    if (triggerBuild) {
         try {
-            await executor.queue.enqueueAt(next, executor.periodicBuildQueue, 'startDelayed', [{ jobId: job.id }]);
+            await postBuildEvent(executor, config);
         } catch (err) {
-            logger.error(`failed to add to delayed queue for job ${job.id}: ${err}`);
+            logger.error(
+                `periodic builds: failed to post build event for job ${job.id} in pipeline ${pipeline.id}: ${err}`
+            );
         }
     }
-    logger.info(`added to delayed queue for job ${job.id}`);
 
     return Promise.resolve();
 }
@@ -405,15 +406,21 @@ async function start(executor, config) {
             }
         ]);
         if (buildStats) {
-            await helper.updateBuild(
-                {
-                    buildId,
-                    token: buildToken,
-                    apiUri,
-                    payload: { stats: build.stats, status: 'QUEUED' }
-                },
-                helper.requestRetryStrategy
-            );
+            await helper
+                .updateBuild(
+                    {
+                        buildId,
+                        token: buildToken,
+                        apiUri,
+                        payload: { stats: build.stats, status: 'QUEUED' }
+                    },
+                    helper.requestRetryStrategy
+                )
+                .catch(err => {
+                    logger.error(`Failed to update build status for build ${buildId}: ${err}`);
+
+                    throw err;
+                });
         }
     }
 
@@ -530,8 +537,11 @@ async function init(executor) {
     executor.scheduler.on('master', state => logger.info(`scheduler became master ${state}`));
     executor.scheduler.on('error', error => logger.info(`scheduler error >> ${error}`));
     executor.scheduler.on('workingTimestamp', timestamp => logger.info(`scheduler working timestamp ${timestamp}`));
+    executor.scheduler.on('cleanStuckWorker', (workerName, errorPayload, delta) =>
+        logger.info(`scheduler failing ${workerName} (stuck for ${delta}s) and failing job ${errorPayload}`)
+    );
     executor.scheduler.on('transferredJob', (timestamp, job) =>
-        logger.info(`scheduler enqueuing job timestamp  >>  ${JSON.stringify(job)}`)
+        logger.info(`scheduler enqueuing job ${timestamp}  >>  ${JSON.stringify(job)}`)
     );
 
     await executor.multiWorker.start();

package/plugins/redis.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const Redis = require('ioredis');
+const logger = require('screwdriver-logger');
+const { connectionDetails } = require('../config/redis');
+
+const redis = new Redis({
+    port: connectionDetails.port,
+    host: connectionDetails.host,
+    password: connectionDetails.options.password,
+    tls: connectionDetails.options.tls
+});
+
+redis.on('connecting', () => {
+    logger.info('Connecting to Redis.');
+});
+redis.on('connect', () => {
+    logger.info('Successfully connected to Redis');
+});
+redis.on('error', err => {
+    if (err.code === 'ECONNREFUSED') {
+        logger.error(`Could not connect to Redis: ${err.message}.`);
+    } else {
+        logger.error(`Redis encountered an error: ${err.message}.`);
+        throw err;
+    }
+});
+
+module.exports = redis;

package/plugins/worker/lib/BlockedBy.js

@@ -3,11 +3,10 @@
 const NodeResque = require('node-resque');
 const hoek = require('@hapi/hoek');
 const logger = require('screwdriver-logger');
-const Redis = require('ioredis');
 const Redlock = require('redlock');
 const helper = require('../../helper');
-const { connectionDetails, runningJobsPrefix, waitingJobsPrefix, queuePrefix } = require('../../../config/redis');
-const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
+const { runningJobsPrefix, waitingJobsPrefix, queuePrefix } = require('../../../config/redis');
+const redis = require('../../redis');
 // https://github.com/mike-marcacci/node-redlock
 const redlock = new Redlock([redis], {
     driftFactor: 0.01, // time in ms

package/plugins/worker/lib/jobs.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const amqp = require('amqp-connection-manager');
-const Redis = require('ioredis');
 const config = require('config');
 const hoek = require('@hapi/hoek');
 const ExecutorRouter = require('screwdriver-executor-router');
@@ -12,7 +11,7 @@ const { BlockedBy } = require('./BlockedBy');
 const { Filter } = require('./Filter');
 const { CacheFilter } = require('./CacheFilter');
 const blockedByConfig = config.get('plugins').blockedBy;
-const { connectionDetails, queuePrefix, runningJobsPrefix, waitingJobsPrefix } = require('../../../config/redis');
+const { queuePrefix, runningJobsPrefix, waitingJobsPrefix } = require('../../../config/redis');
 const rabbitmqConf = require('../../../config/rabbitmq');
 const { amqpURI, exchange, connectOptions } = rabbitmqConf.getConfig();
 const kafkaConfig = require('../../../config/kafka');
@@ -21,8 +20,7 @@ const RETRY_LIMIT = 3;
 // This is in milliseconds, reference: https://github.com/actionhero/node-resque/blob/2ffdf0/lib/plugins/Retry.js#L12
 const RETRY_DELAY = 5 * 1000;
 const DEFAULT_BUILD_TIMEOUT = 90;
-const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
-
+const redis = require('../../redis');
 const ecosystem = config.get('ecosystem');
 const executorConfig = config.get('executor');
 

package/plugins/worker/worker.js

@@ -2,7 +2,6 @@
 
 const NodeResque = require('node-resque');
 const config = require('config');
-const Redis = require('ioredis');
 const logger = require('screwdriver-logger');
 const Redlock = require('redlock');
 const jobs = require('./lib/jobs');
@@ -10,7 +9,7 @@ const timeout = require('./lib/timeout');
 const helper = require('../helper');
 const workerConfig = config.get('worker');
 const { connectionDetails, queuePrefix } = require('../../config/redis');
-const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
+const redis = require('../redis');
 // https://github.com/mike-marcacci/node-redlock
 const redlock = new Redlock([redis], {
     driftFactor: 0.01, // time in ms
@@ -86,29 +85,27 @@ async function invoke() {
             logger.info(`queueWorker->worker[${workerId}] ${job} success ${queue} ${JSON.stringify(job)} >> ${result}`)
         );
         multiWorker.on('failure', (workerId, queue, job, failure) =>
-            helper.updateBuildStatus(
-                {
+            helper
+                .updateBuildStatus({
                     redisInstance: redis,
                     buildId: job.args[0].buildId,
                     status: 'FAILURE',
                     statusMessage: `${failure}`
-                },
-                (err, response) => {
-                    if (!err) {
-                        logger.info(
-                            `queueWorker->worker[${workerId}] ${JSON.stringify(job)} ` +
-                                `failure ${queue} ` +
-                                `${JSON.stringify(job)} >> successfully update build status: ${failure}`
-                        );
-                    } else {
-                        logger.error(
-                            `queueWorker->worker[${workerId}] ${job} failure ` +
-                                `${queue} ${JSON.stringify(job)} ` +
-                                `>> ${failure} ${err} ${JSON.stringify(response)}`
-                        );
-                    }
-                }
-            )
+                })
+                .then(() => {
+                    logger.info(
+                        `queueWorker->worker[${workerId}] ${JSON.stringify(job)} ` +
+                            `failure ${queue} ` +
+                            `${JSON.stringify(job)} >> successfully update build status: ${failure}`
+                    );
+                })
+                .catch(err => {
+                    logger.error(
+                        `queueWorker->worker[${workerId}] ${job} failure ` +
+                            `${queue} ${JSON.stringify(job)} ` +
+                            `>> ${failure} ${err}`
+                    );
+                })
         );
         multiWorker.on('error', (workerId, queue, job, error) =>
             logger.error(`queueWorker->worker[${workerId}] error ${queue} ${JSON.stringify(job)} >> ${error}`)
@@ -126,13 +123,17 @@ async function invoke() {
         scheduler.on('poll', () => logger.info('queueWorker->scheduler polling'));
         scheduler.on('master', state => logger.info(`queueWorker->scheduler became master ${state}`));
         scheduler.on('error', error => logger.info(`queueWorker->scheduler error >> ${error}`));
-        scheduler.on('working_timestamp', timestamp =>
+        scheduler.on('workingTimestamp', timestamp =>
             logger.info(`queueWorker->scheduler working timestamp ${timestamp}`)
         );
-        scheduler.on('transferred_job', (timestamp, job) =>
+        scheduler.on('transferredJob', (timestamp, job) =>
             logger.info(`queueWorker->scheduler enqueuing job timestamp  >> ${timestamp} ${JSON.stringify(job)}`)
         );
-
+        scheduler.on('cleanStuckWorker', (workerName, errorPayload, delta) =>
+            logger.info(
+                `queueWorker->scheduler failing ${workerName} (stuck for ${delta}s) and failing job ${errorPayload}`
+            )
+        );
         multiWorker.start();
 
         await scheduler.connect();