screwdriver-queue-service 2.0.23 → 2.0.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.23",
+  "version": "2.0.27",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {

package/plugins/worker/lib/BlockedBy.js

@@ -3,8 +3,19 @@
 const NodeResque = require('node-resque');
 const hoek = require('@hapi/hoek');
 const logger = require('screwdriver-logger');
+const Redis = require('ioredis');
+const Redlock = require('redlock');
 const helper = require('../../helper');
-const { runningJobsPrefix, waitingJobsPrefix, queuePrefix } = require('../../../config/redis');
+const { connectionDetails, runningJobsPrefix, waitingJobsPrefix, queuePrefix } = require('../../../config/redis');
+const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
+// https://github.com/mike-marcacci/node-redlock
+const redlock = new Redlock([redis], {
+    driftFactor: 0.01, // time in ms
+    retryCount: 5,
+    retryDelay: 200, // time in ms
+    retryJitter: 200 // time in ms
+});
+const REDIS_LOCK_TTL = 10000; // in ms
 const BLOCK_TIMEOUT_BUFFER = 30;
 
 /**
@@ -118,212 +129,192 @@ async function blockedBySelf({ waitingKey, buildId, collapse }) {
 }
 
 /**
- *
- * @param {String} buildConfig
- * @param {Array} blockingBuildIds
- * @param {String} buildId
- * @param {Object} redisConn
+ * Checks if there are any blocking jobs running.
+ * If yes, re-enqueue. If no, check if there is the same job waiting.
+ * If buildId is not the same, re-enqueue. Otherwise, proceeds and set the current job as running
+ * @method checkBlockingJob
+ * @param  {Number}      jobId          Current jobId
+ * @param  {Number}      buildId        Current buildId
+ * @return {Boolean}
  */
-async function checkMultipleBuildsInSameEvent(buildConfig, blockingBuildIds, buildId, redisConn) {
-    if (!buildConfig) return false;
+async function checkBlockingJob({ jobId, buildId }) {
+    logger.info('%s | %s | Processing blocked-by filter', buildId, jobId);
+
+    const runningKey = `${runningJobsPrefix}${jobId}`;
+    const lastRunningKey = `last_${runningJobsPrefix}${jobId}`;
+    const waitingKey = `${waitingJobsPrefix}${jobId}`;
+    const deleteKey = `deleted_${jobId}_${buildId}`;
+    const enforceBlockedBySelf = String(this.options.blockedBySelf) === 'true'; // because kubernetes value is a string
+    const shouldDelete = await this.queueObject.connection.redis.get(deleteKey);
+    const runningBuildId = await this.queueObject.connection.redis.get(runningKey);
+    const lastRunningBuildId = await this.queueObject.connection.redis.get(lastRunningKey);
+    const enableCollapse = String(this.options.collapse) === 'true'; // because kubernetes value is a string
+    const buildConfig = await this.queueObject.connection.redis.hget(`${queuePrefix}buildConfigs`, buildId);
+    const annotations = hoek.reach(JSON.parse(buildConfig), 'annotations', {
+        default: {}
+    });
+    const collapse = hoek.reach(annotations, 'screwdriver.cd/collapseBuilds', {
+        default: enableCollapse,
+        separator: '>'
+    });
+    const timeout = hoek.reach(annotations, 'screwdriver.cd/timeout', {
+        separator: '>'
+    });
+
+    // For retry logic: failed to create pod, so it will retry
+    // Current buildId is already set as runningKey. Should proceed
+    if (parseInt(runningBuildId, 10) === buildId) {
+        return true;
+    }
+
+    // Current build is older than last running build for the same job, discard the build
+    if (collapse && buildId < parseInt(lastRunningBuildId, 10)) {
+        await this.queueObject.connection.redis.lrem(waitingKey, 0, buildId);
+        await helper
+            .updateBuildStatus({
+                redisInstance: this.queueObject.connection.redis,
+                buildId,
+                status: 'COLLAPSED',
+                statusMessage: `Collapsed to build: ${lastRunningBuildId}`
+            })
+            .catch(err => {
+                logger.error(`Failed to update build status to COLLAPSED for build:${buildId}:${err}`);
+            });
+        await this.queueObject.connection.redis.hdel(`${queuePrefix}buildConfigs`, buildId);
 
-    const currentBuild = JSON.parse(buildConfig);
+        logger.info('%s | %s | Remove waiting key and collapse build', buildId, jobId);
 
-    if (!currentBuild || !currentBuild.eventId || !currentBuild.jobId) {
         return false;
     }
 
-    const isSameBuild = await Promise.race(
-        blockingBuildIds.map(async id => {
-            const blockedBuild = JSON.parse(await redisConn.hget(`${queuePrefix}buildConfigs`, id));
-            const hasEventId = blockedBuild && blockedBuild.eventId && blockedBuild.jobId;
-            const isSameJob =
-                hasEventId &&
-                currentBuild.eventId === blockedBuild.eventId &&
-                currentBuild.jobId === blockedBuild.jobId;
-
-            if (isSameJob) {
-                logger.error(`Builds ${id} & ${buildId} have the same event 
-                ${currentBuild.eventId} & same job ${currentBuild.jobId}`);
-            }
+    // If this build is in the delete list (it was aborted)
+    if (shouldDelete !== null) {
+        await this.queueObject.connection.redis.del(deleteKey);
 
-            return isSameJob;
-        })
-    );
+        //  Clean up to prevent race condition: stop and beforePerform happen at the same time
+        //  stop deletes key runningKey and waitingKey
+        //  beforePerform either proceeds or reEnqueue (which adds the key back)
+        await this.queueObject.connection.redis.lrem(waitingKey, 0, buildId);
 
-    return isSameBuild;
-}
+        if (parseInt(runningBuildId, 10) === buildId) {
+            await this.queueObject.connection.redis.del(runningKey);
+        }
 
-class BlockedBy extends NodeResque.Plugin {
-    /**
-     * Construct a new BlockedBy plugin
-     * @method constructor
-     */
-    constructor(worker, func, queue, job, args, options) {
-        super(worker, func, queue, job, args, options);
+        logger.info('%s | %s | Delete runningKey and waitingKey', buildId, jobId);
 
-        this.name = 'BlockedBy';
+        // Should not proceed since this build was previously aborted
+        return false;
     }
 
-    /**
-     * Checks if there are any blocking jobs running.
-     * If yes, re-enqueue. If no, check if there is the same job waiting.
-     * If buildId is not the same, re-enqueue. Otherwise, proceeds and set the current job as running
-     * @method beforePerform
-     * @return {Promise}
-     */
-    async beforePerform() {
-        const { jobId, buildId } = this.args[0];
-
-        logger.info('%s | %s | Processing blocked-by filter', buildId, jobId);
-
-        const runningKey = `${runningJobsPrefix}${jobId}`;
-        const lastRunningKey = `last_${runningJobsPrefix}${jobId}`;
-        const waitingKey = `${waitingJobsPrefix}${jobId}`;
-        const deleteKey = `deleted_${jobId}_${buildId}`;
-        const enforceBlockedBySelf = String(this.options.blockedBySelf) === 'true'; // because kubernetes value is a string
-        const shouldDelete = await this.queueObject.connection.redis.get(deleteKey);
-        const runningBuildId = await this.queueObject.connection.redis.get(runningKey);
-        const lastRunningBuildId = await this.queueObject.connection.redis.get(lastRunningKey);
-        const enableCollapse = String(this.options.collapse) === 'true'; // because kubernetes value is a string
-        const buildConfig = await this.queueObject.connection.redis.hget(`${queuePrefix}buildConfigs`, buildId);
-        const annotations = hoek.reach(JSON.parse(buildConfig), 'annotations', {
-            default: {}
-        });
-        const collapse = hoek.reach(annotations, 'screwdriver.cd/collapseBuilds', {
-            default: enableCollapse,
-            separator: '>'
-        });
-        const timeout = hoek.reach(annotations, 'screwdriver.cd/timeout', {
-            separator: '>'
-        });
+    let blockedBy = this.args[0].blockedBy.split(',').map(jid => `${runningJobsPrefix}${jid}`);
 
-        // For retry logic: failed to create pod, so it will retry
-        // Current buildId is already set as runningKey. Should proceed
-        if (parseInt(runningBuildId, 10) === buildId) {
-            return true;
-        }
+    if (!enforceBlockedBySelf) {
+        blockedBy = blockedBy.filter(key => key !== `${runningJobsPrefix}${jobId}`); // remove itself from blocking list
+    }
 
-        // Current build is older than last running build for the same job, discard the build
-        if (collapse && buildId < parseInt(lastRunningBuildId, 10)) {
-            await this.queueObject.connection.redis.lrem(waitingKey, 0, buildId);
-            await helper
-                .updateBuildStatus({
-                    redisInstance: this.queueObject.connection.redis,
-                    buildId,
-                    status: 'COLLAPSED',
-                    statusMessage: `Collapsed to build: ${lastRunningBuildId}`
-                })
-                .catch(err => {
-                    logger.error(`Failed to update build status to COLLAPSED for build:${buildId}:${err}`);
-                });
-            await this.queueObject.connection.redis.hdel(`${queuePrefix}buildConfigs`, buildId);
+    if (blockedBy.length > 0) {
+        logger.info('%s | %s | BlockedBy list:%s', buildId, jobId, blockedBy);
 
-            logger.info('%s | %s | Remove waiting key and collapse build', buildId, jobId);
+        const blockingBuildIds = [];
 
-            return false;
-        }
+        // Get the blocking job
+        await Promise.all(
+            blockedBy.map(async key => {
+                const val = await this.queueObject.connection.redis.get(key);
 
-        // If this build is in the delete list (it was aborted)
-        if (shouldDelete !== null) {
-            await this.queueObject.connection.redis.del(deleteKey);
+                if (val !== null) {
+                    blockingBuildIds.push(val);
+                }
+            })
+        );
 
-            //  Clean up to prevent race condition: stop and beforePerform happen at the same time
-            //  stop deletes key runningKey and waitingKey
-            //  beforePerform either proceeds or reEnqueue (which adds the key back)
-            await this.queueObject.connection.redis.lrem(waitingKey, 0, buildId);
+        logger.info('%s | %s | blockingBuildIds:%s', buildId, jobId, blockingBuildIds);
 
-            if (parseInt(runningBuildId, 10) === buildId) {
-                await this.queueObject.connection.redis.del(runningKey);
+        // If any blocking job is running, then re-enqueue
+        if (blockingBuildIds.length > 0) {
+            if (enforceBlockedBySelf && collapse) {
+                await collapseBuilds.call(this, {
+                    waitingKey,
+                    buildId,
+                    blockingBuildIds
+                });
+            } else {
+                await this.reEnqueue(waitingKey, buildId, blockingBuildIds);
             }
 
-            logger.info('%s | %s | Delete runningKey and waitingKey', buildId, jobId);
-
-            // Should not proceed since this build was previously aborted
             return false;
         }
+    }
 
-        let blockedBy = this.args[0].blockedBy.split(',').map(jid => `${runningJobsPrefix}${jid}`);
-
-        if (!enforceBlockedBySelf) {
-            blockedBy = blockedBy.filter(key => key !== `${runningJobsPrefix}${jobId}`); // remove itself from blocking list
-        }
+    if (enforceBlockedBySelf) {
+        // only check this if feature is on
+        const blocked = await blockedBySelf.call(this, {
+            // pass in this context
+            waitingKey,
+            buildId,
+            runningBuildId,
+            collapse
+        });
 
-        if (blockedBy.length > 0) {
-            logger.info('%s | %s | BlockedBy list:%s', buildId, jobId, blockedBy);
+        if (blocked) {
+            return false;
+        } // if blocked then cannot proceed
+    } else {
+        // clean up waitingKey
+        await this.queueObject.connection.redis.del(waitingKey);
+    }
 
-            const blockingBuildIds = [];
+    // Register the curent job as running by setting key
+    await this.queueObject.connection.redis.set(runningKey, buildId);
+    // Set lastRunningKey
+    await this.queueObject.connection.redis.set(lastRunningKey, buildId);
 
-            // Get the blocking job
-            await Promise.all(
-                blockedBy.map(async key => {
-                    const val = await this.queueObject.connection.redis.get(key);
+    // Set expire time to take care of the case where
+    // afterPerform failed to call and blocked jobs will be stuck forever
+    await this.queueObject.connection.redis.expire(runningKey, this.blockTimeout(timeout) * 60);
+    await this.queueObject.connection.redis.expire(lastRunningKey, this.blockTimeout(timeout) * 60);
 
-                    if (val !== null) {
-                        blockingBuildIds.push(val);
-                    }
-                })
-            );
+    // Proceed
+    return true;
+}
 
-            logger.info('%s | %s | blockingBuildIds:%s', buildId, jobId, blockingBuildIds);
+class BlockedBy extends NodeResque.Plugin {
+    /**
+     * Construct a new BlockedBy plugin
+     * @method constructor
+     */
+    constructor(worker, func, queue, job, args, options) {
+        super(worker, func, queue, job, args, options);
 
-            // If any blocking job is running, then re-enqueue
-            if (blockingBuildIds.length > 0) {
-                // if build is from same event then don't re-enqueue
-                const isSameBuild = await checkMultipleBuildsInSameEvent(
-                    buildConfig,
-                    blockingBuildIds,
-                    buildId,
-                    this.queueObject.connection.redis
-                );
+        this.name = 'BlockedBy';
+    }
 
-                if (isSameBuild) {
-                    return false;
-                }
-                if (enforceBlockedBySelf && collapse) {
-                    await collapseBuilds.call(this, {
-                        waitingKey,
-                        buildId,
-                        blockingBuildIds
-                    });
-                } else {
-                    await this.reEnqueue(waitingKey, buildId, blockingBuildIds);
-                }
+    /**
+     * Returns true to proceed
+     * @method beforePerform
+     * @return {Promise}
+     */
+    async beforePerform() {
+        const { jobId, buildId } = this.args[0];
+        let lock;
 
-                return false;
-            }
+        try {
+            lock = await redlock.lock(`jobId_${jobId}`, REDIS_LOCK_TTL);
+        } catch (err) {
+            logger.error(`Failed to lock job ${jobId} for ${buildId}: ${err}`);
         }
 
-        if (enforceBlockedBySelf) {
-            // only check this if feature is on
-            const blocked = await blockedBySelf.call(this, {
-                // pass in this context
-                waitingKey,
-                buildId,
-                runningBuildId,
-                collapse
-            });
+        const shouldProceed = await checkBlockingJob.call(this, { jobId, buildId });
 
-            if (blocked) {
-                return false;
-            } // if blocked then cannot proceed
-        } else {
-            // clean up waitingKey
-            await this.queueObject.connection.redis.del(waitingKey);
+        if (lock) {
+            try {
+                await lock.unlock();
+            } catch (err) {
+                logger.error(`Failed to unlock job ${jobId} for ${buildId}: ${err}`);
+            }
         }
 
-        // Register the curent job as running by setting key
-        await this.queueObject.connection.redis.set(runningKey, buildId);
-        // Set lastRunningKey
-        await this.queueObject.connection.redis.set(lastRunningKey, buildId);
-
-        // Set expire time to take care of the case where
-        // afterPerform failed to call and blocked jobs will be stuck forever
-        await this.queueObject.connection.redis.expire(runningKey, this.blockTimeout(timeout) * 60);
-        await this.queueObject.connection.redis.expire(lastRunningKey, this.blockTimeout(timeout) * 60);
-
-        // Proceed
-        return true;
+        return shouldProceed;
     }
 
     /**