screwdriver-queue-service 5.0.3 → 6.0.1

package/plugins/worker/lib/lua/stopBuild.lua

@@ -0,0 +1,133 @@
+--[[
+    stopBuild.lua - Atomic build stop/cleanup
+
+    This script atomically cleans up all Redis state for a stopped build.
+
+    ARGV[1]  = buildId (string)
+    ARGV[2]  = jobId (string)
+    ARGV[3]  = queuePrefix (string, e.g., "resque:")
+    ARGV[4]  = runningJobsPrefix (string, e.g., "running_job_")
+    ARGV[5]  = waitingJobsPrefix (string, e.g., "waiting_job_")
+
+    Returns: JSON string with cleanup result
+    {
+        action: "CLEANED" | "NOT_RUNNING" | "PARTIAL",
+        buildId: string,
+        jobId: string,
+        keysDeleted: {
+            buildConfig: boolean,
+            runningKey: boolean,
+            lastRunningKey: boolean,
+            waitingKey: boolean,
+            timeoutConfig: boolean,
+            deleteKey: boolean
+        }
+    }
+]]
+
+
+-- Parse arguments
+local buildId = ARGV[1]
+local jobId = ARGV[2]
+local queuePrefix = ARGV[3]
+local runningJobsPrefix = ARGV[4]
+local waitingJobsPrefix = ARGV[5]
+
+-- Convert buildId to number for comparison
+local buildIdNum = tonumber(buildId)
+
+-- Build Redis keys
+local buildConfigKey = queuePrefix .. "buildConfigs"
+local timeoutConfigKey = queuePrefix .. "timeoutConfigs"
+local runningKey = runningJobsPrefix .. jobId
+local lastRunningKey = "last_" .. runningJobsPrefix .. jobId
+local waitingKey = waitingJobsPrefix .. jobId
+local deleteKey = "deleted_" .. jobId .. "_" .. buildId
+
+-- Read current state to validate ownership
+
+local currentRunningBuildId = redis.call("GET", runningKey)
+local lastRunningBuildId = redis.call("GET", lastRunningKey)
+local buildConfigExists = redis.call("HEXISTS", buildConfigKey, buildId)
+
+-- Track what gets deleted
+local keysDeleted = {
+    buildConfig = false,
+    runningKey = false,
+    lastRunningKey = false,
+    waitingKey = false,
+    timeoutConfig = false,
+    deleteKey = false
+}
+
+-- Determine what to clean up
+
+-- Convert Redis values to numbers for comparison
+local currentRunningId = currentRunningBuildId and tonumber(currentRunningBuildId) or nil
+local lastRunningId = lastRunningBuildId and tonumber(lastRunningBuildId) or nil
+
+-- Determine if this build "owns" the running keys
+local ownsRunningKey = (currentRunningId == buildIdNum)
+local ownsLastRunningKey = (lastRunningId == buildIdNum)
+
+-- Atomically clean up all keys
+
+-- Always delete buildConfig (unconditionally)
+if buildConfigExists == 1 then
+    redis.call("HDEL", buildConfigKey, buildId)
+    keysDeleted.buildConfig = true
+end
+
+-- Delete runningKey only if it matches this buildId
+if ownsRunningKey then
+    redis.call("DEL", runningKey)
+    keysDeleted.runningKey = true
+end
+
+-- Delete lastRunningKey only if it matches this buildId
+if ownsLastRunningKey then
+    redis.call("DEL", lastRunningKey)
+    keysDeleted.lastRunningKey = true
+end
+
+-- Remove from waiting queue (returns count removed)
+local waitingRemoved = redis.call("LREM", waitingKey, 0, buildId)
+if waitingRemoved > 0 then
+    keysDeleted.waitingKey = true
+end
+
+-- Delete timeout config (unconditionally)
+local timeoutDeleted = redis.call("HDEL", timeoutConfigKey, buildId)
+if timeoutDeleted > 0 then
+    keysDeleted.timeoutConfig = true
+end
+
+-- Delete the deleteKey if it exists
+local deleteKeyExists = redis.call("EXISTS", deleteKey)
+if deleteKeyExists == 1 then
+    redis.call("DEL", deleteKey)
+    keysDeleted.deleteKey = true
+end
+
+-- Report what was cleaned up
+
+-- Determine action based on what was cleaned
+local action = "CLEANED"
+
+if not ownsRunningKey and currentRunningBuildId then
+    -- Different build is running now
+    action = "NOT_RUNNING"
+elseif not keysDeleted.buildConfig and not keysDeleted.runningKey and not keysDeleted.lastRunningKey then
+    -- build already cleaned up
+    action = "ALREADY_CLEAN"
+end
+
+return cjson.encode({
+    action = action,
+    buildId = buildId,
+    jobId = jobId,
+    keysDeleted = keysDeleted,
+    currentRunningBuildId = currentRunningBuildId,
+    ownsRunningKey = ownsRunningKey,
+    ownsLastRunningKey = ownsLastRunningKey
+})

package/plugins/worker/lib/timeout.js

@@ -3,90 +3,149 @@
 const logger = require('screwdriver-logger');
 const helper = require('../../helper');
 const { waitingJobsPrefix, runningJobsPrefix, queuePrefix } = require('../../../config/redis');
+
+let luaScriptLoader;
+
+/**
+ * Get LuaScriptLoader instance
+ * @return {LuaScriptLoader} Lua script loader instance
+ */
+function getLuaScriptLoader() {
+    if (!luaScriptLoader) {
+        // eslint-disable-next-line global-require
+        const worker = require('../worker');
+
+        luaScriptLoader = worker.luaScriptLoader;
+    }
+
+    return luaScriptLoader;
+}
+
 const TIMEOUT_CODE = 3;
-const TIMEOUT_BUFFER = 1;
 const DEFAULT_TIMEOUT = 90;
 const hash = `${queuePrefix}timeoutConfigs`;
-const REDIS_LOCK_TTL = 10000; // in ms
 
 /**
- * Wrapper function to process timeout logic
- * @method process
- * @param {Object} timeoutConfig
+ * Execute timeout cleanup
+ * @param {Object} decision
  * @param {String} buildId
  * @param {Object} redis
  * @param {String} workerId
  * @return {Promise}
  */
-async function process(timeoutConfig, buildId, redis, workerId) {
+async function executeTimeout(decision, buildId, redis, workerId) {
+    const { timeoutMinutes } = decision;
+
+    // Get and update current step
+    let step;
+
     try {
-        const { jobId } = timeoutConfig;
-        const runningKey = `${runningJobsPrefix}${jobId}`;
-        const lastRunningKey = `last_${runningJobsPrefix}${jobId}`;
-        const waitingKey = `${waitingJobsPrefix}${jobId}`;
-        const deleteKey = `deleted_${jobId}_${buildId}`;
-        const timeoutValue = parseInt(timeoutConfig.timeout, 10);
-        const timeout = (Number.isNaN(timeoutValue) ? DEFAULT_TIMEOUT : timeoutValue) + TIMEOUT_BUFFER; // set timeout 1 min more than the launcher
-        const { startTime } = timeoutConfig;
+        step = await helper.getCurrentStep({
+            redisInstance: redis,
+            buildId
+        });
+    } catch (err) {
+        logger.error(`worker[${workerId}] -> No active step found for ${buildId}`);
+    }
 
-        if (!startTime) {
-            // there is no startTime set for the build
-            logger.warn(`worker[${workerId}] -> startTime not set for buildId: ${buildId}`);
+    if (step) {
+        await helper.updateStepStop({
+            redisInstance: redis,
+            buildId,
+            stepName: step.name,
+            code: TIMEOUT_CODE
+        });
+    }
 
-            return;
-        }
+    await helper.updateBuildStatus({
+        redisInstance: redis,
+        buildId,
+        status: 'FAILURE',
+        statusMessage: `Build failed due to timeout (${timeoutMinutes} minutes)`
+    });
 
-        const diffMs = new Date().getTime() - new Date(startTime).getTime();
-        const diffMins = Math.round(diffMs / 60000);
+    logger.info(`worker[${workerId}] -> Timeout cleanup completed for build ${buildId}`);
+}
 
-        // check if build has timed out, if yes abort build
-        if (diffMins > timeout) {
-            let step;
+/**
+ * Handle the decision from Lua script
+ * @param {Object} decision - {action, reason, buildId, ...}
+ * @param {String} buildId - Build ID
+ * @param {Object} redis - Redis instance
+ * @param {String} workerId - Worker ID
+ * @return {Promise}
+ */
+async function handleDecision(decision, buildId, redis, workerId) {
+    switch (decision.action) {
+        case 'TIMEOUT':
+            // Build has timed out - execute cleanup
+            logger.info(`worker[${workerId}] -> Build has timed out ${buildId}`);
+            await executeTimeout(decision, buildId, redis, workerId);
+            break;
 
-            try {
-                step = await helper.getCurrentStep({
-                    redisInstance: redis,
-                    buildId
-                });
-            } catch (err) {
-                logger.error(`worker[${workerId}] -> No active step found for ${buildId}`);
-            }
+        case 'CLEANUP':
+            // Build already completed/cleaned up - just remove from timeout configs
+            logger.info(`worker[${workerId}] -> Build ${buildId} ${decision.reason}, cleaning up`);
+            await redis.hdel(hash, buildId);
+            break;
 
-            if (step) {
-                await helper.updateStepStop({
-                    redisInstance: redis,
-                    buildId,
-                    stepName: step.name,
-                    code: TIMEOUT_CODE
-                });
-            }
+        case 'SKIP':
+            // Build still within timeout - do nothing
+            break;
 
-            await helper.updateBuildStatus({
-                redisInstance: redis,
-                buildId,
-                status: 'FAILURE',
-                statusMessage: 'Build failed due to timeout'
-            });
+        default:
+            logger.error(`worker[${workerId}] -> Unknown timeout action: ${decision.action}`);
+    }
+}
 
-            logger.info(`worker[${workerId}] -> Build has timed out ${buildId}`);
+/**
+ * @method process
+ * @param {Object} timeoutConfig
+ * @param {String} buildId
+ * @param {Object} redis
+ * @param {String} workerId
+ * @return {Promise}
+ */
+async function process(timeoutConfig, buildId, redis, workerId) {
+    const { jobId, startTime, timeout } = timeoutConfig;
 
-            await redis.hdel(`${queuePrefix}buildConfigs`, buildId);
+    logger.info(`worker[${workerId}] -> Checking timeout for build ${buildId} (job ${jobId})`);
 
-            // expire now as build failed
-            await redis.expire(runningKey, 0);
-            await redis.expire(lastRunningKey, 0);
+    if (!startTime) {
+        logger.warn(`worker[${workerId}] -> startTime not set for buildId: ${buildId}`);
+        await redis.hdel(hash, buildId);
 
-            await redis.del(deleteKey);
-            await redis.lrem(waitingKey, 0, buildId);
+        return;
+    }
 
-            // remove from timeout configs after build is timed out
-            await redis.hdel(hash, buildId);
-        }
+    try {
+        const loader = getLuaScriptLoader();
+        const result = await loader.executeScript(
+            'checkTimeout.lua',
+            [],
+            [
+                String(buildId),
+                String(jobId),
+                String(startTime),
+                String(timeout || DEFAULT_TIMEOUT),
+                String(Date.now()),
+                queuePrefix,
+                runningJobsPrefix,
+                waitingJobsPrefix
+            ]
+        );
+
+        const decision = JSON.parse(result);
+
+        logger.info(`worker[${workerId}] -> Build ${buildId}: action=${decision.action}, reason=${decision.reason}`);
+
+        await handleDecision(decision, buildId, redis, workerId);
     } catch (err) {
-        // delete key from redis in case of error to prevent reprocessing
-        await redis.hdel(hash, buildId);
+        logger.error(`Error in timeout check for build ${buildId}: ${err.message}`);
+        logger.error(err.stack);
 
-        logger.error(`worker[${workerId}] -> Error occurred while checking timeout for buildId: ${buildId}, ${err}`);
+        // Delete key from redis in case of error to prevent reprocessing
+        await redis.hdel(hash, buildId);
     }
 }
 
@@ -95,11 +154,10 @@ async function process(timeoutConfig, buildId, redis, workerId) {
  * If yes, abort build.
  * @method check
  * @param {Object} redis
- * @param {Object} redlock
  * @param {String} workerId
  * @return {Promise}
  */
-async function check(redis, redlock, workerId) {
+async function check(redis, workerId) {
     const keys = await redis.hkeys(hash);
 
     if (!keys || keys.length === 0) return;
@@ -107,7 +165,6 @@ async function check(redis, redlock, workerId) {
     await Promise.all(
         keys.map(async buildId => {
             try {
-                const lock = await redlock.lock(buildId, REDIS_LOCK_TTL);
                 const json = await redis.hget(hash, buildId);
 
                 if (!json) return;
@@ -119,10 +176,8 @@ async function check(redis, redlock, workerId) {
                 }
 
                 await process(timeoutConfig, buildId, redis, workerId);
-
-                await lock.unlock();
             } catch (err) {
-                logger.error(`worker[${workerId}] -> Redis locking error ${buildId}: ${err.message}`);
+                logger.error(`worker[${workerId}] -> Error checking timeout for build ${buildId}: ${err.message}`);
             }
         })
     );
@@ -135,13 +190,13 @@ async function check(redis, redlock, workerId) {
 const workerAccessMap = {};
 
 /**
+ * Check timeouts with backoff to avoid excessive polling
  *
  * @param {Object} redis
- * @param {Object} redlock
  * @param {Number} workerId
  * @param {Number} [pollInterval]
  */
-async function checkWithBackOff(redis, redlock, workerId, pollInterval = 60) {
+async function checkWithBackOff(redis, workerId, pollInterval = 60) {
     workerAccessMap[workerId] = workerAccessMap[workerId] || new Date();
     const diffMs = new Date().getTime() - new Date(workerAccessMap[workerId]).getTime();
     const diffSeconds = Math.round(diffMs / 1000);
@@ -150,7 +205,7 @@ async function checkWithBackOff(redis, redlock, workerId, pollInterval = 60) {
     if (diffSeconds >= pollInterval) {
         logger.info('worker[%s] -> Processing timeout checks', workerId);
         workerAccessMap[workerId] = new Date();
-        await check(redis, redlock, workerId);
+        await check(redis, workerId);
     }
 }
 

package/plugins/worker/worker.js

@@ -3,21 +3,16 @@
 const { MultiWorker, Scheduler } = require('node-resque');
 const config = require('config');
 const logger = require('screwdriver-logger');
-const Redlock = require('redlock');
 const jobs = require('./lib/jobs');
 const timeout = require('./lib/timeout');
 const helper = require('../helper');
+const LuaScriptLoader = require('./lib/LuaScriptLoader');
 const workerConfig = config.get('worker');
 const { queueNamespace, queuePrefix } = require('../../config/redis');
 const redis = require('../redis');
-// https://github.com/mike-marcacci/node-redlock
-const redlock = new Redlock([redis], {
-    driftFactor: 0.01, // time in ms
-    retryCount: 5,
-    retryDelay: 200, // time in ms
-    retryJitter: 200 // time in ms
-});
+
 const resqueConnection = { redis, namespace: queueNamespace };
+const luaScriptLoader = new LuaScriptLoader(redis);
 
 /**
  * Shutdown both worker and scheduler and then exit the process
@@ -60,6 +55,10 @@ const scheduler = new Scheduler({ connection: resqueConnection });
  */
 async function invoke() {
     try {
+        logger.info('Loading Lua scripts...');
+        await luaScriptLoader.loadAllScripts();
+        logger.info('Lua scripts loaded successfully');
+
         /* eslint-disable max-len */
         multiWorker.on('start', workerId => logger.info(`queueWorker->worker[${workerId}] started`));
         multiWorker.on('end', workerId => logger.info(`queueWorker->worker[${workerId}] ended`));
@@ -69,7 +68,7 @@ async function invoke() {
         multiWorker.on('poll', async (workerId, queue) => {
             if (queue === 'builds') {
                 logger.info(`queueWorker->worker[${workerId}] polling ${queue}`);
-                await timeout.checkWithBackOff(redis, redlock, workerId);
+                await timeout.checkWithBackOff(redis, workerId);
             }
         });
         multiWorker.on('job', (workerId, queue, job) =>
@@ -165,5 +164,6 @@ module.exports = {
     multiWorker,
     scheduler,
     shutDownAll,
-    cleanUp
+    cleanUp,
+    luaScriptLoader
 };