screwdriver-queue-service 6.0.1 → 6.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "6.0.1",
+  "version": "6.0.2",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {

package/plugins/helper.js

@@ -85,12 +85,15 @@ function formatOptions(method, url, token, json, retryStrategyFn) {
  * Update build status
  * @method updateBuildStatus
  * @param  {Object}  updateConfig build config of the job
+ * @param  {Object}  updateConfig.buildConfig Optional buildConfig to avoid Redis lookup
  * @return {Promise}
  */
 async function updateBuildStatus(updateConfig) {
-    const { redisInstance, status, statusMessage, buildId } = updateConfig;
+    const { redisInstance, status, statusMessage, buildId, buildConfig: providedConfig } = updateConfig;
 
-    const buildConfig = await redisInstance.hget(`${queuePrefix}buildConfigs`, buildId).then(JSON.parse);
+    // Use provided buildConfig if available, otherwise fetch from Redis
+    const buildConfig =
+        providedConfig || (await redisInstance.hget(`${queuePrefix}buildConfigs`, buildId).then(JSON.parse));
 
     if (!buildConfig) return null;
 
@@ -118,11 +121,15 @@ async function updateBuildStatus(updateConfig) {
  * @param {String} stepConfig.buildId
  * @param {String} stepConfig.stepName
  * @param {Integer} stepConfig.code
+ * @param {Object} stepConfig.buildConfig Optional buildConfig to avoid Redis lookup
  * @return {Promise} response body or error
  */
 async function updateStepStop(stepConfig) {
-    const { redisInstance, buildId, stepName, code } = stepConfig;
-    const buildConfig = await redisInstance.hget(`${queuePrefix}buildConfigs`, buildId).then(JSON.parse);
+    const { redisInstance, buildId, stepName, code, buildConfig: providedConfig } = stepConfig;
+
+    // Use provided buildConfig if available, otherwise fetch from Redis
+    const buildConfig =
+        providedConfig || (await redisInstance.hget(`${queuePrefix}buildConfigs`, buildId).then(JSON.parse));
 
     // if buildConfig got deleted already, do not update
     if (!buildConfig) return null;
@@ -149,11 +156,15 @@ async function updateStepStop(stepConfig) {
  * @param {Object} stepConfig
  * @param {Object} stepConfig.redisInstance
  * @param {String} stepConfig.buildId
+ * @param {Object} stepConfig.buildConfig Optional buildConfig to avoid Redis lookup
  * @return {Promise} active step or error
  */
 async function getCurrentStep(stepConfig) {
-    const { redisInstance, buildId } = stepConfig;
-    const buildConfig = await redisInstance.hget(`${queuePrefix}buildConfigs`, buildId).then(JSON.parse);
+    const { redisInstance, buildId, buildConfig: providedConfig } = stepConfig;
+
+    // Use provided buildConfig if available, otherwise fetch from Redis
+    const buildConfig =
+        providedConfig || (await redisInstance.hget(`${queuePrefix}buildConfigs`, buildId).then(JSON.parse));
 
     // if buildConfig got deleted already, do not update
     if (!buildConfig) return null;

package/plugins/worker/lib/BlockedBy.js

@@ -134,11 +134,18 @@ class BlockedBy extends NodeResque.Plugin {
      */
     async reEnqueue(buildConfig, decision) {
         const { buildId } = buildConfig;
-        const blockedBy = decision.blockedBy || [];
+        const blockedByArray = decision.blockedBy || [];
+        const runningBuildId = decision.runningBuildId;
+
+        const allBlockingBuilds = [...blockedByArray];
+
+        if (runningBuildId) {
+            allBlockingBuilds.push(runningBuildId);
+        }
 
         let statusMessage = 'Blocked by these running build(s): ';
 
-        statusMessage += blockedBy
+        statusMessage += allBlockingBuilds
             .map(blockingBuildId => `<a href="/builds/${blockingBuildId}">${blockingBuildId}</a>`)
             .join(', ');
 
@@ -177,7 +184,8 @@ class BlockedBy extends NodeResque.Plugin {
                 redisInstance: this.queueObject.connection.redis,
                 buildId,
                 status: 'COLLAPSED',
-                statusMessage: newestBuild ? `Collapsed to build: ${newestBuild}` : 'Collapsed'
+                statusMessage: newestBuild ? `Collapsed to build: ${newestBuild}` : 'Collapsed',
+                buildConfig
             })
             .catch(err => {
                 logger.error(`Failed to update build status to COLLAPSED for build:${buildId}:${err}`);

package/plugins/worker/lib/lua/startBuild.lua

@@ -44,6 +44,32 @@ local waitingBuilds = redis.call("LRANGE", waitingKey, 0, -1)
 local lastRunningBuildId = redis.call("GET", lastRunningKey)
 local deleteKeyExists = redis.call("EXISTS", deleteKey)
 
+-- Early exit: If buildConfig doesn't exist, build was already processed/deleted
+-- Clean up any remaining state and abort
+if not buildConfig then
+    redis.log(redis.LOG_WARNING, string.format(
+        "[startBuild] BUILD_CONFIG_MISSING: build=%s job=%s - buildConfig deleted externally",
+        buildId, jobId
+    ))
+
+    redis.call("LREM", waitingKey, 0, buildId)
+    redis.call("DEL", deleteKey)
+
+    -- If this build is somehow marked as running, clean that up too
+    if runningBuildId and tostring(runningBuildId) == tostring(buildId) then
+        redis.call("DEL", runningKey)
+    end
+    if lastRunningBuildId and tostring(lastRunningBuildId) == tostring(buildId) then
+        redis.call("DEL", lastRunningKey)
+    end
+
+    return cjson.encode({
+        action = "ABORT",
+        reason = "BUILD_CONFIG_MISSING",
+        buildId = buildId
+    })
+end
+
 -- Check if build was aborted (deleteKey exists)
 local isAborted = (deleteKeyExists == 1)
 
@@ -58,25 +84,19 @@ end
 
 -- Check which dependencies are currently running
 local runningBuilds = {}
+local runningJobIds = {}
 for _, depJobId in ipairs(dependencies) do
     local depRunningBuild = redis.call("GET", runningJobsPrefix .. tostring(depJobId))
     if depRunningBuild then
         table.insert(runningBuilds, depRunningBuild)
+        table.insert(runningJobIds, depJobId)
     end
 end
 
 -- Helper: Check if blocked by dependencies
-local function isBlockedByDependencies(deps, running)
-    local blockedByBuilds = {}
-    for _, dep in ipairs(deps) do
-        for _, runningId in ipairs(running) do
-            if tostring(dep) == tostring(runningId) then
-                table.insert(blockedByBuilds, dep)
-                break
-            end
-        end
-    end
-    return #blockedByBuilds > 0, blockedByBuilds
+-- Returns: isBlocked (boolean), blockedByBuilds (array of buildIds that are blocking)
+local function isBlockedByDependencies(runningBuildIds)
+    return #runningBuildIds > 0, runningBuildIds
 end
 
 -- Helper: Check if blocked by same job
@@ -121,10 +141,17 @@ local function shouldCollapse()
     return false, nil, "IS_NEWEST_BUILD"
 end
 
-local isBlockedByDeps, blockedByBuilds = isBlockedByDependencies(dependencies, runningBuilds)
+local isBlockedByDeps, blockedByBuilds = isBlockedByDependencies(runningBuilds)
 local isBlockedBySelf = isBlockedBySameJob()
 local shouldCollapseFlag, newestBuild, collapseReason = shouldCollapse()
 
+redis.log(redis.LOG_NOTICE, string.format(
+    "[startBuild] build=%s job=%s deps=%d blockedByDeps=%s blockedBySelf=%s runningBuildId=%s lastRunning=%s shouldCollapse=%s deleteKeyExists=%s",
+    buildId, jobId, #dependencies, tostring(isBlockedByDeps), tostring(isBlockedBySelf),
+    tostring(runningBuildId or "nil"), tostring(lastRunningBuildId or "nil"),
+    tostring(shouldCollapseFlag), tostring(deleteKeyExists == 1)
+))
+
 -- Determine final action (Priority: ABORT > COLLAPSE > BLOCK > START)
 local action, reason, actionData
 
@@ -154,7 +181,11 @@ end
 
 -- Update Redis state based on decision
 if action == "ABORT" then
-    -- Build was aborted, no state changes needed
+    -- Build was aborted - clean up waiting queue and configs
+    redis.call("HDEL", buildConfigKey, buildId)
+    redis.call("LREM", waitingKey, 0, buildId)
+    redis.call("DEL", deleteKey)
+
     return cjson.encode({
         action = "ABORT",
         reason = reason,
@@ -163,6 +194,10 @@ if action == "ABORT" then
 
 elseif action == "COLLAPSE" then
     -- Collapse this build - remove from configs and waiting queue
+    redis.log(redis.LOG_NOTICE, string.format(
+        "[startBuild] COLLAPSE: Deleting buildConfig for build=%s job=%s reason=%s",
+        buildId, jobId, reason
+    ))
     redis.call("HDEL", buildConfigKey, buildId)
     redis.call("LREM", waitingKey, 0, buildId)
 
@@ -187,9 +222,6 @@ elseif action == "BLOCK" then
         redis.call("RPUSH", waitingKey, buildId)
     end
 
-    -- Set deleteKey with timeout (prevents zombie builds)
-    redis.call("SET", deleteKey, buildId, "EX", blockTimeout * 60)
-
     return cjson.encode({
         action = "BLOCK",
         reason = reason,

package/plugins/worker/lib/timeout.js

@@ -31,9 +31,10 @@ const hash = `${queuePrefix}timeoutConfigs`;
  * @param {String} buildId
  * @param {Object} redis
  * @param {String} workerId
+ * @param {Object} buildConfig - Build config fetched before Lua execution
  * @return {Promise}
  */
-async function executeTimeout(decision, buildId, redis, workerId) {
+async function executeTimeout(decision, buildId, redis, workerId, buildConfig) {
     const { timeoutMinutes } = decision;
 
     // Get and update current step
@@ -42,7 +43,8 @@ async function executeTimeout(decision, buildId, redis, workerId) {
     try {
         step = await helper.getCurrentStep({
             redisInstance: redis,
-            buildId
+            buildId,
+            buildConfig
         });
     } catch (err) {
         logger.error(`worker[${workerId}] -> No active step found for ${buildId}`);
@@ -53,7 +55,8 @@ async function executeTimeout(decision, buildId, redis, workerId) {
             redisInstance: redis,
             buildId,
             stepName: step.name,
-            code: TIMEOUT_CODE
+            code: TIMEOUT_CODE,
+            buildConfig
         });
     }
 
@@ -61,7 +64,8 @@ async function executeTimeout(decision, buildId, redis, workerId) {
         redisInstance: redis,
         buildId,
         status: 'FAILURE',
-        statusMessage: `Build failed due to timeout (${timeoutMinutes} minutes)`
+        statusMessage: `Build failed due to timeout (${timeoutMinutes} minutes)`,
+        buildConfig
     });
 
     logger.info(`worker[${workerId}] -> Timeout cleanup completed for build ${buildId}`);
@@ -73,14 +77,15 @@ async function executeTimeout(decision, buildId, redis, workerId) {
  * @param {String} buildId - Build ID
  * @param {Object} redis - Redis instance
  * @param {String} workerId - Worker ID
+ * @param {Object} buildConfig - Build config fetched before Lua execution
  * @return {Promise}
  */
-async function handleDecision(decision, buildId, redis, workerId) {
+async function handleDecision(decision, buildId, redis, workerId, buildConfig) {
     switch (decision.action) {
         case 'TIMEOUT':
             // Build has timed out - execute cleanup
             logger.info(`worker[${workerId}] -> Build has timed out ${buildId}`);
-            await executeTimeout(decision, buildId, redis, workerId);
+            await executeTimeout(decision, buildId, redis, workerId, buildConfig);
             break;
 
         case 'CLEANUP':
@@ -119,6 +124,10 @@ async function process(timeoutConfig, buildId, redis, workerId) {
     }
 
     try {
+        // Fetch buildConfig BEFORE Lua execution (Lua script will delete it if timeout)
+        const buildConfigJson = await redis.hget(`${queuePrefix}buildConfigs`, buildId);
+        const buildConfig = buildConfigJson ? JSON.parse(buildConfigJson) : null;
+
         const loader = getLuaScriptLoader();
         const result = await loader.executeScript(
             'checkTimeout.lua',
@@ -139,7 +148,7 @@ async function process(timeoutConfig, buildId, redis, workerId) {
 
         logger.info(`worker[${workerId}] -> Build ${buildId}: action=${decision.action}, reason=${decision.reason}`);
 
-        await handleDecision(decision, buildId, redis, workerId);
+        await handleDecision(decision, buildId, redis, workerId, buildConfig);
     } catch (err) {
         logger.error(`Error in timeout check for build ${buildId}: ${err.message}`);
         logger.error(err.stack);

package/plugins/worker/worker.js

@@ -94,7 +94,8 @@ async function invoke() {
                     redisInstance: redis,
                     buildId: job.args[0].buildId,
                     status: 'FAILURE',
-                    statusMessage: `${failure}`
+                    statusMessage: `${failure}`,
+                    buildConfig: job.args[0] // Pass buildConfig to avoid Redis lookup
                 })
                 .then(() => {
                     logger.info(