screwdriver-queue-service 5.0.2 → 6.0.0

package/plugins/worker/lib/LuaScriptLoader.js

@@ -0,0 +1,232 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const logger = require('screwdriver-logger');
+
+/**
+ * LuaScriptLoader - Manages loading and execution of Lua scripts in Redis
+ *
+ * Features:
+ * - Load Lua scripts into Redis and cache their SHAs
+ * - Auto-reload scripts if Redis loses them (NOSCRIPT error)
+ * - Execute scripts by name with proper error handling
+ * - Hash validation for cache consistency
+ */
+class LuaScriptLoader {
+    /**
+     * Constructor
+     * @param {Object} redis - Redis client instance
+     */
+    constructor(redis) {
+        this.redis = redis;
+        this.scripts = new Map(); // Map<scriptName, {sha, content, hash}>
+        this.scriptDir = path.join(__dirname, 'lua');
+    }
+
+    /**
+     * Load a Lua script and return its SHA
+     * @param {String} scriptName - Script filename (e.g., 'startBuild.lua')
+     * @return {Promise<String>} SHA of loaded script
+     */
+    async loadScript(scriptName) {
+        // Check if already loaded
+        if (this.scripts.has(scriptName)) {
+            const cached = this.scripts.get(scriptName);
+
+            logger.info(`Lua script ${scriptName} already loaded (SHA: ${cached.sha.substring(0, 8)}...)`);
+
+            return cached.sha;
+        }
+
+        const scriptPath = path.join(this.scriptDir, scriptName);
+
+        // Check if file exists
+        if (!fs.existsSync(scriptPath)) {
+            throw new Error(`Lua script not found: ${scriptPath}`);
+        }
+
+        const scriptContent = fs.readFileSync(scriptPath, 'utf8');
+
+        // Load script into Redis
+        const sha = await this.redis.script('LOAD', scriptContent);
+
+        // Cache script info
+        this.scripts.set(scriptName, {
+            sha,
+            content: scriptContent,
+            hash: this.hashScript(scriptContent)
+        });
+
+        logger.info(`Loaded Lua script: ${scriptName} (SHA: ${sha.substring(0, 8)}...)`);
+
+        return sha;
+    }
+
+    /**
+     * Get script SHA (load if not already loaded)
+     * @param {String} scriptName
+     * @return {Promise<String>}
+     */
+    async getScriptSha(scriptName) {
+        if (!this.scripts.has(scriptName)) {
+            await this.loadScript(scriptName);
+        }
+
+        return this.scripts.get(scriptName).sha;
+    }
+
+    /**
+     * Execute a Lua script by name
+     * @param {String} scriptName - Script filename
+     * @param {Array} keys - KEYS array for script (usually empty for our scripts)
+     * @param {Array} args - ARGV array for script
+     * @return {Promise<Any>} Script result
+     */
+    async executeScript(scriptName, keys = [], args = []) {
+        const sha = await this.getScriptSha(scriptName);
+
+        try {
+            // Execute script using EVALSHA (faster than EVAL)
+            const result = await this.redis.evalsha(sha, keys.length, ...keys, ...args);
+
+            return result;
+        } catch (err) {
+            // If script not found in Redis, reload and retry
+            if (err.message && err.message.includes('NOSCRIPT')) {
+                logger.warn(`Script ${scriptName} not found in Redis, reloading...`);
+
+                // Remove from cache and reload
+                this.scripts.delete(scriptName);
+                await this.loadScript(scriptName);
+
+                const newSha = this.scripts.get(scriptName).sha;
+
+                // Retry execution
+                return this.redis.evalsha(newSha, keys.length, ...keys, ...args);
+            }
+
+            // Re-throw other errors
+            throw err;
+        }
+    }
+
+    /**
+     * Load all Lua scripts in the lua directory
+     * @return {Promise<Number>} Number of scripts loaded
+     */
+    async loadAllScripts() {
+        // Find all .lua files in lua directory (not in subdirectories)
+        const files = fs.readdirSync(this.scriptDir);
+        const luaFiles = files.filter(f => f.endsWith('.lua') && !f.startsWith('.'));
+
+        let loadedCount = 0;
+
+        for (const file of luaFiles) {
+            try {
+                await this.loadScript(file);
+                loadedCount += 1;
+            } catch (err) {
+                logger.error(`Failed to load Lua script ${file}: ${err.message}`);
+            }
+        }
+
+        logger.info(`Loaded ${loadedCount} Lua script(s)`);
+
+        return loadedCount;
+    }
+
+    /**
+     * Reload a specific script (useful for hot-reload during development)
+     * @param {String} scriptName
+     * @return {Promise<String>} New SHA
+     */
+    async reloadScript(scriptName) {
+        this.scripts.delete(scriptName);
+
+        return this.loadScript(scriptName);
+    }
+
+    /**
+     * Reload all scripts
+     * @return {Promise<Number>} Number of scripts reloaded
+     */
+    async reloadAllScripts() {
+        this.scripts.clear();
+
+        return this.loadAllScripts();
+    }
+
+    /**
+     * Hash script content for cache validation
+     * @param {String} content
+     * @return {String} Hash (first 8 characters)
+     */
+    hashScript(content) {
+        return crypto.createHash('sha256').update(content).digest('hex').substring(0, 8);
+    }
+
+    /**
+     * Get info about loaded scripts
+     * @return {Array<Object>} Script info
+     */
+    getLoadedScripts() {
+        const scripts = [];
+
+        for (const [name, info] of this.scripts.entries()) {
+            scripts.push({
+                name,
+                sha: info.sha,
+                hash: info.hash,
+                size: info.content.length
+            });
+        }
+
+        return scripts;
+    }
+
+    /**
+     * Check if a script is loaded
+     * @param {String} scriptName
+     * @return {Boolean}
+     */
+    isScriptLoaded(scriptName) {
+        return this.scripts.has(scriptName);
+    }
+
+    /**
+     * Clear all cached scripts (does not remove from Redis)
+     * @return {void}
+     */
+    clearCache() {
+        this.scripts.clear();
+        logger.info('Lua script cache cleared');
+    }
+
+    /**
+     * Validate that a script exists in Redis
+     * @param {String} scriptName
+     * @return {Promise<Boolean>}
+     */
+    async validateScript(scriptName) {
+        if (!this.scripts.has(scriptName)) {
+            return false;
+        }
+
+        const { sha } = this.scripts.get(scriptName);
+
+        try {
+            // Check if script exists in Redis using SCRIPT EXISTS
+            const exists = await this.redis.script('EXISTS', sha);
+
+            return exists[0] === 1;
+        } catch (err) {
+            logger.error(`Error validating script ${scriptName}: ${err.message}`);
+
+            return false;
+        }
+    }
+}
+
+module.exports = LuaScriptLoader;

package/plugins/worker/lib/jobs.js

@@ -41,6 +41,24 @@ const executor = new ExecutorRouter({
     executor: executorPlugins,
     ecosystem
 });
+
+let luaScriptLoader;
+
+/**
+ * Get LuaScriptLoader instance (lazy loaded to avoid circular dependency)
+ * @return {LuaScriptLoader} Lua script loader instance
+ */
+function getLuaScriptLoader() {
+    if (!luaScriptLoader) {
+        // eslint-disable-next-line global-require
+        const worker = require('../worker');
+
+        luaScriptLoader = worker.luaScriptLoader;
+    }
+
+    return luaScriptLoader;
+}
+
 const retryOptions = {
     retryLimit: RETRY_LIMIT,
     retryDelay: RETRY_DELAY
@@ -256,47 +274,77 @@ async function start(buildConfig) {
 
 /**
  * Call executor.stop with the buildConfig
+ *
+ * Uses atomic Lua script for cleanup to prevent partial failures
+ *
  * @method stop
  * @param  {Object}    buildConfig               Configuration object
  * @param  {String}    buildConfig.buildId       Unique ID for a build
  * @param  {String}    buildConfig.jobId         Job that this build belongs to
- * @param  {String}    buildConfig.blockedBy     Jobs that are blocking this job
  * @param  {String}    buildConfig.started       Whether job has started
- * @param  {String}    buildConfig.jobName    Job name
  * @return {Promise}
  */
 async function stop(buildConfig) {
     const started = hoek.reach(buildConfig, 'started', { default: true }); // default value for backward compatibility
-    const { buildId, jobId, jobName } = buildConfig;
-    let stopConfig = { buildId, jobId, jobName };
-    const runningKey = `${runningJobsPrefix}${jobId}`;
+    const { buildId, jobId } = buildConfig;
 
     try {
-        const fullBuildConfig = await redis.hget(`${queuePrefix}buildConfigs`, buildId);
-        const parsedConfig = JSON.parse(fullBuildConfig);
-
-        if (parsedConfig) {
-            stopConfig = {
-                buildId,
-                ...parsedConfig
-            };
+        let stopConfig = buildConfig;
+
+        try {
+            const fullBuildConfig = await redis.hget(`${queuePrefix}buildConfigs`, buildId);
+
+            if (fullBuildConfig) {
+                stopConfig = JSON.parse(fullBuildConfig);
+            }
+        } catch (err) {
+            logger.error(`[Stop Build] Failed to get config for build ${buildId}: ${err.message}`);
         }
-    } catch (err) {
-        logger.error(`[Stop Build] failed to get config for build ${buildId}: ${err.message}`);
-    }
 
-    await redis.hdel(`${queuePrefix}buildConfigs`, buildId);
-    // If this is a running job
-    const runningBuildId = await redis.get(runningKey);
+        const loader = getLuaScriptLoader();
+        const result = await loader.executeScript(
+            'stopBuild.lua',
+            [],
+            [String(buildId), String(jobId), queuePrefix, runningJobsPrefix, waitingJobsPrefix]
+        );
 
-    if (parseInt(runningBuildId, 10) === buildId) {
-        await redis.del(runningKey);
-    }
-    // If this is a waiting job
-    await redis.lrem(`${waitingJobsPrefix}${jobId}`, 0, buildId);
+        const cleanupResult = JSON.parse(result);
+
+        logger.info(
+            '[Stop Build] Atomic cleanup for build %s: action=%s, keysDeleted=%j',
+            buildId,
+            cleanupResult.action,
+            cleanupResult.keysDeleted
+        );
+
+        if (cleanupResult.keysDeleted.runningKey) {
+            logger.info('[Stop Build] Deleted running key for job %s, build %s', jobId, buildId);
+        } else if (cleanupResult.currentRunningBuildId) {
+            logger.info(
+                '[Stop Build] Running key for job %s is %s, not %s (skip delete)',
+                jobId,
+                cleanupResult.currentRunningBuildId,
+                buildId
+            );
+        }
 
-    if (started) {
-        await schedule('stop', stopConfig);
+        if (cleanupResult.keysDeleted.lastRunningKey) {
+            logger.info('[Stop Build] Deleted last running key for job %s, build %s', jobId, buildId);
+        }
+
+        if (cleanupResult.keysDeleted.waitingKey) {
+            logger.info('[Stop Build] Removed build %s from waiting queue for job %s', buildId, jobId);
+        }
+
+        if (started) {
+            await schedule('stop', stopConfig);
+            logger.info('[Stop Build] Executor stop called for job %s, build %s', jobId, buildId);
+        } else {
+            logger.info('[Stop Build] Build %s not started, skipping executor stop', buildId);
+        }
+    } catch (err) {
+        logger.error('[Stop Build] Error in stop for build %s: %s', buildId, err.message);
+        throw err;
     }
 
     return null;

package/plugins/worker/lib/lua/checkTimeout.lua

@@ -0,0 +1,166 @@
+--[[
+    checkTimeout.lua - Atomic timeout check
+
+    This script atomically checks if a build has timed out and takes appropriate action.
+
+    ARGV[1]  = buildId (string)
+    ARGV[2]  = jobId (string)
+    ARGV[3]  = startTime (number, milliseconds timestamp)
+    ARGV[4]  = timeoutMinutes (number)
+    ARGV[5]  = currentTime (number, milliseconds timestamp)
+    ARGV[6]  = queuePrefix (string)
+    ARGV[7]  = runningJobsPrefix (string)
+    ARGV[8]  = waitingJobsPrefix (string)
+
+    Returns: JSON string with decision
+    {
+        action: "TIMEOUT" | "CLEANUP" | "SKIP",
+        reason: string,
+        buildId: string,
+        data: {...}
+    }
+]]
+
+local buildId = ARGV[1]
+local jobId = ARGV[2]
+local startTime = tonumber(ARGV[3])
+local timeoutMinutes = tonumber(ARGV[4])
+local currentTime = tonumber(ARGV[5])
+local queuePrefix = ARGV[6]
+local runningJobsPrefix = ARGV[7]
+local waitingJobsPrefix = ARGV[8]
+
+local buildConfigKey = queuePrefix .. "buildConfigs"
+local timeoutConfigKey = queuePrefix .. "timeoutConfigs"
+local runningKey = runningJobsPrefix .. jobId
+local lastRunningKey = "last_" .. runningJobsPrefix .. jobId
+local waitingKey = waitingJobsPrefix .. jobId
+local deleteKey = "deleted_" .. jobId .. "_" .. buildId
+
+local buildConfig = redis.call("HGET", buildConfigKey, buildId)
+local currentRunningBuildId = redis.call("GET", runningKey)
+local buildConfigExists = (buildConfig ~= false)
+
+-- Helper: Check if timed out
+local function hasTimedOut()
+    if not startTime then
+        return false, 0, timeoutMinutes, "NO_START_TIME"
+    end
+
+    local bufferMinutes = 1
+    local elapsedMs = currentTime - startTime
+    local elapsedMinutes = math.floor(elapsedMs / 60000 + 0.5)
+    local effectiveTimeout = timeoutMinutes + bufferMinutes
+
+    local timedOut = elapsedMinutes > effectiveTimeout
+
+    return timedOut, elapsedMinutes, effectiveTimeout,
+           (timedOut and "TIMEOUT_EXCEEDED" or "WITHIN_TIMEOUT")
+end
+
+-- Helper: Check if eligible for timeout
+local function isEligibleForTimeout()
+    local buildIdNum = tonumber(buildId)
+
+    -- Build config deleted = already completed
+    if not buildConfigExists then
+        return false, "BUILD_COMPLETED", true
+    end
+
+    -- Different build running = not running anymore
+    if currentRunningBuildId then
+        local runningId = tonumber(currentRunningBuildId)
+        if runningId ~= buildIdNum then
+            return false, "NOT_RUNNING", true
+        end
+
+        -- Running build matches = eligible
+        if runningId == buildIdNum then
+            return true, "ELIGIBLE", false
+        end
+    end
+
+    -- No running build = not running
+    return false, "NO_RUNNING_BUILD", true
+end
+
+-- check timeout status
+local timedOut, elapsedMinutes, effectiveTimeout, timeoutReason = hasTimedOut()
+local eligible, eligibleReason, shouldCleanup = isEligibleForTimeout()
+
+-- Determine action
+local action, reason, actionData
+
+if not eligible then
+    if shouldCleanup then
+        action = "CLEANUP"
+        reason = eligibleReason
+        actionData = {shouldCleanup = true}
+    else
+        action = "SKIP"
+        reason = eligibleReason
+    end
+elseif timedOut then
+    action = "TIMEOUT"
+    reason = "BUILD_TIMEOUT"
+    actionData = {
+        elapsedMinutes = elapsedMinutes,
+        timeoutMinutes = effectiveTimeout
+    }
+else
+    action = "SKIP"
+    reason = "WITHIN_TIMEOUT"
+    actionData = {
+        elapsedMinutes = elapsedMinutes,
+        timeoutMinutes = effectiveTimeout
+    }
+end
+
+-- update redis state based on action
+if action == "CLEANUP" then
+    -- Clean up stale timeout config
+    redis.call("HDEL", timeoutConfigKey, buildId)
+
+    return cjson.encode({
+        action = "CLEANUP",
+        reason = reason,
+        buildId = buildId
+    })
+
+elseif action == "TIMEOUT" then
+    -- Build has timed out - clean up all keys
+
+    -- Remove build config
+    redis.call("HDEL", buildConfigKey, buildId)
+
+    -- Expire running keys immediately
+    redis.call("EXPIRE", runningKey, 0)
+    redis.call("EXPIRE", lastRunningKey, 0)
+
+    -- Remove from waiting queue (if present)
+    redis.call("LREM", waitingKey, 0, buildId)
+
+    -- Delete deleteKey
+    redis.call("DEL", deleteKey)
+
+    -- Remove timeout config
+    redis.call("HDEL", timeoutConfigKey, buildId)
+
+    return cjson.encode({
+        action = "TIMEOUT",
+        reason = reason,
+        buildId = buildId,
+        elapsedMinutes = actionData.elapsedMinutes,
+        timeoutMinutes = actionData.timeoutMinutes
+    })
+
+else  -- SKIP
+    -- No action needed
+    return cjson.encode({
+        action = "SKIP",
+        reason = reason,
+        buildId = buildId,
+        elapsedMinutes = actionData and actionData.elapsedMinutes or nil,
+        timeoutMinutes = actionData and actionData.timeoutMinutes or nil
+    })
+end

package/plugins/worker/lib/lua/lib/CollapseDecider.lua

@@ -0,0 +1,155 @@
+--[[
+    This module determines whether a build should be collapsed
+    (discarded in favor of a newer build of the same job).
+]]
+
+local CollapseDecider = {}
+
+--[[
+    Determine if a build should be collapsed
+    @param buildId - Current build ID
+    @param waitingBuilds - Array of waiting build IDs for this job
+    @param lastRunningBuildId - Last build that ran for this job
+    @param collapseEnabled - Whether collapse feature is enabled
+    @return {shouldCollapse, reason, newestBuild, collapseTarget}
+]]
+function CollapseDecider.shouldCollapse(buildId, waitingBuilds, lastRunningBuildId, collapseEnabled)
+    if not collapseEnabled then
+        return {
+            shouldCollapse = false,
+            reason = "COLLAPSE_DISABLED"
+        }
+    end
+
+    if not waitingBuilds or #waitingBuilds == 0 then
+        return {
+            shouldCollapse = false,
+            reason = "NO_WAITING_BUILDS"
+        }
+    end
+
+    -- Find oldest and newest waiting builds
+    local oldestBuild = tonumber(waitingBuilds[1])
+    local newestBuild = tonumber(waitingBuilds[#waitingBuilds])
+    local currentBuild = tonumber(buildId)
+
+    -- Check if current build is older than last running build
+    -- Note: cjson.null is used for JSON null values (not Lua nil)
+    if lastRunningBuildId and lastRunningBuildId ~= cjson.null then
+        local lastRunning = tonumber(lastRunningBuildId)
+        if lastRunning and currentBuild < lastRunning then
+            return {
+                shouldCollapse = true,
+                reason = "OLDER_THAN_LAST_RUNNING",
+                newestBuild = newestBuild,
+                lastRunningBuildId = lastRunning
+            }
+        end
+    end
+
+    -- Check if current build is not the newest waiting build
+    if newestBuild and currentBuild < newestBuild then
+        return {
+            shouldCollapse = true,
+            reason = "NEWER_BUILD_EXISTS",
+            newestBuild = newestBuild
+        }
+    end
+
+    -- Current build is the newest
+    return {
+        shouldCollapse = false,
+        reason = "IS_NEWEST_BUILD"
+    }
+end
+
+--[[
+    Get list of builds that should be collapsed
+    @param waitingBuilds - Array of all waiting build IDs
+    @param newestBuildId - The newest build ID (don't collapse this one)
+    @return Array of build IDs to collapse
+]]
+function CollapseDecider.getBuildsToCollapse(waitingBuilds, newestBuildId)
+    local toCollapse = {}
+    local newest = tonumber(newestBuildId)
+
+    for _, buildId in ipairs(waitingBuilds) do
+        local bid = tonumber(buildId)
+        if bid < newest then
+            table.insert(toCollapse, buildId)
+        end
+    end
+
+    -- Mark as array for cjson (even if empty)
+    setmetatable(toCollapse, cjson.array_mt)
+
+    return toCollapse
+end
+
+--[[
+    Find newest build in a list
+    @param buildIds - Array of build IDs
+    @return Newest build ID (as number) or nil
+]]
+function CollapseDecider.findNewestBuild(buildIds)
+    if not buildIds or #buildIds == 0 then
+        return nil
+    end
+
+    local newest = tonumber(buildIds[1])
+
+    for i = 2, #buildIds do
+        local current = tonumber(buildIds[i])
+        if current > newest then
+            newest = current
+        end
+    end
+
+    return newest
+end
+
+--[[
+    Find oldest build in a list
+    @param buildIds - Array of build IDs
+    @return Oldest build ID (as number) or nil
+]]
+function CollapseDecider.findOldestBuild(buildIds)
+    if not buildIds or #buildIds == 0 then
+        return nil
+    end
+
+    local oldest = tonumber(buildIds[1])
+
+    for i = 2, #buildIds do
+        local current = tonumber(buildIds[i])
+        if current < oldest then
+            oldest = current
+        end
+    end
+
+    return oldest
+end
+
+--[[
+    Check if a build is in the waiting queue
+    @param buildId - Build ID to check
+    @param waitingBuilds - Array of waiting build IDs
+    @return boolean
+]]
+function CollapseDecider.isBuildWaiting(buildId, waitingBuilds)
+    if not waitingBuilds or #waitingBuilds == 0 then
+        return false
+    end
+
+    local targetId = tostring(buildId)
+
+    for _, waitingId in ipairs(waitingBuilds) do
+        if tostring(waitingId) == targetId then
+            return true
+        end
+    end
+
+    return false
+end
+
+return CollapseDecider