screwdriver-queue-service 5.0.3 → 6.0.1

package/plugins/worker/lib/lua/lib/DependencyResolver.lua

@@ -0,0 +1,109 @@
+--[[
+    This module provides functions to resolve build dependencies
+    and determine if a build is blocked by other builds.
+]]
+
+local DependencyResolver = {}
+
+--[[
+    Parse blockedBy value (could be array, single value, or nil)
+    @param blockedBy - Array of job IDs or single job ID
+    @return {blocked, dependencies}
+]]
+function DependencyResolver.parseBlockedBy(blockedBy)
+    -- Check for nil or cjson.null (JavaScript null becomes cjson.null)
+    if not blockedBy or blockedBy == cjson.null then
+        return {blocked = false, dependencies = {}}
+    end
+
+    -- Handle array
+    if type(blockedBy) == "table" then
+        if #blockedBy == 0 then
+            return {blocked = false, dependencies = {}}
+        end
+        return {blocked = true, dependencies = blockedBy}
+    end
+
+    -- Handle single value
+    return {blocked = true, dependencies = {blockedBy}}
+end
+
+--[[
+    Check if build is blocked by running dependencies
+    @param dependencies - Array of job IDs that block this build
+    @param runningBuilds - Array of currently running build IDs
+    @return {blocked, blockedBy, reason}
+]]
+function DependencyResolver.isBlockedByDependencies(dependencies, runningBuilds)
+    local blockedByBuilds = {}
+
+    -- Check if any dependency is in running builds
+    for _, dep in ipairs(dependencies) do
+        for _, running in ipairs(runningBuilds) do
+            -- Convert both to strings for comparison
+            if tostring(dep) == tostring(running) then
+                table.insert(blockedByBuilds, dep)
+                break
+            end
+        end
+    end
+
+    local isBlocked = #blockedByBuilds > 0
+
+    return {
+        blocked = isBlocked,
+        blockedBy = blockedByBuilds,
+        reason = isBlocked and "BLOCKED_BY_DEPENDENCIES" or "NOT_BLOCKED"
+    }
+end
+
+--[[
+    Check if build is blocked by same job (another build of same job running)
+    @param jobId - Current job ID
+    @param runningBuildId - Currently running build ID for this job (or nil)
+    @param buildId - Current build ID
+    @param blockedBySelf - Whether blocking by same job is enabled
+    @return {blocked, reason}
+]]
+function DependencyResolver.isBlockedBySameJob(jobId, runningBuildId, buildId, blockedBySelf)
+    if not blockedBySelf then
+        return {blocked = false, reason = "BLOCKED_BY_SELF_DISABLED"}
+    end
+
+    -- Check for nil or cjson.null (JavaScript null becomes cjson.null)
+    if not runningBuildId or runningBuildId == cjson.null then
+        return {blocked = false, reason = "NO_RUNNING_BUILD"}
+    end
+
+    -- Convert to numbers for comparison
+    local runningId = tonumber(runningBuildId)
+    local currentId = tonumber(buildId)
+
+    if runningId == currentId then
+        return {blocked = false, reason = "SAME_BUILD_RUNNING"}
+    end
+
+    return {
+        blocked = true,
+        reason = "BLOCKED_BY_SAME_JOB",
+        runningBuildId = runningId
+    }
+end
+
+--[[
+    Build blocked message for logging/status
+    @param blockedBy - Array of blocking build IDs
+    @param reason - Reason for blocking
+    @return string message
+]]
+function DependencyResolver.buildBlockedMessage(blockedBy, reason)
+    -- Check for nil, cjson.null, or empty array
+    if not blockedBy or blockedBy == cjson.null or #blockedBy == 0 then
+        return "Build is blocked: " .. reason
+    end
+
+    local buildIds = table.concat(blockedBy, ", ")
+    return "Build is blocked by: " .. buildIds .. " (" .. reason .. ")"
+end
+
+return DependencyResolver

package/plugins/worker/lib/lua/lib/StateValidator.lua

@@ -0,0 +1,179 @@
+--[[
+    StateValidator
+]]
+
+local StateValidator = {}
+
+-- Define state constants
+StateValidator.STATES = {
+    QUEUED = "QUEUED",
+    BLOCKED = "BLOCKED",
+    READY = "READY",
+    RUNNING = "RUNNING",
+    SUCCESS = "SUCCESS",
+    FAILURE = "FAILURE",
+    ABORTED = "ABORTED",
+    COLLAPSED = "COLLAPSED"
+}
+
+-- Define action constants
+StateValidator.ACTIONS = {
+    START = "START",
+    BLOCK = "BLOCK",
+    COLLAPSE = "COLLAPSE",
+    ABORT = "ABORT",
+    SKIP = "SKIP"
+}
+
+--[[
+    Compute action based on current conditions
+    Priority: ABORT > COLLAPSE > BLOCK > START
+
+    @param isAborted - Whether build was aborted
+    @param isBlocked - Whether build is blocked by dependencies
+    @param shouldCollapse - Whether build should be collapsed
+    @param isRetry - Whether this is a retry attempt
+    @return {action, reason, priority, nextState}
+]]
+function StateValidator.computeAction(isAborted, isBlocked, shouldCollapse, isRetry)
+    -- Priority 1: ABORT (highest priority)
+    if isAborted then
+        return {
+            action = StateValidator.ACTIONS.ABORT,
+            reason = "BUILD_ABORTED",
+            priority = 1,
+            nextState = StateValidator.STATES.ABORTED
+        }
+    end
+
+    -- Priority 2: COLLAPSE
+    if shouldCollapse then
+        return {
+            action = StateValidator.ACTIONS.COLLAPSE,
+            reason = "NEWER_BUILD_EXISTS",
+            priority = 2,
+            nextState = StateValidator.STATES.COLLAPSED
+        }
+    end
+
+    -- Priority 3: BLOCK
+    if isBlocked then
+        return {
+            action = StateValidator.ACTIONS.BLOCK,
+            reason = "BLOCKED_BY_DEPENDENCIES",
+            priority = 3,
+            nextState = StateValidator.STATES.BLOCKED
+        }
+    end
+
+    -- Priority 4: START (lowest priority, default action)
+    return {
+        action = StateValidator.ACTIONS.START,
+        reason = "READY",
+        priority = 4,
+        nextState = StateValidator.STATES.RUNNING
+    }
+end
+
+--[[
+    Check if a state is terminal (no further transitions)
+    @param state - State to check
+    @return boolean
+]]
+function StateValidator.isTerminalState(state)
+    return state == StateValidator.STATES.SUCCESS or
+           state == StateValidator.STATES.FAILURE or
+           state == StateValidator.STATES.ABORTED or
+           state == StateValidator.STATES.COLLAPSED
+end
+
+--[[
+    Validate if a state transition is allowed
+    @param currentState - Current state
+    @param nextState - Desired next state
+    @return {valid, reason}
+]]
+function StateValidator.isValidTransition(currentState, nextState)
+    -- Terminal states cannot transition
+    if StateValidator.isTerminalState(currentState) then
+        return {
+            valid = false,
+            reason = "CURRENT_STATE_IS_TERMINAL"
+        }
+    end
+
+    -- Define valid transitions
+    local validTransitions = {
+        [StateValidator.STATES.QUEUED] = {
+            StateValidator.STATES.BLOCKED,
+            StateValidator.STATES.READY,
+            StateValidator.STATES.RUNNING,
+            StateValidator.STATES.ABORTED,
+            StateValidator.STATES.COLLAPSED
+        },
+        [StateValidator.STATES.BLOCKED] = {
+            StateValidator.STATES.READY,
+            StateValidator.STATES.RUNNING,
+            StateValidator.STATES.ABORTED,
+            StateValidator.STATES.COLLAPSED
+        },
+        [StateValidator.STATES.READY] = {
+            StateValidator.STATES.RUNNING,
+            StateValidator.STATES.ABORTED
+        },
+        [StateValidator.STATES.RUNNING] = {
+            StateValidator.STATES.SUCCESS,
+            StateValidator.STATES.FAILURE,
+            StateValidator.STATES.ABORTED
+        }
+    }
+
+    local allowedStates = validTransitions[currentState]
+
+    if not allowedStates then
+        return {
+            valid = false,
+            reason = "NO_TRANSITIONS_FROM_STATE"
+        }
+    end
+
+    -- Check if nextState is in allowed list
+    for _, allowedState in ipairs(allowedStates) do
+        if nextState == allowedState then
+            return {
+                valid = true,
+                reason = "VALID_TRANSITION"
+            }
+        end
+    end
+
+    return {
+        valid = false,
+        reason = "INVALID_TRANSITION"
+    }
+end
+
+--[[
+    Get the current state based on conditions
+    @param hasRunningKey - Whether running key exists
+    @param hasWaitingEntry - Whether build is in waiting queue
+    @param isAborted - Whether build was aborted
+    @return state
+]]
+function StateValidator.inferState(hasRunningKey, hasWaitingEntry, isAborted)
+    if isAborted then
+        return StateValidator.STATES.ABORTED
+    end
+
+    if hasRunningKey then
+        return StateValidator.STATES.RUNNING
+    end
+
+    if hasWaitingEntry then
+        return StateValidator.STATES.BLOCKED
+    end
+
+    return StateValidator.STATES.QUEUED
+end
+
+return StateValidator

package/plugins/worker/lib/lua/lib/TimeoutDecider.lua

@@ -0,0 +1,161 @@
+--[[
+    This module determines whether a build has timed out and what action to take.
+]]
+
+local TimeoutDecider = {}
+
+--[[
+    Parse timeout value, handling NaN cases
+    @param timeoutValue - Timeout value (string or number)
+    @param defaultTimeout - Default timeout in minutes (default: 90)
+    @return number (parsed timeout or default)
+]]
+function TimeoutDecider.parseTimeout(timeoutValue, defaultTimeout)
+    -- Handle cjson.null for defaultTimeout (JavaScript null becomes cjson.null)
+    if not defaultTimeout or defaultTimeout == cjson.null then
+        defaultTimeout = 90
+    end
+
+    -- Handle cjson.null for timeoutValue
+    if not timeoutValue or timeoutValue == cjson.null then
+        return defaultTimeout
+    end
+
+    local parsed = tonumber(timeoutValue)
+
+    if not parsed or parsed ~= parsed then  -- NaN check
+        return defaultTimeout
+    end
+
+    return parsed
+end
+
+--[[
+    Check if a build has timed out
+    @param startTime - Build start timestamp (milliseconds)
+    @param currentTime - Current timestamp (milliseconds)
+    @param timeoutMinutes - Timeout duration in minutes
+    @param bufferMinutes - Buffer time in minutes (default: 1)
+    @return {hasTimedOut, elapsedMinutes, effectiveTimeout, reason}
+]]
+function TimeoutDecider.hasTimedOut(startTime, currentTime, timeoutMinutes, bufferMinutes)
+    -- Handle cjson.null for bufferMinutes (JavaScript null becomes cjson.null)
+    if not bufferMinutes or bufferMinutes == cjson.null then
+        bufferMinutes = 1
+    end
+
+    -- Handle cjson.null for startTime
+    if not startTime or startTime == cjson.null then
+        return {
+            hasTimedOut = false,
+            reason = "NO_START_TIME",
+            elapsedMinutes = 0,
+            effectiveTimeout = timeoutMinutes + bufferMinutes
+        }
+    end
+
+    local elapsedMs = currentTime - startTime
+    local elapsedMinutes = math.floor(elapsedMs / 60000 + 0.5)  -- Round to nearest minute
+    local effectiveTimeout = timeoutMinutes + bufferMinutes
+
+    local timedOut = elapsedMinutes > effectiveTimeout
+
+    return {
+        hasTimedOut = timedOut,
+        elapsedMinutes = elapsedMinutes,
+        effectiveTimeout = effectiveTimeout,
+        reason = timedOut and "TIMEOUT_EXCEEDED" or "WITHIN_TIMEOUT"
+    }
+end
+
+--[[
+    Check if a build is eligible for timeout action
+    @param buildId - Current build ID
+    @param runningBuildId - Build ID from running key (or nil)
+    @param buildConfigExists - Whether build config still exists
+    @return {eligible, reason, shouldCleanup}
+]]
+function TimeoutDecider.isEligibleForTimeout(buildId, runningBuildId, buildConfigExists)
+    -- Parse buildId to number for comparison (may be string from Redis keys)
+    local buildIdNum = tonumber(buildId)
+
+    -- Build config deleted = build already completed
+    if not buildConfigExists then
+        return {
+            eligible = false,
+            reason = "BUILD_COMPLETED",
+            shouldCleanup = true
+        }
+    end
+
+    -- Check for nil or cjson.null (JavaScript null becomes cjson.null)
+    if not runningBuildId or runningBuildId == cjson.null then
+        return {
+            eligible = false,
+            reason = "NO_RUNNING_BUILD",
+            shouldCleanup = true
+        }
+    end
+
+    -- Different build running = this build not running anymore
+    if tonumber(runningBuildId) ~= buildIdNum then
+        return {
+            eligible = false,
+            reason = "NOT_RUNNING",
+            shouldCleanup = true
+        }
+    end
+
+    -- Running build ID matches = eligible for timeout
+    return {
+        eligible = true,
+        reason = "ELIGIBLE",
+        shouldCleanup = false
+    }
+end
+
+--[[
+    Compute action to take based on timeout check
+    @param timeoutCheck - Result from hasTimedOut()
+    @param eligibilityCheck - Result from isEligibleForTimeout()
+    @param timeoutMinutes - Timeout duration for logging
+    @return {action, reason, timeoutMinutes}
+]]
+function TimeoutDecider.computeAction(timeoutCheck, eligibilityCheck, timeoutMinutes)
+    -- If not eligible, determine action based on cleanup need
+    if not eligibilityCheck.eligible then
+        if eligibilityCheck.shouldCleanup then
+            return {
+                action = "CLEANUP",
+                reason = eligibilityCheck.reason,
+                shouldCleanup = true
+            }
+        else
+            return {
+                action = "SKIP",
+                reason = eligibilityCheck.reason,
+                shouldCleanup = false
+            }
+        end
+    end
+
+    -- Eligible - check if timed out
+    if timeoutCheck.hasTimedOut then
+        return {
+            action = "TIMEOUT",
+            reason = "BUILD_TIMEOUT",
+            timeoutMinutes = timeoutCheck.effectiveTimeout,
+            elapsedMinutes = timeoutCheck.elapsedMinutes
+        }
+    end
+
+    -- Eligible but not timed out yet
+    return {
+        action = "SKIP",
+        reason = "WITHIN_TIMEOUT",
+        elapsedMinutes = timeoutCheck.elapsedMinutes,
+        timeoutMinutes = timeoutCheck.effectiveTimeout
+    }
+end
+
+return TimeoutDecider

package/plugins/worker/lib/lua/startBuild.lua

@@ -0,0 +1,217 @@
+--[[
+    startBuild.lua - Atomic build start check
+
+    This script atomically checks if a build can start and updates Redis state accordingly.
+
+    ARGV[1]  = buildId (string)
+    ARGV[2]  = jobId (string)
+    ARGV[3]  = blockedBy (JSON array string, e.g., "[123, 456]" or "[]")
+    ARGV[4]  = collapseEnabled (string: "true" or "false")
+    ARGV[5]  = blockedBySelf (string: "true" or "false")
+    ARGV[6]  = queuePrefix (string, e.g., "buildConfig_")
+    ARGV[7]  = runningJobsPrefix (string, e.g., "running_job_")
+    ARGV[8]  = waitingJobsPrefix (string, e.g., "waiting_job_")
+    ARGV[9]  = blockTimeout (number, in minutes, e.g., 90)
+
+    Returns: JSON string with decision
+    {
+        action: "START" | "BLOCK" | "COLLAPSE" | "ABORT",
+        reason: string,
+        buildId: string,
+        data: {...}
+    }
+]]
+
+local buildId = ARGV[1]
+local jobId = ARGV[2]
+local blockedByJson = ARGV[3]
+local collapseEnabled = ARGV[4] == "true"
+local blockedBySelf = ARGV[5] == "true"
+local queuePrefix = ARGV[6]
+local runningJobsPrefix = ARGV[7]
+local waitingJobsPrefix = ARGV[8]
+local blockTimeout = tonumber(ARGV[9])
+
+local buildConfigKey = queuePrefix .. "buildConfigs"
+local runningKey = runningJobsPrefix .. jobId
+local lastRunningKey = "last_" .. runningJobsPrefix .. jobId
+local waitingKey = waitingJobsPrefix .. jobId
+local deleteKey = "deleted_" .. jobId .. "_" .. buildId
+
+local buildConfig = redis.call("HGET", buildConfigKey, buildId)
+local runningBuildId = redis.call("GET", runningKey)
+local waitingBuilds = redis.call("LRANGE", waitingKey, 0, -1)
+local lastRunningBuildId = redis.call("GET", lastRunningKey)
+local deleteKeyExists = redis.call("EXISTS", deleteKey)
+
+-- Check if build was aborted (deleteKey exists)
+local isAborted = (deleteKeyExists == 1)
+
+-- Parse blockedBy dependencies
+local blockedBy = cjson.decode(blockedByJson)
+local dependencies = {}
+if type(blockedBy) == "table" and #blockedBy > 0 then
+    dependencies = blockedBy
+elseif blockedBy then
+    dependencies = {blockedBy}
+end
+
+-- Check which dependencies are currently running
+local runningBuilds = {}
+for _, depJobId in ipairs(dependencies) do
+    local depRunningBuild = redis.call("GET", runningJobsPrefix .. tostring(depJobId))
+    if depRunningBuild then
+        table.insert(runningBuilds, depRunningBuild)
+    end
+end
+
+-- Helper: Check if blocked by dependencies
+local function isBlockedByDependencies(deps, running)
+    local blockedByBuilds = {}
+    for _, dep in ipairs(deps) do
+        for _, runningId in ipairs(running) do
+            if tostring(dep) == tostring(runningId) then
+                table.insert(blockedByBuilds, dep)
+                break
+            end
+        end
+    end
+    return #blockedByBuilds > 0, blockedByBuilds
+end
+
+-- Helper: Check if blocked by same job
+local function isBlockedBySameJob()
+    if not blockedBySelf then
+        return false
+    end
+    if not runningBuildId then
+        return false
+    end
+    local runningId = tonumber(runningBuildId)
+    local currentId = tonumber(buildId)
+    return runningId ~= currentId
+end
+
+-- Helper: Check if should collapse
+local function shouldCollapse()
+    if not collapseEnabled then
+        return false, nil, "COLLAPSE_DISABLED"
+    end
+
+    if not waitingBuilds or #waitingBuilds == 0 then
+        return false, nil, "NO_WAITING_BUILDS"
+    end
+
+    local newestBuild = tonumber(waitingBuilds[#waitingBuilds])
+    local currentBuild = tonumber(buildId)
+
+    -- Check if older than last running build
+    if lastRunningBuildId then
+        local lastRunning = tonumber(lastRunningBuildId)
+        if currentBuild < lastRunning then
+            return true, newestBuild, "OLDER_THAN_LAST_RUNNING"
+        end
+    end
+
+    -- Check if not the newest waiting build
+    if currentBuild < newestBuild then
+        return true, newestBuild, "NEWER_BUILD_EXISTS"
+    end
+
+    return false, nil, "IS_NEWEST_BUILD"
+end
+
+local isBlockedByDeps, blockedByBuilds = isBlockedByDependencies(dependencies, runningBuilds)
+local isBlockedBySelf = isBlockedBySameJob()
+local shouldCollapseFlag, newestBuild, collapseReason = shouldCollapse()
+
+-- Determine final action (Priority: ABORT > COLLAPSE > BLOCK > START)
+local action, reason, actionData
+
+if isAborted then
+    action = "ABORT"
+    reason = "BUILD_ABORTED"
+
+elseif shouldCollapseFlag then
+    action = "COLLAPSE"
+    reason = collapseReason
+    actionData = {newestBuild = newestBuild}
+
+elseif isBlockedByDeps then
+    action = "BLOCK"
+    reason = "BLOCKED_BY_DEPENDENCIES"
+    actionData = {blockedBy = blockedByBuilds}
+
+elseif isBlockedBySelf then
+    action = "BLOCK"
+    reason = "BLOCKED_BY_SAME_JOB"
+    actionData = {runningBuildId = runningBuildId}
+
+else
+    action = "START"
+    reason = "READY"
+end
+
+-- Update Redis state based on decision
+if action == "ABORT" then
+    -- Build was aborted, no state changes needed
+    return cjson.encode({
+        action = "ABORT",
+        reason = reason,
+        buildId = buildId
+    })
+
+elseif action == "COLLAPSE" then
+    -- Collapse this build - remove from configs and waiting queue
+    redis.call("HDEL", buildConfigKey, buildId)
+    redis.call("LREM", waitingKey, 0, buildId)
+
+    return cjson.encode({
+        action = "COLLAPSE",
+        reason = reason,
+        buildId = buildId,
+        newestBuild = actionData.newestBuild
+    })
+
+elseif action == "BLOCK" then
+    -- Build is blocked - add to waiting queue if not already there
+    local alreadyWaiting = false
+    for _, waitingBuildId in ipairs(waitingBuilds) do
+        if tostring(waitingBuildId) == tostring(buildId) then
+            alreadyWaiting = true
+            break
+        end
+    end
+
+    if not alreadyWaiting then
+        redis.call("RPUSH", waitingKey, buildId)
+    end
+
+    -- Set deleteKey with timeout (prevents zombie builds)
+    redis.call("SET", deleteKey, buildId, "EX", blockTimeout * 60)
+
+    return cjson.encode({
+        action = "BLOCK",
+        reason = reason,
+        buildId = buildId,
+        blockedBy = actionData and actionData.blockedBy or nil,
+        runningBuildId = actionData and actionData.runningBuildId or nil
+    })
+
+else  -- START
+    -- Build can start - update running keys and clean up
+    redis.call("SET", runningKey, buildId, "EX", blockTimeout * 60)
+    redis.call("SET", lastRunningKey, buildId, "EX", blockTimeout * 60 * 2)
+
+    -- Remove from waiting queue if present
+    redis.call("LREM", waitingKey, 0, buildId)
+
+    -- Delete the deleteKey (no longer needed)
+    redis.call("DEL", deleteKey)
+
+    return cjson.encode({
+        action = "START",
+        reason = reason,
+        buildId = buildId
+    })
+end