screwdriver-api 8.0.66 → 8.0.68

package/config/custom-environment-variables.yaml

@@ -361,32 +361,32 @@ scms:
   #         # SCM clone type (https or ssh)
   #         cloneType: SCM_GITLAB_RO_CLONE_TYPE
 webhooks:
-  # Object keyed by scm name with value webhook settings.
-  # Value of webhook settings is an object with the following properties:
-  # Example:
-  # {
-  #  "github": {
-  #   # Obtains the SCM token for a given user. If a user does not have a valid SCM token registered with Screwdriver, it will use this user's token instead.
-  #   "username": "sd-buildbot",
-  #   # Ignore commits made by these users
-  #   "ignoreCommitsBy": [],
-  #   # Restrict PR: all, none, branch, or fork
-  #   "restrictPR": "none",
-  #   # Chain PR: true or false
-  #   "chainPR": false,
-  #   # Upper limit on incoming uploads to builds
-  #   "maxBytes": 1048576 #1MB
-  #   },
-  #  "github.example.com": {
-  #     "username": "someuser",
-  #     "ignoreCommitsBy": ["someuser", "anotheruser"],
-  #     "restrictPR": "branch",
-  #     "chainPR": true,
-  #     "maxBytes": 2097152 #2MB
-  #   }
-  # }
-  __name: WEBHOOK_SETTINGS
-  __format: json
+  scms:
+    # Object keyed by scm name with value webhook settings.
+    # Value of webhook settings is an object with the following properties:
+    # Example:
+    # {
+    #  "github": {
+    #   # Obtains the SCM token for a given user. If a user does not have a valid SCM token registered with Screwdriver, it will use this user's token instead.
+    #   "username": "sd-buildbot",
+    #   # Ignore commits made by these users
+    #   "ignoreCommitsBy": [],
+    #   # Restrict PR: all, none, branch, or fork
+    #   "restrictPR": "none",
+    #   # Chain PR: true or false
+    #   "chainPR": false
+    #   },
+    #  "github.example.com": {
+    #     "username": "someuser",
+    #     "ignoreCommitsBy": ["someuser", "anotheruser"],
+    #     "restrictPR": "branch",
+    #     "chainPR": true
+    #   }
+    # }
+    __name: WEBHOOK_SETTINGS
+    __format: json
+  maxBytes: WEBHOOK_MAX_BYTES
+
 
 bookends:
   # Object keyed by cluster name with value setup/teardown bookend.

package/config/default.yaml

@@ -277,42 +277,40 @@ scms: {}
 #         accessToken: headlesstoken
 #         # SCM clone type (https or ssh)
 #         cloneType: https
-webhooks: 
-  github: 
-    # Obtains the SCM token for a given user. If a user does not have a valid SCM token registered with Screwdriver, it will use this user's token instead.
-    username: sd-buildbot
-    # Ignore commits made by these users
-    ignoreCommitsBy: []
-    # Restrict PR: all, none, branch, or fork
-    restrictPR: none
-    # Chain PR: true or false
-    chainPR: false
-    # Upper limit on incoming uploads to builds
-    maxBytes: 1048576 #1MB
-  # Object keyed by scm name with value webhook settings.
-  # Value of webhook settings is an object with the following properties:
-  # Example:
-  # {
-  #  "github:github.com": {
-  #   # Obtains the SCM token for a given user. If a user does not have a valid SCM token registered with Screwdriver, it will use this user's token instead.
-  #   "username": "sd-buildbot",
-  #   # Ignore commits made by these users
-  #   "ignoreCommitsBy": [],
-  #   # Restrict PR: all, none, branch, or fork
-  #   "restrictPR": "none",
-  #   # Chain PR: true or false
-  #   "chainPR": false,
-  #   # Upper limit on incoming uploads to builds
-  #   "maxBytes": 1048576 #1MB
-  #   },
-  #  "github.example.com": {
-  #     "username": "someuser",
-  #     "ignoreCommitsBy": ["someuser", "anotheruser"],
-  #     "restrictPR": "branch",
-  #     "chainPR": true,
-  #     "maxBytes": 2097152 #2MB
-  #   }
-  # }
+webhooks:
+  scms:
+    github:
+      # Obtains the SCM token for a given user. If a user does not have a valid SCM token registered with Screwdriver, it will use this user's token instead.
+      username: sd-buildbot
+      # Ignore commits made by these users
+      ignoreCommitsBy: []
+      # Restrict PR: all, none, branch, or fork
+      restrictPR: none
+      # Chain PR: true or false
+      chainPR: false
+    # Object keyed by scm name with value webhook settings.
+    # Value of webhook settings is an object with the following properties:
+    # Example:
+    # {
+    #  "github:github.com": {
+    #   # Obtains the SCM token for a given user. If a user does not have a valid SCM token registered with Screwdriver, it will use this user's token instead.
+    #   "username": "sd-buildbot",
+    #   # Ignore commits made by these users
+    #   "ignoreCommitsBy": [],
+    #   # Restrict PR: all, none, branch, or fork
+    #   "restrictPR": "none",
+    #   # Chain PR: true or false
+    #   "chainPR": false
+    #   },
+    #  "github.example.com": {
+    #     "username": "someuser",
+    #     "ignoreCommitsBy": ["someuser", "anotheruser"],
+    #     "restrictPR": "branch",
+    #     "chainPR": true
+    #   }
+    # }
+  # Upper limit on incoming uploads to builds
+  maxBytes: 1048576 # 1MB
 coverage:
   default: "false"
   plugin: sonar

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-api",
-  "version": "8.0.66",
+  "version": "8.0.68",
   "description": "API server for the Screwdriver.cd service",
   "main": "index.js",
   "scripts": {

package/plugins/builds/helper/updateBuild.js

@@ -7,7 +7,7 @@ const { PR_JOB_NAME, PR_STAGE_NAME, STAGE_TEARDOWN_PATTERN } = require('screwdri
 const { getFullStageJobName } = require('../../helper');
 const { updateVirtualBuildSuccess } = require('../triggers/helpers');
 const TERMINAL_STATUSES = ['FAILURE', 'ABORTED', 'UNSTABLE', 'COLLAPSED'];
-const FINISHED_STATUSES = ['FAILURE', 'SUCCESS', 'ABORTED', 'UNSTABLE', 'COLLAPSED'];
+const FINISHED_STATUSES = ['SUCCESS', ...TERMINAL_STATUSES];
 
 /**
  * @typedef {import('screwdriver-models/lib/build')} Build
@@ -140,6 +140,35 @@ async function getStage({ stageFactory, workflowGraph, jobName, pipelineId }) {
     return Promise.resolve(stage);
 }
 
+/**
+ * Get stage build for current event
+ * @param {StageFactory} stageFactory Stage factory
+ * @param {StageBuildFactory} stageBuildFactory stage build factory
+ * @param {Object} workflowGraph workflow graph
+ * @param {String} jobName job name
+ * @param {Number} pipelineId pipeline id
+ * @param {Number} eventId event id
+ *
+ * @returns {StageBuild} stage build for current event
+ */
+async function getStageBuild({ stageFactory, stageBuildFactory, workflowGraph, jobName, pipelineId, eventId }) {
+    const stage = await getStage({
+        stageFactory,
+        workflowGraph,
+        jobName,
+        pipelineId
+    });
+
+    if (stage) {
+        return stageBuildFactory.get({
+            stageId: stage.id,
+            eventId
+        });
+    }
+
+    return null;
+}
+
 /**
  * Get all builds in stage
  *
@@ -241,6 +270,50 @@ function deriveEventStatusFromBuildStatuses(builds) {
     return newEventStatus;
 }
 
+/**
+ * Update stage build's status
+ * Stage build's status can transition as below
+ *   CREATED -> RUNNING -when all builds are succeeded-> SUCCESS
+ *                      -when some builds are terminated-> FAILURE, ABORTED, UNSTABLE, COLLAPSED
+ *
+ * @param {Object} param
+ * @param {StageBuild} param.stageBuild
+ * @param {String} param.newStatus
+ * @param {Job} param.job
+ * @returns {StageBuild} new stage build's status
+ */
+async function updateStageBuildStatus({ stageBuild, newStatus, job }) {
+    if (stageBuild.status === newStatus) {
+        return stageBuild;
+    }
+
+    // Not update when the stage build status is terminal
+    if (TERMINAL_STATUSES.includes(stageBuild.status)) {
+        return stageBuild;
+    }
+
+    const isStageTeardown = STAGE_TEARDOWN_PATTERN.test(job.name);
+    const intermediateStatuses = ['RUNNING', ...TERMINAL_STATUSES];
+    const teardownStatuses = ['RUNNING', ...FINISHED_STATUSES];
+
+    // It means all builds are succeeded if stage build status is not terminal when teardown build is done.
+    if (isStageTeardown && teardownStatuses.includes(newStatus)) {
+        stageBuild.status = newStatus;
+
+        return stageBuild.update();
+    }
+
+    // Stage build status can transition to terminated status in the intermediate builds.
+    // But not update to SUCCESS
+    if (!isStageTeardown && intermediateStatuses.includes(newStatus)) {
+        stageBuild.status = newStatus;
+
+        return stageBuild.update();
+    }
+
+    return stageBuild;
+}
+
 /**
  * Updates the build and trigger its downstream jobs in the workflow
  *
@@ -336,14 +409,9 @@ async function updateBuildAndTriggerDownstreamJobs(config, build, server, userna
             eventId: newEvent.id
         });
 
-        if (stageBuild.status !== newBuild.status) {
-            if (!TERMINAL_STATUSES.includes(stageBuild.status)) {
-                stageBuild.status = newBuild.status;
-                await stageBuild.update();
-            }
-        }
+        const newStageBuild = await updateStageBuildStatus({ stageBuild, newStatus: newBuild.status, job });
 
-        stageBuildHasFailure = TERMINAL_STATUSES.includes(stageBuild.status);
+        stageBuildHasFailure = TERMINAL_STATUSES.includes(newStageBuild.status);
     }
 
     // Guard against triggering non-successful or unstable builds
@@ -417,5 +485,7 @@ async function updateBuildAndTriggerDownstreamJobs(config, build, server, userna
 
 module.exports = {
     updateBuildAndTriggerDownstreamJobs,
-    deriveEventStatusFromBuildStatuses
+    deriveEventStatusFromBuildStatuses,
+    getStageBuild,
+    updateStageBuildStatus
 };

package/plugins/builds/index.js

@@ -44,6 +44,7 @@ const {
     getNextJobStageName
 } = require('./triggers/helpers');
 const { getFullStageJobName } = require('../helper');
+const { updateStageBuildStatus, getStageBuild } = require('./helper/updateBuild');
 
 /**
  * Delete a build
@@ -71,7 +72,7 @@ async function triggerNextJobs(config, app) {
     const currentPipeline = config.pipeline;
     const currentJob = config.job;
     const currentBuild = config.build;
-    const { jobFactory, buildFactory, eventFactory, pipelineFactory } = app;
+    const { jobFactory, buildFactory, eventFactory, pipelineFactory, stageFactory, stageBuildFactory } = app;
 
     /** @type {EventModel} */
     const currentEvent = await eventFactory.get({ id: currentBuild.eventId });
@@ -100,8 +101,8 @@ async function triggerNextJobs(config, app) {
     for (const [nextJobName] of Object.entries(currentPipelineNextJobs)) {
         const nextJob = await getJob(nextJobName, currentPipeline.id, jobFactory);
         const node = currentEvent.workflowGraph.nodes.find(n => n.name === trimJobName(nextJobName));
-        const isNextJobVirtual = node.virtual || false;
-        const nextJobStageName = getNextJobStageName({ stageName: node.stageName, nextJobName });
+        const isNextJobVirtual = node && node.virtual === true;
+        const nextJobStageName = node ? getNextJobStageName({ stageName: node.stageName, nextJobName }) : null;
         const resource = `pipeline:${currentPipeline.id}:groupEvent:${currentEvent.groupEventId}`;
         let lock;
         let nextBuild;
@@ -144,15 +145,31 @@ async function triggerNextJobs(config, app) {
                 );
             }
 
-            if (isNextJobVirtual && nextBuild && nextBuild.status === Status.SUCCESS) {
-                downstreamOfNextJobsToBeProcessed.push({
-                    build: nextBuild,
-                    event: currentEvent,
-                    job: nextJob,
-                    pipeline: currentPipeline,
-                    scmContext: config.scmContext,
-                    username: config.username
+            if (isNextJobVirtual) {
+                const stageBuild = await getStageBuild({
+                    stageFactory,
+                    stageBuildFactory,
+                    workflowGraph: currentEvent.workflowGraph,
+                    jobName: nextJobName,
+                    pipelineId: currentPipeline.id,
+                    eventId: currentEvent.id
                 });
+
+                if (stageBuild) {
+                    await updateStageBuildStatus({ stageBuild, newStatus: nextBuild.status, job: nextJob });
+                }
+
+                // Trigger downstream jobs
+                if (nextBuild && nextBuild.status === Status.SUCCESS) {
+                    downstreamOfNextJobsToBeProcessed.push({
+                        build: nextBuild,
+                        event: currentEvent,
+                        job: nextJob,
+                        pipeline: currentPipeline,
+                        scmContext: config.scmContext,
+                        username: config.username
+                    });
+                }
             }
         } catch (err) {
             logger.error(
@@ -290,8 +307,8 @@ async function triggerNextJobs(config, app) {
             for (const [nextJobName, nextJobInfo] of Object.entries(joinedPipeline.jobs)) {
                 const nextJob = await getJob(nextJobName, joinedPipelineId, jobFactory);
                 const node = externalEvent.workflowGraph.nodes.find(n => n.name === trimJobName(nextJobName));
-                const isNextJobVirtual = node.virtual || false;
-                const nextJobStageName = getNextJobStageName({ stageName: node.stageName, nextJobName });
+                const isNextJobVirtual = node && node.virtual === true;
+                const nextJobStageName = node ? getNextJobStageName({ stageName: node.stageName, nextJobName }) : null;
 
                 const { parentBuilds } = parseJobInfo({
                     joinObj: joinedPipeline.jobs,
@@ -334,15 +351,30 @@ async function triggerNextJobs(config, app) {
                         );
                     }
 
-                    if (isNextJobVirtual && nextBuild && nextBuild.status === Status.SUCCESS) {
-                        downstreamOfNextJobsToBeProcessed.push({
-                            build: nextBuild,
-                            event: currentEvent,
-                            job: nextJob,
-                            pipeline: await nextJob.pipeline,
-                            scmContext: config.scmContext,
-                            username: config.username
+                    if (isNextJobVirtual) {
+                        const stageBuild = await getStageBuild({
+                            stageFactory,
+                            stageBuildFactory,
+                            workflowGraph: currentEvent.workflowGraph,
+                            jobName: nextJob.name,
+                            pipelineId: currentPipeline.id,
+                            eventId: currentEvent.id
                         });
+
+                        if (stageBuild) {
+                            await updateStageBuildStatus({ stageBuild, newStatus: nextBuild.status, job: nextJob });
+                        }
+
+                        if (nextBuild && nextBuild.status === Status.SUCCESS) {
+                            downstreamOfNextJobsToBeProcessed.push({
+                                build: nextBuild,
+                                event: currentEvent,
+                                job: nextJob,
+                                pipeline: await nextJob.pipeline,
+                                scmContext: config.scmContext,
+                                username: config.username
+                            });
+                        }
                     }
                 } catch (err) {
                     logger.error(

package/plugins/events/stopBuilds.js

@@ -8,6 +8,7 @@ const schema = require('screwdriver-data-schema');
 const getSchema = schema.models.event.get;
 const idSchema = schema.models.event.base.extract('id');
 const { deriveEventStatusFromBuildStatuses } = require('../builds/helper/updateBuild');
+const nonTerminatedStatus = ['CREATED', 'RUNNING', 'QUEUED', 'BLOCKED', 'FROZEN'];
 
 module.exports = () => ({
     method: 'PUT',
@@ -78,25 +79,25 @@ module.exports = () => ({
 
             // User has good permissions, get event builds
             const builds = await event.getBuilds();
-            const toUpdate = [];
+            const toUpdateBuilds = [];
             const updatedBuilds = [];
 
             // Update endtime and stop running builds
             // Note: COLLAPSED builds will never run
             builds.forEach(b => {
-                if (['CREATED', 'RUNNING', 'QUEUED', 'BLOCKED', 'FROZEN'].includes(b.status)) {
+                if (nonTerminatedStatus.includes(b.status)) {
                     if (b.status === 'RUNNING') {
                         b.endTime = new Date().toISOString();
                     }
                     b.status = 'ABORTED';
                     b.statusMessage = `Aborted by ${username}`;
 
-                    toUpdate.push(b.update());
+                    toUpdateBuilds.push(b.update());
                 } else {
                     updatedBuilds.push(b);
                 }
             });
-            updatedBuilds.push(...(await Promise.all(toUpdate)));
+            updatedBuilds.push(...(await Promise.all(toUpdateBuilds)));
 
             const newEventStatus = deriveEventStatusFromBuildStatuses(updatedBuilds);
 
@@ -105,6 +106,19 @@ module.exports = () => ({
                 await event.update();
             }
 
+            // Update stageBuild status to ABORTED
+            const stageBuilds = await event.getStageBuilds();
+            const toUpdateStageBuilds = [];
+
+            stageBuilds.forEach(sb => {
+                if (nonTerminatedStatus.includes(sb.status)) {
+                    sb.status = 'ABORTED';
+                    toUpdateStageBuilds.push(sb.update());
+                }
+            });
+
+            await Promise.all(toUpdateStageBuilds);
+
             // everything succeeded, inform the user
             const location = urlLib.format({
                 host: request.headers.host,

package/plugins/pipelines/admins/get.js

@@ -20,7 +20,7 @@ module.exports = () => ({
         },
 
         handler: async (request, h) => {
-            const pipelineFactory = request.server.app.pipelineFactory;
+            const { pipelineFactory } = request.server.app;
             const { scope } = request.auth.credentials;
             const { scmContext, includeUserToken } = request.query;
 

package/plugins/webhooks/index.js

@@ -6,8 +6,7 @@ const boom = require('@hapi/boom');
 const { ValidationError } = require('joi');
 const { startHookEvent } = require('./helper');
 
-const DEFAULT_MAX_BYTES = 1048576;
-const MAX_BYTES_UPPER_BOUND = 5242880; // 5MB
+const DEFAULT_MAX_BYTES = 1048576; // 1MB
 const providerSchema = joi
     .object({
         username: joi.string().required(),
@@ -41,7 +40,7 @@ const webhooksPlugin = {
     name: 'webhooks',
     async register(server, options) {
         const pluginOptions = joi.attempt(
-            options,
+            options.scms,
             joi.object().pattern(joi.string(), providerSchema).min(1).required(),
             'Invalid config for plugin-webhooks'
         );
@@ -59,7 +58,7 @@ const webhooksPlugin = {
                     }
                 },
                 payload: {
-                    maxBytes: MAX_BYTES_UPPER_BOUND,
+                    maxBytes: parseInt(pluginOptions.maxBytes, 10) || DEFAULT_MAX_BYTES,
                     parse: false,
                     output: 'stream'
                 },
@@ -71,17 +70,10 @@ const webhooksPlugin = {
                     let hookId;
 
                     try {
-                        let size = 0;
                         const chunks = [];
 
                         for await (const chunk of request.payload) {
-                            size += chunk.length;
                             chunks.push(chunk);
-                            if (size > DEFAULT_MAX_BYTES) {
-                                throw boom.entityTooLarge(
-                                    `Payload size exceeds the maximum limit of ${DEFAULT_MAX_BYTES} bytes`
-                                );
-                            }
                         }
 
                         const data = Buffer.concat(chunks).toString();