screwdriver-queue-service 5.0.1 → 5.0.3

package/bin/server

@@ -18,6 +18,6 @@ const authConfig = config.get('auth');
 server({
     httpd: httpdConfig,
     ecosystem,
-    queueConfig,
+    queue: queueConfig,
     auth: authConfig
 });

package/config/custom-environment-variables.yaml

@@ -239,6 +239,7 @@ queue:
   prefix: REDIS_QUEUE_PREFIX
   # whether or not to retrieve from redis that the data needed to start periodic builds
   periodicBuildTableEnabled: PERIODIC_BUILD_TABLE_ENABLED
+  queueMaxPayloadSize: QUEUE_MAX_PAYLOAD_SIZE
 
 plugins:
     blockedBy:

package/config/default.yaml

@@ -167,6 +167,8 @@ queue:
   prefix: ""
   # whether or not to retrieve from redis that the data needed to start periodic builds
   periodicBuildTableEnabled: true
+  # max payload size in bytes, default to 10MB
+  queueMaxPayloadSize: 10485760
 
 plugins:
   blockedBy:

package/lib/server.js

@@ -59,7 +59,7 @@ module.exports = async config => {
         // Setup Executor
         const executor = new ExecutorQueue({
             ecosystem: hoek.clone(config.ecosystem),
-            ...config.queueConfig
+            ...config.queue
         });
 
         server.app = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "5.0.1",
+  "version": "5.0.3",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {

package/plugins/queue/index.js

@@ -7,7 +7,7 @@ const scheduler = require('./scheduler');
 
 const queuePlugin = {
     name: 'queue',
-    async register(server) {
+    async register(server, options) {
         /**
          * Exposes an init function to begin the scheduler process
          * @method init
@@ -28,7 +28,7 @@ const queuePlugin = {
             return scheduler.cleanUp(executor);
         });
 
-        server.route([putRoute(), deleteRoute(), statsRoute()]);
+        server.route([putRoute(options), deleteRoute(), statsRoute()]);
     }
 };
 

package/plugins/queue/put.js

@@ -3,7 +3,7 @@
 const logger = require('screwdriver-logger');
 const scheduler = require('./scheduler');
 
-module.exports = () => ({
+module.exports = options => ({
     method: 'POST',
     path: '/queue/message',
     config: {
@@ -14,6 +14,11 @@ module.exports = () => ({
             strategies: ['token'],
             scope: ['sdapi']
         },
+        payload: {
+            maxBytes: parseInt(options.queueMaxPayloadSize, 10) || 5242880, // 5MB default
+            parse: true,
+            output: 'data'
+        },
         handler: async (request, h) => {
             try {
                 const executor = request.server.app.executorQueue;

package/plugins/queue/scheduler.js

@@ -422,51 +422,14 @@ async function start(executor, config) {
             throw value.error;
         }
 
-        let buildUpdatePayload;
-
         if (isVirtualJob(annotations)) {
             // Bypass execution of the build if the job is virtual
-            buildUpdatePayload = {
+            const buildUpdatePayload = {
                 status: 'SUCCESS',
                 statusMessage: 'Skipped execution of the virtual job',
                 statusMessageType: 'INFO'
             };
-        } else {
-            const token = executor.tokenGen(
-                Object.assign(tokenConfig, { scope: ['temporal'] }),
-                TEMPORAL_TOKEN_TIMEOUT
-            );
 
-            // set the start time in the queue
-            Object.assign(config, { token });
-            // Store the config in redis
-            await executor.redisBreaker.runCommand('hset', executor.buildConfigTable, buildId, JSON.stringify(config));
-
-            const blockedBySameJob = reach(config, 'annotations>screwdriver.cd/blockedBySameJob', {
-                separator: '>',
-                default: true
-            });
-            const blockedBySameJobWaitTime = reach(config, 'annotations>screwdriver.cd/blockedBySameJobWaitTime', {
-                separator: '>',
-                default: BLOCKED_BY_SAME_JOB_WAIT_TIME
-            });
-
-            // Note: arguments to enqueue are [queue name, job name, array of args]
-            enq = await executor.queueBreaker.runCommand('enqueue', executor.buildQueue, 'start', [
-                {
-                    buildId,
-                    jobId,
-                    blockedBy: blockedBy.toString(),
-                    blockedBySameJob,
-                    blockedBySameJobWaitTime
-                }
-            ]);
-            if (buildStats) {
-                buildUpdatePayload = { stats: build.stats, status: 'QUEUED' };
-            }
-        }
-
-        if (buildUpdatePayload) {
             await helper
                 .updateBuild(
                     {
@@ -478,10 +441,68 @@ async function start(executor, config) {
                     helper.requestRetryStrategy
                 )
                 .catch(err => {
-                    logger.error(`Failed to update build status for build ${buildId}: ${err}`);
+                    logger.error(`Failed to update virtual build status for build ${buildId}: ${err}`);
 
                     throw err;
                 });
+        } else {
+            if (buildStats) {
+                await helper
+                    .updateBuild(
+                        {
+                            buildId,
+                            token: buildToken,
+                            apiUri,
+                            payload: { stats: build.stats, status: 'QUEUED' }
+                        },
+                        helper.requestRetryStrategy
+                    )
+                    .catch(err => {
+                        logger.error(`Failed to update build status to QUEUED for build ${buildId}: ${err}`);
+                        throw err;
+                    });
+            }
+
+            try {
+                const token = executor.tokenGen(
+                    Object.assign(tokenConfig, { scope: ['temporal'] }),
+                    TEMPORAL_TOKEN_TIMEOUT
+                );
+
+                // set the start time in the queue
+                Object.assign(config, { token });
+                // Store the config in redis
+                await executor.redisBreaker.runCommand(
+                    'hset',
+                    executor.buildConfigTable,
+                    buildId,
+                    JSON.stringify(config)
+                );
+
+                const blockedBySameJob = reach(config, 'annotations>screwdriver.cd/blockedBySameJob', {
+                    separator: '>',
+                    default: true
+                });
+                const blockedBySameJobWaitTime = reach(config, 'annotations>screwdriver.cd/blockedBySameJobWaitTime', {
+                    separator: '>',
+                    default: BLOCKED_BY_SAME_JOB_WAIT_TIME
+                });
+
+                // Note: arguments to enqueue are [queue name, job name, array of args]
+                enq = await executor.queueBreaker.runCommand('enqueue', executor.buildQueue, 'start', [
+                    {
+                        buildId,
+                        jobId,
+                        blockedBy: blockedBy.toString(),
+                        blockedBySameJob,
+                        blockedBySameJobWaitTime
+                    }
+                ]);
+            } catch (err) {
+                logger.error(`Redis enqueue failed for build ${buildId}: ${err}`);
+
+                throw err;
+            }
         }
     }