screwdriver-queue-service 2.0.45 → 2.0.47

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12
+FROM node:18
 
 # Screwdriver Version
 ARG VERSION=latest

package/Dockerfile.local

@@ -1,4 +1,4 @@
-FROM node:12
+FROM node:18
 
 # Create our application directory
 RUN mkdir -p /usr/src/app

package/bin/server

@@ -20,4 +20,4 @@ server({
     ecosystem,
     queueConfig,
     auth: authConfig
-});
+});

package/config/custom-environment-variables.yaml

@@ -237,6 +237,8 @@ queue:
       tls: REDIS_TLS_ENABLED
     slotsRefreshTimeout: REDIS_CLUSTER_SLOTS_REFRESH_TIMEOUT
   prefix: REDIS_QUEUE_PREFIX
+  # whether or not to retrieve from redis that the data needed to start periodic builds
+  periodicBuildTableEnabled: PERIODIC_BUILD_TABLE_ENABLED
 
 plugins:
     blockedBy:

package/config/default.yaml

@@ -165,6 +165,8 @@ queue:
       tls: false
     slotsRefreshTimeout: 1000
   prefix: ""
+  # whether or not to retrieve from redis that the data needed to start periodic builds
+  periodicBuildTableEnabled: true
 
 plugins:
   blockedBy:

package/config/rabbitmq.js

@@ -1,29 +1,12 @@
 'use strict';
 
 const config = require('config');
-
-/**
- * convert value to Boolean
- * @method convertToBool
- * @param {(Boolean|String)} value
- * @return {Boolean}
- */
-function convertToBool(value) {
-    if (typeof value === 'boolean') {
-        return value;
-    }
-
-    // trueList refers to https://yaml.org/type/bool.html
-    const trueList = ['on', 'true', 'yes', 'y'];
-    const lowerValue = String(value).toLowerCase();
-
-    return trueList.includes(lowerValue);
-}
+const helper = require('../plugins/helper');
 
 const rabbitmqConfig = config.get('scheduler').rabbitmq;
 const { protocol, username, password, host, port, exchange, vhost, connectOptions } = rabbitmqConfig;
 const amqpURI = `${protocol}://${username}:${password}@${host}:${port}${vhost}`;
-const schedulerMode = convertToBool(config.get('scheduler').enabled);
+const schedulerMode = helper.convertToBool(config.get('scheduler').enabled);
 
 /**
  * get configurations for rabbitmq

package/lib/queue.js

@@ -30,6 +30,7 @@ module.exports = class ExecutorQueue {
         this.periodicBuildQueue = `${this.prefix}periodicBuilds`;
         this.frozenBuildQueue = `${this.prefix}frozenBuilds`;
         this.buildConfigTable = `${this.prefix}buildConfigs`;
+        this.periodicBuildTable = `${this.prefix}periodicBuildConfigs`;
         this.frozenBuildTable = `${this.prefix}frozenBuildConfigs`;
         this.tokenGen = null;
         this.userTokenGen = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.45",
+  "version": "2.0.47",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {

package/plugins/helper.js

@@ -364,6 +364,24 @@ async function processHooks(apiUri, token, webhookConfig) {
         });
 }
 
+/**
+ * convert value to Boolean
+ * @method convertToBool
+ * @param {(Boolean|String)} value
+ * @return {Boolean}
+ */
+function convertToBool(value) {
+    if (typeof value === 'boolean') {
+        return value;
+    }
+
+    // trueList refers to https://yaml.org/type/bool.html
+    const trueList = ['on', 'true', 'yes', 'y'];
+    const lowerValue = String(value).toLowerCase();
+
+    return trueList.includes(lowerValue);
+}
+
 module.exports = {
     requestRetryStrategy,
     requestRetryStrategyPostEvent,
@@ -376,5 +394,6 @@ module.exports = {
     getPipelineConfig,
     updateBuild,
     notifyJob,
-    processHooks
+    processHooks,
+    convertToBool
 };

package/plugins/queue/scheduler.js

@@ -11,6 +11,7 @@ const helper = require('../helper');
 const { timeOutOfWindows } = require('./utils/freezeWindows');
 const { queueNamespace } = require('../../config/redis');
 const ecosystem = Config.get('ecosystem');
+const periodicBuildTableEnabled = helper.convertToBool(Config.get('queue').periodicBuildTableEnabled);
 const DEFAULT_BUILD_TIMEOUT = 90;
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
@@ -111,9 +112,15 @@ async function postBuildEvent(executor, eventConfig) {
 async function stopPeriodic(executor, config) {
     await executor.connect();
 
-    return executor.queueBreaker.runCommand('delDelayed', executor.periodicBuildQueue, 'startDelayed', [
+    await executor.queueBreaker.runCommand('delDelayed', executor.periodicBuildQueue, 'startDelayed', [
         { jobId: config.jobId }
     ]);
+
+    if (periodicBuildTableEnabled) {
+        return executor.redisBreaker.runCommand('hdel', executor.periodicBuildTable, config.jobId);
+    }
+
+    return Promise.resolve();
 }
 
 /**
@@ -193,6 +200,25 @@ async function startPeriodic(executor, config) {
 
         const next = cron.next(cron.transform(buildCron, job.id));
 
+        if (periodicBuildTableEnabled) {
+            try {
+                // Store the config in redis
+                await executor.redisBreaker.runCommand(
+                    'hset',
+                    executor.periodicBuildTable,
+                    job.id,
+                    JSON.stringify(
+                        Object.assign(config, {
+                            isUpdate: false,
+                            triggerBuild: false
+                        })
+                    )
+                );
+            } catch (err) {
+                logger.error(`failed to store the config in Redis for job ${job.id}: ${err}`);
+            }
+        }
+
         // Note: arguments to enqueueAt are [timestamp, queue name, job name, array of args]
         let shouldRetry = false;
 
@@ -454,42 +480,57 @@ async function init(executor) {
         startDelayed: {
             perform: async config => {
                 try {
-                    const apiUri = ecosystem.api;
                     const { jobId } = config;
 
                     logger.info(`Started processing periodic job ${jobId}`);
 
-                    const buildToken = executor.tokenGen({
-                        jobId,
-                        service: 'queue',
-                        scope: ['build']
-                    });
-
-                    const job = await helper.getJobConfig({
-                        jobId,
-                        token: buildToken,
-                        apiUri
-                    });
-
-                    const pipelineToken = executor.tokenGen({
-                        pipelineId: job.pipelineId,
-                        service: 'queue',
-                        scope: ['pipeline']
-                    });
-
-                    const pipeline = await helper.getPipelineConfig({
-                        pipelineId: job.pipelineId,
-                        token: pipelineToken,
-                        apiUri
-                    });
-
-                    const fullConfig = {
-                        pipeline,
-                        job,
-                        apiUri,
-                        isUpdate: false,
-                        triggerBuild: true
-                    };
+                    let fullConfig;
+
+                    if (periodicBuildTableEnabled) {
+                        const periodicBuildConfig = await executor.redisBreaker.runCommand(
+                            'hget',
+                            executor.periodicBuildTable,
+                            jobId
+                        );
+
+                        fullConfig = Object.assign(JSON.parse(periodicBuildConfig), {
+                            triggerBuild: true
+                        });
+                    } else {
+                        const apiUri = ecosystem.api;
+
+                        const buildToken = executor.tokenGen({
+                            jobId,
+                            service: 'queue',
+                            scope: ['build']
+                        });
+
+                        const job = await helper.getJobConfig({
+                            jobId,
+                            token: buildToken,
+                            apiUri
+                        });
+
+                        const pipelineToken = executor.tokenGen({
+                            pipelineId: job.pipelineId,
+                            service: 'queue',
+                            scope: ['pipeline']
+                        });
+
+                        const pipeline = await helper.getPipelineConfig({
+                            pipelineId: job.pipelineId,
+                            token: pipelineToken,
+                            apiUri
+                        });
+
+                        fullConfig = {
+                            pipeline,
+                            job,
+                            apiUri,
+                            isUpdate: false,
+                            triggerBuild: true
+                        };
+                    }
 
                     return await startPeriodic(executor, fullConfig);
                 } catch (err) {