screwdriver-queue-service 2.0.46 → 3.0.0

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12
+FROM node:18
 
 # Screwdriver Version
 ARG VERSION=latest

package/Dockerfile.local

@@ -1,4 +1,4 @@
-FROM node:12
+FROM node:18
 
 # Create our application directory
 RUN mkdir -p /usr/src/app

package/bin/server

@@ -20,4 +20,4 @@ server({
     ecosystem,
     queueConfig,
     auth: authConfig
-});
+});

package/config/custom-environment-variables.yaml

@@ -237,6 +237,8 @@ queue:
       tls: REDIS_TLS_ENABLED
     slotsRefreshTimeout: REDIS_CLUSTER_SLOTS_REFRESH_TIMEOUT
   prefix: REDIS_QUEUE_PREFIX
+  # whether or not to retrieve from redis that the data needed to start periodic builds
+  periodicBuildTableEnabled: PERIODIC_BUILD_TABLE_ENABLED
 
 plugins:
     blockedBy:

package/config/default.yaml

@@ -165,6 +165,8 @@ queue:
       tls: false
     slotsRefreshTimeout: 1000
   prefix: ""
+  # whether or not to retrieve from redis that the data needed to start periodic builds
+  periodicBuildTableEnabled: true
 
 plugins:
   blockedBy:

package/config/rabbitmq.js

@@ -1,29 +1,12 @@
 'use strict';
 
 const config = require('config');
-
-/**
- * convert value to Boolean
- * @method convertToBool
- * @param {(Boolean|String)} value
- * @return {Boolean}
- */
-function convertToBool(value) {
-    if (typeof value === 'boolean') {
-        return value;
-    }
-
-    // trueList refers to https://yaml.org/type/bool.html
-    const trueList = ['on', 'true', 'yes', 'y'];
-    const lowerValue = String(value).toLowerCase();
-
-    return trueList.includes(lowerValue);
-}
+const helper = require('../plugins/helper');
 
 const rabbitmqConfig = config.get('scheduler').rabbitmq;
 const { protocol, username, password, host, port, exchange, vhost, connectOptions } = rabbitmqConfig;
 const amqpURI = `${protocol}://${username}:${password}@${host}:${port}${vhost}`;
-const schedulerMode = convertToBool(config.get('scheduler').enabled);
+const schedulerMode = helper.convertToBool(config.get('scheduler').enabled);
 
 /**
  * get configurations for rabbitmq

package/lib/queue.js

@@ -30,6 +30,7 @@ module.exports = class ExecutorQueue {
         this.periodicBuildQueue = `${this.prefix}periodicBuilds`;
         this.frozenBuildQueue = `${this.prefix}frozenBuilds`;
         this.buildConfigTable = `${this.prefix}buildConfigs`;
+        this.periodicBuildTable = `${this.prefix}periodicBuildConfigs`;
         this.frozenBuildTable = `${this.prefix}frozenBuildConfigs`;
         this.tokenGen = null;
         this.userTokenGen = null;

package/package.json

@@ -1,59 +1,59 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.46",
+  "version": "3.0.0",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
     "lib": "lib"
   },
   "dependencies": {
-    "@hapi/boom": "^9.1.1",
+    "@hapi/boom": "^10.0.0",
     "@hapi/good": "^9.0.1",
     "@hapi/good-console": "^9.0.1",
     "@hapi/good-squeeze": "^6.0.0",
-    "@hapi/hapi": "^20.2.1",
-    "@hapi/hoek": "^9.1.1",
-    "amqp-connection-manager": "^3.8.1",
-    "amqplib": "^0.8.0",
+    "@hapi/hapi": "^21.1.0",
+    "@hapi/hoek": "^10.0.1",
+    "amqp-connection-manager": "^4.1.9",
+    "amqplib": "^0.10.3",
     "blipp": "^4.0.2",
-    "circuit-fuses": "^4.0.5",
-    "config": "^1.31.0",
+    "circuit-fuses": "^5.0.0",
+    "config": "^3.3.6",
     "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
     "ioredis": "^5.2.3",
-    "joi": "^17.5.0",
-    "js-yaml": "^3.14.1",
+    "joi": "^17.7.0",
+    "js-yaml": "^4.1.0",
     "jsonwebtoken": "^8.5.1",
     "node-resque": "^9.2.0",
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
-    "screwdriver-aws-producer-service": "^1.3.0",
-    "screwdriver-data-schema": "^21.23.0",
-    "screwdriver-executor-docker": "^5.0.2",
-    "screwdriver-executor-jenkins": "^5.0.1",
-    "screwdriver-executor-k8s": "^14.16.0",
+    "screwdriver-aws-producer-service": "^2.0.0",
+    "screwdriver-data-schema": "^22.0.1",
+    "screwdriver-executor-docker": "^6.0.0",
+    "screwdriver-executor-jenkins": "^6.0.0",
+    "screwdriver-executor-k8s": "^15.0.0",
     "screwdriver-executor-k8s-vm": "^4.3.2",
-    "screwdriver-executor-router": "^2.1.2",
-    "screwdriver-logger": "^1.0.2",
-    "screwdriver-request": "^1.0.2",
+    "screwdriver-executor-router": "^3.0.0",
+    "screwdriver-logger": "^2.0.0",
+    "screwdriver-request": "^2.0.1",
     "string-hash": "^1.1.3",
-    "uuid": "^8.0.0"
+    "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "chai": "^4.3.4",
+    "chai": "^4.3.7",
     "chai-jwt": "^2.0.0",
     "coveralls": "^3.0.9",
     "cucumber": "^6.0.5",
-    "eslint": "^6.8.0",
-    "eslint-config-screwdriver": "^5.0.6",
-    "mocha": "^8.3.2",
+    "eslint": "^8.28.0",
+    "eslint-config-screwdriver": "^7.0.0",
+    "mocha": "^10.1.0",
     "mocha-multi-reporters": "^1.5.1",
     "mocha-sonarqube-reporter": "^1.0.2",
     "mockery": "^2.1.0",
     "nyc": "^15.1.0",
-    "sinon": "^9.2.4",
+    "sinon": "^15.0.0",
     "snyk": "^1.814.0",
-    "util": "^0.12.2"
+    "util": "^0.12.5"
   },
   "scripts": {
     "start": "node ./bin/server",
@@ -81,6 +81,9 @@
     "Pritam Paul <pritamstyz4ever@gmail.com>",
     "Tiffany Kyi <tiffanykyi@gmail.com>"
   ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "bugs": {
     "url": "https://github.com/screwdriver-cd/screwdriver/issues"
   },

package/plugins/auth/index.js

@@ -2,12 +2,11 @@
 
 const joi = require('joi');
 const jwt = require('jsonwebtoken');
-const uuid = require('uuid');
-
+const { v4: uuidv4 } = require('uuid');
 const DEFAULT_TIMEOUT = 60; // 1h in minutes
 const ALGORITHM = 'RS256';
 
-const validate = async function() {
+const validate = async function () {
     // The _decoded token signature is validated by jwt.veriry so we can return true
     return { isValid: true };
 };
@@ -81,7 +80,7 @@ const authPlugin = {
             jwt.sign(profile, pluginOptions.jwtPrivateKey, {
                 algorithm: ALGORITHM,
                 expiresIn: buildTimeout * 60, // must be in second
-                jwtid: uuid.v4()
+                jwtid: uuidv4()
             })
         );
     }

package/plugins/helper.js

@@ -364,6 +364,24 @@ async function processHooks(apiUri, token, webhookConfig) {
         });
 }
 
+/**
+ * convert value to Boolean
+ * @method convertToBool
+ * @param {(Boolean|String)} value
+ * @return {Boolean}
+ */
+function convertToBool(value) {
+    if (typeof value === 'boolean') {
+        return value;
+    }
+
+    // trueList refers to https://yaml.org/type/bool.html
+    const trueList = ['on', 'true', 'yes', 'y'];
+    const lowerValue = String(value).toLowerCase();
+
+    return trueList.includes(lowerValue);
+}
+
 module.exports = {
     requestRetryStrategy,
     requestRetryStrategyPostEvent,
@@ -376,5 +394,6 @@ module.exports = {
     getPipelineConfig,
     updateBuild,
     notifyJob,
-    processHooks
+    processHooks,
+    convertToBool
 };

package/plugins/queue/scheduler.js

@@ -11,6 +11,7 @@ const helper = require('../helper');
 const { timeOutOfWindows } = require('./utils/freezeWindows');
 const { queueNamespace } = require('../../config/redis');
 const ecosystem = Config.get('ecosystem');
+const periodicBuildTableEnabled = helper.convertToBool(Config.get('queue').periodicBuildTableEnabled);
 const DEFAULT_BUILD_TIMEOUT = 90;
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
@@ -111,9 +112,15 @@ async function postBuildEvent(executor, eventConfig) {
 async function stopPeriodic(executor, config) {
     await executor.connect();
 
-    return executor.queueBreaker.runCommand('delDelayed', executor.periodicBuildQueue, 'startDelayed', [
+    await executor.queueBreaker.runCommand('delDelayed', executor.periodicBuildQueue, 'startDelayed', [
         { jobId: config.jobId }
     ]);
+
+    if (periodicBuildTableEnabled) {
+        return executor.redisBreaker.runCommand('hdel', executor.periodicBuildTable, config.jobId);
+    }
+
+    return Promise.resolve();
 }
 
 /**
@@ -193,6 +200,25 @@ async function startPeriodic(executor, config) {
 
         const next = cron.next(cron.transform(buildCron, job.id));
 
+        if (periodicBuildTableEnabled) {
+            try {
+                // Store the config in redis
+                await executor.redisBreaker.runCommand(
+                    'hset',
+                    executor.periodicBuildTable,
+                    job.id,
+                    JSON.stringify(
+                        Object.assign(config, {
+                            isUpdate: false,
+                            triggerBuild: false
+                        })
+                    )
+                );
+            } catch (err) {
+                logger.error(`failed to store the config in Redis for job ${job.id}: ${err}`);
+            }
+        }
+
         // Note: arguments to enqueueAt are [timestamp, queue name, job name, array of args]
         let shouldRetry = false;
 
@@ -454,42 +480,57 @@ async function init(executor) {
         startDelayed: {
             perform: async config => {
                 try {
-                    const apiUri = ecosystem.api;
                     const { jobId } = config;
 
                     logger.info(`Started processing periodic job ${jobId}`);
 
-                    const buildToken = executor.tokenGen({
-                        jobId,
-                        service: 'queue',
-                        scope: ['build']
-                    });
-
-                    const job = await helper.getJobConfig({
-                        jobId,
-                        token: buildToken,
-                        apiUri
-                    });
-
-                    const pipelineToken = executor.tokenGen({
-                        pipelineId: job.pipelineId,
-                        service: 'queue',
-                        scope: ['pipeline']
-                    });
-
-                    const pipeline = await helper.getPipelineConfig({
-                        pipelineId: job.pipelineId,
-                        token: pipelineToken,
-                        apiUri
-                    });
-
-                    const fullConfig = {
-                        pipeline,
-                        job,
-                        apiUri,
-                        isUpdate: false,
-                        triggerBuild: true
-                    };
+                    let fullConfig;
+
+                    if (periodicBuildTableEnabled) {
+                        const periodicBuildConfig = await executor.redisBreaker.runCommand(
+                            'hget',
+                            executor.periodicBuildTable,
+                            jobId
+                        );
+
+                        fullConfig = Object.assign(JSON.parse(periodicBuildConfig), {
+                            triggerBuild: true
+                        });
+                    } else {
+                        const apiUri = ecosystem.api;
+
+                        const buildToken = executor.tokenGen({
+                            jobId,
+                            service: 'queue',
+                            scope: ['build']
+                        });
+
+                        const job = await helper.getJobConfig({
+                            jobId,
+                            token: buildToken,
+                            apiUri
+                        });
+
+                        const pipelineToken = executor.tokenGen({
+                            pipelineId: job.pipelineId,
+                            service: 'queue',
+                            scope: ['pipeline']
+                        });
+
+                        const pipeline = await helper.getPipelineConfig({
+                            pipelineId: job.pipelineId,
+                            token: pipelineToken,
+                            apiUri
+                        });
+
+                        fullConfig = {
+                            pipeline,
+                            job,
+                            apiUri,
+                            isUpdate: false,
+                            triggerBuild: true
+                        };
+                    }
 
                     return await startPeriodic(executor, fullConfig);
                 } catch (err) {