screwdriver-queue-service 2.0.16 → 2.0.20

package/lib/queue.js

@@ -35,6 +35,7 @@ module.exports = class ExecutorQueue {
         this.timeoutQueue = `${this.prefix}timeoutConfigs`;
         this.cacheQueue = `${this.prefix}cache`;
         this.unzipQueue = `${this.prefix}unzip`;
+        this.webhookQueue = `${this.prefix}webhooks`;
 
         const redisConnection = { ...config.redisConnection, pkg: 'ioredis' };
 
@@ -57,20 +58,6 @@ module.exports = class ExecutorQueue {
                 this.redis[funcName](...args),
             breakerOptions
         );
-        this.requestRetryStrategy = response => {
-            if (Math.floor(response.statusCode / 100) !== 2) {
-                throw new Error('Retry limit reached');
-            }
-
-            return response;
-        };
-        this.requestRetryStrategyPostEvent = response => {
-            if (Math.floor(response.statusCode / 100) !== 2 && response.statusCode !== 404) {
-                throw new Error('Retry limit reached');
-            }
-
-            return response;
-        };
         this.fuseBox = new FuseBox();
         this.fuseBox.addFuse(this.queueBreaker);
         this.fuseBox.addFuse(this.redisBreaker);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.16",
+  "version": "2.0.20",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -11,17 +11,17 @@
     "@hapi/good": "^9.0.1",
     "@hapi/good-console": "^9.0.1",
     "@hapi/good-squeeze": "^6.0.0",
-    "@hapi/hapi": "^20.1.0",
+    "@hapi/hapi": "^20.2.1",
     "@hapi/hoek": "^9.1.1",
-    "amqp-connection-manager": "^3.2.2",
+    "amqp-connection-manager": "^3.8.1",
     "amqplib": "^0.8.0",
     "blipp": "^4.0.2",
     "circuit-fuses": "^4.0.5",
     "config": "^1.31.0",
-    "cron-parser": "^2.18.0",
+    "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
     "ioredis": "^3.2.2",
-    "joi": "^17.4.0",
+    "joi": "^17.5.0",
     "js-yaml": "^3.14.1",
     "jsonwebtoken": "^8.5.1",
     "laabr": "^6.1.3",
@@ -29,10 +29,10 @@
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
     "screwdriver-aws-producer-service": "^1.1.0",
-    "screwdriver-data-schema": "^21.10.2",
+    "screwdriver-data-schema": "^21.16.0",
     "screwdriver-executor-docker": "^5.0.2",
     "screwdriver-executor-jenkins": "^5.0.1",
-    "screwdriver-executor-k8s": "^14.14.4",
+    "screwdriver-executor-k8s": "^14.16.0",
     "screwdriver-executor-k8s-vm": "^4.3.2",
     "screwdriver-executor-router": "^2.1.2",
     "screwdriver-logger": "^1.0.2",
@@ -53,7 +53,7 @@
     "mockery": "^2.1.0",
     "nyc": "^15.1.0",
     "sinon": "^9.2.4",
-    "snyk": "^1.712.0",
+    "snyk": "^1.814.0",
     "util": "^0.12.2"
   },
   "scripts": {

package/plugins/helper.js

@@ -7,6 +7,34 @@ const { queuePrefix } = require('../config/redis');
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
 
+/**
+ * Callback function to retry when HTTP status code is not 2xx
+ * @param   {Object}    response
+ * @param   {Function}  retryWithMergedOptions
+ * @return  {Object}    response
+ */
+function requestRetryStrategy(response) {
+    if (Math.floor(response.statusCode / 100) !== 2) {
+        throw new Error('Retry limit reached');
+    }
+
+    return response;
+}
+
+/**
+ * Callback function to retry when HTTP status code is not 2xx and 404
+ * @param   {Object}    response
+ * @param   {Function}  retryWithMergedOptions
+ * @return  {Object}    response
+ */
+function requestRetryStrategyPostEvent(response) {
+    if (Math.floor(response.statusCode / 100) !== 2 && response.statusCode !== 404) {
+        throw new Error('Retry limit reached');
+    }
+
+    return response;
+}
+
 /**
  *
  * @param {String} method
@@ -218,11 +246,36 @@ async function updateBuild(updateConfig, retryStrategyFn) {
     );
 }
 
+/**
+ * Post the webhooks process
+ * @method processHooks
+ * @param {String} apiUri
+ * @param {String} token
+ * @param {String} webhookConfig as JSON format
+ * @param {Function} retryStrategyFn
+ * @return {Promise} response or error
+ */
+async function processHooks(apiUri, token, webhookConfig, retryStrategyFn) {
+    return request(formatOptions('POST', `${apiUri}/v4/processHooks`, token, webhookConfig, retryStrategyFn)).then(
+        res => {
+            logger.info(`POST /v4/processHooks completed, ${res.statusCode}, ${JSON.stringify(res.body)}`);
+            if ([200, 201, 204].includes(res.statusCode)) {
+                return res;
+            }
+
+            throw new Error(`Failed to process webhook with ${res.statusCode} code and ${res.body}`);
+        }
+    );
+}
+
 module.exports = {
+    requestRetryStrategy,
+    requestRetryStrategyPostEvent,
     updateBuildStatus,
     updateStepStop,
     getCurrentStep,
     createBuildEvent,
     getPipelineAdmin,
-    updateBuild
+    updateBuild,
+    processHooks
 };

package/plugins/queue/put.js

@@ -36,6 +36,9 @@ module.exports = () => ({
                     case 'unzip':
                         await scheduler.unzipArtifacts(executor, request.payload);
                         break;
+                    case 'webhook':
+                        await scheduler.queueWebhook(executor, request.payload);
+                        break;
                     default:
                         await scheduler.start(executor, request.payload);
                         break;

package/plugins/queue/scheduler.js

@@ -38,7 +38,7 @@ async function postBuildEvent(executor, eventConfig) {
             scope: ['user']
         });
 
-        const admin = await helper.getPipelineAdmin(token, apiUri, pipelineId, executor.requestRetryStrategy);
+        const admin = await helper.getPipelineAdmin(token, apiUri, pipelineId, helper.requestRetryStrategy);
 
         if (admin) {
             logger.info(
@@ -64,7 +64,7 @@ async function postBuildEvent(executor, eventConfig) {
                 buildEvent.buildId = buildId;
             }
 
-            await helper.createBuildEvent(apiUri, jwt, buildEvent, executor.requestRetryStrategyPostEvent);
+            await helper.createBuildEvent(apiUri, jwt, buildEvent, helper.requestRetryStrategyPostEvent);
         } else {
             logger.error(
                 `POST event for pipeline failed as no admin found: ${pipelineId}:${job.name}:${job.id}:${buildId}`
@@ -337,7 +337,7 @@ async function start(executor, config) {
                     apiUri,
                     payload
                 },
-                executor.requestRetryStrategy
+                helper.requestRetryStrategy
             )
             .catch(err => {
                 logger.error(`frozenBuilds: failed to update build status for build ${buildId}: ${err}`);
@@ -399,7 +399,7 @@ async function start(executor, config) {
                     apiUri,
                     payload: { stats: build.stats, status: 'QUEUED' }
                 },
-                executor.requestRetryStrategy
+                helper.requestRetryStrategy
             );
         }
     }
@@ -716,6 +716,30 @@ async function unzipArtifacts(executor, config) {
     return enq;
 }
 
+/**
+ * Pushes webhooks to redis
+ * @async  queueWebhook
+ * @param  {Object} executor
+ * @param  {Object} webhookConfig
+ * @return {Promise}
+ */
+async function queueWebhook(executor, webhookConfig) {
+    await executor.connect();
+
+    return executor.queueBreaker.runCommand(
+        'enqueue',
+        executor.webhookQueue,
+        'sendWebhook',
+        JSON.stringify({
+            webhookConfig,
+            token: executor.tokenGen({
+                service: 'queue',
+                scope: ['webhook_worker']
+            })
+        })
+    );
+}
+
 module.exports = {
     init,
     start,
@@ -728,5 +752,6 @@ module.exports = {
     stopTimer,
     cleanUp,
     clearCache,
-    unzipArtifacts
+    unzipArtifacts,
+    queueWebhook
 };

package/plugins/queue/utils/freezeWindows.js

@@ -74,12 +74,11 @@ const timeOutOfWindow = (cronExp, timeToCheck) => {
     const utcDayOfWeek = timeToCheck.getUTCDay();
     const utcMonth = timeToCheck.getUTCMonth() + 1;
 
-    /* eslint no-underscore-dangle: ["error", { "allow": ["_fields"] }] */
-    const minuteField = cronObj._fields.minute;
-    const hourField = cronObj._fields.hour;
-    const dayOfMonthField = cronObj._fields.dayOfMonth;
-    const dayOfWeekField = cronObj._fields.dayOfWeek;
-    const monthField = cronObj._fields.month;
+    const minuteField = cronObj.fields.minute;
+    const hourField = cronObj.fields.hour;
+    const dayOfMonthField = cronObj.fields.dayOfMonth;
+    const dayOfWeekField = cronObj.fields.dayOfWeek;
+    const monthField = cronObj.fields.month;
 
     const includesMinute = minuteField.includes(utcMinutes);
     const includesHour = hourField.includes(utcHours);

package/plugins/worker/lib/jobs.js

@@ -7,6 +7,7 @@ const hoek = require('@hapi/hoek');
 const ExecutorRouter = require('screwdriver-executor-router');
 const logger = require('screwdriver-logger');
 const AWSProducer = require('screwdriver-aws-producer-service');
+const helper = require('../../helper');
 const { BlockedBy } = require('./BlockedBy');
 const { Filter } = require('./Filter');
 const { CacheFilter } = require('./CacheFilter');
@@ -17,7 +18,7 @@ const { amqpURI, exchange, connectOptions } = rabbitmqConf.getConfig();
 const kafkaEnabled = config.get('kafka').enabled === 'true';
 
 const RETRY_LIMIT = 3;
-// This is in milliseconds, reference: https://github.com/taskrabbit/node-resque/blob/master/lib/plugins/Retry.js#L12
+// This is in milliseconds, reference: https://github.com/actionhero/node-resque/blob/2ffdf0/lib/plugins/Retry.js#L12
 const RETRY_DELAY = 5 * 1000;
 const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
 
@@ -273,6 +274,20 @@ async function clear(cacheConfig) {
     return null;
 }
 
+/**
+ * Send message to processHooks API
+ * @param {String} configs as String
+ */
+async function sendWebhook(configs) {
+    const parsedConfig = JSON.parse(configs);
+    const { webhookConfig, token } = parsedConfig;
+    const apiUri = ecosystem.api;
+
+    await helper.processHooks(apiUri, token, webhookConfig, helper.requestRetryStrategyPostEvent);
+
+    return null;
+}
+
 module.exports = {
     start: {
         plugins: [Filter, 'Retry', BlockedBy],
@@ -295,5 +310,12 @@ module.exports = {
             Retry: retryOptions
         },
         perform: clear
+    },
+    sendWebhook: {
+        plugins: ['Retry'],
+        pluginOptions: {
+            Retry: retryOptions
+        },
+        perform: sendWebhook
     }
 };

package/plugins/worker/worker.js

@@ -42,7 +42,7 @@ async function shutDownAll(worker, scheduler) {
 const multiWorker = new NodeResque.MultiWorker(
     {
         connection: connectionDetails,
-        queues: [`${queuePrefix}builds`, `${queuePrefix}cache`],
+        queues: [`${queuePrefix}builds`, `${queuePrefix}cache`, `${queuePrefix}webhooks`],
         minTaskProcessors: workerConfig.minTaskProcessors,
         maxTaskProcessors: workerConfig.maxTaskProcessors,
         checkTimeout: workerConfig.checkTimeout,