screwdriver-queue-service 2.0.17 → 2.0.21

package/config/custom-environment-variables.yaml

@@ -290,4 +290,6 @@ kafka:
   # Amazon secret access key
   secretAccessKey: KAFKA_ACCESS_KEY_SECRET
   # AWS region 
-  region: AWS_REGION
+  region: AWS_REGION
+  # Flag to use Short Region Name like use2,usw2
+  shortRegion : USE_SHORT_REGION_NAME

package/config/default.yaml

@@ -221,4 +221,6 @@ kafka:
   # Amazon secret access key
   secretAccessKey: KAFKA_ACCESS_KEY_SECRET
   # AWS region 
-  region: AWS_REGION
+  region: AWS_REGION
+  # Flag to use Short Region Name like use2,usw2
+  shortRegion : true

package/config/kafka.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const config = require('config');
+const kafkaConfig = config.get('kafka');
+
+/**
+ * get config
+ * @returns Object containing kafka config values
+ */
+function get() {
+    return {
+        kafkaEnabled: kafkaConfig.enabled === 'true',
+        useShortRegionName: kafkaConfig.shortRegion === 'true'
+    };
+}
+
+module.exports = {
+    get
+};

package/lib/queue.js

@@ -58,20 +58,6 @@ module.exports = class ExecutorQueue {
                 this.redis[funcName](...args),
             breakerOptions
         );
-        this.requestRetryStrategy = response => {
-            if (Math.floor(response.statusCode / 100) !== 2) {
-                throw new Error('Retry limit reached');
-            }
-
-            return response;
-        };
-        this.requestRetryStrategyPostEvent = response => {
-            if (Math.floor(response.statusCode / 100) !== 2 && response.statusCode !== 404) {
-                throw new Error('Retry limit reached');
-            }
-
-            return response;
-        };
         this.fuseBox = new FuseBox();
         this.fuseBox.addFuse(this.queueBreaker);
         this.fuseBox.addFuse(this.redisBreaker);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.17",
+  "version": "2.0.21",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -11,17 +11,17 @@
     "@hapi/good": "^9.0.1",
     "@hapi/good-console": "^9.0.1",
     "@hapi/good-squeeze": "^6.0.0",
-    "@hapi/hapi": "^20.1.0",
+    "@hapi/hapi": "^20.2.1",
     "@hapi/hoek": "^9.1.1",
-    "amqp-connection-manager": "^3.2.2",
+    "amqp-connection-manager": "^3.8.1",
     "amqplib": "^0.8.0",
     "blipp": "^4.0.2",
     "circuit-fuses": "^4.0.5",
     "config": "^1.31.0",
-    "cron-parser": "^2.18.0",
+    "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
     "ioredis": "^3.2.2",
-    "joi": "^17.4.0",
+    "joi": "^17.5.0",
     "js-yaml": "^3.14.1",
     "jsonwebtoken": "^8.5.1",
     "laabr": "^6.1.3",
@@ -29,10 +29,10 @@
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
     "screwdriver-aws-producer-service": "^1.1.0",
-    "screwdriver-data-schema": "^21.10.2",
+    "screwdriver-data-schema": "^21.16.0",
     "screwdriver-executor-docker": "^5.0.2",
     "screwdriver-executor-jenkins": "^5.0.1",
-    "screwdriver-executor-k8s": "^14.14.4",
+    "screwdriver-executor-k8s": "^14.16.0",
     "screwdriver-executor-k8s-vm": "^4.3.2",
     "screwdriver-executor-router": "^2.1.2",
     "screwdriver-logger": "^1.0.2",
@@ -53,7 +53,7 @@
     "mockery": "^2.1.0",
     "nyc": "^15.1.0",
     "sinon": "^9.2.4",
-    "snyk": "^1.712.0",
+    "snyk": "^1.814.0",
     "util": "^0.12.2"
   },
   "scripts": {

package/plugins/helper.js

@@ -7,6 +7,34 @@ const { queuePrefix } = require('../config/redis');
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
 
+/**
+ * Callback function to retry when HTTP status code is not 2xx
+ * @param   {Object}    response
+ * @param   {Function}  retryWithMergedOptions
+ * @return  {Object}    response
+ */
+function requestRetryStrategy(response) {
+    if (Math.floor(response.statusCode / 100) !== 2) {
+        throw new Error('Retry limit reached');
+    }
+
+    return response;
+}
+
+/**
+ * Callback function to retry when HTTP status code is not 2xx and 404
+ * @param   {Object}    response
+ * @param   {Function}  retryWithMergedOptions
+ * @return  {Object}    response
+ */
+function requestRetryStrategyPostEvent(response) {
+    if (Math.floor(response.statusCode / 100) !== 2 && response.statusCode !== 404) {
+        throw new Error('Retry limit reached');
+    }
+
+    return response;
+}
+
 /**
  *
  * @param {String} method
@@ -218,11 +246,36 @@ async function updateBuild(updateConfig, retryStrategyFn) {
     );
 }
 
+/**
+ * Post the webhooks process
+ * @method processHooks
+ * @param {String} apiUri
+ * @param {String} token
+ * @param {String} webhookConfig as JSON format
+ * @param {Function} retryStrategyFn
+ * @return {Promise} response or error
+ */
+async function processHooks(apiUri, token, webhookConfig, retryStrategyFn) {
+    return request(formatOptions('POST', `${apiUri}/v4/processHooks`, token, webhookConfig, retryStrategyFn)).then(
+        res => {
+            logger.info(`POST /v4/processHooks completed, ${res.statusCode}, ${JSON.stringify(res.body)}`);
+            if ([200, 201, 204].includes(res.statusCode)) {
+                return res;
+            }
+
+            throw new Error(`Failed to process webhook with ${res.statusCode} code and ${res.body}`);
+        }
+    );
+}
+
 module.exports = {
+    requestRetryStrategy,
+    requestRetryStrategyPostEvent,
     updateBuildStatus,
     updateStepStop,
     getCurrentStep,
     createBuildEvent,
     getPipelineAdmin,
-    updateBuild
+    updateBuild,
+    processHooks
 };

package/plugins/queue/scheduler.js

@@ -38,7 +38,7 @@ async function postBuildEvent(executor, eventConfig) {
             scope: ['user']
         });
 
-        const admin = await helper.getPipelineAdmin(token, apiUri, pipelineId, executor.requestRetryStrategy);
+        const admin = await helper.getPipelineAdmin(token, apiUri, pipelineId, helper.requestRetryStrategy);
 
         if (admin) {
             logger.info(
@@ -64,7 +64,7 @@ async function postBuildEvent(executor, eventConfig) {
                 buildEvent.buildId = buildId;
             }
 
-            await helper.createBuildEvent(apiUri, jwt, buildEvent, executor.requestRetryStrategyPostEvent);
+            await helper.createBuildEvent(apiUri, jwt, buildEvent, helper.requestRetryStrategyPostEvent);
         } else {
             logger.error(
                 `POST event for pipeline failed as no admin found: ${pipelineId}:${job.name}:${job.id}:${buildId}`
@@ -337,7 +337,7 @@ async function start(executor, config) {
                     apiUri,
                     payload
                 },
-                executor.requestRetryStrategy
+                helper.requestRetryStrategy
             )
             .catch(err => {
                 logger.error(`frozenBuilds: failed to update build status for build ${buildId}: ${err}`);
@@ -399,7 +399,7 @@ async function start(executor, config) {
                     apiUri,
                     payload: { stats: build.stats, status: 'QUEUED' }
                 },
-                executor.requestRetryStrategy
+                helper.requestRetryStrategy
             );
         }
     }
@@ -730,7 +730,13 @@ async function queueWebhook(executor, webhookConfig) {
         'enqueue',
         executor.webhookQueue,
         'sendWebhook',
-        JSON.stringify(webhookConfig)
+        JSON.stringify({
+            webhookConfig,
+            token: executor.tokenGen({
+                service: 'queue',
+                scope: ['webhook_worker']
+            })
+        })
     );
 }
 

package/plugins/queue/utils/freezeWindows.js

@@ -74,12 +74,11 @@ const timeOutOfWindow = (cronExp, timeToCheck) => {
     const utcDayOfWeek = timeToCheck.getUTCDay();
     const utcMonth = timeToCheck.getUTCMonth() + 1;
 
-    /* eslint no-underscore-dangle: ["error", { "allow": ["_fields"] }] */
-    const minuteField = cronObj._fields.minute;
-    const hourField = cronObj._fields.hour;
-    const dayOfMonthField = cronObj._fields.dayOfMonth;
-    const dayOfWeekField = cronObj._fields.dayOfWeek;
-    const monthField = cronObj._fields.month;
+    const minuteField = cronObj.fields.minute;
+    const hourField = cronObj.fields.hour;
+    const dayOfMonthField = cronObj.fields.dayOfMonth;
+    const dayOfWeekField = cronObj.fields.dayOfWeek;
+    const monthField = cronObj.fields.month;
 
     const includesMinute = minuteField.includes(utcMinutes);
     const includesHour = hourField.includes(utcHours);

package/plugins/worker/lib/jobs.js

@@ -7,6 +7,7 @@ const hoek = require('@hapi/hoek');
 const ExecutorRouter = require('screwdriver-executor-router');
 const logger = require('screwdriver-logger');
 const AWSProducer = require('screwdriver-aws-producer-service');
+const helper = require('../../helper');
 const { BlockedBy } = require('./BlockedBy');
 const { Filter } = require('./Filter');
 const { CacheFilter } = require('./CacheFilter');
@@ -14,10 +15,10 @@ const blockedByConfig = config.get('plugins').blockedBy;
 const { connectionDetails, queuePrefix, runningJobsPrefix, waitingJobsPrefix } = require('../../../config/redis');
 const rabbitmqConf = require('../../../config/rabbitmq');
 const { amqpURI, exchange, connectOptions } = rabbitmqConf.getConfig();
-const kafkaEnabled = config.get('kafka').enabled === 'true';
-
+const kafkaConfig = require('../../../config/kafka');
+const { kafkaEnabled, useShortRegionName } = kafkaConfig.get();
 const RETRY_LIMIT = 3;
-// This is in milliseconds, reference: https://github.com/taskrabbit/node-resque/blob/master/lib/plugins/Retry.js#L12
+// This is in milliseconds, reference: https://github.com/actionhero/node-resque/blob/2ffdf0/lib/plugins/Retry.js#L12
 const RETRY_DELAY = 5 * 1000;
 const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
 
@@ -56,6 +57,17 @@ const blockedByOptions = {
 
     collapse: blockedByConfig.collapse
 };
+// AWS region map
+const AWS_REGION_MAP = {
+    north: 'n',
+    west: 'w',
+    northeast: 'nw',
+    east: 'e',
+    south: 's',
+    central: 'c',
+    southeast: 'se'
+};
+
 let rabbitmqConn;
 
 /**
@@ -133,6 +145,23 @@ async function pushToKafka(message, topic) {
     }
 }
 
+/**
+ *
+ * @param {*String} accountId The AWS accountId
+ * @param {*String} region The region name
+ * @returns String topicName
+ */
+function getTopicName(accountId, region) {
+    const items = region.split('-');
+
+    if (items.length < 3 || !useShortRegionName) {
+        return `builds-${accountId}-${region}`;
+    }
+
+    const shortRegion = ''.concat(items[0], AWS_REGION_MAP[items[1]], items[2]);
+
+    return `builds-${accountId}-${shortRegion}`;
+}
 /**
  * Schedule a job based on mode
  * @method schedule
@@ -152,9 +181,8 @@ async function schedule(job, buildConfig) {
 
     if (kafkaEnabled && buildConfig.provider) {
         const { accountId, region } = buildConfig.provider;
-        const topic = `builds-${accountId}-${region}`;
 
-        return pushToKafka(msg, topic);
+        return pushToKafka(msg, getTopicName(accountId, region));
     }
 
     if (rabbitmqConf.getConfig().schedulerMode) {
@@ -273,6 +301,20 @@ async function clear(cacheConfig) {
     return null;
 }
 
+/**
+ * Send message to processHooks API
+ * @param {String} configs as String
+ */
+async function sendWebhook(configs) {
+    const parsedConfig = JSON.parse(configs);
+    const { webhookConfig, token } = parsedConfig;
+    const apiUri = ecosystem.api;
+
+    await helper.processHooks(apiUri, token, webhookConfig, helper.requestRetryStrategyPostEvent);
+
+    return null;
+}
+
 module.exports = {
     start: {
         plugins: [Filter, 'Retry', BlockedBy],
@@ -295,5 +337,12 @@ module.exports = {
             Retry: retryOptions
         },
         perform: clear
+    },
+    sendWebhook: {
+        plugins: ['Retry'],
+        pluginOptions: {
+            Retry: retryOptions
+        },
+        perform: sendWebhook
     }
 };

package/plugins/worker/worker.js

@@ -42,7 +42,7 @@ async function shutDownAll(worker, scheduler) {
 const multiWorker = new NodeResque.MultiWorker(
     {
         connection: connectionDetails,
-        queues: [`${queuePrefix}builds`, `${queuePrefix}cache`],
+        queues: [`${queuePrefix}builds`, `${queuePrefix}cache`, `${queuePrefix}webhooks`],
         minTaskProcessors: workerConfig.minTaskProcessors,
         maxTaskProcessors: workerConfig.maxTaskProcessors,
         checkTimeout: workerConfig.checkTimeout,