screwdriver-queue-service 2.0.18 → 2.0.22

package/config/custom-environment-variables.yaml

@@ -290,4 +290,6 @@ kafka:
   # Amazon secret access key
   secretAccessKey: KAFKA_ACCESS_KEY_SECRET
   # AWS region 
-  region: AWS_REGION
+  region: AWS_REGION
+  # Flag to use Short Region Name like use2,usw2
+  shortRegion : USE_SHORT_REGION_NAME

package/config/default.yaml

@@ -221,4 +221,6 @@ kafka:
   # Amazon secret access key
   secretAccessKey: KAFKA_ACCESS_KEY_SECRET
   # AWS region 
-  region: AWS_REGION
+  region: AWS_REGION
+  # Flag to use Short Region Name like use2,usw2
+  shortRegion : true

package/config/kafka.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const config = require('config');
+const kafkaConfig = config.get('kafka');
+
+/**
+ * get config
+ * @returns Object containing kafka config values
+ */
+function get() {
+    return {
+        kafkaEnabled: kafkaConfig.enabled === 'true',
+        useShortRegionName: kafkaConfig.shortRegion === 'true'
+    };
+}
+
+module.exports = {
+    get
+};

package/lib/queue.js

@@ -58,20 +58,6 @@ module.exports = class ExecutorQueue {
                 this.redis[funcName](...args),
             breakerOptions
         );
-        this.requestRetryStrategy = response => {
-            if (Math.floor(response.statusCode / 100) !== 2) {
-                throw new Error('Retry limit reached');
-            }
-
-            return response;
-        };
-        this.requestRetryStrategyPostEvent = response => {
-            if (Math.floor(response.statusCode / 100) !== 2 && response.statusCode !== 404) {
-                throw new Error('Retry limit reached');
-            }
-
-            return response;
-        };
         this.fuseBox = new FuseBox();
         this.fuseBox.addFuse(this.queueBreaker);
         this.fuseBox.addFuse(this.redisBreaker);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.18",
+  "version": "2.0.22",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -11,28 +11,28 @@
     "@hapi/good": "^9.0.1",
     "@hapi/good-console": "^9.0.1",
     "@hapi/good-squeeze": "^6.0.0",
-    "@hapi/hapi": "^20.1.0",
+    "@hapi/hapi": "^20.2.1",
     "@hapi/hoek": "^9.1.1",
-    "amqp-connection-manager": "^3.2.2",
+    "amqp-connection-manager": "^3.8.1",
     "amqplib": "^0.8.0",
     "blipp": "^4.0.2",
     "circuit-fuses": "^4.0.5",
     "config": "^1.31.0",
-    "cron-parser": "^2.18.0",
+    "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
     "ioredis": "^3.2.2",
-    "joi": "^17.4.0",
+    "joi": "^17.5.0",
     "js-yaml": "^3.14.1",
     "jsonwebtoken": "^8.5.1",
     "laabr": "^6.1.3",
     "node-resque": "^5.5.3",
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
-    "screwdriver-aws-producer-service": "^1.1.0",
-    "screwdriver-data-schema": "^21.10.2",
+    "screwdriver-aws-producer-service": "^1.3.0",
+    "screwdriver-data-schema": "^21.16.0",
     "screwdriver-executor-docker": "^5.0.2",
     "screwdriver-executor-jenkins": "^5.0.1",
-    "screwdriver-executor-k8s": "^14.14.4",
+    "screwdriver-executor-k8s": "^14.16.0",
     "screwdriver-executor-k8s-vm": "^4.3.2",
     "screwdriver-executor-router": "^2.1.2",
     "screwdriver-logger": "^1.0.2",
@@ -53,7 +53,7 @@
     "mockery": "^2.1.0",
     "nyc": "^15.1.0",
     "sinon": "^9.2.4",
-    "snyk": "^1.712.0",
+    "snyk": "^1.814.0",
     "util": "^0.12.2"
   },
   "scripts": {

package/plugins/helper.js

@@ -7,6 +7,34 @@ const { queuePrefix } = require('../config/redis');
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
 
+/**
+ * Callback function to retry when HTTP status code is not 2xx
+ * @param   {Object}    response
+ * @param   {Function}  retryWithMergedOptions
+ * @return  {Object}    response
+ */
+function requestRetryStrategy(response) {
+    if (Math.floor(response.statusCode / 100) !== 2) {
+        throw new Error('Retry limit reached');
+    }
+
+    return response;
+}
+
+/**
+ * Callback function to retry when HTTP status code is not 2xx and 404
+ * @param   {Object}    response
+ * @param   {Function}  retryWithMergedOptions
+ * @return  {Object}    response
+ */
+function requestRetryStrategyPostEvent(response) {
+    if (Math.floor(response.statusCode / 100) !== 2 && response.statusCode !== 404) {
+        throw new Error('Retry limit reached');
+    }
+
+    return response;
+}
+
 /**
  *
  * @param {String} method
@@ -241,6 +269,8 @@ async function processHooks(apiUri, token, webhookConfig, retryStrategyFn) {
 }
 
 module.exports = {
+    requestRetryStrategy,
+    requestRetryStrategyPostEvent,
     updateBuildStatus,
     updateStepStop,
     getCurrentStep,

package/plugins/queue/scheduler.js

@@ -38,7 +38,7 @@ async function postBuildEvent(executor, eventConfig) {
             scope: ['user']
         });
 
-        const admin = await helper.getPipelineAdmin(token, apiUri, pipelineId, executor.requestRetryStrategy);
+        const admin = await helper.getPipelineAdmin(token, apiUri, pipelineId, helper.requestRetryStrategy);
 
         if (admin) {
             logger.info(
@@ -64,7 +64,7 @@ async function postBuildEvent(executor, eventConfig) {
                 buildEvent.buildId = buildId;
             }
 
-            await helper.createBuildEvent(apiUri, jwt, buildEvent, executor.requestRetryStrategyPostEvent);
+            await helper.createBuildEvent(apiUri, jwt, buildEvent, helper.requestRetryStrategyPostEvent);
         } else {
             logger.error(
                 `POST event for pipeline failed as no admin found: ${pipelineId}:${job.name}:${job.id}:${buildId}`
@@ -337,7 +337,7 @@ async function start(executor, config) {
                     apiUri,
                     payload
                 },
-                executor.requestRetryStrategy
+                helper.requestRetryStrategy
             )
             .catch(err => {
                 logger.error(`frozenBuilds: failed to update build status for build ${buildId}: ${err}`);
@@ -399,7 +399,7 @@ async function start(executor, config) {
                     apiUri,
                     payload: { stats: build.stats, status: 'QUEUED' }
                 },
-                executor.requestRetryStrategy
+                helper.requestRetryStrategy
             );
         }
     }
@@ -730,7 +730,13 @@ async function queueWebhook(executor, webhookConfig) {
         'enqueue',
         executor.webhookQueue,
         'sendWebhook',
-        JSON.stringify(webhookConfig)
+        JSON.stringify({
+            webhookConfig,
+            token: executor.tokenGen({
+                service: 'queue',
+                scope: ['webhook_worker']
+            })
+        })
     );
 }
 

package/plugins/queue/utils/freezeWindows.js

@@ -74,12 +74,11 @@ const timeOutOfWindow = (cronExp, timeToCheck) => {
     const utcDayOfWeek = timeToCheck.getUTCDay();
     const utcMonth = timeToCheck.getUTCMonth() + 1;
 
-    /* eslint no-underscore-dangle: ["error", { "allow": ["_fields"] }] */
-    const minuteField = cronObj._fields.minute;
-    const hourField = cronObj._fields.hour;
-    const dayOfMonthField = cronObj._fields.dayOfMonth;
-    const dayOfWeekField = cronObj._fields.dayOfWeek;
-    const monthField = cronObj._fields.month;
+    const minuteField = cronObj.fields.minute;
+    const hourField = cronObj.fields.hour;
+    const dayOfMonthField = cronObj.fields.dayOfMonth;
+    const dayOfWeekField = cronObj.fields.dayOfWeek;
+    const monthField = cronObj.fields.month;
 
     const includesMinute = minuteField.includes(utcMinutes);
     const includesHour = hourField.includes(utcHours);

package/plugins/worker/lib/jobs.js

@@ -15,8 +15,8 @@ const blockedByConfig = config.get('plugins').blockedBy;
 const { connectionDetails, queuePrefix, runningJobsPrefix, waitingJobsPrefix } = require('../../../config/redis');
 const rabbitmqConf = require('../../../config/rabbitmq');
 const { amqpURI, exchange, connectOptions } = rabbitmqConf.getConfig();
-const kafkaEnabled = config.get('kafka').enabled === 'true';
-
+const kafkaConfig = require('../../../config/kafka');
+const { kafkaEnabled, useShortRegionName } = kafkaConfig.get();
 const RETRY_LIMIT = 3;
 // This is in milliseconds, reference: https://github.com/actionhero/node-resque/blob/2ffdf0/lib/plugins/Retry.js#L12
 const RETRY_DELAY = 5 * 1000;
@@ -57,6 +57,17 @@ const blockedByOptions = {
 
     collapse: blockedByConfig.collapse
 };
+// AWS region map
+const AWS_REGION_MAP = {
+    north: 'n',
+    west: 'w',
+    northeast: 'nw',
+    east: 'e',
+    south: 's',
+    central: 'c',
+    southeast: 'se'
+};
+
 let rabbitmqConn;
 
 /**
@@ -125,15 +136,33 @@ async function pushToRabbitMq(message, queue, messageId) {
  * Push message to Kafka topic
  * @param {Object} message  Job and build config metadata
  * @param {String} topic          Topic name
+ * @param {String} messageId The message id
  */
-async function pushToKafka(message, topic) {
-    const conn = await AWSProducer.connect();
+async function pushToKafka(message, topic, messageId) {
+    const producer = await AWSProducer.connect();
 
-    if (conn) {
-        await AWSProducer.sendMessage(message, topic);
+    if (producer) {
+        await AWSProducer.sendMessage(producer, message, topic, messageId);
     }
 }
 
+/**
+ *
+ * @param {*String} accountId The AWS accountId
+ * @param {*String} region The region name
+ * @returns String topicName
+ */
+function getTopicName(accountId, region) {
+    const items = region.split('-');
+
+    if (items.length < 3 || !useShortRegionName) {
+        return `builds-${accountId}-${region}`;
+    }
+
+    const shortRegion = ''.concat(items[0], AWS_REGION_MAP[items[1]], items[2]);
+
+    return `builds-${accountId}-${shortRegion}`;
+}
 /**
  * Schedule a job based on mode
  * @method schedule
@@ -153,9 +182,9 @@ async function schedule(job, buildConfig) {
 
     if (kafkaEnabled && buildConfig.provider) {
         const { accountId, region } = buildConfig.provider;
-        const topic = `builds-${accountId}-${region}`;
+        const messageId = `${job}-${buildConfig.buildId}`;
 
-        return pushToKafka(msg, topic);
+        return pushToKafka(msg, getTopicName(accountId, region), messageId);
     }
 
     if (rabbitmqConf.getConfig().schedulerMode) {
@@ -276,18 +305,14 @@ async function clear(cacheConfig) {
 
 /**
  * Send message to processHooks API
- * @param {String} webhookConfig as String
+ * @param {String} configs as String
  */
-async function sendWebhook(webhookConfig) {
-    const parsedConfig = JSON.parse(webhookConfig);
+async function sendWebhook(configs) {
+    const parsedConfig = JSON.parse(configs);
+    const { webhookConfig, token } = parsedConfig;
     const apiUri = ecosystem.api;
-    const token = executor.tokenGen({
-        service: 'queue',
-        scope: ['webhook_worker']
-    });
-    const retryFn = executor.requestRetryStrategyPostEvent;
 
-    await helper.processHooks(apiUri, token, parsedConfig, retryFn);
+    await helper.processHooks(apiUri, token, webhookConfig, helper.requestRetryStrategyPostEvent);
 
     return null;
 }