screwdriver-queue-service 2.0.19 → 2.0.23

package/config/custom-environment-variables.yaml

@@ -290,4 +290,6 @@ kafka:
   # Amazon secret access key
   secretAccessKey: KAFKA_ACCESS_KEY_SECRET
   # AWS region 
-  region: AWS_REGION
+  region: AWS_REGION
+  # Flag to use Short Region Name like use2,usw2
+  shortRegion : USE_SHORT_REGION_NAME

package/config/default.yaml

@@ -221,4 +221,6 @@ kafka:
   # Amazon secret access key
   secretAccessKey: KAFKA_ACCESS_KEY_SECRET
   # AWS region 
-  region: AWS_REGION
+  region: AWS_REGION
+  # Flag to use Short Region Name like use2,usw2
+  shortRegion : true

package/config/kafka.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const config = require('config');
+const kafkaConfig = config.get('kafka');
+
+/**
+ * get config
+ * @returns Object containing kafka config values
+ */
+function get() {
+    return {
+        kafkaEnabled: kafkaConfig.enabled === 'true',
+        useShortRegionName: kafkaConfig.shortRegion === 'true'
+    };
+}
+
+module.exports = {
+    get
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.19",
+  "version": "2.0.23",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -11,28 +11,28 @@
     "@hapi/good": "^9.0.1",
     "@hapi/good-console": "^9.0.1",
     "@hapi/good-squeeze": "^6.0.0",
-    "@hapi/hapi": "^20.1.0",
+    "@hapi/hapi": "^20.2.1",
     "@hapi/hoek": "^9.1.1",
-    "amqp-connection-manager": "^3.2.2",
+    "amqp-connection-manager": "^3.8.1",
     "amqplib": "^0.8.0",
     "blipp": "^4.0.2",
     "circuit-fuses": "^4.0.5",
     "config": "^1.31.0",
-    "cron-parser": "^2.18.0",
+    "cron-parser": "^4.2.1",
     "hapi-auth-jwt2": "^10.2.0",
     "ioredis": "^3.2.2",
-    "joi": "^17.4.0",
+    "joi": "^17.5.0",
     "js-yaml": "^3.14.1",
     "jsonwebtoken": "^8.5.1",
     "laabr": "^6.1.3",
     "node-resque": "^5.5.3",
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
-    "screwdriver-aws-producer-service": "^1.1.0",
-    "screwdriver-data-schema": "^21.10.2",
+    "screwdriver-aws-producer-service": "^1.3.0",
+    "screwdriver-data-schema": "^21.17.0",
     "screwdriver-executor-docker": "^5.0.2",
     "screwdriver-executor-jenkins": "^5.0.1",
-    "screwdriver-executor-k8s": "^14.14.4",
+    "screwdriver-executor-k8s": "^14.16.0",
     "screwdriver-executor-k8s-vm": "^4.3.2",
     "screwdriver-executor-router": "^2.1.2",
     "screwdriver-logger": "^1.0.2",
@@ -53,7 +53,7 @@
     "mockery": "^2.1.0",
     "nyc": "^15.1.0",
     "sinon": "^9.2.4",
-    "snyk": "^1.712.0",
+    "snyk": "^1.814.0",
     "util": "^0.12.2"
   },
   "scripts": {

package/plugins/queue/utils/freezeWindows.js

@@ -74,12 +74,11 @@ const timeOutOfWindow = (cronExp, timeToCheck) => {
     const utcDayOfWeek = timeToCheck.getUTCDay();
     const utcMonth = timeToCheck.getUTCMonth() + 1;
 
-    /* eslint no-underscore-dangle: ["error", { "allow": ["_fields"] }] */
-    const minuteField = cronObj._fields.minute;
-    const hourField = cronObj._fields.hour;
-    const dayOfMonthField = cronObj._fields.dayOfMonth;
-    const dayOfWeekField = cronObj._fields.dayOfWeek;
-    const monthField = cronObj._fields.month;
+    const minuteField = cronObj.fields.minute;
+    const hourField = cronObj.fields.hour;
+    const dayOfMonthField = cronObj.fields.dayOfMonth;
+    const dayOfWeekField = cronObj.fields.dayOfWeek;
+    const monthField = cronObj.fields.month;
 
     const includesMinute = minuteField.includes(utcMinutes);
     const includesHour = hourField.includes(utcHours);

package/plugins/worker/lib/jobs.js

@@ -15,11 +15,12 @@ const blockedByConfig = config.get('plugins').blockedBy;
 const { connectionDetails, queuePrefix, runningJobsPrefix, waitingJobsPrefix } = require('../../../config/redis');
 const rabbitmqConf = require('../../../config/rabbitmq');
 const { amqpURI, exchange, connectOptions } = rabbitmqConf.getConfig();
-const kafkaEnabled = config.get('kafka').enabled === 'true';
-
+const kafkaConfig = require('../../../config/kafka');
+const { kafkaEnabled, useShortRegionName } = kafkaConfig.get();
 const RETRY_LIMIT = 3;
 // This is in milliseconds, reference: https://github.com/actionhero/node-resque/blob/2ffdf0/lib/plugins/Retry.js#L12
 const RETRY_DELAY = 5 * 1000;
+const DEFAULT_BUILD_TIMEOUT = 90;
 const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
 
 const ecosystem = config.get('ecosystem');
@@ -57,6 +58,17 @@ const blockedByOptions = {
 
     collapse: blockedByConfig.collapse
 };
+// AWS region map
+const AWS_REGION_MAP = {
+    north: 'n',
+    west: 'w',
+    northeast: 'nw',
+    east: 'e',
+    south: 's',
+    central: 'c',
+    southeast: 'se'
+};
+
 let rabbitmqConn;
 
 /**
@@ -125,15 +137,62 @@ async function pushToRabbitMq(message, queue, messageId) {
  * Push message to Kafka topic
  * @param {Object} message  Job and build config metadata
  * @param {String} topic          Topic name
+ * @param {String} messageId The message id
  */
-async function pushToKafka(message, topic) {
-    const conn = await AWSProducer.connect();
+async function pushToKafka(message, topic, messageId) {
+    const producer = await AWSProducer.connect();
 
-    if (conn) {
-        await AWSProducer.sendMessage(message, topic);
+    if (producer) {
+        await AWSProducer.sendMessage(producer, message, topic, messageId);
     }
 }
 
+/**
+ *
+ * @param {*String} accountId The AWS accountId
+ * @param {*String} region The region name
+ * @returns String topicName
+ */
+function getTopicName(accountId, region) {
+    const items = region.split('-');
+
+    if (items.length < 3 || !useShortRegionName) {
+        return `builds-${accountId}-${region}`;
+    }
+
+    const shortRegion = ''.concat(items[0], AWS_REGION_MAP[items[1]], items[2]);
+
+    return `builds-${accountId}-${shortRegion}`;
+}
+
+/**
+ *
+ * @param {String} job  type of job start|stop`
+ * @param {*} buildConfig
+ * @returns
+ */
+function getKafkaMessageRequest(job, buildConfig) {
+    const { accountId, region, executor: executorType } = buildConfig.provider;
+
+    const topic = getTopicName(accountId, region);
+    const messageId = `${job}-${buildConfig.buildId}`;
+
+    const timeout = parseInt(hoek.reach(buildConfig, 'annotations>screwdriver.cd/timeout', { separator: '>' }), 10);
+    const buildTimeout = Number.isNaN(timeout) ? DEFAULT_BUILD_TIMEOUT : timeout;
+
+    const message = {
+        job,
+        executorType,
+        buildConfig: {
+            ...buildConfig,
+            buildTimeout,
+            uiUri: ecosystem.ui,
+            storeUri: ecosystem.store
+        }
+    };
+
+    return { message, topic, messageId };
+}
 /**
  * Schedule a job based on mode
  * @method schedule
@@ -146,18 +205,17 @@ async function schedule(job, buildConfig) {
 
     delete buildConfig.buildClusterName;
 
+    if (kafkaEnabled && buildConfig.provider) {
+        const { message, topic, messageId } = getKafkaMessageRequest(job, buildConfig);
+
+        return pushToKafka(message, topic, messageId);
+    }
+
     const msg = {
         job,
         buildConfig
     };
 
-    if (kafkaEnabled && buildConfig.provider) {
-        const { accountId, region } = buildConfig.provider;
-        const topic = `builds-${accountId}-${region}`;
-
-        return pushToKafka(msg, topic);
-    }
-
     if (rabbitmqConf.getConfig().schedulerMode) {
         try {
             return await pushToRabbitMq(msg, buildCluster, buildConfig.buildId);