screwdriver-queue-service 2.0.14 → 2.0.18

package/config/custom-environment-variables.yaml

@@ -272,3 +272,22 @@ scheduler:
         vhost: RABBITMQ_VHOST
         # Connection options
         connectOptions: RABBITMQ_CONNECT_OPTIONS
+kafka:
+  # flag for kafka broker
+  enabled: KAFKA_ENABLED
+  # kafka brokers list
+  hosts: KAFKA_BROKERS_LIST
+  # sasl options
+  sasl:
+    # sasl mechanism
+    mechanism: SASL_MECHANISM
+    # secret id for sasl/scram
+    secretId: SASL_AWS_SECRET_ID
+  # client id of the producer
+  clientId: KAFKA_CLIENT_ID
+  # Amazon access key
+  accessKeyId: KAFKA_ACCESS_KEY_ID
+  # Amazon secret access key
+  secretAccessKey: KAFKA_ACCESS_KEY_SECRET
+  # AWS region 
+  region: AWS_REGION

package/config/default.yaml

@@ -202,3 +202,23 @@ scheduler:
     vhost: /screwdriver
     # Connect Options
     connectOptions: { json: true, heartbeatIntervalInSeconds: 20, reconnectTimeInSeconds: 30 }
+
+kafka:
+  # flag for kafka broker
+  enabled: false
+  # kafka brokers list
+  hosts: KAFKA_BROKERS_LIST
+  # sasl options
+  sasl:
+    # sasl mechanism
+    mechanism: scram-sha-512
+    # secret id for sasl/scram
+    secretId: fake-secret
+  # client id of the producer
+  clientId: sd-producer
+  # Amazon access key
+  accessKeyId: KAFKA_ACCESS_KEY_ID
+  # Amazon secret access key
+  secretAccessKey: KAFKA_ACCESS_KEY_SECRET
+  # AWS region 
+  region: AWS_REGION

package/lib/queue.js

@@ -34,6 +34,8 @@ module.exports = class ExecutorQueue {
         this.userTokenGen = null;
         this.timeoutQueue = `${this.prefix}timeoutConfigs`;
         this.cacheQueue = `${this.prefix}cache`;
+        this.unzipQueue = `${this.prefix}unzip`;
+        this.webhookQueue = `${this.prefix}webhooks`;
 
         const redisConnection = { ...config.redisConnection, pkg: 'ioredis' };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.14",
+  "version": "2.0.18",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -28,7 +28,8 @@
     "node-resque": "^5.5.3",
     "npm-auto-version": "^1.0.0",
     "redlock": "^4.2.0",
-    "screwdriver-data-schema": "^21.2.5",
+    "screwdriver-aws-producer-service": "^1.1.0",
+    "screwdriver-data-schema": "^21.10.2",
     "screwdriver-executor-docker": "^5.0.2",
     "screwdriver-executor-jenkins": "^5.0.1",
     "screwdriver-executor-k8s": "^14.14.4",
@@ -60,7 +61,8 @@
     "pretest": "eslint . --quiet",
     "test": "nyc --report-dir ./artifacts/coverage --reporter=lcov mocha --reporter mocha-multi-reporters --reporter-options configFile=./mocha.config.json --recursive --timeout 4000 --retries 1 --exit --allow-uncaught true --color true",
     "debug": "node --nolazy ./bin/server",
-    "functional": "cucumber-js --format=progress --tags 'not @ignore' --retry 2 --fail-fast --exit"
+    "functional": "cucumber-js --format=progress --tags 'not @ignore' --retry 2 --fail-fast --exit",
+    "semantic-release": "semantic-release"
   },
   "repository": {
     "type": "git",
@@ -85,9 +87,6 @@
   },
   "homepage": "https://github.com/screwdriver-cd/screwdriver-queue-service#readme",
   "release": {
-    "debug": false,
-    "verifyConditions": {
-      "path": "./node_modules/semantic-release/src/lib/plugin-noop.js"
-    }
+    "debug": false
   }
 }

package/plugins/helper.js

@@ -218,11 +218,34 @@ async function updateBuild(updateConfig, retryStrategyFn) {
     );
 }
 
+/**
+ * Post the webhooks process
+ * @method processHooks
+ * @param {String} apiUri
+ * @param {String} token
+ * @param {String} webhookConfig as JSON format
+ * @param {Function} retryStrategyFn
+ * @return {Promise} response or error
+ */
+async function processHooks(apiUri, token, webhookConfig, retryStrategyFn) {
+    return request(formatOptions('POST', `${apiUri}/v4/processHooks`, token, webhookConfig, retryStrategyFn)).then(
+        res => {
+            logger.info(`POST /v4/processHooks completed, ${res.statusCode}, ${JSON.stringify(res.body)}`);
+            if ([200, 201, 204].includes(res.statusCode)) {
+                return res;
+            }
+
+            throw new Error(`Failed to process webhook with ${res.statusCode} code and ${res.body}`);
+        }
+    );
+}
+
 module.exports = {
     updateBuildStatus,
     updateStepStop,
     getCurrentStep,
     createBuildEvent,
     getPipelineAdmin,
-    updateBuild
+    updateBuild,
+    processHooks
 };

package/plugins/queue/put.js

@@ -33,6 +33,12 @@ module.exports = () => ({
                     case 'cache':
                         await scheduler.clearCache(executor, request.payload);
                         break;
+                    case 'unzip':
+                        await scheduler.unzipArtifacts(executor, request.payload);
+                        break;
+                    case 'webhook':
+                        await scheduler.queueWebhook(executor, request.payload);
+                        break;
                     default:
                         await scheduler.start(executor, request.payload);
                         break;

package/plugins/queue/scheduler.js

@@ -13,6 +13,7 @@ const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
 const EXPIRE_TIME = 1800; // 30 mins
 const TEMPORAL_TOKEN_TIMEOUT = 12 * 60; // 12 hours in minutes
+const TEMPORAL_UNZIP_TOKEN_TIMEOUT = 2 * 60; // 2 hours in minutes
 
 /**
  * Posts a new build event to the API
@@ -651,7 +652,7 @@ async function stop(executor, config) {
 }
 
 /**
- * Cleanup any reladed processing
+ * Cleanup any related processing
  */
 async function cleanUp(executor) {
     try {
@@ -688,6 +689,51 @@ async function clearCache(executor, config) {
     }
 }
 
+/**
+ * Pushes a message to unzip artifacts
+ * @async  unzipArtifacts
+ * @param  {Object} executor
+ * @param  {Object} config               Configuration
+ * @param  {String} config.buildId       Unique ID for a build
+ * @return {Promise}
+ */
+async function unzipArtifacts(executor, config) {
+    await executor.connect();
+    const { buildId } = config;
+    const tokenConfig = {
+        username: buildId,
+        scope: 'unzip_worker'
+    };
+    const token = executor.tokenGen(tokenConfig, TEMPORAL_UNZIP_TOKEN_TIMEOUT);
+
+    const enq = await executor.queueBreaker.runCommand('enqueue', executor.unzipQueue, 'start', [
+        {
+            buildId,
+            token
+        }
+    ]);
+
+    return enq;
+}
+
+/**
+ * Pushes webhooks to redis
+ * @async  queueWebhook
+ * @param  {Object} executor
+ * @param  {Object} webhookConfig
+ * @return {Promise}
+ */
+async function queueWebhook(executor, webhookConfig) {
+    await executor.connect();
+
+    return executor.queueBreaker.runCommand(
+        'enqueue',
+        executor.webhookQueue,
+        'sendWebhook',
+        JSON.stringify(webhookConfig)
+    );
+}
+
 module.exports = {
     init,
     start,
@@ -699,5 +745,7 @@ module.exports = {
     startTimer,
     stopTimer,
     cleanUp,
-    clearCache
+    clearCache,
+    unzipArtifacts,
+    queueWebhook
 };

package/plugins/worker/lib/jobs.js

@@ -6,6 +6,8 @@ const config = require('config');
 const hoek = require('@hapi/hoek');
 const ExecutorRouter = require('screwdriver-executor-router');
 const logger = require('screwdriver-logger');
+const AWSProducer = require('screwdriver-aws-producer-service');
+const helper = require('../../helper');
 const { BlockedBy } = require('./BlockedBy');
 const { Filter } = require('./Filter');
 const { CacheFilter } = require('./CacheFilter');
@@ -13,9 +15,10 @@ const blockedByConfig = config.get('plugins').blockedBy;
 const { connectionDetails, queuePrefix, runningJobsPrefix, waitingJobsPrefix } = require('../../../config/redis');
 const rabbitmqConf = require('../../../config/rabbitmq');
 const { amqpURI, exchange, connectOptions } = rabbitmqConf.getConfig();
+const kafkaEnabled = config.get('kafka').enabled === 'true';
 
 const RETRY_LIMIT = 3;
-// This is in milliseconds, reference: https://github.com/taskrabbit/node-resque/blob/master/lib/plugins/Retry.js#L12
+// This is in milliseconds, reference: https://github.com/actionhero/node-resque/blob/2ffdf0/lib/plugins/Retry.js#L12
 const RETRY_DELAY = 5 * 1000;
 const redis = new Redis(connectionDetails.port, connectionDetails.host, connectionDetails.options);
 
@@ -72,7 +75,7 @@ function getRabbitmqConn() {
     logger.info('Creating new rabbitmq connection.');
 
     rabbitmqConn.on('connect', () => logger.info('Connected to rabbitmq!'));
-    rabbitmqConn.on('disconnect', params => logger.info('Disconnected from rabbitmq.', params.err.stack));
+    rabbitmqConn.on('disconnect', params => logger.info(`Disconnected from rabbitmq: ${params.err.stack}`));
 
     return rabbitmqConn;
 }
@@ -83,17 +86,23 @@ function getRabbitmqConn() {
  * @param {String} queue
  * @param {String} messageId
  */
-function pushToRabbitMq(message, queue, messageId) {
+async function pushToRabbitMq(message, queue, messageId) {
     if (!rabbitmqConf.getConfig().schedulerMode) {
         return Promise.resolve();
     }
-    const channelWrapper = getRabbitmqConn().createChannel({
+
+    const conn = getRabbitmqConn();
+    const channelWrapper = conn.createChannel({
         json: true,
         setup: channel => channel.checkExchange(exchange)
     });
 
     logger.info('publishing msg to rabbitmq: %s', messageId);
 
+    channelWrapper.on('error', (error, { name }) => {
+        logger.error(`channel wrapper error ${error}:${name}`);
+    });
+
     return channelWrapper
         .publish(exchange, queue, message, {
             contentType: 'application/json',
@@ -107,11 +116,24 @@ function pushToRabbitMq(message, queue, messageId) {
         .catch(err => {
             logger.error('publishing failed to rabbitmq: %s', err.message);
             channelWrapper.close();
-
+            conn.close();
             throw err;
         });
 }
 
+/**
+ * Push message to Kafka topic
+ * @param {Object} message  Job and build config metadata
+ * @param {String} topic          Topic name
+ */
+async function pushToKafka(message, topic) {
+    const conn = await AWSProducer.connect();
+
+    if (conn) {
+        await AWSProducer.sendMessage(message, topic);
+    }
+}
+
 /**
  * Schedule a job based on mode
  * @method schedule
@@ -119,7 +141,7 @@ function pushToRabbitMq(message, queue, messageId) {
  * @param  {Object} buildConfig build config
  * @return {Promise}
  */
-function schedule(job, buildConfig) {
+async function schedule(job, buildConfig) {
     const buildCluster = buildConfig.buildClusterName;
 
     delete buildConfig.buildClusterName;
@@ -129,8 +151,20 @@ function schedule(job, buildConfig) {
         buildConfig
     };
 
+    if (kafkaEnabled && buildConfig.provider) {
+        const { accountId, region } = buildConfig.provider;
+        const topic = `builds-${accountId}-${region}`;
+
+        return pushToKafka(msg, topic);
+    }
+
     if (rabbitmqConf.getConfig().schedulerMode) {
-        return pushToRabbitMq(msg, buildCluster, buildConfig.buildId);
+        try {
+            return await pushToRabbitMq(msg, buildCluster, buildConfig.buildId);
+        } catch (err) {
+            logger.error(`err in pushing to rabbitmq: ${err}`);
+            throw err;
+        }
     }
 
     // token is not allowed in executor.stop
@@ -150,15 +184,17 @@ function schedule(job, buildConfig) {
  * @param  {String}    buildConfig.blockedBy     Jobs that are blocking this job
  * @return {Promise}
  */
-function start(buildConfig) {
-    return redis
-        .hget(`${queuePrefix}buildConfigs`, buildConfig.buildId)
-        .then(fullBuildConfig => schedule('start', JSON.parse(fullBuildConfig)))
-        .catch(err => {
-            logger.error(`err in start job: ${err}`);
+async function start(buildConfig) {
+    try {
+        const fullBuildConfig = await redis.hget(`${queuePrefix}buildConfigs`, buildConfig.buildId);
 
-            return Promise.reject(err);
-        });
+        await schedule('start', JSON.parse(fullBuildConfig));
+
+        return null;
+    } catch (err) {
+        logger.error(`err in start job: ${err}`);
+        throw err;
+    }
 }
 
 /**
@@ -169,79 +205,91 @@ function start(buildConfig) {
  * @param  {String}    buildConfig.jobId         Job that this build belongs to
  * @param  {String}    buildConfig.blockedBy     Jobs that are blocking this job
  * @param  {String}    buildConfig.started       Whether job has started
+ * @param  {String}    buildConfig.jobName    Job name
  * @return {Promise}
  */
-function stop(buildConfig) {
+async function stop(buildConfig) {
     const started = hoek.reach(buildConfig, 'started', { default: true }); // default value for backward compatibility
-    const { buildId, jobId } = buildConfig;
-    const stopConfig = { buildId };
+    const { buildId, jobId, jobName } = buildConfig;
+    let stopConfig = { buildId, jobId, jobName };
     const runningKey = `${runningJobsPrefix}${jobId}`;
 
-    return (
-        redis
-            .hget(`${queuePrefix}buildConfigs`, buildId)
-            .then(fullBuildConfig => {
-                const parsedConfig = JSON.parse(fullBuildConfig);
+    try {
+        const fullBuildConfig = await redis.hget(`${queuePrefix}buildConfigs`, buildId);
+        const parsedConfig = JSON.parse(fullBuildConfig);
+
+        if (parsedConfig) {
+            stopConfig = {
+                buildId,
+                ...parsedConfig
+            };
+        }
+    } catch (err) {
+        logger.error(`[Stop Build] failed to get config for build ${buildId}: ${err.message}`);
+    }
 
-                if (parsedConfig && parsedConfig.annotations) {
-                    stopConfig.annotations = parsedConfig.annotations;
-                }
+    await redis.hdel(`${queuePrefix}buildConfigs`, buildId);
+    // If this is a running job
+    const runningBuildId = await redis.get(runningKey);
 
-                if (parsedConfig && parsedConfig.buildClusterName) {
-                    stopConfig.buildClusterName = parsedConfig.buildClusterName;
-                }
+    if (parseInt(runningBuildId, 10) === buildId) {
+        await redis.del(runningKey);
+    }
+    // If this is a waiting job
+    await redis.lrem(`${waitingJobsPrefix}${jobId}`, 0, buildId);
 
-                stopConfig.token = parsedConfig.token;
-            })
-            .catch(err => {
-                logger.error(`[Stop Build] failed to get config for build ${buildId}: ${err.message}`);
-            })
-            .then(() => redis.hdel(`${queuePrefix}buildConfigs`, buildId))
-            // If this is a running job
-            .then(() => redis.get(runningKey))
-            .then(runningBuildId => {
-                if (parseInt(runningBuildId, 10) === buildId) {
-                    return redis.del(runningKey);
-                }
-
-                return null;
-            })
-            // If this is a waiting job
-            .then(() => redis.lrem(`${waitingJobsPrefix}${jobId}`, 0, buildId))
-            .then(() => (started ? schedule('stop', stopConfig) : null))
-    );
+    if (started) {
+        await schedule('stop', stopConfig);
+    }
+
+    return null;
 }
 
 /**
  * Send message to clear cache from disk
  * @param {Object} cacheConfig
  */
-function clear(cacheConfig) {
+async function clear(cacheConfig) {
     const { id, buildClusters } = cacheConfig;
-    let queueName;
+    const data = await redis.hget(`${queuePrefix}buildConfigs`, id);
 
-    return redis
-        .hget(`${queuePrefix}buildConfigs`, id)
-        .then(data => {
-            if (data) {
-                const buildConfig = JSON.parse(data);
+    if (data) {
+        const buildConfig = JSON.parse(data);
 
-                queueName = buildConfig.buildClusterName;
-            }
-        })
-        .then(() => {
-            if (queueName) {
-                return pushToRabbitMq({ job: 'clear', cacheConfig }, queueName, id);
-            }
+        const queueName = buildConfig.buildClusterName;
 
-            if (buildClusters) {
-                return Promise.all(
-                    buildClusters.map(cluster => pushToRabbitMq({ job: 'clear', cacheConfig }, cluster, id))
-                );
-            }
+        if (queueName) {
+            await pushToRabbitMq({ job: 'clear', cacheConfig }, queueName, id);
+        }
+    }
 
-            return null;
-        });
+    if (buildClusters) {
+        await Promise.all(
+            buildClusters.map(async cluster => {
+                return pushToRabbitMq({ job: 'clear', cacheConfig }, cluster, id);
+            })
+        );
+    }
+
+    return null;
+}
+
+/**
+ * Send message to processHooks API
+ * @param {String} webhookConfig as String
+ */
+async function sendWebhook(webhookConfig) {
+    const parsedConfig = JSON.parse(webhookConfig);
+    const apiUri = ecosystem.api;
+    const token = executor.tokenGen({
+        service: 'queue',
+        scope: ['webhook_worker']
+    });
+    const retryFn = executor.requestRetryStrategyPostEvent;
+
+    await helper.processHooks(apiUri, token, parsedConfig, retryFn);
+
+    return null;
 }
 
 module.exports = {
@@ -266,5 +314,12 @@ module.exports = {
             Retry: retryOptions
         },
         perform: clear
+    },
+    sendWebhook: {
+        plugins: ['Retry'],
+        pluginOptions: {
+            Retry: retryOptions
+        },
+        perform: sendWebhook
     }
 };

package/plugins/worker/worker.js

@@ -42,7 +42,7 @@ async function shutDownAll(worker, scheduler) {
 const multiWorker = new NodeResque.MultiWorker(
     {
         connection: connectionDetails,
-        queues: [`${queuePrefix}builds`, `${queuePrefix}cache`],
+        queues: [`${queuePrefix}builds`, `${queuePrefix}cache`, `${queuePrefix}webhooks`],
         minTaskProcessors: workerConfig.minTaskProcessors,
         maxTaskProcessors: workerConfig.maxTaskProcessors,
         checkTimeout: workerConfig.checkTimeout,