screwdriver-queue-service 2.0.43 → 2.0.45

package/config/redis.js

@@ -3,7 +3,7 @@
 const config = require('config');
 
 const queueConfig = config.get('queue');
-const connectionType = queueConfig.connectionType;
+const { connectionType } = queueConfig;
 
 if (!connectionType || (connectionType !== 'redis' && connectionType !== 'redisCluster')) {
     throw new Error(

package/lib/queue.js

@@ -30,7 +30,6 @@ module.exports = class ExecutorQueue {
         this.periodicBuildQueue = `${this.prefix}periodicBuilds`;
         this.frozenBuildQueue = `${this.prefix}frozenBuilds`;
         this.buildConfigTable = `${this.prefix}buildConfigs`;
-        this.periodicBuildTable = `${this.prefix}periodicBuildConfigs`;
         this.frozenBuildTable = `${this.prefix}frozenBuildConfigs`;
         this.tokenGen = null;
         this.userTokenGen = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.43",
+  "version": "2.0.45",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {

package/plugins/helper.js

@@ -169,7 +169,7 @@ async function getCurrentStep(stepConfig) {
             return null;
         }
 
-        logger.error(`PUT /v4/builds/${buildId}/steps?status=active returned non 200, ${res.statusCode}, ${res.body}`);
+        logger.error(`GET /v4/builds/${buildId}/steps?status=active returned non 200, ${res.statusCode}, ${res.body}`);
 
         throw new Error(`Failed to getCurrentStep with ${res.statusCode} code and ${res.body}`);
     });
@@ -214,7 +214,7 @@ async function getPipelineAdmin(token, apiUri, pipelineId, retryStrategyFn) {
         formatOptions('GET', `${apiUri}/v4/pipelines/${pipelineId}/admin`, token, undefined, retryStrategyFn)
     ).then(res => {
         logger.info(
-            `POST /v4/pipelines/${pipelineId}/admin completed with attempts, ${res.statusCode}, ${res.attempts}`
+            `GET /v4/pipelines/${pipelineId}/admin completed with attempts, ${res.statusCode}, ${res.attempts}`
         );
         if (res.statusCode === 200) {
             return res.body;
@@ -247,6 +247,50 @@ async function updateBuild(updateConfig, retryStrategyFn) {
     );
 }
 
+/**
+ * Get job config
+ * @param {Object} config
+ * @param {Number} config.jobId
+ * @param {String} config.token
+ * @param {String} config.apiUri
+ * @param {Object} retryStrategyFn
+ */
+async function getJobConfig(config, retryStrategyFn) {
+    const { jobId, token, apiUri } = config;
+
+    return request(formatOptions('GET', `${apiUri}/v4/jobs/${jobId}`, token, undefined, retryStrategyFn)).then(res => {
+        logger.info(`GET /v4/jobs/${jobId} completed with attempts, ${res.statusCode}, ${res.attempts}`);
+        if (res.statusCode === 200) {
+            return res.body;
+        }
+
+        throw new Error(`Failed to get job config with ${res.statusCode} code and ${JSON.stringify(res.body)}`);
+    });
+}
+
+/**
+ * Get pipeline config
+ * @param {Object} config
+ * @param {Number} config.pipelineId
+ * @param {String} config.token
+ * @param {String} config.apiUri
+ * @param {Object} retryStrategyFn
+ */
+async function getPipelineConfig(config, retryStrategyFn) {
+    const { pipelineId, token, apiUri } = config;
+
+    return request(
+        formatOptions('GET', `${apiUri}/v4/pipelines/${pipelineId}`, token, undefined, retryStrategyFn)
+    ).then(res => {
+        logger.info(`GET /v4/pipelines/${pipelineId} completed with attempts, ${res.statusCode}, ${res.attempts}`);
+        if (res.statusCode === 200) {
+            return res.body;
+        }
+
+        throw new Error(`Failed to get pipeline config with ${res.statusCode} code and ${JSON.stringify(res.body)}`);
+    });
+}
+
 /**
  * Notify user with job status
  * @param {Number} jobId
@@ -328,6 +372,8 @@ module.exports = {
     getCurrentStep,
     createBuildEvent,
     getPipelineAdmin,
+    getJobConfig,
+    getPipelineConfig,
     updateBuild,
     notifyJob,
     processHooks

package/plugins/queue/scheduler.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const Config = require('config');
 const logger = require('screwdriver-logger');
 const configSchema = require('screwdriver-data-schema').config;
 const TOKEN_CONFIG_SCHEMA = configSchema.tokenConfig;
@@ -9,6 +10,7 @@ const cron = require('./utils/cron');
 const helper = require('../helper');
 const { timeOutOfWindows } = require('./utils/freezeWindows');
 const { queueNamespace } = require('../../config/redis');
+const ecosystem = Config.get('ecosystem');
 const DEFAULT_BUILD_TIMEOUT = 90;
 const RETRY_LIMIT = 3;
 const RETRY_DELAY = 5;
@@ -20,11 +22,11 @@ const BLOCKED_BY_SAME_JOB_WAIT_TIME = 5;
 /**
  * Posts a new build event to the API
  * @method postBuildEvent
- * @param {Object} config           Configuration
- * @param {Number} [config.eventId] Optional Parent event ID (optional)
- * @param {Object} config.pipeline  Pipeline of the job
- * @param {Object} config.job       Job object to create periodic builds for
- * @param {String} config.apiUri    Base URL of the Screwdriver API
+ * @param {Object} eventConfig           Configuration
+ * @param {Number} [eventConfig.eventId] Optional Parent event ID (optional)
+ * @param {Object} eventConfig.pipeline  Pipeline of the job
+ * @param {Object} eventConfig.job       Job object to create periodic builds for
+ * @param {String} eventConfig.apiUri    Base URL of the Screwdriver API
  * @return {Promise}
  */
 async function postBuildEvent(executor, eventConfig) {
@@ -109,11 +111,9 @@ async function postBuildEvent(executor, eventConfig) {
 async function stopPeriodic(executor, config) {
     await executor.connect();
 
-    await executor.queueBreaker.runCommand('delDelayed', executor.periodicBuildQueue, 'startDelayed', [
+    return executor.queueBreaker.runCommand('delDelayed', executor.periodicBuildQueue, 'startDelayed', [
         { jobId: config.jobId }
     ]);
-
-    return executor.redisBreaker.runCommand('hdel', executor.periodicBuildTable, config.jobId);
 }
 
 /**
@@ -193,23 +193,6 @@ async function startPeriodic(executor, config) {
 
         const next = cron.next(cron.transform(buildCron, job.id));
 
-        try {
-            // Store the config in redis
-            await executor.redisBreaker.runCommand(
-                'hset',
-                executor.periodicBuildTable,
-                job.id,
-                JSON.stringify(
-                    Object.assign(config, {
-                        isUpdate: false,
-                        triggerBuild: false
-                    })
-                )
-            );
-        } catch (err) {
-            logger.error(`failed to store the config in Redis for job ${job.id}: ${err}`);
-        }
-
         // Note: arguments to enqueueAt are [timestamp, queue name, job name, array of args]
         let shouldRetry = false;
 
@@ -238,7 +221,7 @@ async function startPeriodic(executor, config) {
     }
     logger.info(`added to delayed queue for job ${job.id}`);
 
-    if (triggerBuild) {
+    if (triggerBuild && !job.archived) {
         try {
             await postBuildEvent(executor, config);
         } catch (err) {
@@ -469,22 +452,46 @@ async function init(executor) {
     // Jobs object to register the worker with
     const jobs = {
         startDelayed: {
-            perform: async jobConfig => {
+            perform: async config => {
                 try {
-                    logger.info(`Started processing periodic job ${jobConfig.jobId}`);
+                    const apiUri = ecosystem.api;
+                    const { jobId } = config;
 
-                    const fullConfig = await executor.redisBreaker.runCommand(
-                        'hget',
-                        executor.periodicBuildTable,
-                        jobConfig.jobId
-                    );
+                    logger.info(`Started processing periodic job ${jobId}`);
 
-                    return await startPeriodic(
-                        executor,
-                        Object.assign(JSON.parse(fullConfig), {
-                            triggerBuild: true
-                        })
-                    );
+                    const buildToken = executor.tokenGen({
+                        jobId,
+                        service: 'queue',
+                        scope: ['build']
+                    });
+
+                    const job = await helper.getJobConfig({
+                        jobId,
+                        token: buildToken,
+                        apiUri
+                    });
+
+                    const pipelineToken = executor.tokenGen({
+                        pipelineId: job.pipelineId,
+                        service: 'queue',
+                        scope: ['pipeline']
+                    });
+
+                    const pipeline = await helper.getPipelineConfig({
+                        pipelineId: job.pipelineId,
+                        token: pipelineToken,
+                        apiUri
+                    });
+
+                    const fullConfig = {
+                        pipeline,
+                        job,
+                        apiUri,
+                        isUpdate: false,
+                        triggerBuild: true
+                    };
+
+                    return await startPeriodic(executor, fullConfig);
                 } catch (err) {
                     logger.error(`err in startDelayed job: ${err}`);
                     throw err;

package/plugins/worker/worker.js

@@ -43,10 +43,10 @@ const multiWorker = new MultiWorker(
     {
         connection: resqueConnection,
         queues: [`${queuePrefix}builds`, `${queuePrefix}cache`, `${queuePrefix}webhooks`],
-        minTaskProcessors: workerConfig.minTaskProcessors,
-        maxTaskProcessors: workerConfig.maxTaskProcessors,
-        checkTimeout: workerConfig.checkTimeout,
-        maxEventLoopDelay: workerConfig.maxEventLoopDelay
+        minTaskProcessors: parseInt(workerConfig.minTaskProcessors, 10),
+        maxTaskProcessors: parseInt(workerConfig.maxTaskProcessors, 10),
+        checkTimeout: parseInt(workerConfig.checkTimeout, 10),
+        maxEventLoopDelay: parseInt(workerConfig.maxEventLoopDelay, 10)
     },
     jobs
 );