screwdriver-queue-service 2.0.36 → 2.0.39

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screwdriver-queue-service",
-  "version": "2.0.36",
+  "version": "2.0.39",
   "description": "Screwdriver Queue Service API",
   "main": "app.js",
   "directories": {
@@ -38,7 +38,7 @@
     "screwdriver-logger": "^1.0.2",
     "screwdriver-request": "^1.0.2",
     "string-hash": "^1.1.3",
-    "uuid": "^7.0.2"
+    "uuid": "^8.0.0"
   },
   "devDependencies": {
     "chai": "^4.3.4",

package/plugins/helper.js

@@ -247,6 +247,29 @@ async function updateBuild(updateConfig, retryStrategyFn) {
     );
 }
 
+/**
+ * Notify user with job status
+ * @param {Number} jobId
+ * @param {String} token
+ * @param {Object} payload
+ * @param {String} apiUri
+ * @param {Object} retryStrategyFn
+ */
+async function notifyJob(notifyConfig, retryStrategyFn) {
+    const { token, apiUri, jobId, payload } = notifyConfig;
+
+    return request(formatOptions('POST', `${apiUri}/v4/jobs/${jobId}/notify`, token, payload, retryStrategyFn)).then(
+        res => {
+            logger.info(`POST /v4/jobs/${jobId}/notify completed with attempts, ${res.statusCode}, ${res.attempts}`);
+            if ([200, 201, 204].includes(res.statusCode)) {
+                return res.body;
+            }
+
+            throw new Error(`Could not notify job ${jobId} with ${res.statusCode}code and ${JSON.stringify(res.body)}`);
+        }
+    );
+}
+
 /**
  * Post the webhooks process
  * @method processHooks
@@ -306,5 +329,6 @@ module.exports = {
     createBuildEvent,
     getPipelineAdmin,
     updateBuild,
+    notifyJob,
     processHooks
 };

package/plugins/queue/scheduler.js

@@ -29,12 +29,13 @@ const BLOCKED_BY_SAME_JOB_WAIT_TIME = 5;
 async function postBuildEvent(executor, eventConfig) {
     const { pipeline, job, apiUri, eventId, causeMessage, buildId } = eventConfig;
     const pipelineId = pipeline.id;
+    const jobId = job.id;
 
     try {
         const token = executor.tokenGen({
             pipelineId,
             service: 'queue',
-            jobId: job.id,
+            jobId,
             scmContext: pipeline.scmContext,
             scope: ['user']
         });
@@ -71,6 +72,26 @@ async function postBuildEvent(executor, eventConfig) {
                 `POST event for pipeline failed as no admin found: ${pipelineId}:${job.name}:${job.id}:${buildId}`
             );
 
+            const pipelineToken = executor.tokenGen({
+                pipelineId,
+                service: 'queue',
+                scmContext: pipeline.scmContext,
+                scope: ['pipeline']
+            });
+
+            const status = 'FAILURE';
+            const message = `Pipeline ${pipelineId} does not have admin, unable to start job ${job.name}.`;
+
+            await helper.notifyJob(
+                {
+                    token: pipelineToken,
+                    apiUri,
+                    jobId,
+                    payload: { status, message }
+                },
+                helper.requestRetryStrategyPostEvent
+            );
+
             throw new Error(`Pipeline admin not found, cannot process build ${buildId}`);
         }
     } catch (err) {
@@ -166,15 +187,6 @@ async function startPeriodic(executor, config) {
 
     if (triggerBuild) {
         config.causeMessage = 'Started by periodic build scheduler';
-
-        // Even if post event failed for this event after retry, we should still enqueue the next event
-        try {
-            await postBuildEvent(executor, config);
-        } catch (err) {
-            logger.error(
-                `periodic builds: failed to post build event for job ${job.id} in pipeline ${pipeline.id}: ${err}`
-            );
-        }
     }
 
     if (buildCron && job.state === 'ENABLED' && !job.archived) {
@@ -214,18 +226,28 @@ async function startPeriodic(executor, config) {
                 logger.error(`failed to enqueue for job ${job.id}: ${err}`);
             }
         }
-        if (!shouldRetry) {
-            logger.info(`successfully added to queue for job ${job.id}`);
 
-            return Promise.resolve();
+        if (shouldRetry) {
+            try {
+                await executor.queue.enqueueAt(next, executor.periodicBuildQueue, 'startDelayed', [{ jobId: job.id }]);
+            } catch (err) {
+                logger.error(`failed to add to delayed queue for job ${job.id}: ${err}`);
+            }
+        } else {
+            logger.info(`successfully added to queue for job ${job.id}`);
         }
+    }
+    logger.info(`added to delayed queue for job ${job.id}`);
+
+    if (triggerBuild) {
         try {
-            await executor.queue.enqueueAt(next, executor.periodicBuildQueue, 'startDelayed', [{ jobId: job.id }]);
+            await postBuildEvent(executor, config);
         } catch (err) {
-            logger.error(`failed to add to delayed queue for job ${job.id}: ${err}`);
+            logger.error(
+                `periodic builds: failed to post build event for job ${job.id} in pipeline ${pipeline.id}: ${err}`
+            );
         }
     }
-    logger.info(`added to delayed queue for job ${job.id}`);
 
     return Promise.resolve();
 }
@@ -536,8 +558,11 @@ async function init(executor) {
     executor.scheduler.on('master', state => logger.info(`scheduler became master ${state}`));
     executor.scheduler.on('error', error => logger.info(`scheduler error >> ${error}`));
     executor.scheduler.on('workingTimestamp', timestamp => logger.info(`scheduler working timestamp ${timestamp}`));
+    executor.scheduler.on('cleanStuckWorker', (workerName, errorPayload, delta) =>
+        logger.info(`scheduler failing ${workerName} (stuck for ${delta}s) and failing job ${errorPayload}`)
+    );
     executor.scheduler.on('transferredJob', (timestamp, job) =>
-        logger.info(`scheduler enqueuing job timestamp  >>  ${JSON.stringify(job)}`)
+        logger.info(`scheduler enqueuing job ${timestamp}  >>  ${JSON.stringify(job)}`)
     );
 
     await executor.multiWorker.start();

package/plugins/worker/worker.js

@@ -123,13 +123,17 @@ async function invoke() {
         scheduler.on('poll', () => logger.info('queueWorker->scheduler polling'));
         scheduler.on('master', state => logger.info(`queueWorker->scheduler became master ${state}`));
         scheduler.on('error', error => logger.info(`queueWorker->scheduler error >> ${error}`));
-        scheduler.on('working_timestamp', timestamp =>
+        scheduler.on('workingTimestamp', timestamp =>
             logger.info(`queueWorker->scheduler working timestamp ${timestamp}`)
         );
-        scheduler.on('transferred_job', (timestamp, job) =>
+        scheduler.on('transferredJob', (timestamp, job) =>
             logger.info(`queueWorker->scheduler enqueuing job timestamp  >> ${timestamp} ${JSON.stringify(job)}`)
         );
-
+        scheduler.on('cleanStuckWorker', (workerName, errorPayload, delta) =>
+            logger.info(
+                `queueWorker->scheduler failing ${workerName} (stuck for ${delta}s) and failing job ${errorPayload}`
+            )
+        );
         multiWorker.start();
 
         await scheduler.connect();