ti2 1.0.124 → 1.0.125

package/README.md

@@ -94,6 +94,22 @@ The product cache refresh interval defaults to seven days when no TTR is set.
 Any explicit `ttlForProducts` or plugin `cacheSettings.bookingsProductSearch.ttr`
 value is honored, including `86400` for a one-day interval.
 
+### Optional catalog lifecycle service
+
+Ti2 synchronously asks Filematch to update catalog lifecycle state when an
+integration is removed or reactivated. The call reuses
+`ti2_events2url_eventsURL` and `ti2_events2url_authorization`; Filematch then
+forwards the request to Pyfilematch. Without the events URL, Ti2 cleans its own
+credentials and schedules without making an external catalog request.
+`INTEGRATION_LIFECYCLE_TIMEOUT_MS` controls the request timeout and defaults to
+30 seconds. A save left in `provisioning` blocks deletion until
+it is older than `INTEGRATION_PROVISIONING_TIMEOUT_MS` (five minutes by
+default); deletion then advances the generation before removing local and
+catalog state. Catalog lifecycle consumers must accept newer generations so a
+failed activation followed by deletion can move from generation N to N+1. The
+timeout is a provisioning lease: saves refresh it before catalog activation and
+must still own their generation when marking activation complete.
+
 ## Contributing
 
 Contributions are welcome and ecouraged.

package/api.yml

@@ -101,6 +101,53 @@ components:
       properties:
         value:
           type: string
+    IntegrationCleanup:
+      type: object
+      required:
+        - userId
+        - integrationId
+        - hint
+        - generation
+        - requestId
+        - status
+      properties:
+        userId:
+          type: string
+        integrationId:
+          type: string
+        hint:
+          type: string
+        generation:
+          type: integer
+          minimum: 1
+        requestId:
+          type: string
+        requestedAt:
+          type: string
+          format: date-time
+        retryCount:
+          type: integer
+          minimum: 0
+        status:
+          type: string
+          enum:
+            - deleting
+            - failed
+            - external_cleanup
+            - complete
+        artifacts:
+          type: object
+          additionalProperties: true
+    IntegrationCleanupResponse:
+      type: object
+      required:
+        - message
+        - cleanup
+      properties:
+        message:
+          type: string
+        cleanup:
+          $ref: '#/components/schemas/IntegrationCleanup'
     CreateAppTokenResponse:
       type: object
       required:
@@ -186,6 +233,9 @@ components:
           type: string
           description: An identifier for this token, it should NOT be the token itself but a short safe version of it
           example: 'expediaTestLocation-23'
+        deferCompletion:
+          type: boolean
+          description: Keep re-creation blocked until the caller finalizes external cleanup
     UserAppValidate:
       description: A user + App configuration to validate
       type: object
@@ -254,6 +304,13 @@ components:
                 type: string
               updatedAt:
                 type: string
+              cleanupStatus:
+                type: string
+                description: Present when removal is incomplete and may be retried.
+                enum:
+                  - deleting
+                  - external_cleanup
+                  - failed
     appRequest:
       type: object
       properties:
@@ -982,7 +1039,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Message'
+                $ref: '#/components/schemas/IntegrationCleanupResponse'
     put:
       tags:
         - app
@@ -1020,6 +1077,56 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Message'
+  /{app}/{userId}/cleanup/finalize:
+    post:
+      tags:
+        - app
+        - user
+      summary: Finalize an integration cleanup after external artifacts are removed
+      security:
+        - bearerAuth: ['admin', 'app', 'user']
+      operationId: finalizeIntegrationCleanup
+      parameters:
+        - name: app
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: userId
+          in: path
+          required: true
+          schema:
+            type: string
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - tokenHint
+                - generation
+                - requestId
+              properties:
+                tokenHint:
+                  type: string
+                generation:
+                  type: integer
+                  minimum: 1
+                requestId:
+                  type: string
+                  minLength: 1
+                  maxLength: 64
+                artifacts:
+                  type: object
+                  additionalProperties: true
+      responses:
+        200:
+          description: success
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IntegrationCleanupResponse'
   /{app}/{userId}/{hint}/token:
     get:
       tags:

package/auth/authHandler.js

@@ -38,6 +38,7 @@ const appCheck = async req => {
     return 'Invalid api key';
   }
   req.appRecord = appRecord.dataValues;
+  req.auth = { subject: `app:${req.pathParams.app}` };
   return undefined;
 };
 
@@ -53,6 +54,7 @@ const app = async (req, res, next) => {
 const adminCheck = async req => {
   const token = getToken({ req });
   if (token !== adminKey) return 'Invalid admin key';
+  req.auth = { subject: 'admin' };
   return undefined;
 };
 
@@ -78,6 +80,7 @@ const userCheck = async req => {
   if (req.params.userId && userId !== req.params.userId) {
     return 'Non matching user id';
   }
+  req.auth = { subject: `user:${userId}` };
   return undefined;
 };
 

package/controllers/__tests__/app.js

@@ -1,9 +1,11 @@
 /* globals beforeAll describe it expect */
 const chance = require('chance').Chance();
+const crypto = require('crypto');
 const jwt = require('jwt-promise');
 const R = require('ramda');
 
 const slugify = require('../../test/slugify');
+const { queue } = require('../../worker/queue');
 let appController = require('../app');
 
 const { env: { adminKey, jwtSecret } } = process;
@@ -13,6 +15,7 @@ describe('app', () => {
   const appName = slugify(
     chance.company(),
   );
+  const otherAppName = slugify(chance.company());
   const newApp = {
     name: appName,
     packageName: `ti2-${appName}`,
@@ -21,6 +24,8 @@ describe('app', () => {
   let doApiPost;
   let doApiGet;
   let doApiPut;
+  let db;
+  let plugins;
   let appKey;
   const userId = chance.guid();
   const apiKey = chance.guid();
@@ -34,8 +39,10 @@ describe('app', () => {
   };
   let encodedKey;
   beforeAll(async () => {
-    ({ doApiPost, doApiGet, doApiPut } = await testUtils({
-      plugins: [appName],
+    ({
+      doApiPost, doApiGet, doApiPut, plugins, sqldb: db,
+    } = await testUtils({
+      plugins: [appName, otherAppName],
     }));
     appController = appController([appName]);
   });
@@ -69,6 +76,12 @@ describe('app', () => {
       },
     });
     expect(parseInt(value, 10)).toBeGreaterThan(0);
+    expect(await db.models.CronJobs.count({
+      where: { userId, hint: apiKey.split('-')[0], pluginName: appName },
+    })).toBeGreaterThan(0);
+    expect(await db.models.CronJobs.count({
+      where: { userId, hint: apiKey.split('-')[0], pluginName: otherAppName },
+    })).toBe(0);
   });
   it('should be able to test a user token for the app', async () => {
     const { valid } = await doApiPost({
@@ -96,7 +109,39 @@ describe('app', () => {
           cron: '0 9 * * *',
         });
     });
-    it.todo('if a new version of the plugin has a different set of scheduled tasks, they should be re-synced');
+    it('replaces the repeatable definition when a cron pattern changes', async () => {
+      const plugin = plugins.find(({ name }) => name === appName);
+      plugin.jobs[0].cron = '0 10 * * *';
+
+      await doApiPost({
+        url: `/${appName}/${userId}`,
+        token: appKey,
+        payload: {
+          tokenHint: apiKey.split('-')[0],
+          token,
+        },
+      });
+
+      const jobs = R.path(
+        ['jobs'],
+        await appController.getAppScheduledJobs({ integrationId: appName, userId }),
+      );
+      expect(R.head(R.project(['pluginJobId', 'cron'], jobs))).toEqual({
+        pluginJobId: 'dailyReport',
+        cron: '0 10 * * *',
+      });
+      const repeatId = crypto.createHash('sha256').update(JSON.stringify({
+        pluginName: appName,
+        pluginJobId: 'dailyReport',
+        userId,
+        hint: apiKey.split('-')[0],
+      })).digest('hex');
+      const repeatDefinitions = (await queue.getRepeatableJobs())
+        .filter(({ id }) => id === repeatId);
+      expect(repeatDefinitions).toEqual([
+        expect.objectContaining({ cron: '0 10 * * *' }),
+      ]);
+    });
     let jobId;
     it('should be able to run a job', async () => {
       let status;

package/controllers/__tests__/user.js

@@ -97,11 +97,25 @@ describe('user', () => {
     expect(template).toBeTruthy();
     expect(R.path(['apiKey', 'regExp', 'source'], template)).toBeTruthy();
   });
+  it('returns 404 when deleting a hint with no credential or lifecycle', async () => {
+    const unknownHint = chance.word({ length: 12 });
+    await doApiDelete({
+      url: `/${appName}/${userId}`,
+      token: userKey,
+      payload: { tokenHint: unknownHint },
+      expectStatusCode: 404,
+    });
+
+    expect(await db.models.IntegrationLifecycle.count({
+      where: { userId, integrationId: appName, hint: unknownHint },
+    })).toBe(0);
+  });
   it('should be able to delete a user/app key', async () => {
+    const deletedHint = apiKey.split('-')[0];
     await doApiDelete({
       url: `/${appName}/${userId}`,
       token: userKey,
-      payload: { tokenHint: apiKey.split('-')[0] },
+      payload: { tokenHint: deletedHint },
     });
     const { userAppKeys } = await doApiGet({
       url: `/user/${userId}/apps`,
@@ -116,6 +130,14 @@ describe('user', () => {
         }),
       ]),
     );
+    const lifecycle = await db.models.IntegrationLifecycle.findOne({
+      where: {
+        userId,
+        integrationId: appName,
+        hint: deletedHint,
+      },
+    });
+    expect(lifecycle.requestedBy).toBe(`user:${userId}`);
   });
   it('should be able to create a user/app integration', async () => {
     // set up new token
@@ -145,6 +167,70 @@ describe('user', () => {
       ]),
     );
   });
+  it('reports deferred deletion as pending until tokenHint finalization', async () => {
+    const deferredHint = chance.word({ length: 12 });
+    await doApiPost({
+      url: `/${appName}/${userId}`,
+      token: userKey,
+      payload: { tokenHint: deferredHint, token },
+    });
+
+    const deletion = await doApiDelete({
+      url: `/${appName}/${userId}`,
+      token: userKey,
+      payload: { tokenHint: deferredHint, deferCompletion: true },
+    });
+    expect(deletion).toEqual(expect.objectContaining({
+      message: 'Integration cleanup is awaiting external finalization.',
+      cleanup: expect.objectContaining({ status: 'external_cleanup' }),
+    }));
+
+    const pendingList = await doApiGet({
+      url: `/user/${userId}/apps`,
+      token: userKey,
+    });
+    expect(pendingList.userAppKeys).toEqual(expect.arrayContaining([
+      expect.objectContaining({
+        integrationId: appName,
+        userId,
+        hint: deferredHint,
+        cleanupStatus: 'external_cleanup',
+      }),
+    ]));
+
+    await doApiPost({
+      url: `/${appName}/${userId}/cleanup/finalize`,
+      token: userKey,
+      payload: {
+        tokenHint: deferredHint,
+        generation: deletion.cleanup.generation,
+      },
+      expectStatusCode: 400,
+    });
+
+    const finalized = await doApiPost({
+      url: `/${appName}/${userId}/cleanup/finalize`,
+      token: userKey,
+      payload: {
+        tokenHint: deferredHint,
+        generation: deletion.cleanup.generation,
+        requestId: deletion.cleanup.requestId,
+        artifacts: { test: true },
+      },
+    });
+    expect(finalized.cleanup.status).toBe('complete');
+
+    const completedList = await doApiGet({
+      url: `/user/${userId}/apps`,
+      token: userKey,
+    });
+    expect(completedList.userAppKeys).toEqual(expect.not.arrayContaining([
+      expect.objectContaining({
+        integrationId: appName,
+        hint: deferredHint,
+      }),
+    ]));
+  });
   it('reports post-save cron setup failure while preserving the committed credential', async () => {
     const postCommitUserId = chance.guid();
     const postCommitUserKey = (await doApiPost({

package/controllers/app.js

@@ -5,9 +5,11 @@ const { Umzug, SequelizeStorage } = require('umzug');
 const path = require('path');
 const Sequelize = require('sequelize');
 const fs = require('fs'); // Changed to synchronous fs for initial load
+const crypto = require('crypto');
 const yaml = require('js-yaml');
 const bb = require('bluebird');
 const R = require('ramda');
+const integrationLifecycle = require('../services/integrationLifecycle');
 
 // Load OpenAPI schema
 let openApiSchema = null;
@@ -102,6 +104,7 @@ const createAppToken = async (req, res, next) => {
     },
   } = req;
   let transaction;
+  let lifecycle;
   try {
     const payload = {
       integrationId,
@@ -117,6 +120,12 @@ const createAppToken = async (req, res, next) => {
       transaction,
       lock: transaction.LOCK.UPDATE,
     });
+    lifecycle = await integrationLifecycle.prepareActivation({
+      userId,
+      integrationId,
+      hint,
+      transaction,
+    });
     const integrationCount = await sqldb.UserAppKey.count({
       where: { userId },
       transaction,
@@ -146,7 +155,8 @@ const createAppToken = async (req, res, next) => {
     // Credential persistence is already committed, but cron provisioning must
     // remain caller-visible so missing scheduled functionality is not silent.
     try {
-      await bb.each(req.app.plugins, async plugin => {
+      const integrationPlugins = req.app.plugins.filter(plugin => plugin.name === integrationId);
+      await bb.each(integrationPlugins, async plugin => {
         if (!Array.isArray(plugin.jobs)) return;
         const validJobs = plugin.jobs.filter(job => Boolean(job.cron) && Boolean(job.method));
         await bb.each(validJobs, async job => {
@@ -171,22 +181,25 @@ const createAppToken = async (req, res, next) => {
               },
             } : {}),
             ...(job.params || {}),
+            jobId: crypto.createHash('sha256').update(JSON.stringify({
+              pluginName: plugin.name,
+              pluginJobId: job.method,
+              userId,
+              hint,
+            })).digest('hex'),
             removeOnComplete: false,
           };
           if (existing) {
-            const bullJob = await queue.getJob(job.bullJobId);
-            if (!bullJob) {
+            if (existing.cron !== job.cron) {
+              await removeJob(existing.bullJobId);
               const rawBullJobId = await addJob(jobPayload, jobParams);
               existing.bullJobId = (rawBullJobId && typeof rawBullJobId === 'object'
                 && rawBullJobId.id) ? rawBullJobId.id : rawBullJobId;
+              existing.cron = job.cron;
               await existing.save();
             } else {
-              const bullCron = R.path(
-                ['opts', 'repeat', 'cron'],
-                await queue.getJob(existing.bullJobId),
-              );
-              if (bullCron !== job.cron) {
-                await queue.removeJobs(existing.bullJobId);
+              const bullJob = await queue.getJob(existing.bullJobId);
+              if (!bullJob) {
                 const rawBullJobId = await addJob(jobPayload, jobParams);
                 existing.bullJobId = (rawBullJobId && typeof rawBullJobId === 'object'
                   && rawBullJobId.id) ? rawBullJobId.id : rawBullJobId;
@@ -206,6 +219,12 @@ const createAppToken = async (req, res, next) => {
       throw cronErr;
     }
 
+    await integrationLifecycle.touchActivation(lifecycle);
+    if (R.pathOr(false, ['artifacts', 'requiresCatalogActivation'], lifecycle)) {
+      await integrationLifecycle.activateCatalog(lifecycle);
+    }
+    await integrationLifecycle.completeActivation(lifecycle);
+
     return res.json(responsePayload);
   } catch (err) {
     if (transaction && !transaction.finished) {
@@ -215,6 +234,13 @@ const createAppToken = async (req, res, next) => {
         console.error('Unable to roll back integration credential transaction', rollbackErr);
       }
     }
+    if (lifecycle && (!transaction || transaction.finished)) {
+      try {
+        await integrationLifecycle.failActivation(lifecycle, err);
+      } catch (lifecycleErr) {
+        console.error('Unable to record integration activation failure', lifecycleErr);
+      }
+    }
     return next(err);
   }
 };
@@ -250,24 +276,69 @@ const deleteAppToken = async (req, res, next) => {
   const {
     body: {
       tokenHint: hint,
+      deferCompletion = false,
     },
     params: {
       app: integrationId,
       userId,
     },
   } = req;
-  // check if the user exists
-  const userRecord = await sqldb.User.findOne({ where: { userId } });
-  if (!userRecord) return next({ status: 404, message: 'User does not exists' });
-  const retVal = await sqldb.UserAppKey.destroy({
-    where: {
-      integrationId,
+  try {
+    if (typeof hint !== 'string' || hint.length === 0 || hint.length > 256) {
+      return next({ status: 400, message: 'A valid tokenHint is required' });
+    }
+    const userRecord = await sqldb.User.findOne({ where: { userId } });
+    if (!userRecord) return next({ status: 404, message: 'User does not exists' });
+    const result = await integrationLifecycle.deleteIntegration({
       userId,
+      integrationId,
       hint,
+      requestedBy: R.pathOr('api', ['auth', 'subject'], req),
+      deferCompletion,
+    });
+    return res.json({
+      message: result.status === 'external_cleanup'
+        ? 'Integration cleanup is awaiting external finalization.'
+        : 'Integration cleanup complete.',
+      cleanup: result,
+    });
+  } catch (err) {
+    return next(err);
+  }
+};
+
+const finalizeIntegrationCleanup = async (req, res, next) => {
+  const {
+    body: {
+      tokenHint: hint,
+      generation,
+      requestId,
+      artifacts,
     },
-  });
-  if (retVal === 0) return next({ status: 404, message: 'Key not found' });
-  return res.json({ message: `${retVal} erased` });
+    params: {
+      app: integrationId,
+      userId,
+    },
+  } = req;
+  try {
+    if (typeof hint !== 'string' || hint.length === 0 || hint.length > 256) {
+      return next({ status: 400, message: 'A valid tokenHint is required' });
+    }
+    if (typeof requestId !== 'string' || requestId.length === 0 || requestId.length > 64) {
+      return next({ status: 400, message: 'A valid requestId is required' });
+    }
+    const cleanup = await integrationLifecycle.completeDeletion({
+      userId,
+      integrationId,
+      hint,
+      generation,
+      requestId,
+      externalArtifacts: artifacts,
+    });
+    return res.json({ message: 'Integration cleanup finalized.', cleanup });
+  } catch (err) {
+    return next(err);
+  }
 };
 
 const validateAppToken = plugins => async (req, res, next) => {
@@ -807,6 +878,7 @@ module.exports = plugins => ({
   getAppSettings,
   updateAppToken,
   deleteAppToken,
+  finalizeIntegrationCleanup,
   deleteAppSettings,
   getAppScheduledJobs,
   getJobStatus,

package/controllers/user.js

@@ -1,16 +1,44 @@
 // controllers/user.js
-const { omit } = require('ramda');
+const { Op } = require('sequelize');
 
-const { UserAppKey } = require('../models');
+const { IntegrationLifecycle, UserAppKey } = require('../models');
+
+const pendingCleanupStatuses = ['deleting', 'external_cleanup', 'failed'];
+const integrationKey = ({ integrationId, hint }) => JSON.stringify([integrationId, hint]);
 
 const userAppList = async (req, res, next) => {
   const { params: { userId } } = req;
   try {
-    const userAppKeys = await UserAppKey.findAll({
-      where: { userId },
-      raw: true,
-      attributes: ['integrationId', 'userId', 'hint', 'createdAt', 'updatedAt'],
-    })
+    const [credentials, pendingCleanups] = await Promise.all([
+      UserAppKey.findAll({
+        where: { userId },
+        raw: true,
+        attributes: ['integrationId', 'userId', 'hint', 'createdAt', 'updatedAt'],
+      }),
+      IntegrationLifecycle.findAll({
+        where: {
+          userId,
+          status: { [Op.in]: pendingCleanupStatuses },
+        },
+        raw: true,
+        attributes: ['integrationId', 'userId', 'hint', 'status'],
+      }),
+    ]);
+    const cleanupByIntegration = new Map(
+      pendingCleanups.map(cleanup => [integrationKey(cleanup), cleanup]),
+    );
+    const userAppKeys = credentials.map(credential => {
+      const cleanup = cleanupByIntegration.get(integrationKey(credential));
+      if (!cleanup) return credential;
+      cleanupByIntegration.delete(integrationKey(credential));
+      return { ...credential, cleanupStatus: cleanup.status };
+    });
+    cleanupByIntegration.forEach(cleanup => userAppKeys.push({
+      integrationId: cleanup.integrationId,
+      userId: cleanup.userId,
+      hint: cleanup.hint,
+      cleanupStatus: cleanup.status,
+    }));
     return res.json({ userAppKeys });
   } catch (err) {
     return next(err);