ti2 1.0.123 → 1.0.125

package/controllers/app.js

@@ -5,9 +5,11 @@ const { Umzug, SequelizeStorage } = require('umzug');
 const path = require('path');
 const Sequelize = require('sequelize');
 const fs = require('fs'); // Changed to synchronous fs for initial load
+const crypto = require('crypto');
 const yaml = require('js-yaml');
 const bb = require('bluebird');
 const R = require('ramda');
+const integrationLifecycle = require('../services/integrationLifecycle');
 
 // Load OpenAPI schema
 let openApiSchema = null;
@@ -102,6 +104,7 @@ const createAppToken = async (req, res, next) => {
     },
   } = req;
   let transaction;
+  let lifecycle;
   try {
     const payload = {
       integrationId,
@@ -117,6 +120,12 @@ const createAppToken = async (req, res, next) => {
       transaction,
       lock: transaction.LOCK.UPDATE,
     });
+    lifecycle = await integrationLifecycle.prepareActivation({
+      userId,
+      integrationId,
+      hint,
+      transaction,
+    });
     const integrationCount = await sqldb.UserAppKey.count({
       where: { userId },
       transaction,
@@ -146,7 +155,8 @@ const createAppToken = async (req, res, next) => {
     // Credential persistence is already committed, but cron provisioning must
     // remain caller-visible so missing scheduled functionality is not silent.
     try {
-      await bb.each(req.app.plugins, async plugin => {
+      const integrationPlugins = req.app.plugins.filter(plugin => plugin.name === integrationId);
+      await bb.each(integrationPlugins, async plugin => {
         if (!Array.isArray(plugin.jobs)) return;
         const validJobs = plugin.jobs.filter(job => Boolean(job.cron) && Boolean(job.method));
         await bb.each(validJobs, async job => {
@@ -171,22 +181,25 @@ const createAppToken = async (req, res, next) => {
               },
             } : {}),
             ...(job.params || {}),
+            jobId: crypto.createHash('sha256').update(JSON.stringify({
+              pluginName: plugin.name,
+              pluginJobId: job.method,
+              userId,
+              hint,
+            })).digest('hex'),
             removeOnComplete: false,
           };
           if (existing) {
-            const bullJob = await queue.getJob(job.bullJobId);
-            if (!bullJob) {
+            if (existing.cron !== job.cron) {
+              await removeJob(existing.bullJobId);
               const rawBullJobId = await addJob(jobPayload, jobParams);
               existing.bullJobId = (rawBullJobId && typeof rawBullJobId === 'object'
                 && rawBullJobId.id) ? rawBullJobId.id : rawBullJobId;
+              existing.cron = job.cron;
               await existing.save();
             } else {
-              const bullCron = R.path(
-                ['opts', 'repeat', 'cron'],
-                await queue.getJob(existing.bullJobId),
-              );
-              if (bullCron !== job.cron) {
-                await queue.removeJobs(existing.bullJobId);
+              const bullJob = await queue.getJob(existing.bullJobId);
+              if (!bullJob) {
                 const rawBullJobId = await addJob(jobPayload, jobParams);
                 existing.bullJobId = (rawBullJobId && typeof rawBullJobId === 'object'
                   && rawBullJobId.id) ? rawBullJobId.id : rawBullJobId;
@@ -206,6 +219,12 @@ const createAppToken = async (req, res, next) => {
       throw cronErr;
     }
 
+    await integrationLifecycle.touchActivation(lifecycle);
+    if (R.pathOr(false, ['artifacts', 'requiresCatalogActivation'], lifecycle)) {
+      await integrationLifecycle.activateCatalog(lifecycle);
+    }
+    await integrationLifecycle.completeActivation(lifecycle);
+
     return res.json(responsePayload);
   } catch (err) {
     if (transaction && !transaction.finished) {
@@ -215,6 +234,13 @@ const createAppToken = async (req, res, next) => {
         console.error('Unable to roll back integration credential transaction', rollbackErr);
       }
     }
+    if (lifecycle && (!transaction || transaction.finished)) {
+      try {
+        await integrationLifecycle.failActivation(lifecycle, err);
+      } catch (lifecycleErr) {
+        console.error('Unable to record integration activation failure', lifecycleErr);
+      }
+    }
     return next(err);
   }
 };
@@ -250,24 +276,69 @@ const deleteAppToken = async (req, res, next) => {
   const {
     body: {
       tokenHint: hint,
+      deferCompletion = false,
     },
     params: {
       app: integrationId,
       userId,
     },
   } = req;
-  // check if the user exists
-  const userRecord = await sqldb.User.findOne({ where: { userId } });
-  if (!userRecord) return next({ status: 404, message: 'User does not exists' });
-  const retVal = await sqldb.UserAppKey.destroy({
-    where: {
-      integrationId,
+  try {
+    if (typeof hint !== 'string' || hint.length === 0 || hint.length > 256) {
+      return next({ status: 400, message: 'A valid tokenHint is required' });
+    }
+    const userRecord = await sqldb.User.findOne({ where: { userId } });
+    if (!userRecord) return next({ status: 404, message: 'User does not exists' });
+    const result = await integrationLifecycle.deleteIntegration({
       userId,
+      integrationId,
       hint,
+      requestedBy: R.pathOr('api', ['auth', 'subject'], req),
+      deferCompletion,
+    });
+    return res.json({
+      message: result.status === 'external_cleanup'
+        ? 'Integration cleanup is awaiting external finalization.'
+        : 'Integration cleanup complete.',
+      cleanup: result,
+    });
+  } catch (err) {
+    return next(err);
+  }
+};
+
+const finalizeIntegrationCleanup = async (req, res, next) => {
+  const {
+    body: {
+      tokenHint: hint,
+      generation,
+      requestId,
+      artifacts,
     },
-  });
-  if (retVal === 0) return next({ status: 404, message: 'Key not found' });
-  return res.json({ message: `${retVal} erased` });
+    params: {
+      app: integrationId,
+      userId,
+    },
+  } = req;
+  try {
+    if (typeof hint !== 'string' || hint.length === 0 || hint.length > 256) {
+      return next({ status: 400, message: 'A valid tokenHint is required' });
+    }
+    if (typeof requestId !== 'string' || requestId.length === 0 || requestId.length > 64) {
+      return next({ status: 400, message: 'A valid requestId is required' });
+    }
+    const cleanup = await integrationLifecycle.completeDeletion({
+      userId,
+      integrationId,
+      hint,
+      generation,
+      requestId,
+      externalArtifacts: artifacts,
+    });
+    return res.json({ message: 'Integration cleanup finalized.', cleanup });
+  } catch (err) {
+    return next(err);
+  }
 };
 
 const validateAppToken = plugins => async (req, res, next) => {
@@ -807,6 +878,7 @@ module.exports = plugins => ({
   getAppSettings,
   updateAppToken,
   deleteAppToken,
+  finalizeIntegrationCleanup,
   deleteAppSettings,
   getAppScheduledJobs,
   getJobStatus,

package/controllers/bookings.js

@@ -193,6 +193,14 @@ const createScopedCatalogRefreshError = () => {
   return err;
 };
 
+const createManualCatalogRefreshReasonError = () => {
+  const err = new Error(
+    'A manual catalog refresh requires a non-blank admissionOverrideReason',
+  );
+  err.status = 400;
+  return err;
+};
+
 const createUnsupportedCatalogRefreshError = appKey => {
   const err = new Error(
     `${appKey} does not support complete product catalog refreshes`,
@@ -402,7 +410,7 @@ const $bookingsProductSearch = plugins => async ({
     requestedFullSyncTrigger,
   ) >= 0
     ? requestedFullSyncTrigger
-    : (forceRefresh ? 'scheduled' : 'organic');
+    : (forceRefresh ? 'manual' : 'organic');
   const admissionOverrideReason = typeof requestedOverrideReason === 'string'
     ? requestedOverrideReason.trim() || undefined
     : undefined;
@@ -431,6 +439,14 @@ const $bookingsProductSearch = plugins => async ({
   const isScopedSearch = hasProductSearchSelector(normalizedRequestBody);
   const isDeclaredFullCatalogRefresh = forceRefresh
     && ['scheduled', 'manual'].indexOf(fullSyncTrigger) >= 0;
+  if (
+    forceRefresh
+    && !cacheOnly
+    && fullSyncTrigger === 'manual'
+    && !admissionOverrideReason
+  ) {
+    throw createManualCatalogRefreshReasonError();
+  }
   if (isDeclaredFullCatalogRefresh && isScopedSearch) {
     throw createScopedCatalogRefreshError();
   }

package/controllers/user.js

@@ -1,16 +1,44 @@
 // controllers/user.js
-const { omit } = require('ramda');
+const { Op } = require('sequelize');
 
-const { UserAppKey } = require('../models');
+const { IntegrationLifecycle, UserAppKey } = require('../models');
+
+const pendingCleanupStatuses = ['deleting', 'external_cleanup', 'failed'];
+const integrationKey = ({ integrationId, hint }) => JSON.stringify([integrationId, hint]);
 
 const userAppList = async (req, res, next) => {
   const { params: { userId } } = req;
   try {
-    const userAppKeys = await UserAppKey.findAll({
-      where: { userId },
-      raw: true,
-      attributes: ['integrationId', 'userId', 'hint', 'createdAt', 'updatedAt'],
-    })
+    const [credentials, pendingCleanups] = await Promise.all([
+      UserAppKey.findAll({
+        where: { userId },
+        raw: true,
+        attributes: ['integrationId', 'userId', 'hint', 'createdAt', 'updatedAt'],
+      }),
+      IntegrationLifecycle.findAll({
+        where: {
+          userId,
+          status: { [Op.in]: pendingCleanupStatuses },
+        },
+        raw: true,
+        attributes: ['integrationId', 'userId', 'hint', 'status'],
+      }),
+    ]);
+    const cleanupByIntegration = new Map(
+      pendingCleanups.map(cleanup => [integrationKey(cleanup), cleanup]),
+    );
+    const userAppKeys = credentials.map(credential => {
+      const cleanup = cleanupByIntegration.get(integrationKey(credential));
+      if (!cleanup) return credential;
+      cleanupByIntegration.delete(integrationKey(credential));
+      return { ...credential, cleanupStatus: cleanup.status };
+    });
+    cleanupByIntegration.forEach(cleanup => userAppKeys.push({
+      integrationId: cleanup.integrationId,
+      userId: cleanup.userId,
+      hint: cleanup.hint,
+      cleanupStatus: cleanup.status,
+    }));
     return res.json({ userAppKeys });
   } catch (err) {
     return next(err);

package/migrations/20260914180000-create-integration-lifecycles.js

@@ -0,0 +1,67 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.createTable('IntegrationLifecycles', {
+      userId: {
+        type: Sequelize.STRING(64),
+        allowNull: false,
+        primaryKey: true,
+      },
+      integrationId: {
+        type: Sequelize.STRING(100),
+        allowNull: false,
+        primaryKey: true,
+      },
+      hint: {
+        type: Sequelize.STRING(256),
+        allowNull: false,
+        primaryKey: true,
+      },
+      generation: {
+        type: Sequelize.INTEGER.UNSIGNED,
+        allowNull: false,
+        defaultValue: 1,
+      },
+      status: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      requestId: {
+        type: Sequelize.STRING(64),
+        allowNull: false,
+      },
+      retryCount: {
+        type: Sequelize.INTEGER.UNSIGNED,
+        allowNull: false,
+        defaultValue: 0,
+      },
+      requestedBy: {
+        type: Sequelize.STRING,
+        allowNull: true,
+      },
+      requestedAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+      artifacts: {
+        type: Sequelize.JSON,
+        allowNull: true,
+      },
+      lastError: {
+        type: Sequelize.TEXT,
+        allowNull: true,
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        allowNull: false,
+      },
+    });
+  },
+
+  down: async queryInterface => {
+    await queryInterface.dropTable('IntegrationLifecycles');
+  },
+};

package/models/integrationlifecycle.js

@@ -0,0 +1,44 @@
+const Sequelize = require('sequelize');
+const db = require('./db');
+
+const IntegrationLifecycle = db.define('IntegrationLifecycle', {
+  userId: {
+    type: Sequelize.STRING(64),
+    primaryKey: true,
+  },
+  integrationId: {
+    type: Sequelize.STRING(100),
+    primaryKey: true,
+  },
+  hint: {
+    type: Sequelize.STRING(256),
+    primaryKey: true,
+  },
+  generation: {
+    type: Sequelize.INTEGER.UNSIGNED,
+    allowNull: false,
+    defaultValue: 1,
+  },
+  status: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  requestId: {
+    type: Sequelize.STRING(64),
+    allowNull: false,
+  },
+  retryCount: {
+    type: Sequelize.INTEGER.UNSIGNED,
+    allowNull: false,
+    defaultValue: 0,
+  },
+  requestedBy: Sequelize.STRING,
+  requestedAt: {
+    type: Sequelize.DATE,
+    allowNull: false,
+  },
+  artifacts: Sequelize.JSON,
+  lastError: Sequelize.TEXT,
+}, {});
+
+module.exports = IntegrationLifecycle;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ti2",
-  "version": "1.0.123",
+  "version": "1.0.125",
   "description": "Tourist Industry Exchange (TI2)",
   "main": "index.js",
   "scripts": {

package/services/__tests__/integrationLifecycle.db.js

@@ -0,0 +1,251 @@
+/* globals afterAll afterEach beforeEach describe expect it jest */
+
+jest.mock('axios', () => ({ post: jest.fn() }));
+jest.mock('../../worker/queue', () => ({ removeJob: jest.fn() }));
+
+const { v4: uuidv4 } = require('uuid');
+const axios = require('axios');
+
+const sqldb = require('../../models');
+const { removeJob } = require('../../worker/queue');
+const { deleteIntegration } = require('../integrationLifecycle');
+
+const originalEventsUrl = process.env.ti2_events2url_eventsURL;
+const originalEventsAuthorization = process.env.ti2_events2url_authorization;
+
+const deferred = () => {
+  let resolve;
+  let reject;
+  const promise = new Promise((resolvePromise, rejectPromise) => {
+    resolve = resolvePromise;
+    reject = rejectPromise;
+  });
+  return { promise, reject, resolve };
+};
+
+describe('integrationLifecycle database transitions', () => {
+  let userId;
+  const hint = 'Shared desk';
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+    process.env.ti2_events2url_eventsURL = 'http://filematch.test/ti2events';
+    process.env.ti2_events2url_authorization = 'Bearer service-token';
+    userId = `tc-1460-${uuidv4()}`;
+    await sqldb.User.create({ userId });
+  });
+
+  afterEach(async () => {
+    await sqldb.CronJobs.destroy({ where: { userId } });
+    await sqldb.UserAppKey.destroy({ where: { userId } });
+    await sqldb.UserIntegrationSettings.destroy({ where: { userId } });
+    await sqldb.IntegrationLifecycle.destroy({ where: { userId } });
+    await sqldb.User.destroy({ where: { userId } });
+  });
+
+  afterAll(() => {
+    if (originalEventsUrl === undefined) {
+      delete process.env.ti2_events2url_eventsURL;
+    } else {
+      process.env.ti2_events2url_eventsURL = originalEventsUrl;
+    }
+    if (originalEventsAuthorization === undefined) {
+      delete process.env.ti2_events2url_authorization;
+    } else {
+      process.env.ti2_events2url_authorization = originalEventsAuthorization;
+    }
+  });
+
+  const createActiveLifecycle = () => sqldb.IntegrationLifecycle.create({
+    userId,
+    integrationId: 'tourplan',
+    hint,
+    generation: 1,
+    status: 'active',
+    requestId: uuidv4(),
+    retryCount: 0,
+    requestedAt: new Date(),
+  });
+
+  it('returns 404 without lifecycle or catalog work for an unknown hint', async () => {
+    await expect(deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:test',
+    })).rejects.toMatchObject({ status: 404 });
+
+    expect(axios.post).not.toHaveBeenCalled();
+    expect(await sqldb.IntegrationLifecycle.count({ where: { userId } })).toBe(0);
+  });
+
+  it('removes only schedules owned by the deleted integration', async () => {
+    await createActiveLifecycle();
+    await sqldb.CronJobs.bulkCreate([
+      {
+        pluginName: 'tourplan',
+        pluginJobId: 'dailyReport',
+        userId,
+        hint,
+        bullJobId: 'repeat:tourplan:1',
+        cron: '0 9 * * *',
+      },
+      {
+        pluginName: 'sellouts',
+        pluginJobId: 'dailyReport',
+        userId,
+        hint,
+        bullJobId: 'repeat:sellouts:1',
+        cron: '0 9 * * *',
+      },
+    ]);
+    axios.post.mockResolvedValue({ data: { status: 'deleted' } });
+
+    await deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:test',
+    });
+
+    expect(removeJob).toHaveBeenCalledTimes(1);
+    expect(removeJob).toHaveBeenCalledWith('repeat:tourplan:1');
+    expect(await sqldb.CronJobs.count({
+      where: { userId, hint, pluginName: 'tourplan' },
+    })).toBe(0);
+    expect(await sqldb.CronJobs.count({
+      where: { userId, hint, pluginName: 'sellouts' },
+    })).toBe(1);
+  });
+
+  it('does not let a late failure overwrite external cleanup', async () => {
+    await createActiveLifecycle();
+    const successfulCatalog = deferred();
+    const failedCatalog = deferred();
+    const successfulCatalogCalled = deferred();
+    const failedCatalogCalled = deferred();
+    axios.post
+      .mockImplementationOnce(() => {
+        successfulCatalogCalled.resolve();
+        return successfulCatalog.promise;
+      })
+      .mockImplementationOnce(() => {
+        failedCatalogCalled.resolve();
+        return failedCatalog.promise;
+      });
+
+    const successfulDeletion = deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:first',
+      deferCompletion: true,
+    });
+    await successfulCatalogCalled.promise;
+    const failedDeletion = deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:second',
+      deferCompletion: true,
+    });
+    await failedCatalogCalled.promise;
+
+    successfulCatalog.resolve({ data: { status: 'deleted' } });
+    await expect(successfulDeletion).resolves.toEqual(expect.objectContaining({
+      status: 'external_cleanup',
+    }));
+    failedCatalog.reject(new Error('catalog unavailable'));
+    await expect(failedDeletion).rejects.toMatchObject({ status: 503 });
+
+    const lifecycle = await sqldb.IntegrationLifecycle.findOne({
+      where: { userId, integrationId: 'tourplan', hint },
+    });
+    expect(lifecycle.status).toBe('external_cleanup');
+  });
+
+  it('lets a successful concurrent attempt recover a failed attempt', async () => {
+    await createActiveLifecycle();
+    const failedCatalog = deferred();
+    const successfulCatalog = deferred();
+    const failedCatalogCalled = deferred();
+    const successfulCatalogCalled = deferred();
+    axios.post
+      .mockImplementationOnce(() => {
+        failedCatalogCalled.resolve();
+        return failedCatalog.promise;
+      })
+      .mockImplementationOnce(() => {
+        successfulCatalogCalled.resolve();
+        return successfulCatalog.promise;
+      });
+
+    const failedDeletion = deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:first',
+      deferCompletion: true,
+    });
+    await failedCatalogCalled.promise;
+    const successfulDeletion = deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:second',
+      deferCompletion: true,
+    });
+    await successfulCatalogCalled.promise;
+
+    failedCatalog.reject(new Error('catalog unavailable'));
+    await expect(failedDeletion).rejects.toMatchObject({ status: 503 });
+    successfulCatalog.resolve({ data: { status: 'deleted' } });
+    await expect(successfulDeletion).resolves.toEqual(expect.objectContaining({
+      status: 'external_cleanup',
+    }));
+
+    const lifecycle = await sqldb.IntegrationLifecycle.findOne({
+      where: { userId, integrationId: 'tourplan', hint },
+    });
+    expect(lifecycle.status).toBe('external_cleanup');
+  });
+
+  it('reports the persisted winner when concurrent deletes disagree on finalization', async () => {
+    await createActiveLifecycle();
+    const deferredCatalog = deferred();
+    const immediateCatalogCalled = deferred();
+    axios.post
+      .mockImplementationOnce(() => deferredCatalog.promise)
+      .mockImplementationOnce(() => {
+        immediateCatalogCalled.resolve();
+        return Promise.resolve({ data: { status: 'deleted' } });
+      });
+
+    const deferredDeletion = deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:first',
+      deferCompletion: true,
+    });
+    const immediateDeletion = deleteIntegration({
+      userId,
+      integrationId: 'tourplan',
+      hint,
+      requestedBy: 'user:second',
+      deferCompletion: false,
+    });
+    await immediateCatalogCalled.promise;
+
+    const immediateResult = await immediateDeletion;
+    deferredCatalog.resolve({ data: { status: 'deleted' } });
+    const deferredResult = await deferredDeletion;
+    const lifecycle = await sqldb.IntegrationLifecycle.findOne({
+      where: { userId, integrationId: 'tourplan', hint },
+    });
+
+    expect(immediateResult.status).toBe('complete');
+    expect(deferredResult.status).toBe('complete');
+    expect(lifecycle.status).toBe('complete');
+  });
+});