ti2 1.0.124 → 1.0.125

package/services/integrationLifecycle.js

@@ -0,0 +1,496 @@
+const axios = require('axios');
+const crypto = require('crypto');
+const { Op } = require('sequelize');
+
+const sqldb = require('../models');
+const { removeJob } = require('../worker/queue');
+
+const DEFAULT_PROVISIONING_TIMEOUT_MS = 5 * 60e3;
+
+const identityWhere = ({ userId, integrationId, hint }) => ({
+  userId,
+  integrationId,
+  hint,
+});
+
+const lifecycleError = (status, message) => {
+  const error = new Error(message);
+  error.status = status;
+  return error;
+};
+
+const newRequestId = () => crypto.randomBytes(16).toString('hex');
+
+const provisioningTimeoutMs = () => {
+  const configured = Number(process.env.INTEGRATION_PROVISIONING_TIMEOUT_MS);
+  return Number.isFinite(configured) && configured > 0
+    ? configured
+    : DEFAULT_PROVISIONING_TIMEOUT_MS;
+};
+
+const provisioningIsStale = lifecycle => {
+  const lastUpdatedAt = lifecycle.updatedAt || lifecycle.requestedAt;
+  const lastUpdatedMs = new Date(lastUpdatedAt).getTime();
+  return Number.isFinite(lastUpdatedMs)
+    && Date.now() - lastUpdatedMs >= provisioningTimeoutMs();
+};
+
+const catalogLifecycleEndpoint = () => {
+  const eventsUrl = process.env.ti2_events2url_eventsURL;
+  if (!eventsUrl) return null;
+  const endpoint = new URL(eventsUrl);
+  endpoint.pathname = '/productSync/integration-lifecycle';
+  endpoint.search = '';
+  endpoint.hash = '';
+  return endpoint.toString();
+};
+
+const callCatalogLifecycle = async ({
+  action,
+  userId,
+  integrationId,
+  hint,
+  generation,
+}) => {
+  const endpoint = catalogLifecycleEndpoint();
+  if (!endpoint) {
+    return {
+      status: action === 'activate' ? 'active' : 'deleted',
+      generation,
+      skipped: true,
+      reason: 'catalog_lifecycle_not_configured',
+    };
+  }
+  try {
+    const response = await axios.post(
+      endpoint,
+      {
+        action,
+        companyId: userId,
+        integrationId,
+        hint,
+        generation,
+      },
+      {
+        headers: {
+          Authorization: process.env.ti2_events2url_authorization,
+        },
+        timeout: Number(process.env.INTEGRATION_LIFECYCLE_TIMEOUT_MS) || 30e3,
+      },
+    );
+    return response.data;
+  } catch (error) {
+    const responseData = error.response && error.response.data;
+    if (responseData && ['retry', 'superseded'].includes(responseData.status)) {
+      return responseData;
+    }
+    throw error;
+  }
+};
+
+const prepareActivation = async ({
+  userId,
+  integrationId,
+  hint,
+  transaction,
+}) => {
+  const where = identityWhere({ userId, integrationId, hint });
+  let lifecycle = await sqldb.IntegrationLifecycle.findOne({
+    where,
+    transaction,
+    lock: transaction.LOCK.UPDATE,
+  });
+  if (lifecycle && ['deleting', 'external_cleanup', 'failed'].includes(lifecycle.status)) {
+    throw lifecycleError(409, 'Integration removal must finish before this integration can be saved again.');
+  }
+  if (!lifecycle) {
+    lifecycle = await sqldb.IntegrationLifecycle.create({
+      ...where,
+      generation: 1,
+      status: 'provisioning',
+      requestId: newRequestId(),
+      retryCount: 0,
+      requestedAt: new Date(),
+      artifacts: { requiresCatalogActivation: false },
+    }, { transaction });
+  } else if (lifecycle.status === 'complete') {
+    lifecycle.generation += 1;
+    lifecycle.status = 'provisioning';
+    lifecycle.requestId = newRequestId();
+    lifecycle.retryCount = 0;
+    lifecycle.requestedAt = new Date();
+    lifecycle.lastError = null;
+    lifecycle.artifacts = { requiresCatalogActivation: true };
+    await lifecycle.save({ transaction });
+  } else {
+    lifecycle.status = 'provisioning';
+    lifecycle.lastError = null;
+    if (!lifecycle.artifacts || typeof lifecycle.artifacts !== 'object') {
+      lifecycle.artifacts = { requiresCatalogActivation: false };
+    }
+    await lifecycle.save({ transaction });
+  }
+  return lifecycle;
+};
+
+const activateCatalog = async lifecycle => {
+  const result = await callCatalogLifecycle({
+    action: 'activate',
+    userId: lifecycle.userId,
+    integrationId: lifecycle.integrationId,
+    hint: lifecycle.hint,
+    generation: lifecycle.generation,
+  });
+  if (result && result.status === 'active') return result;
+  if (result && result.status === 'superseded') {
+    throw lifecycleError(409, 'A newer integration lifecycle already exists.');
+  }
+  if (result && result.status === 'retry') {
+    throw lifecycleError(503, 'Product catalog activation is not ready and can be retried.');
+  }
+  throw lifecycleError(502, 'Product catalog activation returned an unexpected response.');
+};
+
+const activationWhere = lifecycle => ({
+  ...identityWhere(lifecycle),
+  generation: lifecycle.generation,
+  status: 'provisioning',
+});
+
+const assertActivationOwnership = updatedRows => {
+  if (updatedRows !== 1) {
+    throw lifecycleError(409, 'Integration save was superseded by a newer lifecycle.');
+  }
+};
+
+const touchActivation = async lifecycle => {
+  const [updatedRows] = await sqldb.IntegrationLifecycle.update({
+    updatedAt: new Date(),
+  }, {
+    where: activationWhere(lifecycle),
+  });
+  if (updatedRows === 1) return;
+  // MySQL reports zero changed rows when a sub-second heartbeat rounds to the
+  // same stored timestamp, so confirm the fenced row before treating it as lost.
+  const current = await sqldb.IntegrationLifecycle.findOne({
+    where: activationWhere(lifecycle),
+  });
+  if (!current) assertActivationOwnership(updatedRows);
+};
+
+const completeActivation = async lifecycle => {
+  const [updatedRows] = await sqldb.IntegrationLifecycle.update({
+    status: 'active',
+    artifacts: null,
+    lastError: null,
+  }, {
+    where: activationWhere(lifecycle),
+  });
+  assertActivationOwnership(updatedRows);
+};
+
+const failActivation = (lifecycle, error) => sqldb.IntegrationLifecycle.update({
+  status: 'failed_activation',
+  lastError: error.message,
+}, {
+  where: {
+    ...identityWhere(lifecycle),
+    generation: lifecycle.generation,
+    status: 'provisioning',
+  },
+});
+
+const beginDeletion = async ({
+  userId,
+  integrationId,
+  hint,
+  requestedBy,
+}) => {
+  const where = identityWhere({ userId, integrationId, hint });
+  return sqldb.sequelize.transaction(async transaction => {
+    await sqldb.User.findOne({
+      where: { userId },
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+    });
+    let lifecycle = await sqldb.IntegrationLifecycle.findOne({
+      where,
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+    });
+    if (lifecycle && ['complete', 'external_cleanup'].includes(lifecycle.status)) {
+      return lifecycle;
+    }
+    if (
+      lifecycle
+      && lifecycle.status === 'provisioning'
+      && !provisioningIsStale(lifecycle)
+    ) {
+      throw lifecycleError(409, 'Integration save is still finishing. Retry removal shortly.');
+    }
+    if (!lifecycle) {
+      const credential = await sqldb.UserAppKey.findOne({
+        where,
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+      if (!credential) throw lifecycleError(404, 'Integration credential not found.');
+      lifecycle = await sqldb.IntegrationLifecycle.create({
+        ...where,
+        generation: 1,
+        status: 'deleting',
+        requestId: newRequestId(),
+        retryCount: 0,
+        requestedBy,
+        requestedAt: new Date(),
+      }, { transaction });
+      return lifecycle;
+    }
+    if (
+      ['active', 'failed_activation'].includes(lifecycle.status)
+      || (lifecycle.status === 'provisioning' && provisioningIsStale(lifecycle))
+    ) {
+      // Deletion intentionally advances beyond a failed/stale activation. The catalog
+      // contract accepts a higher generation so an activation that committed remotely
+      // before TI2 crashed cannot resurrect the integration.
+      lifecycle.generation += 1;
+      lifecycle.requestId = newRequestId();
+      lifecycle.retryCount = 0;
+      lifecycle.requestedAt = new Date();
+    } else {
+      lifecycle.retryCount += 1;
+    }
+    lifecycle.status = 'deleting';
+    lifecycle.requestedBy = requestedBy;
+    lifecycle.lastError = null;
+    await lifecycle.save({ transaction });
+    return lifecycle;
+  });
+};
+
+const cleanupSchedules = async ({
+  userId,
+  integrationId,
+  hint,
+}) => {
+  const where = {
+    userId,
+    hint,
+    pluginName: integrationId,
+  };
+  return sqldb.sequelize.transaction(async transaction => {
+    const schedules = await sqldb.CronJobs.findAll({
+      where,
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+    });
+    /* eslint-disable no-await-in-loop */
+    // Sequential removal keeps each Bull deletion paired with its locked database row.
+    for (const schedule of schedules) { // eslint-disable-line no-restricted-syntax
+      const references = await sqldb.CronJobs.findAll({
+        where: { bullJobId: schedule.bullJobId },
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+      if (references.length === 1) await removeJob(schedule.bullJobId);
+      await schedule.destroy({ transaction });
+    }
+    /* eslint-enable no-await-in-loop */
+    return schedules.length;
+  });
+};
+
+const cleanupCredentialAndSettings = ({ userId, integrationId, hint }) => (
+  sqldb.sequelize.transaction(async transaction => {
+    await sqldb.User.findOne({
+      where: { userId },
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+    });
+    const credentials = await sqldb.UserAppKey.destroy({
+      where: identityWhere({ userId, integrationId, hint }),
+      transaction,
+    });
+    const remainingHints = await sqldb.UserAppKey.count({
+      where: { userId, integrationId },
+      transaction,
+    });
+    const settings = remainingHints === 0
+      ? await sqldb.UserIntegrationSettings.destroy({
+        where: { userId, integrationId },
+        transaction,
+      })
+      : 0;
+    return { credentials, settings };
+  })
+);
+
+const deleteIntegration = async ({
+  userId,
+  integrationId,
+  hint,
+  requestedBy,
+  deferCompletion = false,
+}) => {
+  const lifecycle = await beginDeletion({
+    userId,
+    integrationId,
+    hint,
+    requestedBy,
+  });
+  if (['complete', 'external_cleanup'].includes(lifecycle.status)) {
+    return lifecycle.get({ plain: true });
+  }
+
+  const where = identityWhere({ userId, integrationId, hint });
+  let failureStage = 'schedule cleanup';
+  let observedArtifacts = lifecycle.artifacts || {};
+  try {
+    const cronJobs = await cleanupSchedules({ userId, integrationId, hint });
+    const previousArtifacts = { ...observedArtifacts };
+    delete previousArtifacts.failureStage;
+    observedArtifacts = {
+      ...previousArtifacts,
+      cronJobs: Math.max(previousArtifacts.cronJobs || 0, cronJobs),
+    };
+    await sqldb.IntegrationLifecycle.update({ artifacts: observedArtifacts }, {
+      where: {
+        ...where,
+        generation: lifecycle.generation,
+        status: { [Op.in]: ['deleting', 'failed'] },
+      },
+    });
+    failureStage = 'credential and settings cleanup';
+    const { credentials, settings } = await cleanupCredentialAndSettings({
+      userId,
+      integrationId,
+      hint,
+    });
+    observedArtifacts = {
+      ...observedArtifacts,
+      credentials: Math.max(observedArtifacts.credentials || 0, credentials),
+      settings: Math.max(observedArtifacts.settings || 0, settings),
+    };
+    await sqldb.IntegrationLifecycle.update({ artifacts: observedArtifacts }, {
+      where: {
+        ...where,
+        generation: lifecycle.generation,
+        status: { [Op.in]: ['deleting', 'failed'] },
+      },
+    });
+    failureStage = 'product catalog cleanup';
+    const catalog = await callCatalogLifecycle({
+      action: 'delete',
+      userId,
+      integrationId,
+      hint,
+      generation: lifecycle.generation,
+    });
+    if (catalog && catalog.status === 'superseded') {
+      throw lifecycleError(409, 'A newer product catalog lifecycle already exists.');
+    }
+    if (catalog && catalog.status === 'retry') {
+      throw lifecycleError(503, 'Product catalog cleanup is not ready and can be retried.');
+    }
+    if (!catalog || catalog.status !== 'deleted') {
+      throw lifecycleError(502, 'Product catalog cleanup returned an unexpected response.');
+    }
+    const artifacts = {
+      ...observedArtifacts,
+      catalog,
+    };
+    const status = deferCompletion ? 'external_cleanup' : 'complete';
+    const [updatedRows] = await sqldb.IntegrationLifecycle.update({
+      status,
+      artifacts,
+      lastError: null,
+    }, {
+      where: {
+        ...where,
+        generation: lifecycle.generation,
+        status: { [Op.in]: ['deleting', 'failed'] },
+      },
+    });
+    if (updatedRows === 0) {
+      const persisted = await sqldb.IntegrationLifecycle.findOne({ where });
+      if (
+        persisted
+        && persisted.generation === lifecycle.generation
+        && ['complete', 'external_cleanup'].includes(persisted.status)
+      ) {
+        return persisted.get({ plain: true });
+      }
+      throw lifecycleError(409, 'Integration cleanup state changed before completion.');
+    }
+    return {
+      ...where,
+      generation: lifecycle.generation,
+      requestId: lifecycle.requestId,
+      requestedAt: lifecycle.requestedAt,
+      retryCount: lifecycle.retryCount,
+      status,
+      artifacts,
+    };
+  } catch (error) {
+    await sqldb.IntegrationLifecycle.update({
+      status: 'failed',
+      artifacts: { ...observedArtifacts, failureStage },
+      lastError: error.message,
+    }, {
+      where: {
+        ...where,
+        generation: lifecycle.generation,
+        status: 'deleting',
+      },
+    });
+    if (error.status) throw error;
+    throw lifecycleError(
+      503,
+      `Integration cleanup stopped at ${failureStage} and can be retried: ${error.message}`,
+    );
+  }
+};
+
+const completeDeletion = async ({
+  userId,
+  integrationId,
+  hint,
+  generation,
+  requestId,
+  externalArtifacts,
+}) => sqldb.sequelize.transaction(async transaction => {
+  const where = identityWhere({ userId, integrationId, hint });
+  const lifecycle = await sqldb.IntegrationLifecycle.findOne({
+    where,
+    transaction,
+    lock: transaction.LOCK.UPDATE,
+  });
+  if (!lifecycle || lifecycle.generation !== generation) {
+    throw lifecycleError(409, 'Integration cleanup generation no longer matches.');
+  }
+  if (lifecycle.requestId !== requestId) {
+    throw lifecycleError(409, 'Integration cleanup request no longer owns this generation.');
+  }
+  if (lifecycle.status === 'complete') return lifecycle.get({ plain: true });
+  if (lifecycle.status !== 'external_cleanup') {
+    throw lifecycleError(409, 'Integration cleanup is not ready to finalize.');
+  }
+  lifecycle.status = 'complete';
+  lifecycle.artifacts = {
+    ...(lifecycle.artifacts || {}),
+    external: externalArtifacts || {},
+  };
+  lifecycle.lastError = null;
+  await lifecycle.save({ transaction });
+  return lifecycle.get({ plain: true });
+});
+
+module.exports = {
+  activateCatalog,
+  completeActivation,
+  completeDeletion,
+  deleteIntegration,
+  failActivation,
+  prepareActivation,
+  touchActivation,
+};

package/worker/__tests__/queue-remove.js

@@ -0,0 +1,77 @@
+/* globals beforeEach describe expect it jest */
+
+const crypto = require('crypto');
+
+const mockQueue = {
+  getJob: jest.fn(),
+  getRepeatableJobs: jest.fn(),
+  on: jest.fn(),
+  removeRepeatableByKey: jest.fn(),
+};
+const mockRedis = { del: jest.fn() };
+
+jest.mock('bull', () => jest.fn(() => mockQueue));
+jest.mock('ioredis', () => jest.fn(() => mockRedis));
+
+jest.resetModules();
+const { removeJob } = require('../queue');
+
+const occurrenceId = ({
+  name, id, key, timestamp = 1700000000000,
+}) => {
+  const namespace = crypto.createHash('md5').update(key).digest('hex');
+  const repeatHash = crypto.createHash('md5')
+    .update(`${name}${id ? `${id}:` : ':'}${namespace}`)
+    .digest('hex');
+  return `repeat:${repeatHash}:${timestamp}`;
+};
+
+describe('removeJob', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockQueue.getJob.mockResolvedValue(null);
+  });
+
+  it('removes only the repeat definition that owns the stored occurrence id', async () => {
+    const target = {
+      name: '__default__',
+      id: 'company-a-tourplan-desk-a',
+      key: '__default__:company-a-tourplan-desk-a:::0 9 * * *',
+    };
+    const sameCronOtherIntegration = {
+      name: '__default__',
+      id: 'company-a-tourplan-desk-b',
+      key: '__default__:company-a-tourplan-desk-b:::0 9 * * *',
+    };
+    mockQueue.getRepeatableJobs.mockResolvedValue([sameCronOtherIntegration, target]);
+    const jobId = occurrenceId(target);
+
+    await removeJob(jobId);
+
+    expect(mockQueue.removeRepeatableByKey).toHaveBeenCalledTimes(1);
+    expect(mockQueue.removeRepeatableByKey).toHaveBeenCalledWith(target.key);
+    expect(mockRedis.del).toHaveBeenCalledWith(jobId);
+  });
+
+  it('treats an already-removed repeat definition as an idempotent success', async () => {
+    mockQueue.getRepeatableJobs.mockResolvedValue([]);
+
+    await expect(removeJob('repeat:missing:1700000000000')).resolves.toBeUndefined();
+
+    expect(mockQueue.removeRepeatableByKey).not.toHaveBeenCalled();
+    expect(mockRedis.del).toHaveBeenCalledWith('repeat:missing:1700000000000');
+  });
+
+  it('removes a repeat definition referenced by a legacy custom job id', async () => {
+    const target = {
+      id: 'company-a-tourplan-desk-a',
+      key: '__default__:company-a-tourplan-desk-a:::0 9 * * *',
+    };
+    mockQueue.getRepeatableJobs.mockResolvedValue([target]);
+
+    await removeJob(target.id);
+
+    expect(mockQueue.removeRepeatableByKey).toHaveBeenCalledWith(target.key);
+    expect(mockRedis.del).toHaveBeenCalledWith(target.id);
+  });
+});

package/worker/queue.js

@@ -1,8 +1,8 @@
 const Queue = require('bull');
+const crypto = require('crypto');
 const R = require('ramda');
 const Redis = require('ioredis');
-const sqldb = require('../models');
-const { env: { redisHost, redisPort } } = process;
+
 const REDIS_URL = process.env.REDIS_URL || 'redis://redis:6379';
 const itemsTTL = 3 * 60 * 60; // 3 hours
 
@@ -40,71 +40,60 @@ const addJob = async (payload, paramsParam) => {
     removeOnComplete: true,
     ...params,
   });
-  // For repeat jobs, we need to store the full repeat key
-  const id = job.opts.repeat ? job.opts.jobId : job.id;
-  return id;
+  // Repeat jobs use an occurrence ID containing Bull's repeat-definition hash.
+  // Persisting it lets removeJob identify one exact definition without matching by cron.
+  return job.id;
 };
 
 const saveResult = async ({ id, resultValue }) => {
   await redisResults.set(id, JSON.stringify(resultValue), 'EX', itemsTTL);
 };
 
-const removeJob = async (jobId) => {
-  // For repeat jobs, the ID is in the format 'repeat:jobId:timestamp'
+const removeJob = async jobId => {
+  if (!jobId) return;
+  // Bull occurrence IDs use the format 'repeat:<repeat-definition-hash>:<timestamp>'.
   const jobIdParts = jobId.split(':');
   const isRepeatJob = jobIdParts[0] === 'repeat';
 
   if (isRepeatJob) {
-    // Get all repeatable jobs
-    const repeatableJobs = await queue.getRepeatableJobs();
-
-    // Find the job with matching cron pattern
-    const cronJob = await sqldb.CronJobs.findOne({
-      where: {
-        bullJobId: jobId,
-      },
-    });
-
-    if (cronJob) {
-      // First remove all repeatable jobs with matching cron pattern
-      for (const repeatableJob of repeatableJobs) {
-        if (repeatableJob.cron === cronJob.cron) {
-          await queue.removeRepeatableByKey(repeatableJob.key);
-        }
-      }
-
-      // Then remove all jobs with this pattern
-      const jobs = await queue.getJobs(['active', 'wait', 'delayed']);
-      for (const job of jobs) {
-        if (job.opts.repeat && job.opts.jobId === jobId) {
-          await job.remove();
-        }
-      }
-
-      // Wait for the queue to process the removals
-      await new Promise(resolve => setTimeout(resolve, 500));
-
-      // Clean up any remaining jobs
-      const remainingJobs = await queue.getJobs(['active', 'wait', 'delayed']);
-      for (const job of remainingJobs) {
-        if (job.opts.repeat && job.opts.jobId === jobId) {
-          await job.remove();
-        }
+    const job = await queue.getJob(jobId);
+    let repeatKey = R.path(['opts', 'repeat', 'key'], job);
+    if (!repeatKey) {
+      // Bull v4 does not expose occurrence-to-definition lookup. Recreate its
+      // internal hash only for legacy rows whose scheduled occurrence has expired.
+      const repeatHash = jobIdParts[1];
+      const repeatableJobs = await queue.getRepeatableJobs();
+      const repeatable = repeatableJobs.find(candidate => {
+        const candidateJobId = candidate.id ? `${candidate.id}:` : ':';
+        const namespace = crypto.createHash('md5').update(candidate.key).digest('hex');
+        const candidateHash = crypto.createHash('md5')
+          .update(`${candidate.name}${candidateJobId}${namespace}`)
+          .digest('hex');
+        return candidateHash === repeatHash;
+      });
+      repeatKey = repeatable && repeatable.key;
+    }
+    if (repeatKey) await queue.removeRepeatableByKey(repeatKey);
+    if (job) {
+      try {
+        await job.remove();
+      } catch (error) {
+        if (error.message && !error.message.includes('not found')) throw error;
       }
-
-      await redisResults.del(jobId);
-      return;
     }
-
-    // If we couldn't find the repeatable job, just delete from the database
-    // This can happen if the job was already removed from Bull but still exists in our database
+    await redisResults.del(jobId);
     return;
   }
 
   // If not a repeatable job, try to remove as a regular job
   const job = await queue.getJob(jobId);
   if (!job) {
-    throw new Error('Job not found');
+    // Older CronJobs rows stored opts.jobId instead of the repeat occurrence ID.
+    const repeatableJobs = await queue.getRepeatableJobs() || [];
+    const repeatable = repeatableJobs.find(candidate => candidate.id === jobId);
+    if (repeatable) await queue.removeRepeatableByKey(repeatable.key);
+    await redisResults.del(jobId);
+    return;
   }
   await job.remove();
   await redisResults.del(jobId);