ti2 1.0.123 → 1.0.125

package/README.md

@@ -73,11 +73,12 @@ calling the integration; `cacheFound` distinguishes a cache miss from a cached
 empty catalog.
 
 A forced request is a full-catalog refresh by default. `fullSyncTrigger` may be
-`scheduled`, `manual`, or `organic`; scheduled and manual refreshes reject
-`searchInput`, `optionId`, `productId`, `productName`, or `lastUpdatedFrom`
-selectors. An empty `searchInput` or `searchInput: "*"` represents the full
-catalog. Scheduler-owned requests may also carry `fullSyncStartedAt`,
-`fullSyncAdmissionToken`, and an `admissionOverrideReason`.
+`scheduled`, `manual`, or `organic`; an omitted trigger on a forced request is
+treated as manual. Manual refreshes require a non-blank `admissionOverrideReason`.
+Scheduled and manual refreshes reject `searchInput`, `optionId`, `productId`,
+`productName`, or `lastUpdatedFrom` selectors. An empty `searchInput` or
+`searchInput: "*"` represents the full catalog. Scheduler-owned requests may
+also carry `fullSyncStartedAt` and `fullSyncAdmissionToken`.
 
 Unscoped forced responses include `catalogRefreshOutcome`, `cacheUpdated`,
 `cachePreserved`, and `cachedProductCount`. Consumers that update a downstream
@@ -93,6 +94,22 @@ The product cache refresh interval defaults to seven days when no TTR is set.
 Any explicit `ttlForProducts` or plugin `cacheSettings.bookingsProductSearch.ttr`
 value is honored, including `86400` for a one-day interval.
 
+### Optional catalog lifecycle service
+
+Ti2 synchronously asks Filematch to update catalog lifecycle state when an
+integration is removed or reactivated. The call reuses
+`ti2_events2url_eventsURL` and `ti2_events2url_authorization`; Filematch then
+forwards the request to Pyfilematch. Without the events URL, Ti2 cleans its own
+credentials and schedules without making an external catalog request.
+`INTEGRATION_LIFECYCLE_TIMEOUT_MS` controls the request timeout and defaults to
+30 seconds. A save left in `provisioning` blocks deletion until
+it is older than `INTEGRATION_PROVISIONING_TIMEOUT_MS` (five minutes by
+default); deletion then advances the generation before removing local and
+catalog state. Catalog lifecycle consumers must accept newer generations so a
+failed activation followed by deletion can move from generation N to N+1. The
+timeout is a provisioning lease: saves refresh it before catalog activation and
+must still own their generation when marking activation complete.
+
 ## Contributing
 
 Contributions are welcome and ecouraged.

package/api.yml

@@ -101,6 +101,53 @@ components:
       properties:
         value:
           type: string
+    IntegrationCleanup:
+      type: object
+      required:
+        - userId
+        - integrationId
+        - hint
+        - generation
+        - requestId
+        - status
+      properties:
+        userId:
+          type: string
+        integrationId:
+          type: string
+        hint:
+          type: string
+        generation:
+          type: integer
+          minimum: 1
+        requestId:
+          type: string
+        requestedAt:
+          type: string
+          format: date-time
+        retryCount:
+          type: integer
+          minimum: 0
+        status:
+          type: string
+          enum:
+            - deleting
+            - failed
+            - external_cleanup
+            - complete
+        artifacts:
+          type: object
+          additionalProperties: true
+    IntegrationCleanupResponse:
+      type: object
+      required:
+        - message
+        - cleanup
+      properties:
+        message:
+          type: string
+        cleanup:
+          $ref: '#/components/schemas/IntegrationCleanup'
     CreateAppTokenResponse:
       type: object
       required:
@@ -186,6 +233,9 @@ components:
           type: string
           description: An identifier for this token, it should NOT be the token itself but a short safe version of it
           example: 'expediaTestLocation-23'
+        deferCompletion:
+          type: boolean
+          description: Keep re-creation blocked until the caller finalizes external cleanup
     UserAppValidate:
       description: A user + App configuration to validate
       type: object
@@ -254,6 +304,13 @@ components:
                 type: string
               updatedAt:
                 type: string
+              cleanupStatus:
+                type: string
+                description: Present when removal is incomplete and may be retried.
+                enum:
+                  - deleting
+                  - external_cleanup
+                  - failed
     appRequest:
       type: object
       properties:
@@ -455,7 +512,7 @@ components:
         forceRefresh:
           type: boolean
           default: false
-          description: Call the plugin directly. Full-catalog refreshes require the app's weeklyProductCatalogSync capability and default to a scheduled trigger when fullSyncTrigger is omitted.
+          description: Call the plugin directly. Full-catalog refreshes require the app's weeklyProductCatalogSync capability and default to a manual trigger when fullSyncTrigger is omitted. Manual refreshes require admissionOverrideReason.
         cacheOnly:
           type: boolean
           default: false
@@ -466,14 +523,14 @@ components:
           description: Execute the request through the Ti2 worker.
         fullSyncTrigger:
           type: string
-          description: Catalog refresh owner. Scheduled and manual triggers reject selector fields.
+          description: Catalog refresh owner. Forced requests default to manual when omitted. Scheduled and manual triggers reject selector fields.
           enum:
             - organic
             - scheduled
             - manual
         admissionOverrideReason:
           type: string
-          description: Audit reason for an admitted manual refresh override.
+          description: Required non-blank audit reason for a manual refresh override.
         fullSyncStartedAt:
           type: number
           description: Start time supplied by the full-sync admission owner.
@@ -982,7 +1039,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Message'
+                $ref: '#/components/schemas/IntegrationCleanupResponse'
     put:
       tags:
         - app
@@ -1020,6 +1077,56 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Message'
+  /{app}/{userId}/cleanup/finalize:
+    post:
+      tags:
+        - app
+        - user
+      summary: Finalize an integration cleanup after external artifacts are removed
+      security:
+        - bearerAuth: ['admin', 'app', 'user']
+      operationId: finalizeIntegrationCleanup
+      parameters:
+        - name: app
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: userId
+          in: path
+          required: true
+          schema:
+            type: string
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - tokenHint
+                - generation
+                - requestId
+              properties:
+                tokenHint:
+                  type: string
+                generation:
+                  type: integer
+                  minimum: 1
+                requestId:
+                  type: string
+                  minLength: 1
+                  maxLength: 64
+                artifacts:
+                  type: object
+                  additionalProperties: true
+      responses:
+        200:
+          description: success
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IntegrationCleanupResponse'
   /{app}/{userId}/{hint}/token:
     get:
       tags:

package/auth/authHandler.js

@@ -38,6 +38,7 @@ const appCheck = async req => {
     return 'Invalid api key';
   }
   req.appRecord = appRecord.dataValues;
+  req.auth = { subject: `app:${req.pathParams.app}` };
   return undefined;
 };
 
@@ -53,6 +54,7 @@ const app = async (req, res, next) => {
 const adminCheck = async req => {
   const token = getToken({ req });
   if (token !== adminKey) return 'Invalid admin key';
+  req.auth = { subject: 'admin' };
   return undefined;
 };
 
@@ -78,6 +80,7 @@ const userCheck = async req => {
   if (req.params.userId && userId !== req.params.userId) {
     return 'Non matching user id';
   }
+  req.auth = { subject: `user:${userId}` };
   return undefined;
 };
 

package/controllers/README.md

@@ -29,6 +29,7 @@ The `$bookingsProductSearch` function in `controllers/bookings.js` implements a
     *   **Condition 3: `forceRefresh` is true**:
         *   This is the catalog refresh owner. The system fetches fresh data from the plugin.
         *   Ti2 rejects the request unless the host declared `weeklyProductCatalogSync: true` for the plugin. Manual audit reasons may override timing admission, but not catalog-completeness eligibility.
+        *   An omitted `fullSyncTrigger` is treated as `manual`, and manual refreshes require a non-blank `admissionOverrideReason`. Weekly scheduler requests must explicitly send `fullSyncTrigger: scheduled`.
         *   Unless `fullSyncTrigger: organic` is explicit, a force refresh is a scheduled or manual full-catalog request and rejects `searchInput`, `optionId`, `productId`, `productName`, and `lastUpdatedFrom` selectors. `searchInput: "*"` is unscoped.
         *   The `fetchFromPluginAndCache` helper function is invoked. This function:
             *   Sets the `pluginExecutionLockKey` before calling the plugin to prevent other concurrent direct calls.

package/controllers/__tests__/app.js

@@ -1,9 +1,11 @@
 /* globals beforeAll describe it expect */
 const chance = require('chance').Chance();
+const crypto = require('crypto');
 const jwt = require('jwt-promise');
 const R = require('ramda');
 
 const slugify = require('../../test/slugify');
+const { queue } = require('../../worker/queue');
 let appController = require('../app');
 
 const { env: { adminKey, jwtSecret } } = process;
@@ -13,6 +15,7 @@ describe('app', () => {
   const appName = slugify(
     chance.company(),
   );
+  const otherAppName = slugify(chance.company());
   const newApp = {
     name: appName,
     packageName: `ti2-${appName}`,
@@ -21,6 +24,8 @@ describe('app', () => {
   let doApiPost;
   let doApiGet;
   let doApiPut;
+  let db;
+  let plugins;
   let appKey;
   const userId = chance.guid();
   const apiKey = chance.guid();
@@ -34,8 +39,10 @@ describe('app', () => {
   };
   let encodedKey;
   beforeAll(async () => {
-    ({ doApiPost, doApiGet, doApiPut } = await testUtils({
-      plugins: [appName],
+    ({
+      doApiPost, doApiGet, doApiPut, plugins, sqldb: db,
+    } = await testUtils({
+      plugins: [appName, otherAppName],
     }));
     appController = appController([appName]);
   });
@@ -69,6 +76,12 @@ describe('app', () => {
       },
     });
     expect(parseInt(value, 10)).toBeGreaterThan(0);
+    expect(await db.models.CronJobs.count({
+      where: { userId, hint: apiKey.split('-')[0], pluginName: appName },
+    })).toBeGreaterThan(0);
+    expect(await db.models.CronJobs.count({
+      where: { userId, hint: apiKey.split('-')[0], pluginName: otherAppName },
+    })).toBe(0);
   });
   it('should be able to test a user token for the app', async () => {
     const { valid } = await doApiPost({
@@ -96,7 +109,39 @@ describe('app', () => {
           cron: '0 9 * * *',
         });
     });
-    it.todo('if a new version of the plugin has a different set of scheduled tasks, they should be re-synced');
+    it('replaces the repeatable definition when a cron pattern changes', async () => {
+      const plugin = plugins.find(({ name }) => name === appName);
+      plugin.jobs[0].cron = '0 10 * * *';
+
+      await doApiPost({
+        url: `/${appName}/${userId}`,
+        token: appKey,
+        payload: {
+          tokenHint: apiKey.split('-')[0],
+          token,
+        },
+      });
+
+      const jobs = R.path(
+        ['jobs'],
+        await appController.getAppScheduledJobs({ integrationId: appName, userId }),
+      );
+      expect(R.head(R.project(['pluginJobId', 'cron'], jobs))).toEqual({
+        pluginJobId: 'dailyReport',
+        cron: '0 10 * * *',
+      });
+      const repeatId = crypto.createHash('sha256').update(JSON.stringify({
+        pluginName: appName,
+        pluginJobId: 'dailyReport',
+        userId,
+        hint: apiKey.split('-')[0],
+      })).digest('hex');
+      const repeatDefinitions = (await queue.getRepeatableJobs())
+        .filter(({ id }) => id === repeatId);
+      expect(repeatDefinitions).toEqual([
+        expect.objectContaining({ cron: '0 10 * * *' }),
+      ]);
+    });
     let jobId;
     it('should be able to run a job', async () => {
       let status;

package/controllers/__tests__/bookings-searchProducts.js

@@ -134,7 +134,7 @@ describe('user: bookings controller - searchProducts', () => {
       const response = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${testHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true },
+        payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
         expectStatusCode: 500,
       });
 
@@ -146,6 +146,7 @@ describe('user: bookings controller - searchProducts', () => {
       // NOTE: we SHOULD NOT need to remove the cache first, since we are forceRefreshing, we are testing the endpoint get's called while having a cache created
       const payload = {
         forceRefresh: true,
+        fullSyncTrigger: 'scheduled',
       };
       const { products } = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${testHint}/search`,
@@ -253,7 +254,10 @@ describe('user: bookings controller - searchProducts', () => {
         await doApiPost({
           url: `/products/${testAppName}/${testUserId}/${testHint}/search`,
           token: userToken,
-          payload: { forceRefresh: true },
+          payload: {
+            forceRefresh: true,
+            admissionOverrideReason: 'legacy manual refresh',
+          },
         });
         const cacheSavePayload = emitSpy.mock.calls
           .filter(call => call[0] === 'bookingsProductSearch:cache:save')
@@ -265,6 +269,8 @@ describe('user: bookings controller - searchProducts', () => {
         expect(cacheSavePayload.hint).toBe(testHint);
         expect(cacheSavePayload.userIdHash).toBeTruthy();
         expect(cacheSavePayload.hintHash).toBeTruthy();
+        expect(cacheSavePayload.fullSyncTrigger).toBe('manual');
+        expect(cacheSavePayload.admissionOverrideReason).toBe('legacy manual refresh');
         emitSpy.mockRestore();
       });
 
@@ -355,7 +361,7 @@ describe('user: bookings controller - searchProducts', () => {
         await doApiPost({
           url: `/products/${testAppName}/${testUserId}/${doNotCallHint}/search`,
           token: userToken,
-          payload: { forceRefresh: true },
+          payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
         }).then(({ products: p }) => {
           products = p; // Assign to the outer scoped 'products'
         });
@@ -415,7 +421,11 @@ describe('user: bookings controller - searchProducts', () => {
         await doApiPost({
           url: `/products/${testAppName}/${testUserId}/${configuredOmitHint}/search`,
           token: userToken,
-          payload: { forceRefresh: true, omitServiceCodes: [] },
+          payload: {
+            forceRefresh: true,
+            fullSyncTrigger: 'scheduled',
+            omitServiceCodes: [],
+          },
         });
 
         expect(travelgatePlugin.searchProducts).toHaveBeenCalledTimes(1);
@@ -642,7 +652,7 @@ describe('user: bookings controller - searchProducts', () => {
       const result = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${staleCacheTestHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true },
+        payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
       });
 
       const cached = await cache.get({ pluginName: testAppName, key: cacheKeyForTest });
@@ -687,7 +697,7 @@ describe('user: bookings controller - searchProducts', () => {
       await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${staleCacheTestHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true },
+        payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
       });
 
       const cached = await cache.get({ pluginName: testAppName, key: cacheKeyForTest });
@@ -725,7 +735,7 @@ describe('user: bookings controller - searchProducts', () => {
       const result = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${staleCacheTestHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true },
+        payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
       });
 
       const cached = await cache.get({ pluginName: testAppName, key: cacheKeyForTest });
@@ -764,15 +774,22 @@ describe('user: bookings controller - searchProducts', () => {
       },
     );
 
-    it('rejects a forced selector when fullSyncTrigger is omitted', async () => {
+    it.each([
+      ['an omitted trigger', { forceRefresh: true }],
+      ['a blank reason', {
+        forceRefresh: true,
+        fullSyncTrigger: 'manual',
+        admissionOverrideReason: '  ',
+      }],
+    ])('rejects a manual catalog refresh with %s', async (_description, payload) => {
       const response = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${staleCacheTestHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true, optionId: 'option-1' },
+        payload,
         expectStatusCode: 400,
       });
 
-      expect(response.message).toContain('must not include product selectors');
+      expect(response.message).toContain('requires a non-blank admissionOverrideReason');
       expect(travelgatePlugin.searchProducts).not.toHaveBeenCalled();
     });
 
@@ -847,7 +864,7 @@ describe('user: bookings controller - searchProducts', () => {
       const result = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${configuredHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true },
+        payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
       });
 
       expect(result).toEqual(expect.objectContaining({
@@ -1130,7 +1147,7 @@ describe('Bookings Product Search Lock Mechanism (Job Queuing on Stale Cache)',
     const makeForceRequest = () => doApiPost({
       url: `/products/${testAppName}/${testUserId}/${ttrTestHint}/search`,
       token: userToken,
-      payload: { forceRefresh: true },
+      payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
     });
 
     const requestPromises = [makeForceRequest(), makeForceRequest(), makeForceRequest()];
@@ -1185,7 +1202,7 @@ describe('Bookings Product Search Lock Mechanism (Job Queuing on Stale Cache)',
     const makeForceRequest = () => doApiPost({
       url: `/products/${testAppName}/${testUserId}/${ttrTestHint}/search`,
       token: userToken,
-      payload: { forceRefresh: true },
+      payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
     });
 
     const requestPromises = [makeForceRequest(), makeForceRequest(), makeForceRequest()];
@@ -1236,7 +1253,7 @@ describe('Bookings Product Search Lock Mechanism (Job Queuing on Stale Cache)',
       const result = await doApiPost({
         url: `/products/${testAppName}/${testUserId}/${ttrTestHint}/search`,
         token: userToken,
-        payload: { forceRefresh: true },
+        payload: { forceRefresh: true, fullSyncTrigger: 'scheduled' },
       });
 
       expect(result).toEqual(expect.objectContaining({

package/controllers/__tests__/user.js

@@ -97,11 +97,25 @@ describe('user', () => {
     expect(template).toBeTruthy();
     expect(R.path(['apiKey', 'regExp', 'source'], template)).toBeTruthy();
   });
+  it('returns 404 when deleting a hint with no credential or lifecycle', async () => {
+    const unknownHint = chance.word({ length: 12 });
+    await doApiDelete({
+      url: `/${appName}/${userId}`,
+      token: userKey,
+      payload: { tokenHint: unknownHint },
+      expectStatusCode: 404,
+    });
+
+    expect(await db.models.IntegrationLifecycle.count({
+      where: { userId, integrationId: appName, hint: unknownHint },
+    })).toBe(0);
+  });
   it('should be able to delete a user/app key', async () => {
+    const deletedHint = apiKey.split('-')[0];
     await doApiDelete({
       url: `/${appName}/${userId}`,
       token: userKey,
-      payload: { tokenHint: apiKey.split('-')[0] },
+      payload: { tokenHint: deletedHint },
     });
     const { userAppKeys } = await doApiGet({
       url: `/user/${userId}/apps`,
@@ -116,6 +130,14 @@ describe('user', () => {
         }),
       ]),
     );
+    const lifecycle = await db.models.IntegrationLifecycle.findOne({
+      where: {
+        userId,
+        integrationId: appName,
+        hint: deletedHint,
+      },
+    });
+    expect(lifecycle.requestedBy).toBe(`user:${userId}`);
   });
   it('should be able to create a user/app integration', async () => {
     // set up new token
@@ -145,6 +167,70 @@ describe('user', () => {
       ]),
     );
   });
+  it('reports deferred deletion as pending until tokenHint finalization', async () => {
+    const deferredHint = chance.word({ length: 12 });
+    await doApiPost({
+      url: `/${appName}/${userId}`,
+      token: userKey,
+      payload: { tokenHint: deferredHint, token },
+    });
+
+    const deletion = await doApiDelete({
+      url: `/${appName}/${userId}`,
+      token: userKey,
+      payload: { tokenHint: deferredHint, deferCompletion: true },
+    });
+    expect(deletion).toEqual(expect.objectContaining({
+      message: 'Integration cleanup is awaiting external finalization.',
+      cleanup: expect.objectContaining({ status: 'external_cleanup' }),
+    }));
+
+    const pendingList = await doApiGet({
+      url: `/user/${userId}/apps`,
+      token: userKey,
+    });
+    expect(pendingList.userAppKeys).toEqual(expect.arrayContaining([
+      expect.objectContaining({
+        integrationId: appName,
+        userId,
+        hint: deferredHint,
+        cleanupStatus: 'external_cleanup',
+      }),
+    ]));
+
+    await doApiPost({
+      url: `/${appName}/${userId}/cleanup/finalize`,
+      token: userKey,
+      payload: {
+        tokenHint: deferredHint,
+        generation: deletion.cleanup.generation,
+      },
+      expectStatusCode: 400,
+    });
+
+    const finalized = await doApiPost({
+      url: `/${appName}/${userId}/cleanup/finalize`,
+      token: userKey,
+      payload: {
+        tokenHint: deferredHint,
+        generation: deletion.cleanup.generation,
+        requestId: deletion.cleanup.requestId,
+        artifacts: { test: true },
+      },
+    });
+    expect(finalized.cleanup.status).toBe('complete');
+
+    const completedList = await doApiGet({
+      url: `/user/${userId}/apps`,
+      token: userKey,
+    });
+    expect(completedList.userAppKeys).toEqual(expect.not.arrayContaining([
+      expect.objectContaining({
+        integrationId: appName,
+        hint: deferredHint,
+      }),
+    ]));
+  });
   it('reports post-save cron setup failure while preserving the committed credential', async () => {
     const postCommitUserId = chance.guid();
     const postCommitUserKey = (await doApiPost({