ti2 1.0.16 → 1.0.17

package/.eslintrc.js

@@ -42,4 +42,5 @@ module.exports = {
       },
     ],
   },
+  ignorePatterns: ['**/docs/*'],
 };

package/.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     timeout-minutes: 10
     runs-on: ubuntu-latest
     container:
-      image: node:erbium
+      image: node:16.15.0
     services:
       mysql:
         image: mysql
@@ -17,9 +17,12 @@ jobs:
           MYSQL_ROOT_PASSWORD: root
         ports:
           - 3306:3306
+      redis:
+        image: redis
     env:
       CI: true
       DB_URL: 'mysql://root:root@mysql/ti2development'
+      REDIS_URL: 'redis://redis:6379'
       dbCryptoKey: 'YmYzYzE5OWMyNDcwY2I0NzdkOTA3YjFlMDkxN2MxN2I='
       jwtSecret: '9rifawckwztuKehocJOtGM6deTqICGWzIyc5CqgNyi'
       frontendKey: 'lYjEJHzb7lbQ1hk3ypjX'
@@ -30,10 +33,11 @@ jobs:
       with:
         node-version: 12
     - name: install dependencies
-      run: npm i
+      run: npm ci
     - name: create/migrate the database
       run: npx sequelize db:create && npx sequelize db:migrate
     - name: seed the database
       run: npx sequelize db:seed:all
+    - run: RUNNER_TRACKING_ID="" && (nohup node ./test/worker.js&)
     - name: run the tests
       run: npx jest -i --coverage --forceExit

package/.nvmrc

@@ -0,0 +1 @@
+v16.15.0

package/api.yml

@@ -29,6 +29,34 @@ components:
       properties:
         value:
           type: string
+    JobSpec:
+      type: object
+      properties:
+        payload:
+          type: object
+          description: The job app parameters
+          required: true
+          example: '{
+            "method": "dailyReport"
+          }'
+        jobParams:
+          type: object
+          description: Additional bull job params
+          example: '{
+            "delay": "5000"
+          }'
+    JobStatus:
+      type: object
+      properties:
+        jobId:
+          type: string
+        status:
+          type: string
+          example: 'active'
+        result:
+          type: object
+          description: The job return values
+          example: '{ "result": "value" }'
     App:
       type: object
       properties:
@@ -534,6 +562,128 @@ paths:
           content:
             application/json:
               schema: '#/components/schemas/Message'
+
+  /{app}/{userId}/jobRun:
+    security:
+      - bearerAuth: ['app', 'user', 'admin']
+    post:
+      tags:
+        - jobs
+      summary: Run a Job
+      security:
+        - bearerAuth: ['app', 'user', 'admin']
+      operationId: runAppJob
+      parameters:
+        - name: app
+          in: path
+          description: app that defines the job
+          example: 'ventrata'
+          required: true
+          schema:
+            type: string
+        - name: userId
+          in: path
+          description: user to run the schedule app against
+          example: '536830b6ed19afa44a000002'
+          required: true
+          schema:
+            type: string
+      requestBody:
+        description: The job parameters
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/JobSpec'
+      responses:
+        200:
+          description: success
+          content:
+            application/json:
+              schema: '#/components/schemas/JobStatus'
+  /{app}/{userId}/{hint}/jobRun:
+    security:
+      - bearerAuth: ['app', 'user', 'admin']
+    post:
+      tags:
+        - jobs
+      summary: Run a schedule Job
+      security:
+        - bearerAuth: ['app', 'user', 'admin']
+      operationId: runAppJob
+      parameters:
+        - name: app
+          in: path
+          description: app that defines the job
+          example: 'ventrata'
+          required: true
+          schema:
+            type: string
+        - name: userId
+          in: path
+          description: user to run the schedule app against
+          example: '536830b6ed19afa44a000002'
+          required: true
+          schema:
+            type: string
+        - name: hint
+          in: path
+          description: a specific token hint to be used
+          required: true
+          example: 'travelgate-australia'
+          schema:
+            type: string
+      requestBody:
+        description: The job parameters
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/JobSpec'
+      responses:
+        200:
+          description: success
+          content:
+            application/json:
+              schema: '#/components/schemas/JobStatus'
+  /{app}/{userId}/{jobId}/jobStatus:
+    security:
+      - bearerAuth: ['app', 'user', 'admin']
+    get:
+      tags:
+        - jobs
+      summary: Get the status and/or result from a job
+      security:
+        - bearerAuth: ['app', 'user', 'admin']
+      operationId: getJobStatus
+      parameters:
+        - name: app
+          in: path
+          description: app that defines the job
+          example: 'ventrata'
+          required: true
+          schema:
+            type: string
+        - name: userId
+          in: path
+          description: user that owns the job
+          example: '536830b6ed19afa44a000002'
+          required: true
+          schema:
+            type: string
+        - name: jobId
+          in: path
+          description: jobId to get the status for
+          example: '536830b6ed19afa44a000002'
+          required: true
+          schema:
+            type: string
+      responses:
+        200:
+          description: success
+          content:
+            application/json:
+              schema: '#/components/schemas/JobStatus'
   /user/{userId}/apps:
     get:
       security:

package/auth/authHandler.js

@@ -68,10 +68,10 @@ const frontend = async (req, res, next) => {
 };
 
 const adminOrUser = levels => async (req, res, next) => {
-  result = await Promise.all(levels.map(
+  const result = await Promise.all(levels.map(
     level => level(req, res, err => err),
   ));
-  if (result.some(val => (!Boolean(val)))) {
+  if (result.some(val => (!val))) {
     return next();
   }
   return next(result[0]);

package/closeAll.js

@@ -0,0 +1,6 @@
+const { redisResults, queue } = require('./worker/queue');
+
+module.exports = async () => {
+  await queue.close();
+  await redisResults.quit();
+};

package/controllers/__tests__/admin.js

@@ -1,4 +1,4 @@
-/* globals describe it expect */
+/* globals beforeAll describe it expect */
 
 const chance = require('chance').Chance();
 const testUtils = require('../../test/utils');
@@ -15,11 +15,18 @@ describe('admin', () => {
     packageName: `ti2-${appName}`,
     adminEmail: chance.email(),
   };
-  const { doApiPost, doApiGet } = testUtils({
-    plugins: [appName],
-  });
   let appKey;
   const userId = '536830b6ed19afa44a000002';
+  let doApiPost; let
+    doApiGet;
+  beforeAll(async () => {
+    ({
+      doApiGet,
+      doApiPost,
+    } = await testUtils({
+      plugins: [appName],
+    }));
+  });
   it('should be able to create an app', async () => {
     ({ value: appKey } = await doApiPost({
       url: '/app',
@@ -33,11 +40,7 @@ describe('admin', () => {
       url: '/apps',
       token: adminKey,
     });
-    expect(integrations).toEqual(
-      expect.arrayContaining([
-        expect.objectContaining(newApp),
-      ]),
-    );
+    expect(integrations.find(({ name }) => name === newApp.name)).toBeTruthy();
   });
   it('should be able to reset an app\'s key', async () => {
     const url = `/app/resetAppKey/${newApp.name}`;

package/controllers/__tests__/app.js

@@ -1,9 +1,11 @@
-/* globals describe it expect */
-
+/* globals beforeAll describe it expect */
 const chance = require('chance').Chance();
 const jwt = require('jwt-promise');
+const R = require('ramda');
+
 const testUtils = require('../../test/utils');
 const slugify = require('../../test/slugify');
+const appController = require('../app');
 
 const { env: { adminKey, jwtSecret } } = process;
 
@@ -16,9 +18,9 @@ describe('app', () => {
     packageName: `ti2-${appName}`,
     adminEmail: chance.email(),
   };
-  const { doApiPost, doApiGet, doApiPut } = testUtils({
-    plugins: [appName],
-  });
+  let doApiPost;
+  let doApiGet;
+  let doApiPut;
   let appKey;
   const userId = chance.guid();
   const apiKey = chance.guid();
@@ -31,6 +33,11 @@ describe('app', () => {
     payload: { lorem: chance.paragraph() },
   };
   let encodedKey;
+  beforeAll(async () => {
+    ({ doApiPost, doApiGet, doApiPut } = await testUtils({
+      plugins: [appName],
+    }));
+  });
   it('should create a new app', async () => {
     ({ value: appKey } = await doApiPost({
       url: '/app',
@@ -62,6 +69,51 @@ describe('app', () => {
     });
     expect(parseInt(value, 10)).toBeGreaterThan(0);
   });
+  describe('jobs', () => {
+    it('the scheduled job should have been created for the user', async () => {
+      const jobs = R.path(
+        ['jobs'],
+        await appController.getAppScheduledJobs({
+          integrationId: appName,
+          userId,
+        }),
+      );
+      expect(Array.isArray(jobs)).toBeTruthy();
+      expect(R.head(R.project(['pluginJobId', 'cron'], jobs)))
+        .toEqual({
+          pluginJobId: 'dailyReport',
+          cron: '0 9 * * *',
+        });
+    });
+    it.todo('if a new version of the plugin has a different set of scheduled tasks, they shoul dbe re-syncjed');
+    let jobId;
+    it('should be able to run a job', async () => {
+      let status;
+      ({ jobId, status } = await doApiPost({
+        url: `/${appName}/${userId}/jobRun`,
+        token: appKey,
+        payload: {
+          payload: {
+            method: 'dailyReport',
+          },
+        },
+      }));
+      expect(jobId).toBeTruthy();
+      expect(status).toBeTruthy();
+    });
+    it('should be able to get the status of a ran job', async () => {
+      await global.sleep(5e3);
+      const url = `/${appName}/${userId}/${jobId}/jobStatus`;
+      const jobStatus = await doApiGet({
+        url,
+        token: appKey,
+        payload: {},
+      });
+      expect(jobStatus.jobId).toBeTruthy();
+      expect(jobStatus.status).toBe('success');
+      expect(jobStatus.result).toEqual({ someVal: true });
+    }, 6e3);
+  });
   it('should be able to get a list of all tokens related to the app', async () => {
     const { userAppKeys } = await doApiGet({
       url: `/app/tokens/${appName}`,

package/controllers/__tests__/bookings.js

@@ -1,4 +1,4 @@
-/* globals describe it expect */
+/* globals beforeAll describe it expect */
 
 const chance = require('chance').Chance();
 const testUtils = require('../../test/utils');
@@ -22,13 +22,19 @@ describe('user: booking search', () => {
     tokenHint: 'testingToken',
     token,
   };
-  const {
-    doApiDelete,
-    doApiGet,
-    doApiPost,
-    plugins,
-  } = testUtils({
-    plugins: [newApp.name],
+  let doApiDelete;
+  let doApiGet;
+  let doApiPost;
+  let plugins;
+  beforeAll(async () => {
+    ({
+      doApiDelete,
+      doApiGet,
+      doApiPost,
+      plugins,
+    } = await testUtils({
+      plugins: [newApp.name],
+    }));
   });
 
   let userToken;
@@ -39,13 +45,17 @@ describe('user: booking search', () => {
         token: adminKey,
         payload: { tokenHint: 'testingToken' },
       });
-    } catch {}
+    } catch (err) {
+      console.debug(err);
+    }
     try {
       await doApiDelete({
         url: '/app/travelgate',
         token: adminKey,
       });
-    } catch {}
+    } catch (err) {
+      console.debug(err);
+    }
   });
   it('create the travelgate app', async () => {
     const { value: apiKey } = await doApiPost({

package/controllers/__tests__/ping.js

@@ -1,8 +1,13 @@
-/* global describe, it, expect */
-
-const { doApiGet } = require('../../test/utils')();
+/* global beforeAll, describe, it, expect */
+const testUtils = require('../../test/utils');
 
 describe('ping', () => {
+  let doApiGet;
+  beforeAll(async () => {
+    ({
+      doApiGet,
+    } = await testUtils());
+  });
   it('should get a server pong', async () => {
     const response = await doApiGet({
       url: '/ping',

package/controllers/__tests__/user.js

@@ -15,9 +15,8 @@ describe('user', () => {
     packageName: `ti2-${appName}`,
     adminEmail: chance.email(),
   };
-  const { doApiPost, doApiGet, doApiDelete } = testUtils({
-    plugins: [appName],
-  });
+  let doApiGet; let doApiPost; let
+    doApiDelete;
   let appKey;
   const userId = chance.guid();
   let apiKey = chance.guid();
@@ -27,6 +26,13 @@ describe('user', () => {
     apiKey,
   };
   beforeAll(async () => {
+    ({
+      doApiDelete,
+      doApiGet,
+      doApiPost,
+    } = await testUtils({
+      plugins: [appName],
+    }));
     // create an App
     ({ value: appKey } = await doApiPost({
       url: '/app',

package/controllers/app.js

@@ -5,8 +5,15 @@ const { Umzug, SequelizeStorage } = require('umzug');
 const path = require('path');
 const Sequelize = require('sequelize');
 const fs = require('fs').promises;
+const bb = require('bluebird');
+const R = require('ramda');
 
 const sqldb = require('../models');
+const {
+  queue,
+  addJob,
+  jobStatus,
+} = require('../worker/queue');
 
 const { env: { jwtSecret } } = process;
 
@@ -48,6 +55,61 @@ const createAppToken = async (req, res, next) => {
     });
     if (userAppKeyDup) await userAppKeyDup.destroy();
     const newAppKey = await sqldb.UserAppKey.create(payload);
+    // create any cronjobs related to the app
+    await bb.each(req.app.plugins, async plugin => {
+      if (Array.isArray(plugin.jobs)) {
+        await bb.each(plugin.jobs.filter(job => Boolean(job.cron)), async job => {
+          const where = {
+            pluginName: plugin.name,
+            pluginJobId: job.method,
+            userId,
+            hint,
+            cron: job.cron,
+          };
+          const existing = await sqldb.CronJobs.findOne({
+            where: R.omit(['cron'], where),
+          });
+          const jobPayload = {
+            ...where,
+            ...job.payload,
+          };
+          const jobParams = {
+            ...(job.cron ? {
+              repeat: {
+                cron: job.cron,
+              },
+            } : {}),
+            ...(job.params || {}),
+            removeOnComplete: false,
+          };
+          let bullJobId;
+          if (existing) {
+            const bullJob = await queue.getJob(job.bullJobId);
+            if (!bullJob) {
+              bullJobId = await addJob(jobPayload, jobParams);
+              existing.bullJobId = bullJobId;
+              await existing.save();
+            } else {
+              // make sure the cron is the same
+              const bullCron = R.path(
+                ['opts', 'repeat', 'cron'],
+                await queue.getJob(bullJobId),
+              );
+              if (bullCron !== job.cron) {
+                await queue.removeJobs(bullJobId);
+                bullJobId = await addJob(jobPayload, jobParams);
+                existing.bullJobId = bullJobId;
+                await existing.save();
+              }
+            }
+          } else {
+            bullJobId = await addJob(jobPayload, jobParams);
+            await sqldb.CronJobs.create({ ...where, bullJobId });
+          }
+        });
+      }
+    });
+
     return res.json({ value: newAppKey.get('id').toString() });
   } catch (err) {
     return next(err);
@@ -133,10 +195,70 @@ const migrateApp = async ({ integrationId, action }) => {
   throw Error('No recognized action');
 };
 
+const getAppScheduledJobs = async ({
+  integrationId,
+  userId,
+  hint,
+}) => {
+  const where = R.reject(R.isNil)({
+    pluginName: integrationId,
+    userId,
+    hint,
+  });
+  const jobs = await sqldb.CronJobs.findAll({
+    where,
+    raw: true,
+  });
+  return { jobs };
+};
+
+const runAppJob = async (req, res, next) => {
+  const {
+    body: {
+      payload,
+      jobParams,
+    },
+    params: {
+      app: pluginName,
+      hint,
+      userId,
+    },
+  } = req;
+  try {
+    const bullJobId = await addJob({
+      ...payload,
+      pluginName,
+      hint,
+      userId,
+    }, jobParams);
+    assert(bullJobId);
+    return res.json(await jobStatus({ jobId: bullJobId }));
+  } catch (err) {
+    return next(err);
+  }
+};
+
+const getJobStatus = async (req, res, next) => {
+  const {
+    params: {
+      jobId,
+    },
+  } = req;
+  try {
+    const returnValue = await jobStatus({ jobId });
+    return res.json(returnValue);
+  } catch (err) {
+    return next(err);
+  }
+};
+
 module.exports = {
   createAppToken,
   deleteAppToken,
+  getAppScheduledJobs,
+  getJobStatus,
   jwtEncode,
   listAppTokens,
   migrateApp,
+  runAppJob,
 };

package/controllers/ping.js

@@ -20,7 +20,8 @@ const ping = (_req, res, next) => {
     });
   } catch (err) {
     return next(err);
-  } };
+  }
+};
 
 module.exports = {
   ping,

package/controllers/user.js

@@ -1,11 +1,7 @@
 // controllers/user.js
-const { omit, flatten } = require('ramda');
-const assert = require('assert');
-const Promise = require('bluebird');
-const moment = require('moment');
+const { omit } = require('ramda');
 
-const { UserAppKey, Sequelize: { Op } } = require('../models');
-// const { load } = require('../../plugins/index.js');
+const { UserAppKey } = require('../models');
 
 const userAppList = async (req, res, next) => {
   const { params: { userId } } = req;

package/index.js

@@ -5,6 +5,9 @@ const swaggerUi = require('swagger-ui-express');
 const yaml = require('js-yaml');
 const fs = require('fs');
 const { pickBy } = require('ramda');
+const bb = require('bluebird');
+const { Op } = require('sequelize');
+const R = require('ramda');
 
 const schema = yaml.load(fs.readFileSync(`${__dirname}/api.yml`));
 
@@ -14,18 +17,20 @@ const adminController = require('./controllers/admin');
 const appController = require('./controllers/app');
 const userController = require('./controllers/user');
 const bookingsController = require('./controllers/bookings');
+const { Integration } = require('./models');
 
-module.exports = ({
+module.exports = async ({
   apiDocs = true,
   elasticLogIndex = 'apilog_ti2',
   elasticLogsClient,
   plugins: pluginsParam = {},
   port: portParam,
   startServer = true,
+  worker = false,
 }) => {
   const port = portParam || process.env.PORT || 10010;
-  const app = express();
-  const plugins = Object.entries(pluginsParam).map(([pluginName, Plugin]) => {
+  // create the plugin instances
+  const plugins = await bb.map(Object.entries(pluginsParam), async ([pluginName, Plugin]) => {
     // pass all env variables
     const pluginEnv = pickBy(
       (_val, key) => key.substring(0, `ti2_${pluginName}`.length)
@@ -37,9 +42,34 @@ module.exports = ({
       const nuName = attr.replace(/_/g, '-').replace(`ti2-${pluginName}-`, '');
       params[nuName] = value;
     });
-    return new Plugin({ name: pluginName, ...params });
+    const pluginInstance = await new Plugin({ name: pluginName, ...params });
+    return pluginInstance;
+  });
+  if (worker) {
+    return require('./worker/index')({ plugins });
+  }
+  // make sure all plugins have a DB entry
+  const pluginNames = plugins.map(R.prop('name'));
+  const matchedIntegrations = await Integration.findAll({
+    attributes: ['name'],
+    where: {
+      name: {
+        [Op.in]: pluginNames,
+      },
+    },
+    raw: true,
   });
-  // console.log({ plugins });
+  const missingIntegrations = R.difference(pluginNames, matchedIntegrations.map(R.prop('name')));
+  if (missingIntegrations.length > 0) {
+    // need to crete the missing integrations
+    await Integration.bulkCreate(missingIntegrations.map(name => ({
+      name,
+      packageName: `ti2-${name}`,
+      adminEmail: `ti2+${name}@localhost.local`,
+    })));
+  }
+  // create the API Web server
+  const app = express();
   const api = {
     ...pingController,
     ...adminController,
@@ -141,9 +171,12 @@ module.exports = ({
     connect(app);
     app.use(middleware.mock());
     // global error Handling
-    app.use((err, req, res, next) => {
+    app.use((err, req, res) => {
       // console.log(req.headers.['X-Request-Id'], err);
       res.status(err.status || 500);
+      if ((process.env.JEST_WORKER_ID)) {
+        console.debug(err);
+      }
       return res.json({
         message: err.message || 'Internal Error',
       });
@@ -156,8 +189,11 @@ module.exports = ({
   // first create a generic "terminator"
   const terminator = sig => {
     if (typeof sig === 'string') {
-      console.log('%s: Received %s - terminating ti2 ...',
-        Date(Date.now()), sig);
+      console.log(
+        '%s: Received %s - terminating ti2 ...',
+        Date(Date.now()),
+        sig,
+      );
       process.exit(1);
     }
     console.log('%s: Node server stopped.', Date(Date.now()));

package/jest.setup.js

@@ -0,0 +1,11 @@
+/* global afterAll */
+
+const closeAll = require('./closeAll');
+
+afterAll(async () => {
+  await closeAll();
+});
+
+global.sleep = (ms = 300) => new Promise(r => {
+  setTimeout(r, ms);
+});

package/migrations/20200128162519-deletedAtForIntegrations.js

@@ -1,4 +1,3 @@
-
 module.exports = {
   up: (queryInterface, Sequelize) => queryInterface.addColumn(
     'Integrations',

package/migrations/20210408024232-expand-JWT-field-size.js

@@ -3,8 +3,8 @@ module.exports = {
     type: Sequelize.TEXT,
     allowNull: false,
   }),
-  down: (queryInterface, Sequelize) =>  queryInterface.changeColumn('UserAppKeys', 'appKey', {
+  down: (queryInterface, Sequelize) => queryInterface.changeColumn('UserAppKeys', 'appKey', {
     type: Sequelize.STRING,
     allowNull: false,
-  })
+  }),
 };

package/migrations/20220426153807-cronJobs.js

@@ -0,0 +1,41 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => queryInterface
+    .createTable('CronJobs', {
+      pluginName: {
+        allowNull: false,
+        primaryKey: true,
+        type: Sequelize.STRING,
+      },
+      userId: {
+        allowNull: false,
+        primaryKey: true,
+        type: Sequelize.STRING,
+      },
+      hint: {
+        allowNull: false,
+        primaryKey: true,
+        type: Sequelize.STRING,
+      },
+      pluginJobId: {
+        allowNull: false,
+        type: Sequelize.STRING,
+      },
+      bullJobId: {
+        allowNull: false,
+        type: Sequelize.STRING,
+      },
+      cron: {
+        allowNull: false,
+        type: Sequelize.STRING,
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE,
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE,
+      },
+    }),
+  down: queryInterface => queryInterface.dropTable('CronJobs'),
+};

package/models/cronjobs.js

@@ -0,0 +1,31 @@
+const Sequelize = require('sequelize');
+const db = require('./db');
+
+const CronJobs = db.define('CronJobs', {
+  pluginName: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  userId: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  hint: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  pluginJobId: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  bullJobId: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  cron: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+}, {});
+
+module.exports = CronJobs;

package/models/index.js

@@ -10,6 +10,7 @@ fs
   .readdirSync(__dirname)
   .filter(file => (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js'))
   .forEach(file => {
+    // eslint-disable-next-line import/no-dynamic-require, global-require
     const model = require(path.join(__dirname, file));
     models[model.name] = model;
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ti2",
-  "version": "1.0.16",
+  "version": "1.0.17",
   "description": "Tourist Industry Exchange (TI2)",
   "main": "index.js",
   "scripts": {
@@ -8,21 +8,28 @@
     "sequelize": "sequelize-cli",
     "doc": "jsdoc -c jsdoc.conf.json -u tutorials"
   },
+  "jest": {
+    "setupFilesAfterEnv": [
+      "./jest.setup.js"
+    ]
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ti2travel/ti2.git"
   },
   "bin": "./cli.js",
   "author": "",
-  "license": "GPL-3",
+  "license": "GPL-3.0",
   "bugs": {
     "url": "https://github.com/ti2travel/ti2/issues"
   },
   "homepage": "https://github.com/ti2travel/ti2#readme",
   "dependencies": {
     "bluebird": "^3.7.2",
+    "bull": "^4.8.2",
     "commander": "^9.2.0",
     "express": "^4.17.2",
+    "ioredis": "^5.0.4",
     "js-yaml": "^4.1.0",
     "jsonwebtoken": "^8.5.1",
     "jwt-promise": "^1.0.0",
@@ -34,17 +41,18 @@
     "swagger-express-middleware": "3.0.0-alpha.5",
     "swagger-routes-express": "3.3.1",
     "swagger-ui-express": "4.3.0",
+    "throng": "^5.0.0",
     "umzug": "^3.1.1",
     "uninstall": "^0.0.0"
   },
   "devDependencies": {
+    "@salvador/minami": "^1.2.3",
     "chance": "^1.1.8",
     "eslint": "^8.7.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-import": "^2.25.4",
     "jest": "^27.4.7",
     "jsdoc": "^3.6.10",
-    "minami": "git+https://github.com/MyNameReallySux/minami.git#sort-data",
     "supertest": "^6.2.2"
   }
 }

package/test/plugin.js

@@ -1,4 +1,15 @@
 /* global jest, expect */
+/* eslint class-methods-use-this: "off", max-len: "off" */
+
+const jestPlugin = (() => {
+  if (typeof jest === 'undefined') {
+    return {
+      fn: fn => fn,
+    };
+  }
+  return jest;
+})();
+
 const chance = require('chance').Chance();
 
 class Plugin {
@@ -15,16 +26,16 @@ class Plugin {
       this[attr] = value;
     });
     // mock implementations
-    this.validateToken = jest.fn(() => true);
-    this.getProfile = jest.fn(() => {});
-    this.updateProfile = jest.fn(() => {});
-    this.getProduct = jest.fn(() => ({ products: [] }));
-    this.getProducts = jest.fn(() => true);
-    this.createLocation = jest.fn(() => ({ locationId: chance.guid() }));
-    this.updateLocation = jest.fn(() => true);
-    this.searchBooking = jest.fn(() => ({ bookings: [] }));
-    this.searchProducts = jest.fn(() => ({ bookings: [], products: [] }));
-    this.searchAvailability = jest.fn(({
+    this.validateToken = jestPlugin.fn(() => true);
+    this.getProfile = jestPlugin.fn(() => {});
+    this.updateProfile = jestPlugin.fn(() => {});
+    this.getProduct = jestPlugin.fn(() => ({ products: [] }));
+    this.getProducts = jestPlugin.fn(() => true);
+    this.createLocation = jestPlugin.fn(() => ({ locationId: chance.guid() }));
+    this.updateLocation = jestPlugin.fn(() => true);
+    this.searchBooking = jestPlugin.fn(() => ({ bookings: [] }));
+    this.searchProducts = jestPlugin.fn(() => ({ bookings: [], products: [] }));
+    this.searchAvailability = jestPlugin.fn(({
       token,
       payload: {
         travelDateStart,
@@ -40,9 +51,23 @@ class Plugin {
       expect(Array.isArray(occupancies)).toBeTruthy();
       return { availability: [{ id: chance.guid() }] };
     });
+    this.searchQuote = jestPlugin.fn(() => ({ quote: [{ id: chance.guid() }] }));
+    this.createBooking = jestPlugin.fn(() => {});
+
+    /**
+     * Background and schedule Jos
+     */
+    this.jobs = [{
+      method: 'dailyReport',
+      payload: {
+        action: 'report',
+      },
+      cron: '0 9 * * *',
+    }];
 
-    this.searchQuote = jest.fn(() => ({ quote: [{ id: chance.guid() }] }));
-    this.createBooking = jest.fn(() => {});
+    this.dailyReport = jestPlugin.fn(() => ({
+      someVal: true,
+    }));
   }
 
   /**
@@ -258,7 +283,7 @@ class Plugin {
     return true;
   }
 
-  searchHotelBooking = jest.fn(() => ({ bookings: [] }));
+  searchHotelBooking = jestPlugin.fn(() => ({ bookings: [] }));
 
   /**
    * Booking Object
@@ -363,6 +388,7 @@ class Plugin {
   searchAvailability() {
     // return (args);
   }
+
   /**
    * Retrieve an availability calendar over a date range
    * @async
@@ -379,6 +405,7 @@ class Plugin {
   availabilityCalndar() {
     // return (args);
   }
+
   /**
    * A quote return spec
    * @typedef {Object} Quote

package/test/utils.js

@@ -3,6 +3,7 @@
 const chance = require('chance').Chance();
 const request = require('supertest');
 const assert = require('assert');
+
 const appReq = require('../index');
 const Plugin = require('./plugin');
 const slugify = require('./slugify');
@@ -10,12 +11,10 @@ const sqldb = require('../models/db');
 
 const { env: { adminKey } } = process;
 
-const timeout = ms => (new Promise(resolve => setTimeout(resolve, ms)));
-
 afterAll(async () => {
   await sqldb.connectionManager.close();
 });
-module.exports = (appParams = {}) => {
+module.exports = async (appParams = {}) => {
   const plugins = (() => {
     if (Array.isArray(appParams.plugins)) {
       const retVal = {};
@@ -28,7 +27,7 @@ module.exports = (appParams = {}) => {
       mockName: Plugin,
     };
   })();
-  const app = appReq({
+  const app = await appReq({
     startServer: false,
     ...appParams,
     plugins,
@@ -104,7 +103,6 @@ module.exports = (appParams = {}) => {
     doApiPost,
     doApiPut,
     slugify,
-    timeout,
     plugins: app.plugins,
   };
 };

package/test/worker.js

@@ -0,0 +1,5 @@
+// tests only file, should not be execute by itself
+
+const worker = require('../worker/index')({ plugins: [] });
+
+worker.then(() => {});

package/tutorials/setup-your-instance.md

@@ -23,6 +23,7 @@ User + app integration credentials can be added after the app has been added to
 A MySQl instance is required and the following environment variables are required:
 
 - DB_URL (mysql connection in url format)
+- REDIS_URL (a redis database url to store the background queue)
 - dbCryptoKey (integration details encription key, should be a 32 chars base64 encoded random string)
 - adminKey (a key for admin related requests)
 - jwtKey (a key to encrypt user sessions request)
@@ -31,6 +32,7 @@ A MySQl instance is required and the following environment variables are require
 
 ```
 DB_URL=mysql://root:@mysqlserver/ti2development
+REDIS_URL=redis://redis:6379
 dbCryptoKey=cG9VYmJyQ2tlSDVHeXQ4RUN4VlNieEJRejNYWDlZU0g=
 jwtSecret=C8k0mrHVfWVTP7pIoZEHRrvFgTULhw3E4swDq1aoDH4P
 frontendKey=IpOiLahrGQxsQimwoK7Z
@@ -61,21 +63,57 @@ then just create an entry file like this :
 
 ```javascript
 // index.js
-const ti2 = require('ti2')({
-  plugins: {
-    travelgate: require('ti2-travelgate'),
-    ventrata: require('ti2-ventrata'),
-    tourconnect: require('ti2-tourconnect'),
-  },
-});
-```
 
-and stat the server using node:
+module.exports = (async () => {
+    const ti2 = await require('ti2')({
+      plugins: {
+        ventrata: require('ti2-ventrata'),
+        travelgate: require('ti2-travelgate'),
+        tourconnect: require('ti2-tourconnect'),
+      },
+    });
+    return ti2;
+})();
 
 ```
+
+and start the server:
+
+```bash
 $ node index.js
 ```
 
+in order to support the background job queue (required by some plugins) you should consider the following example:
+
+```javascript
+const plugins = {
+  ventrata: require('ti2-ventrata'),
+  travelgate: require('ti2-travelgate'),
+  tourconnect: require('ti2-tourconnect'),
+};
+
+module.exports = (async () => {
+  if (process.argv.indexOf('worker') > 0) {
+    const worker = await require('ti2')({
+      plugins,
+      worker: true,
+    });
+    return worker;
+  } else {
+    const ti2 = await require('ti2')({
+      plugins,
+    });
+    return ti2;
+  }
+})();
+```
+
+and start a background worker (on a second terminal), required for some plugins:
+
+```bash
+$ node index.js worker
+```
+
 ## API logging
 
 Currently logging is supported via elastic search, an elasticLogClient value can ben passed down, example:
@@ -87,12 +125,26 @@ const elasticLogsClient = (() => {
   return new Client({ node: elasticLogs });
 })();
 
-const ti2 = require('ti2')({
-  elasticLogsClient,
-  plugins: {
-    ventrata: require('ti2-ventrata'),
-    travelgate: require('ti2-travelgate'),
-    tourconnect: require('ti2-tourconnect'),
-  },
-});
+const plugins = {
+  ventrata: require('ti2-ventrata'),
+  travelgate: require('ti2-travelgate'),
+  tourconnect: require('ti2-tourconnect'),
+};
+
+module.exports = (async () => {
+  if (process.argv.indexOf('worker') > 0) {
+    const worker = await require('ti2')({
+      elasticLogsClient,
+      plugins,
+      worker: true,
+    });
+    return worker;
+  } else {
+    const ti2 = await require('ti2')({
+      elasticLogsClient,
+      plugins,
+    });
+    return ti2;
+  }
+})();
 ```

package/worker/index.js

@@ -0,0 +1,53 @@
+const throng = require('throng');
+require('util').inspect.defaultOptions.depth = null;
+const fakePlugin = require('../test/plugin');
+
+const { queue, saveResult } = require('./queue');
+
+const workers = process.env.WEB_CONCURRENCY || 2;
+
+const maxJobsPerWorker = 1;
+
+const worker = ({ plugins: pluginsParam }) => (id, disconnect) => {
+  console.log(`Started worker ${id}`);
+
+  const bye = () => {
+    console.log(`Worker ${id} exiting`);
+    disconnect();
+    process.exit(0);
+  };
+
+  process.on('SIGTERM', bye);
+  process.on('SIGUSR2', bye);
+  process.on('SIGINT', bye);
+
+  queue.process(maxJobsPerWorker, async job => {
+    const {
+      data: {
+        pluginName,
+        method,
+        userId,
+        inTesting,
+      },
+      id: jobId,
+      data: params,
+    } = job;
+    console.log(`job ${jobId} > running ${pluginName}:${method} for ${userId}`);
+    const plugins = await (async () => {
+      if (inTesting) {
+        return [...pluginsParam, await new fakePlugin({ name: pluginName })];
+      }
+      return pluginsParam;
+    })();
+    const thePlugin = plugins.find(({ name }) => name === pluginName);
+    const resultValue = await thePlugin[method](params);
+    console.log(`job ${jobId} > completed`);
+    await saveResult({ id: jobId, resultValue });
+  });
+};
+
+module.exports = async args => throng({
+  workers,
+  worker: worker(args),
+  signals: ['SIGUSR2', 'SIGTERM', 'SIGINT'],
+});

package/worker/queue.js

@@ -0,0 +1,88 @@
+const Queue = require('bull');
+const Redis = require('ioredis');
+const R = require('ramda');
+
+const REDIS_URL = process.env.REDIS_URL || 'redis://redis:6379';
+const itemsTTL = 3 * 60 * 60; // 3 hours
+
+const queue = new Queue('work', `${REDIS_URL}/0`);
+const redisResults = new Redis(`${REDIS_URL}/1`);
+
+const getPending = async () => {
+  const pending = await queue.count();
+  const running = await queue.getActiveCount();
+  return pending + running;
+};
+
+const allDone = async () => {
+  const left1 = await getPending();
+  if (left1 === 0) return true;
+  return (new Promise(resolve => {
+    queue.on('global:completed', async () => {
+      const left = await getPending();
+      if (left === 0) resolve();
+    });
+  }));
+};
+
+queue.on('failed', (job, err) => {
+  console.log(`job ${job.id} failed`, err);
+});
+
+const addJob = async (payload, paramsParam) => {
+  const params = paramsParam || {};
+  const inTesting = Boolean(process.env.JEST_WORKER_ID);
+  const id = R.path(['id'], await queue.add({
+    ...payload,
+    inTesting,
+  }, {
+    removeOnComplete: true,
+    ...params,
+  }));
+  return id;
+};
+
+const saveResult = async ({ id, resultValue }) => {
+  await redisResults.set(id, JSON.stringify(resultValue), 'EX', itemsTTL);
+};
+
+const jobStatus = async ({ jobId }) => {
+  try {
+    const job = await queue.getJob(jobId);
+    if (!job) {
+      // check if it is done
+      const result = JSON.parse(await redisResults.get(jobId));
+      // await global.sleep();
+      if (result) {
+        return {
+          jobId,
+          status: 'success',
+          result,
+        };
+      }
+      return { // 404
+        jobId,
+        status: 'failed',
+      };
+    }
+    const state = await job.getState();
+    return {
+      jobId,
+      status: state.toLowerCase(),
+    };
+  } catch (err) {
+    console.log(`could not get job status for ${jobId}`, err);
+    return { // 500
+      jobId,
+      status: 'failed',
+    };
+  }
+};
+module.exports = {
+  addJob,
+  allDone,
+  jobStatus,
+  queue,
+  saveResult,
+  redisResults,
+};