ti2 1.0.15 → 1.0.18

package/controllers/app.js

@@ -1,6 +1,19 @@
 const jwt = require('jwt-promise');
 const { omit } = require('ramda');
+const assert = require('assert');
+const { Umzug, SequelizeStorage } = require('umzug');
+const path = require('path');
+const Sequelize = require('sequelize');
+const fs = require('fs').promises;
+const bb = require('bluebird');
+const R = require('ramda');
+
 const sqldb = require('../models');
+const {
+  queue,
+  addJob,
+  jobStatus,
+} = require('../worker/queue');
 
 const { env: { jwtSecret } } = process;
 
@@ -42,6 +55,61 @@ const createAppToken = async (req, res, next) => {
     });
     if (userAppKeyDup) await userAppKeyDup.destroy();
     const newAppKey = await sqldb.UserAppKey.create(payload);
+    // create any cronjobs related to the app
+    await bb.each(req.app.plugins, async plugin => {
+      if (Array.isArray(plugin.jobs)) {
+        await bb.each(plugin.jobs.filter(job => Boolean(job.cron)), async job => {
+          const where = {
+            pluginName: plugin.name,
+            pluginJobId: job.method,
+            userId,
+            hint,
+            cron: job.cron,
+          };
+          const existing = await sqldb.CronJobs.findOne({
+            where: R.omit(['cron'], where),
+          });
+          const jobPayload = {
+            ...where,
+            ...job.payload,
+          };
+          const jobParams = {
+            ...(job.cron ? {
+              repeat: {
+                cron: job.cron,
+              },
+            } : {}),
+            ...(job.params || {}),
+            removeOnComplete: false,
+          };
+          let bullJobId;
+          if (existing) {
+            const bullJob = await queue.getJob(job.bullJobId);
+            if (!bullJob) {
+              bullJobId = await addJob(jobPayload, jobParams);
+              existing.bullJobId = bullJobId;
+              await existing.save();
+            } else {
+              // make sure the cron is the same
+              const bullCron = R.path(
+                ['opts', 'repeat', 'cron'],
+                await queue.getJob(bullJobId),
+              );
+              if (bullCron !== job.cron) {
+                await queue.removeJobs(bullJobId);
+                bullJobId = await addJob(jobPayload, jobParams);
+                existing.bullJobId = bullJobId;
+                await existing.save();
+              }
+            }
+          } else {
+            bullJobId = await addJob(jobPayload, jobParams);
+            await sqldb.CronJobs.create({ ...where, bullJobId });
+          }
+        });
+      }
+    });
+
     return res.json({ value: newAppKey.get('id').toString() });
   } catch (err) {
     return next(err);
@@ -83,9 +151,114 @@ const listAppTokens = async (req, res, next) => {
   }
 };
 
+const migrateApp = async ({ integrationId, action }) => {
+  const { sequelize } = sqldb;
+  assert(integrationId);
+  assert(action);
+  const migrationsPath = path.join(
+    __dirname,
+    '../',
+    '../',
+    `ti2-${integrationId}`,
+    'migrations',
+  );
+  try {
+    await fs.access(migrationsPath);
+  } catch (err) {
+    throw Error(`Could not find any migrations for ${integrationId} on ${migrationsPath}`);
+  }
+  const umzug = new Umzug({
+    migrations: {
+      glob: `${migrationsPath}/*.js`,
+      resolve: ({ name, path: migPath, context }) => {
+        const migration = require(migPath);
+        return {
+          name,
+          up: async () => migration.up(context, Sequelize),
+          down: async () => migration.down(context, Sequelize),
+        };
+      },
+    },
+    context: sequelize.getQueryInterface(),
+    storage: new SequelizeStorage({
+      sequelize,
+      tableName: `SequelizeMeta-${integrationId}`,
+    }),
+    logger: console,
+  });
+  if (action === 'migrate') {
+    return umzug.up();
+  }
+  if (action === 'revert') {
+    return umzug.down({ to: 0 });
+  }
+  throw Error('No recognized action');
+};
+
+const getAppScheduledJobs = async ({
+  integrationId,
+  userId,
+  hint,
+}) => {
+  const where = R.reject(R.isNil)({
+    pluginName: integrationId,
+    userId,
+    hint,
+  });
+  const jobs = await sqldb.CronJobs.findAll({
+    where,
+    raw: true,
+  });
+  return { jobs };
+};
+
+const runAppJob = async (req, res, next) => {
+  const {
+    body: {
+      payload,
+      jobParams,
+    },
+    params: {
+      app: pluginName,
+      hint,
+      userId,
+    },
+  } = req;
+  try {
+    const bullJobId = await addJob({
+      ...payload,
+      pluginName,
+      hint,
+      userId,
+    }, jobParams);
+    assert(bullJobId);
+    return res.json(await jobStatus({ jobId: bullJobId }));
+  } catch (err) {
+    return next(err);
+  }
+};
+
+const getJobStatus = async (req, res, next) => {
+  const {
+    params: {
+      jobId,
+    },
+  } = req;
+  try {
+    const returnValue = await jobStatus({ jobId });
+    return res.json(returnValue);
+  } catch (err) {
+    return next(err);
+  }
+};
+
 module.exports = {
-  jwtEncode,
   createAppToken,
   deleteAppToken,
+  getAppScheduledJobs,
+  getJobStatus,
+  jwtEncode,
   listAppTokens,
+  migrateApp,
+  runAppJob,
 };

package/controllers/ping.js

@@ -20,7 +20,8 @@ const ping = (_req, res, next) => {
     });
   } catch (err) {
     return next(err);
-  } };
+  }
+};
 
 module.exports = {
   ping,

package/controllers/user.js

@@ -1,11 +1,7 @@
 // controllers/user.js
-const { omit, flatten } = require('ramda');
-const assert = require('assert');
-const Promise = require('bluebird');
-const moment = require('moment');
+const { omit } = require('ramda');
 
-const { UserAppKey, Sequelize: { Op } } = require('../models');
-// const { load } = require('../../plugins/index.js');
+const { UserAppKey } = require('../models');
 
 const userAppList = async (req, res, next) => {
   const { params: { userId } } = req;

package/index.js

@@ -5,6 +5,9 @@ const swaggerUi = require('swagger-ui-express');
 const yaml = require('js-yaml');
 const fs = require('fs');
 const { pickBy } = require('ramda');
+const bb = require('bluebird');
+const { Op } = require('sequelize');
+const R = require('ramda');
 
 const schema = yaml.load(fs.readFileSync(`${__dirname}/api.yml`));
 
@@ -14,18 +17,20 @@ const adminController = require('./controllers/admin');
 const appController = require('./controllers/app');
 const userController = require('./controllers/user');
 const bookingsController = require('./controllers/bookings');
+const { Integration } = require('./models');
 
-module.exports = ({
+module.exports = async ({
   apiDocs = true,
   elasticLogIndex = 'apilog_ti2',
   elasticLogsClient,
   plugins: pluginsParam = {},
   port: portParam,
   startServer = true,
+  worker = false,
 }) => {
   const port = portParam || process.env.PORT || 10010;
-  const app = express();
-  const plugins = Object.entries(pluginsParam).map(([pluginName, Plugin]) => {
+  // create the plugin instances
+  const plugins = await bb.map(Object.entries(pluginsParam), async ([pluginName, Plugin]) => {
     // pass all env variables
     const pluginEnv = pickBy(
       (_val, key) => key.substring(0, `ti2_${pluginName}`.length)
@@ -37,9 +42,34 @@ module.exports = ({
       const nuName = attr.replace(/_/g, '-').replace(`ti2-${pluginName}-`, '');
       params[nuName] = value;
     });
-    return new Plugin({ name: pluginName, ...params });
+    const pluginInstance = await new Plugin({ name: pluginName, ...params });
+    return pluginInstance;
+  });
+  if (worker) {
+    return require('./worker/index')({ plugins });
+  }
+  // make sure all plugins have a DB entry
+  const pluginNames = plugins.map(R.prop('name'));
+  const matchedIntegrations = await Integration.findAll({
+    attributes: ['name'],
+    where: {
+      name: {
+        [Op.in]: pluginNames,
+      },
+    },
+    raw: true,
   });
-  // console.log({ plugins });
+  const missingIntegrations = R.difference(pluginNames, matchedIntegrations.map(R.prop('name')));
+  if (missingIntegrations.length > 0) {
+    // need to crete the missing integrations
+    await Integration.bulkCreate(missingIntegrations.map(name => ({
+      name,
+      packageName: `ti2-${name}`,
+      adminEmail: `ti2+${name}@localhost.local`,
+    })));
+  }
+  // create the API Web server
+  const app = express();
   const api = {
     ...pingController,
     ...adminController,
@@ -141,9 +171,12 @@ module.exports = ({
     connect(app);
     app.use(middleware.mock());
     // global error Handling
-    app.use((err, req, res, next) => {
+    app.use((err, req, res) => {
       // console.log(req.headers.['X-Request-Id'], err);
       res.status(err.status || 500);
+      if ((process.env.JEST_WORKER_ID)) {
+        console.debug(err);
+      }
       return res.json({
         message: err.message || 'Internal Error',
       });
@@ -156,8 +189,11 @@ module.exports = ({
   // first create a generic "terminator"
   const terminator = sig => {
     if (typeof sig === 'string') {
-      console.log('%s: Received %s - terminating ti2 ...',
-        Date(Date.now()), sig);
+      console.log(
+        '%s: Received %s - terminating ti2 ...',
+        Date(Date.now()),
+        sig,
+      );
       process.exit(1);
     }
     console.log('%s: Node server stopped.', Date(Date.now()));

package/jest.setup.js

@@ -0,0 +1,11 @@
+/* global afterAll */
+
+const closeAll = require('./closeAll');
+
+afterAll(async () => {
+  await closeAll();
+});
+
+global.sleep = (ms = 300) => new Promise(r => {
+  setTimeout(r, ms);
+});

package/jsdoc.conf.json

@@ -23,6 +23,6 @@
         "recurse": true,
         "sort": false,
         "readme": "README.md",
-        "template": "node_modules/minami"
+        "template": "node_modules/@salvador/minami"
     }
 }

package/migrations/20200128162519-deletedAtForIntegrations.js

@@ -1,4 +1,3 @@
-
 module.exports = {
   up: (queryInterface, Sequelize) => queryInterface.addColumn(
     'Integrations',

package/migrations/20210408024232-expand-JWT-field-size.js

@@ -3,8 +3,8 @@ module.exports = {
     type: Sequelize.TEXT,
     allowNull: false,
   }),
-  down: (queryInterface, Sequelize) =>  queryInterface.changeColumn('UserAppKeys', 'appKey', {
+  down: (queryInterface, Sequelize) => queryInterface.changeColumn('UserAppKeys', 'appKey', {
     type: Sequelize.STRING,
     allowNull: false,
-  })
+  }),
 };

package/migrations/20220426153807-cronJobs.js

@@ -0,0 +1,41 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => queryInterface
+    .createTable('CronJobs', {
+      pluginName: {
+        allowNull: false,
+        primaryKey: true,
+        type: Sequelize.STRING,
+      },
+      userId: {
+        allowNull: false,
+        primaryKey: true,
+        type: Sequelize.STRING,
+      },
+      hint: {
+        allowNull: false,
+        primaryKey: true,
+        type: Sequelize.STRING,
+      },
+      pluginJobId: {
+        allowNull: false,
+        type: Sequelize.STRING,
+      },
+      bullJobId: {
+        allowNull: false,
+        type: Sequelize.STRING,
+      },
+      cron: {
+        allowNull: false,
+        type: Sequelize.STRING,
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE,
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE,
+      },
+    }),
+  down: queryInterface => queryInterface.dropTable('CronJobs'),
+};

package/models/cronjobs.js

@@ -0,0 +1,31 @@
+const Sequelize = require('sequelize');
+const db = require('./db');
+
+const CronJobs = db.define('CronJobs', {
+  pluginName: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  userId: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  hint: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  pluginJobId: {
+    type: Sequelize.STRING,
+    primaryKey: true,
+  },
+  bullJobId: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+  cron: {
+    type: Sequelize.STRING,
+    allowNull: false,
+  },
+}, {});
+
+module.exports = CronJobs;

package/models/db.js

@@ -7,7 +7,6 @@ const {
 } = process;
 
 const options = {
-  operatorsAliases: false,
   logging: false,
 };
 

package/models/index.js

@@ -10,6 +10,7 @@ fs
   .readdirSync(__dirname)
   .filter(file => (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js'))
   .forEach(file => {
+    // eslint-disable-next-line import/no-dynamic-require, global-require
     const model = require(path.join(__dirname, file));
     models[model.name] = model;
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ti2",
-  "version": "1.0.15",
+  "version": "1.0.18",
   "description": "Tourist Industry Exchange (TI2)",
   "main": "index.js",
   "scripts": {
@@ -8,19 +8,28 @@
     "sequelize": "sequelize-cli",
     "doc": "jsdoc -c jsdoc.conf.json -u tutorials"
   },
+  "jest": {
+    "setupFilesAfterEnv": [
+      "./jest.setup.js"
+    ]
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ti2travel/ti2.git"
   },
+  "bin": "./cli.js",
   "author": "",
-  "license": "GPL-3",
+  "license": "GPL-3.0",
   "bugs": {
     "url": "https://github.com/ti2travel/ti2/issues"
   },
   "homepage": "https://github.com/ti2travel/ti2#readme",
   "dependencies": {
     "bluebird": "^3.7.2",
+    "bull": "^4.8.2",
+    "commander": "^9.2.0",
     "express": "^4.17.2",
+    "ioredis": "^5.0.4",
     "js-yaml": "^4.1.0",
     "jsonwebtoken": "^8.5.1",
     "jwt-promise": "^1.0.0",
@@ -32,16 +41,18 @@
     "swagger-express-middleware": "3.0.0-alpha.5",
     "swagger-routes-express": "3.3.1",
     "swagger-ui-express": "4.3.0",
+    "throng": "^5.0.0",
+    "umzug": "^3.1.1",
     "uninstall": "^0.0.0"
   },
   "devDependencies": {
+    "@salvador/minami": "^1.2.3",
     "chance": "^1.1.8",
     "eslint": "^8.7.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-import": "^2.25.4",
     "jest": "^27.4.7",
     "jsdoc": "^3.6.10",
-    "minami": "git+https://github.com/MyNameReallySux/minami.git#sort-data",
     "supertest": "^6.2.2"
   }
 }

package/test/plugin.js

@@ -1,4 +1,15 @@
 /* global jest, expect */
+/* eslint class-methods-use-this: "off", max-len: "off" */
+
+const jestPlugin = (() => {
+  if (typeof jest === 'undefined') {
+    return {
+      fn: fn => fn,
+    };
+  }
+  return jest;
+})();
+
 const chance = require('chance').Chance();
 
 class Plugin {
@@ -15,16 +26,16 @@ class Plugin {
       this[attr] = value;
     });
     // mock implementations
-    this.validateToken = jest.fn(() => true);
-    this.getProfile = jest.fn(() => {});
-    this.updateProfile = jest.fn(() => {});
-    this.getProduct = jest.fn(() => ({ products: [] }));
-    this.getProducts = jest.fn(() => true);
-    this.createLocation = jest.fn(() => ({ locationId: chance.guid() }));
-    this.updateLocation = jest.fn(() => true);
-    this.searchBooking = jest.fn(() => ({ bookings: [] }));
-    this.searchProducts = jest.fn(() => ({ bookings: [], products: [] }));
-    this.searchAvailability = jest.fn(({
+    this.validateToken = jestPlugin.fn(() => true);
+    this.getProfile = jestPlugin.fn(() => {});
+    this.updateProfile = jestPlugin.fn(() => {});
+    this.getProduct = jestPlugin.fn(() => ({ products: [] }));
+    this.getProducts = jestPlugin.fn(() => true);
+    this.createLocation = jestPlugin.fn(() => ({ locationId: chance.guid() }));
+    this.updateLocation = jestPlugin.fn(() => true);
+    this.searchBooking = jestPlugin.fn(() => ({ bookings: [] }));
+    this.searchProducts = jestPlugin.fn(() => ({ bookings: [], products: [] }));
+    this.searchAvailability = jestPlugin.fn(({
       token,
       payload: {
         travelDateStart,
@@ -40,9 +51,23 @@ class Plugin {
       expect(Array.isArray(occupancies)).toBeTruthy();
       return { availability: [{ id: chance.guid() }] };
     });
+    this.searchQuote = jestPlugin.fn(() => ({ quote: [{ id: chance.guid() }] }));
+    this.createBooking = jestPlugin.fn(() => {});
+
+    /**
+     * Background and schedule Jos
+     */
+    this.jobs = [{
+      method: 'dailyReport',
+      payload: {
+        action: 'report',
+      },
+      cron: '0 9 * * *',
+    }];
 
-    this.searchQuote = jest.fn(() => ({ quote: [{ id: chance.guid() }] }));
-    this.createBooking = jest.fn(() => {});
+    this.dailyReport = jestPlugin.fn(() => ({
+      someVal: true,
+    }));
   }
 
   /**
@@ -258,7 +283,7 @@ class Plugin {
     return true;
   }
 
-  searchHotelBooking = jest.fn(() => ({ bookings: [] }));
+  searchHotelBooking = jestPlugin.fn(() => ({ bookings: [] }));
 
   /**
    * Booking Object
@@ -363,6 +388,7 @@ class Plugin {
   searchAvailability() {
     // return (args);
   }
+
   /**
    * Retrieve an availability calendar over a date range
    * @async
@@ -379,6 +405,7 @@ class Plugin {
   availabilityCalndar() {
     // return (args);
   }
+
   /**
    * A quote return spec
    * @typedef {Object} Quote

package/test/utils.js

@@ -3,6 +3,7 @@
 const chance = require('chance').Chance();
 const request = require('supertest');
 const assert = require('assert');
+
 const appReq = require('../index');
 const Plugin = require('./plugin');
 const slugify = require('./slugify');
@@ -10,12 +11,10 @@ const sqldb = require('../models/db');
 
 const { env: { adminKey } } = process;
 
-const timeout = ms => (new Promise(resolve => setTimeout(resolve, ms)));
-
 afterAll(async () => {
   await sqldb.connectionManager.close();
 });
-module.exports = (appParams = {}) => {
+module.exports = async (appParams = {}) => {
   const plugins = (() => {
     if (Array.isArray(appParams.plugins)) {
       const retVal = {};
@@ -28,7 +27,7 @@ module.exports = (appParams = {}) => {
       mockName: Plugin,
     };
   })();
-  const app = appReq({
+  const app = await appReq({
     startServer: false,
     ...appParams,
     plugins,
@@ -104,7 +103,6 @@ module.exports = (appParams = {}) => {
     doApiPost,
     doApiPut,
     slugify,
-    timeout,
     plugins: app.plugins,
   };
 };

package/test/worker.js

@@ -0,0 +1,5 @@
+// tests only file, should not be execute by itself
+
+const worker = require('../worker/index')({ plugins: [] });
+
+worker.then(() => {});

package/tutorials/plugin-development.md

@@ -216,14 +216,17 @@ Ti2 uses [Sequelize v6.13](https://sequelize.org/v6) which is the database ORM w
 
 Migrations should be run after the fact, from the root of ti2 instance like so:
 
-
 ```
-$  npm explore ti2<pluginName> -- npx sequelize db:migrate"
+$ ti2 dbapp [plugin name] migrate
 ```
 
 We strongly encourage all the plugin's table names are prefixed with the plugin name (i.e. ti2-pluginName-cacheTable).
 
-This should be executed after the ti2 project migrations have been executed.
+This should be executed after all ti2 host migrations had been executed.
+
+```
+$ ti2 db migrate
+```
 
 ## Scheduled Task (to be released on v2)