ti2 1.0.14 → 1.0.17

package/tutorials/plugin-development.md

@@ -216,14 +216,17 @@ Ti2 uses [Sequelize v6.13](https://sequelize.org/v6) which is the database ORM w
 
 Migrations should be run after the fact, from the root of ti2 instance like so:
 
-
 ```
-$  npm explore ti2<pluginName> -- npx sequelize db:migrate"
+$ ti2 dbapp [plugin name] migrate
 ```
 
 We strongly encourage all the plugin's table names are prefixed with the plugin name (i.e. ti2-pluginName-cacheTable).
 
-This should be executed after the ti2 project migrations have been executed.
+This should be executed after all ti2 host migrations had been executed.
+
+```
+$ ti2 db migrate
+```
 
 ## Scheduled Task (to be released on v2)
 

package/tutorials/setup-your-instance.md

@@ -23,6 +23,7 @@ User + app integration credentials can be added after the app has been added to
 A MySQl instance is required and the following environment variables are required:
 
 - DB_URL (mysql connection in url format)
+- REDIS_URL (a redis database url to store the background queue)
 - dbCryptoKey (integration details encription key, should be a 32 chars base64 encoded random string)
 - adminKey (a key for admin related requests)
 - jwtKey (a key to encrypt user sessions request)
@@ -31,6 +32,7 @@ A MySQl instance is required and the following environment variables are require
 
 ```
 DB_URL=mysql://root:@mysqlserver/ti2development
+REDIS_URL=redis://redis:6379
 dbCryptoKey=cG9VYmJyQ2tlSDVHeXQ4RUN4VlNieEJRejNYWDlZU0g=
 jwtSecret=C8k0mrHVfWVTP7pIoZEHRrvFgTULhw3E4swDq1aoDH4P
 frontendKey=IpOiLahrGQxsQimwoK7Z
@@ -52,7 +54,7 @@ $ npm i ti2 ti2-tourconnect
 After installing the ti2 package you need to run the database migrations for ti2:
 
 ```
-$  npm explore ti2<pluginName> -- npx sequelize db:migrate"
+$  ti2 db migrate
 ```
 
 ### Entry File
@@ -61,21 +63,57 @@ then just create an entry file like this :
 
 ```javascript
 // index.js
-const ti2 = require('ti2')({
-  plugins: {
-    travelgate: require('ti2-travelgate'),
-    ventrata: require('ti2-ventrata'),
-    tourconnect: require('ti2-tourconnect'),
-  },
-});
-```
 
-and stat the server using node:
+module.exports = (async () => {
+    const ti2 = await require('ti2')({
+      plugins: {
+        ventrata: require('ti2-ventrata'),
+        travelgate: require('ti2-travelgate'),
+        tourconnect: require('ti2-tourconnect'),
+      },
+    });
+    return ti2;
+})();
 
 ```
+
+and start the server:
+
+```bash
 $ node index.js
 ```
 
+in order to support the background job queue (required by some plugins) you should consider the following example:
+
+```javascript
+const plugins = {
+  ventrata: require('ti2-ventrata'),
+  travelgate: require('ti2-travelgate'),
+  tourconnect: require('ti2-tourconnect'),
+};
+
+module.exports = (async () => {
+  if (process.argv.indexOf('worker') > 0) {
+    const worker = await require('ti2')({
+      plugins,
+      worker: true,
+    });
+    return worker;
+  } else {
+    const ti2 = await require('ti2')({
+      plugins,
+    });
+    return ti2;
+  }
+})();
+```
+
+and start a background worker (on a second terminal), required for some plugins:
+
+```bash
+$ node index.js worker
+```
+
 ## API logging
 
 Currently logging is supported via elastic search, an elasticLogClient value can ben passed down, example:
@@ -87,12 +125,26 @@ const elasticLogsClient = (() => {
   return new Client({ node: elasticLogs });
 })();
 
-const ti2 = require('ti2')({
-  elasticLogsClient,
-  plugins: {
-    ventrata: require('ti2-ventrata'),
-    travelgate: require('ti2-travelgate'),
-    tourconnect: require('ti2-tourconnect'),
-  },
-});
+const plugins = {
+  ventrata: require('ti2-ventrata'),
+  travelgate: require('ti2-travelgate'),
+  tourconnect: require('ti2-tourconnect'),
+};
+
+module.exports = (async () => {
+  if (process.argv.indexOf('worker') > 0) {
+    const worker = await require('ti2')({
+      elasticLogsClient,
+      plugins,
+      worker: true,
+    });
+    return worker;
+  } else {
+    const ti2 = await require('ti2')({
+      elasticLogsClient,
+      plugins,
+    });
+    return ti2;
+  }
+})();
 ```

package/worker/index.js

@@ -0,0 +1,53 @@
+const throng = require('throng');
+require('util').inspect.defaultOptions.depth = null;
+const fakePlugin = require('../test/plugin');
+
+const { queue, saveResult } = require('./queue');
+
+const workers = process.env.WEB_CONCURRENCY || 2;
+
+const maxJobsPerWorker = 1;
+
+const worker = ({ plugins: pluginsParam }) => (id, disconnect) => {
+  console.log(`Started worker ${id}`);
+
+  const bye = () => {
+    console.log(`Worker ${id} exiting`);
+    disconnect();
+    process.exit(0);
+  };
+
+  process.on('SIGTERM', bye);
+  process.on('SIGUSR2', bye);
+  process.on('SIGINT', bye);
+
+  queue.process(maxJobsPerWorker, async job => {
+    const {
+      data: {
+        pluginName,
+        method,
+        userId,
+        inTesting,
+      },
+      id: jobId,
+      data: params,
+    } = job;
+    console.log(`job ${jobId} > running ${pluginName}:${method} for ${userId}`);
+    const plugins = await (async () => {
+      if (inTesting) {
+        return [...pluginsParam, await new fakePlugin({ name: pluginName })];
+      }
+      return pluginsParam;
+    })();
+    const thePlugin = plugins.find(({ name }) => name === pluginName);
+    const resultValue = await thePlugin[method](params);
+    console.log(`job ${jobId} > completed`);
+    await saveResult({ id: jobId, resultValue });
+  });
+};
+
+module.exports = async args => throng({
+  workers,
+  worker: worker(args),
+  signals: ['SIGUSR2', 'SIGTERM', 'SIGINT'],
+});

package/worker/queue.js

@@ -0,0 +1,88 @@
+const Queue = require('bull');
+const Redis = require('ioredis');
+const R = require('ramda');
+
+const REDIS_URL = process.env.REDIS_URL || 'redis://redis:6379';
+const itemsTTL = 3 * 60 * 60; // 3 hours
+
+const queue = new Queue('work', `${REDIS_URL}/0`);
+const redisResults = new Redis(`${REDIS_URL}/1`);
+
+const getPending = async () => {
+  const pending = await queue.count();
+  const running = await queue.getActiveCount();
+  return pending + running;
+};
+
+const allDone = async () => {
+  const left1 = await getPending();
+  if (left1 === 0) return true;
+  return (new Promise(resolve => {
+    queue.on('global:completed', async () => {
+      const left = await getPending();
+      if (left === 0) resolve();
+    });
+  }));
+};
+
+queue.on('failed', (job, err) => {
+  console.log(`job ${job.id} failed`, err);
+});
+
+const addJob = async (payload, paramsParam) => {
+  const params = paramsParam || {};
+  const inTesting = Boolean(process.env.JEST_WORKER_ID);
+  const id = R.path(['id'], await queue.add({
+    ...payload,
+    inTesting,
+  }, {
+    removeOnComplete: true,
+    ...params,
+  }));
+  return id;
+};
+
+const saveResult = async ({ id, resultValue }) => {
+  await redisResults.set(id, JSON.stringify(resultValue), 'EX', itemsTTL);
+};
+
+const jobStatus = async ({ jobId }) => {
+  try {
+    const job = await queue.getJob(jobId);
+    if (!job) {
+      // check if it is done
+      const result = JSON.parse(await redisResults.get(jobId));
+      // await global.sleep();
+      if (result) {
+        return {
+          jobId,
+          status: 'success',
+          result,
+        };
+      }
+      return { // 404
+        jobId,
+        status: 'failed',
+      };
+    }
+    const state = await job.getState();
+    return {
+      jobId,
+      status: state.toLowerCase(),
+    };
+  } catch (err) {
+    console.log(`could not get job status for ${jobId}`, err);
+    return { // 500
+      jobId,
+      status: 'failed',
+    };
+  }
+};
+module.exports = {
+  addJob,
+  allDone,
+  jobStatus,
+  queue,
+  saveResult,
+  redisResults,
+};