sockethub 4.0.0 → 5.0.0-alpha.0

package/src/platform-instance.ts

@@ -3,13 +3,13 @@ import { join } from 'path';
 import { debug, Debugger } from 'debug';
 import Queue from 'bull';
 
-import redisConfig from "./config";
-import crypto from "./crypto";
-import { ActivityObject, Job } from "./sockethub";
+import config from "./config";
+import { ActivityStream, JobDataDecrypted, JobEncrypted } from "./sockethub";
 import { getSocket } from "./serve";
+import { decryptJobData } from "./common";
 
 // collection of platform instances, stored by `id`
-export const platformInstances = new Map();
+export const platformInstances = new Map<string, PlatformInstance>();
 
 export interface PlatformInstanceParams {
   identifier: string;
@@ -18,26 +18,32 @@ export interface PlatformInstanceParams {
   actor?: string;
 }
 
-interface MessageFromPlatform extends Array<string|ActivityObject>{
-  0: string, 1: ActivityObject, 2: string}
+interface MessageFromPlatform extends Array<string|ActivityStream>{
+  0: string, 1: ActivityStream, 2: string}
 export interface MessageFromParent extends Array<string|any>{0: string, 1: any}
 
+interface PlatformConfig {
+  persist?: boolean;
+  requireCredentials?: Array<string>;
+}
 
 export default class PlatformInstance {
   flaggedForTermination: boolean = false;
   id: string;
   queue: Queue;
+  completedJobHandlers: Map<string, Function> = new Map();
+  config: PlatformConfig = {};
   readonly name: string;
   readonly process: ChildProcess;
   readonly debug: Debugger;
   readonly parentId: string;
   readonly sessions: Set<string> = new Set();
-  public readonly global?: boolean = false;
-  private readonly actor?: string;
-  private readonly sessionCallbacks: object = {
+  readonly sessionCallbacks: object = {
     'close': (() => new Map())(),
-    'message': (() => new Map())(),
+    'message': (() => new Map())()
   };
+  public readonly global?: boolean = false;
+  private readonly actor?: string;
 
   constructor(params: PlatformInstanceParams) {
     this.id = params.identifier;
@@ -57,37 +63,46 @@ export default class PlatformInstance {
   /**
    * Destroys all references to this platform instance, internal listeners and controlled processes
    */
-  public destroy() {
+  public async destroy() {
+    this.debug(`cleaning up`);
     this.flaggedForTermination = true;
-    platformInstances.delete(this.id);
     try {
-      this.queue.clean(0);
+      await this.queue.removeAllListeners();
+    } catch (e) { }
+    try {
+      await this.queue.obliterate({ force: true });
     } catch (e) { }
     try {
-      this.process.removeAllListeners('close');
-      this.process.unref();
-      this.process.kill();
+      delete this.queue;
+      await this.process.removeAllListeners('close');
+      await this.process.unref();
+      await this.process.kill();
     } catch (e) { }
+    platformInstances.delete(this.id);
   }
 
   /**
    * When jobs are completed or failed, we prepare the results and send them to the client socket
    */
   public initQueue(secret: string) {
-    this.queue = new Queue(this.parentId + this.id, redisConfig);
-    this.queue.on('global:completed', (jobId, result) => {
-      this.queue.getJob(jobId).then((job) => {
-        job.data.msg = crypto.decrypt(job.data.msg, secret);
-        this.handleJobResult('completed', job, result);
+    this.queue = new Queue(this.parentId + this.id, { redis: config.get('redis') });
+
+    this.queue.on('global:completed', (jobId, resultString) => {
+      const result = resultString ? JSON.parse(resultString) : "";
+      this.queue.getJob(jobId).then(async (job: JobEncrypted) => {
+        await this.handleJobResult('completed', decryptJobData(job, secret), result);
+        await job.remove();
       });
     });
+
     this.queue.on('global:error', (jobId, result) => {
       this.debug("unknown queue error", jobId, result);
     });
+
     this.queue.on('global:failed', (jobId, result) => {
-      this.queue.getJob(jobId).then((job) => {
-        job.data.msg = crypto.decrypt(job.data.msg, secret);
-        this.handleJobResult('failed', job, result);
+      this.queue.getJob(jobId).then(async (job: JobEncrypted) => {
+        await this.handleJobResult('failed', decryptJobData(job, secret), result);
+        await job.remove();
       });
     });
   }
@@ -111,54 +126,58 @@ export default class PlatformInstance {
    * Sends a message to client (user), can be registered with an event emitted from the platform
    * process.
    * @param sessionId ID of the socket connection to send the message to
-   * @param type type of message to emit. 'message', 'completed', 'failed'
    * @param msg ActivityStream object to send to client
    */
-  public sendToClient(sessionId: string, type: string, msg: ActivityObject) {
+  public sendToClient(sessionId: string, msg: ActivityStream) {
     getSocket(sessionId).then((socket) => {
       try {
         // this property should never be exposed externally
         delete msg.sessionSecret;
       } catch (e) {}
       msg.context = this.name;
-      socket.emit(type, msg);
+      if ((msg.type === 'error') && (typeof msg.actor === 'undefined') && (this.actor)) {
+        // ensure an actor is present if not otherwise defined
+        msg.actor = { id: this.actor };
+      }
+      socket.emit('message', msg);
     }, (err) => this.debug(`sendToClient ${err}`));
   }
 
   // send message to every connected socket associated with this platform instance.
-  private async broadcastToSharedPeers(sessionId: string, msg: ActivityObject) {
+  private async broadcastToSharedPeers(sessionId: string, msg: ActivityStream) {
     for (let sid of this.sessions.values()) {
       if (sid !== sessionId) {
         this.debug(`broadcasting message to ${sid}`);
-        this.sendToClient(sid, 'message', msg);
+        await this.sendToClient(sid, msg);
       }
     }
   }
 
   // handle job results coming in on the queue from platform instances
-  private handleJobResult(type: string, job: Job, result) {
-    this.debug(`job ${job.data.title}: ${type}`);
-
-    if ((type === 'completed') && (result)) {
-      job.data.msg.object = {
-        '@type': 'result',
-        content: result
-      };
-    } else if (type === 'failed') {
-      job.data.msg.object = {
-        '@type': 'error',
-        content: result ? result : "job failed for unknown reason"
-      };
+  private async handleJobResult(type: string, jobData: JobDataDecrypted, result) {
+    this.debug(`${type} job ${jobData.title}`);
+    delete jobData.msg.sessionSecret;
+    let msg = jobData.msg;
+    if (type === 'failed') {
+      msg.error = result ? result : "job failed for unknown reason";
+      if ((this.config.persist) && (this.config.requireCredentials.includes(jobData.msg.type))) {
+        this.debug(`critical job type ${jobData.msg.type} failed, terminating platform instance`);
+        await this.destroy();
+      }
     }
 
-    // send message to client as completed for failed job
-    this.sendToClient(job.data.sessionId, type, job.data.msg);
+    // send result to client
+    const callback = this.completedJobHandlers.get(jobData.title);
+    if (callback) {
+      callback(msg);
+      this.completedJobHandlers.delete(jobData.title);
+    } else {
+      await this.sendToClient(jobData.sessionId, msg);
+    }
 
     // let all related peers know of result as an independent message
     // (not as part of a job completion, or failure)
-    this.broadcastToSharedPeers(job.data.sessionId, job.data.msg);
-
-    job.remove();
+    await this.broadcastToSharedPeers(jobData.sessionId, msg);
   }
 
   /**
@@ -166,19 +185,16 @@ export default class PlatformInstance {
    * @param sessionId
    * @param errorMessage
    */
-  private reportError(sessionId: string, errorMessage: any) {
-    const errorObject: ActivityObject = {
+  private async reportError(sessionId: string, errorMessage: any) {
+    const errorObject: ActivityStream = {
       context: this.name,
-      '@type': 'error',
-      target: this.actor,
-      object: {
-        '@type': 'error',
-        content: errorMessage
-      }
+      type: 'error',
+      actor: { id: this.actor },
+      error: errorMessage
     };
-    this.sendToClient(sessionId, 'message', errorObject);
+    this.sendToClient(sessionId, errorObject);
     this.sessions.clear();
-    this.destroy();
+    await this.destroy();
   }
 
   /**
@@ -211,7 +227,7 @@ export default class PlatformInstance {
           this.reportError(sessionId, data[1]);
         } else {
           // treat like a message to clients
-          this.sendToClient(sessionId, 'message', data[1]);
+          this.sendToClient(sessionId, data[1]);
         }
       }
     };

package/src/platform.ts

@@ -1,11 +1,11 @@
 import debug from 'debug';
 import hash from "object-hash";
-import redisConfig from './config';
-import crypto from "./crypto";
+import config from './config';
 import Queue from 'bull';
-import { getSessionStore, getPlatformId, Store } from "./common";
-import { ActivityObject, Job } from "./sockethub";
+import { getPlatformId, decryptJobData } from "./common";
+import { ActivityStream, JobDataDecrypted, JobEncrypted } from "./sockethub";
 import { MessageFromParent } from './platform-instance';
+import { getSessionStore } from "./store";
 
 // command-line params
 const parentId = process.argv[2];
@@ -23,7 +23,7 @@ logger(`platform handler initialized for ${platformName} ${identifier}`);
 
 export interface PlatformSession {
   debug(msg: string): void;
-  sendToClient(msg: ActivityObject, special?: string): void;
+  sendToClient(msg: ActivityStream, special?: string): void;
   updateActor(credentials: object): void;
 }
 
@@ -31,7 +31,8 @@ export interface PlatformSession {
  * Handle any uncaught errors from the platform by alerting the worker and shutting down.
  */
 process.on('uncaughtException', (err) => {
-  console.log(`\nUNCAUGHT EXCEPTION IN PLATFORM\n`);
+  console.log('EXCEPTION IN PLATFORM');
+  // eslint-disable-next-line security-node/detect-crlf
   console.log(err.stack);
   process.send(['error', err.toString()]);
   process.exit(1);
@@ -70,7 +71,7 @@ const platform = new PlatformModule(platformSession);
  */
 function getCredentials(actorId: string, sessionId: string, sessionSecret: string, cb: Function) {
   if (platform.config.noCredentials) { return cb(); }
-  const store: Store = getSessionStore(parentId, parentSecret1, sessionId, sessionSecret);
+  const store = getSessionStore(parentId, parentSecret1, sessionId, sessionSecret);
   store.get(actorId, (err, credentials) => {
     if (platform.config.persist) {
       // don't continue if we don't get credentials
@@ -83,7 +84,7 @@ function getCredentials(actorId: string, sessionId: string, sessionSecret: strin
     if (platform.credentialsHash) {
       if (platform.credentialsHash !== hash(credentials.object)) {
         return cb('provided credentials do not match existing platform instance for actor '
-            + platform.actor['@id']);
+            + platform.actor.id);
       }
     } else {
       platform.credentialsHash = hash(credentials.object);
@@ -97,29 +98,48 @@ function getCredentials(actorId: string, sessionId: string, sessionSecret: strin
  * @param secret the secret used to decrypt credentials
  */
 function getJobHandler(secret: string) {
-  return (job: Job, done: Function) => {
-    job.data.msg = crypto.decrypt(job.data.msg, secret);
-    const jobLog = debug(`${loggerPrefix}:${job.data.sessionId}`);
-    jobLog(`job ${job.data.title}: ${job.data.msg['@type']}`);
-    const sessionSecret = job.data.msg.sessionSecret;
-    delete job.data.msg.sessionSecret;
-
-    return getCredentials(job.data.msg.actor['@id'], job.data.sessionId, sessionSecret,
+  return (job: JobEncrypted, done: Function) => {
+    const jobData: JobDataDecrypted = decryptJobData(job, secret);
+    const jobLog = debug(`${loggerPrefix}:${jobData.sessionId}`);
+    jobLog(`received ${jobData.title} ${jobData.msg.type}`);
+    const sessionSecret = jobData.msg.sessionSecret;
+    delete jobData.msg.sessionSecret;
+
+    return getCredentials(jobData.msg.actor.id, jobData.sessionId, sessionSecret,
       (err, credentials) => {
         if (err) { return done(new Error(err)); }
+        let jobCallbackCalled = false;
         const doneCallback = (err, result) => {
+          if (jobCallbackCalled) { return; }
+          jobCallbackCalled = true;
           if (err) {
-            done(new Error(err));
+            jobLog(`errored ${jobData.title} ${jobData.msg.type}`);
+            let errMsg;
+            // some error objects (eg. TimeoutError) don't interoplate correctly to human-readable
+            // so we have to do this little dance
+            try {
+              errMsg = err.toString();
+            } catch (e) {
+              errMsg = err;
+            }
+            done(new Error(errMsg));
           } else {
+            jobLog(`completed ${jobData.title} ${jobData.msg.type}`);
             done(null, result);
           }
         };
         if ((Array.isArray(platform.config.requireCredentials)) &&
-          (platform.config.requireCredentials.includes(job.data.msg['@type']))) {
+          (platform.config.requireCredentials.includes(jobData.msg.type))) {
           // add the credentials object if this method requires it
-          platform[job.data.msg['@type']](job.data.msg, credentials, doneCallback);
+          platform[jobData.msg.type](jobData.msg, credentials, doneCallback);
+        } else if (platform.config.persist) {
+          if (platform.initialized) {
+            platform[jobData.msg.type](jobData.msg, doneCallback);
+          } else {
+            done(new Error(`${jobData.msg.type} called on uninitialized platform`));
+          }
         } else {
-          platform[job.data.msg['@type']](job.data.msg, doneCallback);
+          platform[jobData.msg.type](jobData.msg, doneCallback);
         }
       });
   };
@@ -131,7 +151,7 @@ function getJobHandler(secret: string) {
  * @param command string containing the type of command to be sent. 'message' or 'close'
  */
 function getSendFunction(command: string) {
-  return function (msg: ActivityObject, special?: string) {
+  return function (msg: ActivityStream, special?: string) {
     process.send([command, msg, special]);
   };
 }
@@ -142,8 +162,8 @@ function getSendFunction(command: string) {
  * @param credentials
  */
 function updateActor(credentials) {
-  identifier = getPlatformId(platformName, credentials.actor['@id']);
-  logger(`platform actor updated to ${credentials.actor['@id']} identifier ${identifier}`);
+  identifier = getPlatformId(platformName, credentials.actor.id);
+  logger(`platform actor updated to ${credentials.actor.id} identifier ${identifier}`);
   logger = debug(`sockethub:platform:${identifier}`);
   platform.credentialsHash = hash(credentials.object);
   platform.debug = debug(`sockethub:platform:${platformName}:${identifier}`);
@@ -162,7 +182,7 @@ function startQueueListener(refresh: boolean = false) {
     logger('start queue called multiple times, skipping');
     return;
   }
-  const queue = new Queue(parentId + identifier, redisConfig);
+  const queue = new Queue(parentId + identifier, { redis: config.get('redis') });
   queueStarted = true;
   logger('listening on the queue for incoming jobs');
   queue.process(getJobHandler(secret));

package/src/process-manager.ts

@@ -1,7 +1,7 @@
 import init from './bootstrap/init';
-import PlatformInstance, { platformInstances, PlatformInstanceParams } from "./platform-instance";
+import PlatformInstance, {
+  platformInstances, PlatformInstanceParams, MessageFromParent } from "./platform-instance";
 import { getPlatformId } from "./common";
-import { MessageFromParent } from "./platform-instance";
 
 class ProcessManager {
   private readonly parentId: string;
@@ -16,14 +16,17 @@ class ProcessManager {
 
   get(platform: string, actorId: string, sessionId?: string): PlatformInstance {
     const platformDetails = init.platforms.get(platform);
+    let pi;
 
     if (platformDetails.config.persist) {
       // ensure process is started - one for each actor
-      return this.ensureProcess(platform, sessionId, actorId);
+      pi = this.ensureProcess(platform, sessionId, actorId);
     } else {
       // ensure process is started - one for all jobs
-      return this.ensureProcess(platform);
+      pi = this.ensureProcess(platform);
     }
+    pi.config = platformDetails.config;
+    return pi;
   }
 
   private createPlatformInstance(identifier: string, platform: string,

package/src/routes.test.ts

@@ -1,12 +1,19 @@
+import { expect } from 'chai';
+import * as sinon from 'sinon';
 import { existsSync } from "fs";
 
 import routes, { basePaths, examplePaths, examplePages } from "./routes";
 
 describe('routes/base', () => {
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
   it('can find each of the base files it serves', () => {
     Object.values(basePaths).forEach((fwd: string) => {
       try {
-        expect(existsSync(fwd)).toBeTruthy();
+        expect(existsSync(fwd)).to.be.true;
       } catch (e) {
         throw new Error(`Unable to resolve path ${fwd}`);
       }
@@ -16,7 +23,7 @@ describe('routes/base', () => {
   it('can find each of the example files it serves', () => {
     Object.values(examplePaths).forEach((fwd: string) => {
       try {
-        expect(existsSync(fwd)).toBeTruthy();
+        expect(existsSync(fwd)).to.be.true;
       } catch (e) {
         throw new Error(`Unable to resolve path ${fwd}`);
       }
@@ -26,7 +33,7 @@ describe('routes/base', () => {
   it('can find each of the example page files it serves', () => {
     Object.values(examplePages).forEach((fwd: string) => {
       try {
-        expect(existsSync(fwd)).toBeTruthy();
+        expect(existsSync(fwd)).to.be.true;
       } catch (e) {
         throw new Error(`Unable to resolve path ${fwd}`);
       }
@@ -34,20 +41,26 @@ describe('routes/base', () => {
   });
 
   it('adds base routes', () => {
-    let app = {
-      get: jest.fn()
+    const app = {
+      get: sinon.spy()
     };
-    routes.setup(app);
-    expect(app['get']).toBeCalledTimes(Object.keys(basePaths).length);
+    routes.setup(app, false);
+    sinon.assert.callCount(
+      app.get,
+      Object.keys(basePaths).length
+    );
   });
 
   it('adds base and example routes', () => {
-    let app = {
-      get: jest.fn()
+    const app = {
+      get: sinon.spy()
     };
     routes.setup(app, true);
-    expect(app['get']).toBeCalledTimes(
-      Object.keys(basePaths).length + Object.keys(examplePaths).length + Object.keys(examplePages).length
+    sinon.assert.callCount(
+      app.get,
+      Object.keys(basePaths).length
+      + Object.keys(examplePaths).length
+      + Object.keys(examplePages).length
     );
   });
 
@@ -63,12 +76,13 @@ describe('routes/base', () => {
     function verifyPathRoutes(pathMap) {
       Object.keys(pathMap).forEach((path) => {
         const res = {
-          setHeader: jest.fn(),
-          sendFile: jest.fn()
+          setHeader: sinon.spy(),
+          sendFile: sinon.spy()
         };
+        expect(pathMap[path].endsWith('.ejs')).to.be.false;
         routeHandlers[path]({url: path}, res);
-        expect(res.setHeader).toHaveBeenCalled();
-        expect(res.sendFile).toHaveBeenCalledWith(pathMap[path]);
+        sinon.assert.called(res.setHeader);
+        sinon.assert.calledWith(res.sendFile, pathMap[path]);
       });
     }
     verifyPathRoutes(basePaths);
@@ -76,10 +90,11 @@ describe('routes/base', () => {
 
     Object.keys(examplePages).forEach((path) => {
       const res = {
-        render: jest.fn()
+        render: sinon.spy()
       };
+      expect(examplePages[path].endsWith('.ejs')).to.be.true;
       routeHandlers[path]({url: path}, res);
-      expect(res.render).toHaveBeenCalled();
+      sinon.assert.called(res.render);
     });
   });
 });

package/src/routes.ts

@@ -1,31 +1,31 @@
 import path from 'path';
 import config from "./config";
+import debug from 'debug';
 
 const debug_scope = process.env.DEBUG || '',
+      logger      = debug('sockethub:routes'),
       address     = config.get('public:protocol') + '://' +
                     config.get('public:host') + ':' +
                     config.get('public:port') +
                     config.get('public:path');
 
 export const basePaths = {
-  '/sockethub-client.js': path.resolve(`${__dirname}/js/client.js`),
+  '/sockethub-client.js': path.resolve(`${__dirname}/../dist/sockethub-client.js`),
+  '/sockethub-client.min.js': path.resolve(`${__dirname}/../dist/sockethub-client.min.js`),
+  '/sockethub-client.js.map': path.resolve(`${__dirname}/../dist/sockethub-client.js.map`),
   '/socket.io.js': path.resolve(`${__dirname}/../node_modules/socket.io/client-dist/socket.io.js`),
   '/socket.io.js.map': path.resolve(
     `${__dirname}/../node_modules/socket.io/client-dist/socket.io.js.map`),
-  // '/activity-streams.js':
-  // path.resolve(`${__dirname}/../node_modules/activity-streams/browser/activity-streams.js`),
-  '/activity-streams.min.js':
-    path.resolve(`${__dirname}/../node_modules/activity-streams/browser/activity-streams.min.js`),
 };
 
 export const examplePaths = {
-  '/': path.resolve(`${__dirname}/../views/index.ejs`),
   '/jquery.js': path.resolve(`${__dirname}/../node_modules/jquery/dist/jquery.min.js`),
   '/jquery.min.map': path.resolve(`${__dirname}/../node_modules/jquery/dist/jquery.min.map`),
   '/examples/shared.js': path.resolve(`${__dirname}/../views/examples/shared.js`)
 };
 
 export const examplePages = {
+  '/': path.resolve(`${__dirname}/../views/index.ejs`),
   '/examples/dummy': path.resolve(`${__dirname}/../views/examples/dummy.ejs`),
   '/examples/feeds': path.resolve(`${__dirname}/../views/examples/feeds.ejs`),
   '/examples/irc': path.resolve(`${__dirname}/../views/examples/irc.ejs`),
@@ -42,6 +42,7 @@ function prepFileRoutes(pathMap) {
         path: key
       },
       route: (req, res) => {
+        logger(`serving resource ${req.url}`);
         res.setHeader('Access-Control-Allow-Origin', '*');
         res.sendFile(pathMap[req.url]);
       }
@@ -60,6 +61,7 @@ Object.keys(examplePages).forEach((key) => {
       path: key
     },
     route: (req, res) => {
+      logger(`serving page ${req.url}`);
       res.render(examplePages[req.url], {
         debug_scope: debug_scope,
         address: address,

package/src/serve.ts

@@ -59,7 +59,7 @@ class Serve {
 
 const serve = new Serve();
 
-interface SocketInstance {
+export interface SocketInstance {
   id: string;
   emit: Function;
 }