telemersive-bus 0.6.19 → 0.6.20

package/lib/BusManager.js

@@ -21,6 +21,15 @@ const houseKeepingTimeOut_onPeerJoined = 5000;
 // OS ports — without this delay a room restart races against a still-dying proxy and fails
 // to bind the same port.
 const proxyShutdownSettleTime = 3000;
+// how often the switchboard is asked whether it still holds the proxies it was
+// told to run. deliberately on its own timer rather than part of housekeeping:
+// housekeeping only runs when a peer joins or leaves, so a room in the middle of
+// a performance can go for a very long time without a single cycle.
+const switchBoardCheckInterval = 15000;
+// how often a single port is sent to the switchboard again before it is left
+// alone. a port that never starts is a real fault, and retrying it every round
+// forever would only bury it in noise.
+const maxPortRepairAttempts = 3;
 const ROOM_ID_MIN = 11;
 const ROOM_ID_MAX = 50;
 
@@ -52,6 +61,14 @@ class BusManager {
         this.resolveHousekeepingCycle = null;
         this.chatManagers = {};
         this.switchBoardURI = null;
+        this.switchBoardBaseURI = null;
+        // the switchboard's instance id as last seen. a different one means the
+        // switchboard was restarted and has forgotten every proxy we asked for.
+        this.switchBoardInstance = null;
+        this.switchBoardWatch = null;
+        // per room, how often each port has been sent to the switchboard again
+        // without it taking. keyed by room name, then by port.
+        this.portRepairAttempts = {};
     }
 
     /*********************************************************************
@@ -68,7 +85,134 @@ class BusManager {
      * @param _port switchboard port
      */
     configureSwitchBoard = (_url, _port) => {
-        this.switchBoardURI = 'http://' + _url + ":" + _port + "/proxies/";
+        this.switchBoardBaseURI = 'http://' + _url + ":" + _port;
+        this.switchBoardURI = this.switchBoardBaseURI + "/proxies/";
+    }
+
+    /**
+     * start watching whether the switchboard still holds the proxies it was told
+     * to run.
+     *
+     * proxies live only in the switchboard's memory, so a restart leaves every
+     * room without a media path while the rooms themselves carry on: peers keep
+     * answering their pings, no room is torn down, and nothing would ever ask for
+     * the ports again. This manager is the only part that knows which proxies a
+     * room should have, so it is the one that has to notice and send them again.
+     */
+    startSwitchBoardWatch = () => {
+        if (this.switchBoardBaseURI === null || this.switchBoardWatch !== null) {
+            return;
+        }
+        this.switchBoardWatch = setInterval(() => {
+            this.checkSwitchBoard().catch((err) => {
+                console.error(`      ...: ${err}. Could not check the switchboard.`);
+            });
+        }, switchBoardCheckInterval);
+    }
+
+    stopSwitchBoardWatch = () => {
+        if (this.switchBoardWatch !== null) {
+            clearInterval(this.switchBoardWatch);
+            this.switchBoardWatch = null;
+        }
+    }
+
+    /**
+     * ask the switchboard what it is running and rebuild whatever is missing.
+     *
+     * only ports the switchboard does not know about are sent again. a port it
+     * knows but is not running is its own business: it revives those itself, and
+     * re-sending one would reset the switchboard's own give-up counter and leave
+     * the two of us restarting the same broken proxy forever.
+     */
+    checkSwitchBoard = async () => {
+        const health = await this.getSwitchBoardHealth();
+        if (health === null) {
+            return;
+        }
+        if (this.switchBoardInstance !== null && this.switchBoardInstance !== health.instance) {
+            console.log(`  <- switchboard was restarted - it has forgotten the ports of ${Object.keys(this.rooms).length} room(s)`);
+        }
+        this.switchBoardInstance = health.instance;
+
+        for (const roomName of Object.keys(this.rooms)) {
+            const room = this.rooms[roomName];
+            if (room === undefined || room.flagRoom4cleanup) {
+                // a room on its way out must not have its ports built up again
+                continue;
+            }
+            const expected = this.roomProxyDefinitions(room.roomId);
+            const held = (health.rooms && health.rooms[roomName]) ? health.rooms[roomName] : 0;
+            if (held >= expected.length) {
+                continue;
+            }
+            await this.repairServerSidePortScripts(roomName, room.roomId);
+        }
+    }
+
+    /**
+     * send the build commands again for the ports of a room the switchboard does
+     * not have.
+     */
+    repairServerSidePortScripts = async (_roomName, _roomId) => {
+        const held = await this.getSwitchBoardRoom(_roomName);
+        if (held === null) {
+            return;
+        }
+        const attempts = this.portRepairAttempts[_roomName] || {};
+        this.portRepairAttempts[_roomName] = attempts;
+        const missing = this.roomProxyDefinitions(_roomId).filter(
+            (definition) => !Object.prototype.hasOwnProperty.call(held, String(definition.port)));
+        const worthTrying = missing.filter(
+            (definition) => (attempts[definition.port] || 0) < maxPortRepairAttempts);
+        if (worthTrying.length === 0) {
+            return;
+        }
+        console.log(`        -> room '${_roomName}' is missing ${missing.length} port(s) on the switchboard - sending ${worthTrying.length} of them again`);
+        const failed = await this.addSwitchBoardProxies(_roomName, worthTrying);
+        const failedPorts = new Set(failed.map((definition) => definition.port));
+        for (const definition of worthTrying) {
+            if (failedPorts.has(definition.port)) {
+                attempts[definition.port] = (attempts[definition.port] || 0) + 1;
+                if (attempts[definition.port] >= maxPortRepairAttempts) {
+                    console.error(`        <- giving up on port ${definition.port} of room '${_roomName}' after ${maxPortRepairAttempts} attempts`);
+                }
+            } else {
+                delete attempts[definition.port];
+            }
+        }
+        console.log(`        <- restored ${worthTrying.length - failed.length} of ${worthTrying.length} port(s) for room ${_roomName}`);
+    }
+
+    /**
+     * what the switchboard is currently running. returns null when it cannot be
+     * reached, so a momentary hiccup is simply retried on the next round.
+     */
+    getSwitchBoardHealth = async () => {
+        try {
+            const res = await superagent.get(this.switchBoardBaseURI + '/health');
+            return JSON.parse(res.text);
+        } catch (err) {
+            console.error(`      ...: ${err}. Could not reach the switchboard.`);
+            return null;
+        }
+    }
+
+    /**
+     * the proxies the switchboard holds for a room, keyed by port. an unknown
+     * room answers 404, which means it holds none of them.
+     */
+    getSwitchBoardRoom = async (_roomName) => {
+        try {
+            const res = await superagent.get(this.switchBoardBaseURI + '/rooms/' + encodeURIComponent(_roomName));
+            return JSON.parse(res.text);
+        } catch (err) {
+            if (err.status === 404) {
+                return {};
+            }
+            console.error(`      ...: ${err}. Could not read room '${_roomName}' from the switchboard.`);
+            return null;
+        }
     }
 
     /*********************************************************************
@@ -112,12 +256,15 @@ class BusManager {
         console.log(`-> clean out the house ... `);
         // we wait a moment to start with houskeeping
         setTimeout(this.startHousekeeping, houseKeepingTimeOut);
+        // and keep an eye on the switchboard, independently of housekeeping
+        this.startSwitchBoardWatch();
     };
 
     /**
      * disconnect from mqtt broker
      */
     disconnectServer = async () => {
+        this.stopSwitchBoardWatch();
         await this.communicator.disconnect();
         this.communicator.clearSubscriptions();
     };
@@ -673,45 +820,74 @@ class BusManager {
     /**
      * Start server side port scripts
      */
+    /**
+     * the full set of proxies a room is made of.
+     *
+     * this is the single description of a room's ports: it is used both to build
+     * a new room and to work out what is missing from one that already exists, so
+     * the two can not drift apart.
+     */
+    roomProxyDefinitions = (_roomId) => {
+        const definitions = [];
+        /** the ultragrid needs two hundert range */
+        for (let i = 0; i < 20; i++) {
+            // OSC proxies
+            definitions.push({
+                port: _roomId * 1000 + i * 10 + 9, many_port: _roomId * 1000 + i * 10 + 9,
+                type: 'many2manyBi', description: "UDP proxy for OSC, channel:" + i});
+            // Ultragrid video and audio proxies
+            definitions.push({
+                port: _roomId * 1000 + i * 10 + 2, many_port: _roomId * 1000 + i * 10 + 6,
+                type: 'one2manyMo', description: "Ultragrid-video proxy, channel:" + i});
+            definitions.push({
+                port: _roomId * 1000 + i * 10 + 4, many_port: _roomId * 1000 + i * 10 + 8,
+                type: 'one2manyMo', description: "Ultragrid-audio / NatNet-data proxy, channel:" + i});
+            // NatNet2OSC and NatNetBridge proxies
+            definitions.push({
+                port: _roomId * 1000 + i * 10 + 0, many_port: _roomId * 1000 + i * 10 + 1,
+                type: 'one2manyBi', description: "MoCap/NatNet-Ctrl proxy, channel:" + i});
+        }
+        // Stage Control Many to many proxy
+        definitions.push({
+            port: _roomId * 1000 + 902, many_port: _roomId * 1000 + 902,
+            type: 'many2manyBi', description: "UDP proxy for Open Stage Control"});
+        definitions.push({
+            port: _roomId * 1000 + 900, many_port: _roomId * 1000 + 902,
+            type: 'OpenStageControl', description: "Open Stage Control instance"});
+        return definitions;
+    }
+
+    /**
+     * ask the switchboard to run the given proxies. returns the definitions it
+     * would not start, so the caller can tell a fully built room from a partial
+     * one instead of the failures only being logged and forgotten.
+     */
+    addSwitchBoardProxies = async (_roomName, _definitions) => {
+        const failed = [];
+        for (const definition of _definitions) {
+            try {
+                await this.addSwitchBoardProxy(_roomName, definition.port,
+                    definition.many_port, definition.type, definition.description);
+            } catch (err) {
+                failed.push(definition);
+                console.error(`      ...: ${err}. Interrupting process of starting up ${definition.type} script on port ${definition.port}.`);
+            }
+        }
+        return failed;
+    }
+
     startServerSidePortScripts = async (_roomName, _roomId) => {
         if(this.switchBoardURI !== null){
             console.log(`        -> starting ports for room '${_roomName}' on range '${_roomId}000 - ${_roomId}999`);
-            /** the ultragrid needs two hundert range */
-            for (let i = 0; i < 20; i++) {
-                // OSC proxies
-                try{
-                    await this.addSwitchBoardProxy(_roomName,_roomId * 1000 + i * 10 + 9, _roomId * 1000 + i * 10 + 9, 'many2manyBi', "UDP proxy for OSC, channel:" + i);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of starting up many2manyBi script on port ${_roomId * 1000 + i * 10 + 9}.`);
-                }
-                // Ultragrid video and audio proxies
-                try{
-                    await this.addSwitchBoardProxy(_roomName, _roomId * 1000 + i * 10 + 2, _roomId * 1000 + i * 10 + 6,'one2manyMo', "Ultragrid-video proxy, channel:" + i);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of starting up one2manyMo script on port ${_roomId * 1000 + i * 10 + 2}.`);
-                }
-                try{
-                    await this.addSwitchBoardProxy(_roomName, _roomId * 1000 + i * 10 + 4, _roomId * 1000 + i * 10 + 8, 'one2manyMo', "Ultragrid-audio / NatNet-data proxy, channel:" + i);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of starting up one2manyMo script on port ${_roomId * 1000 + i * 10 + 4}.`);
-                }
-                // NatNet2OSC and NatNetBridge proxies
-                try{
-                    await this.addSwitchBoardProxy(_roomName, _roomId * 1000 + i * 10 + 0, _roomId * 1000 + i * 10 + 1, 'one2manyBi', "MoCap/NatNet-Ctrl proxy, channel:" + i);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of starting up one2manyBi script on port ${_roomId * 1000 + i * 10 + 0}.`);
-                }
+            let failed = await this.addSwitchBoardProxies(_roomName, this.roomProxyDefinitions(_roomId));
+            if (failed.length > 0) {
+                // retry once before giving up: a port that is not created here is
+                // simply absent, and nothing downstream would ever notice it.
+                console.log(`        -> retrying ${failed.length} port(s) for room '${_roomName}' that could not be started`);
+                failed = await this.addSwitchBoardProxies(_roomName, failed);
             }
-            // Stage Control Many to many proxy
-            try{
-                await this.addSwitchBoardProxy(_roomName,_roomId * 1000 + 902, _roomId * 1000 + 902, 'many2manyBi', "UDP proxy for Open Stage Control");
-            } catch(err){
-                console.error(`      ...: ${err}. Interrupting process of starting up many2manyBi script on port ${_roomId * 1000 + 902}.`);
-            }
-            try{
-                await this.addSwitchBoardProxy(_roomName,_roomId * 1000 + 900, _roomId * 1000 + 902, 'OpenStageControl', "Open Stage Control instance");
-            } catch(err){
-                console.error(`      ...: ${err}. Interrupting process of starting up many2manyBi script on port ${_roomId * 1000 + 902}.`);
+            if (failed.length > 0) {
+                console.error(`        <- room ${_roomName} is missing ${failed.length} of ${this.roomProxyDefinitions(_roomId).length} ports: ${failed.map(d => d.port).join(', ')}`);
             }
             console.log(`        <- started all ports for room ${_roomName}`);
         } else {
@@ -725,43 +901,19 @@ class BusManager {
     stopServerSidePortScripts = async (_roomName, _roomId) => {
         if(this.switchBoardURI !== null) {
             console.log(`        -> stopping ports for room '${_roomName}' on range '${_roomId}000 - ${_roomId}999`);
-            for (let i = 0; i < 20; i++) {
-                try{
-                    await this.deleteSwitchBoardProxy(_roomId * 1000 + i * 10 + 9);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of stopping many2manyBi script on port ${_roomId * 1000 + i * 10 + 9}.`);
-                }
-                try{
-                    await this.deleteSwitchBoardProxy(_roomId * 1000 + i * 10 + 2);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of stopping one2manyMo script on port ${_roomId * 1000 + i * 10 + 2}.`);
-                }
-                try{
-                    await this.deleteSwitchBoardProxy(_roomId * 1000 + i * 10 + 4);
-                } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of stopping one2manyMo script on port ${_roomId * 1000 + i * 10 + 4}.`);
-                }
+            for (const definition of this.roomProxyDefinitions(_roomId)) {
                 try{
-                    await this.deleteSwitchBoardProxy(_roomId * 1000 + i * 10 + 0);
+                    await this.deleteSwitchBoardProxy(definition.port);
                 } catch(err){
-                    console.error(`      ...: ${err}. Interrupting process of stopping one2manyBi script on port ${_roomId * 1000 + i * 10 + 0}.`);
+                    console.error(`      ...: ${err}. Interrupting process of stopping ${definition.type} script on port ${definition.port}.`);
                 }
             }
-            try{
-                await this.deleteSwitchBoardProxy(_roomId * 1000 + 902);
-            } catch(err){
-                console.error(`      ...: ${err}. Interrupting process of stopping many2manyBi script on port ${_roomId * 1000 + 902}.`);
-            }
-            try{
-                await this.deleteSwitchBoardProxy(_roomId * 1000 + 900);
-            } catch(err){
-                console.error(`      ...: ${err}. Interrupting process of stopping open stage control instance on port ${_roomId * 1000 + 900}.`);
-            }
             console.log(`        <- stopped all ports for room ${_roomName}`);
             // wait for proxy processes to fully exit and release OS ports before allowing
             // the room to be recreated. without this, a restart races against dying proxies.
             await new Promise(resolve => setTimeout(resolve, proxyShutdownSettleTime));
             delete this.rooms[_roomName];
+            delete this.portRepairAttempts[_roomName];
             console.log(`        <- removed room ${_roomName}`);
         }else {
             console.log(`        ...stopped no ports for room ${_roomName}: no switchboard url/port defined`);
@@ -780,8 +932,18 @@ class BusManager {
             //let reply = JSON.parse(res["text"]);
         } catch (err) {
             if (err.response?.text) {
-                let reply = JSON.parse(err.response["text"]);
-                throw new Error(reply["msg"]);
+                // the switchboard answers errors as json, but an unhandled one
+                // arrives as an html page - do not let parsing that hide the
+                // actual failure behind a SyntaxError.
+                try {
+                    const reply = JSON.parse(err.response["text"]);
+                    throw new Error(reply["msg"]);
+                } catch (parseErr) {
+                    if (parseErr instanceof SyntaxError) {
+                        throw new Error(`switchboard responded ${err.status}`);
+                    }
+                    throw parseErr;
+                }
             }
             throw err;
         }
@@ -797,8 +959,18 @@ class BusManager {
             //let reply = JSON.parse(res["text"]);
         } catch (err) {
             if (err.response?.text) {
-                let reply = JSON.parse(err.response["text"]);
-                throw new Error(reply["msg"]);
+                // the switchboard answers errors as json, but an unhandled one
+                // arrives as an html page - do not let parsing that hide the
+                // actual failure behind a SyntaxError.
+                try {
+                    const reply = JSON.parse(err.response["text"]);
+                    throw new Error(reply["msg"]);
+                } catch (parseErr) {
+                    if (parseErr instanceof SyntaxError) {
+                        throw new Error(`switchboard responded ${err.status}`);
+                    }
+                    throw parseErr;
+                }
             }
             throw err;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "telemersive-bus",
-  "version": "0.6.19",
+  "version": "0.6.20",
   "description": "MQTT based data protocol to manage unlimited peers, connected by rooms, where all peers joined to a room can exchange private data. It provides a simple chat mechanism, latency pinging, publish-subscribe mechanism (based on mqtt) and a OSC-like data stream.  ",
   "main": "index.js",
   "directories": {