roster-server 3.2.1 → 3.4.0

package/README.md

@@ -184,6 +184,30 @@ Isolation needs a Roster-owned listener, so it applies only when `start()` initi
 
 Each isolated process loads Node.js and its site, so memory use grows with the number of site directories. Code that runs when a site's module loads runs in that site's process, not in the main process.
 
+### Socket sites
+
+A site directory whose entry exports an object with a `connection(socket, head)` method receives raw client connections instead of requests. Use it for a site that terminates TLS itself, for example to spread connections across its own worker processes:
+
+```javascript
+// www/game.example.com/index.js
+export default {
+    async start({ certificate }) {
+        // Runs once before the first connection, e.g. to fork workers.
+        // certificate(servername) resolves { key, cert } PEM strings; cert includes the chain.
+    },
+    connection(socket, head) {
+        // Terminate TLS here, or pass socket and head to another process.
+    },
+    async close() {
+        // Release the site's resources; runs after Roster stops accepting connections.
+    }
+};
+```
+
+In production, `socket` is the client's TCP connection and `head` holds the TLS ClientHello that Roster already read; put it back with `socket.unshift(head)` before terminating TLS, and do not `resume()` the socket after wrapping it. In local mode, the site's assigned port accepts plain TCP connections, `socket` arrives paused, and `head` is empty. `connection()` runs synchronously; if it throws, Roster destroys the socket.
+
+`certificate(servername)` returns the certificate Roster would serve for that host, with the same wildcard and zone matching. A missing certificate is issued by the main process first, as for any isolated site. Roster also renews it; call `certificate()` again to pick up a renewal, for example on an interval or when a cached copy gets old. It is not available in local mode. Request plugins, `www.` redirects, and `onClose` hooks do not apply: the site owns every connection it receives. A socket site must run isolated; Roster rejects it when it would be served by the main process.
+
 ## Startup and shutdown
 
 Concurrent `init()` calls share initialization, so each factory runs once per registered domain/port. Concurrent `start()` calls share startup. In standalone and local modes, `await roster.start()` waits for listeners to bind. Initialization or bind failures reject and clean up resources already initialized by the instance.

package/index.js

@@ -8,7 +8,7 @@ const crypto = require('crypto');
 const { EventEmitter } = require('events');
 const Greenlock = require('./vendor/greenlock-express/greenlock-express.js');
 const GreenlockShim = require('./vendor/greenlock-express/greenlock-shim.js');
-const { locateSiteEntry, resolveSiteApp } = require('./lib/resolve-site-app.js');
+const { locateSiteEntry, resolveSiteApp, isSocketSite } = require('./lib/resolve-site-app.js');
 const { IsolatedSite } = require('./lib/isolated-site.js');
 const { parseClientHello } = require('./lib/client-hello.js');
 const log = require('lemonlog')('roster');
@@ -902,6 +902,9 @@ class Roster {
         this._sitesByPort = {};
         for (const [hostKey, siteApp] of Object.entries(this.sites)) {
             this._assertOpen();
+            if (isSocketSite(siteApp)) {
+                throw new Error(`${hostKey} serves raw connections and must run as an isolated site directory`);
+            }
             if (hostKey.startsWith('www.') || typeof siteApp !== 'function') continue;
             const { domain, port } = this.parseDomainWithPort(hostKey);
             if (!this._sitesByPort[port]) {

package/lib/resolve-site-app.js

@@ -41,4 +41,12 @@ async function resolveSiteApp(domainPath, options = {}) {
     return { siteApp, type: 'js' };
 }
 
-module.exports = { locateSiteEntry, resolveSiteApp };
+/**
+ * A socket site exports `{ start?(context), connection(socket, head), close?() }` and serves
+ * raw client connections itself, including TLS. Only an isolated site process can hand it sockets.
+ */
+function isSocketSite(siteApp) {
+    return typeof siteApp === 'object' && siteApp !== null && typeof siteApp.connection === 'function';
+}
+
+module.exports = { locateSiteEntry, resolveSiteApp, isSocketSite };

package/lib/site-worker.js

@@ -2,8 +2,9 @@
 
 // Entry point of an isolated site process, forked by IsolatedSite.
 const https = require('https');
+const net = require('net');
 const Roster = require('../index.js');
-const { resolveSiteApp } = require('./resolve-site-app.js');
+const { resolveSiteApp, isSocketSite } = require('./resolve-site-app.js');
 const { serializeError } = require('./isolated-site.js');
 const log = require('lemonlog')('roster');
 
@@ -11,6 +12,8 @@ class SiteWorkerRoster extends Roster {
     constructor(site) {
         super({ ...site.options, isolation: false, autoCertificates: false, handleSignals: false });
         this._site = site;
+        this._socketSite = null;
+        this._socketSiteClose = null;
         this._httpsServers = new Map();
         this._certificateRequests = new Map();
         this._nextCertificateRequest = 0;
@@ -31,9 +34,69 @@ class SiteWorkerRoster extends Roster {
             log.warn(`⚠️  No index file (js/mjs/cjs or index.html) found in ${sitePath}`);
             return;
         }
+        if (isSocketSite(resolved.siteApp)) {
+            this._socketSite = resolved.siteApp;
+            return;
+        }
         for (const key of keys) this.sites[key] = resolved.siteApp;
     }
 
+    get socketSite() {
+        return this._socketSite !== null;
+    }
+
+    // The site terminates TLS with certificates Roster resolves: read from the store, or issued by the main process.
+    startSocketSite() {
+        return this._socketSite.start?.({ certificate: servername => this.ensureCertificate(servername) });
+    }
+
+    // In local mode a socket site gets plain connections on its assigned ports, with no ClientHello.
+    async serveLocalSockets() {
+        const head = Buffer.alloc(0);
+        for (const key of this._site.keys) {
+            if (key.startsWith('www.')) continue;
+            const { domain, port } = this.parseDomainWithPort(key);
+            const server = net.createServer({ pauseOnConnect: true });
+            const listening = this._listen(server, this._site.localPorts[`${domain}:${port}`], 'localhost');
+            server.on('connection', socket => {
+                // The site owns the connection from here; it may pass it to another process.
+                this._sockets.delete(socket);
+                this._handOff(head, socket);
+            });
+            await listening;
+            log.info(`🌐 ${domain} → tcp://localhost:${this._site.localPorts[`${domain}:${port}`]} (socket site)`);
+        }
+    }
+
+    _handOff(head, socket) {
+        if (this._closing || socket.destroyed) {
+            socket.destroy();
+            return;
+        }
+        try {
+            this._socketSite.connection(socket, head);
+        } catch (error) {
+            socket.destroy();
+            log.error(`❌ Socket site ${this._site.name} rejected a connection:`, error);
+        }
+    }
+
+    close() {
+        if (!this._socketSite) return super.close();
+        // Stop accepting first, then let the site drain the connections it was handed.
+        this._socketSiteClose ??= Promise.allSettled([super.close()]).then(async ([listeners]) => {
+            const errors = listeners.status === 'rejected' ? [listeners.reason] : [];
+            try {
+                await this._socketSite.close?.();
+            } catch (error) {
+                errors.push(error);
+            }
+            const flat = errors.flatMap(error => error instanceof AggregateError ? error.errors : [error]);
+            if (flat.length) throw new AggregateError(flat, 'Roster close failed');
+        });
+        return this._socketSiteClose;
+    }
+
     generateConfigJson() {}
 
     assignPortToDomain(domain, port) {
@@ -81,6 +144,10 @@ class SiteWorkerRoster extends Roster {
     }
 
     accept(port, head, socket) {
+        if (this._socketSite) {
+            this._handOff(head, socket);
+            return;
+        }
         const server = this._httpsServers.get(port);
         if (this._closing || !server) {
             socket.destroy();
@@ -103,10 +170,13 @@ async function start(site) {
         for (const plugin of site.plugins) {
             roster.use(require(plugin.module)[plugin.export](plugin.options));
         }
-        if (roster.local) {
+        await roster.init();
+        if (roster.socketSite) {
+            await roster.startSocketSite();
+            if (roster.local) await roster.serveLocalSockets();
+        } else if (roster.local) {
             await roster.start();
         } else {
-            await roster.init();
             roster.serveHttps();
         }
         process.send({ type: 'ready' });

package/package.json

@@ -1,54 +1,54 @@
 {
-  "name": "roster-server",
-  "version": "3.2.1",
-  "description": "👾 RosterServer - A domain host router to host multiple HTTPS.",
-  "main": "index.js",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/clasen/RosterServer.git"
-  },
-  "keywords": [
-    "greenlock",
-    "https",
-    "domain",
-    "multi-domain",
-    "letsencrypt",
-    "hosting",
-    "ssl",
-    "host",
-    "vhost",
-    "virtual-host",
-    "socket.io",
-    "websocket",
-    "express",
-    "greenlock-express",
-    "shotx",
-    "bun",
-    "clasen"
-  ],
-  "author": "Martin Clasen",
-  "license": "MIT",
-  "bugs": {
-    "url": "https://github.com/clasen/RosterServer/issues"
-  },
-  "homepage": "https://github.com/clasen/RosterServer#readme",
-  "dependencies": {
-    "@greenlock/manager": "^3.1.0",
-    "@root/acme": "^3.1.0",
-    "@root/csr": "^0.8.1",
-    "@root/encoding": "^1.0.1",
-    "@root/keypairs": "^0.10.0",
-    "@root/mkdirp": "^1.0.0",
-    "@root/request": "^1.6.1",
-    "acme-dns-01-cli": "^3.0.7",
-    "acme-http-01-standalone": "^3.0.5",
-    "cert-info": "^1.5.1",
-    "greenlock-store-fs": "^3.2.2",
-    "lemonlog": "^1.2.0",
-    "redirect-https": "^1.3.1",
-    "safe-replace": "^1.1.0"
-  },
-  "scripts": {
-    "test": "node --test 'test/**/*.test.js'"
-  }
-}
+    "name": "roster-server",
+    "version": "3.4.0",
+    "description": "👾 RosterServer - A domain host router to host multiple HTTPS.",
+    "main": "index.js",
+    "scripts": {
+        "test": "node --test 'test/**/*.test.js'"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/clasen/RosterServer.git"
+    },
+    "keywords": [
+        "greenlock",
+        "https",
+        "domain",
+        "multi-domain",
+        "letsencrypt",
+        "hosting",
+        "ssl",
+        "host",
+        "vhost",
+        "virtual-host",
+        "socket.io",
+        "websocket",
+        "express",
+        "greenlock-express",
+        "shotx",
+        "bun",
+        "clasen"
+    ],
+    "author": "Martin Clasen",
+    "license": "MIT",
+    "bugs": {
+        "url": "https://github.com/clasen/RosterServer/issues"
+    },
+    "homepage": "https://github.com/clasen/RosterServer#readme",
+    "dependencies": {
+        "@greenlock/manager": "^3.1.0",
+        "@root/acme": "^3.1.0",
+        "@root/csr": "^0.8.1",
+        "@root/encoding": "^1.0.1",
+        "@root/keypairs": "^0.10.0",
+        "@root/mkdirp": "^1.0.0",
+        "@root/request": "^1.6.1",
+        "acme-dns-01-cli": "^3.0.7",
+        "acme-http-01-standalone": "^3.0.5",
+        "cert-info": "^1.5.1",
+        "greenlock-store-fs": "^3.2.2",
+        "lemonlog": "^1.2.0",
+        "redirect-https": "^1.3.1",
+        "safe-replace": "^1.1.0"
+    }
+}

package/test/isolation.test.js

@@ -405,6 +405,91 @@ it('closes isolated sites when the main process dies without closing', { timeout
     assert.equal(fs.readFileSync(marker, 'utf8'), 'closed');
 });
 
+function socketSite(server, marker, start = 'async start() {}') {
+    return `const fs = require('fs');
+let server = ${server};
+module.exports = {
+    ${start},
+    connection(socket, head) {
+        fs.writeFileSync(${JSON.stringify(marker)} + '.head', String(head.length));
+        // A ClientHello read by Roster goes back on the stream; a local connection arrives paused.
+        if (head.length > 0) socket.unshift(head);
+        server.emit('connection', socket);
+        if (head.length === 0) socket.resume();
+    },
+    reply(req, res) {
+        res.end(JSON.stringify({ pid: process.pid, address: req.socket.remoteAddress }));
+    },
+    close() {
+        fs.writeFileSync(${JSON.stringify(marker)}, 'closed');
+    }
+};`;
+}
+
+it('hands raw client sockets with their ClientHello to a socket site that terminates TLS', { timeout: 20000 }, async t => {
+    const marker = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'roster-socket-site-')), 'closed');
+    t.after(() => fs.rmSync(path.dirname(marker), { recursive: true, force: true }));
+    // The site gets its certificate from Roster's store through start(), not from files it reads.
+    const start = `async start({ certificate }) {
+        const pems = await certificate('raw.example.com');
+        server = require('https').createServer(pems, (req, res) => module.exports.reply(req, res));
+    }`;
+    const { roster, port } = await startProduction(t, {
+        'raw.example.com': socketSite('null', marker, start),
+        'plain.example.com': REPORT_SITE
+    }, { certificates: ['plain.example.com', 'raw.example.com'] });
+    const raw = JSON.parse((await get(port, 'raw.example.com')).body);
+    const plain = JSON.parse((await get(port, 'plain.example.com')).body);
+    assert.notEqual(raw.pid, process.pid);
+    assert.notEqual(raw.pid, plain.pid);
+    assert.match(raw.address, /127\.0\.0\.1$/);
+    assert.ok(Number(fs.readFileSync(`${marker}.head`, 'utf8')) > 0);
+    await roster.close();
+    assert.equal(fs.readFileSync(marker, 'utf8'), 'closed');
+    assert.equal(isRunning(raw.pid), false);
+});
+
+it('issues a missing certificate through the main process when a socket site asks for it', { timeout: 20000 }, async t => {
+    const marker = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'roster-socket-site-')), 'closed');
+    t.after(() => fs.rmSync(path.dirname(marker), { recursive: true, force: true }));
+    const store = path.join(path.dirname(marker), 'greenlock.d');
+    const issued = [];
+    const start = `async start({ certificate }) {
+        server = require('https').createServer(await certificate('fresh.example.com'), (req, res) => module.exports.reply(req, res));
+    }`;
+    const { port } = await startProduction(t, { 'fresh.example.com': socketSite('null', marker, start) }, {
+        options: { greenlockStorePath: store },
+        issue: async ({ servername }) => {
+            issued.push(servername);
+            installCertificate(store, servername);
+        }
+    });
+    assert.deepEqual(issued, ['fresh.example.com']);
+    assert.notEqual(JSON.parse((await get(port, 'fresh.example.com')).body).pid, process.pid);
+});
+
+it('gives a socket site plain connections on its local port', { timeout: 20000 }, async t => {
+    const marker = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'roster-socket-site-')), 'closed');
+    t.after(() => fs.rmSync(path.dirname(marker), { recursive: true, force: true }));
+    const { wwwPath } = createWww(t, {
+        'raw.example': socketSite(`require('http').createServer((req, res) => module.exports.reply(req, res))`, marker)
+    });
+    const roster = new Roster({ local: true, wwwPath, minLocalPort: 19370, maxLocalPort: 19379 });
+    t.after(() => roster.close().catch(() => {}));
+    await roster.start();
+    const { port } = new URL(roster.getUrl('raw.example'));
+    assert.notEqual(JSON.parse((await localGet(port)).body).pid, process.pid);
+    assert.equal(fs.readFileSync(`${marker}.head`, 'utf8'), '0');
+    await roster.close();
+    assert.equal(fs.readFileSync(marker, 'utf8'), 'closed');
+});
+
+it('refuses a socket site served by the main process', { timeout: 20000 }, async t => {
+    const { wwwPath } = createWww(t, { 'raw.example': `module.exports = { connection() {} };` });
+    const roster = new Roster({ local: true, wwwPath, isolation: false, minLocalPort: 19380, maxLocalPort: 19389 });
+    await assert.rejects(roster.start(), /must run as an isolated site directory/);
+});
+
 it('rejects combining isolation with the legacy cluster launcher', () => {
     assert.throws(() => new Roster({ cluster: true, isolation: true }), /cannot be combined/);
     assert.equal(new Roster({ cluster: true }).isolation, null);