pinokiod 8.2.31 → 8.2.33

package/kernel/anywhere/forward.js

@@ -30,6 +30,14 @@ function openPath(label, port, to) {
   return `/anywhere/peers/${encodeURIComponent(label)}/open?port=${Number(port)}&to=${encodeURIComponent(to || '/')}`
 }
 
+// The headers go out as the incoming request's headers. Setting them in http-proxy's proxyReq
+// hook instead would throw: with the proxy agent the hook fires once the CONNECT completes, and
+// by then the outgoing headers are already written.
+function mark(req) {
+  for (const name of STRIPPED_HEADERS) delete req.headers[name]
+  req.headers[FORWARDED_HEADER] = '1'
+}
+
 class Forwarders {
   // proxyUrl: () => 'http://127.0.0.1:<port>' or null while the daemon is down
   // serverPort: () => pinokiod's own port
@@ -74,8 +82,6 @@ class Forwarders {
     const label = String(dnsName).split('.')[0]
     const target = `${this.scheme}://${dnsName}:${port}`
     const proxy = httpProxy.createProxyServer({ target, agent: this.agent(), changeOrigin: true, secure: true, ws: true })
-    proxy.on('proxyReq', (proxyReq) => { proxyReq.setHeader(FORWARDED_HEADER, '1') })
-    proxy.on('proxyReqWs', (proxyReq) => { proxyReq.setHeader(FORWARDED_HEADER, '1') })
     proxy.on('error', (error, req, res) => {
       const message = `Could not reach ${dnsName}: ${error && error.message ? error.message : error}`
       if (res && typeof res.writeHead === 'function' && !res.headersSent) {
@@ -85,7 +91,6 @@ class Forwarders {
         res.destroy()
       }
     })
-    const strip = (req) => { for (const name of STRIPPED_HEADERS) delete req.headers[name] }
     const server = http.createServer((req, res) => {
       const url = new URL(req.url, 'http://127.0.0.1')
       if (url.pathname === OPEN_PATH) {
@@ -100,11 +105,11 @@ class Forwarders {
         res.end()
         return
       }
-      strip(req)
+      mark(req)
       proxy.web(req, res)
     })
     server.on('upgrade', (req, socket, head) => {
-      strip(req)
+      mark(req)
       proxy.ws(req, socket, head)
     })
     // every connection on both sides, upgraded ones included, so stopAll can end a stream a
@@ -138,7 +143,7 @@ class Forwarders {
   }
 }
 
-// Resolves { status, body }; rejects on a transport failure or when the whole
+// Resolves { status, headers, raw, body }; rejects on a transport failure or when the whole
 // request, the proxy's CONNECT included, exceeds the deadline.
 function requestThroughProxy(agent, url, { method = 'GET', headers = {}, body = null, timeoutMs = 4000 } = {}) {
   return new Promise((resolve, reject) => {
@@ -150,7 +155,11 @@ function requestThroughProxy(agent, url, { method = 'GET', headers = {}, body =
     const req = lib.request(target, { method, headers, agent }, (res) => {
       const chunks = []
       res.on('data', (chunk) => chunks.push(chunk))
-      res.on('end', () => { clearTimeout(deadline); resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }) })
+      res.on('end', () => {
+        clearTimeout(deadline)
+        const raw = Buffer.concat(chunks)
+        resolve({ status: res.statusCode, headers: res.headers, raw, body: raw.toString('utf8') })
+      })
       res.on('error', (error) => { clearTimeout(deadline); reject(error) })
     })
     req.on('error', (error) => { clearTimeout(deadline); reject(error) })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.2.31",
+  "version": "8.2.33",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -1069,7 +1069,7 @@ class Server {
   anywherePeerAllowed(req) {
     if (!this.isLocalRequest(req)) return false
     const origin = req.get("Origin")
-    if (!origin) return true
+    if (!origin || origin === "https://pinokio.co") return true
     try { return new URL(origin).host.toLowerCase() === String(req.get("host")).toLowerCase() } catch (_) { return false }
   }
   // the reason the peer routes cannot work right now, or null
@@ -1096,6 +1096,7 @@ class Server {
       const apps = Array.isArray(status.apps) ? status.apps : []
       return {
         name: label,
+        address: `https://${peer.dnsName}`,
         apps: apps.map((app) => {
           const { remote, ...rest } = app
           return Object.assign(rest, { open: this.anywherePeerOpenPath(label, remote) })
@@ -1135,31 +1136,35 @@ class Server {
     if (location.origin !== new URL(base).origin) location = new URL(base)
     return { status: 302, headers: { Location: location.href, "Cache-Control": "no-store" } }
   }
-  // POST /anywhere/peers/<name>/apps/<launch|stop>: the peer's answer, an open URL turned into this node's open route
-  async relayAnywherePeerAction(name, action, body = {}) {
+  // /anywhere/peers/<name>/<path>: the peer's own API through the daemon's proxy (API spec, Relays).
+  // JSON routes get the peer's published addresses turned into this node's open routes, absolute,
+  // since the client may be pinokio.co on another origin; an asset is passed through as it is.
+  async relayAnywherePeer(name, method, relayPath, body = {}) {
     const anywhere = this.kernel.anywhere
     const peer = anywhere.peerFor(name)
     if (!peer) return { status: 400, body: { error: `Not a computer on your network: ${name}` } }
     const unavailable = this.anywherePeerUnavailable()
     if (unavailable) return { status: 502, body: { error: unavailable } }
+    const json = relayPath.startsWith("anywhere/")
     let answer
     try {
-      answer = await anywhere.forwarders.request(`https://${peer.dnsName}/anywhere/apps/${action}`, {
-        method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body || {}), timeoutMs: 30000,
-      })
+      answer = await anywhere.forwarders.request(`https://${peer.dnsName}/${relayPath}`, method === "POST"
+        ? { method, headers: { "Content-Type": "application/json" }, body: JSON.stringify(body || {}), timeoutMs: 30000 }
+        : { method, timeoutMs: 8000 })
     } catch (error) {
       return { status: 502, body: { error: `Could not reach ${peer.name}: ${error && error.message ? error.message : error}` } }
     }
+    if (!json) return { status: answer.status, raw: answer.raw, contentType: answer.headers["content-type"] || "application/octet-stream" }
     let parsed
     try { parsed = JSON.parse(answer.body) } catch (_) {
       return { status: 502, body: { error: `${peer.name} did not answer with JSON (HTTP ${answer.status})` } }
     }
-    // only the peer's own published address becomes this node's open route; any other URL passes through
-    if (parsed && parsed.action === "open" && typeof parsed.url === "string") {
-      let host = null
-      try { host = new URL(parsed.url).hostname.toLowerCase() } catch (_) { }
-      if (host === peer.dnsName.toLowerCase()) parsed.url = this.anywherePeerOpenPath(peer.dnsName.split(".")[0], parsed.url)
+    const own = (url) => { try { return new URL(url).hostname.toLowerCase() === peer.dnsName.toLowerCase() } catch (_) { return false } }
+    const route = (url) => `http://127.0.0.1:${anywhere.serverPort()}${this.anywherePeerOpenPath(peer.dnsName.split(".")[0], url)}`
+    if (parsed && Array.isArray(parsed.apps)) {
+      for (const app of parsed.apps) if (app && typeof app.remote === "string" && own(app.remote)) app.remote = route(app.remote)
     }
+    if (parsed && parsed.action === "open" && typeof parsed.url === "string" && own(parsed.url)) parsed.url = route(parsed.url)
     return { status: answer.status, body: parsed }
   }
   // the app's menu as launch targets: scripts and web addresses, never interface links;
@@ -7002,6 +7007,12 @@ class Server {
         this.kernel.vault.automaticScans.supported)
       next()
     })
+    // pinokio.co's Remote tab reaches the peer routes from a browser on this computer; Chrome's
+    // local network access wants this on the preflight, which the cors middleware below answers
+    this.app.use("/anywhere/peers", (req, res, next) => {
+      if (req.get("Access-Control-Request-Private-Network")) res.set("Access-Control-Allow-Private-Network", "true")
+      next()
+    })
     this.app.use(cors({
       origin: '*'
     }));
@@ -14512,7 +14523,8 @@ class Server {
       try {
         const answer = await fn(req)
         if (answer.headers) res.set(answer.headers)
-        if (answer.body === undefined) res.status(answer.status).end()
+        if (answer.raw) res.status(answer.status).type(answer.contentType).send(answer.raw)
+        else if (answer.body === undefined) res.status(answer.status).end()
         else res.status(answer.status).json(answer.body)
       } catch (error) {
         res.status(500).json({ error: error && error.message ? error.message : String(error) })
@@ -14520,8 +14532,13 @@ class Server {
     })
     this.app.get("/anywhere/peers", anywherePeerRoute(() => this.composeAnywherePeers()))
     this.app.get("/anywhere/peers/:name/open", anywherePeerRoute((req) => this.anywherePeerOpen(req.params.name, req.query)))
-    this.app.post("/anywhere/peers/:name/apps/launch", anywherePeerRoute((req) => this.relayAnywherePeerAction(req.params.name, "launch", req.body)))
-    this.app.post("/anywhere/peers/:name/apps/stop", anywherePeerRoute((req) => this.relayAnywherePeerAction(req.params.name, "stop", req.body)))
+    // the peer's own API under its prefix: a client swaps the peer's address for this prefix and changes nothing else
+    const anywhereRelay = (method, relayPath) => anywherePeerRoute((req) => this.relayAnywherePeer(req.params.name, method, relayPath(req), req.body))
+    this.app.get("/anywhere/peers/:name/anywhere/status", anywhereRelay("GET", () => "anywhere/status"))
+    this.app.get("/anywhere/peers/:name/anywhere/apps", anywhereRelay("GET", () => "anywhere/apps"))
+    this.app.post("/anywhere/peers/:name/anywhere/apps/launch", anywhereRelay("POST", () => "anywhere/apps/launch"))
+    this.app.post("/anywhere/peers/:name/anywhere/apps/stop", anywhereRelay("POST", () => "anywhere/apps/stop"))
+    this.app.get("/anywhere/peers/:name/asset/*", anywhereRelay("GET", (req) => "asset/" + encodeURI(req.params[0])))
     const anywhereAction = (label, fn) => ex(async (req, res) => {
       const anywhere = this.kernel.anywhere
       if (!anywhere) {

package/server/views/anywhere.ejs

@@ -517,7 +517,7 @@ document.addEventListener('DOMContentLoaded', function() {
       button.disabled = true
       othersNote = ''
       try {
-        const response = await fetch(`/anywhere/peers/${encodeURIComponent(peer)}/apps/stop`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: button.dataset.stop }) })
+        const response = await fetch(`/anywhere/peers/${encodeURIComponent(peer)}/anywhere/apps/stop`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: button.dataset.stop }) })
         const answer = await response.json().catch(() => ({}))
         if (!response.ok) othersNote = `Could not stop it on ${peer}: ${answer.error || `HTTP ${response.status}`}`
       } catch (e) {

package/test/anywhere-forward.test.js

@@ -55,6 +55,39 @@ test('a forwarder relays requests with the forwarded header, the peer as Host, a
   assert.equal(target.seen[0].headers['x-pinokio-source-host'], undefined)
 })
 
+// A forward proxy for plain http targets: the real ProxyAgent sends it absolute URLs, and
+// connects asynchronously, which is what broke the proxyReq-hook version in production.
+async function forwardProxy(t) {
+  const server = http.createServer((req, res) => {
+    const target = new URL(req.url)
+    const upstream = http.request({ host: target.hostname, port: target.port, path: target.pathname + target.search, method: req.method, headers: req.headers }, (answer) => {
+      res.writeHead(answer.statusCode, answer.headers)
+      answer.pipe(res)
+    })
+    upstream.on('error', (error) => { res.writeHead(502); res.end(error.message) })
+    req.pipe(upstream)
+  })
+  server.listen(0, '127.0.0.1')
+  await once(server, 'listening')
+  t.after(() => { server.closeAllConnections(); server.close() })
+  return `http://127.0.0.1:${server.address().port}`
+}
+
+test('through the real proxy agent, whose connection completes later, the forwarded header still goes out and nothing throws', async (t) => {
+  const target = await peer(t)
+  const proxyUrl = await forwardProxy(t)
+  const forwarders = new Forwarders({ proxyUrl: () => proxyUrl, serverPort: () => 42000, scheme: 'http' })
+  t.after(() => forwarders.stopAll())
+  const local = await forwarders.open('127.0.0.1', target.port)
+  const answer = await get(local, '/ui/?a=1', { 'X-Pinokio-Peer': '1' })
+  assert.equal(answer.status, 200)
+  const body = JSON.parse(answer.body)
+  assert.equal(body.forwarded, '1')
+  assert.equal(body.peer, null)
+  assert.equal(body.host, `127.0.0.1:${target.port}`)
+  assert.equal(target.seen.length, 1)
+})
+
 test('two opens of the same peer and port at once share one forwarder', async (t) => {
   const target = await peer(t)
   const forwarders = new Forwarders({ proxyUrl: () => null, serverPort: () => 42000, scheme: 'http' })

package/test/anywhere-others.test.js

@@ -71,7 +71,7 @@ test('with none found the section says so, and a stop is relayed to the peer the
   stop.click()
   await new Promise((resolve) => setImmediate(resolve)); await new Promise((resolve) => setImmediate(resolve))
   const posted = h.state.calls.find((call) => call.method === 'POST')
-  assert.deepEqual([posted.url, posted.body], ['/anywhere/peers/pinokio-1/apps/stop', '{"id":"comfy"}'])
+  assert.deepEqual([posted.url, posted.body], ['/anywhere/peers/pinokio-1/anywhere/apps/stop', '{"id":"comfy"}'])
   assert.equal(h.state.calls.filter((call) => call.url === '/anywhere/peers').length, 4, 'load, two polls, and once more after the stop')
   assert.equal(h.section().querySelector('.anywhere-error'), null)
   h.state.stopFails = true

package/test/anywhere-peers.test.js

@@ -43,7 +43,7 @@ test('peers lists the online computers that answered a status, with apps linked
   })
   const out = await Server.prototype.composeAnywherePeers.call(fake(anywhere))
   assert.equal(out.status, 200)
-  assert.deepEqual(out.body, { peers: [{ name: 'pinokio-1', apps: [
+  assert.deepEqual(out.body, { peers: [{ name: 'pinokio-1', address: 'https://pinokio-1.tail1.ts.net', apps: [
     { id: 'comfy', name: 'ComfyUI', state: 'ready', exposed: true, open: '/anywhere/peers/pinokio-1/open?port=43000&to=%2Fui%2F%3Fa%3D1%23v' },
     { id: 'forge', name: 'Forge', state: 'starting', exposed: false, open: null },
   ] }] })
@@ -83,6 +83,28 @@ test('open validates the peer, the port and the path, then redirects to the forw
   assert.deepEqual(unavailable, { status: 502, body: { error: 'Turn Remote off and on to reach your other Pinokios' } }, 'a daemon from before the proxy existed')
 })
 
+test('the relays carry the peer\'s API under its prefix, with its own addresses made absolute open routes', async () => {
+  const anywhere = anywhereWith({
+    'https://pinokio-1.tail1.ts.net/anywhere/status': json({ phase: 'ready', apps: [
+      { id: 'comfy', name: 'ComfyUI', remote: 'https://pinokio-1.tail1.ts.net:43000/ui/?a=1#v' },
+      { id: 'ext', name: 'Ext', remote: 'https://example.com/' },
+      { id: 'forge', name: 'Forge', remote: null },
+    ] }),
+    'https://pinokio-1.tail1.ts.net/anywhere/apps': json({ apps: [{ id: 'comfy', name: 'ComfyUI', icon: '/asset/api/comfy/icon.png' }] }),
+    'https://pinokio-1.tail1.ts.net/asset/api/comfy/icon.png': async () => ({ status: 200, headers: { 'content-type': 'image/png' }, raw: Buffer.from([0x89, 0x50]), body: '' }),
+  })
+  const server = fake(anywhere)
+  const status = await Server.prototype.relayAnywherePeer.call(server, 'pinokio-1', 'GET', 'anywhere/status')
+  assert.equal(status.status, 200)
+  assert.deepEqual(status.body.apps.map((app) => app.remote), [
+    'http://127.0.0.1:42000/anywhere/peers/pinokio-1/open?port=43000&to=%2Fui%2F%3Fa%3D1%23v', 'https://example.com/', null,
+  ], 'the peer\'s own address becomes an absolute open route; anything else passes through')
+  const catalog = await Server.prototype.relayAnywherePeer.call(server, 'pinokio-1', 'GET', 'anywhere/apps')
+  assert.deepEqual(catalog.body, { apps: [{ id: 'comfy', name: 'ComfyUI', icon: '/asset/api/comfy/icon.png' }] })
+  const icon = await Server.prototype.relayAnywherePeer.call(server, 'pinokio-1', 'GET', 'asset/api/comfy/icon.png')
+  assert.deepEqual([icon.status, icon.contentType, [...icon.raw]], [200, 'image/png', [0x89, 0x50]], 'an asset is passed through as bytes')
+})
+
 test('launch and stop relay the peer\'s status and body, turning an open URL into the open route', async () => {
   const sent = []
   const anywhere = anywhereWith({
@@ -90,20 +112,20 @@ test('launch and stop relay the peer\'s status and body, turning an open URL int
     'https://pinokio-1.tail1.ts.net/anywhere/apps/stop': json({ error: 'Unknown app: x' }, 400),
   })
   const server = fake(anywhere)
-  const launch = await Server.prototype.relayAnywherePeerAction.call(server, 'pinokio-1', 'launch', { id: 'comfy' })
-  assert.deepEqual(launch, { status: 200, body: { action: 'open', url: '/anywhere/peers/pinokio-1/open?port=43000&to=%2F' } })
+  const launch = await Server.prototype.relayAnywherePeer.call(server, 'pinokio-1', 'POST', 'anywhere/apps/launch', { id: 'comfy' })
+  assert.deepEqual(launch, { status: 200, body: { action: 'open', url: 'http://127.0.0.1:42000/anywhere/peers/pinokio-1/open?port=43000&to=%2F' } })
   assert.equal(sent[0].method, 'POST')
   assert.equal(sent[0].body, '{"id":"comfy"}')
   assert.equal(sent[0].headers['Content-Type'], 'application/json')
-  assert.deepEqual(await Server.prototype.relayAnywherePeerAction.call(server, 'pinokio-1', 'stop', { id: 'x' }), { status: 400, body: { error: 'Unknown app: x' } })
-  assert.equal((await Server.prototype.relayAnywherePeerAction.call(server, 'nobody', 'stop', {})).status, 400)
+  assert.deepEqual(await Server.prototype.relayAnywherePeer.call(server, 'pinokio-1', 'POST', 'anywhere/apps/stop', { id: 'x' }), { status: 400, body: { error: 'Unknown app: x' } })
+  assert.equal((await Server.prototype.relayAnywherePeer.call(server, 'nobody', 'POST', 'anywhere/apps/stop', {})).status, 400)
 })
 
 test('a launch whose default is an outside address passes through unchanged', async () => {
   const anywhere = anywhereWith({
     'https://pinokio-1.tail1.ts.net/anywhere/apps/launch': json({ action: 'open', url: 'https://github.com/x/y' }),
   })
-  const launch = await Server.prototype.relayAnywherePeerAction.call(fake(anywhere), 'pinokio-1', 'launch', { id: 'docs' })
+  const launch = await Server.prototype.relayAnywherePeer.call(fake(anywhere), 'pinokio-1', 'POST', 'anywhere/apps/launch', { id: 'docs' })
   assert.deepEqual(launch.body, { action: 'open', url: 'https://github.com/x/y' })
 })
 
@@ -113,12 +135,12 @@ test('a peer that cannot be reached or answers without JSON relays as 502', asyn
     'https://other.tail1.ts.net/anywhere/apps/stop': async () => ({ status: 404, headers: {}, body: '<html>' }),
   })
   const server = fake(anywhere)
-  const down = await Server.prototype.relayAnywherePeerAction.call(server, 'pinokio-1', 'stop', { id: 'x' })
+  const down = await Server.prototype.relayAnywherePeer.call(server, 'pinokio-1', 'POST', 'anywhere/apps/stop', { id: 'x' })
   assert.equal(down.status, 502)
   assert.match(down.body.error, /Could not reach STUDIO: no answer/)
-  const old = await Server.prototype.relayAnywherePeerAction.call(server, 'other', 'stop', { id: 'x' })
+  const old = await Server.prototype.relayAnywherePeer.call(server, 'other', 'POST', 'anywhere/apps/stop', { id: 'x' })
   assert.deepEqual(old, { status: 502, body: { error: 'other did not answer with JSON (HTTP 404)' } })
-  const stopped = await Server.prototype.relayAnywherePeerAction.call(fake(anywhereWith({}, { proxy: false })), 'other', 'stop', {})
+  const stopped = await Server.prototype.relayAnywherePeer.call(fake(anywhereWith({}, { proxy: false })), 'other', 'POST', 'anywhere/apps/stop', {})
   assert.equal(stopped.status, 502)
 })
 
@@ -133,7 +155,9 @@ test('the peer routes answer a browser on this computer only, never a tailnet cl
   assert.equal(allowed(request({ remote: '::1', host: '[::1]:42000', origin: 'http://[::1]:42000' })), true)
   assert.equal(allowed(request({ remote: '::ffff:127.0.0.1', host: '127.0.0.1:42000', origin: 'http://127.0.0.1:42000' })), true)
   assert.equal(allowed(request({ origin: 'http://localhost:3000' })), false, 'another local site is not this page')
-  assert.equal(allowed(request({ origin: 'https://pinokio.co' })), false)
+  assert.equal(allowed(request({ origin: 'https://pinokio.co' })), true, 'the Remote tab in a browser on this computer')
+  assert.equal(allowed(request({ origin: 'https://pinokio.co', headers: { 'X-Forwarded-For': '100.1.1.1' } })), false, 'but not through Serve')
+  assert.equal(allowed(request({ origin: 'https://pinokio.co.evil' })), false)
   assert.equal(allowed(request({ origin: 'not a url' })), false)
   assert.equal(allowed(request({ host: 'pinokio.tail1.ts.net' })), false, 'the node\'s own name')
   assert.equal(allowed(request({ headers: { 'X-Forwarded-For': '100.1.1.1' } })), false, 'through Serve with a forged loopback Host')