pinokiod 3.8.33 → 3.8.34

package/kernel/caddy.js

@@ -9,8 +9,9 @@ class Caddy {
     let peers = this.kernel.peer.get()
     for(let ip of peers) {
       let account = this.kernel.peer.account(ip)
+      let namespaced_url = `pinokio.${account}.localhost`
       payload.apps.http.servers.main.routes.push({
-        "match": [{ "host": [`pinokio.${account}.localhost`] }],
+        "match": [{ "host": [namespaced_url] }],
         "handle": [
           {
             "handler": "reverse_proxy",
@@ -33,6 +34,14 @@ class Caddy {
     return payload
   }
   home () {
+    let internal_ip = "127.0.0.1:42000"
+    let namespaced_url = "pinokio.localhost"
+
+    if (!this.mapping[internal_ip]) {
+      this.mapping[internal_ip] = new Set()
+    }
+    this.mapping[internal_ip].add(namespaced_url)
+
     return {
       "apps": {
         "http": {
@@ -61,11 +70,11 @@ class Caddy {
                   ]
                 },
                 {
-                  "match": [{ "host": [`pinokio.localhost`] }],
+                  "match": [{ "host": [namespaced_url] }],
                   "handle": [
                     {
                       "handler": "reverse_proxy",
-                      "upstreams": [{ "dial": "127.0.0.1:42000" }]
+                      "upstreams": [{ "dial": internal_ip }]
                     },
                     {
                       "handler": "headers",
@@ -100,55 +109,98 @@ class Caddy {
         for(let key in local_variables) {
           let val = local_variables[key]
           if (val.startsWith("http")) {
-            val = val.replace(/https?:\/\//, '')
 
+            // val :- https?:(some url)
+
+
+            let internal_ip = val.replace(/https?:\/\//, '')
 
-            // replace 127,0.0.1 or localhost with the 192.168 address
-            val = val.replace(/localhost|127\.0\.0\.1/g, ip)
             let api_path = path.resolve(this.kernel.peer.homedir(ip), "api")
             let rel_path = path.relative(api_path, script_path)
             let name = rel_path.split(path.sep)[0]
-            console.log({ api_path, rel_path, name, val })
+            console.log({ api_path, rel_path, name, internal_ip })
 
+            let chunks = internal_ip.split(":")
+            if (chunks.length === 2) {
+              // 1. if the URL is a localhost url with port
+              let INTERNAL_HOST = chunks[0]
+              let INTERNAL_PORT = chunks[1]
 
-            // create a proxy proxy
-            // - get next available port and expose
-            payload.apps.http.servers.main.routes.push({
-              "match": [{
-                "host": [":PORT_PLACEHOLDER_" + val],
-              }],
-              "handle": [
-                {
-                  "handler": "reverse_proxy",
-                  "upstreams": [{ "dial": val }]
-                },
-                {
-                  "handler": "headers",
-                  "response": {
-                    "set": {
-                      "Access-Control-Allow-Origin": ["*"],
-                      "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                      "Access-Control-Allow-Headers": ["*"],
-                      "Vary": ["Origin"]
-                    }
-                  }
-                }
-              ]
-            })
+              this.ip_mapping[INTERNAL_PORT] = ip
 
 
-            // proxy proxy => proxy
-            if (key === "url") {
-              let url = `${name}.${account}.localhost`.toLowerCase()
-              this.mapping[val] = url
+              // replace 127,0.0.1 or localhost with the 192.168 address
+              //val = val.replace(/localhost|127\.0\.0\.1/g, ip)
+
+
+              // create a proxy proxy
+              // - get next available port and expose
+              payload.apps.http.servers[":" + INTERNAL_PORT] = {
+                "listen": [":PORT_PLACEHOLDER_" + INTERNAL_PORT],
+                "routes": [{
+                  "handle": [
+                    {
+                      "handler": "reverse_proxy",
+                      "upstreams": [{ "dial": internal_ip }]
+                    },
+                    {
+                      "handler": "headers",
+                      "response": {
+                        "set": {
+                          "Access-Control-Allow-Origin": ["*"],
+                          "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                          "Access-Control-Allow-Headers": ["*"],
+                          "Vary": ["Origin"]
+                        }
+                      }
+                    }
+                  ]
+                }]
+              }
+
+              // proxy proxy => proxy
+              if (key === "url") {
+                let namespaced_url = `${name}.${account}.localhost`.toLowerCase()
+                if (!this.mapping[internal_ip]) {
+                  this.mapping[internal_ip] = new Set()
+                }
+                this.mapping[internal_ip].add(namespaced_url)
+                payload.apps.http.servers.main.routes.push({
+                  "match": [{
+                    "host": [namespaced_url]
+                  }],
+                  "handle": [
+                    {
+                      "handler": "reverse_proxy",
+                      "upstreams": [{ "dial": ip + ":PORT_PLACEHOLDER_" + INTERNAL_PORT }]
+                    },
+                    {
+                      "handler": "headers",
+                      "response": {
+                        "set": {
+                          "Access-Control-Allow-Origin": ["*"],
+                          "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                          "Access-Control-Allow-Headers": ["*"],
+                          "Vary": ["Origin"]
+                        }
+                      }
+                    }
+                  ]
+                })
+              }
+              let namespaced_url = `${name}.${account}.localhost`.toLowerCase()
+              if (!this.mapping[internal_ip]) {
+                this.mapping[internal_ip] = new Set()
+              }
+              this.mapping[internal_ip].add(namespaced_url)
               payload.apps.http.servers.main.routes.push({
                 "match": [{
-                  "host": [url]
+                  "host": [namespaced_url],
                 }],
                 "handle": [
                   {
                     "handler": "reverse_proxy",
-                    "upstreams": [{ "dial": ":PORT_PLACEHOLDER_" + val }]
+                    "upstreams": [{ "dial": ip + ":PORT_PLACEHOLDER_" + INTERNAL_PORT }]
                   },
                   {
                     "handler": "headers",
@@ -163,29 +215,12 @@ class Caddy {
                   }
                 ]
               })
+            } else {
+              
+              // 2. if otherwise (anything, like https://google.com)
+              // DO NOT create a proxy
             }
-            payload.apps.http.servers.main.routes.push({
-              "match": [{
-                "host": [`${key}.${name}${account}.localhost`.toLowerCase()],
-              }],
-              "handle": [
-                {
-                  "handler": "reverse_proxy",
-                  "upstreams": [{ "dial": ":PORT_PLACEHOLDER_" + val }]
-                },
-                {
-                  "handler": "headers",
-                  "response": {
-                    "set": {
-                      "Access-Control-Allow-Origin": ["*"],
-                      "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                      "Access-Control-Allow-Headers": ["*"],
-                      "Vary": ["Origin"]
-                    }
-                  }
-                }
-              ]
-            })
+
           }
         }
       }
@@ -203,16 +238,45 @@ class Caddy {
         let val = local_variables[key]
         if (val.startsWith("http")) {
           val = val.replace(/https?:\/\//, '')
+          let internal_ip = val
           let api_path = this.kernel.path("api")
           let rel_path = path.relative(api_path, script_path)
           let name = rel_path.split(path.sep)[0]
           console.log({ api_path, rel_path, name, val })
-          if (key === "url") {
-            let url = `${name}.localhost`.toLowerCase()
-            this.mapping[val] = url
+          let chunks = internal_ip.split(":")
+          if (chunks.length === 2) {
+            if (key === "url") {
+              let url = `${name}.localhost`.toLowerCase()
+              if (!this.mapping[val]) {
+                this.mapping[val] = new Set()
+              }
+              this.mapping[val].add(url)
+              payload.apps.http.servers.main.routes.push({
+                "match": [{
+                  "host": [url]
+                }],
+                "handle": [
+                  {
+                    "handler": "reverse_proxy",
+                    "upstreams": [{ "dial": val }]
+                  },
+                  {
+                    "handler": "headers",
+                    "response": {
+                      "set": {
+                        "Access-Control-Allow-Origin": ["*"],
+                        "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                        "Access-Control-Allow-Headers": ["*"],
+                        "Vary": ["Origin"]
+                      }
+                    }
+                  }
+                ]
+              })
+            }
             payload.apps.http.servers.main.routes.push({
               "match": [{
-                "host": [url]
+                "host": [`${key}.${name}.localhost`.toLowerCase()],
               }],
               "handle": [
                 {
@@ -232,29 +296,10 @@ class Caddy {
                 }
               ]
             })
+          } else {
+            // 2. if otherwise (anything, like https://google.com)
+            // do NOT create a proxy
           }
-          payload.apps.http.servers.main.routes.push({
-            "match": [{
-              "host": [`${key}.${name}.localhost`.toLowerCase()],
-            }],
-            "handle": [
-              {
-                "handler": "reverse_proxy",
-                "upstreams": [{ "dial": val }]
-              },
-              {
-                "handler": "headers",
-                "response": {
-                  "set": {
-                    "Access-Control-Allow-Origin": ["*"],
-                    "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                    "Access-Control-Allow-Headers": ["*"],
-                    "Vary": ["Origin"]
-                  }
-                }
-              }
-            ]
-          })
         }
       }
     }
@@ -272,29 +317,15 @@ class Caddy {
       let caddy_mapping = this.kernel.peer.caddy(ip)
       console.log({ caddy_mapping })
       for(let internal_ip in caddy_mapping) {
-        let INTERNAL_PORT = internal_ip.split(":")[1]
-        let https_domain = caddy_mapping[internal_ip]
-//        let remote_app_ip = app_ip.replace(/localhost|127\.0\.0\.1/g, ip)
-//        console.log({ https_domain, app_ip, ip })
-        // https_domain <port>.localhost
-//        let re = new RegExp(ip)
-
-        // insert the $account value right before the last part (1234.localhost => 1234.x.localhost)
-        let chunks = https_domain.split(".")
-        let namespaced_url
-        if (chunks.length === 3) {
-          namespaced_url = https_domain
-        } else if (chunks.length === 2) {
-          chunks.splice(chunks.length - 1, 0, account)
-          namespaced_url = chunks.join(".")
-        }
-        this.mapping[internal_ip] = namespaced_url
-        console.log({ namespaced_url, https_domain,  internal_ip })
-
-
         // create a proxy proxy
         // 1. create an exposed port listening on 0.0.0.0
         // 2. the exposed port points to the internal ip
+        let chunks = internal_ip.split(":")
+        let INTERNAL_HOST = chunks[0]
+        let INTERNAL_PORT = chunks[1]
+
+        this.ip_mapping[INTERNAL_PORT] = ip
+
         payload.apps.http.servers[":" + INTERNAL_PORT] = {
           "listen": [":PORT_PLACEHOLDER_" + INTERNAL_PORT],
           "routes": [{
@@ -317,30 +348,55 @@ class Caddy {
             ]
           }]
         }
+        let https_domains = caddy_mapping[internal_ip]
+        console.log(">>>>>>>> https_domains", https_domains)
 
-        // 1. point proxy URL to the 127.0.0.1:EXPOSED_PORT
-        payload.apps.http.servers.main.routes.push({
-          "match": [{
-            "host": [namespaced_url],
-          }],
-          "handle": [
-            {
-              "handler": "reverse_proxy",
-              "upstreams": [{ "dial": ip + ":PORT_PLACEHOLDER_" + INTERNAL_PORT }]
-            },
-            {
-              "handler": "headers",
-              "response": {
-                "set": {
-                  "Access-Control-Allow-Origin": ["*"],
-                  "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                  "Access-Control-Allow-Headers": ["*"],
-                  "Vary": ["Origin"]
+        // 1. if https_domains is made up of one item <port>.localhost,
+        // add <port>.<account>.localhost
+        // 2. if https_domains is made up of two items <port>.localhost and <port>.<account>.localhost, don't do anything
+        for(let https_domain of https_domains) {
+          // insert the $account value right before the last part (1234.localhost => 1234.x.localhost)
+          let chunks = https_domain.split(".")
+          let namespaced_url
+          if (chunks.length === 3) {
+            namespaced_url = https_domain
+          } else if (chunks.length === 2) {
+            chunks.splice(chunks.length - 1, 0, account)
+            namespaced_url = chunks.join(".")
+          }
+          console.log("##x", { chunks, namespaced_url, https_domain })
+          if (!this.mapping[internal_ip]) {
+            this.mapping[internal_ip] = new Set()
+          }
+          console.log("this.mapping", this.mapping)
+          console.log("internal_ip", internal_ip)
+          console.log("result", this.mapping[internal_ip])
+          this.mapping[internal_ip].add(namespaced_url)
+          console.log({ namespaced_url, https_domain,  internal_ip })
+          // 1. point proxy URL to the 127.0.0.1:EXPOSED_PORT
+          payload.apps.http.servers.main.routes.push({
+            "match": [{
+              "host": [namespaced_url],
+            }],
+            "handle": [
+              {
+                "handler": "reverse_proxy",
+                "upstreams": [{ "dial": ip + ":PORT_PLACEHOLDER_" + INTERNAL_PORT }]
+              },
+              {
+                "handler": "headers",
+                "response": {
+                  "set": {
+                    "Access-Control-Allow-Origin": ["*"],
+                    "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                    "Access-Control-Allow-Headers": ["*"],
+                    "Vary": ["Origin"]
+                  }
                 }
               }
-            }
-          ]
-        })
+            ]
+          })
+        }
       }
     }
     return payload
@@ -353,7 +409,10 @@ class Caddy {
     }
     for(let item of list) {
       let url = `${item.port}.localhost`
-      this.mapping[item.ip] = url
+      if (!this.mapping[item.ip]) {
+        this.mapping[item.ip] = new Set()
+      }
+      this.mapping[item.ip].add(url)
       payload.apps.http.servers.main.routes.push({
         "match": [{
           "host": [url],
@@ -386,7 +445,6 @@ class Caddy {
     }
   }
   async update() {
-    this.mapping = {}
     let payload = await this.payload()
     console.log("Make request to caddy", JSON.stringify(payload, null, 2))
     let response = await axios.post('http://127.0.0.1:2019/load', payload, {
@@ -397,6 +455,9 @@ class Caddy {
     console.log("Caddy Response", { response })
   }
   async payload () {
+    this.mapping = {}
+    this.ip_mapping = {}
+    this.port_mapping = {}   // 127.0.0.1 => 192.168,..
 
     // localhost home
     let payload = this.home()
@@ -422,9 +483,20 @@ class Caddy {
     if (count > 0) {
       let ports = await this.kernel.ports(count)
       console.log({ count, ports })
-      payload = Util.fill_object(payload, /PORT_PLACEHOLDER_.+/, ports)
+      let response = Util.fill_object(payload, /PORT_PLACEHOLDER_.+/, ports)
+      payload = response.result
+      let replaced_map = response.replaced_map
+      for(let key in replaced_map) {
+        let original_port = key.replace("PORT_PLACEHOLDER_", "")
+        this.port_mapping[original_port] = replaced_map[key]
+      }
     }
 
+    for(let key in this.mapping) {
+      this.mapping[key] = Array.from(this.mapping[key])
+    }
+
+
     return payload
   }
 }

package/kernel/peer.js

@@ -14,7 +14,7 @@ class PeerDiscovery {
   account(ip) {
     return this.data[ip].account.username
   }
-  home(ip) {
+  homedir(ip) {
     return this.data[ip].homedir
   }
   caddy(ip) {
@@ -36,7 +36,6 @@ class PeerDiscovery {
           this.peers.add(ip);
           let interval = setInterval(() => {
             axios.get(`http://${ip}:42000/pinokio/mem`).then((res) => {
-              console.log("axios", res.data)
               this.data[ip] = res.data
               if (res.data.caddy) {
                 console.log(`Discovered peer: ${ip}`);

package/kernel/util.js

@@ -308,7 +308,9 @@ const update_env = async (filepath, changes) => {
   await fs.promises.writeFile(filepath, newval)
 };
 function fill_object(obj, pattern, list) {
+  console.log("FILL_OBJECT")
   const map = {};
+  const replaced_map = {}
   let index = 0;
 
   function recurse(value) {
@@ -317,6 +319,7 @@ function fill_object(obj, pattern, list) {
         if (!(match in map)) {
           if (index >= list.length) throw new Error("Not enough items provided");
           map[match] = list[index++];
+          replaced_map[match] = map[match]
         }
         return map[match];
       });
@@ -332,7 +335,8 @@ function fill_object(obj, pattern, list) {
     return value;
   }
 
-  return recurse(obj);
+  let res = recurse(obj);
+  return { result: res, replaced_map }
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "3.8.33",
+  "version": "3.8.34",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -2659,14 +2659,15 @@ class Server {
       })
     }))
     this.app.get("/proxy", ex(async (req, res) => {
+      await this.kernel.caddy.update()
       let list = await this.kernel.processes.refresh()
       list = list.map((item) => {
         return {
-          proxy: this.kernel.caddy.mapping[item.ip],
+          proxy: this.kernel.caddy.mapping[item.ip] || [],
           ...item
         }
       })
-      await this.kernel.caddy.update()
+      console.log("list", list)
 
       let peers = this.kernel.peer.get()
 
@@ -2728,27 +2729,23 @@ class Server {
         let pinokio = (await this.kernel.loader.load(p)).resolved
         let p2 = path.resolve(apipath, folder, "pinokio_meta.json")
         let pinokio2 = (await this.kernel.loader.load(p2)).resolved
-        let cc;
         if (pinokio) {
-          cc = {
-            name: pinokio.title,
-            description: pinokio.description,
-            link: `/pinokio/browser/${folder}/browse#n1`,
-            icon: pinokio.icon ? `/api/${folder}/${pinokio.icon}?raw=true` : null
-          }
-        }
-        if (pinokio2) {
-          if (pinokio2.title) cc.title = pinokio2.title
-          if (pinokio2.description) cc.description = pinokio2.description
-          if (pinokio2.icon) cc.icon = pinokio2.icon
-        }
-        if (cc) {
+          let cc = Object.assign({} , pinokio , pinokio2);
+          console.log({ pinokio, pinokio2, cc })
+          cc.link = `/pinokio/browser/${folder}/browse#n1`,
+          cc.icon = cc.icon ? `/api/${folder}/${cc.icon}?raw=true` : null
+          cc.name = cc.title
+          console.log("after", { pinokio, pinokio2, cc })
           apps.push(cc)
         }
       }
+
+      console.log({ port_mapping: this.kernel.caddy.port_mapping, ip_mapping: this.kernel.caddy.ip_mapping })
       res.render("proxy", {
         peers,
         list,
+        port_mapping: this.kernel.caddy.port_mapping,
+        ip_mapping: this.kernel.caddy.ip_mapping,
         agent: this.agent,
         theme: this.theme,
         items: proxies,

package/server/views/proxy.ejs

@@ -54,6 +54,12 @@
 header .runner {
   padding: 10px 0 0;
 }
+.proc-table {
+  display: flex;
+}
+.proc-table .title {
+  flex-grow: 1;
+}
 .proxy {
   padding: 0 10px;
   text-align: right;
@@ -147,21 +153,27 @@ body.dark .browser-options-row {
   margin-top: 10px;
 }
 .header-label {
+  padding: 0;
   font-size: 50px;
   letter-spacing: -1px;
-  padding: 50px 0 0;
   text-align: center;
   text-transform: capitalize;
 }
 .header-label-sub {
   font-size: 25px;
   letter-spacing: -1px;
-  padding: 15px;
+  padding: 0 10px 10px;
   font-weight: bold;
 }
+.header-container {
+  margin: 10px;
+  background: rgba(0, 0, 100, 0.04);
+  padding: 20px;
+  box-sizing: border-box;
+}
 .container {
   display: grid;
-  grid-template-columns: repeat(4, 1fr);
+  grid-template-columns: repeat(2, 1fr);
 }
 .dark .container-row {
   background: rgba(255, 255, 255, 0.04);
@@ -176,6 +188,25 @@ body.dark .browser-options-row {
   text-align:center;
   padding: 10px;
 }
+.container-row table {
+  width: 100%;
+}
+table a {
+  text-decoration: none;
+  color: royalblue;
+}
+.container-row th {
+  text-align: left;
+  font-size: 18px;
+  padding: 5px;
+  text-transform: uppercase;
+}
+.container-row td.title {
+}
+.container-row td {
+  font-size: 14px;
+  padding: 5px;
+}
 @media only screen and (max-width: 480px) {
   .btn2 {
     padding: 5px;
@@ -207,37 +238,48 @@ body.dark .browser-options-row {
 </h1>
 </header>
 <main>
-  <div class='header-label'><i class='fa-solid fa-wifi'></i> Local Sharing</div>
+  <div class='header-label'><i class='fa-solid fa-wifi'></i> Networking</div>
   <div class='desc'>Serve Pinokio, installed apps, and even external apps to any device on the same network.</div>
+  <div class='header-container'>
+    <div class='header-label-sub'><i class="fa-solid fa-podcast"></i> Local Network</div>
+    <% peers.forEach((item) => { %>
+      <a target="_blank" href="http://<%=item%>" class='line align-top'>
+        <h3 class='table-grid'><%=item%></h3>
+      </a>
+    <% }) %>
+  </div>
   <div class='container'>
     <div class='container-row'>
-      <div class='header-label-sub'><i class="fa-solid fa-microchip"></i> Peers</div>
-      <% peers.forEach((item) => { %>
-        <a target="_blank" href="http://<%=item%>" class='line align-top'>
-          <h3 class='table-grid'><%=item%></h3>
-        </a>
-      <% }) %>
-    </div>
-    <div class='container-row'>
-      <div class='header-label-sub'><i class="fa-solid fa-microchip"></i> Apps</div>
-      <div class='line align-top'>
-        <h3 class='table-grid'>
-          <div class='col'>Name</div>
-          <div class='col'>Pid</div>
-          <div class='col'>Port</div>
-        </h3>
-      </div>
-      <% list.forEach((item) => { %>
-        <a target="_blank" href="https://<%=item.proxy%>" class='line align-top'>
-          <h3 class='table-grid'>
-            <div class='col'><%=item.name%></div>
-            <div class='col'><%=item.pid%></div>
-            <div class='col'><%=item.port%></div>
-          </h3>
-        </a>
-      <% }) %>
+      <div class='header-label-sub'><i class="fa-solid fa-microchip"></i> System</div>
+      <table>
+        <tr>
+          <th>Server</th>
+          <th>https</th>
+          <th>http</th>
+        </tr>
+        <% list.forEach((item) => { %>
+          <tr> 
+            <td class='title'><%=item.name%></td>
+            <td>
+              <% item.proxy.forEach((ip) => { %>
+                <div>
+                  <a target="_blank" href="https://<%=ip%>">https://<%=ip%></a>
+                </div>
+              <% }) %>
+            </td>
+            <td>
+              <div>
+                <a target="_blank" href="http://localhost:<%=item.port%>">http://localhost:<%=item.port%></a>
+              </div>
+              <div>
+                <a target="_blank" href="http://<%=ip_mapping[item.port]%>:<%=port_mapping[item.port]%>">http://<%=ip_mapping[item.port]%>:<%=port_mapping[item.port]%></a>
+              </div>
+            </td>
+          </tr>
+        <% }) %>
+      </table>
     </div>
-    <div class='container-row'>
+    <div class='container-row hidden'>
       <div class='header-label-sub'><i class="fa-solid fa-microchip"></i> System Programs</div>
       <div class='line align-top'>
         <h3>
@@ -286,7 +328,7 @@ body.dark .browser-options-row {
       <% }) %>
     </div>
     <div class='container-row'>
-      <div class='header-label-sub'><i class="fa-solid fa-dice-d6"></i> Application Programs</div>
+      <div class='header-label-sub'><i class="fa-solid fa-dice-d6"></i> Pinokio</div>
       <% apps.forEach((item) => { %>
         <a class='line align-top' href="<%=item.link%>">
           <h3>