pinokiod 3.41.0 → 3.43.0

package/kernel/prototype.js

@@ -8,7 +8,6 @@ class Proto {
     this.kernel = kernel
   }
   async init() {
-    console.log("Proto init")
     this.items = []
     this.kv = {}
     if (this.kernel.bin.installed && this.kernel.bin.installed.conda && this.kernel.bin.installed.conda.has("git")) {
@@ -48,7 +47,6 @@ class Proto {
         })
       }
     }
-    console.log("Proto init done")
   }
   async ai() {
     let ai_path = this.kernel.path("prototype/system/ai/new/static")
@@ -78,6 +76,10 @@ class Proto {
   }
   async create(req, ondata) {
     try {
+      if (req.client) {
+        this.kernel.client = req.client
+      }
+      console.log("REQ", JSON.stringify(req, null, 2))
       let projectType = req.params.projectType
       let startType = req.params.cliType || req.params.startType
 
@@ -88,27 +90,39 @@ class Proto {
       payload.input = req.params
 
 
-      await fs.promises.mkdir(payload.cwd)
+      // 1. move mkdir into each launcher
+      // 2. run logic
+      // 3. add docs
 
-      let default_icon_path = path.resolve(__dirname, "../server/public/pinokio-black.png")
-      let icon_path = path.resolve(payload.cwd, "icon.png")
-      await fs.promises.cp(default_icon_path, icon_path)
+//      await fs.promises.mkdir(payload.cwd)
 
-      let mod_path = this.kernel.path("prototype/system", projectType, startType)
-      let mod = await this.kernel.require(mod_path)
+      if (projectType === "blank") {
+        return { success: "/p/" + name + "/dev" }
+      }
 
-      await mod(payload, ondata, this.kernel)
+//      let default_icon_path = path.resolve(__dirname, "../server/public/pinokio-black.png")
+//      let icon_path = path.resolve(payload.cwd, "icon.png")
+//      await fs.promises.cp(default_icon_path, icon_path)
 
-      // copy readme
-      let readme_path = this.kernel.path("prototype/PINOKIO.md")
-      await fs.promises.cp(readme_path, path.resolve(cwd, name, "PINOKIO.md"))
+      // run the init logic
+      let mod_path = this.kernel.path("prototype/system", projectType, startType)
+      let mod = await this.kernel.require(mod_path)
+      let response = await mod(payload, ondata, this.kernel)
 
-      // copy pterm.md
-      let cli_readme_path = this.kernel.path("prototype/PTERM.md")
-      await fs.promises.cp(cli_readme_path, path.resolve(cwd, name, "PTERM.md"))
+//      // copy readme
+//      let readme_path = this.kernel.path("prototype/PINOKIO.md")
+//      await fs.promises.cp(readme_path, path.resolve(cwd, name, "PINOKIO.md"))
+//
+//      // copy pterm.md
+//      let cli_readme_path = this.kernel.path("prototype/PTERM.md")
+//      await fs.promises.cp(cli_readme_path, path.resolve(cwd, name, "PTERM.md"))
 
 
-      return { success: "/p/" + name }
+      if (response) {
+        return response
+      } else {
+        return { success: "/p/" + name + "/dev" }
+      }
     } catch (e) {
       console.log("ERROR", e)
       return { error: e.stack }

package/kernel/router/common.js

@@ -33,7 +33,7 @@ class Common extends Processor {
       handler = override_handler
     }
     this.router.config.apps.http.servers.main.routes.push({
-      "match": [{ "host": [match] }],
+      "match": [{ "host": Array.isArray(match) ? match : [match] }],
       "handle": handler
     })
     this.router.add({ host, dial, match })

package/kernel/router/connector.js

@@ -47,9 +47,7 @@ class Connector extends Processor {
       }
     }
     this.router.config.apps.http.servers.main.routes.push({
-      "match": [{
-        "host": [match],
-      }],
+      "match": [{ "host": [match] }],
       "handle": [
         {
           "handler": "reverse_proxy",

package/kernel/router/index.js

@@ -5,10 +5,12 @@ const Util = require('../util')
 const LocalhostHomeRouter = require('./localhost_home_router')
 const LocalhostVariableRouter = require('./localhost_variable_router')
 const LocalhostPortRouter = require('./localhost_port_router')
+const LocalhostStaticRouter = require('./localhost_static_router')
 const PeerHomeRouter = require('./peer_home_router')
 const PeerVariableRouter = require('./peer_variable_router')
 const PeerPortRouter = require('./peer_port_router')
 const PeerPeerRouter = require('./peer_peer_router')
+const PeerStaticRouter = require('./peer_static_router')
 const CustomDomainRouter = require('./custom_domain_router')
 const Environment = require("../environment")
 class Router {
@@ -21,7 +23,9 @@ class Router {
     this.peer_variable_router = new PeerVariableRouter(this)
     this.peer_port_router = new PeerPortRouter(this)
     this.peer_peer_router = new PeerPeerRouter(this)
+    this.peer_static_router = new PeerStaticRouter(this)
     this.custom_domain_router = new CustomDomainRouter(this)
+    this.localhost_static_router = new LocalhostStaticRouter(this)
     this.default_prefix = "pinokio"
     this.default_suffix = "localhost"
     this.default_match = this.default_prefix + "." + this.default_suffix
@@ -33,6 +37,7 @@ class Router {
     this.port_mapping = {}   // 127.0.0.1 => 192.168,..
     this.local_network_mapping = {}
     this.custom_routers = {}
+    this.rewrite_mapping = {}
   }
   async init() {
     // if ~/pinokio/network doesn't exist, clone
@@ -95,13 +100,24 @@ class Router {
       }
     }
 
-    let env = await Environment.get(this.kernel.homedir)
+    let env = await Environment.get(this.kernel.homedir, this.kernel)
     this.active = false
     if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="1" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="true")) {
       this.active = true
     }
 
 
+  }
+  async add_rewrite({ peer, route, match, dial }) {
+    if (!this.rewrite_mapping[dial]) {
+      this.rewrite_mapping[dial] = {}
+    }
+    if (!this.rewrite_mapping[dial][route]) {
+      this.rewrite_mapping[dial][route] = []
+    }
+    if (!this.rewrite_mapping[dial][route].includes(match)) {
+      this.rewrite_mapping[dial][route].push(match)
+    }
   }
   async add({ host, match, dial }) {
     if (!this._mapping[host]) {
@@ -110,7 +126,13 @@ class Router {
     if (!this._mapping[host][dial]) {
       this._mapping[host][dial] = new Set()
     }
-    this._mapping[host][dial].add(match)
+    if (Array.isArray(match)) {
+      for(let m of match) {
+        this._mapping[host][dial].add(m)
+      }
+    } else {
+      this._mapping[host][dial].add(match)
+    }
   }
   _info() {
     let mapping = {}
@@ -194,6 +216,18 @@ class Router {
     }
   }
 
+  async static() {
+    this.localhost_static_router.handle()
+    for(let host in this.kernel.peer.info) {
+      let info = this.kernel.peer.info[host]
+      if (info.rewrite_mapping) {
+        for(let name in info.rewrite_mapping) {
+          this.peer_static_router.handle(info.rewrite_mapping[name])
+        }
+      }
+    }
+    this.mapping = this._mapping
+  }
 
   // set local config
   async local() {
@@ -208,7 +242,7 @@ class Router {
         for(let host in this.kernel.peer.info) {
           let peer = this.kernel.peer.info[host]
           if (peer.host === this.kernel.peer.host) {
-            this.peer_home_router.handle(peer)
+//            this.peer_home_router.handle(peer)
             this.peer_variable_router.handle(peer)
             this.peer_port_router.handle(peer)
           }
@@ -274,7 +308,7 @@ class Router {
 //        console.log("Caddy Response", { response })
         this.old_config = this.config
       } catch (e) {
-        console.log("Caddy Request Failed", e.cause)
+        console.log("Caddy Request Failed", e)
       }
     }
   }

package/kernel/router/localhost_home_router.js

@@ -3,7 +3,11 @@ class LocalhostHomeRouter {
     this.router = router
   }
   handle () {
-    this.router.add({ host: this.router.kernel.peer.host, dial: this.router.default_host + ":" + this.router.default_port, match: this.router.default_match })
+    this.router.add({
+      host: this.router.kernel.peer.host,
+      dial: this.router.default_host + ":" + this.router.default_port,
+      match: this.router.default_match
+    })
     this.router.config = {
       "apps": {
         "tls": {

package/kernel/router/localhost_port_router.js

@@ -5,8 +5,34 @@ class LocalhostPortRouter {
     this.common = new Common(router)
   }
   handle (proc) {
+    let name_match
+    for(let api_name in this.router.kernel.pinokio_configs) {
+      let config = this.router.kernel.pinokio_configs[api_name]
+      if (config.dns) {
+        let root_routes = config.dns["@"]
+        if (root_routes && root_routes.length > 0) {
+          for(let route of root_routes) {
+            if (route === ":" + proc.port) {
+              // matched
+              name_match = api_name
+              break;
+            }
+          }
+        }
+      }
+    }
+
+    let match
+    if (name_match) {
+      match = [
+        `${name_match}.localhost`,
+        `${proc.port}.localhost`
+      ]
+    } else {
+      match = `${proc.port}.localhost`
+    }
     this.common.handle({
-      match: `${proc.port}.localhost`,
+      match,
       dial: proc.ip,
       host: this.router.kernel.peer.host,
     })

package/kernel/router/localhost_static_router.js

@@ -0,0 +1,93 @@
+const path = require('path')
+const Common = require('./common')
+const Rewriter = require('./rewriter')
+const Connector = require('./connector')
+const Processor = require('./processor')
+class LocalhostStaticRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.common = new Common(router)
+    this.connector = new Connector(router)
+    this.rewriter = new Rewriter(router)
+  }
+  handle () {
+    let configs = []
+    for(let api_name in this.router.kernel.pinokio_configs) {
+      let config = this.router.kernel.pinokio_configs[api_name]
+      if (config.dns) {
+        configs.push({
+          api_name,
+          config
+        })
+      }
+    }
+    for(let { api_name, config } of configs) {
+      for(let domain in config.dns) {
+        let localhost_match
+        let peer_match
+        if (domain === "@") {
+          localhost_match = `${api_name}.localhost`.toLowerCase()
+          peer_match = `${api_name}.${this.router.kernel.peer.name}.localhost`.toLowerCase()
+        } else {
+          localhost_match = `${domain}.${api_name}.localhost`.toLowerCase()
+          peer_match = `${domain}.${api_name}.${this.router.kernel.peer.name}.localhost`.toLowerCase()
+        }
+
+        let routes = config.dns[domain]
+        for(let route of routes) {
+          if (!route.startsWith("$") && !route.startsWith(":")) {
+            let chunks = route.split("/")
+            let local_dial = `${this.router.default_host}:${this.router.default_port}`
+            let peer_dial = `${this.router.kernel.peer.host}:${this.router.default_port}`
+            let rewrite = `/asset/api/${api_name}`
+            this.rewriter.handle({
+              route: rewrite,
+              match: [localhost_match],
+              dial: local_dial,
+            })
+            this.rewriter.handle({
+              route: rewrite,
+              match: [peer_match],
+              dial: peer_dial,
+            })
+
+//            this.router.add_rewrite({ route: new_path, match, peer, dial })
+
+
+/*
+        name: web
+        internal_router: ["127.0.0.1:42000/asset/api/web", "https://web.localhost"]
+        external_ip: ["192.168.1.49:42000/asset/api/web"]
+        external_router ["https://web.x.localhost"]
+        */
+
+            
+
+            this.router.rewrite_mapping[api_name] = {
+              name: api_name,
+              internal_router: [
+                `${local_dial}${rewrite}`,
+                localhost_match
+              ],
+              external_ip: `${peer_dial}${rewrite}`,
+              external_router: [
+                peer_match
+              ]
+            }
+//            this.connector.handle({
+//              match: peer_match,
+//              connector: {
+//                host: this.router.kernel.peer.host,
+//                port: this.router.default_port,
+//              },
+//              dial,
+//              host: this.router.kernel.peer.host,
+//            })
+          }
+        } 
+      }
+    }
+  }
+}
+module.exports = LocalhostStaticRouter

package/kernel/router/localhost_variable_router.js

@@ -14,20 +14,25 @@ class LocalhostVariableRouter extends Processor {
         let val = local_variables_for_script[key]
         if (typeof val === "string" && val.startsWith("http")) {
           let dial = val.replace(/https?:\/\//, '')
-          let name = this.api_name(this.router.kernel.platform, this.router.kernel.homedir, script_path)
-          if (this.has_port(dial)) {
-            if (key === "url") {
+          if (dial.endsWith("/")) {
+            dial = dial.slice(0, -1)
+          }
+          let api_name = this.api_name(this.router.kernel.platform, this.router.kernel.homedir, script_path)
+          let domain = this.domain(api_name, key)
+          if (domain) {
+            if (this.has_port(dial)) {
+              let match
+              if (domain === "@") {
+                match = `${api_name}.localhost`.toLowerCase()
+              } else {
+                match = `${domain}.${api_name}.localhost`.toLowerCase()
+              }
               this.common.handle({
-                match: `${name}.localhost`.toLowerCase(),
+                match,
                 dial,
                 host: this.router.kernel.peer.host,
               })
             }
-            this.common.handle({
-              match: `${key}.${name}.localhost`.toLowerCase(),
-              dial,
-              host: this.router.kernel.peer.host,
-            })
           }
         }
       }

package/kernel/router/peer_peer_router.js

@@ -6,6 +6,9 @@ class PeerPeerRouter {
   }
   handle(peer) {
     for(let dial in peer.router) {
+      if (dial.endsWith("/")) {
+        dial = dial.slice(0, -1)
+      }
       let matches = peer.router[dial]
       let exposed_matches = matches.filter((m) => {
         return m.endsWith(`${peer.name}.localhost`)

package/kernel/router/peer_static_router.js

@@ -0,0 +1,43 @@
+const path = require('path')
+const Common = require('./common')
+const Rewriter = require('./rewriter')
+const Connector = require('./connector')
+const Processor = require('./processor')
+class PeerStaticRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.common = new Common(router)
+    this.connector = new Connector(router)
+    this.rewriter = new Rewriter(router)
+  }
+  handle (rewrite_mapping) {
+    /*
+      rewrite_mapping: {
+        name: "test",
+        internal_router: [
+          "127.0.0.1:42000/asset/api/test",
+          "test.localhost"
+        ],
+        external_ip: "192.168.1.49:42000/asset/api/test",
+        external_router: [
+          "test.x.localhost"
+        ]
+      }
+
+      connect 
+        dial: 192.168.1.49:42000 
+        rewrite: /asset/api/test
+        match: test.x.localhost
+    */ 
+    let url = new URL("http://" + rewrite_mapping.external_ip)
+    let dial = url.host
+    let rewrite = url.pathname
+    this.rewriter.handle({
+      route: url.pathname,
+      match: rewrite_mapping.external_router,
+      dial: url.host,
+    })
+  }
+}
+module.exports = PeerStaticRouter

package/kernel/router/peer_variable_router.js

@@ -14,12 +14,22 @@ class PeerVariableRouter extends Processor {
         let val = local_variables[key]
         if (typeof val === "string" && val.startsWith("http")) {
           let dial = val.replace(/https?:\/\//, '')
-          let name = this.api_name(peer.platform, peer.home, script_path)
-          if (this.has_port(dial)) {
-            let parsed_dial = this.parse_ip(dial)
-            if (key === "url") {
+          if (dial.endsWith("/")) {
+            dial = dial.slice(0, -1)
+          }
+          let api_name = this.api_name(peer.platform, peer.home, script_path)
+          let domain = this.domain(api_name, key)
+          if (domain) {
+            if (this.has_port(dial)) {
+              let match
+              if (domain === "@") {
+                match = `${api_name}.${peer.name}.localhost`.toLowerCase()
+              } else {
+                match = `${domain}.${api_name}.${peer.name}.localhost`.toLowerCase()
+              }
+              let parsed_dial = this.parse_ip(dial)
               this.connector.handle({
-                match: `${name}.${peer.name}.localhost`.toLowerCase(),
+                match,
                 connector: {
                   host: peer.host,
                   port: parsed_dial.port
@@ -28,15 +38,6 @@ class PeerVariableRouter extends Processor {
                 host: peer.host,
               })
             }
-            this.connector.handle({
-              match:`${key}.${name}.${peer.name}.localhost`.toLowerCase(),
-              connector: {
-                host: peer.host,
-                port: parsed_dial.port
-              },
-              dial,
-              host: peer.host,
-            })
           }
         }
       }

package/kernel/router/processor.js

@@ -23,9 +23,34 @@ class Processor {
   }
   parse_ip(str) {
     let chunks = str.split(":")
-    let port = chunks[chunks.length-1]
+    let port = chunks[chunks.length-1].replace(/[^0-9]/g, '')
     let host = chunks.slice(0, chunks.length-1).join(":")
     return { host, port }
   }
+  domain (api_name, key) {
+    let config = this.router.kernel.pinokio_configs[api_name]
+    let dns = config.dns
+    for(let domain in dns) {
+      let routes = dns[domain]
+      for(let route of routes) {
+        // $local.url@start ==> start.js/start.json 'url' local variable
+        if (route.startsWith("$")) {
+          let chunks = route.split("@")
+          if (chunks.length === 2) {
+            let [memory, filepath] = chunks
+            if (memory.startsWith("$local.")) {
+              let varname = memory.replace("$local.", "") 
+              if (key === varname) {
+                return domain
+              }
+            }
+          }
+        } else {
+          // file path
+          return domain
+        }
+      }
+    }
+  }
 }
 module.exports = Processor

package/kernel/router/rewriter.js

@@ -0,0 +1,59 @@
+const Processor = require('./processor')
+class Rewriter extends Processor {
+  constructor (router) {
+    super()
+    this.router = router
+  }
+  handle({ match, dial, route }) {
+    //let rewrite = `${route}/{path}`
+    //let rewrite = `${route}{http.request.uri}`
+    /*
+    let handler = [{
+      "handler": "rewrite",
+      "uri": rewrite,
+    }, {
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+            "X-Forwarded-Proto": ["https"],
+            "X-Forwarded-Host": ["{http.request.host}"]
+          }
+        },
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
+    }]
+    */
+
+    // stript the leading /asset/ => (/asset/api/test => /api/test)
+    const asset_path = this.router.kernel.path(route.replace(/\/asset\//, ''))
+    let handler = [{
+      "handler": "file_server",
+      "root": asset_path,
+      "browse": { },
+      "index_names": ["index.html"]
+    }]
+
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
+    this.router.config.apps.http.servers.main.routes.push({
+      "match": [{
+        "host": match ,
+      }],
+      "handle": handler
+    })
+  }
+}
+module.exports = Rewriter

package/kernel/scripts/git/commit

@@ -1,5 +1,14 @@
 {
   "run": [{
+    "when": "{{args.message && args.message.length > 0}}",
+    "method": "jump",
+    "params": {
+      "id": "commit",
+      "params": {
+        "commit": "{{args.message}}"
+      }
+    }
+  }, {
     "method": "input",
     "params": {
       "form": [{
@@ -8,13 +17,14 @@
       }]
     }
   }, {
+    "id": "commit",
     "method": "shell.run",
     "params": {
       "chain": true,
       "path": "{{args.cwd}}",
       "message": [
         "git add .",
-        "git commit -am {{input.commit}}"
+        "git commit -am \"{{input.commit}}\""
       ]
     }
   }]

package/kernel/shell.js

@@ -109,9 +109,14 @@ class Shell {
 //    this.env.npm_config_globalconfig = this.kernel.path("global_npmrc")
 
     // First override this.env with system env
-    let system_env = await Environment.get(this.kernel.homedir)
+    let system_env = await Environment.get(this.kernel.homedir, this.kernel)
     this.env = Object.assign(this.env, system_env)
 
+    let hf_keys = await this.kernel.connect.keys("huggingface")
+    if (hf_keys && hf_keys.access_token) {
+      this.env.HF_TOKEN = hf_keys.access_token
+    }
+
     // if the shell is running from a script file, the params.$parent will include the path to the parent script
     // this means we need to apply app environment as well
     if (params.$parent) {
@@ -123,10 +128,10 @@ class Shell {
       }
 
       // initialize folders
-      await Environment.init_folders(api_path)
+      await Environment.init_folders(api_path, this.kernel)
 
       // apply app env to this.env
-      let app_env = await Environment.get(api_path)
+      let app_env = await Environment.get(api_path, this.kernel)
       this.env = Object.assign(this.env, app_env)
     }
     let PATH_KEY;