pinokiod 3.15.26 → 3.16.0

package/kernel/bin/index.js

@@ -77,6 +77,7 @@ class Bin {
         "User-Agent": userAgent
       }
     }
+    console.log("Download", opts)
     if (process.env.HTTP_PROXY && process.env.HTTP_PROXY.length > 0) {
       opts.httpRequestOptions = { agent }
       opts.httpsRequestOptions = { agent }

package/kernel/index.js

@@ -36,6 +36,9 @@ const Router = require("./router")
 const Procs = require('./procs')
 const Peer = require('./peer')
 const Connect = require('./connect')
+const { DownloaderHelper } = require('node-downloader-helper');
+const { ProxyAgent } = require('proxy-agent');
+const fakeUa = require('fake-useragent');
 //const kill = require('./tree-kill');
 const kill = require('kill-sync')
 const VARS = {
@@ -285,16 +288,15 @@ class Kernel {
     }
   }
   async refresh(notify_peers) {
-    console.log("kernel.refresh")
     const ts = Date.now()
     let network_active = await this.network_active()
-    console.log({ network_active })
+//    console.log({ network_active })
     if (!network_active) {
       return
     }
     let network_running = await this.network_running()
-    console.log({ network_running })
-    console.log({ refreshing: this.processes.refreshing })
+//    console.log({ network_running })
+//    console.log({ refreshing: this.processes.refreshing })
     if (network_running) {
 
 
@@ -305,58 +307,58 @@ class Kernel {
       }
 
       // 1. get the process list
-      console.time("> 1. Proc Refresh"+ts)
+//      console.time("> 1. Proc Refresh"+ts)
       await this.processes.refresh()
 
       // diff check
       let new_config = JSON.stringify(this.processes.info)
-      if (this.old_config !== new_config) {
-        console.log("Proc config has changed")
-        console.log("old", this.old_config)
-        console.log("new", new_config)
-      } else {
-        console.log("Proc config is the same")
-      }
+//      if (this.old_config !== new_config) {
+//        console.log("Proc config has changed")
+//        console.log("old", this.old_config)
+//        console.log("new", new_config)
+//      } else {
+//        console.log("Proc config is the same")
+//      }
       this.old_config = new_config
 
-      console.timeEnd("> 1. Proc Refresh"+ts)
+//      console.timeEnd("> 1. Proc Refresh"+ts)
 
       // 2. refresh peer info to reflect the proc info
-      console.time("> 2. Peer Refresh"+ts)
+//      console.time("> 2. Peer Refresh"+ts)
       await this.peer.refresh()
-      console.timeEnd("> 2. Peer Refresh"+ts)
+//      console.timeEnd("> 2. Peer Refresh"+ts)
 
       // 3. load custom routers from ~/pinokio/network
-      console.time("> 3. Router Init"+ts)
+//      console.time("> 3. Router Init"+ts)
       await this.router.init()
-      console.timeEnd("> 3. Router Init"+ts)
+//      console.timeEnd("> 3. Router Init"+ts)
 
       // 4. set current local host router info
-      console.time("> 4. Router Local"+ts)
+//      console.time("> 4. Router Local"+ts)
       await this.router.local()
-      console.timeEnd("> 4. Router Local"+ts)
+//      console.timeEnd("> 4. Router Local"+ts)
 
       // 5. refresh peer info to reflect the updated router info
-      console.time("> 5. Peer Refresh"+ts)
+//      console.time("> 5. Peer Refresh"+ts)
       await this.peer.refresh()
-      console.timeEnd("> 5. Peer Refresh"+ts)
+//      console.timeEnd("> 5. Peer Refresh"+ts)
 
       // 6. tell peers to refresh
       if (notify_peers) {
-        console.time("> 6. Peer Notify Peers"+ts)
+//        console.time("> 6. Peer Notify Peers"+ts)
         await this.peer.notify_peers()
-        console.timeEnd("> 6. Peer Notify Peers"+ts)
+//        console.timeEnd("> 6. Peer Notify Peers"+ts)
       }
 
       // 7. update remote router
-      console.time("> 7. Router Remote"+ts)
+//      console.time("> 7. Router Remote"+ts)
       await this.router.remote()
-      console.timeEnd("> 7. Router Remote"+ts)
+//      console.timeEnd("> 7. Router Remote"+ts)
 
       // 8. update caddy config
-      console.time("> 8. Router Update"+ts)
+//      console.time("> 8. Router Update"+ts)
       await this.router.update()
-      console.timeEnd("> 8. Router Update"+ts)
+//      console.timeEnd("> 8. Router Update"+ts)
 
       // 9. announce self to the peer network
       this.peer.announce()
@@ -382,6 +384,62 @@ class Kernel {
       }
     }
   }
+  async download(options, ondata) {
+    console.log("download", { options })
+    const agent = new ProxyAgent();
+    const userAgent = fakeUa()
+    let url = options.uri
+    let cwd = options.path
+    const opts = {
+      override: true,
+      headers: {
+        "User-Agent": userAgent
+      }
+    }
+    if (options.filename) {
+      opts.fileName = options.filename
+    }
+    console.log("Download", opts)
+    if (process.env.HTTP_PROXY && process.env.HTTP_PROXY.length > 0) {
+      opts.httpRequestOptions = { agent }
+      opts.httpsRequestOptions = { agent }
+    }
+    if (process.env.HTTPS_PROXY && process.env.HTTPS_PROXY.length > 0) {
+      opts.httpRequestOptions = { agent }
+      opts.httpsRequestOptions = { agent }
+    }
+    const dl = new DownloaderHelper(url, cwd, opts)
+    ondata({ raw: `\r\nDownloading ${url} to ${cwd}...\r\n` })
+    let res = await new Promise((resolve, reject) => {
+      dl.on('end', () => {
+        ondata({ raw: `\r\nDownload Complete!\r\n` })
+        resolve()
+      })
+      dl.on('error', (err) => {
+        ondata({ raw: `\r\nDownload Failed: ${err.message}!\r\n` })
+        reject(err)
+      })
+      dl.on('progress', (stats) => {
+        let p = Math.floor(stats.progress)
+        let str = ""
+        for(let i=0; i<p; i++) {
+          str += "#"
+        }
+        for(let i=p; i<100; i++) {
+          str += "-"
+        }
+        ondata({ raw: `\r${str}` })
+      })
+      dl.start().catch((err) => {
+        ondata({ raw: `\r\nDownload Failed: ${err.message}!\r\n` })
+        reject(err)
+      })
+    })
+
+  /*
+    await this.exec({ message: `aria2 -o download.zip ${url}` })
+    */
+  }
   async _log(data, group, info) {
     if (group) {
 

package/kernel/procs.js

@@ -7,27 +7,65 @@ const limit = pLimit(10)
 const platform = os.platform();
 const isWin = platform === 'win32';
 const cls = isWin ? 'cls' : 'clear'
+const net = require('net');
 class Procs {
   constructor (kernel) {
 //    console.log("Initializing procs")
     this.kernel = kernel
     this.cache = {}
   }
-  async isHttp(localAddress) {
-    if (this.cache.hasOwnProperty(localAddress)) {
-      console.log("Use cached", localAddress)
-      return this.cache[localAddress]
-    }
-    console.log("Not cached", localAddress)
-    try {
-      await axios.head(`http://${localAddress}`, { timeout: 3000 });
-      this.cache[localAddress] = true
-      return true;
-    } catch (err) {
-//      console.log("HEAD ERROR",{ localAddress, err })
-      this.cache[localAddress] = false
-      return false;
+  async isHttp(port) {
+  //async isHttp(localAddress) {
+//    if (this.cache.hasOwnProperty(localAddress)) {
+////      console.log("Use cached", localAddress)
+//      return this.cache[localAddress]
+//    }
+//    console.log("Not cached", localAddress)
+    //try {
+    //  //await axios.head(`http://${localAddress}`, { timeout: 3000 });
+    //  await axios.get(`http://${localAddress}`, { timeout: 3000 });
+    //  this.cache[localAddress] = true
+    //  return true;
+    //} catch (err) {
+    //  console.log("HEAD ERROR",{ localAddress, err })
+    //  this.cache[localAddress] = false
+    //  return false;
+    //}
+    if (this.cache.hasOwnProperty("" + port)) {
+      return this.cache["" + port]
     }
+    let ip = "127.0.0.1"
+    let timeout = 1000
+    let response = await new Promise(resolve => {
+      const socket = new net.Socket();
+      let response = '';
+      let resolved = false;
+      socket.setTimeout(timeout);
+      socket.connect(port, ip, () => {
+        // Use a nonsense method to trigger a 400/405/501 from real HTTP servers
+        socket.write(`FOO / HTTP/1.1\r\nHost: ${ip}\r\nConnection: close\r\n\r\n`);
+      });
+      socket.on('data', chunk => {
+        response += chunk.toString();
+        if (/^HTTP\/\d+\.\d+ \d+/.test(response)) {
+          resolved = true;
+          socket.destroy();
+          resolve(true); // Valid HTTP response detected
+        }
+      });
+      socket.on('error', () => {
+        if (!resolved) resolve(false);
+      });
+      socket.on('timeout', () => {
+        if (!resolved) resolve(false);
+        socket.destroy();
+      });
+      socket.on('close', () => {
+        if (!resolved) resolve(false);
+      });
+    });
+    this.cache["" + port] = response
+    return response
   }
   newline(id) {
     this.kernel.shell.emit({
@@ -124,13 +162,13 @@ class Procs {
       }
     }
 //    console.timeEnd("###### Line parsing")
-    console.time("########## http_check")
-    const http_check = await Promise.all(results.map(({ ip }) => {
+//    console.time("########## http_check")
+    const http_check = await Promise.all(results.map(({ port }) => {
       return limit(() => {
-        return this.isHttp(ip) 
+        return this.isHttp(port) 
       })
     }))
-    console.timeEnd("########## http_check")
+//    console.timeEnd("########## http_check")
     let filtered = []
     for(let i=0; i<http_check.length; i++) {
       if (http_check[i]) {
@@ -292,7 +330,7 @@ class Procs {
 //      console.log("get ports")
       this.getPortPidList((portPidList) => {
 //        console.log("done: get ports")
-        console.log({ portPidList })
+//        console.log({ portPidList })
 //        console.timeEnd(">>>>>>>>GET PORTS " + ts)
 //        console.time(">>>>>>> GET PIDS " + ts)
         // if there's any new port, run getPidToNameMap

package/kernel/router/index.js

@@ -208,9 +208,9 @@ class Router {
   // update caddy config
   async update() {
     if (JSON.stringify(this.config) === JSON.stringify(this.old_config)) {
-      console.log("######### config hasn't updated")
+//      console.log("######### config hasn't updated")
     } else {
-      console.log("######### caddy config has updated. refresh")
+//      console.log("######### caddy config has updated. refresh")
 //      console.log("Old", JSON.stringify(this.old_config, null, 2))
 //      console.log("New", JSON.stringify(this.config, null, 2))
       try {

package/kernel/shell.js

@@ -98,6 +98,7 @@ class Shell {
     this.env.CMAKE_OBJECT_PATH_MAX = 1024
     this.env.PYTORCH_ENABLE_MPS_FALLBACK = 1
     this.env.TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD = 1
+    //this.env.NODE_EXTRA_CA_CERTS = this.kernel.path("cache/XDG_DATA_HOME/caddy/pki/authorities/local/root.crt")
 //    this.env.PIP_REQUIRE_VIRTUALENV = "true"
 //    this.env.NPM_CONFIG_USERCONFIG = this.kernel.path("user_npmrc")
 //    this.env.NPM_CONFIG_GLOBALCONFIG = this.kernel.path("global_npmrc")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "3.15.26",
+  "version": "3.16.0",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -4274,7 +4274,6 @@ class Server {
     }))
     this.app.post("/pinokio/peer/refresh", ex(async (req, res) => {
       // refresh and broadcast
-      console.log("kernel.refresh /pinokio/peer/refresh")
       await this.kernel.refresh()
       res.json({ success: true })
     }))