pinokiod 1.0.51 → 1.0.53

package/kernel/api/fs/index.js

@@ -113,40 +113,99 @@ class FS {
       // pip sharing
       // 1. get all pip items
       let p
+      let drive_path
       if (kernel.platform === "win32") {
         p = path.resolve(req.cwd, req.params.venv, "Lib", "site-packages")
+        drive_path = "pip/Lib/site-packages"
       } else {
         p = path.resolve(req.cwd, req.params.venv, "lib", "python3.10", "site-packages")
+        drive_path = "pip/lib/python3.10/site-packages"
       }
       const res = await kernel.python.call(
-        "pip",
-        path.resolve(__dirname, "../../python/pip.py"),
-        "list",
+        "mod",
+        kernel.bin.path("py"),
+        "get",
         [p],
         ondata
       )
-      const d = {}
+      console.log("RES", res)
       // symlink:
       //  from => :package_path
       //  to => /drives/packages/pip/:package_name/:package_version
+      const drivePath = kernel.path("drive")
+      const drive = new Pdrive(drivePath)
       for(let name in res) {
-        const package_path = res[name].path
-        const package_name = res[name].name
+        /*
+          torch: {
+            version: "2.1",
+            copy: [
+              '../../../bin/torchrun',
+              '../../../bin/convert-onnx-to-caffe2',
+              '../../../bin/convert-caffe2-to-onnx'
+            ],
+            move: [
+              "torchgen",
+              "functorch",
+              "torch"
+            ]
+          },
+
+
+          => 
+
+          {
+            copy: {
+              '../../../bin/torchrun': path.resolve(p, '../../../bin/torchrun')
+              '../../../bin/convert-onnx-to-caffe2': path.resolve(p, '../../../bin/convert-onnx-to-caffe2'),
+              '../../../bin/convert-caffe2-to-onnx': path.resolve(p, '../../../bin/convert-caffe2-to-onnx')
+            },
+            move: {
+              "torchgen": "torchgen",
+              "functorch": "functorch",
+              "torch": "torch"
+            },
+            parent: "pip/torch/2.1/lib/python3.10/site-packages"
+          }
+        */
+        const copy = res[name].copy
+        const move = res[name].move
         const version = res[name].version
-        d[`${package_name}/${version}`] = package_path
+
+        console.log({ name, copy, move, version })
+
+        if (version) {
+          let site_packages_root;
+          if (kernel.platform === "win32") {
+            site_packages_root = `pip/${name}/${version}/Lib/site-packages`
+          } else {
+            site_packages_root = `pip/${name}/${version}/lib/python3.10/site-packages`
+          }
+          const d = {
+            copy: {},
+            move: {},
+            parent: site_packages_root
+          }
+          for(let relpath of copy) {
+            d.copy[relpath] = kernel.path("drive", p, relpath)
+          }
+          for(let relpath of move) {
+            d.move[relpath] = kernel.path("drive", p, relpath)
+          }
+          console.log("D", name, d)
+          ondata({
+            raw: "\r\nLINKING DRIVES:\r\n" + JSON.stringify(d, null, 2).replace(/\n/g, "\r\n")
+          })
+          await drive.create(d)
+          ondata({
+            raw: "Done.\r\n"
+          })
+        } else {
+          ondata({
+            raw: "\r\nNo version specified: " + name + " => Ignore"
+          })
+        }
+
       }
-      ondata({
-        raw: "\r\nLINKING DRIVES:\r\n" + JSON.stringify(d, null, 2).replace(/\n/g, "\r\n")
-      })
-      const drivePath = kernel.path("drive")
-      const drive = new Pdrive(drivePath)
-      await drive.create({
-        drive: d,
-        parent: "packages/pip"
-      })
-      ondata({
-        raw: "Done.\r\n"
-      })
     } else if (req.params.drive) {
       const drivePath = kernel.path("drive")
       const drive = new Pdrive(drivePath)

package/kernel/bin/py.js

@@ -0,0 +1,20 @@
+class Py {
+  async install(req, ondata) {
+    await this.kernel.exec({
+      message: "git clone https://github.com/pinokiocomputer/python py",
+      path: this.kernel.path("bin")
+    }, ondata)
+    await this.kernel.exec({
+      message: "pip install -r requirements.txt",
+      path: this.kernel.bin.path("py")
+    }, ondata)
+  }
+  async installed() {
+    let exists = await this.kernel.exists(this.kernel.bin.path("py"))
+    return exists
+  }
+  async uninstall(req, ondata) {
+    await this.kernel.bin.rm(this.kernel.bin.path("py"), ondata)
+  }
+}
+module.exports = Py

package/kernel/python/index.js

@@ -7,20 +7,10 @@ class Python {
     console.log("CALL", { modname, modpath, method, args })
     // if this.proc doesn't exist, create
     if (!this.proc) {
-      await this.kernel.exec({
-        message: [
-          "which pip",
-          "pip install fastapi importlib_metadata uvicorn[standard]",
-        ],
-        conda: "base"
-      }, (s) => {
-        ondata(s)
-      })
-      console.log("Start uvicorn", __dirname)
       await new Promise((resolve, reject) => {
         this.kernel.exec({
           message: "uvicorn --port 42001 server:app",
-          path: __dirname,
+          path: modpath,
           conda: "base"
         }, (s) => {
   //        process.stdout.write(s)

package/kernel/python/mod.py

@@ -0,0 +1,40 @@
+import sys
+import importlib
+import importlib_metadata
+import os
+# Get all the pip packages for the base path
+def get(sitepackages_base):
+  sys.path=[sitepackages_base]
+  importlib.invalidate_caches()
+  dists = importlib_metadata.packages_distributions()
+  dict = {}
+  for package_name in dists:
+    for dist_name in dists[package_name]:
+      dict[dist_name] = {}
+      # get package path
+      try:
+        meta = importlib_metadata.metadata(dist_name)
+        files = importlib_metadata.files(dist_name)
+        move = set()
+        copy = set()
+        for file in files:
+          filepath = os.path.join(sitepackages_base, file)
+          dirpath = os.path.abspath(os.path.dirname(filepath))
+          try:
+            relpath = Path(dirpath).relative_to(sitepackages_base)
+            # site package handling
+            chunks = str(relpath).split(os.sep)
+            if len(chunks) > 1:
+              folder = chunks[0]
+              move.add(os.path.join(sitepackages_base, folder))
+          except:
+            # non site package handling
+            # add the full file paths => so they can be copied
+            copy.add(str(filepath))
+        dict[dist_name]["copy"] = list(copy)
+        dict[dist_name]["move"] = list(move)
+      except Exception as error:
+        print(f"#ERR {error}")
+  return dict
+if __name__ == "__main__":
+  get("/Users/x/pinokio/api/comfyui.git/app/env/lib/python3.10/site-packages")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "1.0.51",
+  "version": "1.0.53",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -43,7 +43,7 @@
     "node-calls-python": "^1.8.2",
     "node-downloader-helper": "^2.1.9",
     "node-pty-prebuilt-multiarch-cp": "^0.10.1-pre.48",
-    "pdrive": "^0.0.14",
+    "pdrive": "^0.0.23",
     "portfinder": "^1.0.32",
     "random-useragent": "^0.5.0",
     "rimraf": "^5.0.0",

package/server/index.js

@@ -289,6 +289,8 @@ class Server {
         type: "conda",
         name: "nodejs",
         args: "-c conda-forge"
+      }, {
+        name: "py"
       }]
       let platform = os.platform()
       if (platform === "win32") {
@@ -528,6 +530,8 @@ class Server {
           type: "conda",
           name: "nodejs",
           args: "-c conda-forge"
+        }, {
+          name: "py"
         }]
         let platform = os.platform()
         if (platform === "win32") {

package/kernel/python/pip.py

@@ -1,79 +0,0 @@
-import pkg_resources
-import platform
-from pathlib import Path
-import site
-import sysconfig
-import sys
-import importlib
-import importlib_metadata
-import os
-
-# Get all the pip packages for the base path
-def list(sitepackages_base):
-#  print(f"LIST {sitepackages_base}")
-  sys.path=[sitepackages_base]
-  importlib.invalidate_caches()
-
-#  sys.path.insert(0, sitepackages_base)
-  print(f"sys.path={sys.path}")
-  print(f"sys.meta_path={sys.meta_path}")
-  print(f"sys.path_hooks={sys.path_hooks}")
-  #dists = importlib_metadata.packages_distributions()
-  dists = importlib_metadata.packages_distributions()
-  print(f"dists={dists}")
-#  print(f"DISTS {dists}")
-
-#  print(f"SYSPATH {sys.path}")
-#  for key in sys.modules:
-#    print(f"## {key}: {sys.modules[key]}")
-#  print(f"sys.modules={sys.modules}")
-  #print(f"BEFORE PATH: {sys.path}")
-  #print(f"AFTER PATH: {sys.path}")
-#  pkg_resources.working_set.add_entry(sitepackages_base)
-  sitepackages_path = sysconfig.get_paths()["purelib"]
-  dict = {}
-#  packages = importlib.resources.files(package)
-  for package_name in dists:
-    print(f"package_name={package_name}")
-    for dist_name in dists[package_name]:
-      # get package path
-      try:
-        meta = importlib_metadata.metadata(dist_name)
-        files = importlib_metadata.files(dist_name)
-
-        for file in files:
-
-          #if str(file).startswith("_"):
-          if str(file).startswith("__pycache__"):
-            continue
-          if str(file).startswith("."):
-            continue
-
-
-          # if the resolved path
-          abspath = os.path.join(sitepackages_base, file)
-          is_child = Path(abspath).is_relative_to(sitepackages_base) and (sitepackages_base in abspath)
-          if is_child:
-            relative = str(file)
-            chunks = relative.split(os.sep)
-            if len(chunks) > 1:
-              folder = chunks[0]
-              package_path = str(os.path.join(sitepackages_base, folder))
-              if not hasattr(dict, folder):
-                dict[folder] = {}
-
-              try:
-                dict[folder]["version"] = meta["Version"]
-              except:
-                print('no version')
-              dict[folder]["name"] = folder
-              dict[folder]["path"] = package_path
-      except Exception as error:
-        print(f"#ERR {error}")
-        # nothing
-
-    # get dist path (only if there's a version)
-
-  print(f"DICT={dict}")
-
-  return dict