pinokiod 1.0.47 → 1.0.49

package/kernel/api/fs/index.js

@@ -95,13 +95,58 @@ class FS {
     //      }
     //    }
     //
+    //    2. pip
+    //
+    //      1.1. Share all pip
+    //      {
+    //        “method”: “fs.share”,
+    //        “params”: {
+    //          "venv": "app/env"
+    //        }
+    //      }
+    //
     /////////////////////////////////////////////////////////////////////////////
 
-    console.log("fs.share", req.params)
-
     ondata({ raw: "\r\ncreating a shared drive:\r\n" + JSON.stringify(req.params, null, 2).replace(/\n/g, "\r\n") })
 
-    if (req.params.drive) {
+    if (req.params.venv) {
+      // pip sharing
+      // 1. get all pip items
+      let p
+      if (kernel.platform === "win32") {
+        p = path.resolve(req.cwd, req.params.venv, "Lib", "site-packages")
+      } else {
+        p = path.resolve(req.cwd, req.params.venv, "lib", "python3.10", "site-packages")
+      }
+      const res = await kernel.python.call(
+        "pip",
+        path.resolve(__dirname, "../../python/pip.py"),
+        "list",
+        [p]
+      )
+      const d = {}
+      // symlink:
+      //  from => :package_path
+      //  to => /drives/packages/pip/:package_name/:package_version
+      for(let name in res) {
+        const package_path = res[name].path
+        const package_name = res[name].name
+        const version = res[name].version
+        d[`${package_name}/${version}`] = package_path
+      }
+      ondata({
+        raw: "\r\nLINKING DRIVES:\r\n" + JSON.stringify(d, null, 2).replace(/\n/g, "\r\n")
+      })
+      const drivePath = kernel.path("drive")
+      const drive = new Pdrive(drivePath)
+      await drive.create({
+        drive: d,
+        parent: "packages/pip"
+      })
+      ondata({
+        raw: "Done.\r\n"
+      })
+    } else if (req.params.drive) {
       const drivePath = kernel.path("drive")
       const drive = new Pdrive(drivePath)
       for(const route in req.params.drive) {

package/kernel/bin/index.js

@@ -24,8 +24,6 @@ const fakeUa = require('fake-useragent');
 const fse = require('fs-extra')
 
 
-
-
 //const Puppet = require("./puppeteer")
 class Bin {
   constructor(kernel) {
@@ -437,7 +435,7 @@ class Bin {
           } else if (type === "pip") {
             const message = (install ? install : `pip install ${name} ${args}`)
             ondata({ html: `<b>${progress} Installing ${name}</b><br>${message}` }, "notify")
-            await this.exec({ message }, ondata)
+            await this.exec({ message, conda: "base" }, ondata)
           } else if (type === "brew") {
             const message = (install ? install : `brew install ${name} ${args}`)
             ondata({ html: `<b>${progress} Installing ${name}</b><br>${message}` }, "notify")

package/kernel/index.js

@@ -9,6 +9,7 @@ const portfinder = require('portfinder');
 const Loader = require("./loader")
 const Bin = require('./bin')
 const Api = require("./api")
+const Python = require("./python")
 const Template = require('./template')
 const Shells = require("./shells")
 const Key = require("./key")
@@ -184,6 +185,7 @@ class Kernel {
     this.loader = new Loader()
     this.bin = new Bin(this)
     this.api = new Api(this)
+    this.python = new Python(this)
     this.shell = new Shells(this)
     this.system = system
     this.keys = {}

package/kernel/python/index.js

@@ -0,0 +1,49 @@
+const path = require('path')
+const nodecallspython = require("node-calls-python");
+class Python {
+  constructor(kernel) {
+    this.kernel = kernel
+  }
+  async call(modname, modpath, method, args) {
+    // if this.proc doesn't exist, create
+    if (!this.proc) {
+      await this.kernel.exec({
+        message: [
+          "which pip",
+          "pip install fastapi importlib_metadata uvicorn[standard]",
+        ],
+        conda: "base"
+      }, (s) => {
+      })
+      await new Promise((resolve, reject) => {
+        this.kernel.exec({
+          message: "uvicorn --port 42001 server:app",
+          path: __dirname,
+          conda: "base"
+        }, (s) => {
+  //        process.stdout.write(s)
+          if (/Uvicorn running on/i.test(s.cleaned)) {
+            resolve()
+          }
+        })
+      })
+      console.log("Server Started")
+      this.proc = true
+    }
+
+    let res = await fetch("http://127.0.0.1:42001/call", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        name: modname,
+        path: modpath,
+        method,
+        params: args
+      })
+    }).then((res) => {
+      return res.json()
+    })
+    return res
+  }
+}
+module.exports = Python

package/kernel/python/pip.py

@@ -0,0 +1,79 @@
+import pkg_resources
+import platform
+from pathlib import Path
+import site
+import sysconfig
+import sys
+import importlib
+import importlib_metadata
+import os
+
+# Get all the pip packages for the base path
+def list(sitepackages_base):
+#  print(f"LIST {sitepackages_base}")
+  sys.path=[sitepackages_base]
+  importlib.invalidate_caches()
+
+#  sys.path.insert(0, sitepackages_base)
+  print(f"sys.path={sys.path}")
+  print(f"sys.meta_path={sys.meta_path}")
+  print(f"sys.path_hooks={sys.path_hooks}")
+  #dists = importlib_metadata.packages_distributions()
+  dists = importlib_metadata.packages_distributions()
+  print(f"dists={dists}")
+#  print(f"DISTS {dists}")
+
+#  print(f"SYSPATH {sys.path}")
+#  for key in sys.modules:
+#    print(f"## {key}: {sys.modules[key]}")
+#  print(f"sys.modules={sys.modules}")
+  #print(f"BEFORE PATH: {sys.path}")
+  #print(f"AFTER PATH: {sys.path}")
+#  pkg_resources.working_set.add_entry(sitepackages_base)
+  sitepackages_path = sysconfig.get_paths()["purelib"]
+  dict = {}
+#  packages = importlib.resources.files(package)
+  for package_name in dists:
+    print(f"package_name={package_name}")
+    for dist_name in dists[package_name]:
+      # get package path
+      try:
+        meta = importlib_metadata.metadata(dist_name)
+        files = importlib_metadata.files(dist_name)
+
+        for file in files:
+
+          #if str(file).startswith("_"):
+          if str(file).startswith("__pycache__"):
+            continue
+          if str(file).startswith("."):
+            continue
+
+
+          # if the resolved path
+          abspath = os.path.join(sitepackages_base, file)
+          is_child = Path(abspath).is_relative_to(sitepackages_base) and (sitepackages_base in abspath)
+          if is_child:
+            relative = str(file)
+            chunks = relative.split(os.sep)
+            if len(chunks) > 1:
+              folder = chunks[0]
+              package_path = str(os.path.join(sitepackages_base, folder))
+              if not hasattr(dict, folder):
+                dict[folder] = {}
+
+              try:
+                dict[folder]["version"] = meta["Version"]
+              except:
+                print('no version')
+              dict[folder]["name"] = folder
+              dict[folder]["path"] = package_path
+      except Exception as error:
+        print(f"#ERR {error}")
+        # nothing
+
+    # get dist path (only if there's a version)
+
+  print(f"DICT={dict}")
+
+  return dict

package/kernel/python/server.py

@@ -0,0 +1,27 @@
+from fastapi import FastAPI, Body
+from typing import List
+from pydantic import BaseModel
+import sys
+import os
+
+import importlib
+class Params(BaseModel):
+  values: List[str]
+
+app = FastAPI()
+#@app.get("/")
+#async def root():
+# return {"greeting":"Hello world"}
+
+ # { name: "pip.py", method: "list", params: [ ] }
+@app.post("/call")
+async def call(item: dict = Body(...)):
+  print(f"item={item}")
+  sys.path.insert(0, os.path.dirname(item["path"]))
+#async def call(path: str, method: str, params: Params):
+  mod = importlib.import_module(item["name"], item["path"])
+  print(f"Mod = {mod}")
+  fun = getattr(mod, item["method"])
+  print(f"Fun = {fun}")
+  result = fun(*item["params"])
+  return result

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "1.0.47",
+  "version": "1.0.49",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -40,6 +40,7 @@
     "marked": "^5.0.1",
     "mime-types": "^2.1.35",
     "multer": "^1.4.5-lts.1",
+    "node-calls-python": "^1.8.2",
     "node-downloader-helper": "^2.1.9",
     "node-pty-prebuilt-multiarch-cp": "^0.10.1-pre.48",
     "pdrive": "^0.0.14",