pyodide 0.18.2 → 0.19.0-alpha.1

package/load-pyodide.js

@@ -1,7 +1,11 @@
 import { Module } from "./module.js";
 
 const IN_NODE =
-  typeof process !== "undefined" && process.release.name !== "undefined";
+  typeof process !== "undefined" &&
+  process.release &&
+  process.release.name === "node" &&
+  typeof process.browser ===
+    "undefined"; /* This last condition checks if we run the browser shim of process */
 
 /** @typedef {import('./pyproxy.js').PyProxy} PyProxy */
 /** @private */
@@ -14,7 +18,7 @@ export async function initializePackageIndex(indexURL) {
   baseURL = indexURL;
   let package_json;
   if (IN_NODE) {
-    const fsPromises = await import("fs/promises");
+    const fsPromises = await import(/* webpackIgnore: true */ "fs/promises");
     const package_string = await fsPromises.readFile(
       `${indexURL}packages.json`
     );
@@ -39,6 +43,17 @@ export async function initializePackageIndex(indexURL) {
   }
 }
 
+export async function _fetchBinaryFile(indexURL, path) {
+  if (IN_NODE) {
+    const fsPromises = await import(/* webpackIgnore: true */ "fs/promises");
+    const tar_buffer = await fsPromises.readFile(`${indexURL}${path}`);
+    return tar_buffer.buffer;
+  } else {
+    let response = await fetch(`${indexURL}${path}`);
+    return await response.arrayBuffer();
+  }
+}
+
 ////////////////////////////////////////////////////////////
 // Package loading
 const DEFAULT_CHANNEL = "default channel";
@@ -61,17 +76,21 @@ function _uri_to_package_name(package_uri) {
 export let loadScript;
 if (globalThis.document) {
   // browser
-  loadScript = (url) => import(url);
+  loadScript = async (url) => await import(/* webpackIgnore: true */ url);
 } else if (globalThis.importScripts) {
   // webworker
   loadScript = async (url) => {
     // This is async only for consistency
     globalThis.importScripts(url);
   };
-} else if (typeof process !== "undefined" && process.release.name === "node") {
-  const pathPromise = import("path").then((M) => M.default);
+} else if (IN_NODE) {
+  const pathPromise = import(/* webpackIgnore: true */ "path").then(
+    (M) => M.default
+  );
   const fetchPromise = import("node-fetch").then((M) => M.default);
-  const vmPromise = import("vm").then((M) => M.default);
+  const vmPromise = import(/* webpackIgnore: true */ "vm").then(
+    (M) => M.default
+  );
   loadScript = async (url) => {
     if (url.includes("://")) {
       // If it's a url, have to load it with fetch and then eval it.
@@ -89,6 +108,23 @@ if (globalThis.document) {
   throw new Error("Cannot determine runtime environment");
 }
 
+function addPackageToLoad(name, toLoad) {
+  name = name.toLowerCase();
+  if (toLoad.has(name)) {
+    return;
+  }
+  toLoad.set(name, DEFAULT_CHANNEL);
+  // If the package is already loaded, we don't add dependencies, but warn
+  // the user later. This is especially important if the loaded package is
+  // from a custom url, in which case adding dependencies is wrong.
+  if (loadedPackages[name] !== undefined) {
+    return;
+  }
+  for (let dep_name of Module.packages[name].depends) {
+    addPackageToLoad(dep_name, toLoad);
+  }
+}
+
 function recursiveDependencies(
   names,
   _messageCallback,
@@ -96,23 +132,6 @@ function recursiveDependencies(
   sharedLibsOnly
 ) {
   const toLoad = new Map();
-
-  const addPackage = (name) => {
-    name = name.toLowerCase();
-    if (toLoad.has(name)) {
-      return;
-    }
-    toLoad.set(name, DEFAULT_CHANNEL);
-    // If the package is already loaded, we don't add dependencies, but warn
-    // the user later. This is especially important if the loaded package is
-    // from a custom url, in which case adding dependencies is wrong.
-    if (loadedPackages[name] !== undefined) {
-      return;
-    }
-    for (let dep_name of Module.packages[name].depends) {
-      addPackage(dep_name);
-    }
-  };
   for (let name of names) {
     const pkgname = _uri_to_package_name(name);
     if (toLoad.has(pkgname) && toLoad.get(pkgname) !== name) {
@@ -129,7 +148,7 @@ function recursiveDependencies(
     }
     name = name.toLowerCase();
     if (name in Module.packages) {
-      addPackage(name);
+      addPackageToLoad(name, toLoad);
       continue;
     }
     errorCallback(`Skipping unknown package '${name}'`);
@@ -147,24 +166,46 @@ function recursiveDependencies(
   return toLoad;
 }
 
-async function _loadPackage(names, messageCallback, errorCallback) {
-  // toLoad is a map pkg_name => pkg_uri
-  let toLoad = recursiveDependencies(names, messageCallback, errorCallback);
+// locateFile is the function used by the .js file to locate the .data file
+// given the filename
+Module.locateFile = function (path) {
+  // handle packages loaded from custom URLs
+  let pkg = path.replace(/\.data$/, "");
+  const toLoad = Module.locateFile_packagesToLoad;
+  if (toLoad && toLoad.has(pkg)) {
+    let package_uri = toLoad.get(pkg);
+    if (package_uri != DEFAULT_CHANNEL) {
+      return package_uri.replace(/\.js$/, ".data");
+    }
+  }
+  return baseURL + path;
+};
 
-  // locateFile is the function used by the .js file to locate the .data
-  // file given the filename
-  Module.locateFile = (path) => {
-    // handle packages loaded from custom URLs
-    let pkg = path.replace(/\.data$/, "");
-    if (toLoad.has(pkg)) {
-      let package_uri = toLoad.get(pkg);
-      if (package_uri != DEFAULT_CHANNEL) {
-        return package_uri.replace(/\.js$/, ".data");
+// When the JS loads, it synchronously adds a runDependency to emscripten. It
+// then loads the data file, and removes the runDependency from emscripten.
+// This function returns a promise that resolves when there are no pending
+// runDependencies.
+function waitRunDependency() {
+  const promise = new Promise((r) => {
+    Module.monitorRunDependencies = (n) => {
+      if (n === 0) {
+        r();
       }
-    }
-    return baseURL + path;
-  };
+    };
+  });
+  // If there are no pending dependencies left, monitorRunDependencies will
+  // never be called. Since we can't check the number of dependencies,
+  // manually trigger a call.
+  Module.addRunDependency("dummy");
+  Module.removeRunDependency("dummy");
+  return promise;
+}
 
+async function _loadPackage(names, messageCallback, errorCallback) {
+  // toLoad is a map pkg_name => pkg_uri
+  let toLoad = recursiveDependencies(names, messageCallback, errorCallback);
+  // Tell Module.locateFile about the packages we're loading
+  Module.locateFile_packagesToLoad = toLoad;
   if (toLoad.size === 0) {
     return Promise.resolve("No new packages to load");
   } else {
@@ -172,8 +213,8 @@ async function _loadPackage(names, messageCallback, errorCallback) {
     messageCallback(`Loading ${packageNames}`);
   }
 
-  // This is a collection of promises that resolve when the package's JS file
-  // is loaded. The promises already handle error and never fail.
+  // This is a collection of promises that resolve when the package's JS file is
+  // loaded. The promises already handle error and never fail.
   let scriptPromises = [];
 
   for (let [pkg, uri] of toLoad) {
@@ -204,26 +245,6 @@ async function _loadPackage(names, messageCallback, errorCallback) {
     );
   }
 
-  // When the JS loads, it synchronously adds a runDependency to emscripten.
-  // It then loads the data file, and removes the runDependency from
-  // emscripten. This function returns a promise that resolves when there are
-  // no pending runDependencies.
-  function waitRunDependency() {
-    const promise = new Promise((r) => {
-      Module.monitorRunDependencies = (n) => {
-        if (n === 0) {
-          r();
-        }
-      };
-    });
-    // If there are no pending dependencies left, monitorRunDependencies will
-    // never be called. Since we can't check the number of dependencies,
-    // manually trigger a call.
-    Module.addRunDependency("dummy");
-    Module.removeRunDependency("dummy");
-    return promise;
-  }
-
   // We must start waiting for runDependencies *after* all the JS files are
   // loaded, since the number of runDependencies may happen to equal zero
   // between package files loading.
@@ -253,13 +274,11 @@ async function _loadPackage(names, messageCallback, errorCallback) {
 
   // We have to invalidate Python's import caches, or it won't
   // see the new files.
-  Module.runPythonSimple(
-    "import importlib\n" + "importlib.invalidate_caches()\n"
-  );
+  Module.importlib.invalidate_caches();
 }
 
-// This is a promise that is resolved iff there are no pending package loads.
-// It never fails.
+// This is a promise that is resolved iff there are no pending package loads. It
+// never fails.
 let _package_lock = Promise.resolve();
 
 /**
@@ -286,6 +305,51 @@ async function acquirePackageLock() {
  */
 export let loadedPackages = {};
 
+let sharedLibraryWasmPlugin;
+let origWasmPlugin;
+let wasmPluginIndex;
+function initSharedLibraryWasmPlugin() {
+  for (let p in Module.preloadPlugins) {
+    if (Module.preloadPlugins[p].canHandle("test.so")) {
+      origWasmPlugin = Module.preloadPlugins[p];
+      wasmPluginIndex = p;
+      break;
+    }
+  }
+  sharedLibraryWasmPlugin = {
+    canHandle: origWasmPlugin.canHandle,
+    handle(byteArray, name, onload, onerror) {
+      origWasmPlugin.handle(byteArray, name, onload, onerror);
+      origWasmPlugin.asyncWasmLoadPromise = (async () => {
+        await origWasmPlugin.asyncWasmLoadPromise;
+        Module.loadDynamicLibrary(name, {
+          global: true,
+          nodelete: true,
+        });
+      })();
+    },
+  };
+}
+
+// override the load plugin so that it calls "Module.loadDynamicLibrary" on any
+// .so files.
+// this only needs to be done for shared library packages because we assume that
+// if a package depends on a shared library it needs to have access to it. not
+// needed for .so in standard module because those are linked together
+// correctly, it is only where linking goes across modules that it needs to be
+// done. Hence, we only put this extra preload plugin in during the shared
+// library load
+function useSharedLibraryWasmPlugin() {
+  if (!sharedLibraryWasmPlugin) {
+    initSharedLibraryWasmPlugin();
+  }
+  Module.preloadPlugins[wasmPluginIndex] = sharedLibraryWasmPlugin;
+}
+
+function restoreOrigWasmPlugin() {
+  Module.preloadPlugins[wasmPluginIndex] = origWasmPlugin;
+}
+
 /**
  * @callback LogFn
  * @param {string} msg
@@ -297,17 +361,17 @@ export let loadedPackages = {};
  * Load a package or a list of packages over the network. This installs the
  * package in the virtual filesystem. The package needs to be imported from
  * Python before it can be used.
- * @param {string | string[] | PyProxy} names Either a single package name or URL
- * or a list of them. URLs can be absolute or relative. The URLs must have
- * file name
- * ``<package-name>.js`` and there must be a file called
+ *
+ * @param {string | string[] | PyProxy} names Either a single package name or
+ * URL or a list of them. URLs can be absolute or relative. The URLs must have
+ * file name ``<package-name>.js`` and there must be a file called
  * ``<package-name>.data`` in the same directory. The argument can be a
- * ``PyProxy`` of a list, in which case the list will be converted to
- * Javascript and the ``PyProxy`` will be destroyed.
+ * ``PyProxy`` of a list, in which case the list will be converted to JavaScript
+ * and the ``PyProxy`` will be destroyed.
  * @param {LogFn=} messageCallback A callback, called with progress messages
  *    (optional)
- * @param {LogFn=} errorCallback A callback, called with error/warning
- *    messages (optional)
+ * @param {LogFn=} errorCallback A callback, called with error/warning messages
+ *    (optional)
  * @async
  */
 export async function loadPackage(names, messageCallback, errorCallback) {
@@ -340,58 +404,23 @@ export async function loadPackage(names, messageCallback, errorCallback) {
   } catch (e) {
     // do nothing - let the main load throw any errors
   }
-  // override the load plugin so that it imports any dlls also
-  // this only needs to be done for shared library packages because
-  // we assume that if a package depends on a shared library
-  // it needs to have access to it.
-  // not needed for so in standard module because those are linked together
-  // correctly, it is only where linking goes across modules that it needs to
-  // be done. Hence we only put this extra preload plugin in during the shared
-  // library load
-  let oldPlugin;
-  for (let p in Module.preloadPlugins) {
-    if (Module.preloadPlugins[p].canHandle("test.so")) {
-      oldPlugin = Module.preloadPlugins[p];
-      break;
-    }
-  }
-  let dynamicLoadHandler = {
-    get: function (obj, prop) {
-      if (prop === "handle") {
-        return function (bytes, name) {
-          obj[prop].apply(obj, arguments);
-          this["asyncWasmLoadPromise"] = this["asyncWasmLoadPromise"].then(
-            function () {
-              Module.loadDynamicLibrary(name, {
-                global: true,
-                nodelete: true,
-              });
-            }
-          );
-        };
-      } else {
-        return obj[prop];
-      }
-    },
-  };
-  var loadPluginOverride = new Proxy(oldPlugin, dynamicLoadHandler);
-  // restore the preload plugin
-  Module.preloadPlugins.unshift(loadPluginOverride);
 
   let releaseLock = await acquirePackageLock();
   try {
+    useSharedLibraryWasmPlugin();
     await _loadPackage(
       sharedLibraryNames,
       messageCallback || console.log,
       errorCallback || console.error
     );
-    Module.preloadPlugins.shift(loadPluginOverride);
+    restoreOrigWasmPlugin();
     await _loadPackage(
       names,
       messageCallback || console.log,
       errorCallback || console.error
     );
   } finally {
+    restoreOrigWasmPlugin();
     releaseLock();
   }
 }

package/module.js

@@ -1,19 +1,26 @@
+/**
+ * @typedef {import('emscripten').Module} Module
+ */
+
 /**
  * The Emscripten Module.
  *
- * @private @type {import('emscripten').Module}
+ * @private
+ * @type {Module}
  */
 export let Module = {};
 Module.noImageDecoding = true;
 Module.noAudioDecoding = true;
 Module.noWasmDecoding = false; // we preload wasm using the built in plugin now
 Module.preloadedWasm = {};
+Module.preRun = [];
 
 /**
  *
- * @param {undefined|(() => string)} stdin
- * @param {undefined|((text: string) => void)} stdout
- * @param {undefined|((text: string) => void)} stderr
+ * @param {undefined | function(): string} stdin
+ * @param {undefined | function(string)} stdout
+ * @param {undefined | function(string)} stderr
+ * @private
  */
 export function setStandardStreams(stdin, stdout, stderr) {
   // For stdout and stderr, emscripten provides convenient wrappers that save us the trouble of converting the bytes into a string
@@ -27,11 +34,9 @@ export function setStandardStreams(stdin, stdout, stderr) {
 
   // For stdin, we have to deal with the low level API ourselves
   if (stdin) {
-    Module.preRun = [
-      function () {
-        Module.FS.init(createStdinWrapper(stdin), null, null);
-      },
-    ];
+    Module.preRun.push(function () {
+      Module.FS.init(createStdinWrapper(stdin), null, null);
+    });
   }
 }
 
@@ -79,3 +84,26 @@ function createStdinWrapper(stdin) {
   }
   return stdinWrapper;
 }
+
+/**
+ * Make the home directory inside the virtual file system,
+ * then change the working directory to it.
+ *
+ * @param {string} path
+ * @private
+ */
+export function setHomeDirectory(path) {
+  Module.preRun.push(function () {
+    const fallbackPath = "/";
+    try {
+      Module.FS.mkdirTree(path);
+    } catch (e) {
+      console.error(`Error occurred while making a home directory '${path}':`);
+      console.error(e);
+      console.error(`Using '${fallbackPath}' for a home directory instead`);
+      path = fallbackPath;
+    }
+    Module.ENV.HOME = path;
+    Module.FS.chdir(path);
+  });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pyodide",
-  "version": "0.18.2",
+  "version": "0.19.0-alpha.1",
   "description": "The Pyodide JavaScript package",
   "keywords": [
     "python",