pyodide 0.18.0 → 0.19.0

package/pyodide.js

@@ -1,17 +1,16 @@
 /**
  * The main bootstrap code for loading pyodide.
  */
-import { Module, setStandardStreams } from "./module.js";
+import { Module, setStandardStreams, setHomeDirectory } from "./module.js";
 import {
   loadScript,
   initializePackageIndex,
+  _fetchBinaryFile,
   loadPackage,
 } from "./load-pyodide.js";
 import { makePublicAPI, registerJsModule } from "./api.js";
 import "./pyproxy.gen.js";
 
-import { wrapNamespace } from "./pyproxy.gen.js";
-
 /**
  * @typedef {import('./pyproxy.gen').PyProxy} PyProxy
  * @typedef {import('./pyproxy.gen').PyProxyWithLength} PyProxyWithLength
@@ -36,7 +35,7 @@ import { wrapNamespace } from "./pyproxy.gen.js";
  * @private
  */
 Module.dump_traceback = function () {
-  let fd_stdout = 1;
+  const fd_stdout = 1;
   Module.__Py_DumpTraceback(fd_stdout, Module._PyGILState_GetThisThreadState());
 };
 
@@ -44,21 +43,26 @@ let fatal_error_occurred = false;
 /**
  * Signal a fatal error.
  *
- * Dumps the Python traceback, shows a Javascript traceback, and prints a clear
+ * Dumps the Python traceback, shows a JavaScript traceback, and prints a clear
  * message indicating a fatal error. It then dummies out the public API so that
  * further attempts to use Pyodide will clearly indicate that Pyodide has failed
- * and can no longer be used. pyodide._module is left accessible and it is
+ * and can no longer be used. pyodide._module is left accessible, and it is
  * possible to continue using Pyodide for debugging purposes if desired.
  *
  * @argument e {Error} The cause of the fatal error.
  * @private
  */
 Module.fatal_error = function (e) {
+  if (e.pyodide_fatal_error) {
+    return;
+  }
   if (fatal_error_occurred) {
     console.error("Recursive call to fatal_error. Inner error was:");
     console.error(e);
     return;
   }
+  // Mark e so we know not to handle it later in EM_JS wrappers
+  e.pyodide_fatal_error = true;
   fatal_error_occurred = true;
   console.error(
     "Pyodide has suffered a fatal error. Please report this to the Pyodide maintainers."
@@ -98,128 +102,179 @@ Module.fatal_error = function (e) {
   throw e;
 };
 
+let runPythonInternal_dict; // Initialized in finalizeBootstrap
 /**
- * Run Python code in the simplest way possible. The primary purpose of this
- * method is for bootstrapping. It is also useful for debugging: If the Python
- * interpreter is initialized successfully then it should be possible to use
- * this method to run Python code even if everything else in the Pyodide
- * `core` module fails.
- *
- * The differences are:
- *    1. `runPythonSimple` doesn't return anything (and so won't leak
- *        PyProxies)
- *    2. `runPythonSimple` doesn't require access to any state on the
- *       Javascript `pyodide` module.
- *    3. `runPython` uses `pyodide.eval_code`, whereas `runPythonSimple` uses
- *       `PyRun_String` which is the C API for `eval` / `exec`.
- *    4. `runPythonSimple` runs with `globals` a separate dict which is called
- *       `init_dict` (keeps global state private)
- *    5. `runPythonSimple` doesn't dedent the argument
- *
- * When `core` initialization is completed, the globals for `runPythonSimple`
- * is made available as `Module.init_dict`.
- *
+ * Just like `runPython` except uses a different globals dict and gets
+ * `eval_code` from `_pyodide` so that it can work before `pyodide` is imported.
  * @private
  */
-Module.runPythonSimple = function (code) {
-  let code_c_string = Module.stringToNewUTF8(code);
-  let errcode;
-  try {
-    errcode = Module._run_python_simple_inner(code_c_string);
-  } catch (e) {
-    Module.fatal_error(e);
-  } finally {
-    Module._free(code_c_string);
-  }
-  if (errcode === -1) {
-    Module._pythonexc2js();
-  }
+Module.runPythonInternal = function (code) {
+  return Module._pyodide._base.eval_code(code, runPythonInternal_dict);
 };
 
 /**
- * The Javascript/Wasm call stack is too small to handle the default Python call
- * stack limit of 1000 frames. Here, we determine the Javascript call stack
- * depth available, and then divide by 50 (determined heuristically) to set the
- * maximum Python call stack depth.
- *
+ * A proxy around globals that falls back to checking for a builtin if has or
+ * get fails to find a global with the given key. Note that this proxy is
+ * transparent to js2python: it won't notice that this wrapper exists at all and
+ * will translate this proxy to the globals dictionary.
  * @private
  */
-function fixRecursionLimit() {
-  let depth = 0;
-  function recurse() {
-    depth += 1;
-    recurse();
-  }
-  try {
-    recurse();
-  } catch (err) {}
+function wrapPythonGlobals(globals_dict, builtins_dict) {
+  return new Proxy(globals_dict, {
+    get(target, symbol) {
+      if (symbol === "get") {
+        return (key) => {
+          let result = target.get(key);
+          if (result === undefined) {
+            result = builtins_dict.get(key);
+          }
+          return result;
+        };
+      }
+      if (symbol === "has") {
+        return (key) => target.has(key) || builtins_dict.has(key);
+      }
+      return Reflect.get(target, symbol);
+    },
+  });
+}
 
-  let recursionLimit = Math.min(depth / 25, 500);
-  Module.runPythonSimple(
-    `import sys; sys.setrecursionlimit(int(${recursionLimit}))`
+function unpackPyodidePy(pyodide_py_tar) {
+  const fileName = "/pyodide_py.tar";
+  let stream = Module.FS.open(fileName, "w");
+  Module.FS.write(
+    stream,
+    new Uint8Array(pyodide_py_tar),
+    0,
+    pyodide_py_tar.byteLength,
+    undefined,
+    true
   );
+  Module.FS.close(stream);
+  const code_ptr = Module.stringToNewUTF8(`
+import shutil
+shutil.unpack_archive("/pyodide_py.tar", "/lib/python3.9/site-packages/")
+del shutil
+import importlib
+importlib.invalidate_caches()
+del importlib
+    `);
+  let errcode = Module._PyRun_SimpleString(code_ptr);
+  if (errcode) {
+    throw new Error("OOPS!");
+  }
+  Module._free(code_ptr);
+  Module.FS.unlink(fileName);
+}
+
+/**
+ * This function is called after the emscripten module is finished initializing,
+ * so eval_code is newly available.
+ * It finishes the bootstrap so that once it is complete, it is possible to use
+ * the core `pyodide` apis. (But package loading is not ready quite yet.)
+ * @private
+ */
+function finalizeBootstrap(config) {
+  // First make internal dict so that we can use runPythonInternal.
+  // runPythonInternal uses a separate namespace, so we don't pollute the main
+  // environment with variables from our setup.
+  runPythonInternal_dict = Module._pyodide._base.eval_code("{}");
+  Module.importlib = Module.runPythonInternal("import importlib; importlib");
+  let import_module = Module.importlib.import_module;
+
+  Module.sys = import_module("sys");
+  Module.sys.path.insert(0, config.homedir);
+
+  // Set up globals
+  let globals = Module.runPythonInternal("import __main__; __main__.__dict__");
+  let builtins = Module.runPythonInternal("import builtins; builtins.__dict__");
+  Module.globals = wrapPythonGlobals(globals, builtins);
+
+  // Set up key Javascript modules.
+  let importhook = Module._pyodide._importhook;
+  importhook.register_js_finder();
+  importhook.register_js_module("js", config.jsglobals);
+
+  let pyodide = makePublicAPI();
+  importhook.register_js_module("pyodide_js", pyodide);
+
+  // import pyodide_py. We want to ensure that as much stuff as possible is
+  // already set up before importing pyodide_py to simplify development of
+  // pyodide_py code (Otherwise it's very hard to keep track of which things
+  // aren't set up yet.)
+  Module.pyodide_py = import_module("pyodide");
+  Module.version = Module.pyodide_py.__version__;
+
+  // copy some last constants onto public API.
+  pyodide.pyodide_py = Module.pyodide_py;
+  pyodide.version = Module.version;
+  pyodide.globals = Module.globals;
+  return pyodide;
 }
+
 /**
  * Load the main Pyodide wasm module and initialize it.
  *
- * Only one copy of Pyodide can be loaded in a given Javascript global scope
+ * Only one copy of Pyodide can be loaded in a given JavaScript global scope
  * because Pyodide uses global variables to load packages. If an attempt is made
  * to load a second copy of Pyodide, :any:`loadPyodide` will throw an error.
  * (This can be fixed once `Firefox adopts support for ES6 modules in webworkers
  * <https://bugzilla.mozilla.org/show_bug.cgi?id=1247687>`_.)
  *
- * @param {{ indexURL : string, fullStdLib? : boolean = true, stdin?: () => string, stdout?: (text: string) => void, stderr?: (text: string) => void }} config
  * @param {string} config.indexURL - The URL from which Pyodide will load
  * packages
+ * @param {string} config.homedir - The home directory which Pyodide will use inside virtual file system
+ * Default: /home/pyodide
  * @param {boolean} config.fullStdLib - Load the full Python standard library.
  * Setting this to false excludes following modules: distutils.
  * Default: true
- * @param {undefined | (() => string)} config.stdin - Override the standard input callback. Should ask the user for one line of input.
+ * @param {undefined | function(): string} config.stdin - Override the standard input callback. Should ask the user for one line of input.
  * Default: undefined
- * @param {undefined | ((text: string) => void)} config.stdout - Override the standard output callback.
+ * @param {undefined | function(string)} config.stdout - Override the standard output callback.
  * Default: undefined
- * @param {undefined | ((text: string) => void)} config.stderr - Override the standard error output callback.
+ * @param {undefined | function(string)} config.stderr - Override the standard error output callback.
  * Default: undefined
  * @returns The :ref:`js-api-pyodide` module.
  * @memberof globalThis
  * @async
  */
 export async function loadPyodide(config) {
+  if (globalThis.__pyodide_module) {
+    throw new Error("Pyodide is already loading.");
+  }
+  if (!config.indexURL) {
+    throw new Error("Please provide indexURL parameter to loadPyodide");
+  }
+
+  loadPyodide.inProgress = true;
+  // A global "mount point" for the package loaders to talk to pyodide
+  // See "--export-name=__pyodide_module" in buildpkg.py
+  globalThis.__pyodide_module = Module;
+
   const default_config = {
     fullStdLib: true,
     jsglobals: globalThis,
     stdin: globalThis.prompt ? globalThis.prompt : undefined,
+    homedir: "/home/pyodide",
   };
   config = Object.assign(default_config, config);
-  if (globalThis.__pyodide_module) {
-    if (globalThis.languagePluginURL) {
-      throw new Error(
-        "Pyodide is already loading because languagePluginURL is defined."
-      );
-    } else {
-      throw new Error("Pyodide is already loading.");
-    }
-  }
-  // A global "mount point" for the package loaders to talk to pyodide
-  // See "--export-name=__pyodide_module" in buildpkg.py
-  globalThis.__pyodide_module = Module;
-  loadPyodide.inProgress = true;
-  if (!config.indexURL) {
-    throw new Error("Please provide indexURL parameter to loadPyodide");
-  }
-  let baseURL = config.indexURL;
-  if (!baseURL.endsWith("/")) {
-    baseURL += "/";
+
+  if (!config.indexURL.endsWith("/")) {
+    config.indexURL += "/";
   }
-  Module.indexURL = baseURL;
-  let packageIndexReady = initializePackageIndex(baseURL);
+  Module.indexURL = config.indexURL;
+  let packageIndexReady = initializePackageIndex(config.indexURL);
+  let pyodide_py_tar_promise = _fetchBinaryFile(
+    config.indexURL,
+    "pyodide_py.tar"
+  );
 
   setStandardStreams(config.stdin, config.stdout, config.stderr);
+  setHomeDirectory(config.homedir);
 
-  Module.locateFile = (path) => baseURL + path;
   let moduleLoaded = new Promise((r) => (Module.postRun = r));
 
-  const scriptSrc = `${baseURL}pyodide.asm.js`;
+  const scriptSrc = `${config.indexURL}pyodide.asm.js`;
   await loadScript(scriptSrc);
 
   // _createPyodideModule is specified in the Makefile by the linker flag:
@@ -230,85 +285,18 @@ export async function loadPyodide(config) {
   // being called.
   await moduleLoaded;
 
-  fixRecursionLimit();
-  let pyodide = makePublicAPI();
-
-  // Bootstrap steps:
-  //
-  //   1. _pyodide_core is ready now so we can call _pyodide.register_js_finder
-  //   2. Use the jsfinder to register the js and pyodide_js packages
-  //   3. Import pyodide, this requires _pyodide_core, js and pyodide_js to be
-  //      ready.
-  //   4. Add the pyodide_py and Python __main__.__dict__ objects to pyodide_js
-  Module.runPythonSimple(`
-def temp(pyodide_js, Module, jsglobals):
-  from _pyodide._importhook import register_js_finder
-  jsfinder = register_js_finder()
-  jsfinder.register_js_module("js", jsglobals)
-  jsfinder.register_js_module("pyodide_js", pyodide_js)
-
-  import pyodide
-  import __main__
-  import builtins
-
-  globals = __main__.__dict__
-  globals.update(builtins.__dict__)
-
-  Module.version = pyodide.__version__
-  Module.globals = globals
-  Module.builtins = builtins.__dict__
-  Module.pyodide_py = pyodide
-  print("Python initialization complete")
-`);
-
-  Module.init_dict.get("temp")(pyodide, Module, config.jsglobals);
-  // Module.runPython works starting from here!
-
-  // Wrap "globals" in a special Proxy that allows `pyodide.globals.x` access.
-  // TODO: Should we have this?
-  Module.globals = wrapNamespace(Module.globals);
+  const pyodide_py_tar = await pyodide_py_tar_promise;
+  unpackPyodidePy(pyodide_py_tar);
+  Module._pyodide_init();
 
-  pyodide.globals = Module.globals;
-  pyodide.pyodide_py = Module.pyodide_py;
-  pyodide.version = Module.version;
+  let pyodide = finalizeBootstrap(config);
+  // Module.runPython works starting here.
 
   await packageIndexReady;
   if (config.fullStdLib) {
     await loadPackage(["distutils"]);
   }
-
+  pyodide.runPython("print('Python initialization complete')");
   return pyodide;
 }
 globalThis.loadPyodide = loadPyodide;
-
-if (globalThis.languagePluginUrl) {
-  console.warn(
-    "languagePluginUrl is deprecated and will be removed in version 0.18.0, " +
-      "instead use loadPyodide({ indexURL : <some_url>})"
-  );
-
-  /**
-   * A deprecated parameter that specifies the Pyodide ``indexURL``. If present,
-   * Pyodide will automatically invoke
-   * ``loadPyodide({indexURL : languagePluginUrl})``
-   * and will store the resulting promise in
-   * :any:`globalThis.languagePluginLoader`. Use :any:`loadPyodide`
-   * directly instead of defining this.
-   *
-   * @type String
-   * @deprecated Will be removed in version 0.18.0
-   */
-  globalThis.languagePluginUrl;
-
-  /**
-   * A deprecated promise that resolves to ``undefined`` when Pyodide is
-   * finished loading. Only created if :any:`languagePluginUrl` is
-   * defined. Instead use :any:`loadPyodide`.
-   *
-   * @type Promise
-   * @deprecated Will be removed in version 0.18.0
-   */
-  globalThis.languagePluginLoader = loadPyodide({
-    indexURL: globalThis.languagePluginUrl,
-  }).then((pyodide) => (self.pyodide = pyodide));
-}