pyodide 0.19.1 → 0.20.1-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pyodide",
-  "version": "0.19.1",
+  "version": "0.20.1-alpha.1",
   "description": "The Pyodide JavaScript package",
   "keywords": [
     "python",
@@ -16,22 +16,50 @@
   },
   "license": "Apache-2.0",
   "devDependencies": {
+    "@rollup/plugin-commonjs": "^21.0.1",
+    "@rollup/plugin-node-resolve": "^13.1.3",
+    "@types/assert": "^1.5.6",
+    "@types/emscripten": "^1.39.5",
+    "@types/expect": "^24.3.0",
+    "@types/mocha": "^9.1.0",
+    "@types/node": "^17.0.25",
+    "@types/node-fetch": "^2.6.1",
+    "@types/ws": "^8.5.3",
+    "dts-bundle-generator": "^6.7.0",
+    "error-stack-parser": "^2.0.6",
+    "mocha": "^9.0.2",
     "prettier": "^2.2.1",
-    "terser": "^5.7.0",
     "rollup": "^2.48.0",
     "rollup-plugin-terser": "^7.0.2",
+    "rollup-plugin-ts": "^2.0.5",
+    "terser": "^5.7.0",
+    "ts-mocha": "^9.0.2",
     "tsd": "^0.15.1",
-    "typescript": "^4.2.4",
-    "mocha": "^9.0.2",
-    "@types/emscripten": "^1.39.5"
+    "typedoc": "^0.22.11",
+    "typescript": "^4.2.4"
+  },
+  "main": "pyodide.js",
+  "exports": {
+    ".": {
+      "require": "./pyodide.js",
+      "import": "./pyodide.mjs"
+    },
+    "./pyodide.mjs": "./pyodide.mjs",
+    "./pyodide.js": "./pyodide.js",
+    "./package.json": "./package.json"
   },
-  "type": "module",
+  "files": [
+    "pyodide*",
+    "packages.json",
+    "distutils.tar",
+    "console.html"
+  ],
   "scripts": {
-    "test": "mocha"
+    "test": "ts-mocha -p tsconfig.test.json test/**/*.test.ts"
   },
   "mocha": {
     "timeout": 30000,
-    "file": "./test/conftest.js"
+    "file": "test/conftest.ts"
   },
   "tsd": {
     "compilerOptions": {
@@ -43,6 +71,7 @@
   },
   "dependencies": {
     "base-64": "^1.0.0",
-    "node-fetch": "^2.6.1"
+    "node-fetch": "^2.6.1",
+    "ws": "^8.5.0"
   }
 }

package/pyodide.ts

@@ -0,0 +1,301 @@
+/**
+ * The main bootstrap code for loading pyodide.
+ */
+import ErrorStackParser from "error-stack-parser";
+import { loadScript, _loadBinaryFile, initNodeModules } from "./compat";
+
+import { createModule, setStandardStreams, setHomeDirectory } from "./module";
+
+import type { PyodideInterface } from "./api.js";
+import type { PyProxy, PyProxyDict } from "./pyproxy.gen";
+
+export type {
+  PyProxy,
+  PyProxyWithLength,
+  PyProxyDict,
+  PyProxyWithGet,
+  PyProxyWithSet,
+  PyProxyWithHas,
+  PyProxyIterable,
+  PyProxyIterator,
+  PyProxyAwaitable,
+  PyProxyBuffer,
+  PyProxyCallable,
+  TypedArray,
+  PyBuffer,
+} from "./pyproxy.gen";
+
+export type Py2JsResult = any;
+
+/**
+ * A proxy around globals that falls back to checking for a builtin if has or
+ * get fails to find a global with the given key. Note that this proxy is
+ * transparent to js2python: it won't notice that this wrapper exists at all and
+ * will translate this proxy to the globals dictionary.
+ * @private
+ */
+function wrapPythonGlobals(
+  globals_dict: PyProxyDict,
+  builtins_dict: PyProxyDict
+) {
+  return new Proxy(globals_dict, {
+    get(target, symbol) {
+      if (symbol === "get") {
+        return (key: any) => {
+          let result = target.get(key);
+          if (result === undefined) {
+            result = builtins_dict.get(key);
+          }
+          return result;
+        };
+      }
+      if (symbol === "has") {
+        return (key: any) => target.has(key) || builtins_dict.has(key);
+      }
+      return Reflect.get(target, symbol);
+    },
+  });
+}
+
+function unpackPyodidePy(Module: any, pyodide_py_tar: Uint8Array) {
+  const fileName = "/pyodide_py.tar";
+  let stream = Module.FS.open(fileName, "w");
+  Module.FS.write(
+    stream,
+    pyodide_py_tar,
+    0,
+    pyodide_py_tar.byteLength,
+    undefined,
+    true
+  );
+  Module.FS.close(stream);
+  const code_ptr = Module.stringToNewUTF8(`
+from sys import version_info
+pyversion = f"python{version_info.major}.{version_info.minor}"
+import shutil
+shutil.unpack_archive("/pyodide_py.tar", f"/lib/{pyversion}/site-packages/")
+del shutil
+import importlib
+importlib.invalidate_caches()
+del importlib
+    `);
+  let errcode = Module._PyRun_SimpleString(code_ptr);
+  if (errcode) {
+    throw new Error("OOPS!");
+  }
+  Module._free(code_ptr);
+  Module.FS.unlink(fileName);
+}
+
+/**
+ * This function is called after the emscripten module is finished initializing,
+ * so eval_code is newly available.
+ * It finishes the bootstrap so that once it is complete, it is possible to use
+ * the core `pyodide` apis. (But package loading is not ready quite yet.)
+ * @private
+ */
+function finalizeBootstrap(API: any, config: ConfigType) {
+  // First make internal dict so that we can use runPythonInternal.
+  // runPythonInternal uses a separate namespace, so we don't pollute the main
+  // environment with variables from our setup.
+  API.runPythonInternal_dict = API._pyodide._base.eval_code("{}") as PyProxy;
+  API.importlib = API.runPythonInternal("import importlib; importlib");
+  let import_module = API.importlib.import_module;
+
+  API.sys = import_module("sys");
+  API.sys.path.insert(0, config.homedir);
+
+  // Set up globals
+  let globals = API.runPythonInternal(
+    "import __main__; __main__.__dict__"
+  ) as PyProxyDict;
+  let builtins = API.runPythonInternal(
+    "import builtins; builtins.__dict__"
+  ) as PyProxyDict;
+  API.globals = wrapPythonGlobals(globals, builtins);
+
+  // Set up key Javascript modules.
+  let importhook = API._pyodide._importhook;
+  importhook.register_js_finder();
+  importhook.register_js_module("js", config.jsglobals);
+
+  let pyodide = API.makePublicAPI();
+  importhook.register_js_module("pyodide_js", pyodide);
+
+  // import pyodide_py. We want to ensure that as much stuff as possible is
+  // already set up before importing pyodide_py to simplify development of
+  // pyodide_py code (Otherwise it's very hard to keep track of which things
+  // aren't set up yet.)
+  API.pyodide_py = import_module("pyodide");
+  API.package_loader = import_module("pyodide._package_loader");
+  API.version = API.pyodide_py.__version__;
+
+  // copy some last constants onto public API.
+  pyodide.pyodide_py = API.pyodide_py;
+  pyodide.version = API.version;
+  pyodide.globals = API.globals;
+  return pyodide;
+}
+
+declare function _createPyodideModule(Module: any): Promise<void>;
+
+/**
+ *  If indexURL isn't provided, throw an error and catch it and then parse our
+ *  file name out from the stack trace.
+ *
+ *  Question: But getting the URL from error stack trace is well... really
+ *  hacky. Can't we use
+ *  [`document.currentScript`](https://developer.mozilla.org/en-US/docs/Web/API/Document/currentScript)
+ *  or
+ *  [`import.meta.url`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import.meta)
+ *  instead?
+ *
+ *  Answer: `document.currentScript` works for the browser main thread.
+ *  `import.meta` works for es6 modules. In a classic webworker, I think there
+ *  is no approach that works. Also we would need some third approach for node
+ *  when loading a commonjs module using `require`. On the other hand, this
+ *  stack trace approach works for every case without any feature detection
+ *  code.
+ */
+function calculateIndexURL(): string {
+  let err;
+  try {
+    throw new Error();
+  } catch (e) {
+    err = e;
+  }
+  let fileName = ErrorStackParser.parse(err)[0].fileName!;
+  return fileName.slice(0, fileName.lastIndexOf("/"));
+}
+
+/**
+ * See documentation for loadPyodide.
+ * @private
+ */
+export type ConfigType = {
+  indexURL: string;
+  homedir: string;
+  fullStdLib?: boolean;
+  stdin?: () => string;
+  stdout?: (msg: string) => void;
+  stderr?: (msg: string) => void;
+  jsglobals?: object;
+};
+
+/**
+ * Load the main Pyodide wasm module and initialize it.
+ *
+ * Only one copy of Pyodide can be loaded in a given JavaScript global scope
+ * because Pyodide uses global variables to load packages. If an attempt is made
+ * to load a second copy of Pyodide, :any:`loadPyodide` will throw an error.
+ * (This can be fixed once `Firefox adopts support for ES6 modules in webworkers
+ * <https://bugzilla.mozilla.org/show_bug.cgi?id=1247687>`_.)
+ *
+ * @returns The :ref:`js-api-pyodide` module.
+ * @memberof globalThis
+ * @async
+ */
+export async function loadPyodide(
+  options: {
+    /**
+     * The URL from which Pyodide will load the main Pyodide runtime and
+     * packages. Defaults to the url that pyodide is loaded from with the file
+     * name (pyodide.js or pyodide.mjs) removed. It is recommended that you
+     * leave this undefined, providing an incorrect value can cause broken
+     * behavior.
+     */
+    indexURL?: string;
+
+    /**
+     * The home directory which Pyodide will use inside virtual file system. Default: "/home/pyodide"
+     */
+    homedir?: string;
+
+    /** Load the full Python standard library.
+     * Setting this to false excludes following modules: distutils.
+     * Default: true
+     */
+    fullStdLib?: boolean;
+    /**
+     * Override the standard input callback. Should ask the user for one line of input.
+     */
+    stdin?: () => string;
+    /**
+     * Override the standard output callback.
+     * Default: undefined
+     */
+    stdout?: (msg: string) => void;
+    /**
+     * Override the standard error output callback.
+     * Default: undefined
+     */
+    stderr?: (msg: string) => void;
+    jsglobals?: object;
+  } = {}
+): Promise<PyodideInterface> {
+  if (!options.indexURL) {
+    options.indexURL = calculateIndexURL();
+  }
+
+  const default_config = {
+    fullStdLib: true,
+    jsglobals: globalThis,
+    stdin: globalThis.prompt ? globalThis.prompt : undefined,
+    homedir: "/home/pyodide",
+  };
+  const config = Object.assign(default_config, options) as ConfigType;
+  if (!config.indexURL.endsWith("/")) {
+    config.indexURL += "/";
+  }
+  await initNodeModules();
+  const pyodide_py_tar_promise = _loadBinaryFile(
+    config.indexURL,
+    "pyodide_py.tar"
+  );
+
+  const Module = createModule();
+  const API: any = { config };
+  Module.API = API;
+
+  setStandardStreams(Module, config.stdin, config.stdout, config.stderr);
+  setHomeDirectory(Module, config.homedir);
+
+  const moduleLoaded = new Promise((r) => (Module.postRun = r));
+
+  // locateFile tells Emscripten where to find the data files that initialize
+  // the file system.
+  Module.locateFile = (path: string) => config.indexURL + path;
+  const scriptSrc = `${config.indexURL}pyodide.asm.js`;
+  await loadScript(scriptSrc);
+
+  // _createPyodideModule is specified in the Makefile by the linker flag:
+  // `-s EXPORT_NAME="'_createPyodideModule'"`
+  await _createPyodideModule(Module);
+
+  // There is some work to be done between the module being "ready" and postRun
+  // being called.
+  await moduleLoaded;
+
+  // Disable further loading of Emscripten file_packager stuff.
+  Module.locateFile = (path: string) => {
+    throw new Error("Didn't expect to load any more file_packager files!");
+  };
+
+  const pyodide_py_tar = await pyodide_py_tar_promise;
+  unpackPyodidePy(Module, pyodide_py_tar);
+  Module._pyodide_init();
+
+  const pyodide = finalizeBootstrap(API, config);
+  // API.runPython works starting here.
+  if (!pyodide.version.includes("dev")) {
+    // Currently only used in Node to download packages the first time they are
+    // loaded. But in other cases it's harmless.
+    API.setCdnUrl(`https://cdn.jsdelivr.net/pyodide/v${pyodide.version}/full/`);
+  }
+  await API.packageIndexReady;
+  if (config.fullStdLib) {
+    await pyodide.loadPackage(["distutils"]);
+  }
+  pyodide.runPython("print('Python initialization complete')");
+  return pyodide;
+}

package/pyodide.umd.ts

@@ -0,0 +1,3 @@
+import { loadPyodide } from "./pyodide";
+export { loadPyodide };
+(globalThis as any).loadPyodide = loadPyodide;