requirejs-esm 1.1.0 → 2.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "requirejs-esm",
-  "version": "1.1.0",
+  "version": "2.1.0",
   "description": "A RequireJS plugin converting JavaScript modules from ESM to AMD.",
   "author": "Ferdinand Prantl <prantlf@gmail.com>",
   "license": "MIT",
@@ -21,13 +21,16 @@
   "engines": {
     "node": ">=14"
   },
-  "main": "dist/plugin.js",
+  "main": "dist/api.js",
+  "module": "src/api.js",
+  "types": "src/api.d.ts",
   "bin": {
     "esm2requirejs": "bin/esm2requirejs.js"
   },
   "files": [
     "bin",
-    "dist"
+    "dist",
+    "src"
   ],
   "scripts": {
     "prepare": "npm run build && npm run demo",
@@ -39,10 +42,11 @@
     "demo:local": "npm run optimize:local && npm run minify:local",
     "optimize:local": "r.js -o demo-local/build.config.js",
     "minify:local": "terser -cm --ecma 2018 --comments false --source-map content=demo-local/main-built.js.map --source-map includeSources --source-map url=main-built.min.js.map -o demo-local/main-built.min.js demo-local/main-built.js",
-    "lint": "eslint -c .eslintrc.browser.yml src && eslint -c .eslintrc.node.yml *.js perf/*.js bin 'test/*.js'",
+    "lint": "eslint -c .eslintrc.browser.yml src && eslint -c .eslintrc.node.yml perf/*.js bin 'test/*.js'",
     "check": "tehanu test/*.js",
     "cover": "c8 node test && c8 --no-clean ./bin/esm2requirejs.js && c8 --no-clean ./bin/esm2requirejs.js test/input/esm-import-all.js && c8 --no-clean ./bin/esm2requirejs.js dummy || c8 report -r text -r lcov | grep -Ev '(\\.\\.\\.)|(index.mjs)|(source-map.js)|(base64-vlq.js)|(base64.js)|(binary-search.js)|(mapping-list.js)|(read-wasm-browser.js)|(url-browser.js)|(wasm.js)|(regexes.js)|(public-api.js)|(All files)' && c8 check-coverage",
-    "test": "npm run lint && npm run cover"
+    "test": "npm run lint && npm run cover",
+    "start": "python3 -m http.server"
   },
   "tehanu": {
     "autostart": false
@@ -78,29 +82,28 @@
       "@semantic-release/git"
     ]
   },
-  "devDependencies": {
+  "dependencies": {
     "@prantlf/convert-source-map": "^2.0.0",
-    "@prantlf/requirejs": "^2.5.0",
+    "astring": "^1.8.1",
+    "charcodes": "^0.2.0",
+    "commander": "^9.1.0",
+    "meriyah": "^4.2.1",
+    "punycode": "^2.1.1",
+    "source-map": "^0.8.0-beta.0",
+    "tiny-glob": "^0.2.9"
+  },
+  "devDependencies": {
+    "@prantlf/requirejs": "^3.0.0",
     "@rollup/plugin-commonjs": "^21.0.3",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^13.1.3",
     "@semantic-release/changelog": "^6.0.1",
     "@semantic-release/git": "^10.0.1",
-    "astring": "^1.8.1",
     "c8": "^7.11.0",
-    "charcodes": "^0.2.0",
-    "connect": "^3.7.0",
-    "connect-block-favicon": "^1.0.4",
-    "escodegen": "^2.0.0",
-    "eslint": "^8.12.0",
+    "eslint": "^8.13.0",
     "lit-html": "^1",
-    "meriyah": "^4.2.1",
-    "morgan": "^1.10.0",
     "rollup": "^2.70.1",
-    "serve-index": "^1.9.1",
-    "serve-static": "^1.15.0",
-    "source-map": "^0.8.0-beta.0",
-    "tehanu": "^0.2.1",
+    "tehanu": "^0.2.2",
     "tehanu-repo-coco": "^0.0.2",
     "tehanu-teru": "^0.2.2",
     "terser": "^5.12.1"
@@ -112,12 +115,5 @@
     "amd",
     "esm",
     "es6"
-  ],
-  "dependencies": {
-    "commander": "^9.1.0",
-    "cors": "^2.8.5",
-    "mime": "^3.0.0",
-    "polka": "^0.5.2",
-    "tiny-glob": "^0.2.9"
-  }
+  ]
 }

package/src/api.d.ts

@@ -0,0 +1,14 @@
+type NeedsResolve = (sourcePath: string, currentFile: string) => boolean
+
+interface ResolveOptions {
+  pluginName?: string
+  needsResolve?: NeedsResolve
+}
+
+declare function resolvePath(sourcePath: string, currentFile: string, options?: ResolveOptions): string
+
+type ResolvePath = ((sourcePath: string, currentFile: string, options?: ResolveOptions) => string) | false
+
+declare function transform(contents: string, path: string, options?: {
+  pluginName?: string /*= 'esm'' */, resolvePath?: ResolvePath,
+  sourceMap?: boolean /*= true */ }): string

package/src/api.js

@@ -0,0 +1,2 @@
+export { resolvePath } from './resolve-path'
+export { default as transform } from './transform'

package/src/fetch-text.js

@@ -0,0 +1,35 @@
+/* global require */
+
+let fetchText
+
+// Initialise the fetchText variable with a function to download
+// from a URL or to read from the file system.
+/* istanbul ignore if */
+if (typeof window !== 'undefined' && window.navigator && window.document) {
+  fetchText = (url, callback) => {
+    const xhr = new XMLHttpRequest()
+    xhr.open('GET', url, true)
+    xhr.onreadystatechange = () => {
+      if (xhr.readyState === 4) {
+        if (xhr.status === 200) {
+          callback(null, xhr.responseText)
+        } else {
+          callback(new Error(xhr.statusText))
+        }
+      }
+    }
+    xhr.send(null)
+  }
+} else {
+  const { readFileSync } = require.nodeRequire ? require.nodeRequire('fs') : require('fs')
+  fetchText = (path, callback) => {
+    // Asynchronous reading is not possible during the build in the optimizer.
+    try {
+      callback(null, readFileSync(path, 'utf8'))
+    } catch (error) {
+      callback(error)
+    }
+  }
+}
+
+export default fetchText

package/src/plugin.js

@@ -0,0 +1,147 @@
+/* global module */
+
+import fetchText from './fetch-text'
+import writeText from './write-text'
+import { childFile } from './resolve-path'
+import { setSkipModules, skipModule } from './skip-module'
+import transformSource from './transform'
+
+const {
+  // Allow using a different plugin alias than `esm` in the source code.
+  pluginName,
+  // Assume that the original sources are JavaScript files by default.
+  fileExtension = '.js',
+  // Flags to enforce or suppress the transpilation of not yet defined modules.
+  mixedAmdAndEsm,
+  onlyAmd,
+  // List of module names not to check and transform.
+  skipModules = [],
+  // Method to update paths of module dependencies, to prefix JavaScript module
+  // name with `esm!`, above all.
+  resolvePath,
+  // ecmaVersion,
+  // Boolean or object with booleans { inline, content }.
+  sourceMap,
+  // Enable console logging.
+  verbose,
+  // Directory to save a copy of the transformed modules.
+  debugDir
+} = typeof module !== 'undefined' && module.config && module.config() || {}
+
+const buildMap = {}
+
+setSkipModules(skipModules)
+
+//>>excludeEnd('excludeEsm')
+export default {
+  load(name, req, onload, reqConfig) {
+//>>excludeStart('excludeEsm', pragmas.excludeEsm)
+    const { isBuild } = reqConfig
+    // if (!bundledModules) {
+    //   bundledModules = new Set()
+    //   const { bundles } = reqConfig
+    //   if (bundles) {
+    //     verbose && console.log('esm: initialising bundles', name)
+    //     for (const bundle in bundles) {
+    //       for (const module of bundles[bundle]) {
+    //         bundledModules.add(module)
+    //       }
+    //     }
+    //   }
+    // }
+
+    // Paths relative to the current directory include the file extension.
+    // Otherwise the file extension must not be used for JavaScript modules.
+    // The file name should include the orioginal extension in the source maps.
+    const file = name.endsWith(fileExtension) ? name : name + fileExtension
+    // Compilation and bundling of module sub-trees can be skipped, if those
+    // are mapped to the pseudo-path `empty:`, meaning that those modules
+    // are external and will be loaded during the runtime.
+    const url = req.toUrl(file)
+    if (url.startsWith('empty:')) {
+      verbose && console.log('esm: skipping', name, 'mapped to', url)
+      return onload()
+    }
+
+    verbose && console.log('esm: loading', name)
+    // If the module has been already defined from a module bundle, it was
+    // already transpiled, when the output bundle was written. No need to
+    // re-transpile it. This can happen only during the runtime.
+    //
+    // The re-transpilation is however necessary, if a testing page or a unit
+    // test loads source ESM modules with extensions, which are going to be
+    // consumed by a bundled module. Then the bundled module has to prefix
+    // each of its dependencies with `esm!` to ensure their transpilation.
+    // This mixed mode can to be enabled by the flag `mixedAmdAndEsm`, if needed.
+    //
+    // If the whole application is transpiled, there is no need to transpile
+    // ESM modules or prefix dependencies of AMD modules. Even not yet defined
+    // modules can be loaded just by `require` to get better performance.
+    if (!mixedAmdAndEsm && !isBuild && req.specified(name) ||
+        /*bundledModules.has(name) ||*/ onlyAmd || skipModule(name) ||
+        // If the module was bundled, it had to be already transpiled.
+        !isBuild && childFile(url) !== childFile(file)) {
+      verbose && console.log('esm: delegating', name)
+//>>excludeEnd('excludeEsm')
+      return req([name], onload, onload.error)
+//>>excludeStart('excludeEsm', pragmas.excludeEsm)
+    }
+
+    // Fetch the text of the source module by AJAX and transpile it.
+    verbose && console.log('esm: fetching', name, 'mapped to', url)
+    fetchText(url, async (error, text) => {
+      if (error) {
+        verbose && console.log('esm: missing', name)
+        return onload.error(error)
+      }
+
+      let code, updated
+      try {
+        verbose && console.log('esm: transforming', name);
+        ({ code, updated } = transformSource(text, file, {
+          pluginName,
+          resolvePath,
+          /*ecmaVersion,*/
+          // Always produce the source maps when transpiling in the browser, otherwise
+          // the debugging would me impossible. When building and bundling, check if
+          // the source maps were enabled for the output.
+          sourceMap: sourceMap || !isBuild
+        }))
+        if (!updated) {
+          verbose && console.log('esm: retaining', name)
+          // return req([name], onload, onload.error)
+        } else if (isBuild && debugDir) {
+          writeText(`${debugDir}/${file}`, code)
+        }
+      } catch (error) {
+        // RequireJS did not always log this error.
+        console.error(`Transforming "${name}" (resolved to "${url}") failed:`)
+        console.error(error)
+        return onload.error(error)
+      }
+
+      // Remember the transpiled content for the writing phase during the build.
+      if (isBuild) {
+        buildMap[name] = code
+      }
+
+      verbose && console.log('esm: returning', name)
+      onload.fromText(code)
+    })
+  },
+
+  write(pluginName, moduleName, write) {
+    const code = buildMap[moduleName]
+    if (code) {
+      // Add the transpiled module under the original name. Earlier modules
+      // refer it with that name, before the module was converted to ESM.
+      write.asModule(moduleName, code)
+      // Add a stub of the module with the name prefixed by `esm!`. Modules
+      // compiled with esm refer it with that name and this stub will simplify
+      // the module loading by skipping the plugin evaluation.
+      write.asModule(`${pluginName}!${moduleName}`,
+        '\ndefine([\'' + moduleName + '\'], res => res);\n')
+    }
+//>>excludeEnd('excludeEsm')
+  }
+}

package/src/resolve-path.js

@@ -0,0 +1,80 @@
+// Returns the child file name by cutting the directoris before the last slash,
+// including the slash. If there is no slash in the path - the path is just
+// a file name, it will return the file name. If the path includes a URL
+// query starting with the question mark, it will cut it away including the
+// question mark.
+export function childFile (path) {
+  const lastSlash = path.lastIndexOf('/')
+  if (lastSlash > 0) {
+    path = path.substring(lastSlash + 1)
+  }
+  const questionMark = path.lastIndexOf('?')
+  return questionMark > 0 ? path.substring(0, questionMark) : path
+}
+
+// Returns the parent directory by cutting the file name after the last slash,
+// leaving the slash in the result. If there is no slash in the path - the path
+// is just a file name, it will return `undefined`.
+function parentDir (path) {
+  const lastSlash = path.lastIndexOf('/')
+  return lastSlash > 0 ? path.substring(0, lastSlash + 1) : undefined
+}
+
+// Trims the leading "./" off the path.
+function shortenPath(path) {
+  while (path.charAt(0) === '.' && path.charAt(1) === '/') {
+    path = path.substring(2)
+  }
+  return path
+}
+
+// Checks if the path starts with "../".
+function pointsToParent(path) {
+  return path.charAt(0) === '.' && path.charAt(1) === '.' && path.charAt(2) === '/'
+}
+
+// Joins two paths together and avoids leaving "/./" or "/../" in the middle.
+function joinPath (first, second) {
+  // If the second part starts with "./", it can be safely removed.
+  second = shortenPath(second)
+  // The parent path can be undefined, if the file is located in the current directory.
+  if (first !== undefined) {
+    // As long as "../" can be removed from the second path, shorten the first one.
+    while (pointsToParent(second)) {
+      // Remove the leading "../" and trim futher all leading "./".
+      second = shortenPath(second.substring(3))
+      // Cut the last directory from the first path.
+      first = parentDir(first)
+      // If the last part of the first path was removed and there is no parent
+      // directory to go further, return the rest of the second path.
+      if (first === undefined) {
+        return second
+      }
+    }
+    // Return what remains from the first path concatenated with the second one.
+    second = first + second
+  }
+  return second
+}
+
+// Ensures that every JavaScript dependency will be prefixed by `esm!`.
+// Relative paths will be converted to be relative to the parent module.
+export function resolvePath (sourcePath, currentFile, { pluginName, needsResolve } = {}) {
+  // Ignore paths with other plugins applied and the three built-in
+  // pseudo-modules of RequireJS.
+  if (sourcePath.includes('!') || sourcePath === 'require' ||
+      sourcePath === 'module' || sourcePath === 'exports') return
+
+  // If `sourcePath` is relative to `currentFile` - starts with ./ or ../ -
+  // prepend the parent directory of `currentFile` to it. This was needed
+  // for modules located outside the source root (`baseUrl`), which were
+  // mapped there using the `paths` of `map` configuration properties.
+  if ((sourcePath.charAt(0) === '.' && (sourcePath.charAt(1) === '/' ||
+       sourcePath.charAt(1) === '.' && sourcePath.charAt(2) === '/')) &&
+      !(needsResolve && needsResolve(sourcePath, currentFile))) {
+    sourcePath = joinPath(parentDir(currentFile), sourcePath)
+    if (sourcePath.endsWith('.js')) sourcePath = sourcePath.substring(0, sourcePath.length - 3)
+  }
+
+  return pluginName ? `${pluginName}!${sourcePath}` : sourcePath
+}

package/src/skip-module.js

@@ -0,0 +1,18 @@
+let skipModules = []
+
+export function setSkipModules(names) {
+  skipModules = names
+}
+
+// Checks if a module name is included in the list of modules to skip
+// the transformation.
+export function skipModule(name) {
+  for (const skipModule of skipModules) {
+    // Recognise an asterisk at the beginning
+    if (skipModule.startsWith('*')) {
+      if (name.indexOf(skipModule.substring(1)) >= 0) return true
+    }
+    // Accept a path prefix as well.
+    else if (name.startsWith(skipModule)) return true
+  }
+}

package/src/transform.js

@@ -0,0 +1,47 @@
+import { resolvePath as originalResolvePath } from './resolve-path'
+// import { parse } from 'acorn'
+import { parseModule } from 'meriyah'
+import { generate } from 'astring'
+import { SourceMapGenerator } from 'source-map'
+import convert from '@prantlf/convert-source-map'
+import transformModules from './transformer'
+
+export default function transform(text, file, {
+  // Allow using a different plugin alias than `esm` in the source code.
+  pluginName = 'esm',
+  // Method to update paths of module dependencies, to prefix JavaScript module
+  // name with `esm!`, above all.
+  resolvePath = originalResolvePath,
+  // ecmaVersion = 2020,
+  sourceMap
+} = {}) {
+  // const ast = parse(text, { ecmaVersion, sourceType: 'module', locations: true })
+  let ast = parseModule(text, { next: true, loc: true })
+
+  const options = { sourceFileName: file, pluginName, resolvePath, originalResolvePath }
+  transformModules(ast, options)
+
+  const { updated } = options
+  let code, map
+  if (updated) {
+    if (sourceMap === true) {
+      sourceMap = { inline: true, content: true }
+    }
+    const mapGenerator = sourceMap ? new SourceMapGenerator({ file }) : undefined
+    code = generate(ast, { sourceMap: mapGenerator })
+    if (sourceMap) {
+      if (sourceMap.content) {
+        mapGenerator.setSourceContent(file, text)
+      }
+      if (sourceMap.inline) {
+        code += convert.fromObject(mapGenerator.toJSON()).toComment()
+      } else {
+        map = mapGenerator.toJSON()
+      }
+    }
+  } else {
+    code = text
+  }
+
+  return { code, map, updated }
+}

package/src/transformer/amd.js

@@ -0,0 +1,98 @@
+import { replaceLiteral } from './converters'
+
+// Detects if an expression calls define or require function.
+// Returns information about an AMD module false, { deps } or [{ deps }, ...].
+function detectDefineOrRequireCall(expr) {
+  if (expr.type !== 'CallExpression') return false
+
+  const args = expr.arguments
+  const { length } = args
+  if (length === 0) return false
+
+  const { callee } = expr
+  let func
+  // namespace.define(...)
+  if (callee.type === 'MemberExpression') {
+    const { object } = callee
+    if (object.type !== 'Identifier') return false
+    func = callee.property
+  } else {
+    func = callee
+  }
+  if (func.type !== 'Identifier') return false
+
+  // define('name', [deps], factory)
+  if (func.name === 'define') {
+    let index = 0
+    let arg = args[index]
+    let deps
+    if (arg.type === 'Literal') {
+      if (length <= ++index || typeof arg.value !== 'string') return false
+      arg = args[index]
+    }
+    if (arg.type === 'ArrayExpression') {
+      deps = arg
+      if (length <= ++index) return false
+      arg = args[index]
+    }
+    return (arg.type === 'FunctionExpression' || arg.type === 'ObjectExpression') &&
+      { deps: deps }
+  }
+
+  // require([deps], success, error)
+  if (func.name === 'require') {
+    const deps = args[0]
+    return deps.type === 'ArrayExpression' && length >= 2 &&
+      args[1].type === 'FunctionExpression' && { deps: deps }
+  }
+}
+
+// Detects if a program contains statements calling define or require function.
+// Returns information about AMD modules false, { deps } or [{ deps }, ...].
+function detectDefineOrRequire(stat) {
+  if (stat.type !== 'ExpressionStatement') return false
+  const { expression } = stat
+
+  // multiple define/require statements in one file
+  if (expression.type === 'SequenceExpression') {
+    return expression
+      .expressions
+      .map(detectDefineOrRequireCall)
+      .reduce((all, one) => one ? all.concat(one) : all, [])
+  }
+
+  return detectDefineOrRequireCall(expression)
+}
+
+// Detects if a program contains statements calling define or require function.
+// Returns information about the AMD modules [{ deps }, ...] or [].
+export function detectDefinesOrRequires(program) {
+  const { body } = program
+  return body
+    .map(detectDefineOrRequire)
+    .reduce((all, one) => one ? all.concat(one) : all, [])
+}
+
+// Updates dependency paths to be prefixed by `esm!` or otherwise updated.
+export function updateAmdDeps(amd, options) {
+  const { deps } = amd
+  if (!deps) return
+
+  const { sourceFileName: parentName } = options
+  const { elements } = deps
+  if (!elements.length) return
+
+  const { resolvePath } = options
+  let updated
+  for (const element of elements) {
+    if (element.type === 'Literal') {
+      const moduleName = element.value
+      const newModuleName = resolvePath(moduleName, parentName, options)
+      if (newModuleName && newModuleName !== moduleName) {
+        replaceLiteral(element, newModuleName)
+        updated = true
+      }
+    }
+  }
+  return updated
+}

package/src/transformer/converters.js

@@ -0,0 +1,102 @@
+import {
+  identifier, memberExpression, assignmentExpression, expressionStatement
+} from './factories'
+import { isValidIdentifier } from './validators'
+import { isIdentifierChar } from './identifier'
+
+export function toIdentifier(input) {
+  input = input + ''
+
+  let name = ''
+  for (const c of input) {
+    name += isIdentifierChar(c.codePointAt(0)) ? c : '-'
+  }
+
+  name = name.replace(/^[-0-9]+/, '')
+
+  name = name.replace(/[-\s]+(.)?/g, function (match, c) {
+    return c ? c.toUpperCase() : ''
+  })
+
+  if (!isValidIdentifier(name)) {
+    name = `_${name}`
+  }
+
+  return name || '_'
+}
+
+export function toExpression(node) {
+  if (node.type === 'ExpressionStatement') {
+    node = node.expression
+  }
+
+  const { type } = node
+
+  if (type.endsWith('Expression')) return node
+
+  if (type === 'ClassDeclaration') {
+    node.type = 'ClassExpression'
+  } else if (type === 'FunctionDeclaration') {
+    node.type = 'FunctionExpression'
+  }
+
+  if (!node.type.endsWith('Expression')) {
+    throw new Error(`cannot turn ${node.type} to an expression`)
+  }
+
+  return node
+}
+
+export function toStatement(node, ignore) {
+  const { type } = node
+
+  if (type.endsWith('Statement')) return node
+
+  let mustHaveId = false
+  let newType
+
+  if (type === 'ClassDeclaration' || type === 'ClassExpression') {
+    mustHaveId = true
+    newType = 'ClassDeclaration'
+  } else if (type === 'FunctionDeclaration' || type === 'FunctionExpression' ||
+             type === 'ArrowFunctionExpression') {
+    mustHaveId = true
+    newType = 'FunctionDeclaration'
+  } else if (type === 'AssignmentExpression') {
+    return expressionStatement(node)
+  }
+
+  if (mustHaveId && !node.id) {
+    newType = false
+  }
+
+  if (!newType) {
+    if (ignore) {
+      return false
+    } else {
+      throw new Error(`cannot turn ${node.type} to a statement`)
+    }
+  }
+
+  node.type = newType
+
+  return node
+}
+
+export function replaceLiteral(literal, value) {
+  literal.value = value
+  const { raw } = literal
+  if (raw === undefined) return
+  if (typeof value === 'string') {
+    literal.raw = String(raw).charAt(0) === "'" ?
+      `'${value.replaceAll("'", "\\'")}'` : `"${value.replaceAll('"', '\\"')}"`
+  } else {
+    literal.raw = String(value)
+  }
+}
+
+// Returns a statement assigning a property value to the exports object.
+export function exportStatement(exportsVar, key, value) {
+  return toStatement(assignmentExpression('=',
+    memberExpression(exportsVar, identifier(key)), value))
+}