webpack 4.14.0 → 4.15.1

package/lib/util/deterministicGrouping.js

@@ -0,0 +1,251 @@
+"use strict";
+
+// Simulations show these probabilities for a single change
+// 93.1% that one group is invalidated
+// 4.8% that two groups are invalidated
+// 1.1% that 3 groups are invalidated
+// 0.1% that 4 or more groups are invalidated
+//
+// And these for removing/adding 10 lexically adjacent files
+// 64.5% that one group is invalidated
+// 24.8% that two groups are invalidated
+// 7.8% that 3 groups are invalidated
+// 2.7% that 4 or more groups are invalidated
+//
+// And these for removing/adding 3 random files
+// 0% that one group is invalidated
+// 3.7% that two groups are invalidated
+// 80.8% that 3 groups are invalidated
+// 12.3% that 4 groups are invalidated
+// 3.2% that 5 or more groups are invalidated
+
+/**
+ *
+ * @param {string} a key
+ * @param {string} b key
+ * @returns {number} the similarity as number
+ */
+const similarity = (a, b) => {
+	const l = Math.min(a.length, b.length);
+	let dist = 0;
+	for (let i = 0; i < l; i++) {
+		const ca = a.charCodeAt(i);
+		const cb = b.charCodeAt(i);
+		dist += Math.max(0, 10 - Math.abs(ca - cb));
+	}
+	return dist;
+};
+
+/**
+ * @param {string} a key
+ * @param {string} b key
+ * @returns {string} the common part and a single char for the difference
+ */
+const getName = (a, b) => {
+	const l = Math.min(a.length, b.length);
+	let r = "";
+	for (let i = 0; i < l; i++) {
+		const ca = a.charAt(i);
+		const cb = b.charAt(i);
+		r += ca;
+		if (ca === cb) {
+			continue;
+		}
+		return r;
+	}
+	return a;
+};
+
+/**
+ * @template T
+ */
+class Node {
+	/**
+	 * @param {T} item item
+	 * @param {string} key key
+	 * @param {number} size size
+	 */
+	constructor(item, key, size) {
+		this.item = item;
+		this.key = key;
+		this.size = size;
+	}
+}
+
+/**
+ * @template T
+ */
+class Group {
+	/**
+	 * @param {Node<T>[]} nodes nodes
+	 * @param {number[]} similarities similarities between the nodes (length = nodes.length - 1)
+	 */
+	constructor(nodes, similarities) {
+		this.nodes = nodes;
+		this.similarities = similarities;
+		this.size = nodes.reduce((size, node) => size + node.size, 0);
+		/** @type {string} */
+		this.key = undefined;
+	}
+}
+
+/**
+ * @template T
+ * @typedef {Object} GroupedItems<T>
+ * @property {string} key
+ * @property {T[]} items
+ * @property {number} size
+ */
+
+/**
+ * @template T
+ * @typedef {Object} Options
+ * @property {number} maxSize maximum size of a group
+ * @property {number} minSize minimum size of a group (preferred over maximum size)
+ * @property {Iterable<T>} items a list of items
+ * @property {function(T): number} getSize function to get size of an item
+ * @property {function(T): string} getKey function to get the key of an item
+ */
+
+/**
+ * @template T
+ * @param {Options<T>} options options object
+ * @returns {GroupedItems<T>[]} grouped items
+ */
+module.exports = ({ maxSize, minSize, items, getSize, getKey }) => {
+	/** @type {Group<T>[]} */
+	const result = [];
+
+	const nodes = Array.from(
+		items,
+		item => new Node(item, getKey(item), getSize(item))
+	);
+
+	/** @type {Node<T>[]} */
+	const initialNodes = [];
+
+	// return nodes bigger than maxSize directly as group
+	for (const node of nodes) {
+		if (node.size >= maxSize) {
+			result.push(new Group([node], []));
+		} else {
+			initialNodes.push(node);
+		}
+	}
+
+	if (initialNodes.length > 0) {
+		// lexically ordering of keys
+		initialNodes.sort((a, b) => {
+			if (a.key < b.key) return -1;
+			if (a.key > b.key) return 1;
+			return 0;
+		});
+
+		// calculate similarities between lexically adjacent nodes
+		/** @type {number[]} */
+		const similarities = [];
+		for (let i = 1; i < initialNodes.length; i++) {
+			const a = initialNodes[i - 1];
+			const b = initialNodes[i];
+			similarities.push(similarity(a.key, b.key));
+		}
+
+		const queue = [new Group(initialNodes, similarities)];
+
+		while (queue.length) {
+			const group = queue.pop();
+			// only groups bigger than maxSize need to be splitted
+			if (group.size < maxSize) {
+				result.push(group);
+				continue;
+			}
+
+			// find unsplittable area from left and right
+			// going minSize from left and right
+			let left = 0;
+			let leftSize = 0;
+			while (leftSize < minSize) {
+				leftSize += group.nodes[left].size;
+				left++;
+			}
+			let right = group.nodes.length - 1;
+			let rightSize = 0;
+			while (rightSize < minSize) {
+				rightSize += group.nodes[right].size;
+				right--;
+			}
+
+			if (left - 1 > right) {
+				// can't split group while holding minSize
+				// because minSize is preferred of maxSize we return
+				// the group here even while it's too big
+				// To avoid this make sure maxSize > minSize * 3
+				result.push(group);
+				continue;
+			}
+			if (left <= right) {
+				// when there is a area between left and right
+				// we look for best split point
+				// we split at the minimum similiarity
+				// here key space is separated the most
+				let best = left - 1;
+				let bestSimilarity = group.similarities[best];
+				for (let i = left; i <= right; i++) {
+					const similarity = group.similarities[i];
+					if (similarity < bestSimilarity) {
+						best = i;
+						bestSimilarity = similarity;
+					}
+				}
+				left = best + 1;
+				right = best;
+			}
+
+			// create two new groups for left and right area
+			// and queue them up
+			const rightNodes = [group.nodes[right + 1]];
+			/** @type {number[]} */
+			const rightSimilaries = [];
+			for (let i = right + 2; i < group.nodes.length; i++) {
+				rightSimilaries.push(group.similarities[i - 1]);
+				rightNodes.push(group.nodes[i]);
+			}
+			queue.push(new Group(rightNodes, rightSimilaries));
+
+			const leftNodes = [group.nodes[0]];
+			/** @type {number[]} */
+			const leftSimilaries = [];
+			for (let i = 1; i < left; i++) {
+				leftSimilaries.push(group.similarities[i - 1]);
+				leftNodes.push(group.nodes[i]);
+			}
+			queue.push(new Group(leftNodes, leftSimilaries));
+		}
+	}
+
+	// lexically ordering
+	result.sort((a, b) => {
+		if (a.nodes[0].key < b.nodes[0].key) return -1;
+		if (a.nodes[0].key > b.nodes[0].key) return 1;
+		return 0;
+	});
+
+	// give every group a name
+	for (let i = 0; i < result.length; i++) {
+		const group = result[i];
+		const first = group.nodes[0];
+		const last = group.nodes[group.nodes.length - 1];
+		let name = getName(first.key, last.key);
+		group.key = name;
+	}
+
+	// return the results
+	return result.map(group => {
+		/** @type {GroupedItems} */
+		return {
+			key: group.key,
+			items: group.nodes.map(node => node.item),
+			size: group.size
+		};
+	});
+};

package/lib/util/identifier.js

@@ -74,3 +74,30 @@ exports.makePathsRelative = (context, identifier, cache) => {
 		return relativePath;
 	}
 };
+
+/**
+ * @param {string} context absolute context path
+ * @param {string} request any request string may containing absolute paths, query string, etc.
+ * @returns {string} a new request string avoiding absolute paths when possible
+ */
+exports.contextify = (context, request) => {
+	return request
+		.split("!")
+		.map(r => {
+			const splitPath = r.split("?", 2);
+			if (/^[a-zA-Z]:\\/.test(splitPath[0])) {
+				splitPath[0] = path.win32.relative(context, splitPath[0]);
+				if (!/^[a-zA-Z]:\\/.test(splitPath[0])) {
+					splitPath[0] = splitPath[0].replace(/\\/g, "/");
+				}
+			}
+			if (/^\//.test(splitPath[0])) {
+				splitPath[0] = path.posix.relative(context, splitPath[0]);
+			}
+			if (!/^(\.\.\/|\/|[a-zA-Z]:\\)/.test(splitPath[0])) {
+				splitPath[0] = "./" + splitPath[0];
+			}
+			return splitPath.join("?");
+		})
+		.join("!");
+};

package/lib/wasm/WasmMainTemplatePlugin.js

@@ -8,9 +8,10 @@ const Template = require("../Template");
 const WebAssemblyUtils = require("./WebAssemblyUtils");
 
 /** @typedef {import("../Module")} Module */
+/** @typedef {import("../MainTemplate")} MainTemplate */
 
 // Get all wasm modules
-function getAllWasmModules(chunk) {
+const getAllWasmModules = chunk => {
 	const wasmModules = chunk.getAllAsyncChunks();
 	const array = [];
 	for (const chunk of wasmModules) {
@@ -22,7 +23,7 @@ function getAllWasmModules(chunk) {
 	}
 
 	return array;
-}
+};
 
 /**
  * generates the import object function for a module
@@ -30,7 +31,7 @@ function getAllWasmModules(chunk) {
  * @param {boolean} mangle mangle imports
  * @returns {string} source code
  */
-function generateImportObject(module, mangle) {
+const generateImportObject = (module, mangle) => {
 	const waitForInstances = new Map();
 	const properties = [];
 	const usedWasmDependencies = WebAssemblyUtils.getUsedDependencies(
@@ -151,7 +152,7 @@ function generateImportObject(module, mangle) {
 			"},"
 		]);
 	}
-}
+};
 
 class WasmMainTemplatePlugin {
 	constructor({ generateLoadBinaryCode, supportsStreaming, mangleImports }) {
@@ -159,6 +160,11 @@ class WasmMainTemplatePlugin {
 		this.supportsStreaming = supportsStreaming;
 		this.mangleImports = mangleImports;
 	}
+
+	/**
+	 * @param {MainTemplate} mainTemplate main template
+	 * @returns {void}
+	 */
 	apply(mainTemplate) {
 		mainTemplate.hooks.localVars.tap(
 			"WasmMainTemplatePlugin",

package/lib/wasm/WebAssemblyGenerator.js

@@ -32,21 +32,21 @@ const WebAssemblyExportImportedDependency = require("../dependencies/WebAssembly
  * @param {ArrayBuffer} ab - original binary
  * @returns {ArrayBufferTransform} transform
  */
-function preprocess(ab) {
+const preprocess = ab => {
 	const optBin = shrinkPaddedLEB128(new Uint8Array(ab));
 	return optBin.buffer;
-}
+};
 
 /**
  * @template T
  * @param {Function[]} fns transforms
  * @returns {Function} composed transform
  */
-function compose(...fns) {
+const compose = (...fns) => {
 	return fns.reduce((prevFn, nextFn) => {
 		return value => nextFn(prevFn(value));
 	}, value => value);
-}
+};
 
 // TODO replace with @callback
 
@@ -70,7 +70,7 @@ const removeStartFunc = state => bin => {
  * @param {Object} ast - Module's AST
  * @returns {Array<t.ModuleImport>} - nodes
  */
-function getImportedGlobals(ast) {
+const getImportedGlobals = ast => {
 	const importedGlobals = [];
 
 	t.traverse(ast, {
@@ -82,9 +82,9 @@ function getImportedGlobals(ast) {
 	});
 
 	return importedGlobals;
-}
+};
 
-function getCountImportedFunc(ast) {
+const getCountImportedFunc = ast => {
 	let count = 0;
 
 	t.traverse(ast, {
@@ -96,7 +96,7 @@ function getCountImportedFunc(ast) {
 	});
 
 	return count;
-}
+};
 
 /**
  * Get next type index
@@ -104,7 +104,7 @@ function getCountImportedFunc(ast) {
  * @param {Object} ast - Module's AST
  * @returns {t.Index} - index
  */
-function getNextTypeIndex(ast) {
+const getNextTypeIndex = ast => {
 	const typeSectionMetadata = t.getSectionMetadata(ast, "type");
 
 	if (typeof typeSectionMetadata === "undefined") {
@@ -112,7 +112,7 @@ function getNextTypeIndex(ast) {
 	}
 
 	return t.indexLiteral(typeSectionMetadata.vectorOfSize.value);
-}
+};
 
 /**
  * Get next func index
@@ -125,7 +125,7 @@ function getNextTypeIndex(ast) {
  * @param {Number} countImportedFunc - number of imported funcs
  * @returns {t.Index} - index
  */
-function getNextFuncIndex(ast, countImportedFunc) {
+const getNextFuncIndex = (ast, countImportedFunc) => {
 	const funcSectionMetadata = t.getSectionMetadata(ast, "func");
 
 	if (typeof funcSectionMetadata === "undefined") {
@@ -135,7 +135,7 @@ function getNextFuncIndex(ast, countImportedFunc) {
 	const vectorOfSize = funcSectionMetadata.vectorOfSize.value;
 
 	return t.indexLiteral(vectorOfSize + countImportedFunc);
-}
+};
 
 /**
  * Create a init instruction for a global

package/lib/wasm/WebAssemblyInInitialChunkError.js

@@ -0,0 +1,88 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+*/
+"use strict";
+
+const WebpackError = require("../WebpackError");
+
+/** @typedef {import("../Module")} Module */
+/** @typedef {import("../RequestShortener")} RequestShortener */
+
+/**
+ * @param {Module} module module to get chains from
+ * @param {RequestShortener} requestShortener to make readable identifiers
+ * @returns {string[]} all chains to the module
+ */
+const getInitialModuleChains = (module, requestShortener) => {
+	const queue = [
+		{ head: module, message: module.readableIdentifier(requestShortener) }
+	];
+	/** @type {Set<string>} */
+	const results = new Set();
+	/** @type {Set<string>} */
+	const incompleteResults = new Set();
+	/** @type {Set<Module>} */
+	const visitedModules = new Set();
+
+	for (const chain of queue) {
+		const { head, message } = chain;
+		let final = true;
+		/** @type {Set<Module>} */
+		const alreadyReferencedModules = new Set();
+		for (const reason of head.reasons) {
+			const newHead = reason.module;
+			if (newHead) {
+				if (!newHead.getChunks().some(c => c.canBeInitial())) continue;
+				final = false;
+				if (alreadyReferencedModules.has(newHead)) continue;
+				alreadyReferencedModules.add(newHead);
+				const moduleName = newHead.readableIdentifier(requestShortener);
+				const detail = reason.explanation ? ` (${reason.explanation})` : "";
+				const newMessage = `${moduleName}${detail} --> ${message}`;
+				if (visitedModules.has(newHead)) {
+					incompleteResults.add(`... --> ${newMessage}`);
+					continue;
+				}
+				visitedModules.add(newHead);
+				queue.push({
+					head: newHead,
+					message: newMessage
+				});
+			} else {
+				final = false;
+				const newMessage = reason.explanation
+					? `(${reason.explanation}) --> ${message}`
+					: message;
+				results.add(newMessage);
+			}
+		}
+		if (final) {
+			results.add(message);
+		}
+	}
+	for (const result of incompleteResults) {
+		results.add(result);
+	}
+	return Array.from(results);
+};
+
+module.exports = class WebAssemblyInInitialChunkError extends WebpackError {
+	/**
+	 * @param {Module} module WASM module
+	 * @param {RequestShortener} requestShortener request shortener
+	 */
+	constructor(module, requestShortener) {
+		const moduleChains = getInitialModuleChains(module, requestShortener);
+		const message = `WebAssembly module is included in initial chunk.
+This is not allowed, because WebAssembly download and compilation must happen asynchronous.
+Add an async splitpoint (i. e. import()) somewhere between your entrypoint and the WebAssembly module:
+${moduleChains.map(s => `* ${s}`).join("\n")}`;
+
+		super(message);
+		this.name = "WebAssemblyInInitialChunkError";
+		this.hideStack = true;
+		this.module = module;
+
+		Error.captureStackTrace(this, this.constructor);
+	}
+};

package/lib/wasm/WebAssemblyModulesPlugin.js

@@ -10,12 +10,19 @@ const WebAssemblyGenerator = require("./WebAssemblyGenerator");
 const WebAssemblyJavascriptGenerator = require("./WebAssemblyJavascriptGenerator");
 const WebAssemblyImportDependency = require("../dependencies/WebAssemblyImportDependency");
 const WebAssemblyExportImportedDependency = require("../dependencies/WebAssemblyExportImportedDependency");
+const WebAssemblyInInitialChunkError = require("./WebAssemblyInInitialChunkError");
+
+/** @typedef {import("../Compiler")} Compiler */
 
 class WebAssemblyModulesPlugin {
 	constructor(options) {
 		this.options = options;
 	}
 
+	/**
+	 * @param {Compiler} compiler compiler
+	 * @returns {void}
+	 */
 	apply(compiler) {
 		compiler.hooks.compilation.tap(
 			"WebAssemblyModulesPlugin",
@@ -78,6 +85,27 @@ class WebAssemblyModulesPlugin {
 						return result;
 					}
 				);
+
+				compilation.hooks.afterChunks.tap("WebAssemblyModulesPlugin", () => {
+					const initialWasmModules = new Set();
+					for (const chunk of compilation.chunks) {
+						if (chunk.canBeInitial()) {
+							for (const module of chunk.modulesIterable) {
+								if (module.type.startsWith("webassembly")) {
+									initialWasmModules.add(module);
+								}
+							}
+						}
+					}
+					for (const module of initialWasmModules) {
+						compilation.errors.push(
+							new WebAssemblyInInitialChunkError(
+								module,
+								compilation.requestShortener
+							)
+						);
+					}
+				});
 			}
 		);
 	}

package/lib/webpack.web.js

@@ -9,7 +9,7 @@ const WebEnvironmentPlugin = require("./web/WebEnvironmentPlugin");
 const WebpackOptionsApply = require("./WebpackOptionsApply");
 const WebpackOptionsDefaulter = require("./WebpackOptionsDefaulter");
 
-function webpack(options, callback) {
+const webpack = (options, callback) => {
 	new WebpackOptionsDefaulter().process(options);
 
 	const compiler = new Compiler();
@@ -22,7 +22,7 @@ function webpack(options, callback) {
 		compiler.run(callback);
 	}
 	return compiler;
-}
+};
 module.exports = webpack;
 
 webpack.WebpackOptionsDefaulter = WebpackOptionsDefaulter;

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "webpack",
-  "version": "4.14.0",
+  "version": "4.15.1",
   "author": "Tobias Koppers @sokra",
   "description": "Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.",
   "license": "MIT",
   "dependencies": {
-    "@webassemblyjs/ast": "1.5.12",
-    "@webassemblyjs/helper-module-context": "1.5.12",
-    "@webassemblyjs/wasm-edit": "1.5.12",
-    "@webassemblyjs/wasm-opt": "1.5.12",
-    "@webassemblyjs/wasm-parser": "1.5.12",
+    "@webassemblyjs/ast": "1.5.13",
+    "@webassemblyjs/helper-module-context": "1.5.13",
+    "@webassemblyjs/wasm-edit": "1.5.13",
+    "@webassemblyjs/wasm-opt": "1.5.13",
+    "@webassemblyjs/wasm-parser": "1.5.13",
     "acorn": "^5.6.2",
     "acorn-dynamic-import": "^3.0.0",
     "ajv": "^6.1.0",
@@ -34,6 +34,7 @@
   "devDependencies": {
     "@types/node": "^9.6.4",
     "@types/tapable": "^1.0.1",
+    "@types/webpack-sources": "^0.1.4",
     "benchmark": "^2.1.1",
     "bundle-loader": "~0.5.0",
     "codacy-coverage": "^2.0.1",
@@ -122,7 +123,7 @@
     "code-lint": "eslint --cache setup lib bin hot buildin benchmark \"test/*.js\" \"test/**/webpack.config.js\" \"examples/**/webpack.config.js\" \"schemas/**/*.js\"",
     "type-lint": "tsc --pretty",
     "fix": "yarn code-lint --fix",
-    "pretty": "prettier --write \"setup/**/*.js\" \"lib/**/*.js\" \"bin/*.js\" \"hot/*.js\" \"buildin/*.js\" \"benchmark/**/*.js\" \"test/*.js\" \"test/**/webpack.config.js\" \"examples/**/webpack.config.js\" \"schemas/**/*.js\" \"declarations.d.ts\"",
+    "pretty": "prettier --write \"setup/**/*.js\" \"lib/**/*.js\" \"bin/*.js\" \"hot/*.js\" \"buildin/*.js\" \"benchmark/**/*.js\" \"test/*.js\" \"test/**/webpack.config.js\" \"examples/**/webpack.config.js\" \"schemas/**/*.js\" \"declarations.d.ts\" \"tsconfig.json\"",
     "schema-lint": "node --max-old-space-size=4096 node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.lint.js\" --no-verbose",
     "benchmark": "node --max-old-space-size=4096 --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.benchmark.js\" --runInBand",
     "cover": "yarn cover:init && yarn cover:all && yarn cover:report",

package/schemas/WebpackOptions.json

@@ -1351,7 +1351,12 @@
                   ]
                 },
                 "minSize": {
-                  "description": "Minimal size for the created chunk",
+                  "description": "Minimal size for the created chunks",
+                  "type": "number",
+                  "minimum": 0
+                },
+                "maxSize": {
+                  "description": "Maximal size hint for the created chunks",
                   "type": "number",
                   "minimum": 0
                 },
@@ -1394,6 +1399,32 @@
                   "type": "string",
                   "minLength": 1
                 },
+                "hidePathInfo": {
+                  "description": "Prevents exposing path info when creating names for parts splitted by maxSize",
+                  "type": "boolean"
+                },
+                "fallbackCacheGroup": {
+                  "description": "Options for modules not selected by any other cache group",
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "minSize": {
+                      "description": "Minimal size for the created chunk",
+                      "type": "number",
+                      "minimum": 0
+                    },
+                    "maxSize": {
+                      "description": "Maximal size hint for the created chunks",
+                      "type": "number",
+                      "minimum": 0
+                    },
+                    "automaticNameDelimiter": {
+                      "description": "Sets the name delimiter for created chunks",
+                      "type": "string",
+                      "minLength": 1
+                    }
+                  }
+                },
                 "cacheGroups": {
                   "description": "Assign modules to a cache group (modules from different cache groups are tried to keep in separate chunks)",
                   "type": "object",
@@ -1460,6 +1491,11 @@
                             "type": "number",
                             "minimum": 0
                           },
+                          "maxSize": {
+                            "description": "Maximal size hint for the created chunks",
+                            "type": "number",
+                            "minimum": 0
+                          },
                           "minChunks": {
                             "description": "Minimum number of times a module has to be duplicated until it's considered for splitting",
                             "type": "number",