tldts-utils 6.1.81

package/dist/cjs/index.js

@@ -0,0 +1,698 @@
+'use strict';
+
+var fs = require('fs');
+var path = require('path');
+
+/** Highest positive signed 32-bit float value */
+const maxInt = 2147483647; // aka. 0x7FFFFFFF or 2^31-1
+
+/** Bootstring parameters */
+const base = 36;
+const tMin = 1;
+const tMax = 26;
+const skew = 38;
+const damp = 700;
+const initialBias = 72;
+const initialN = 128; // 0x80
+const delimiter = '-'; // '\x2D'
+const regexNonASCII = /[^\0-\x7F]/; // Note: U+007F DEL is excluded too.
+const regexSeparators = /[\x2E\u3002\uFF0E\uFF61]/g; // RFC 3490 separators
+
+/** Error messages */
+const errors = {
+	'overflow': 'Overflow: input needs wider integers to process',
+	'not-basic': 'Illegal input >= 0x80 (not a basic code point)',
+	'invalid-input': 'Invalid input'
+};
+
+/** Convenience shortcuts */
+const baseMinusTMin = base - tMin;
+const floor = Math.floor;
+const stringFromCharCode = String.fromCharCode;
+
+/*--------------------------------------------------------------------------*/
+
+/**
+ * A generic error utility function.
+ * @private
+ * @param {String} type The error type.
+ * @returns {Error} Throws a `RangeError` with the applicable error message.
+ */
+function error(type) {
+	throw new RangeError(errors[type]);
+}
+
+/**
+ * A generic `Array#map` utility function.
+ * @private
+ * @param {Array} array The array to iterate over.
+ * @param {Function} callback The function that gets called for every array
+ * item.
+ * @returns {Array} A new array of values returned by the callback function.
+ */
+function map(array, callback) {
+	const result = [];
+	let length = array.length;
+	while (length--) {
+		result[length] = callback(array[length]);
+	}
+	return result;
+}
+
+/**
+ * A simple `Array#map`-like wrapper to work with domain name strings or email
+ * addresses.
+ * @private
+ * @param {String} domain The domain name or email address.
+ * @param {Function} callback The function that gets called for every
+ * character.
+ * @returns {String} A new string of characters returned by the callback
+ * function.
+ */
+function mapDomain(domain, callback) {
+	const parts = domain.split('@');
+	let result = '';
+	if (parts.length > 1) {
+		// In email addresses, only the domain name should be punycoded. Leave
+		// the local part (i.e. everything up to `@`) intact.
+		result = parts[0] + '@';
+		domain = parts[1];
+	}
+	// Avoid `split(regex)` for IE8 compatibility. See #17.
+	domain = domain.replace(regexSeparators, '\x2E');
+	const labels = domain.split('.');
+	const encoded = map(labels, callback).join('.');
+	return result + encoded;
+}
+
+/**
+ * Creates an array containing the numeric code points of each Unicode
+ * character in the string. While JavaScript uses UCS-2 internally,
+ * this function will convert a pair of surrogate halves (each of which
+ * UCS-2 exposes as separate characters) into a single code point,
+ * matching UTF-16.
+ * @see `punycode.ucs2.encode`
+ * @see <https://mathiasbynens.be/notes/javascript-encoding>
+ * @memberOf punycode.ucs2
+ * @name decode
+ * @param {String} string The Unicode input string (UCS-2).
+ * @returns {Array} The new array of code points.
+ */
+function ucs2decode(string) {
+	const output = [];
+	let counter = 0;
+	const length = string.length;
+	while (counter < length) {
+		const value = string.charCodeAt(counter++);
+		if (value >= 0xD800 && value <= 0xDBFF && counter < length) {
+			// It's a high surrogate, and there is a next character.
+			const extra = string.charCodeAt(counter++);
+			if ((extra & 0xFC00) == 0xDC00) { // Low surrogate.
+				output.push(((value & 0x3FF) << 10) + (extra & 0x3FF) + 0x10000);
+			} else {
+				// It's an unmatched surrogate; only append this code unit, in case the
+				// next code unit is the high surrogate of a surrogate pair.
+				output.push(value);
+				counter--;
+			}
+		} else {
+			output.push(value);
+		}
+	}
+	return output;
+}
+
+/**
+ * Converts a digit/integer into a basic code point.
+ * @see `basicToDigit()`
+ * @private
+ * @param {Number} digit The numeric value of a basic code point.
+ * @returns {Number} The basic code point whose value (when used for
+ * representing integers) is `digit`, which needs to be in the range
+ * `0` to `base - 1`. If `flag` is non-zero, the uppercase form is
+ * used; else, the lowercase form is used. The behavior is undefined
+ * if `flag` is non-zero and `digit` has no uppercase form.
+ */
+const digitToBasic = function(digit, flag) {
+	//  0..25 map to ASCII a..z or A..Z
+	// 26..35 map to ASCII 0..9
+	return digit + 22 + 75 * (digit < 26) - ((flag != 0) << 5);
+};
+
+/**
+ * Bias adaptation function as per section 3.4 of RFC 3492.
+ * https://tools.ietf.org/html/rfc3492#section-3.4
+ * @private
+ */
+const adapt = function(delta, numPoints, firstTime) {
+	let k = 0;
+	delta = firstTime ? floor(delta / damp) : delta >> 1;
+	delta += floor(delta / numPoints);
+	for (/* no initialization */; delta > baseMinusTMin * tMax >> 1; k += base) {
+		delta = floor(delta / baseMinusTMin);
+	}
+	return floor(k + (baseMinusTMin + 1) * delta / (delta + skew));
+};
+
+/**
+ * Converts a string of Unicode symbols (e.g. a domain name label) to a
+ * Punycode string of ASCII-only symbols.
+ * @memberOf punycode
+ * @param {String} input The string of Unicode symbols.
+ * @returns {String} The resulting Punycode string of ASCII-only symbols.
+ */
+const encode = function(input) {
+	const output = [];
+
+	// Convert the input in UCS-2 to an array of Unicode code points.
+	input = ucs2decode(input);
+
+	// Cache the length.
+	const inputLength = input.length;
+
+	// Initialize the state.
+	let n = initialN;
+	let delta = 0;
+	let bias = initialBias;
+
+	// Handle the basic code points.
+	for (const currentValue of input) {
+		if (currentValue < 0x80) {
+			output.push(stringFromCharCode(currentValue));
+		}
+	}
+
+	const basicLength = output.length;
+	let handledCPCount = basicLength;
+
+	// `handledCPCount` is the number of code points that have been handled;
+	// `basicLength` is the number of basic code points.
+
+	// Finish the basic string with a delimiter unless it's empty.
+	if (basicLength) {
+		output.push(delimiter);
+	}
+
+	// Main encoding loop:
+	while (handledCPCount < inputLength) {
+
+		// All non-basic code points < n have been handled already. Find the next
+		// larger one:
+		let m = maxInt;
+		for (const currentValue of input) {
+			if (currentValue >= n && currentValue < m) {
+				m = currentValue;
+			}
+		}
+
+		// Increase `delta` enough to advance the decoder's <n,i> state to <m,0>,
+		// but guard against overflow.
+		const handledCPCountPlusOne = handledCPCount + 1;
+		if (m - n > floor((maxInt - delta) / handledCPCountPlusOne)) {
+			error('overflow');
+		}
+
+		delta += (m - n) * handledCPCountPlusOne;
+		n = m;
+
+		for (const currentValue of input) {
+			if (currentValue < n && ++delta > maxInt) {
+				error('overflow');
+			}
+			if (currentValue === n) {
+				// Represent delta as a generalized variable-length integer.
+				let q = delta;
+				for (let k = base; /* no condition */; k += base) {
+					const t = k <= bias ? tMin : (k >= bias + tMax ? tMax : k - bias);
+					if (q < t) {
+						break;
+					}
+					const qMinusT = q - t;
+					const baseMinusT = base - t;
+					output.push(
+						stringFromCharCode(digitToBasic(t + qMinusT % baseMinusT, 0))
+					);
+					q = floor(qMinusT / baseMinusT);
+				}
+
+				output.push(stringFromCharCode(digitToBasic(q, 0)));
+				bias = adapt(delta, handledCPCountPlusOne, handledCPCount === basicLength);
+				delta = 0;
+				++handledCPCount;
+			}
+		}
+
+		++delta;
+		++n;
+
+	}
+	return output.join('');
+};
+
+/**
+ * Converts a Unicode string representing a domain name or an email address to
+ * Punycode. Only the non-ASCII parts of the domain name will be converted,
+ * i.e. it doesn't matter if you call it with a domain that's already in
+ * ASCII.
+ * @memberOf punycode
+ * @param {String} input The domain name or email address to convert, as a
+ * Unicode string.
+ * @returns {String} The Punycode representation of the given domain name or
+ * email address.
+ */
+const toASCII = function(input) {
+	return mapDomain(input, function(string) {
+		return regexNonASCII.test(string)
+			? 'xn--' + encode(string)
+			: string;
+	});
+};
+
+/**
+ * Parse public suffix list and invoke callback on each rule.
+ */
+var parse = (body, cb) => {
+    const beginPrivateDomains = '// ===begin private domains===';
+    let isIcann = true;
+    // Iterate on lines and extract public suffix rules
+    const lines = body.split('\n');
+    for (let line of lines) {
+        line = line.trim().toLowerCase();
+        // Ignore comments
+        if (line.length === 0 || line.startsWith('//')) {
+            if (line.startsWith(beginPrivateDomains)) {
+                isIcann = false;
+            }
+            continue;
+        }
+        let isException = false;
+        let isWildcard = false;
+        let isNormal = false;
+        // Select correct section to insert the rule
+        if (line.startsWith('!')) {
+            isException = true;
+            line = line.slice(1);
+        }
+        else if (line.startsWith('*.')) {
+            isWildcard = true;
+        }
+        else {
+            isNormal = true;
+        }
+        // Specification says that we should only consider the content of a line up
+        // to the first whitespace encountered.
+        const spaceIndex = line.indexOf(' ');
+        if (spaceIndex !== -1) {
+            line = line.substr(0, spaceIndex);
+        }
+        // If suffix is not ascii, we index the suffix twice so that we support IDNA
+        // labels as well. This allows to not have to perform the conversion at
+        // runtime.
+        const encoded = toASCII(line);
+        if (line !== encoded) {
+            cb({
+                isException,
+                isIcann,
+                isNormal,
+                isWildcard,
+                rule: encoded,
+            });
+        }
+        cb({
+            isException,
+            isIcann,
+            isNormal,
+            isWildcard,
+            rule: line,
+        });
+    }
+};
+
+function findBaseDir (target) {
+    let baseDir = __dirname;
+    while (!fs.existsSync(path.resolve(baseDir, target))) {
+        baseDir = path.resolve(baseDir, '../');
+    }
+    return path.resolve(baseDir, target);
+}
+
+function loadPublicSuffixList () {
+    return fs.readFileSync(findBaseDir('./publicsuffix/public_suffix_list.dat'), {
+        encoding: 'utf-8',
+    });
+}
+
+/**
+ * This builder implements a probabilistic data-structure to store the public
+ * suffixes from Mozilla's 'publicsuffix' project. The goals are:
+ *
+ *   1. Allow storing the parsed rules in a very compact form (<30KB)
+ *   2. Allow super fast load time with no parsing required (typed array)
+ *   2. Super fast lookup times (~2M+ per second)
+ *
+ * These are achieved by hashing each available suffix as a 32bits number and
+ * storing them in a typed array which is bundled in the library itself. This
+ * way the only cost of loading the rules is paid by the JavaScript engine
+ * loading the source and the size of the rules is ~29KB. Matching is also
+ * extremely fast, at around 2M+ lookups per second.
+ *
+ * **WARNING**: The drawback of this structure is that, because we use hashes of
+ * suffixes instead of their initial values, there can be collisions (think
+ * bloom filters). Some future work involves estimating the probability of
+ * collisions. Only use this structure if you are fine with this limitation and
+ * need the most compact/fast data structure available.
+ *
+ * Data Structure
+ * ==============
+ *
+ * The basic idea behind the packed hashes structure is that suffixes will be
+ * grouped by number of labels then hashed to 32bits integers using a fast
+ * hashing function. For examples:
+ *
+ *  1 label:
+ *    com   -> hash(com)
+ *    fr    -> hash(fr)
+ *    gov   -> hash(gov)
+ *  2 labels:
+ *    co.uk -> hash(co.uk)
+ *
+ * Then the hashes are organized in a typed array from least number of labels to
+ * most number of labels, and in each section, hashes are sorted so that we can
+ * perform lookups using a binary search:
+ *
+ * |#1 length|suffix 1|suffix 2|...|#2 length|suffix 1|suffix 2|...|...
+ *  ^                               ^ number of suffixes with 2 labels
+ *  | number of suffix with 1 label
+ *
+ *  In the example above:
+ *
+ *    3|hash(com)|hash(fr)|hash(gov)|1|hash(co.uk)
+ *    ^                              ^ we have on suffix with 2 labels (co.uk)
+ *    | we have 3 suffixes with one label
+ *
+ * In practice, the data-structure is slightly more complex as we have to handle
+ * the ICANN and PRIVATE sections of the list separately (which allows to pick
+ * one or the other at runtime; or both, which is the default) as well as
+ * wildcards and exceptions rules. The structure looks more like:
+ *
+ *  |icann exceptions|private exceptions|icann wildcards|private wildcards|icann rules|private rules
+ *
+ *  Still ordered from least number of labels to most number of labels. This
+ *  allows the matching to be a forward progression from beginning to end,
+ *  potentially skipping some sections (e.g.: private) by just incrementing the
+ *  index.
+ *
+ * Finding a Match
+ * ===============
+ *
+ * Finding a match can be done efficiently, without any string copy and by a
+ * single pass over the input (from end to beginning). The algorithm is as
+ * follows:
+ *
+ * - For each character `c` in hostname, from end to start:
+ *     1. If `c` is '.' then we reached the end of a label:
+ *         * Use a binary search in the section containing suffixes of the same
+ *           size as our current label to check if our hash is there.
+ *         * Move forward to the next section by incrementing the index by the
+ *           number of suffixes in this section.
+ *     2. Update the hash with the current character
+ *
+ * This does not account for wildcards and exceptions, but explains the general
+ * idea of the matching algorithm. Have a look at the source bellow for more
+ * details.
+ *
+ * Some nice properties of this algorithm:
+ * 1. Pretty straight-forward implementation
+ * 2. Matching is always going forward in the array
+ * 3. Sections can be skipped by just incrementing the index
+ * 4. Hash can be computed on-the-fly from end to start without any string copy
+ */
+/**
+ * Compute 32 bits hash of `str` backward.
+ */
+function fastHash(str) {
+    let hash = 5381;
+    for (let j = str.length - 1; j >= 0; j -= 1) {
+        hash = (hash * 33) ^ str.charCodeAt(j);
+    }
+    return hash >>> 0;
+}
+/**
+ * Build packed typed array given the raw public list as a string.
+ */
+var buildHashes = (body) => {
+    const rules = {
+        icann: [],
+        priv: [],
+    };
+    const wildcards = {
+        icann: [],
+        priv: [],
+    };
+    const exceptions = {
+        icann: [],
+        priv: [],
+    };
+    // Keep track of maximum number of labels seen in a given rule. This allows us
+    // to make sure that all sections of the typed array (wildcards, exceptions
+    // and normal rules) have the same size and we do not need to check that while
+    // matching.
+    let maximumNumberOfLabels = 0;
+    // Iterate on public suffix rules
+    parse(body, ({ rule, isIcann, isException, isWildcard, isNormal }) => {
+        // Select correct section to insert the rule
+        let hashesPerLabels = null;
+        if (isException) {
+            hashesPerLabels = isIcann ? exceptions.icann : exceptions.priv;
+        }
+        else if (isWildcard) {
+            hashesPerLabels = isIcann ? wildcards.icann : wildcards.priv;
+            // eslint-disable-next-line no-param-reassign
+            rule = rule.slice(2);
+        }
+        else if (isNormal) {
+            hashesPerLabels = isIcann ? rules.icann : rules.priv;
+        }
+        if (hashesPerLabels === null) {
+            return;
+        }
+        // Count number of labels in this suffix
+        const numberOfLabels = rule.split('.').length;
+        // Experimentally, we start ignoring suffixes with only one label since the
+        // fallback to '*' will yield the same result. This helps reduce the size of
+        // the bundle further and speed-up matching. One potential side-effect is
+        // that we might not accurately return the information about if a suffix is
+        // from the ICANN or PRIVATE section.
+        if (numberOfLabels === 1 &&
+            (hashesPerLabels === rules.icann || hashesPerLabels === rules.priv)) {
+            return;
+        }
+        maximumNumberOfLabels = Math.max(maximumNumberOfLabels, numberOfLabels);
+        let hashesForLabel = hashesPerLabels[numberOfLabels];
+        if (hashesForLabel === undefined) {
+            hashesForLabel = [];
+            hashesPerLabels[numberOfLabels] = hashesForLabel;
+        }
+        hashesForLabel.push(fastHash(rule));
+    });
+    // Pack everything together
+    const chunks = [];
+    const pushHashes = (hashes = []) => {
+        chunks.push([
+            hashes.length,
+            ...hashes.sort((a, b) => {
+                if (a < b) {
+                    return -1;
+                }
+                if (a > b) {
+                    return 1;
+                }
+                return 0;
+            }),
+        ]);
+    };
+    for (let label = 1; label <= maximumNumberOfLabels; label += 1) {
+        pushHashes(exceptions.icann[label]);
+        pushHashes(exceptions.priv[label]);
+        pushHashes(wildcards.icann[label]);
+        pushHashes(wildcards.priv[label]);
+        pushHashes(rules.icann[label]);
+        pushHashes(rules.priv[label]);
+    }
+    return new Uint32Array([
+        maximumNumberOfLabels,
+        ...[].concat(...chunks),
+    ]);
+};
+
+/**
+ * This builder implements a fast and compact DAWG data-structure (a.k.a.:
+ * direct acyclic word graph, with suffix and prefix compression of the
+ * top-level domains) using JavaScript Objects. It is a good trade-off between
+ * correctness, speed and space. This is much faster then using a linear scan
+ * over all the rules, but slightly slower than the probabilistic
+ * data-structure implemented in `bin/builders/hashes.js`.
+ *
+ * Here is a simple example:
+ *
+ *   co.uk
+ *   gov.uk
+ *   fr
+ *
+ *  <root>
+ *    |_ uk
+ *       |_ co *
+ *       |_ gov *
+ *    |_ fr *
+ *
+ *  Finding a match is then a matter of iterating on labels of a hostname
+ *  backward and following the branches of the Trie. Whenever we find a terminal
+ *  node (indicated with a '*' above), we consider this a match.
+ */
+function convertToCompactTrie(trie) {
+    return [
+        trie.$,
+        Object.fromEntries(Object.entries(trie.succ).map(([label, succ]) => [
+            label,
+            typeof succ === 'string' ? succ : convertToCompactTrie(succ),
+        ])),
+    ];
+}
+/**
+ * Insert a public suffix rule in the `trie`.
+ */
+function insertInTrie({ parts, isIcann, }, trie) {
+    let node = trie;
+    for (const part of parts) {
+        let nextNode = node.succ[part];
+        if (nextNode === undefined) {
+            nextNode = { $: 0, succ: {} };
+            node.succ[part] = nextNode;
+        }
+        node = nextNode;
+    }
+    node.$ = isIcann ? 1 : 2;
+    return trie;
+}
+let nextId = 0;
+/**
+ * Compress the given `trie` into a DAWG (by compressing common suffixes as
+ * well as prefixes). This form is very efficient but cannot be dumped as a
+ * simple JSON into the file. The solution is to create intermediary variables
+ * for each shared sub-tree, which are then used in the main DAWG.
+ */
+function compressToDAWG(trie, name) {
+    const nodesByHash = new Map();
+    const replaceNodes = new Map();
+    (function groupNodesByHash(node) {
+        // Get a sorted list of next labels from this node
+        const nexts = Object.entries(node.succ)
+            .filter(([n]) => n !== '$')
+            .sort();
+        // Create a setter which can be used later to replace the value of a node.
+        // Each node is associated with a lambda function accepting one argument:
+        // the new value of the node. Because the context of the node (parent and
+        // label) is captured in the closure, it allows to just call it later, in a
+        // different context, to change the value of a given node in-place.
+        nexts.forEach(([n, succ]) => {
+            replaceNodes.set(succ, (newNode) => {
+                node.succ[n] = newNode;
+            });
+        });
+        // Compute a unique hash for this node recursively.
+        const hash = `(${node.$},${[
+            ...nexts.map(([c, succ]) => `${c},${groupNodesByHash(succ)}`),
+        ].join('|')})`;
+        // Keep track of all node's hashes
+        let nodes = nodesByHash.get(hash);
+        if (nodes === undefined) {
+            nodes = [];
+            nodesByHash.set(hash, nodes);
+        }
+        nodes.push(node);
+        return hash;
+    })(trie);
+    // Given `nodesByHash`, which associates a list of nodes to each hash
+    // encountered in the previous step, we will detect all the sub-trees being
+    // seen more than once and store them in a variable to allow sharing with
+    // different parts in the DAWG.
+    const variables = [];
+    nodesByHash.forEach((nodes) => {
+        if (nodes.length > 1) {
+            // Dump one of the nodes (they are all the same so it does not matter which one)
+            variables.push(`_${nextId}: ITrie = ${JSON.stringify(convertToCompactTrie(nodes[0]))}`);
+            // Replace all the other ones by the name of this new variable
+            nodes.forEach((node) => {
+                var _a;
+                // @ts-expect-error
+                (_a = replaceNodes.get(node)) === null || _a === void 0 ? void 0 : _a(`_${nextId}`);
+            });
+            nextId += 1;
+        }
+    });
+    const output = [];
+    // Create TypeScript source code to declare all of these variables. Because
+    // the value of nodes has been set to a value of the form '"_id"', we need to
+    // replace this on the fly to '_id'. We use a RegExp for this.
+    if (variables.length !== 0) {
+        let variablesSourceCode = `const ${variables.join(',')};`;
+        for (let i = 0; i < nextId; i += 1) {
+            variablesSourceCode = variablesSourceCode.replace(new RegExp(`"_${i}"`, 'g'), `_${i}`);
+        }
+        output.push(variablesSourceCode);
+    }
+    // Dump the root of the DAWG as well, and perform the same replacement of _id
+    // parts than for the variables above.
+    let serializedTrie = JSON.stringify(convertToCompactTrie(trie));
+    for (let i = 0; i < nextId; i += 1) {
+        serializedTrie = serializedTrie.replace(new RegExp(`"_${i}"`, 'g'), `_${i}`);
+    }
+    output.push(`const ${name}: ITrie = ${serializedTrie};`);
+    return output.join('\n');
+}
+function convertToTypeScriptSource(rules, exceptions, { includePrivate }) {
+    return `
+export type ITrie = [${includePrivate ? '0 | 1 | 2' : '0 | 1'}, { [label: string]: ITrie}];
+
+export const exceptions: ITrie = (function() {
+  ${compressToDAWG(exceptions, 'exceptions')}
+  return exceptions;
+})();
+
+export const rules: ITrie = (function() {
+  ${compressToDAWG(rules, 'rules')}
+  return rules;
+})();
+`;
+}
+var buildTrie = (body, { includePrivate }) => {
+    const exceptions = { $: 0, succ: {} };
+    const rules = { $: 0, succ: {} };
+    // Iterate on public suffix rules
+    parse(body, ({ rule, isIcann, isException }) => {
+        if (isIcann || includePrivate) {
+            insertInTrie({ isIcann, parts: rule.split('.').reverse() }, isException ? exceptions : rules);
+        }
+    });
+    return convertToTypeScriptSource(rules, exceptions, { includePrivate });
+};
+
+function update () {
+    console.log('Updating rules...');
+    const publicSuffixList = loadPublicSuffixList();
+    // Build trie and update TypeScript file
+    fs.writeFileSync(findBaseDir('./tldts/src/data/trie.ts'), buildTrie(publicSuffixList, { includePrivate: true }), 'utf-8');
+    // Build trie and update TypeScript file (ICANN only)
+    fs.writeFileSync(findBaseDir('./tldts-icann/src/data/trie.ts'), buildTrie(publicSuffixList, { includePrivate: false }), 'utf-8');
+    // Build hashes and update TypeScript file
+    const packed = buildHashes(publicSuffixList);
+    fs.writeFileSync(findBaseDir('./tldts-experimental/src/data/hashes.ts'), `
+// Code automatically generated using ./bin/builders/hashes.ts
+export default new Uint32Array([${Array.from(packed).toString()}]);
+`, 'utf-8');
+}
+
+exports.loadPublicSuffixList = loadPublicSuffixList;
+exports.parsePublicSuffixRules = parse;
+exports.updateInternalDataStructures = update;
+//# sourceMappingURL=index.js.map