whatwg-url 17.0.0 → 17.1.0

package/README.md

@@ -4,7 +4,7 @@ whatwg-url is a full implementation of the WHATWG [URL Standard](https://url.spe
 
 ## Specification conformance
 
-whatwg-url is currently up to date with the URL spec up to commit [3c83874](https://github.com/whatwg/url/commit/3c83874ae1eac84ceb71d8c4344673251c982bbe) and the web platform tests up to commit [2a91c2e](https://github.com/web-platform-tests/wpt/commit/2a91c2e71952bdceef8b2825ce3a0a0dc73aa588).
+whatwg-url is currently up to date with the URL spec up to commit [9dc3827](https://github.com/whatwg/url/commit/9dc3827fc722ac4af3f11061aa3e9adb44a17c8b).
 
 For `file:` URLs, whose [origin is left unspecified](https://url.spec.whatwg.org/#concept-url-origin), whatwg-url chooses to use a new opaque origin (which serializes to `"null"`).
 
@@ -19,6 +19,8 @@ The main API is provided by the [`URL`](https://url.spec.whatwg.org/#url-class)
 The following methods are exported for use by places like jsdom that need to implement things like [`HTMLHyperlinkElementUtils`](https://html.spec.whatwg.org/#htmlhyperlinkelementutils). They mostly operate on or return an "internal URL" or ["URL record"](https://url.spec.whatwg.org/#concept-url) type.
 
 - [URL parser](https://url.spec.whatwg.org/#concept-url-parser): `parseURL(input, { baseURL, encoding = "UTF-8" })`
+- URL parser with [validation errors](https://url.spec.whatwg.org/#validation-error): `parseURLWithValidationErrors(input, { baseURL, encoding = "UTF-8" })`
+- [Valid URL string](https://url.spec.whatwg.org/#valid-url-string) checker: `isValidURLString(input, { baseURL })`
 - [Basic URL parser](https://url.spec.whatwg.org/#concept-basic-url-parser): `basicURLParse(input, { baseURL, url, stateOverride, encoding = "UTF-8" })`
 - [URL serializer](https://url.spec.whatwg.org/#concept-url-serializer): `serializeURL(urlRecord, excludeFragment)`
 - [Host serializer](https://url.spec.whatwg.org/#concept-host-serializer): `serializeHost(hostFromURLRecord)`
@@ -71,6 +73,17 @@ These properties should be treated with care, as in general changing them will c
 
 The return value of "failure" in the spec is represented by `null`. That is, functions like `parseURL` and `basicURLParse` can return _either_ a URL record _or_ `null`.
 
+The `parseURLWithValidationErrors` function returns an object with a `url` property and a `validationErrors` property. The `url` property is a URL record or `null`, and the `validationErrors` property is an array of validation error names reported while parsing.
+
+```js
+const result = parseURLWithValidationErrors("https://example.org/%s");
+
+console.log(result.url === null); // false
+console.log(result.validationErrors); // [ "invalid-URL-unit" ]
+```
+
+The `isValidURLString` function implements the grammar-based validity checker from the URL Standard's URL writing section. This is separate from parser validation errors, so it can disagree with `parseURLWithValidationErrors()` in some cases. Pass a `baseURL` URL record to check relative-URL strings.
+
 ### `whatwg-url/webidl2js-wrapper` module
 
 This module exports the `URL` and `URLSearchParams` [interface wrappers API](https://github.com/jsdom/webidl2js#for-interfaces) generated by [webidl2js](https://github.com/jsdom/webidl2js).

package/index.js

@@ -3,6 +3,7 @@
 const { URL, URLSearchParams } = require("./webidl2js-wrapper");
 const urlStateMachine = require("./lib/url-state-machine");
 const percentEncoding = require("./lib/percent-encoding");
+const urlStringValidator = require("./lib/url-string-validator");
 
 const sharedGlobalObject = { Array, Object, Promise, String, TypeError };
 URL.install(sharedGlobalObject, ["Window"]);
@@ -12,6 +13,8 @@ exports.URL = sharedGlobalObject.URL;
 exports.URLSearchParams = sharedGlobalObject.URLSearchParams;
 
 exports.parseURL = urlStateMachine.parseURL;
+exports.parseURLWithValidationErrors = urlStateMachine.parseURLWithValidationErrors;
+exports.isValidURLString = urlStringValidator.isValidURLString;
 exports.basicURLParse = urlStateMachine.basicURLParse;
 exports.serializeURL = urlStateMachine.serializeURL;
 exports.serializePath = urlStateMachine.serializePath;

package/lib/infra.js

@@ -18,9 +18,19 @@ function isASCIIHex(c) {
   return isASCIIDigit(c) || (c >= 0x41 && c <= 0x46) || (c >= 0x61 && c <= 0x66);
 }
 
+function isASCIIString(string) {
+  return !/[^\u0000-\u007F]/u.test(string);
+}
+
+function isNoncharacter(c) {
+  return (c >= 0xFDD0 && c <= 0xFDEF) || (c & 0xFFFE) === 0xFFFE;
+}
+
 module.exports = {
   isASCIIDigit,
   isASCIIAlpha,
   isASCIIAlphanumeric,
-  isASCIIHex
+  isASCIIHex,
+  isASCIIString,
+  isNoncharacter
 };

package/lib/url-miscellaneous.js

@@ -0,0 +1,221 @@
+"use strict";
+
+const tr46 = require("tr46");
+
+const infra = require("./infra");
+
+function p(char) {
+  return char.codePointAt(0);
+}
+
+const failure = Symbol("failure");
+
+const specialSchemes = {
+  ftp: 21,
+  file: null,
+  http: 80,
+  https: 443,
+  ws: 80,
+  wss: 443
+};
+
+const urlCodePoints = new Set([
+  p("!"), p("$"), p("&"), p("'"), p("("), p(")"), p("*"), p("+"), p(","), p("-"), p("."), p("/"),
+  p(":"), p(";"), p("="), p("?"), p("@"), p("_"), p("~")
+]);
+
+const forbiddenHostCodePoints = new Set([
+  0x00, 0x09, 0x0A, 0x0D, 0x20, p("#"), p("/"), p(":"), p("<"), p(">"), p("?"), p("@"), p("["),
+  p("\\"), p("]"), p("^"), p("|")
+]);
+
+function isURLCodePoint(c) {
+  return infra.isASCIIAlphanumeric(c) ||
+    urlCodePoints.has(c) ||
+    (c >= 0xA0 && c <= 0x10FFFD && (c < 0xD800 || c > 0xDFFF) && !infra.isNoncharacter(c));
+}
+
+function isInvalidURLCodePoint(c) {
+  return !isURLCodePoint(c) && c !== p("%");
+}
+
+function isPercentEncodedByteAt(input, index) {
+  return index + 2 < input.length &&
+    input[index] === "%" &&
+    infra.isASCIIHex(input.charCodeAt(index + 1)) &&
+    infra.isASCIIHex(input.charCodeAt(index + 2));
+}
+
+function containsPercentEncodedByte(input) {
+  return /%[0-9A-Fa-f]{2}/u.test(input);
+}
+
+function containsForbiddenHostCodePoint(string) {
+  return [...string].some(c => forbiddenHostCodePoints.has(c.codePointAt(0)));
+}
+
+function containsForbiddenDomainCodePoint(string) {
+  return [...string].some(c => {
+    const cp = c.codePointAt(0);
+    return forbiddenHostCodePoints.has(cp) || (cp >= 0x00 && cp <= 0x1F) || cp === p("%") || cp === 0x7F;
+  });
+}
+
+function domainParserToASCII(domain, beStrict) {
+  return tr46.toASCII(domain, {
+    checkHyphens: beStrict,
+    checkBidi: true,
+    checkJoiners: true,
+    useSTD3ASCIIRules: beStrict,
+    transitionalProcessing: false,
+    verifyDNSLength: beStrict,
+    ignoreInvalidPunycode: false
+  });
+}
+
+function parseIPv4Number(input, validationErrors = null) {
+  if (input === "") {
+    return failure;
+  }
+
+  let validationErrorSeen = false;
+  let R = 10;
+
+  if (input.length >= 2 && input.charAt(0) === "0" && input.charAt(1).toLowerCase() === "x") {
+    validationErrorSeen = true;
+    input = input.substring(2);
+    R = 16;
+  } else if (input.length >= 2 && input.charAt(0) === "0") {
+    validationErrorSeen = true;
+    input = input.substring(1);
+    R = 8;
+  }
+
+  if (input === "") {
+    validationErrors?.push("IPv4-non-decimal-part");
+    return 0;
+  }
+
+  let regex = /[^0-7]/u;
+  if (R === 10) {
+    regex = /[^0-9]/u;
+  }
+  if (R === 16) {
+    regex = /[^0-9A-Fa-f]/u;
+  }
+
+  if (regex.test(input)) {
+    return failure;
+  }
+
+  if (validationErrorSeen) {
+    validationErrors?.push("IPv4-non-decimal-part");
+  }
+
+  return parseInt(input, R);
+}
+
+function endsInANumber(input) {
+  const parts = input.split(".");
+  if (parts[parts.length - 1] === "") {
+    if (parts.length === 1) {
+      return false;
+    }
+    parts.pop();
+  }
+
+  const last = parts[parts.length - 1];
+  if (/^[0-9]+$/u.test(last)) {
+    return true;
+  }
+
+  if (parseIPv4Number(last) !== failure) {
+    return true;
+  }
+
+  return false;
+}
+
+function domainParser(domain, validationErrors = null, beStrict = false) {
+  // A domain-to-ASCII validation error is reported whenever the strict Unicode ToASCII (CheckHyphens,
+  // UseSTD3ASCIIRules, and VerifyDnsLength all true) fails, even when the relaxed parameters used
+  // for non-strict parsing succeed. This step does not itself fail the algorithm.
+  //
+  // Spec divergence (performance): the URL Standard runs this strict pass unconditionally, but when
+  // beStrict is false its only effect is that validation error, so we skip it when we are neither
+  // being strict nor collecting validation errors.
+  if (beStrict || validationErrors) {
+    const strictResult = domainParserToASCII(domain, true);
+    if (strictResult === null) {
+      validationErrors?.push("domain-to-ASCII");
+    }
+
+    if (beStrict) {
+      return strictResult === null ? failure : strictResult;
+    }
+  }
+
+  let result;
+  if (infra.isASCIIString(domain)) {
+    // For web compatibility an ASCII domain is returned lowercased regardless of ToASCII's outcome.
+    result = domain.toLowerCase();
+  } else {
+    result = domainParserToASCII(domain, false);
+    if (result === null) {
+      return failure;
+    }
+  }
+
+  if (result === "") {
+    return failure;
+  }
+  if (containsForbiddenDomainCodePoint(result)) {
+    return failure;
+  }
+
+  return result;
+}
+
+function isSpecialScheme(scheme) {
+  return specialSchemes[scheme.toLowerCase()] !== undefined;
+}
+
+function isSpecialSchemeExceptFile(scheme) {
+  return isSpecialScheme(scheme) && scheme.toLowerCase() !== "file";
+}
+
+function defaultPort(scheme) {
+  return specialSchemes[scheme.toLowerCase()];
+}
+
+function isWindowsDriveLetterCodePoints(cp1, cp2) {
+  return infra.isASCIIAlpha(cp1) && (cp2 === p(":") || cp2 === p("|"));
+}
+
+function isWindowsDriveLetterString(string) {
+  return string.length === 2 && infra.isASCIIAlpha(string.codePointAt(0)) && (string[1] === ":" || string[1] === "|");
+}
+
+function isNormalizedWindowsDriveLetterString(string) {
+  return string.length === 2 && infra.isASCIIAlpha(string.codePointAt(0)) && string[1] === ":";
+}
+
+module.exports = {
+  containsForbiddenHostCodePoint,
+  containsPercentEncodedByte,
+  defaultPort,
+  domainParser,
+  endsInANumber,
+  failure,
+  forbiddenHostCodePoints,
+  isInvalidURLCodePoint,
+  isNormalizedWindowsDriveLetterString,
+  isPercentEncodedByteAt,
+  parseIPv4Number,
+  isSpecialScheme,
+  isSpecialSchemeExceptFile,
+  isURLCodePoint,
+  isWindowsDriveLetterCodePoints,
+  isWindowsDriveLetterString,
+  p
+};