parse-domain 2.2.0 → 2.3.2

package/lib/parseDomain.js

@@ -6,53 +6,35 @@ var privateTrie = require("../lists/private.complete");
 
 var normalize = require("./normalize.js");
 
-var lookUp = require("./tries/lookUp"); // eslint-disable-next-line
+var lookUp = require("./trie/lookUp"); // eslint-disable-next-line
 
 
-var urlParts = /^(:?\/\/|https?:\/\/)?([^/]*@)?(.+?)(:\d{2,5})?([/?].*)?$/; // 1 = protocol, 2 = auth, 3 = domain, 4 = port, 5 = path
+var urlParts = /^(:?\/\/|https?:\/\/)?([^/]*@)?(.+?)(:\d{2,5})?([/?].*)?$/; // 1 = protocol, 2 = auth, 3 = hostname, 4 = port, 5 = path
 
 var dot = /\./g;
 var emptyArr = [];
 
-function matchTld(domain, options) {
+function matchTld(hostname, options) {
   // for potentially unrecognized tlds, try matching against custom tlds
   if (options.customTlds) {
     // try matching against a built regexp of custom tlds
-    var tld = domain.match(options.customTlds);
+    var tld = hostname.match(options.customTlds);
 
     if (tld !== null) {
       return tld[0];
     }
   }
 
-  var tries = (options.privateTlds ? [privateTrie] : emptyArr).concat(icannTrie);
-  var _iteratorNormalCompletion = true;
-  var _didIteratorError = false;
-  var _iteratorError = undefined;
+  var domains = hostname.split(".");
+  var icannTlds = lookUp(icannTrie, domains);
+  var privateTlds = options.privateTlds ? lookUp(privateTrie, domains) : emptyArr;
 
-  try {
-    for (var _iterator = tries[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
-      var trie = _step.value;
-
-      var _tld = lookUp(trie, domain);
+  if (privateTlds.length > icannTlds.length) {
+    return "." + privateTlds.join(".");
+  }
 
-      if (_tld !== null) {
-        return "." + _tld;
-      }
-    }
-  } catch (err) {
-    _didIteratorError = true;
-    _iteratorError = err;
-  } finally {
-    try {
-      if (!_iteratorNormalCompletion && _iterator["return"] != null) {
-        _iterator["return"]();
-      }
-    } finally {
-      if (_didIteratorError) {
-        throw _iteratorError;
-      }
-    }
+  if (icannTlds.length > 0) {
+    return "." + icannTlds.join(".");
   }
 
   return null;
@@ -80,36 +62,40 @@ function parseDomain(url, options) {
   var normalizedUrl = normalize.url(url);
   var tld = null;
   var urlSplit;
-  var domain;
+  var hostname;
 
   if (!normalizedUrl) {
     return null;
   }
 
-  var normalizedOptions = normalize.options(options); // urlSplit can't be null because urlParts will always match at the third capture
+  var normalizedOptions = normalize.options(options);
+  urlSplit = normalizedUrl.match(urlParts); // urlSplit is null if the url contains certain characters like '\n', '\r'.
+
+  if (urlSplit === null) {
+    return null;
+  }
 
-  urlSplit = normalizedUrl.match(urlParts);
-  domain = urlSplit[3]; // domain will now be something like sub.domain.example.com
+  hostname = urlSplit[3]; // domain will now be something like sub.domain.example.com
 
-  tld = matchTld(domain, normalizedOptions);
+  tld = matchTld(hostname, normalizedOptions);
 
   if (tld === null) {
     return null;
   } // remove tld and split by dot
 
 
-  urlSplit = domain.slice(0, -tld.length).split(dot);
+  urlSplit = hostname.slice(0, -tld.length).split(dot);
 
   if (tld.charAt(0) === ".") {
     // removes the remaining dot, if present (added to handle localhost)
     tld = tld.slice(1);
   }
 
-  domain = urlSplit.pop();
+  hostname = urlSplit.pop();
   var subdomain = urlSplit.join(".");
   return {
     tld: tld,
-    domain: domain,
+    domain: hostname,
     subdomain: subdomain
   };
 }

package/lib/trie/characters.js

@@ -0,0 +1,16 @@
+"use strict";
+
+module.exports = {
+  UP: "<",
+  // one level up
+  SAME: ",",
+  // same level
+  DOWN: ">",
+  // one level down
+  RESET: "|",
+  // reset level index and start new
+  WILDCARD: "*",
+  // as defined by publicsuffix.org
+  EXCEPTION: "!" // as defined by publicsuffix.org
+
+};

package/lib/trie/lookUp.js

@@ -0,0 +1,33 @@
+"use strict";
+
+var characters = require("./characters.js");
+
+function lookUp(trie, domains) {
+  var domainsToCheck = domains.slice();
+  var topLevel = [];
+  var node = trie;
+
+  while (domainsToCheck.length) {
+    var domain = domainsToCheck.pop();
+
+    if (node.children.has(characters.WILDCARD)) {
+      if (node.children.has(characters.EXCEPTION + domain)) {
+        break;
+      }
+
+      node = node.children.get(characters.WILDCARD);
+    } else {
+      if (node.children.has(domain) === false) {
+        break;
+      }
+
+      node = node.children.get(domain);
+    }
+
+    topLevel.unshift(domain);
+  }
+
+  return topLevel;
+}
+
+module.exports = lookUp;

package/lib/trie/nodes.js

@@ -0,0 +1,40 @@
+"use strict";
+
+function createNode() {
+  var domain = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null;
+  return {
+    domain: domain,
+    children: new Map(),
+    parent: null
+  };
+}
+
+function createOrGetChild(parent, domain) {
+  var child = parent.children.get(domain);
+
+  if (child === undefined) {
+    child = createNode(domain);
+    adoptChild(parent, child);
+  }
+
+  return child;
+}
+
+function adoptChild(parent, child) {
+  if (typeof child.domain !== "string") {
+    throw new Error("Cannot adopt child: child.domain must be a string");
+  }
+
+  if (parent.children.has(child.domain)) {
+    throw new Error("Cannot adopt child: parent has already a child with the domain '".concat(child.domain, "'"));
+  }
+
+  parent.children.set(child.domain, child);
+  child.parent = parent;
+}
+
+module.exports = {
+  createNode: createNode,
+  createOrGetChild: createOrGetChild,
+  adoptChild: adoptChild
+};

package/lib/trie/parse.js

@@ -0,0 +1,64 @@
+"use strict";
+
+var characters = require("./characters");
+
+var _require = require("./nodes"),
+    createNode = _require.createNode,
+    createOrGetChild = _require.createOrGetChild; // Parsing is complex... :)
+// eslint-disable-next-line complexity
+
+
+function parse(serialized) {
+  var rootNode = createNode();
+  var domain = "";
+  var parentNode = rootNode;
+  var node;
+
+  function addDomain() {
+    node = createOrGetChild(parentNode, domain);
+    domain = "";
+  }
+
+  for (var i = 0; i < serialized.length; i++) {
+    var _char = serialized.charAt(i);
+
+    switch (_char) {
+      case characters.SAME:
+        {
+          addDomain();
+          continue;
+        }
+
+      case characters.DOWN:
+        {
+          addDomain();
+          parentNode = node;
+          continue;
+        }
+
+      case characters.RESET:
+        {
+          addDomain();
+          parentNode = rootNode;
+          continue;
+        }
+
+      case characters.UP:
+        {
+          addDomain();
+          parentNode = parentNode.parent;
+          continue;
+        }
+    }
+
+    domain += _char;
+  }
+
+  if (domain !== "") {
+    addDomain();
+  }
+
+  return rootNode;
+}
+
+module.exports = parse;

package/lib/{tries → trie}/serializeTrie.js

@@ -4,7 +4,7 @@ var _LINE_FILTERS;
 
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
 
-var SEPARATORS = require("./separators");
+var characters = require("./characters");
 
 var TYPE_COMPLETE = "complete";
 var TYPE_LIGHT = "light";
@@ -54,14 +54,14 @@ function lineToString(line, i, arr) {
 
     if (indexOfDifference === 0) {
       // line and prevLine are completely different
-      separatorFromPrev = SEPARATORS.RESET;
+      separatorFromPrev = characters.RESET;
     } else if (indexOfDifference === prevLine.length - 1) {
       // only the last part of line and prevLine are different
-      separatorFromPrev = SEPARATORS.SAME;
+      separatorFromPrev = characters.SAME;
     } else if (indexOfDifference > prevLine.length - 1) {
       // we don't need to go up the hierarchy first because prevLine is part of line
       // so let's just start with an initial down separator
-      separatorFromPrev = SEPARATORS.DOWN;
+      separatorFromPrev = characters.DOWN;
     } else {
       // line and prevLine are different, but share a common root at indexOfDifference - 1
       // we now need to go up the hierarchy to the common root
@@ -76,11 +76,11 @@ function lineToString(line, i, arr) {
         throw new Error("Cannot serialize trie: The public suffix list contains unexpected rules.");
       }
 
-      separatorFromPrev = new Array(levelsUp).fill(SEPARATORS.UP).join("");
+      separatorFromPrev = new Array(levelsUp).fill(characters.UP).join("");
     }
   }
 
-  return separatorFromPrev + line.slice(indexOfDifference).join(SEPARATORS.DOWN);
+  return separatorFromPrev + line.slice(indexOfDifference).join(characters.DOWN);
 }
 
 function serializeTrie(parsedList, type) {

package/lists/icann.complete.js

@@ -1,5 +1,5 @@
 "use strict";
 
-const parseTrie = require("../lib/tries/parseTrie");
+const parse = require("../lib/trie/parse");
 
-module.exports = parseTrie(require("../build/tries/icann.complete.json").trie);
+module.exports = parse(require("../build/tries/icann.complete.json").trie);

package/lists/icann.light.js

@@ -1,5 +1,5 @@
 "use strict";
 
-const parseTrie = require("../lib/tries/parseTrie");
+const parse = require("../lib/trie/parse");
 
-module.exports = parseTrie(require("../build/tries/icann.light.json").trie);
+module.exports = parse(require("../build/tries/icann.light.json").trie);

package/lists/private.complete.js

@@ -1,5 +1,5 @@
 "use strict";
 
-const parseTrie = require("../lib/tries/parseTrie");
+const parse = require("../lib/trie/parse");
 
-module.exports = parseTrie(require("../build/tries/private.complete.json").trie);
+module.exports = parse(require("../build/tries/private.complete.json").trie);

package/package.json

@@ -1,8 +1,9 @@
 {
   "name": "parse-domain",
-  "version": "2.2.0",
+  "version": "2.3.2",
   "description": "Splits an url into sub-domain, domain and effective top-level-domain",
   "main": "./lib/parseDomain.js",
+  "types": "./src/index.d.ts",
   "scripts": {
     "build": "run-s build:lib build:tries",
     "build:lib": "run-s build:lib:clean build:lib:babel",
@@ -39,7 +40,8 @@
     "chai": "^4.2.0",
     "got": "^8.3.2",
     "mkdirp": "^0.5.1",
-    "mocha": "^6.1.4"
+    "mocha": "^6.1.4",
+    "npm-run-all": "^4.1.5"
   },
   "devDependencies": {
     "@babel/cli": "^7.4.4",
@@ -49,7 +51,6 @@
     "eslint-config-peerigon": "^17.1.0",
     "eslint-plugin-import": "^2.17.2",
     "eslint-plugin-jsdoc": "^4.8.4",
-    "npm-run-all": "^4.1.5",
     "nyc": "^14.1.1",
     "rimraf": "^2.6.3",
     "standard-version": "^6.0.1",
@@ -60,6 +61,7 @@
     "scripts",
     "lists",
     "test",
-    "build"
+    "build",
+    "src/index.d.ts"
   ]
 }

package/scripts/build-tries.js

@@ -12,7 +12,7 @@ const got = require("got");
 const mkdirp = require("mkdirp");
 
 try {
-    require.resolve("../lib/tries/parsePubSuffixList");
+    require.resolve("../lib/trie/parsePubSuffixList");
 } catch (error) {
     // This conditions occurs when the CI system or a developer checks out the repo for the first time.
     // It happens because the postinstall hook kicks in *before* the lib is built.
@@ -20,12 +20,13 @@ try {
     console.error("Lib does not exist yet, skipping build-tries step.");
     process.exit(0); // eslint-disable-line no-process-exit
 }
-const parsePubSuffixList = require("../lib/tries/parsePubSuffixList");
-const serializeTrie = require("../lib/tries/serializeTrie");
+const parsePubSuffixList = require("../lib/trie/parsePubSuffixList");
+const serializeTrie = require("../lib/trie/serializeTrie");
 
 const PUBLIC_SUFFIX_URL = "https://publicsuffix.org/list/public_suffix_list.dat";
 const rootPath = path.resolve(__dirname, "..");
 const triesPath = path.resolve(rootPath, "build", "tries");
+const metaJsonFilename = path.resolve(triesPath, "meta.json");
 const tries = [
     {
         listName: "icann",
@@ -59,7 +60,6 @@ got(PUBLIC_SUFFIX_URL, {timeout: 60 * 1000})
                 return {
                     path: path.resolve(triesPath, trie.filename),
                     content: JSON.stringify({
-                        updatedAt: new Date().toISOString(),
                         trie: serializeTrie(parsedList, trie.type),
                     }),
                 };
@@ -80,14 +80,22 @@ got(PUBLIC_SUFFIX_URL, {timeout: 60 * 1000})
         });
 
         process.stderr.write("ok" + os.EOL);
+
+        fs.writeFileSync(
+            metaJsonFilename,
+            JSON.stringify({
+                updatedAt: new Date().toISOString(),
+            }),
+            "utf8"
+        );
     })
     .catch(err => {
         console.error("");
         console.error(`Could not update list of known top-level domains for parse-domain because of "${err.message}"`);
 
-        const prebuiltList = JSON.parse(fs.readFileSync(path.resolve(triesPath, tries[0].filename), "utf8"));
+        const metaJson = JSON.parse(fs.readFileSync(metaJsonFilename, "utf8"));
 
-        console.error("Using possibly outdated prebuilt list from " + new Date(prebuiltList.updatedAt).toDateString());
+        console.error("Using possibly outdated prebuilt list from " + new Date(metaJson.updatedAt).toDateString());
 
         // We can recover using the (possibly outdated) prebuilt list, hence exit code 0
         process.exit(0); // eslint-disable-line no-process-exit

package/src/index.d.ts

@@ -0,0 +1,18 @@
+export as namespace parseDomain;
+
+declare namespace parseDomain {
+    interface ParseOptions {
+        customTlds?: RegExp | Array<string>;
+        privateTlds?: boolean;
+    }
+
+    interface ParsedDomain {
+        domain: string;
+        subdomain: string;
+        tld: string;
+    }
+}
+
+declare function parseDomain(url: string, options?: parseDomain.ParseOptions): parseDomain.ParsedDomain | null;
+
+export = parseDomain;

package/test/parseDomain.test.js

@@ -141,9 +141,17 @@ describe("parseDomain(url)", () => {
     it("should return null if the given value is not a string", () => {
         expect(parseDomain(undefined)).to.equal(null);
         expect(parseDomain({})).to.equal(null);
+    });
+
+    it("should return null if the given string is not a valid URL", () => {
+        expect(parseDomain("\xa0")).to.equal(null);
         expect(parseDomain("")).to.equal(null);
         expect(parseDomain(" ")).to.equal(null);
-        expect(parseDomain("\xa0")).to.equal(null);
+        expect(parseDomain("http://hell.d\ne.ibm.com")).to.equal(null);
+    });
+
+    it("should return null if the given is an empty string with a space character", () => {
+        expect(parseDomain(" ")).to.equal(null);
     });
 
     it("should work with domains that could match multiple tlds", () => {
@@ -172,15 +180,6 @@ describe("parseDomain(url)", () => {
         });
     });
 
-    // See https://github.com/peerigon/parse-domain/pull/65
-    it("should parse police.uk as tld", () => {
-        expect(parseDomain("example.police.uk")).to.eql({
-            subdomain: "",
-            domain: "example",
-            tld: "police.uk",
-        });
-    });
-
     it("should also work with custom top-level domains passed as regexps", () => {
         const options = {customTlds: /(\.local|localhost)$/};
 
@@ -208,4 +207,147 @@ describe("parseDomain(url)", () => {
             tld: "com",
         });
     });
+
+    describe("real-world use cases", () => {
+        // See https://github.com/peerigon/parse-domain/pull/65
+        it("should parse police.uk as tld", () => {
+            expect(parseDomain("example.police.uk")).to.eql({
+                subdomain: "",
+                domain: "example",
+                tld: "police.uk",
+            });
+        });
+
+        // See https://github.com/peerigon/parse-domain/issues/67
+        it("should parse gouv.fr as tld", () => {
+            expect(parseDomain("dev.classea12.beta.gouv.fr", {privateTlds: true})).to.eql({
+                tld: "gouv.fr",
+                domain: "beta",
+                subdomain: "dev.classea12",
+            });
+        });
+    });
+
+    describe("official test suite", () => {
+        it("passes all inputs", () => {
+            function checkPublicSuffix(input, expectedTld) {
+                console.log(input);
+                const result = parseDomain(input, {privateTlds: true});
+
+                if (expectedTld === null) {
+                    // When the expectedTld is null it means that the domain
+                    // should not be registerable.
+                    // Our module will return an empty string for the domain
+                    // in that case. The tld should match the input.
+                    expect(result.tld).to.equal(input.toLowerCase());
+                    expect(result.domain).to.equal("");
+                } else {
+                    expect(result.domain + "." + result.tld).to.equal(expectedTld);
+                }
+            }
+
+            // Any copyright is dedicated to the Public Domain.
+            // https://creativecommons.org/publicdomain/zero/1.0/
+
+            // null input.
+            // checkPublicSuffix(null, null);
+            // Mixed case.
+            checkPublicSuffix("COM", null);
+            checkPublicSuffix("example.COM", "example.com");
+            checkPublicSuffix("WwW.example.COM", "example.com");
+            // Leading dot.
+            // TODO: Make this test work
+            // checkPublicSuffix(".com", null);
+            // checkPublicSuffix(".example", null);
+            // checkPublicSuffix(".example.com", null);
+            // checkPublicSuffix(".example.example", null);
+            // Unlisted TLD.
+            // This is a strange test case. Why should 'example.example' be registerable when 'example' is unknown?
+            // checkPublicSuffix("example", null);
+            // checkPublicSuffix("example.example", "example.example");
+            // checkPublicSuffix("b.example.example", "example.example");
+            // checkPublicSuffix("a.b.example.example", "example.example");
+            // Listed, but non-Internet, TLD.
+            // checkPublicSuffix('local', null);
+            // checkPublicSuffix('example.local', null);
+            // checkPublicSuffix('b.example.local', null);
+            // checkPublicSuffix('a.b.example.local', null);
+            // TLD with only 1 rule.
+            checkPublicSuffix("biz", null);
+            checkPublicSuffix("domain.biz", "domain.biz");
+            checkPublicSuffix("b.domain.biz", "domain.biz");
+            checkPublicSuffix("a.b.domain.biz", "domain.biz");
+            // TLD with some 2-level rules.
+            checkPublicSuffix("com", null);
+            checkPublicSuffix("example.com", "example.com");
+            checkPublicSuffix("b.example.com", "example.com");
+            checkPublicSuffix("a.b.example.com", "example.com");
+            checkPublicSuffix("uk.com", null);
+            checkPublicSuffix("example.uk.com", "example.uk.com");
+            checkPublicSuffix("b.example.uk.com", "example.uk.com");
+            checkPublicSuffix("a.b.example.uk.com", "example.uk.com");
+            checkPublicSuffix("test.ac", "test.ac");
+            // TLD with only 1 (wildcard) rule.
+            checkPublicSuffix("mm", null);
+            checkPublicSuffix("c.mm", null);
+            checkPublicSuffix("b.c.mm", "b.c.mm");
+            checkPublicSuffix("a.b.c.mm", "b.c.mm");
+            // More complex TLD.
+            checkPublicSuffix("jp", null);
+            checkPublicSuffix("test.jp", "test.jp");
+            checkPublicSuffix("www.test.jp", "test.jp");
+            checkPublicSuffix("ac.jp", null);
+            checkPublicSuffix("test.ac.jp", "test.ac.jp");
+            checkPublicSuffix("www.test.ac.jp", "test.ac.jp");
+            checkPublicSuffix("kyoto.jp", null);
+            checkPublicSuffix("test.kyoto.jp", "test.kyoto.jp");
+            checkPublicSuffix("ide.kyoto.jp", null);
+            checkPublicSuffix("b.ide.kyoto.jp", "b.ide.kyoto.jp");
+            checkPublicSuffix("a.b.ide.kyoto.jp", "b.ide.kyoto.jp");
+            checkPublicSuffix("c.kobe.jp", null);
+            checkPublicSuffix("b.c.kobe.jp", "b.c.kobe.jp");
+            checkPublicSuffix("a.b.c.kobe.jp", "b.c.kobe.jp");
+            checkPublicSuffix("city.kobe.jp", "city.kobe.jp");
+            checkPublicSuffix("www.city.kobe.jp", "city.kobe.jp");
+            // TLD with a wildcard rule and exceptions.
+            checkPublicSuffix("ck", null);
+            checkPublicSuffix("test.ck", null);
+            checkPublicSuffix("b.test.ck", "b.test.ck");
+            checkPublicSuffix("a.b.test.ck", "b.test.ck");
+            checkPublicSuffix("www.ck", "www.ck");
+            checkPublicSuffix("www.www.ck", "www.ck");
+            // US K12.
+            checkPublicSuffix("us", null);
+            checkPublicSuffix("test.us", "test.us");
+            checkPublicSuffix("www.test.us", "test.us");
+            checkPublicSuffix("ak.us", null);
+            checkPublicSuffix("test.ak.us", "test.ak.us");
+            checkPublicSuffix("www.test.ak.us", "test.ak.us");
+            checkPublicSuffix("k12.ak.us", null);
+            checkPublicSuffix("test.k12.ak.us", "test.k12.ak.us");
+            checkPublicSuffix("www.test.k12.ak.us", "test.k12.ak.us");
+            // IDN labels.
+            checkPublicSuffix("食狮.com.cn", "食狮.com.cn");
+            checkPublicSuffix("食狮.公司.cn", "食狮.公司.cn");
+            checkPublicSuffix("www.食狮.公司.cn", "食狮.公司.cn");
+            checkPublicSuffix("shishi.公司.cn", "shishi.公司.cn");
+            checkPublicSuffix("公司.cn", null);
+            checkPublicSuffix("食狮.中国", "食狮.中国");
+            checkPublicSuffix("www.食狮.中国", "食狮.中国");
+            checkPublicSuffix("shishi.中国", "shishi.中国");
+            checkPublicSuffix("中国", null);
+            // Same as above, but punycoded.
+            // Punycode is currently not supported
+            // TODO: Make these tests work
+            // checkPublicSuffix("xn--85x722f.com.cn", "xn--85x722f.com.cn");
+            // checkPublicSuffix("xn--85x722f.xn--55qx5d.cn", "xn--85x722f.xn--55qx5d.cn");
+            // checkPublicSuffix("www.xn--85x722f.xn--55qx5d.cn", "xn--85x722f.xn--55qx5d.cn");
+            // checkPublicSuffix("shishi.xn--55qx5d.cn", "shishi.xn--55qx5d.cn");
+            // checkPublicSuffix("xn--55qx5d.cn", null);
+            // checkPublicSuffix("xn--85x722f.xn--fiqs8s", "xn--85x722f.xn--fiqs8s");
+            // checkPublicSuffix("www.xn--85x722f.xn--fiqs8s", "xn--85x722f.xn--fiqs8s");
+            // checkPublicSuffix("shishi.xn--fiqs8s", "shishi.xn--fiqs8s");
+            // checkPublicSuffix("xn--fiqs8s", null);
+        });
+    });
 });