tldts-tests 7.0.32 → 7.1.1

package/src/tldts-tests.ts

@@ -1,8 +1,53 @@
 import { expect } from 'chai';
 import 'mocha';
+import { existsSync, readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
 
 import { loadPublicSuffixList, parsePublicSuffixRules } from 'tldts-utils';
 
+/**
+ * A single entry of the WHATWG URL test data (web-platform-tests). Entries are
+ * either a comment string or a parsed-result object.
+ */
+interface WhatwgUrlTestCase {
+  input: string;
+  base?: string | null;
+  hostname?: string;
+  protocol?: string;
+  failure?: boolean;
+}
+
+/** Locate a repo fixture whether tests run from `src` (ts-node) or `dist`. */
+function resolveFixture(relativePath: string): string {
+  let dir = __dirname;
+  while (!existsSync(resolve(dir, relativePath)) && dir !== dirname(dir)) {
+    dir = dirname(dir);
+  }
+  return resolve(dir, relativePath);
+}
+
+/** True if `value` contains any non-ASCII (> U+007F) code unit. */
+function containsNonAscii(value: string): boolean {
+  for (let i = 0; i < value.length; i += 1) {
+    if (value.charCodeAt(i) > 127) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * WHATWG URL test data, vendored verbatim from web-platform-tests:
+ * https://github.com/web-platform-tests/wpt/blob/258f285de043b79e44324228c0fd800b38d21879/url/resources/urltestdata.json
+ * Re-pin by re-downloading url/resources/urltestdata.json from that repo.
+ */
+const WHATWG_URL_TEST_DATA = JSON.parse(
+  readFileSync(
+    resolveFixture('packages/tldts-tests/src/data/urltestdata.json'),
+    'utf-8',
+  ),
+) as (string | WhatwgUrlTestCase)[];
+
 interface Options {
   extractHostname?: boolean;
   allowPrivateDomains?: boolean;
@@ -147,6 +192,11 @@ export default function test(
         'while',
         'with',
         'yield',
+        // Object.prototype-member labels (lowercase, so they survive hostname
+        // lowercasing): must be treated as ordinary labels, never resolve
+        // against the prototype chain during trie traversal.
+        'constructor',
+        '__proto__',
       ].forEach((keyword) => {
         it(keyword, () => {
           expect(tldts.getDomain(`https://${keyword}.com`)).to.equal(
@@ -975,6 +1025,258 @@ export default function test(
     }
   });
 
+  // `getHostname` must pick the same host substring a compliant WHATWG parser
+  // would, for tldts' scope. Intentional deviations (no IDNA/IPv4/IPv6
+  // normalisation, trailing-dot stripping, lenient host:port/email) are pinned
+  // in the "intentional deviations" block below.
+  describe('WHATWG URL hostname compliance', () => {
+    // --- Web Platform Tests: the browsers' own URL conformance corpus ---
+    describe('web-platform-tests url/resources/urltestdata.json', () => {
+      const SPECIAL_SCHEME = new Set([
+        'ftp:',
+        'file:',
+        'http:',
+        'https:',
+        'ws:',
+        'wss:',
+      ]);
+
+      // In-scope = cases where tldts is expected to match the compliant parser
+      // exactly. We exclude representation differences tldts does not perform
+      // (IDN/punycode, IPv4 normalisation, IPv6) and inputs that rely on a base
+      // URL (tldts has no base concept). `failure` cases are excluded because
+      // tldts is best-effort, not strict-reject.
+      const inScope = (o: WhatwgUrlTestCase): boolean => {
+        if (o.failure) {
+          return false;
+        }
+        const hostname = o.hostname;
+        if (typeof hostname !== 'string') {
+          return false;
+        }
+        if (containsNonAscii(o.input)) {
+          return false; // non-ASCII input (IDN) — tldts keeps Unicode
+        }
+        if (hostname.startsWith('xn--')) {
+          return false; // punycode output
+        }
+        if (hostname.startsWith('[')) {
+          return false; // IPv6 (tldts strips brackets, does not compress)
+        }
+        if (o.protocol === 'file:') {
+          return false; // file drive-letter / backslash quirks
+        }
+        if (/^[0-9.]+$/.test(hostname) && !o.input.includes(hostname)) {
+          return false; // IPv4 normalisation (e.g. 0x7f.1 -> 127.0.0.1)
+        }
+        const schemeMatch = /^([a-zA-Z][a-zA-Z0-9+.-]*:)\/\//.exec(o.input);
+        if (schemeMatch !== null) {
+          const scheme = schemeMatch[1];
+          return (
+            scheme !== undefined && SPECIAL_SCHEME.has(scheme.toLowerCase())
+          );
+        }
+        // Bare input (no scheme, not a path) is only meaningful without a base.
+        const bare =
+          !/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(o.input) &&
+          !o.input.startsWith('/');
+        return bare && (o.base === null || o.base === undefined);
+      };
+
+      // tldts strips trailing dots, and reports "no host" as `null` (not "").
+      const expectedHost = (hostname: string): string | null => {
+        let host = hostname;
+        while (host.length > 0 && host.endsWith('.')) {
+          host = host.slice(0, -1);
+        }
+        return host === '' ? null : host;
+      };
+
+      // Documented intentional deviations (NOT boundary bugs): each is an
+      // IDNA / percent-decode / shallow-validation difference. Removing an entry
+      // once tldts handles it becomes the regression gate.
+      const KNOWN_DIVERGENCES = new Set<string>([
+        'http://./', // all-dot host: WHATWG collapses empty labels (IDNA); tldts keeps '.'
+        'http://../', // idem
+        'http://!"$&\'()*+,-.;=_`{}~/', // punctuation host: WHATWG allows, tldts shallow-validation rejects
+        'wss://!"$&\'()*+,-.;=_`{}~/', // idem
+        'https://a%C2%ADb/', // WHATWG percent-decodes + IDNA-maps the soft hyphen; tldts does neither
+      ]);
+
+      for (const entry of WHATWG_URL_TEST_DATA) {
+        if (typeof entry === 'string' || !inScope(entry)) {
+          continue;
+        }
+        const { input } = entry;
+        if (KNOWN_DIVERGENCES.has(input)) {
+          continue;
+        }
+        const expected = expectedHost(entry.hostname!);
+        it(JSON.stringify(input), () => {
+          expect(tldts.getHostname(input), input).to.equal(expected);
+        });
+      }
+    });
+
+    // --- Curated boundary cases (the bugs this change fixes) ---
+    describe('boundary detection', () => {
+      it("treats '\\' as '/' for special schemes", () => {
+        // WHATWG host-state: backslash is a host terminator for special URLs.
+        expect(tldts.getHostname('https:\\\\example.com\\path')).to.equal(
+          'example.com',
+        );
+      });
+
+      it("does not let a '\\' smuggle the host into userinfo", () => {
+        // Security: 'http://example.com\@evil.com' must resolve to example.com,
+        // not evil.com (backslash is normalised to '/' before the '@').
+        expect(tldts.getHostname('http://example.com\\@evil.com')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('parses a special scheme written without "//"', () => {
+        // WHATWG special-authority-slashes state: the authority is parsed even
+        // when the "//" is missing.
+        expect(tldts.getHostname('https:example.com')).to.equal('example.com');
+        expect(tldts.getHostname('http:example.com/foo')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('removes an embedded ASCII tab in the scheme', () => {
+        // WHATWG basic parser step 2: tab/newline are removed before parsing.
+        expect(tldts.getHostname('htt\tp://example.com/')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('removes an embedded newline / carriage return in the host', () => {
+        expect(tldts.getHostname('http://exa\nmple.com/')).to.equal(
+          'example.com',
+        );
+        expect(tldts.getHostname('http://exa\rmple.com/')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('returns null for file:/// (empty authority)', () => {
+        // WHATWG file-host state: an empty buffer means an empty host.
+        expect(tldts.getHostname('file:///etc/passwd')).to.equal(null);
+        expect(tldts.getHostname('file:/etc/passwd')).to.equal(null);
+      });
+
+      it('returns the host for file://host/path', () => {
+        expect(tldts.getHostname('file://server/share')).to.equal('server');
+      });
+
+      it('returns null for opaque (non-special, no "//") schemes', () => {
+        // WHATWG opaque-path state: there is no host.
+        expect(tldts.getHostname('foo:example.com/bar')).to.equal(null);
+        expect(tldts.getHostname('javascript:alert(1)')).to.equal(null);
+        expect(tldts.getHostname('urn:isbn:9780307476463')).to.equal(null);
+        expect(tldts.getHostname('mailto:example.com')).to.equal(null); // no '@'
+      });
+
+      it('extracts the host past an empty userinfo', () => {
+        // WHATWG authority-state: 'http://@host' has empty userinfo, host=host.
+        expect(tldts.getHostname('http://@example.com')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('lower-cases the scheme when matching (case-insensitive)', () => {
+        expect(tldts.getHostname('HTTPS:EXAMPLE.COM')).to.equal('example.com');
+      });
+
+      it('recognises every special scheme written without "//"', () => {
+        // Exercises the ws / wss / ftp / http / https / file classifier arms.
+        expect(tldts.getHostname('ws:example.com')).to.equal('example.com');
+        expect(tldts.getHostname('wss:example.com')).to.equal('example.com');
+        expect(tldts.getHostname('ftp:example.com')).to.equal('example.com');
+        expect(tldts.getHostname('file://h/x')).to.equal('h');
+      });
+
+      it('treats look-alike non-special schemes as opaque (no host)', () => {
+        // Scheme lengths 1..5 that all fail the special-scheme classifier.
+        expect(tldts.getHostname('w:example.com')).to.equal(null);
+        expect(tldts.getHostname('wx:example.com')).to.equal(null);
+        expect(tldts.getHostname('htu:example.com')).to.equal(null);
+        expect(tldts.getHostname('htxp:example.com')).to.equal(null);
+        expect(tldts.getHostname('htxps:example.com')).to.equal(null);
+      });
+
+      it('keeps a bare host:port (lenient superset)', () => {
+        // tldts accepts hostnames, not only URLs; a strict parser treats the
+        // left side as a scheme. The "port" must be numeric.
+        expect(tldts.getHostname('example.co.uk:8080/p?q#h')).to.equal(
+          'example.co.uk',
+        );
+      });
+
+      it('treats host:<non-numeric-port> as an opaque scheme (=> null)', () => {
+        // Documented limit of the host:port heuristic.
+        expect(tldts.getHostname('example.com:foo')).to.equal(null);
+      });
+
+      it('keeps a scheme-relative reference', () => {
+        expect(tldts.getHostname('//user@example.co.uk:8080/x')).to.equal(
+          'example.co.uk',
+        );
+      });
+    });
+
+    // --- Intentional deviations, pinned so they cannot change silently ---
+    describe('intentional deviations (pinned)', () => {
+      it('keeps Unicode hosts (no IDNA/punycode)', () => {
+        expect(tldts.getHostname('http://bücher.example/')).to.equal(
+          'bücher.example',
+        );
+      });
+
+      it('keeps the literal IPv4 (no normalisation)', () => {
+        expect(tldts.getHostname('http://0x7f.1/')).to.equal('0x7f.1');
+      });
+
+      it('returns bracket-less, uncompressed IPv6', () => {
+        expect(tldts.getHostname('http://[2001:0DB8::1]:8080/')).to.equal(
+          '2001:0db8::1',
+        );
+      });
+
+      it('strips trailing dots', () => {
+        expect(tldts.getHostname('http://example.com./')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('is best-effort on an out-of-range port (WHATWG rejects)', () => {
+        expect(tldts.getHostname('http://example.com:99999/')).to.equal(
+          'example.com',
+        );
+      });
+
+      it('extracts the host from an email-like input', () => {
+        expect(tldts.getHostname('Pelé@example.com')).to.equal('example.com');
+      });
+
+      it('keeps an all-dot host instead of collapsing it (no IDNA)', () => {
+        expect(tldts.getHostname('http://./')).to.equal('.');
+        expect(tldts.getHostname('http://../')).to.equal('.');
+      });
+
+      it('rejects punctuation hosts WHATWG allows (shallow validation)', () => {
+        expect(tldts.getHostname('http://!"$&\'()*+,-.;=_`{}~/')).to.equal(
+          null,
+        );
+      });
+
+      it('does not percent-decode the host (no IDNA mapping)', () => {
+        expect(tldts.getHostname('https://a%C2%ADb/')).to.equal(null);
+      });
+    });
+  });
+
   describe('wildcard tests', () => {
     parsePublicSuffixRules(loadPublicSuffixList(), (rule) => {
       if (
@@ -1043,4 +1345,114 @@ export default function test(
       }
     });
   });
+
+  describe('options handling (regression for shared default-options reuse)', () => {
+    // Calling a method with no options must behave identically to passing `{}`.
+    // The no-options path now returns a shared cached defaults object instead of
+    // allocating one per call; this guards that the shared object stays read-only.
+    it('omitting options equals passing {} for every public method', () => {
+      const url = 'https://sub.example.co.uk/path?q=1';
+      expect(tldts.parse(url)).to.deep.equal(tldts.parse(url, {}));
+      expect(tldts.getHostname(url)).to.equal(tldts.getHostname(url, {}));
+      expect(tldts.getDomain(url)).to.equal(tldts.getDomain(url, {}));
+      expect(tldts.getPublicSuffix(url)).to.equal(
+        tldts.getPublicSuffix(url, {}),
+      );
+      expect(tldts.getSubdomain(url)).to.equal(tldts.getSubdomain(url, {}));
+      expect(tldts.getDomainWithoutSuffix(url)).to.equal(
+        tldts.getDomainWithoutSuffix(url, {}),
+      );
+    });
+
+    // A prior call with custom options must not corrupt the shared defaults that
+    // a subsequent no-options call relies on (co.uk is ICANN in all packages).
+    it('a custom-options call does not corrupt later default-options results', () => {
+      const url = 'https://sub.example.co.uk/';
+      const before = tldts.getDomain(url); // default options
+      // interleave calls with assorted non-default options
+      tldts.getDomain(url, {
+        allowPrivateDomains: true,
+        validateHostname: false,
+        detectIp: false,
+      });
+      tldts.getDomain('https://other.example.com/', { mixedInputs: false });
+      const after = tldts.getDomain(url); // default options again
+      expect(after).to.equal(before);
+      expect(after).to.equal('example.co.uk');
+    });
+  });
+
+  describe('public-suffix reconstruction (single-slice offset)', () => {
+    // Rules match across multiple labels: the suffix is rebuilt via a single
+    // hostname.slice at the computed offset (was parts.slice(i).join('.')).
+    it('returns a multi-label rules suffix unchanged', () => {
+      expect(tldts.getPublicSuffix('not.evil.co.uk')).to.equal('co.uk');
+      expect(tldts.getDomain('a.b.c.example.co.uk')).to.equal('example.co.uk');
+    });
+
+    // Exception match: suffix offset uses index+1 (the exception removes its
+    // own left-most label, e.g. !www.ck -> suffix 'ck').
+    it('returns an exception suffix unchanged', () => {
+      expect(tldts.getPublicSuffix('www.ck')).to.equal('ck');
+    });
+  });
+
+  describe('hostname validation dedupe', () => {
+    // Already-valid lowercase hostname: extractHostname returns the SAME string
+    // reference, so the post-extraction validation is skipped — result must
+    // still be that hostname.
+    it('keeps an already-valid hostname', () => {
+      expect(tldts.getHostname('example.co.uk')).to.equal('example.co.uk');
+      expect(tldts.getHostname('sub.example.com')).to.equal('sub.example.com');
+    });
+
+    // Trailing dot is trimmed by extractHostname -> returned string differs from
+    // the input -> the skip guard must NOT fire -> trimmed hostname is validated.
+    it('still validates a trailing-dot hostname (guard does not fire)', () => {
+      expect(tldts.getHostname('miam.miam.google.com.')).to.equal(
+        'miam.miam.google.com',
+      );
+    });
+
+    // Uppercase input is not a "valid hostname" to the validator, so the full
+    // extraction + lowercasing path runs and the guard does not fire.
+    it('lowercases an uppercase hostname', () => {
+      expect(tldts.getHostname('EXAMPLE.COM')).to.equal('example.com');
+    });
+
+    // An invalid hostname must still be rejected: the guard only skips the
+    // re-scan when the input was ALREADY valid, so it can never keep an invalid one.
+    it('still rejects an invalid hostname', () => {
+      expect(tldts.getHostname('foo-.com')).to.equal(null);
+    });
+  });
+
+  describe('wildcard suffix reconstruction', () => {
+    // *.sch.uk where the matched suffix spans every label of the hostname:
+    // in the packed-hash impl this takes the backward-scan fallback branch
+    // (matchLabels === number of hashed labels). Must return the full suffix.
+    it('returns a wildcard suffix that spans the whole hostname', () => {
+      expect(tldts.getPublicSuffix('foo.sch.uk')).to.equal('foo.sch.uk');
+    });
+
+    // *.sch.uk with an extra leading label (non-fallback wildcard branch).
+    it('returns a wildcard suffix with a leading subdomain present', () => {
+      expect(tldts.getPublicSuffix('pupils.school.sch.uk')).to.equal(
+        'school.sch.uk',
+      );
+    });
+  });
+
+  describe('prototype-pollution-safe trie traversal', () => {
+    // A label equal to an Object.prototype member, sitting where the trie has a
+    // '*' wildcard child (*.ck), must match the wildcard — NOT resolve to
+    // Object.prototype.constructor / the __proto__ accessor. Guards the
+    // hasOwnProperty check (or, post-Stage-2, the substring-keyed lookup).
+    it('treats constructor/__proto__ labels as ordinary labels under a wildcard', () => {
+      expect(tldts.getPublicSuffix('b.constructor.ck')).to.equal(
+        'constructor.ck',
+      );
+      expect(tldts.getPublicSuffix('b.__proto__.ck')).to.equal('__proto__.ck');
+    });
+  });
 }