rdapper 0.2.0 → 0.4.0

package/README.md

@@ -4,7 +4,7 @@ RDAP‑first domain registration lookups with WHOIS fallback. Produces a single,
 
 - RDAP discovery via IANA bootstrap (`https://data.iana.org/rdap/dns.json`)
 - WHOIS TCP 43 client with TLD discovery, registrar referral follow, and curated exceptions
-- Normalized output: registrar, contacts, nameservers, statuses, dates, DNSSEC, source metadata
+- Normalized output: registrar, contacts, nameservers, statuses, dates, DNSSEC, privacy flag, source metadata
 - TypeScript types included; ESM‑only; no external HTTP client (uses global `fetch`)
 
 ### [🦉 See it in action on hoot.sh!](https://hoot.sh)
@@ -49,6 +49,10 @@ await isAvailable("likely-unregistered-thing-320485230458.com"); // => false
 - `rdapOnly?: boolean` – Only attempt RDAP; do not fall back to WHOIS.
 - `whoisOnly?: boolean` – Skip RDAP and query WHOIS directly.
 - `followWhoisReferral?: boolean` – Follow registrar referral from the TLD WHOIS (default `true`).
+- `maxWhoisReferralHops?: number` – Maximum registrar WHOIS referral hops to follow (default `2`).
+- `rdapFollowLinks?: boolean` – Follow related/entity RDAP links to enrich data (default `true`).
+- `maxRdapLinkHops?: number` – Maximum RDAP related link hops to follow (default `2`).
+- `rdapLinkRels?: string[]` – RDAP link rel values to consider (default `["related","entity","registrar","alternate"]`).
 - `customBootstrapUrl?: string` – Override RDAP bootstrap URL.
 - `whoisHints?: Record<string, string>` – Override/add authoritative WHOIS per TLD (keys are lowercase TLDs, values may include or omit `whois://`).
 - `includeRaw?: boolean` – Include `rawRdap`/`rawWhois` in the returned record (default `false`).
@@ -113,6 +117,7 @@ interface DomainRecord {
     country?: string;
     countryCode?: string;
   }>;
+  privacyEnabled?: boolean;   // registrant appears privacy-redacted based on keyword heuristics
   whoisServer?: string;       // authoritative WHOIS queried (if any)
   rdapServers?: string[];     // RDAP base URLs tried
   rawRdap?: unknown;          // raw RDAP JSON (only when options.includeRaw)
@@ -144,10 +149,11 @@ interface DomainRecord {
 - RDAP
   - Discovers base URLs for the TLD via IANA’s RDAP bootstrap JSON.
   - Tries each base until one responds successfully; parses standard RDAP domain JSON.
+  - Optionally follows related/entity links to registrar RDAP resources and merges results (bounded by hop limits).
   - Normalizes registrar (from `entities`), contacts (vCard), nameservers (`ipAddresses`), events (created/changed/expiration), statuses, and DNSSEC (`secureDNS`).
 - WHOIS
   - Discovers the authoritative TLD WHOIS via `whois.iana.org` (TCP 43), with curated exceptions for tricky zones and public SLDs.
-  - Queries the TLD WHOIS; if a registrar referral is present and `followWhoisReferral !== false`, follows one hop to the registrar WHOIS.
+  - Queries the TLD WHOIS and follows registrar referrals recursively up to `maxWhoisReferralHops` (unless disabled).
   - Normalizes common key/value variants across gTLD/ccTLD formats (dates, statuses, nameservers, contacts). Availability is inferred from common phrases (best‑effort heuristic).
 
 Timeouts are enforced per request using a simple race against `timeoutMs` (default 15s). All network I/O is performed with global `fetch` (RDAP) and a raw TCP socket (WHOIS).

package/dist/index.d.ts

@@ -32,21 +32,37 @@ interface StatusEvent {
   raw?: string;
 }
 interface DomainRecord {
+  /** Normalized domain name */
   domain: string;
+  /** Terminal TLD */
   tld: string;
+  /** Whether the domain is registered */
   isRegistered: boolean;
+  /** Whether the domain is internationalized (IDN) */
   isIDN?: boolean;
+  /** Unicode name */
   unicodeName?: string;
+  /** Punycode name */
   punycodeName?: string;
+  /** Registry operator */
   registry?: string;
+  /** Registrar */
   registrar?: RegistrarInfo;
+  /** Reseller (if applicable) */
   reseller?: string;
+  /** EPP status codes */
   statuses?: StatusEvent[];
+  /** Creation date in ISO 8601 */
   creationDate?: string;
+  /** Updated date in ISO 8601 */
   updatedDate?: string;
+  /** Expiration date in ISO 8601 */
   expirationDate?: string;
+  /** Deletion date in ISO 8601 */
   deletionDate?: string;
+  /** Transfer lock */
   transferLock?: boolean;
+  /** DNSSEC data (if available) */
   dnssec?: {
     enabled: boolean;
     dsRecords?: Array<{
@@ -56,24 +72,51 @@ interface DomainRecord {
       digest?: string;
     }>;
   };
+  /** Nameservers */
   nameservers?: Nameserver[];
+  /** Contacts (registrant, admin, tech, billing, abuse, etc.) */
   contacts?: Contact[];
+  /** Best guess as to whether registrant is redacted based on keywords */
+  privacyEnabled?: boolean;
+  /** Authoritative WHOIS queried (if any) */
   whoisServer?: string;
+  /** RDAP base URLs tried */
   rdapServers?: string[];
+  /** Raw RDAP JSON */
   rawRdap?: unknown;
+  /** Raw WHOIS text (last authoritative) */
   rawWhois?: string;
+  /** Which source produced data */
   source: LookupSource;
+  /** ISO 8601 timestamp at time of lookup */
   fetchedAt: string;
+  /** Warnings generated during lookup */
   warnings?: string[];
 }
 interface LookupOptions {
+  /** Total timeout budget */
   timeoutMs?: number;
+  /** Don't fall back to WHOIS */
   rdapOnly?: boolean;
+  /** Don't attempt RDAP */
   whoisOnly?: boolean;
+  /** Follow referral server (default true) */
   followWhoisReferral?: boolean;
+  /** Maximum registrar WHOIS referral hops (default 2) */
+  maxWhoisReferralHops?: number;
+  /** Follow RDAP related/entity links (default true) */
+  rdapFollowLinks?: boolean;
+  /** Maximum RDAP related link fetches (default 2) */
+  maxRdapLinkHops?: number;
+  /** RDAP link rels to consider (default ["related","entity","registrar","alternate"]) */
+  rdapLinkRels?: string[];
+  /** Override IANA bootstrap */
   customBootstrapUrl?: string;
+  /** Override/add authoritative WHOIS per TLD */
   whoisHints?: Record<string, string>;
+  /** Include rawRdap/rawWhois in results (default false) */
   includeRaw?: boolean;
+  /** Optional cancellation signal */
   signal?: AbortSignal;
 }
 interface LookupResult {

package/dist/index.js

@@ -9,14 +9,14 @@ function toISO(dateLike) {
 	const raw = String(dateLike).trim();
 	if (!raw) return void 0;
 	for (const re of [
-		/^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})(?:Z)?$/,
-		/^(\d{4})\/(\d{2})\/(\d{2})[ T](\d{2}):(\d{2}):(\d{2})$/,
+		/^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})(?:Z|([+-]\d{2})(?::?(\d{2}))?)?$/,
+		/^(\d{4})\/(\d{2})\/(\d{2})[ T](\d{2}):(\d{2}):(\d{2})(?:Z|([+-]\d{2})(?::?(\d{2}))?)?$/,
 		/^(\d{2})-([A-Za-z]{3})-(\d{4})$/,
 		/^([A-Za-z]{3})\s+(\d{1,2})\s+(\d{4})$/
 	]) {
 		const m = raw.match(re);
 		if (!m) continue;
-		const d = parseWithRegex(m, re);
+		const d = parseDateWithRegex(m, re);
 		if (d) return toIsoFromDate(d);
 	}
 	const native = new Date(raw);
@@ -29,7 +29,7 @@ function toIsoFromDate(d) {
 		return;
 	}
 }
-function parseWithRegex(m, _re) {
+function parseDateWithRegex(m, _re) {
 	const monthMap = {
 		jan: 0,
 		feb: 1,
@@ -46,8 +46,16 @@ function parseWithRegex(m, _re) {
 	};
 	try {
 		if (m[0].includes(":")) {
-			const [_$1, y, mo, d, hh, mm, ss] = m;
-			return new Date(Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(hh), Number(mm), Number(ss)));
+			const [_$1, y, mo, d, hh, mm, ss, offH, offM] = m;
+			let dt = Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(hh), Number(mm), Number(ss));
+			if (offH) {
+				const sign = offH.startsWith("-") ? -1 : 1;
+				const hours = Math.abs(Number(offH));
+				const minutes = offM ? Number(offM) : 0;
+				const offsetMs = sign * (hours * 60 + minutes) * 60 * 1e3;
+				dt -= offsetMs;
+			}
+			return new Date(dt);
 		}
 		if (m[0].includes("-")) {
 			const [_$1, dd$1, monStr$1, yyyy$1] = m;
@@ -167,6 +175,149 @@ async function fetchRdapDomain(domain, baseUrl, options) {
 	};
 }
 
+//#endregion
+//#region src/rdap/links.ts
+/** Extract candidate RDAP link URLs from an RDAP document. */
+function extractRdapRelatedLinks(doc, opts) {
+	const rels = (opts?.rdapLinkRels?.length ? opts.rdapLinkRels : [
+		"related",
+		"entity",
+		"registrar",
+		"alternate"
+	]).map((r) => r.toLowerCase());
+	const d = doc ?? {};
+	const arr = Array.isArray(d?.links) ? d.links : [];
+	const out = [];
+	for (const link of arr) {
+		const rel = String(link.rel || "").toLowerCase();
+		const type = String(link.type || "").toLowerCase();
+		if (!rels.includes(rel)) continue;
+		if (type && !/application\/rdap\+json/i.test(type)) continue;
+		const url = link.href || link.value;
+		if (url && /^https?:\/\//i.test(url)) out.push(url);
+	}
+	return Array.from(new Set(out));
+}
+
+//#endregion
+//#region src/rdap/merge.ts
+/** Merge RDAP documents with a conservative, additive strategy. */
+function mergeRdapDocs(baseDoc, others) {
+	const merged = { ...baseDoc };
+	for (const doc of others) {
+		const cur = doc ?? {};
+		merged.status = uniqStrings([...toStringArray(merged.status), ...toStringArray(cur.status)]);
+		merged.events = uniqBy([...toArray(merged.events), ...toArray(cur.events)], (e) => `${String(e?.eventAction ?? "").toLowerCase()}|${String(e?.eventDate ?? "")}`);
+		merged.nameservers = uniqBy([...toArray(merged.nameservers), ...toArray(cur.nameservers)], (n) => `${String(n?.ldhName ?? n?.unicodeName ?? "").toLowerCase()}`);
+		merged.entities = uniqBy([...toArray(merged.entities), ...toArray(cur.entities)], (e) => `${String(e?.handle ?? "").toLowerCase()}|${String(JSON.stringify(e?.roles || [])).toLowerCase()}|${String(JSON.stringify(e?.vcardArray || [])).toLowerCase()}`);
+		if (merged.secureDNS == null && cur.secureDNS != null) merged.secureDNS = cur.secureDNS;
+		if (merged.port43 == null && cur.port43 != null) merged.port43 = cur.port43;
+		const mergedRemarks = merged.remarks;
+		const curRemarks = cur.remarks;
+		if (Array.isArray(mergedRemarks) || Array.isArray(curRemarks)) {
+			const a = toArray(mergedRemarks);
+			const b = toArray(curRemarks);
+			merged.remarks = [...a, ...b];
+		}
+	}
+	return merged;
+}
+/** Fetch and merge RDAP related documents up to a hop limit. */
+async function fetchAndMergeRdapRelated(domain, baseDoc, opts) {
+	const tried = [];
+	if (opts?.rdapFollowLinks === false) return {
+		merged: baseDoc,
+		serversTried: tried
+	};
+	const maxHops = Math.max(0, opts?.maxRdapLinkHops ?? 2);
+	if (maxHops === 0) return {
+		merged: baseDoc,
+		serversTried: tried
+	};
+	const visited = /* @__PURE__ */ new Set();
+	let current = baseDoc;
+	let hops = 0;
+	while (hops < maxHops) {
+		const nextBatch = extractRdapRelatedLinks(current, { rdapLinkRels: opts?.rdapLinkRels }).filter((u) => !visited.has(u));
+		if (nextBatch.length === 0) break;
+		const fetchedDocs = [];
+		for (const url of nextBatch) {
+			visited.add(url);
+			try {
+				const { json } = await fetchRdapUrl(url, opts);
+				tried.push(url);
+				const ldh = String(json?.ldhName ?? "").toLowerCase();
+				const uni = String(json?.unicodeName ?? "").toLowerCase();
+				if (ldh && !sameDomain(ldh, domain)) continue;
+				if (uni && !sameDomain(uni, domain)) continue;
+				fetchedDocs.push(json);
+			} catch {}
+		}
+		if (fetchedDocs.length === 0) break;
+		current = mergeRdapDocs(current, fetchedDocs);
+		hops += 1;
+	}
+	return {
+		merged: current,
+		serversTried: tried
+	};
+}
+async function fetchRdapUrl(url, options) {
+	const res = await withTimeout(fetch(url, {
+		method: "GET",
+		headers: { accept: "application/rdap+json, application/json" },
+		signal: options?.signal
+	}), options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, "RDAP link fetch timeout");
+	if (!res.ok) {
+		const bodyText = await res.text();
+		throw new Error(`RDAP ${res.status}: ${bodyText.slice(0, 500)}`);
+	}
+	const json = await res.json();
+	return {
+		url,
+		json
+	};
+}
+function toArray(val) {
+	return Array.isArray(val) ? val : [];
+}
+function toStringArray(val) {
+	return Array.isArray(val) ? val.map((v) => String(v)) : [];
+}
+function uniqStrings(arr) {
+	return Array.from(new Set(arr));
+}
+function uniqBy(arr, key) {
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (const item of arr) {
+		const k = key(item);
+		if (seen.has(k)) continue;
+		seen.add(k);
+		out.push(item);
+	}
+	return out;
+}
+function sameDomain(a, b) {
+	return a.toLowerCase() === b.toLowerCase();
+}
+
+//#endregion
+//#region src/lib/privacy.ts
+const PRIVACY_NAME_KEYWORDS = [
+	"redacted",
+	"privacy",
+	"private",
+	"withheld",
+	"not disclosed",
+	"protected",
+	"protection"
+];
+function isPrivacyName(value) {
+	const v = value.toLowerCase();
+	return PRIVACY_NAME_KEYWORDS.some((k) => v.includes(k));
+}
+
 //#endregion
 //#region src/lib/text.ts
 function uniq(arr) {
@@ -247,6 +398,8 @@ function normalizeRdap(inputDomain, tld, rdap, rdapServersTried, fetchedAtISO, i
 		return n;
 	}).filter((n) => !!n.host) : void 0;
 	const contacts = extractContacts(doc.entities);
+	const registrant = contacts?.find((c) => c.type === "registrant");
+	const privacyEnabled = !!(registrant && [registrant.name, registrant.organization].filter(Boolean).some(isPrivacyName));
 	const statuses = Array.isArray(doc.status) ? doc.status.filter((s) => typeof s === "string").map((s) => ({
 		status: s,
 		raw: s
@@ -291,6 +444,7 @@ function normalizeRdap(inputDomain, tld, rdap, rdapServersTried, fetchedAtISO, i
 			host: n.host.toLowerCase()
 		}))) : void 0,
 		contacts,
+		privacyEnabled: privacyEnabled ? true : void 0,
 		whoisServer,
 		rdapServers: rdapServersTried,
 		rawRdap: includeRaw ? rdap : void 0,
@@ -592,18 +746,26 @@ function normalizeWhois(domain, tld, whoisText, whoisServer, fetchedAtISO, inclu
 		"domain registration date",
 		"domain create date",
 		"created",
-		"registered"
+		"registered",
+		"domain record activated"
 	]);
 	const updatedDate = anyValue(map, [
 		"updated date",
 		"last updated",
 		"last modified",
-		"modified"
+		"modified",
+		"domain record last updated"
 	]);
 	const expirationDate = anyValue(map, [
 		"registry expiry date",
+		"registry expiration date",
 		"expiry date",
 		"expiration date",
+		"registrar registration expiration date",
+		"registrar registration expiry date",
+		"registrar expiration date",
+		"registrar expiry date",
+		"domain expires",
 		"paid-till",
 		"expires on",
 		"renewal date"
@@ -656,6 +818,8 @@ function normalizeWhois(domain, tld, whoisText, whoisServer, fetchedAtISO, inclu
 		return ns;
 	}).filter((x) => !!x)) : void 0;
 	const contacts = collectContacts(map);
+	const registrant = contacts?.find((c) => c.type === "registrant");
+	const privacyEnabled = !!(registrant && [registrant.name, registrant.organization].filter(Boolean).some(isPrivacyName));
 	const dnssecRaw = (map.dnssec?.[0] || "").toLowerCase();
 	const dnssec = dnssecRaw ? { enabled: /signed|yes|true/.test(dnssecRaw) } : void 0;
 	const transferLock = !!statuses?.some((s) => /transferprohibited/i.test(s.status));
@@ -678,6 +842,7 @@ function normalizeWhois(domain, tld, whoisText, whoisServer, fetchedAtISO, inclu
 		dnssec,
 		nameservers,
 		contacts,
+		privacyEnabled: privacyEnabled ? true : void 0,
 		whoisServer,
 		rdapServers: void 0,
 		rawRdap: void 0,
@@ -771,6 +936,37 @@ function multi(map, keys) {
 	}
 }
 
+//#endregion
+//#region src/whois/referral.ts
+/**
+* Follow registrar WHOIS referrals up to a configured hop limit.
+* Returns the last successful WHOIS response (best-effort; keeps original on failures).
+*/
+async function followWhoisReferrals(initialServer, domain, opts) {
+	const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2);
+	let current = await whoisQuery(initialServer, domain, opts);
+	if (opts?.followWhoisReferral === false || maxHops === 0) return current;
+	const visited = new Set([normalize(current.serverQueried)]);
+	let hops = 0;
+	while (hops < maxHops) {
+		const next = extractWhoisReferral(current.text);
+		if (!next) break;
+		const normalized = normalize(next);
+		if (visited.has(normalized)) break;
+		visited.add(normalized);
+		try {
+			current = await whoisQuery(next, domain, opts);
+		} catch {
+			break;
+		}
+		hops += 1;
+	}
+	return current;
+}
+function normalize(server) {
+	return server.replace(/^whois:\/\//i, "").toLowerCase();
+}
+
 //#endregion
 //#region src/index.ts
 /**
@@ -792,9 +988,10 @@ async function lookupDomain(domain, opts) {
 				tried.push(base);
 				try {
 					const { json } = await fetchRdapDomain(domain, base, opts);
+					const rdapEnriched = await fetchAndMergeRdapRelated(domain, json, opts);
 					return {
 						ok: true,
-						record: normalizeRdap(domain, tld, json, tried, now, !!opts?.includeRaw)
+						record: normalizeRdap(domain, tld, rdapEnriched.merged, [...tried, ...rdapEnriched.serversTried], now, !!opts?.includeRaw)
 					};
 				} catch {}
 			}
@@ -813,13 +1010,7 @@ async function lookupDomain(domain, opts) {
 				error: `No WHOIS server discovered for TLD '${tld}'. This registry may not publish public WHOIS over port 43.${hint}`
 			};
 		}
-		let res = await whoisQuery(whoisServer, domain, opts);
-		if (opts?.followWhoisReferral !== false) {
-			const referral = extractWhoisReferral(res.text);
-			if (referral && referral.toLowerCase() !== whoisServer.toLowerCase()) try {
-				res = await whoisQuery(referral, domain, opts);
-			} catch {}
-		}
+		const res = await followWhoisReferrals(whoisServer, domain, opts);
 		if (publicSuffix.includes(".") && /no match|not found/i.test(res.text) && opts?.followWhoisReferral !== false) {
 			const candidates = [];
 			const ps = publicSuffix.toLowerCase();
@@ -827,10 +1018,10 @@ async function lookupDomain(domain, opts) {
 			if (exception) candidates.push(exception);
 			for (const server of candidates) try {
 				const alt = await whoisQuery(server, domain, opts);
-				if (alt.text && !/error/i.test(alt.text)) {
-					res = alt;
-					break;
-				}
+				if (alt.text && !/error/i.test(alt.text)) return {
+					ok: true,
+					record: normalizeWhois(domain, tld, alt.text, alt.serverQueried, now, !!opts?.includeRaw)
+				};
 			} catch {}
 		}
 		return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rdapper",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "license": "MIT",
   "description": "🎩 RDAP/WHOIS fetcher, parser, and normalizer for Node",
   "repository": {