spamscanner 7.0.6 → 7.0.7

package/dist/esm/cli.js

@@ -3001,13 +3001,15 @@ function endsInsideTag(html, end, link) {
   return joined !== host && (!isHost(host) || isHost(joined) && registrableDomain(joined) !== registrableDomain(host));
 }
 var MAX_SHOWN_HTML = 2e5;
-function extractLinks(text, html, { maxLength = 5e5 } = {}) {
+function extractLinks(text, html, { maxLength = 5e5, bareHosts = false } = {}) {
   text = typeof text === "string" ? foldToAscii(text) : text;
   html = typeof html === "string" ? foldToAscii(html) : html;
   const links = [];
   const seen = /* @__PURE__ */ new Set();
+  const linkHosts = /* @__PURE__ */ new Set();
+  const bareFound = [];
   const names = /* @__PURE__ */ new Map();
-  const add = (raw, shown = null) => {
+  const add = (raw, shown = null, bare = false) => {
     let href = trimEndOf(trimControls(raw.replaceAll(/[\t\n\r]/g, "")).trim(), `).,;:!?'"]`);
     if (/^www\./i.test(href)) {
       href = `http://${href}`;
@@ -3031,11 +3033,13 @@ ${shown || ""}`;
     }
     seen.add(key);
     const host = url.hostname.toLowerCase().replace(/\.$/, "").replace(/^\[(.*)]$/, "$1");
+    linkHosts.add(host);
     links.push({
       url: url.href.slice(0, MAX_URL),
       host,
       text: shown,
-      userinfo: Boolean(url.username || url.password)
+      userinfo: Boolean(url.username || url.password),
+      ...bare && { bare: true }
     });
   };
   if (typeof html === "string" && html) {
@@ -3064,10 +3068,92 @@ ${shown || ""}`;
         }
         add(link);
       }
+      if (bareHosts && index < 2) {
+        bareLinkHosts(read, (host) => {
+          bareFound.push(host);
+        });
+      }
+    }
+  }
+  for (const host of bareFound) {
+    if (!linkHosts.has(host)) {
+      add(`http://${host}/`, null, true);
     }
   }
   return links;
 }
+var IN_WORD = /[\p{L}\p{N}\p{M}_%]/u;
+var MAX_BARE_NAME = 760;
+function isHostCode(code) {
+  return code >= 97 && code <= 122 || code >= 65 && code <= 90 || code >= 48 && code <= 57 || code === 45 || code === 46;
+}
+function endsLink(char) {
+  return /[\s<>"'\u0060]/u.test(char);
+}
+function bareLinkHosts(text, add) {
+  const found = /* @__PURE__ */ new Set();
+  let previous = -1;
+  let read = 0;
+  let space = -1;
+  let scheme = -1;
+  for (let slash = text.indexOf("/"); slash !== -1; slash = text.indexOf("/", slash + 1)) {
+    const floor = Math.max(previous + 1, slash - MAX_BARE_NAME);
+    previous = slash;
+    let end = slash;
+    while (end > floor && text.codePointAt(end - 1) >= 48 && text.codePointAt(end - 1) <= 57) {
+      end--;
+    }
+    end = end > floor && text[end - 1] === ":" ? end - 1 : slash;
+    let start = end;
+    while (start > floor && isHostCode(text.codePointAt(start - 1))) {
+      start--;
+    }
+    if (start === end || start > 0 && (text[start - 1] === "/" || IN_WORD.test(text[start - 1]))) {
+      continue;
+    }
+    for (; read < start; read++) {
+      if (endsLink(text[read])) {
+        space = read;
+      } else if (text[read] === "/" && text[read - 1] === "/" && text[read - 2] === ":") {
+        scheme = read;
+      }
+    }
+    if (scheme > space) {
+      continue;
+    }
+    while (end > start && text[end - 1] === ".") {
+      end--;
+    }
+    const labels = text.slice(start, end).split(".");
+    const tld = labels.pop();
+    if (!/^(?:[a-z]{2,24}|xn--[a-z\d-]{1,59})$/i.test(tld)) {
+      continue;
+    }
+    const parts = [];
+    while (labels.length > 0 && parts.length < 11) {
+      const label = labels.pop();
+      if (!/^[a-z\d-]{1,63}$/i.test(label)) {
+        break;
+      }
+      parts.unshift(label);
+    }
+    if (parts.length === 0) {
+      continue;
+    }
+    parts[0] = parts[0].replace(/^-+/, "");
+    if (!parts[0]) {
+      continue;
+    }
+    const host = `${parts.join(".")}.${tld}`.toLowerCase();
+    if (found.has(host)) {
+      continue;
+    }
+    found.add(host);
+    if (isHost(host)) {
+      add(host);
+    }
+  }
+}
 function hostOf(link) {
   const authority = link.replace(/^https?:\/\//i, "").split(/[/?#\\]/)[0];
   return trimEndOf(authority.slice(authority.lastIndexOf("@") + 1).replace(/:\d*$/, ""), `).,;:!?'"]`);
@@ -5104,17 +5190,47 @@ async function expandAttachments(attachments = [], errors = [], parse = parseMes
 
 // src/reputation.js
 import { isIP as isIP2 } from "node:net";
-import { domainToASCII } from "node:url";
+import { domainToASCII as domainToASCII2 } from "node:url";
 
 // src/dnsbl.js
 import dns from "node:dns";
 import { isIP } from "node:net";
-function isLookupName(name) {
-  if (typeof name !== "string") {
-    return false;
+import { domainToASCII } from "node:url";
+import { getPublicSuffix } from "tldts";
+var LOOKUP_LABEL = /^[a-z\d_-]{1,63}$/;
+function lookupName(host) {
+  if (typeof host !== "string") {
+    return null;
+  }
+  let end = host.length;
+  while (end > 0 && host[end - 1] === ".") {
+    end--;
+  }
+  let name = host.slice(0, end).toLowerCase();
+  if (/[\u0080-\u{10FFFF}]/u.test(name)) {
+    name = domainToASCII(name) || name;
+  }
+  if (isIP(name.replaceAll(/^\[|]$/g, "")) !== 0) {
+    return null;
+  }
+  const labels = name.split(".");
+  let start = labels.length;
+  let length = -1;
+  while (start > 0 && LOOKUP_LABEL.test(labels[start - 1]) && length + 1 + labels[start - 1].length <= 253) {
+    start--;
+    length += 1 + labels[start].length;
+  }
+  if (start === labels.length) {
+    return null;
+  }
+  if (start === 0) {
+    return name;
   }
-  const host = name.replaceAll(/^\[|]$/g, "").replace(/\.$/, "");
-  return host !== "" && isIP(host) === 0 && host.length <= 253 && host.split(".").every((label) => label.length > 0 && label.length <= 63);
+  const suffix = labels.slice(start).join(".");
+  if (start === labels.length - 1 || getPublicSuffix(suffix, { allowPrivateDomains: true }) === suffix) {
+    return null;
+  }
+  return suffix;
 }
 var CLOUDFLARE_MALWARE = ["1.1.1.2", "1.0.0.2"];
 var CLOUDFLARE_FAMILY = ["1.1.1.3", "1.0.0.3"];
@@ -5313,9 +5429,10 @@ var DnsChecker = class {
     return results.filter(Boolean);
   }
   /**
-   * Ask Cloudflare's filtering resolvers about host names. A name blocked by
-   * the malware resolver is "malware"; one blocked only by the family
-   * resolver is "adult". The hosts that got no answer (from the malware
+   * Ask Cloudflare's filtering resolvers about host names, each by the name
+   * it is looked up as (see lookupName), once; the findings name it. A name
+   * blocked by the malware resolver is "malware"; one blocked only by the
+   * family resolver is "adult". The hosts that got no answer (from the malware
    * resolver, or from the family resolver when asked) are listed in the
    * result's (non-enumerable) `unanswered` property.
    * @param {string[]} hosts
@@ -5328,10 +5445,8 @@ var DnsChecker = class {
   async checkCloudflare(hosts = [], options = {}) {
     const { adult = true, malwareServers = CLOUDFLARE_MALWARE, familyServers = CLOUDFLARE_FAMILY } = options;
     const unanswered = [];
-    const results = await Promise.all(hosts.map(async (host) => {
-      if (!isLookupName(host)) {
-        return null;
-      }
+    const names = [...new Set(hosts.map((host) => lookupName(host)).filter(Boolean))];
+    const results = await Promise.all(names.map(async (host) => {
       const malware = await this.answer(host, malwareServers);
       if (malware?.includes("0.0.0.0")) {
         return { host, category: "malware" };
@@ -5366,7 +5481,7 @@ function asciiHostname(value) {
   if (lower === "" || /[\0-,/:-^`{-\u007F]/.test(lower)) {
     return null;
   }
-  const ascii = domainToASCII(lower);
+  const ascii = domainToASCII2(lower);
   return ascii.length <= 253 && /^[\w-]{1,63}(?:\.[\w-]{1,63})*$/.test(ascii) ? ascii : null;
 }
 function normalizeEntry(value) {
@@ -5945,7 +6060,7 @@ import net from "node:net";
 var package_default = {
   name: "spamscanner",
   description: "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
-  version: "7.0.6",
+  version: "7.0.7",
   author: "Forward Email (https://forwardemail.net)",
   bin: {
     spamscanner: "./dist/esm/cli.js"
@@ -8887,15 +9002,23 @@ var SpamScanner = class {
           message: `A link shows "${link.text}" but goes to ${link.host}`
         }));
       }
-      seenHosts.add(destination(link));
+      const name = lookupName(destination(link));
+      if (name) {
+        seenHosts.add(name);
+      }
     }
-    for (const link of [...links.filter((link2) => link2.text), ...links.filter((link2) => !link2.text)]) {
+    const found = links.filter((link) => !link.bare);
+    const bare = links.filter((link) => link.bare);
+    for (const link of [...found.filter((link2) => link2.text), ...found.filter((link2) => !link2.text)]) {
       lookalike(destination(link), { url: link.url });
     }
+    for (const link of bare) {
+      lookalike(link.host, { source: "text" });
+    }
     for (const host of hosts) {
       lookalike(host, { source: "text" });
     }
-    const ordered = [...new Set([...links.filter((link) => link.text), ...links.filter((link) => !link.text)].map((link) => destination(link)))];
+    const ordered = [...new Set([...found.filter((link) => link.text), ...found.filter((link) => !link.text), ...bare].map((link) => lookupName(destination(link))).filter(Boolean))];
     const firsts = /* @__PURE__ */ new Map();
     for (const host of ordered) {
       const domain = registrableDomain(host);
@@ -8909,9 +9032,9 @@ var SpamScanner = class {
     let answered = 0;
     const cloudflare = async () => {
       await withTimeout(Promise.all(checked.map(async (host) => {
-        const found = await this.dns.checkCloudflare([host], { adult: config.phishing.adult });
-        blocked.push(...found);
-        if (!found.unanswered?.length) {
+        const found2 = await this.dns.checkCloudflare([host], { adult: config.phishing.adult });
+        blocked.push(...found2);
+        if (!found2.unanswered?.length) {
           answered++;
         }
       })), config.timeout, null);
@@ -9095,7 +9218,7 @@ var SpamScanner = class {
     const linkLength = Number.isFinite(config.phishing.maxLength) && config.phishing.maxLength > 0 ? config.phishing.maxLength : 1e6;
     const text = typeof mail.text === "string" ? mail.text : "";
     const html = typeof mail.html === "string" ? mail.html : "";
-    const checkedLinks = extractLinks(text.slice(0, linkLength), html.slice(0, linkLength * 2), { maxLength: linkLength * 2 });
+    const checkedLinks = extractLinks(text.slice(0, linkLength), html.slice(0, linkLength * 2), { maxLength: linkLength * 2, bareHosts: true });
     const linksTruncated = text.length > linkLength || html.length > linkLength * 2;
     const plainObfuscation = obfuscationOf(`${mail.subject || ""}
 ${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLength });
@@ -9236,7 +9359,8 @@ ${visibleText(html.slice(0, config.maxLength * 2)).text}`, { maxLength: config.m
           riskScore: Math.max(0, ...links.homographs.map((item) => item.riskScore))
         }
       },
-      links: checkedLinks.map((link) => link.url),
+      // (the links of the message, not hosts found without one)
+      links: checkedLinks.filter((link) => !link.bare).map((link) => link.url),
       language: extracted.language,
       tokens: extracted.words,
       mail,