spamscanner 7.0.6 → 7.0.8

package/dist/esm/cli.js

@@ -2739,6 +2739,9 @@ function unwrapLink(address2) {
   }
   for (let depth = 0; depth < 3; depth++) {
     const target = gatewayTarget(url);
+    if (!target) {
+      break;
+    }
     let next;
     try {
       next = new URL(target);
@@ -3001,13 +3004,15 @@ function endsInsideTag(html, end, link) {
   return joined !== host && (!isHost(host) || isHost(joined) && registrableDomain(joined) !== registrableDomain(host));
 }
 var MAX_SHOWN_HTML = 2e5;
-function extractLinks(text, html, { maxLength = 5e5 } = {}) {
+function extractLinks(text, html, { maxLength = 5e5, bareHosts = false } = {}) {
   text = typeof text === "string" ? foldToAscii(text) : text;
   html = typeof html === "string" ? foldToAscii(html) : html;
   const links = [];
   const seen = /* @__PURE__ */ new Set();
+  const linkHosts = /* @__PURE__ */ new Set();
+  const bareFound = [];
   const names = /* @__PURE__ */ new Map();
-  const add = (raw, shown = null) => {
+  const add = (raw, shown = null, bare = false) => {
     let href = trimEndOf(trimControls(raw.replaceAll(/[\t\n\r]/g, "")).trim(), `).,;:!?'"]`);
     if (/^www\./i.test(href)) {
       href = `http://${href}`;
@@ -3031,11 +3036,13 @@ ${shown || ""}`;
     }
     seen.add(key);
     const host = url.hostname.toLowerCase().replace(/\.$/, "").replace(/^\[(.*)]$/, "$1");
+    linkHosts.add(host);
     links.push({
       url: url.href.slice(0, MAX_URL),
       host,
       text: shown,
-      userinfo: Boolean(url.username || url.password)
+      userinfo: Boolean(url.username || url.password),
+      ...bare && { bare: true }
     });
   };
   if (typeof html === "string" && html) {
@@ -3064,10 +3071,92 @@ ${shown || ""}`;
         }
         add(link);
       }
+      if (bareHosts && index < 2) {
+        bareLinkHosts(read, (host) => {
+          bareFound.push(host);
+        });
+      }
+    }
+  }
+  for (const host of bareFound) {
+    if (!linkHosts.has(host)) {
+      add(`http://${host}/`, null, true);
     }
   }
   return links;
 }
+var IN_WORD = /[\p{L}\p{N}\p{M}_%]/u;
+var MAX_BARE_NAME = 760;
+function isHostCode(code) {
+  return code >= 97 && code <= 122 || code >= 65 && code <= 90 || code >= 48 && code <= 57 || code === 45 || code === 46;
+}
+function endsLink(char) {
+  return /[\s<>"'\u0060]/u.test(char);
+}
+function bareLinkHosts(text, add) {
+  const found = /* @__PURE__ */ new Set();
+  let previous = -1;
+  let read = 0;
+  let space = -1;
+  let scheme = -1;
+  for (let slash = text.indexOf("/"); slash !== -1; slash = text.indexOf("/", slash + 1)) {
+    const floor = Math.max(previous + 1, slash - MAX_BARE_NAME);
+    previous = slash;
+    let end = slash;
+    while (end > floor && text.codePointAt(end - 1) >= 48 && text.codePointAt(end - 1) <= 57) {
+      end--;
+    }
+    end = end > floor && text[end - 1] === ":" ? end - 1 : slash;
+    let start = end;
+    while (start > floor && isHostCode(text.codePointAt(start - 1))) {
+      start--;
+    }
+    if (start === end || start > 0 && (text[start - 1] === "/" || IN_WORD.test(text[start - 1]))) {
+      continue;
+    }
+    for (; read < start; read++) {
+      if (endsLink(text[read])) {
+        space = read;
+      } else if (text[read] === "/" && text[read - 1] === "/" && text[read - 2] === ":") {
+        scheme = read;
+      }
+    }
+    if (scheme > space) {
+      continue;
+    }
+    while (end > start && text[end - 1] === ".") {
+      end--;
+    }
+    const labels = text.slice(start, end).split(".");
+    const tld = labels.pop();
+    if (!/^(?:[a-z]{2,24}|xn--[a-z\d-]{1,59})$/i.test(tld)) {
+      continue;
+    }
+    const parts = [];
+    while (labels.length > 0 && parts.length < 11) {
+      const label = labels.pop();
+      if (!/^[a-z\d-]{1,63}$/i.test(label)) {
+        break;
+      }
+      parts.unshift(label);
+    }
+    if (parts.length === 0) {
+      continue;
+    }
+    parts[0] = parts[0].replace(/^-+/, "");
+    if (!parts[0]) {
+      continue;
+    }
+    const host = `${parts.join(".")}.${tld}`.toLowerCase();
+    if (found.has(host)) {
+      continue;
+    }
+    found.add(host);
+    if (isHost(host)) {
+      add(host);
+    }
+  }
+}
 function hostOf(link) {
   const authority = link.replace(/^https?:\/\//i, "").split(/[/?#\\]/)[0];
   return trimEndOf(authority.slice(authority.lastIndexOf("@") + 1).replace(/:\d*$/, ""), `).,;:!?'"]`);
@@ -5065,24 +5154,38 @@ async function parseWithin(raw, parse = parseMessage, deadline = Date.now() + PA
     }
   }
 }
-async function expandAttachments(attachments = [], errors = [], parse = parseMessage) {
+async function expandAttachments(attachments = [], errors = [], parse = parseMessage, unopened = []) {
   const all = [];
   const deadline = Date.now() + PARSE_TIMEOUT;
   let budget = MAX_NESTED_BYTES;
   let nested = 0;
+  let tooMany = false;
   const visit = async (list, prefix, depth) => {
     for (const attachment of list) {
       if (prefix) {
         if (nested >= MAX_ATTACHMENTS) {
+          if (!tooMany) {
+            tooMany = true;
+            unopened.push({ filename: prefix, reason: `more than ${MAX_ATTACHMENTS} attachments in attached messages` });
+          }
           return;
         }
         nested++;
       }
       const name = attachment.filename || (prefix ? "unnamed attachment" : "");
+      const filename = prefix ? `${prefix}/${name}` : name || "attached message";
       all.push(prefix ? { ...attachment, filename: `${prefix}/${name}` } : attachment);
       const { content } = attachment;
       const attached = attachment.contentType === "message/rfc822" || /\.eml$/i.test(attachment.filename || "");
-      if (attached && depth < MAX_NESTED_DEPTH && Buffer5.isBuffer(content) && content.length <= budget) {
+      if (attached && Buffer5.isBuffer(content) && depth >= MAX_NESTED_DEPTH) {
+        unopened.push({ filename, reason: `nested more than ${MAX_NESTED_DEPTH} deep` });
+        continue;
+      }
+      if (attached && Buffer5.isBuffer(content) && content.length > budget) {
+        unopened.push({ filename, reason: `past ${MAX_NESTED_BYTES / 1024 / 1024} MB of attached messages` });
+        continue;
+      }
+      if (attached && Buffer5.isBuffer(content)) {
         budget -= content.length;
         let inner;
         try {
@@ -5104,17 +5207,47 @@ async function expandAttachments(attachments = [], errors = [], parse = parseMes
 
 // src/reputation.js
 import { isIP as isIP2 } from "node:net";
-import { domainToASCII } from "node:url";
+import { domainToASCII as domainToASCII2 } from "node:url";
 
 // src/dnsbl.js
 import dns from "node:dns";
 import { isIP } from "node:net";
-function isLookupName(name) {
-  if (typeof name !== "string") {
-    return false;
+import { domainToASCII } from "node:url";
+import { getPublicSuffix } from "tldts";
+var LOOKUP_LABEL = /^[a-z\d_-]{1,63}$/;
+function lookupName(host) {
+  if (typeof host !== "string") {
+    return null;
+  }
+  let end = host.length;
+  while (end > 0 && host[end - 1] === ".") {
+    end--;
+  }
+  let name = host.slice(0, end).toLowerCase();
+  if (/[\u0080-\u{10FFFF}]/u.test(name)) {
+    name = domainToASCII(name) || name;
   }
-  const host = name.replaceAll(/^\[|]$/g, "").replace(/\.$/, "");
-  return host !== "" && isIP(host) === 0 && host.length <= 253 && host.split(".").every((label) => label.length > 0 && label.length <= 63);
+  if (isIP(name.replaceAll(/^\[|]$/g, "")) !== 0) {
+    return null;
+  }
+  const labels = name.split(".");
+  let start = labels.length;
+  let length = -1;
+  while (start > 0 && LOOKUP_LABEL.test(labels[start - 1]) && length + 1 + labels[start - 1].length <= 253) {
+    start--;
+    length += 1 + labels[start].length;
+  }
+  if (start === labels.length) {
+    return null;
+  }
+  if (start === 0) {
+    return name;
+  }
+  const suffix = labels.slice(start).join(".");
+  if (start === labels.length - 1 || getPublicSuffix(suffix, { allowPrivateDomains: true }) === suffix) {
+    return null;
+  }
+  return suffix;
 }
 var CLOUDFLARE_MALWARE = ["1.1.1.2", "1.0.0.2"];
 var CLOUDFLARE_FAMILY = ["1.1.1.3", "1.0.0.3"];
@@ -5313,9 +5446,10 @@ var DnsChecker = class {
     return results.filter(Boolean);
   }
   /**
-   * Ask Cloudflare's filtering resolvers about host names. A name blocked by
-   * the malware resolver is "malware"; one blocked only by the family
-   * resolver is "adult". The hosts that got no answer (from the malware
+   * Ask Cloudflare's filtering resolvers about host names, each by the name
+   * it is looked up as (see lookupName), once; the findings name it. A name
+   * blocked by the malware resolver is "malware"; one blocked only by the
+   * family resolver is "adult". The hosts that got no answer (from the malware
    * resolver, or from the family resolver when asked) are listed in the
    * result's (non-enumerable) `unanswered` property.
    * @param {string[]} hosts
@@ -5328,10 +5462,8 @@ var DnsChecker = class {
   async checkCloudflare(hosts = [], options = {}) {
     const { adult = true, malwareServers = CLOUDFLARE_MALWARE, familyServers = CLOUDFLARE_FAMILY } = options;
     const unanswered = [];
-    const results = await Promise.all(hosts.map(async (host) => {
-      if (!isLookupName(host)) {
-        return null;
-      }
+    const names = [...new Set(hosts.map((host) => lookupName(host)).filter(Boolean))];
+    const results = await Promise.all(names.map(async (host) => {
       const malware = await this.answer(host, malwareServers);
       if (malware?.includes("0.0.0.0")) {
         return { host, category: "malware" };
@@ -5366,7 +5498,7 @@ function asciiHostname(value) {
   if (lower === "" || /[\0-,/:-^`{-\u007F]/.test(lower)) {
     return null;
   }
-  const ascii = domainToASCII(lower);
+  const ascii = domainToASCII2(lower);
   return ascii.length <= 253 && /^[\w-]{1,63}(?:\.[\w-]{1,63})*$/.test(ascii) ? ascii : null;
 }
 function normalizeEntry(value) {
@@ -5945,7 +6077,7 @@ import net from "node:net";
 var package_default = {
   name: "spamscanner",
   description: "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
-  version: "7.0.6",
+  version: "7.0.8",
   author: "Forward Email (https://forwardemail.net)",
   bin: {
     spamscanner: "./dist/esm/cli.js"
@@ -8747,14 +8879,16 @@ var SpamScanner = class {
    * a header field ("Name: value", or an mbox "From " line) is read as the
    * body of a message with no header, so its text is the input; other
    * input mailparser cannot read gives a message with no text. Other errors
-   * are thrown.
+   * are thrown. `error` is the parser's limit the message exceeds, so that
+   * only its first parts were read (as scan() reports it, see parseError),
+   * or null.
    * @param {Buffer|string|Uint8Array|AsyncIterable} source
-   * @returns {Promise<{raw: Buffer, mail: object}>}
+   * @returns {Promise<{raw: Buffer, mail: object, error: string|null}>}
    */
   async parse(source) {
     const raw = await toBuffer(source);
-    const { mail } = await parseWithin(withHeader(raw));
-    return { raw, mail };
+    const { mail, error } = await parseWithin(withHeader(raw));
+    return { raw, mail, error: error ?? null };
   }
   /**
    * Classifier features of a parsed message.
@@ -8887,15 +9021,23 @@ var SpamScanner = class {
           message: `A link shows "${link.text}" but goes to ${link.host}`
         }));
       }
-      seenHosts.add(destination(link));
+      const name = lookupName(destination(link));
+      if (name) {
+        seenHosts.add(name);
+      }
     }
-    for (const link of [...links.filter((link2) => link2.text), ...links.filter((link2) => !link2.text)]) {
+    const found = links.filter((link) => !link.bare);
+    const bare = links.filter((link) => link.bare);
+    for (const link of [...found.filter((link2) => link2.text), ...found.filter((link2) => !link2.text)]) {
       lookalike(destination(link), { url: link.url });
     }
+    for (const link of bare) {
+      lookalike(link.host, { source: "text" });
+    }
     for (const host of hosts) {
       lookalike(host, { source: "text" });
     }
-    const ordered = [...new Set([...links.filter((link) => link.text), ...links.filter((link) => !link.text)].map((link) => destination(link)))];
+    const ordered = [...new Set([...found.filter((link) => link.text), ...found.filter((link) => !link.text), ...bare].map((link) => lookupName(destination(link))).filter(Boolean))];
     const firsts = /* @__PURE__ */ new Map();
     for (const host of ordered) {
       const domain = registrableDomain(host);
@@ -8909,9 +9051,9 @@ var SpamScanner = class {
     let answered = 0;
     const cloudflare = async () => {
       await withTimeout(Promise.all(checked.map(async (host) => {
-        const found = await this.dns.checkCloudflare([host], { adult: config.phishing.adult });
-        blocked.push(...found);
-        if (!found.unanswered?.length) {
+        const found2 = await this.dns.checkCloudflare([host], { adult: config.phishing.adult });
+        blocked.push(...found2);
+        if (!found2.unanswered?.length) {
           answered++;
         }
       })), config.timeout, null);
@@ -9095,7 +9237,7 @@ var SpamScanner = class {
     const linkLength = Number.isFinite(config.phishing.maxLength) && config.phishing.maxLength > 0 ? config.phishing.maxLength : 1e6;
     const text = typeof mail.text === "string" ? mail.text : "";
     const html = typeof mail.html === "string" ? mail.html : "";
-    const checkedLinks = extractLinks(text.slice(0, linkLength), html.slice(0, linkLength * 2), { maxLength: linkLength * 2 });
+    const checkedLinks = extractLinks(text.slice(0, linkLength), html.slice(0, linkLength * 2), { maxLength: linkLength * 2, bareHosts: true });
     const linksTruncated = text.length > linkLength || html.length > linkLength * 2;
     const plainObfuscation = obfuscationOf(`${mail.subject || ""}
 ${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLength });
@@ -9106,7 +9248,8 @@ ${visibleText(html.slice(0, config.maxLength * 2)).text}`, { maxLength: config.m
       mixed: Math.max(plainObfuscation.mixed, shownObfuscation.mixed),
       styled: plainObfuscation.styled || shownObfuscation.styled
     };
-    const attached = await expandAttachments(mail.attachments, parseErrors);
+    const unopened = [];
+    const attached = await expandAttachments(mail.attachments, parseErrors, void 0, unopened);
     const attachments = config.attachments === false ? [] : inspectAttachments(attached).filter((item) => config.macros !== false || !["macro", "pdf_active", "rtf_object"].includes(item.type)).map((item) => finding(item));
     const [links, authentication, checkedReputation, dnsbl, virusScan, toxicity, nsfw] = await Promise.all([
       this.checkLinks(checkedLinks, config, extracted.hosts, fromDomain(mail)),
@@ -9227,6 +9370,13 @@ ${visibleText(html.slice(0, config.maxLength * 2)).text}`, { maxLength: config.m
         // scanned, as {filename, reason}: the same on every try, so not a
         // failure.
         virusScanSkipped: virusScan.skipped,
+        // Attached messages not opened, past the limits attached messages
+        // are read within (nested more than three deep, more than 100
+        // attachments in them, or more than 25 MB of them), as
+        // {filename, reason}: their attachments were not inspected. Not
+        // scored (forwarded messages can be nested so), for a caller that
+        // needs a message read whole, e.g. to block what it carries.
+        unopenedAttachments: unopened,
         // Whether the message has more text than phishing.maxLength, whose
         // links were not read.
         linksTruncated,
@@ -9236,7 +9386,8 @@ ${visibleText(html.slice(0, config.maxLength * 2)).text}`, { maxLength: config.m
           riskScore: Math.max(0, ...links.homographs.map((item) => item.riskScore))
         }
       },
-      links: checkedLinks.map((link) => link.url),
+      // (the links of the message, not hosts found without one)
+      links: checkedLinks.filter((link) => !link.bare).map((link) => link.url),
       language: extracted.language,
       tokens: extracted.words,
       mail,