spamscanner 7.0.14 → 7.0.15

package/dist/esm/index.js

@@ -3305,6 +3305,7 @@ var EXECUTABLE_EXTENSIONS = /* @__PURE__ */ new Set([
   "apk",
   "app",
   "appimage",
+  "appinstaller",
   "application",
   "appref-ms",
   "appx",
@@ -3341,6 +3342,7 @@ var EXECUTABLE_EXTENSIONS = /* @__PURE__ */ new Set([
   "inf",
   "ins",
   "ipa",
+  "iqy",
   "iso",
   "isp",
   "its",
@@ -3349,6 +3351,7 @@ var EXECUTABLE_EXTENSIONS = /* @__PURE__ */ new Set([
   "js",
   "jse",
   "ksh",
+  "library-ms",
   "lnk",
   "mad",
   "maf",
@@ -3380,6 +3383,8 @@ var EXECUTABLE_EXTENSIONS = /* @__PURE__ */ new Set([
   "mst",
   "msu",
   "nsh",
+  "one",
+  "onepkg",
   "ops",
   "osd",
   "pcd",
@@ -3410,6 +3415,8 @@ var EXECUTABLE_EXTENSIONS = /* @__PURE__ */ new Set([
   "scpt",
   "scr",
   "sct",
+  "search-ms",
+  "searchconnector-ms",
   "settingcontent-ms",
   "sh",
   "shb",
@@ -3637,6 +3644,14 @@ function extensionOf(name) {
   }
   return whole;
 }
+var COMPRESSED_EXTENSIONS = /* @__PURE__ */ new Set(["gz", "bz2", "xz", "zst"]);
+function uncompressedName(name) {
+  if (!COMPRESSED_EXTENSIONS.has(extensionOf(name))) {
+    return String(name);
+  }
+  const trimmed = trimEndOf(String(name).replace(ALL_BIDI_CONTROLS, ""), ". ");
+  return trimmed.slice(0, trimmed.lastIndexOf("."));
+}
 function lastExtension(base) {
   const dot = base.lastIndexOf(".");
   return dot === -1 ? "" : base.slice(dot + 1).trim();
@@ -3689,7 +3704,20 @@ function inspectAttachments(attachments = []) {
     } else if (RISKY_EXTENSIONS.has(extension)) {
       add("risky_attachment", `Attachment "${name}" is a risky file type (.${extension})`, { extension });
     }
+    const held = COMPRESSED_EXTENSIONS.has(extension) ? extensionOf(uncompressedName(filename)) : "";
+    if (EXECUTABLE_EXTENSIONS.has(held)) {
+      const script = SCRIPT_EXTENSIONS.has(held);
+      add(script ? "script_in_archive" : "executable_in_archive", `Archive "${name}" contains ${script ? "a script" : "a program"} (${shorten(uncompressedName(filename).split(/[/\\]/).pop())})`, { extension: held, extensions: [held] });
+    }
+    const webPage = detected?.type === "html" || detected?.type === "svg" || ["htm", "html", "mht", "mhtml", "shtml", "xht", "xhtml", "svg"].includes(extension);
+    const addWebPage = () => {
+      const active = content !== null && /<script\b|<form\b|\bon(?:load|error|click)\s*=|javascript:|window\.location|document\.location|atob\(/i.test(content.subarray(0, 2e5).toString("utf8"));
+      add("html_attachment", `Attachment "${name}" is ${detected?.type === "svg" || extension === "svg" ? "an SVG image" : "a web page"}${active ? " with scripts or forms" : ""}`, { extension: extension || detected?.type, active });
+    };
     if (!content) {
+      if (webPage) {
+        addWebPage();
+      }
       continue;
     }
     const zipFamily = ZIP_FAMILY.has(detected?.type) || extension === "zip" && !detected?.executable && entriesOf(content).length > 0;
@@ -3701,10 +3729,10 @@ function inspectAttachments(attachments = []) {
       } else if (office && entries.some((entry) => MACRO_EXTENSIONS.has(extensionOf(entry.name)))) {
         add("macro", `Attachment "${name}" is an Office document holding another with macros`, { extension });
       }
-      const inner = entries.filter((entry) => EXECUTABLE_EXTENSIONS.has(extensionOf(entry.name)) && !(office && OFFICE_EMBEDDED.has(extensionOf(entry.name))));
-      const details = { extension: extensionOf(inner[0]?.name ?? ""), extensions: [...new Set(inner.map((entry) => extensionOf(entry.name)))] };
+      const inner = entries.filter((entry) => EXECUTABLE_EXTENSIONS.has(extensionOf(uncompressedName(entry.name))) && !(office && OFFICE_EMBEDDED.has(extensionOf(entry.name))));
+      const details = { extension: extensionOf(uncompressedName(inner[0]?.name ?? "")), extensions: [...new Set(inner.map((entry) => extensionOf(uncompressedName(entry.name))))] };
       const listed = inner.slice(0, 3).map((entry) => shorten(entry.name.split("/").pop())).join(", ");
-      if (inner.some((entry) => !SCRIPT_EXTENSIONS.has(extensionOf(entry.name)))) {
+      if (inner.some((entry) => !SCRIPT_EXTENSIONS.has(extensionOf(uncompressedName(entry.name))))) {
         add("executable_in_archive", `Archive "${name}" contains ${inner.length === 1 ? "a program" : `${inner.length} programs`} (${listed})`, details);
       } else if (inner.length > 0) {
         add("script_in_archive", `Archive "${name}" contains ${inner.length === 1 ? "a script" : `${inner.length} scripts`} (${listed})`, details);
@@ -3723,10 +3751,8 @@ function inspectAttachments(attachments = []) {
     } else if (detected?.type === "rtf" && /\\object\b|\\objdata\b|\\objupdate\b/.test(content.toString("latin1"))) {
       add("rtf_object", `RTF document "${name}" embeds an OLE object`, { extension: "rtf" });
     }
-    if (detected?.type === "html" || detected?.type === "svg" || ["htm", "html", "shtml", "xhtml", "svg"].includes(extension)) {
-      const text = content.subarray(0, 2e5).toString("utf8");
-      const active = /<script\b|<form\b|\bon(?:load|error|click)\s*=|javascript:|window\.location|document\.location|atob\(/i.test(text);
-      add("html_attachment", `Attachment "${name}" is ${detected?.type === "svg" || extension === "svg" ? "an SVG image" : "a web page"}${active ? " with scripts or forms" : ""}`, { extension: extension || detected?.type, active });
+    if (webPage) {
+      addWebPage();
     }
   }
   return findings;
@@ -3901,6 +3927,7 @@ var MAX_LINE = 998;
 var MAX_FIELD = 8192;
 var MAX_HEADER = 512 * 1024;
 var MAX_REPEATED = 100;
+var MIN_LINE_COST = 64;
 var KEPT_FIELDS = /* @__PURE__ */ new Set([
   "from",
   "sender",
@@ -3935,18 +3962,106 @@ var MAX_ATTACHMENTS = 100;
 var MAX_NESTED_BYTES = 25 * 1024 * 1024;
 var MAX_ATTACHED_MESSAGES = 100;
 var PARSE_TIMEOUT = 6e4;
+var NAMED_TEXT = /* @__PURE__ */ Symbol("named text parts");
+var ENCODED_FILES = /^(?:begin [0-7]{3,4} |=ybegin\b[^\r\n\u2028\u2029]*?\bname=)([^\r\n\u2028\u2029]+)\r?$/gm;
+var MAX_ENCODED_FILES = 1e3;
+var foundIn = /* @__PURE__ */ new WeakMap();
+function addFinding(list, part) {
+  const found = foundIn.get(list) ?? /* @__PURE__ */ new Set();
+  foundIn.set(list, found);
+  const types = inspectAttachments([part]).map((finding2) => finding2.type);
+  if (types.some((type) => !found.has(type))) {
+    list.push(part);
+    for (const type of types) {
+      found.add(type);
+    }
+  }
+}
+function parametersOf(value) {
+  const parameters = [];
+  let start = -1;
+  let quoted = false;
+  for (let i = 0; i < value.length; i++) {
+    if (quoted && value[i] === "\\") {
+      i++;
+    } else if (value[i] === '"') {
+      quoted = !quoted;
+    } else if (value[i] === ";" && !quoted) {
+      if (start !== -1) {
+        parameters.push(value.slice(start, i));
+      }
+      start = i + 1;
+    }
+  }
+  if (start !== -1) {
+    parameters.push(value.slice(start));
+  }
+  return parameters;
+}
+var MAX_NAME_FIELDS = 8;
+var MAX_NAME_PARAMETERS = 32;
+function repeatedNames(node) {
+  const names = [];
+  const { headers, libmime } = node ?? {};
+  if (typeof headers?.get !== "function" || typeof libmime?.parseHeaderValue !== "function") {
+    return names;
+  }
+  const read = (value, parameter) => {
+    let name = "";
+    try {
+      name = libmime.decodeWords(String(libmime.parseHeaderValue(value, { stripComments: true }).params[parameter] ?? "")).trim();
+    } catch {
+    }
+    if (name !== "") {
+      names.push(name);
+    }
+  };
+  for (const [field, parameter] of [["content-disposition", "filename"], ["content-type", "name"]]) {
+    const values = headers.get(field).slice(0, MAX_NAME_FIELDS);
+    for (const value of values) {
+      const parameters = parametersOf(value).slice(0, MAX_NAME_PARAMETERS).map((text) => ({ text, key: text.slice(0, Math.max(0, text.indexOf("="))).trim().toLowerCase() }));
+      const repeated = parameters.filter(({ key }) => (key === parameter || key.startsWith(`${parameter}*`)) && parameters.filter((other) => other.key === key).length > 1);
+      if (values.length > 1) {
+        read(value, parameter);
+      }
+      for (const { text } of repeated) {
+        read(`x;${text}`, text.slice(0, text.indexOf("=")).trim().toLowerCase().replace(/\*.*$/, ""));
+      }
+    }
+  }
+  return names;
+}
+function namedTextParts(mail) {
+  return mail[NAMED_TEXT] ?? [];
+}
 async function simpleParse(raw, { timeout = PARSE_TIMEOUT, parser = simpleParser } = {}) {
   const input = Readable.from([finishable(unfoldHeaders(raw))], { objectMode: false, autoDestroy: false });
   input.on("error", () => {
   });
+  const named = [];
   const { pipe } = input;
   input.pipe = function(parser2, options) {
     parser2.splitter?.on("error", (error) => input.destroy(error));
+    const { createNode } = parser2;
+    if (typeof createNode === "function") {
+      parser2.createNode = function(node) {
+        const created = createNode.call(this, node);
+        if (created?.isAttachment === false && node?.filename) {
+          addFinding(named, { filename: String(node.filename), contentType: created.contentType, content: null });
+        }
+        for (const filename of repeatedNames(node)) {
+          if (filename !== node?.filename) {
+            addFinding(named, { filename, contentType: created?.contentType || "application/octet-stream", content: null });
+          }
+        }
+        return created;
+      };
+    }
     return pipe.call(this, parser2, options);
   };
   let timer;
   try {
-    return await Promise.race([
+    const mail = await Promise.race([
       parser(input, PARSER_OPTIONS),
       new Promise((resolve, reject) => {
         timer = setTimeout(() => {
@@ -3956,6 +4071,17 @@ async function simpleParse(raw, { timeout = PARSE_TIMEOUT, parser = simpleParser
         }, timeout);
       })
     ]);
+    if (mail && typeof mail === "object") {
+      let count = 0;
+      for (const [, filename] of String(mail.text ?? "").matchAll(ENCODED_FILES)) {
+        addFinding(named, { filename: filename.trim(), contentType: "application/octet-stream", content: null });
+        if (++count >= MAX_ENCODED_FILES) {
+          break;
+        }
+      }
+      Object.defineProperty(mail, NAMED_TEXT, { value: named });
+    }
+    return mail;
   } finally {
     clearTimeout(timer);
     input.destroy();
@@ -4088,6 +4214,9 @@ function unfoldHeaders(raw) {
   chunks.push(raw.subarray(copied));
   return Buffer2.concat(chunks);
 }
+function lineCost(line) {
+  return Math.max(line.length, MIN_LINE_COST);
+}
 function upToLine(raw, offset) {
   return raw.subarray(0, Math.max(raw.lastIndexOf(10, offset - 1) + 1, 0));
 }
@@ -4114,9 +4243,9 @@ function smallerHeader(raw) {
     const whole = text.slice(start, stop).replace(/\r$/, "");
     if (last && (whole.startsWith(" ") || whole.startsWith("	"))) {
       const line = last.address ? whole : whole.slice(0, MAX_LINE);
-      if (last.address || last.size + line.length <= MAX_FIELD) {
+      if (last.address || last.size + lineCost(line) <= MAX_FIELD) {
         last.lines.push(line);
-        last.size += line.length;
+        last.size += lineCost(line);
       }
     } else {
       const name = whole.slice(0, whole.indexOf(":")).trim().toLowerCase();
@@ -4126,7 +4255,7 @@ function smallerHeader(raw) {
         name,
         address: address2,
         lines: [line],
-        size: line.length
+        size: lineCost(line)
       });
     }
     start = stop + 1;
@@ -4137,7 +4266,7 @@ function smallerHeader(raw) {
     const bounded = boundAddressField(line, Number.POSITIVE_INFINITY);
     fromCut ||= field.name === "from" && bounded !== line;
     field.lines = [bounded];
-    field.size = bounded.length;
+    field.size = lineCost(bounded);
   }
   const total = fields.reduce((sum, field) => sum + field.size, 0);
   const counts = /* @__PURE__ */ new Map();
@@ -4170,7 +4299,7 @@ async function parseWithin(raw, parse = parseMessage, deadline = Date.now() + PA
         shorter = fewerParts(input);
       } else if (limit === HEADER_TOO_LARGE) {
         const smaller = smallerHeader(input);
-        shorter = smaller.raw.length < input.length ? smaller.raw : upToLine(input, MAX_PREFIX_BYTES);
+        shorter = smaller.raw.length < input.length ? smaller.raw : upToLine(input, input.length > MAX_PREFIX_BYTES ? MAX_PREFIX_BYTES : Math.floor(input.length / 2));
         fromCut ||= smaller.raw === shorter && smaller.fromCut;
       }
       if (shorter.length >= input.length) {
@@ -4182,7 +4311,13 @@ async function parseWithin(raw, parse = parseMessage, deadline = Date.now() + PA
   }
 }
 async function expandAttachments(attachments = [], errors = [], parse = parseMessage, gets = {}) {
-  const { unopened = [], mails = [] } = Array.isArray(gets) ? { unopened: gets } : gets ?? {};
+  const { unopened = [], mails = [], named = [] } = Array.isArray(gets) ? { unopened: gets } : gets ?? {};
+  const addNamed = (...parts) => {
+    for (const part of parts) {
+      addFinding(named, part);
+    }
+  };
+  const nameOf = (value) => trimEndOf(String(value ?? "").trim().toLowerCase(), ". ");
   const all = [];
   const deadline = Date.now() + PARSE_TIMEOUT;
   let budget = MAX_NESTED_BYTES;
@@ -4204,6 +4339,10 @@ async function expandAttachments(attachments = [], errors = [], parse = parseMes
       const name = attachment.filename || (prefix ? "unnamed attachment" : "");
       const filename = prefix ? `${prefix}/${name}` : name || "attached message";
       all.push(prefix ? { ...attachment, filename: `${prefix}/${name}` } : attachment);
+      const typeName = String(attachment.headers?.get?.("content-type")?.params?.name ?? "").trim();
+      if (typeName && nameOf(typeName) !== nameOf(attachment.filename)) {
+        addNamed({ filename: prefix ? `${prefix}/${typeName}` : typeName, contentType: attachment.contentType, content: null });
+      }
       const { content } = attachment;
       const attached = isAttachedMessage(attachment);
       if (attached && Buffer2.isBuffer(content) && depth >= MAX_NESTED_DEPTH) {
@@ -4232,7 +4371,9 @@ async function expandAttachments(attachments = [], errors = [], parse = parseMes
           errors.push(inner.error);
         }
         mails.push(inner.mail);
-        await visit(inner.mail.attachments, prefix ? `${prefix}/${name}` : name || "attached message", depth + 1);
+        const carrier = prefix ? `${prefix}/${name}` : name || "attached message";
+        addNamed(...namedTextParts(inner.mail).map((part) => ({ ...part, filename: `${carrier}/${part.filename}` })));
+        await visit(inner.mail.attachments, carrier, depth + 1);
       }
     }
   };
@@ -6354,7 +6495,7 @@ function withoutBinaryValues(text, limit = Number.POSITIVE_INFINITY) {
   }
   return out.join("\n");
 }
-var HTML_TYPES = /* @__PURE__ */ new Set(["text/html", "application/xhtml+xml", "image/svg+xml", "text/xml", "application/xml"]);
+var HTML_TYPES = /* @__PURE__ */ new Set(["text/html", "text/x-amp-html", "application/xhtml+xml", "image/svg+xml", "text/xml", "application/xml"]);
 var HTML_EXTENSIONS = /* @__PURE__ */ new Set(["html", "htm", "shtml", "xhtml", "xht", "svg", "xml"]);
 var CARD_TYPES = /* @__PURE__ */ new Set(["text/calendar", "text/x-vcalendar", "application/ics", "text/vcard", "text/x-vcard", "text/directory"]);
 var CARD_EXTENSIONS = /* @__PURE__ */ new Set(["ics", "ical", "ifb", "vcs", "vcf", "vcard"]);
@@ -7609,18 +7750,27 @@ function parseVerdict(text) {
     sources.push([text.slice(0, 2e4), answered]);
   }
   for (const [source, wanted] of sources) {
+    let found = null;
+    let end = -1;
     for (const candidate of jsonObjects(source)) {
-      if (!wanted(candidate)) {
+      if (!wanted(candidate) || candidate[0] < end) {
         continue;
       }
       try {
         const result = normalizeVerdict(JSON.parse(candidate[1]));
         if (result) {
-          return result;
+          if (found && found.verdict !== result.verdict) {
+            return null;
+          }
+          found ??= result;
+          end = candidate[0] + candidate[1].length;
         }
       } catch {
       }
     }
+    if (found) {
+      return found;
+    }
   }
   const prose = visible.toLowerCase();
   let deniedEnd = -1;
@@ -8319,7 +8469,7 @@ function scoreResults(results, options) {
 var package_default = {
   name: "spamscanner",
   description: "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
-  version: "7.0.14",
+  version: "7.0.15",
   author: "Forward Email (https://forwardemail.net)",
   bin: {
     spamscanner: "./dist/esm/cli.js"
@@ -10523,14 +10673,15 @@ ${visibleText(html.slice(0, config.maxLength * 2)).text}`, { maxLength: config.m
     };
     const unopened = [];
     const opened = [];
-    const attached = await expandAttachments(mail.attachments, parseErrors, void 0, { unopened, mails: opened });
+    const named = [];
+    const attached = await expandAttachments(mail.attachments, parseErrors, void 0, { unopened, mails: opened, named });
     const carried = config.phishing.attachments === true && (config.phishing.cloudflare || config.dnsbl.domain?.length > 0) ? carriedLinkHosts(mail, {
       mails: opened,
       attachments: attached,
       maxHosts: config.phishing.maxHosts,
       hostsInText: config.phishing.hostsInText === true
     }) : null;
-    const attachments = config.attachments === false ? [] : inspectAttachments(attached).filter((item) => config.macros !== false || !["macro", "pdf_active", "rtf_object"].includes(item.type)).map((item) => finding(item));
+    const attachments = config.attachments === false ? [] : inspectAttachments([...attached, ...namedTextParts(mail), ...named]).filter((item) => config.macros !== false || !["macro", "pdf_active", "rtf_object"].includes(item.type)).map((item) => finding(item));
     const [links, authentication, checkedReputation, dnsbl, virusScan, toxicity, nsfw] = await Promise.all([
       this.checkLinks(checkedLinks, config, extracted.hosts, { from: fromDomain(mail), carried }),
       config.authenticationResults && typeof config.authenticationResults === "object" ? normalizeAuthOutput(config.authenticationResults) : config.authentication && session.remoteAddress ? withTimeout(authenticate(raw, {