spamscanner 7.0.1 → 7.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/esm/index.js CHANGED
@@ -7,9 +7,581 @@ import { simpleParser as simpleParser2 } from "mailparser";
7
7
  // src/attachments.js
8
8
  import { Buffer } from "node:buffer";
9
9
 
10
+ // src/linear.js
11
+ import { createRequire } from "node:module";
12
+ function loadRe2(load = createRequire(import.meta.url)) {
13
+ try {
14
+ return load("re2");
15
+ } catch {
16
+ return null;
17
+ }
18
+ }
19
+ var RE2 = loadRe2();
20
+ var BOUNDED = /{([01]),(\d{2,})}/g;
21
+ var WHITESPACE = String.raw`\t\n\x0B\f\r \x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}`;
22
+ function re2Source(source) {
23
+ let out = "";
24
+ let inClass = false;
25
+ for (let i = 0; i < source.length; i++) {
26
+ const char = source[i];
27
+ if (char === "\\") {
28
+ const next = source[++i] ?? "";
29
+ if (next === "s") {
30
+ out += inClass ? WHITESPACE : `[${WHITESPACE}]`;
31
+ } else if (next === "S") {
32
+ if (inClass) {
33
+ throw new TypeError(String.raw`\S inside a character class has no RE2 equivalent here`);
34
+ }
35
+ out += `[^${WHITESPACE}]`;
36
+ } else {
37
+ out += `\\${next}`;
38
+ }
39
+ continue;
40
+ }
41
+ if (char === "[" && !inClass) {
42
+ inClass = true;
43
+ } else if (char === "]" && inClass) {
44
+ inClass = false;
45
+ }
46
+ out += char;
47
+ }
48
+ return out.replaceAll(BOUNDED, (_, min) => min === "0" ? "*" : "+");
49
+ }
50
+ function linear(source, flags = "", Engine = RE2) {
51
+ return Engine ? new Engine(re2Source(source), flags) : new RegExp(source, flags);
52
+ }
53
+ function trimEndOf(value, characters) {
54
+ let end = value.length;
55
+ while (end > 0 && characters.includes(value[end - 1])) {
56
+ end--;
57
+ }
58
+ return value.slice(0, end);
59
+ }
60
+ function replaceTags(html, replacement = " ") {
61
+ let out = "";
62
+ let from = 0;
63
+ let open = html.indexOf("<");
64
+ while (open !== -1) {
65
+ const close = html.indexOf(">", open + 1);
66
+ if (close === -1) {
67
+ break;
68
+ }
69
+ out += `${html.slice(from, open)}${replacement}`;
70
+ from = close + 1;
71
+ open = html.indexOf("<", from);
72
+ }
73
+ return out + html.slice(from);
74
+ }
75
+ function caseless(word) {
76
+ return [...word].map((letter) => `[${letter}${letter.toUpperCase()}]`).join("");
77
+ }
78
+ var FOLDS_TO_ASCII = /[\u017F\u212A]/g;
79
+ function foldToAscii(text) {
80
+ return text.replaceAll(FOLDS_TO_ASCII, (character) => character === "\u017F" ? "s" : "k");
81
+ }
82
+ var OPENER = linear(String.raw`<(${caseless("script")}|${caseless("style")})\b`, "g");
83
+ var CLOSERS = {
84
+ script: linear(String.raw`<\/${caseless("script")}\s*>`, "g"),
85
+ style: linear(String.raw`<\/${caseless("style")}\s*>`, "g")
86
+ };
87
+
10
88
  // src/tokenizer.js
11
89
  import { parse as parseDomain } from "tldts";
12
90
 
91
+ // src/html.js
92
+ var LATIN1 = "nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr deg plusmn sup2 sup3 acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest Agrave Aacute Acirc Atilde Auml Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ETH Ntilde Ograve Oacute Ocirc Otilde Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig agrave aacute acirc atilde auml aring aelig ccedil egrave eacute ecirc euml igrave iacute icirc iuml eth ntilde ograve oacute ocirc otilde ouml divide oslash ugrave uacute ucirc uuml yacute thorn yuml".split(" ");
93
+ var ENTITIES = new Map([
94
+ ...LATIN1.map((name, index) => [name, String.fromCodePoint(160 + index)]),
95
+ ["amp", "&"],
96
+ ["lt", "<"],
97
+ ["gt", ">"],
98
+ ["quot", '"'],
99
+ ["apos", "'"],
100
+ ["AMP", "&"],
101
+ ["LT", "<"],
102
+ ["GT", ">"],
103
+ ["QUOT", '"'],
104
+ ["COPY", "\xA9"],
105
+ ["REG", "\xAE"],
106
+ ["OElig", "\u0152"],
107
+ ["oelig", "\u0153"],
108
+ ["Scaron", "\u0160"],
109
+ ["scaron", "\u0161"],
110
+ ["Yuml", "\u0178"],
111
+ ["fnof", "\u0192"],
112
+ ["circ", "\u02C6"],
113
+ ["tilde", "\u02DC"],
114
+ ["ensp", "\u2002"],
115
+ ["emsp", "\u2003"],
116
+ ["thinsp", "\u2009"],
117
+ ["zwnj", "\u200C"],
118
+ ["zwj", "\u200D"],
119
+ ["lrm", "\u200E"],
120
+ ["rlm", "\u200F"],
121
+ ["ndash", "\u2013"],
122
+ ["mdash", "\u2014"],
123
+ ["lsquo", "\u2018"],
124
+ ["rsquo", "\u2019"],
125
+ ["sbquo", "\u201A"],
126
+ ["ldquo", "\u201C"],
127
+ ["rdquo", "\u201D"],
128
+ ["bdquo", "\u201E"],
129
+ ["dagger", "\u2020"],
130
+ ["Dagger", "\u2021"],
131
+ ["bull", "\u2022"],
132
+ ["hellip", "\u2026"],
133
+ ["permil", "\u2030"],
134
+ ["prime", "\u2032"],
135
+ ["Prime", "\u2033"],
136
+ ["lsaquo", "\u2039"],
137
+ ["rsaquo", "\u203A"],
138
+ ["euro", "\u20AC"],
139
+ ["trade", "\u2122"],
140
+ ["larr", "\u2190"],
141
+ ["uarr", "\u2191"],
142
+ ["rarr", "\u2192"],
143
+ ["darr", "\u2193"],
144
+ ["hearts", "\u2665"],
145
+ ["check", "\u2713"]
146
+ ]);
147
+ var LEGACY = [...LATIN1, "amp", "lt", "gt", "quot", "AMP", "LT", "GT", "QUOT", "COPY", "REG"].sort((a, b) => b.length - a.length);
148
+ var WINDOWS_1252 = [8364, 129, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338, 141, 381, 143, 144, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482, 353, 8250, 339, 157, 382, 376];
149
+ var REFERENCE = /&(?:#([xX])([\da-fA-F]+)|#(\d+)|([a-zA-Z][a-zA-Z\d]{0,31}))(;?)/g;
150
+ function fromCode(code) {
151
+ if (code === 0 || code > 1114111 || code >= 55296 && code <= 57343) {
152
+ return "\uFFFD";
153
+ }
154
+ if (code >= 128 && code <= 159) {
155
+ return String.fromCodePoint(WINDOWS_1252[code - 128]);
156
+ }
157
+ return String.fromCodePoint(code);
158
+ }
159
+ function decodeEntities(value, { attribute = false } = {}) {
160
+ return value.replaceAll(REFERENCE, (...found) => {
161
+ const [match, , hex, decimal, name, semicolon, offset] = found;
162
+ if (hex !== void 0 || decimal !== void 0) {
163
+ return fromCode(hex === void 0 ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
164
+ }
165
+ if (semicolon) {
166
+ return ENTITIES.get(name) ?? match;
167
+ }
168
+ const legacy = LEGACY.find((known) => name.startsWith(known));
169
+ if (!legacy) {
170
+ return match;
171
+ }
172
+ const rest = match.slice(legacy.length + 1);
173
+ const next = rest[0] ?? value[offset + match.length] ?? "";
174
+ if (attribute && /[\da-zA-Z=]/.test(next)) {
175
+ return match;
176
+ }
177
+ return ENTITIES.get(legacy) + rest;
178
+ });
179
+ }
180
+ var UNSHOWN = /* @__PURE__ */ new Set(["script", "style", "title", "noembed", "noframes", "iframe"]);
181
+ var RAW_SHOWN = /* @__PURE__ */ new Set(["textarea", "xmp"]);
182
+ var RAW_END = Object.fromEntries([...UNSHOWN, ...RAW_SHOWN].map((name) => [name, linear(String.raw`<\/${caseless(name)}[\s/>]`, "g")]));
183
+ var BREAKS = /* @__PURE__ */ new Set(["address", "applet", "area", "article", "aside", "audio", "blockquote", "body", "br", "button", "canvas", "caption", "center", "col", "colgroup", "dd", "details", "dialog", "dir", "div", "dl", "dt", "embed", "fieldset", "figcaption", "figure", "footer", "form", "frame", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hgroup", "hr", "html", "iframe", "image", "img", "input", "keygen", "legend", "li", "listing", "main", "marquee", "math", "menu", "meter", "nav", "object", "ol", "optgroup", "option", "p", "picture", "plaintext", "pre", "progress", "section", "select", "summary", "svg", "table", "tbody", "td", "textarea", "tfoot", "th", "thead", "tr", "ul", "video", "xmp"]);
184
+ var SPECIAL = /* @__PURE__ */ new Set(["address", "applet", "area", "article", "aside", "base", "blockquote", "body", "button", "caption", "center", "col", "colgroup", "dd", "details", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hgroup", "html", "iframe", "li", "listing", "main", "marquee", "menu", "nav", "object", "ol", "p", "plaintext", "pre", "section", "select", "summary", "table", "tbody", "td", "template", "textarea", "tfoot", "th", "thead", "tr", "ul", "xmp"]);
185
+ var FORMATTING = /* @__PURE__ */ new Set(["a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u"]);
186
+ var LINE = /* @__PURE__ */ new Set(["br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "table", "blockquote", "pre"]);
187
+ var VOID = /* @__PURE__ */ new Set(["area", "base", "basefont", "bgsound", "br", "col", "command", "embed", "frame", "hr", "image", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"]);
188
+ var CLOSES_PARAGRAPH = /* @__PURE__ */ new Set(["address", "article", "aside", "blockquote", "center", "dd", "details", "dialog", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "li", "main", "menu", "nav", "ol", "p", "pre", "section", "table", "ul", "xmp"]);
189
+ var PARAGRAPH = [/* @__PURE__ */ new Set(["p"]), /* @__PURE__ */ new Set(["button", "table", "td", "th", "caption", "marquee", "object", "applet", "html"])];
190
+ var IMPLIED = {
191
+ li: [/* @__PURE__ */ new Set(["li"]), /* @__PURE__ */ new Set(["ul", "ol", "menu"])],
192
+ dt: [/* @__PURE__ */ new Set(["dt", "dd"]), /* @__PURE__ */ new Set(["dl"])],
193
+ dd: [/* @__PURE__ */ new Set(["dt", "dd"]), /* @__PURE__ */ new Set(["dl"])],
194
+ td: [/* @__PURE__ */ new Set(["td", "th"]), /* @__PURE__ */ new Set(["tr", "table"])],
195
+ th: [/* @__PURE__ */ new Set(["td", "th"]), /* @__PURE__ */ new Set(["tr", "table"])],
196
+ tr: [/* @__PURE__ */ new Set(["tr"]), /* @__PURE__ */ new Set(["table", "tbody", "thead", "tfoot"])],
197
+ tbody: [/* @__PURE__ */ new Set(["tbody", "thead", "tfoot"]), /* @__PURE__ */ new Set(["table"])],
198
+ thead: [/* @__PURE__ */ new Set(["tbody", "thead", "tfoot"]), /* @__PURE__ */ new Set(["table"])],
199
+ tfoot: [/* @__PURE__ */ new Set(["tbody", "thead", "tfoot"]), /* @__PURE__ */ new Set(["table"])],
200
+ option: [/* @__PURE__ */ new Set(["option"]), /* @__PURE__ */ new Set(["select", "datalist", "optgroup"])],
201
+ optgroup: [/* @__PURE__ */ new Set(["optgroup", "option"]), /* @__PURE__ */ new Set(["select"])]
202
+ };
203
+ var MAX_OPEN = 1024;
204
+ var MAX_SEARCH = 64;
205
+ var MAX_ATTRIBUTES = 64;
206
+ var MAX_STYLE = 1e4;
207
+ var COMMENT_END = linear("--!?>", "g");
208
+ var WHITESPACE2 = /* @__PURE__ */ new Set([" ", " ", "\n", "\r", "\f"]);
209
+ function readTag(html, start) {
210
+ const closing = html[start + 1] === "/";
211
+ let i = start + (closing ? 2 : 1);
212
+ const nameStart = i;
213
+ while (i < html.length && !WHITESPACE2.has(html[i]) && html[i] !== "/" && html[i] !== ">") {
214
+ i++;
215
+ }
216
+ const name = html.slice(nameStart, Math.min(i, nameStart + 64)).toLowerCase();
217
+ const attributes = /* @__PURE__ */ new Map();
218
+ let slash = false;
219
+ while (i < html.length) {
220
+ const char = html[i];
221
+ if (char === ">") {
222
+ return {
223
+ end: i + 1,
224
+ name,
225
+ closing,
226
+ attributes,
227
+ selfClosing: slash
228
+ };
229
+ }
230
+ if (WHITESPACE2.has(char) || char === "/") {
231
+ slash = char === "/" || slash && WHITESPACE2.has(char);
232
+ i++;
233
+ continue;
234
+ }
235
+ slash = false;
236
+ const attributeStart = i++;
237
+ while (i < html.length && !WHITESPACE2.has(html[i]) && html[i] !== "/" && html[i] !== ">" && html[i] !== "=") {
238
+ i++;
239
+ }
240
+ const attribute = html.slice(attributeStart, i).toLowerCase();
241
+ while (WHITESPACE2.has(html[i])) {
242
+ i++;
243
+ }
244
+ let value = "";
245
+ if (html[i] === "=") {
246
+ i++;
247
+ while (WHITESPACE2.has(html[i])) {
248
+ i++;
249
+ }
250
+ if (html[i] === '"' || html[i] === "'") {
251
+ const close = html.indexOf(html[i], i + 1);
252
+ if (close === -1) {
253
+ break;
254
+ }
255
+ value = html.slice(i + 1, close);
256
+ i = close + 1;
257
+ } else {
258
+ const valueStart = i;
259
+ while (i < html.length && !WHITESPACE2.has(html[i]) && html[i] !== ">") {
260
+ i++;
261
+ }
262
+ value = html.slice(valueStart, i);
263
+ }
264
+ }
265
+ if (!attributes.has(attribute) && attributes.size < MAX_ATTRIBUTES) {
266
+ attributes.set(attribute, value);
267
+ }
268
+ }
269
+ return { end: -1 };
270
+ }
271
+ function declarations(style) {
272
+ let css = "";
273
+ let from = 0;
274
+ const text = style.slice(0, MAX_STYLE);
275
+ while (from < text.length) {
276
+ const open = text.indexOf("/*", from);
277
+ if (open === -1) {
278
+ css += text.slice(from);
279
+ break;
280
+ }
281
+ css += text.slice(from, open);
282
+ const close = text.indexOf("*/", open + 2);
283
+ if (close === -1) {
284
+ break;
285
+ }
286
+ from = close + 2;
287
+ }
288
+ const found = /* @__PURE__ */ new Map();
289
+ for (const declaration of css.split(";")) {
290
+ const colon = declaration.indexOf(":");
291
+ if (colon === -1) {
292
+ continue;
293
+ }
294
+ const property = declaration.slice(0, colon).trim().toLowerCase();
295
+ let value = declaration.slice(colon + 1).trim().toLowerCase();
296
+ const important = /!\s*important$/.test(value);
297
+ if (important) {
298
+ value = value.replace(/!\s*important$/, "").trim();
299
+ }
300
+ const before = found.get(property);
301
+ if (!before || important || !before.important) {
302
+ found.set(property, { value, important });
303
+ }
304
+ }
305
+ return found;
306
+ }
307
+ var MAX_VALUE = 64;
308
+ var LENGTH = /^([+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)(px|pt|pc|in|cm|mm|q|em|rem|%|ex|ch|)$/;
309
+ var UNITS = {
310
+ px: 1,
311
+ pt: 4 / 3,
312
+ pc: 16,
313
+ in: 96,
314
+ cm: 96 / 2.54,
315
+ mm: 96 / 25.4,
316
+ q: 96 / 101.6,
317
+ rem: 16
318
+ };
319
+ var SIZE_KEYWORDS = {
320
+ "xx-small": 9,
321
+ "x-small": 10,
322
+ small: 13,
323
+ medium: 16,
324
+ large: 18,
325
+ "x-large": 24,
326
+ "xx-large": 32,
327
+ "xxx-large": 48
328
+ };
329
+ function pixels(value, parent) {
330
+ const length = value.length > MAX_VALUE ? null : LENGTH.exec(value);
331
+ if (!length) {
332
+ return null;
333
+ }
334
+ const number = Number(length[1]);
335
+ if (!Number.isFinite(number)) {
336
+ return null;
337
+ }
338
+ const unit = length[2];
339
+ if (unit === "em") {
340
+ return number * parent;
341
+ }
342
+ if (unit === "%") {
343
+ return number * parent / 100;
344
+ }
345
+ if (unit === "ex" || unit === "ch") {
346
+ return number * parent / 2;
347
+ }
348
+ return number * (UNITS[unit] ?? 1);
349
+ }
350
+ function fontSize(value, parent) {
351
+ if (Object.hasOwn(SIZE_KEYWORDS, value)) {
352
+ return SIZE_KEYWORDS[value];
353
+ }
354
+ if (value === "smaller") {
355
+ return parent / 1.2;
356
+ }
357
+ if (value === "larger") {
358
+ return parent * 1.2;
359
+ }
360
+ const size = pixels(value, parent);
361
+ return size === null || size < 0 ? null : size;
362
+ }
363
+ function shorthandSize(value, parent) {
364
+ for (const token of value.slice(0, MAX_STYLE).split(/\s+/)) {
365
+ const [size, lineHeight] = token.split("/");
366
+ if (lineHeight === void 0 && /^[+-]?[\d.]+$/.test(size)) {
367
+ continue;
368
+ }
369
+ const pixelSize = fontSize(size, parent);
370
+ if (pixelSize !== null) {
371
+ return pixelSize;
372
+ }
373
+ }
374
+ return null;
375
+ }
376
+ function isTransparent(color) {
377
+ if (color === "transparent") {
378
+ return true;
379
+ }
380
+ const hex = /^#(?:[\da-f]{3}(0)|[\da-f]{6}(00))$/.exec(color);
381
+ if (hex) {
382
+ return true;
383
+ }
384
+ const functional = /^(?:rgb|hsl)a?\((.*)\)$/.exec(color);
385
+ if (!functional) {
386
+ return false;
387
+ }
388
+ const slash = functional[1].lastIndexOf("/");
389
+ const parts = functional[1].split(",");
390
+ if (slash === -1 && parts.length < 4) {
391
+ return false;
392
+ }
393
+ const alpha = (slash === -1 ? parts.at(-1) : functional[1].slice(slash + 1)).trim();
394
+ return ZERO_NUMBER.test(alpha);
395
+ }
396
+ var ZERO_NUMBER = /^[+-]?(?:0+(?:\.0*)?|\.0+)%?$/;
397
+ var isZero = (value) => {
398
+ const size = pixels(value, 16);
399
+ return size !== null && size <= 0;
400
+ };
401
+ function styleState(style, parent) {
402
+ const css = declarations(style);
403
+ const value = (property) => css.get(property)?.value;
404
+ const opacity = value("opacity");
405
+ const overflow = [value("overflow"), value("overflow-x"), value("overflow-y")].some((item) => item === "hidden" || item === "clip");
406
+ const noHeight = (isZero(value("height") ?? "1") || isZero(value("max-height") ?? "1")) && isZero(value("min-height") ?? "0");
407
+ const noWidth = (isZero(value("width") ?? "1") || isZero(value("max-width") ?? "1")) && isZero(value("min-width") ?? "0");
408
+ const indent = pixels(value("text-indent") ?? "0", parent.size);
409
+ const offPage = ["absolute", "fixed"].includes(value("position")) && ["left", "top"].some((side) => (pixels(value(side) ?? "0", parent.size) ?? 0) <= -999);
410
+ const hides = value("display") === "none" || opacity !== void 0 && (ZERO_NUMBER.test(opacity) || opacity.length <= MAX_VALUE && Number.parseFloat(opacity) < 0) || overflow && (noHeight || noWidth) || indent !== null && indent <= -999 || offPage;
411
+ const visibility = value("visibility");
412
+ const color = value("color");
413
+ const size = value("font-size") === void 0 ? value("font") === void 0 ? null : shorthandSize(value("font"), parent.size) : fontSize(value("font-size"), parent.size);
414
+ return {
415
+ hides,
416
+ display: value("display"),
417
+ visible: visibility === "hidden" || visibility === "collapse" ? false : visibility === "visible" ? true : parent.visible,
418
+ transparent: color === void 0 ? parent.transparent : isTransparent(color),
419
+ size: size ?? parent.size
420
+ };
421
+ }
422
+ var ROOT = {
423
+ visible: true,
424
+ transparent: false,
425
+ size: 16,
426
+ hides: false
427
+ };
428
+ function visibleText(html) {
429
+ if (typeof html !== "string" || html === "") {
430
+ return { text: "", hidden: 0 };
431
+ }
432
+ const parts = [];
433
+ let hidden = 0;
434
+ const stack = [];
435
+ const open = /* @__PURE__ */ new Map();
436
+ let hiding = 0;
437
+ let collapsed = 0;
438
+ const state = () => stack.at(-1) ?? ROOT;
439
+ const shown = () => hiding === 0 && collapsed === 0 && state().visible && !state().transparent && state().size > 1;
440
+ const write = (text) => {
441
+ if (text === "") {
442
+ return;
443
+ }
444
+ if (shown()) {
445
+ parts.push(text);
446
+ } else {
447
+ hidden += text.replaceAll(/\s+/g, "").length;
448
+ }
449
+ };
450
+ const popTo = (index) => {
451
+ while (stack.length > index) {
452
+ const element = stack.pop();
453
+ open.set(element.name, open.get(element.name) - 1);
454
+ if (element.hides) {
455
+ hiding--;
456
+ }
457
+ collapsed -= (element.collapses ? 1 : 0) - (element.reveals ? 1 : 0);
458
+ }
459
+ };
460
+ const closeImplied = ([names, limits]) => {
461
+ for (let i = stack.length - 1; i >= Math.max(0, stack.length - MAX_SEARCH); i--) {
462
+ if (names.has(stack[i].name)) {
463
+ popTo(i);
464
+ return;
465
+ }
466
+ if (limits.has(stack[i].name)) {
467
+ return;
468
+ }
469
+ }
470
+ };
471
+ let from = 0;
472
+ while (from < html.length) {
473
+ const lt = html.indexOf("<", from);
474
+ if (lt === -1) {
475
+ write(decodeEntities(html.slice(from)));
476
+ break;
477
+ }
478
+ write(decodeEntities(html.slice(from, lt)));
479
+ from = lt + 1;
480
+ const next = html[lt + 1] ?? "";
481
+ if (html.startsWith("<!--", lt)) {
482
+ if (html.startsWith(">", lt + 4) || html.startsWith("->", lt + 4)) {
483
+ from = html.indexOf(">", lt + 4) + 1;
484
+ continue;
485
+ }
486
+ COMMENT_END.lastIndex = lt + 4;
487
+ const end = COMMENT_END.exec(html);
488
+ from = end === null ? html.length : end.index + end[0].length;
489
+ continue;
490
+ }
491
+ const closingTag = next === "/";
492
+ const letter = /[a-zA-Z]/.test(closingTag ? html[lt + 2] ?? "" : next);
493
+ if (!letter) {
494
+ if (closingTag && lt + 2 >= html.length) {
495
+ write("</");
496
+ break;
497
+ }
498
+ if (next === "!" || next === "?" || closingTag && html[lt + 2] !== ">") {
499
+ const end = html.indexOf(">", lt + 2);
500
+ from = end === -1 ? html.length : end + 1;
501
+ } else if (closingTag) {
502
+ from = lt + 3;
503
+ } else {
504
+ write("<");
505
+ }
506
+ continue;
507
+ }
508
+ const tag = readTag(html, lt);
509
+ if (tag.end === -1) {
510
+ break;
511
+ }
512
+ from = tag.end;
513
+ const { name } = tag;
514
+ if (tag.closing) {
515
+ const through = SPECIAL.has(name) || FORMATTING.has(name);
516
+ const limit = through ? 0 : Math.max(0, stack.length - MAX_SEARCH);
517
+ for (let i = stack.length - 1; open.get(name) > 0 && i >= limit; i--) {
518
+ if (stack[i].name === name) {
519
+ popTo(i);
520
+ break;
521
+ }
522
+ if (!through && SPECIAL.has(stack[i].name)) {
523
+ break;
524
+ }
525
+ }
526
+ } else {
527
+ if (CLOSES_PARAGRAPH.has(name)) {
528
+ closeImplied(PARAGRAPH);
529
+ }
530
+ if (IMPLIED[name]) {
531
+ closeImplied(IMPLIED[name]);
532
+ }
533
+ const foreign = (open.get("svg") ?? 0) + (open.get("math") ?? 0) > 0;
534
+ if (!VOID.has(name) && !(foreign && tag.selfClosing)) {
535
+ if (stack.length >= MAX_OPEN) {
536
+ popTo(stack.length - 1);
537
+ }
538
+ const style = tag.attributes.get("style");
539
+ const parent = state();
540
+ const element = {
541
+ ...style === void 0 ? {
542
+ visible: parent.visible,
543
+ transparent: parent.transparent,
544
+ size: parent.size,
545
+ hides: false
546
+ } : styleState(decodeEntities(style, { attribute: true }), parent),
547
+ name
548
+ };
549
+ if (tag.attributes.has("hidden") && (element.display === void 0 || element.display === "none") || name === "template") {
550
+ element.hides = true;
551
+ }
552
+ element.collapses = name === "details" && !tag.attributes.has("open");
553
+ element.reveals = name === "summary" && parent.collapses === true;
554
+ stack.push(element);
555
+ open.set(name, (open.get(name) ?? 0) + 1);
556
+ if (element.hides) {
557
+ hiding++;
558
+ }
559
+ collapsed += (element.collapses ? 1 : 0) - (element.reveals ? 1 : 0);
560
+ }
561
+ if (name === "plaintext") {
562
+ parts.push("\n");
563
+ write(html.slice(tag.end));
564
+ break;
565
+ }
566
+ if ((UNSHOWN.has(name) || RAW_SHOWN.has(name)) && !(foreign && tag.selfClosing)) {
567
+ const closer = RAW_END[name];
568
+ closer.lastIndex = tag.end;
569
+ const close = closer.exec(html);
570
+ const end = close === null ? html.length : close.index;
571
+ if (RAW_SHOWN.has(name)) {
572
+ const raw = html.slice(tag.end, end);
573
+ write(name === "textarea" ? decodeEntities(raw) : raw);
574
+ }
575
+ from = end;
576
+ }
577
+ }
578
+ if (BREAKS.has(name)) {
579
+ parts.push(LINE.has(name) ? "\n" : " ");
580
+ }
581
+ }
582
+ return { text: parts.join(""), hidden };
583
+ }
584
+
13
585
  // src/language.js
14
586
  import { franc } from "franc";
15
587
  var SCRIPTS = [
@@ -415,13 +987,30 @@ var LEET = {
415
987
  8: "b"
416
988
  };
417
989
  var UNSPACED = /* @__PURE__ */ new Set(["Han", "Hiragana", "Katakana", "Thai", "Lao", "Khmer", "Myanmar"]);
418
- var URL_PATTERN = /\b(?:https?:\/\/|www\.)[^\s<>"'`]+|\b[a-z\d][a-z\d-]*(?:\.[a-z\d-]+)*\.[a-z]{2,24}\/[^\s<>"'`]*/giu;
419
- var EMAIL_PATTERN = new RegExp("[\\p{L}\\p{N}._%+-]+@[\\p{L}\\p{N}-]+(?:\\.[\\p{L}\\p{N}-]+)*\\.\\p{L}{2,24}", "gu");
420
- var PHONE_PATTERN = /(?:\+|0{2})\d[\d\s().-]{7,}\d|\(\d{3}\)\s?\d{3}[\s.-]\d{4}|\b(?:\d{3}[.-]){2}\d{4}\b/g;
421
- var IP_PATTERN = /\b(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}\b/g;
422
- var MONEY_PATTERN = /[$€£¥₹₽₩₺₦]\s?\d|\d\s?(?:[$€£¥₹₽₩₺₦]|usd|eur|gbp|dollars?|euros?)\b/giu;
423
- var BITCOIN_PATTERN = /\b(?:bc1[ac-hj-np-z02-9]{25,60}|[13][a-km-zA-HJ-NP-Z1-9]{25,34})\b/g;
424
- var CARD_PATTERN = /\b(?:\d[ -]?){13,19}\b/g;
990
+ var URL_PATTERN = linear(String.raw`\b(?:https?:\/\/|www\.)[^\s<>"'\x60]+|\b[a-z\d][a-z\d-]{0,62}(?:\.[a-z\d-]{1,63}){0,10}\.[a-z]{2,24}\/[^\s<>"'\x60]*`, "giu");
991
+ var EMAIL_SOURCE = String.raw`[\p{L}\p{N}._%+-]{1,64}@([\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63}){0,10}\.\p{L}{2,24})`;
992
+ var EMAIL_PATTERN = linear(EMAIL_SOURCE, "gu");
993
+ var PHONE_PATTERN = linear(String.raw`(?:\+|0{2})\d[\d\s().-]{7,}\d|\(\d{3}\)\s?\d{3}[\s.-]\d{4}|\b(?:\d{3}[.-]){2}\d{4}\b`, "g");
994
+ var IP_PATTERN = linear(String.raw`\b(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}\b`, "g");
995
+ var MONEY_PATTERN = linear(String.raw`[$€£¥₹₽₩₺₦]\s?\d|\d\s?(?:[$€£¥₹₽₩₺₦]|usd|eur|gbp|dollars?|euros?)\b`, "giu");
996
+ var BITCOIN_PATTERN = linear(String.raw`\b(?:bc1[ac-hj-np-z02-9]{25,60}|[13][a-km-zA-HJ-NP-Z1-9]{25,34})\b`, "g");
997
+ var CARD_PATTERN = linear(String.raw`\b(?:\d[ -]?){13,19}\b`, "g");
998
+ var ANCHOR_START = linear(String.raw`<a\b`, "gi");
999
+ var HREF = linear(String.raw`\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))`, "i");
1000
+ var ANCHOR_CLOSE = linear(String.raw`<\/a\s*>`, "gi");
1001
+ var SHOWN_URL = linear(String.raw`^(?:https?:\/\/|www\.)\S+$|^[a-z\d-]{1,63}(?:\.[a-z\d-]{1,63}){1,10}(?:\/\S*)?$`, "i");
1002
+ var IMG_PIXEL = linear(String.raw`<img\b[^<>]*\b(?:width|height)\s*=\s*["']?[01]["'\s>]`, "i");
1003
+ var IMG = linear(String.raw`<img\b`, "gi");
1004
+ var BARE_HOST = linear(String.raw`\b[a-z\d][a-z\d-]{0,62}(?:\.[a-z\d-]{1,63}){0,9}\.[a-z]{2,24}\b`, "gi");
1005
+ var MAX_BARE_HOSTS = 50;
1006
+ var NAME_END = /[\p{L}\p{N}\p{M}_]$/u;
1007
+ function continuesName(text, index) {
1008
+ if (NAME_END.test(text.slice(Math.max(0, index - 2), index))) {
1009
+ return true;
1010
+ }
1011
+ return (text[index - 1] === "." || text[index - 1] === "-") && NAME_END.test(text.slice(Math.max(0, index - 3), index - 1));
1012
+ }
1013
+ var FILE_TLDS = /* @__PURE__ */ new Set(["ai", "app", "md", "mov", "pm", "ps", "py", "rs", "sh", "so", "zip"]);
425
1014
  var SHORTENERS = /* @__PURE__ */ new Set(["bit.ly", "tinyurl.com", "goo.gl", "t.co", "ow.ly", "is.gd", "buff.ly", "rebrand.ly", "cutt.ly", "shorturl.at", "tiny.cc", "rb.gy", "bit.do", "v.gd", "s.id", "t.ly", "lnkd.in", "qrco.de", "shorturl.asia", "urlz.fr"]);
426
1015
  var FREEMAIL = /* @__PURE__ */ new Set(["gmail.com", "googlemail.com", "yahoo.com", "ymail.com", "hotmail.com", "outlook.com", "live.com", "msn.com", "aol.com", "icloud.com", "me.com", "mail.com", "gmx.com", "gmx.de", "gmx.net", "web.de", "yandex.ru", "yandex.com", "mail.ru", "qq.com", "163.com", "126.com", "proton.me", "protonmail.com", "zoho.com", "naver.com", "daum.net", "rediffmail.com", "libero.it", "orange.fr", "laposte.net", "wp.pl", "o2.pl", "seznam.cz", "rambler.ru", "tutanota.com", "tuta.com"]);
427
1016
  var segmenters = /* @__PURE__ */ new Map();
@@ -456,6 +1045,9 @@ function chunk(text, size = 1e3) {
456
1045
  }
457
1046
  return pieces;
458
1047
  }
1048
+ function readableText(text) {
1049
+ return typeof text === "string" ? text.replaceAll(OBFUSCATING, "").normalize("NFKC") : "";
1050
+ }
459
1051
  function normalizeText(text) {
460
1052
  if (typeof text !== "string" || text === "") {
461
1053
  return { text: "", invisible: 0, styled: false };
@@ -523,6 +1115,24 @@ function segmentWords(text, options = {}) {
523
1115
  leet
524
1116
  };
525
1117
  }
1118
+ var LATIN = new RegExp("\\p{Script=Latin}", "u");
1119
+ var CYRILLIC_OR_GREEK = /[\p{Script=Cyrillic}\p{Script=Greek}]/u;
1120
+ function obfuscationOf(text, options = {}) {
1121
+ const { maxLength = 1e5 } = options;
1122
+ const normalized = normalizeText(typeof text === "string" ? text.slice(0, maxLength) : "");
1123
+ let mixed = 0;
1124
+ for (const piece of chunk(normalized.text)) {
1125
+ if (!LATIN.test(piece) || !CYRILLIC_OR_GREEK.test(piece)) {
1126
+ continue;
1127
+ }
1128
+ for (const { segment, isWordLike } of segmenter().segment(piece)) {
1129
+ if (isWordLike && mixedScripts(segment)) {
1130
+ mixed++;
1131
+ }
1132
+ }
1133
+ }
1134
+ return { invisible: normalized.invisible, mixed, styled: normalized.styled };
1135
+ }
526
1136
  function bucket(n, steps) {
527
1137
  let i = 0;
528
1138
  while (i < steps.length && n > steps[i]) {
@@ -535,10 +1145,12 @@ function registrableDomain(host) {
535
1145
  return parsed.domain || parsed.hostname || host;
536
1146
  }
537
1147
  function extractLinks(text, html) {
1148
+ text = typeof text === "string" ? foldToAscii(text) : text;
1149
+ html = typeof html === "string" ? foldToAscii(html) : html;
538
1150
  const links = [];
539
1151
  const seen = /* @__PURE__ */ new Set();
540
1152
  const add = (raw, shown = null) => {
541
- let href = raw.trim().replace(/[).,;:!?'"\]]+$/, "");
1153
+ let href = trimEndOf(raw.trim(), `).,;:!?'"]`);
542
1154
  if (/^www\./i.test(href)) {
543
1155
  href = `http://${href}`;
544
1156
  }
@@ -563,15 +1175,12 @@ ${shown || ""}`;
563
1175
  links.push({ url: url.href, host: url.hostname.toLowerCase().replace(/\.$/, ""), text: shown });
564
1176
  };
565
1177
  if (typeof html === "string" && html) {
566
- const anchor = /<a\b[^>]{0,2000}?\bhref\s*=\s*(?:"([^"]{1,4000})"|'([^']{1,4000})'|([^\s>]{1,4000}))[^>]{0,2000}>([\s\S]{0,2000}?)<\/a\s*>/gi;
567
- for (const match of html.matchAll(anchor)) {
568
- const href = decodeEntities(match[1] || match[2] || match[3]);
569
- const inner = decodeEntities(match[4].replaceAll(/<[^>]*>/g, "")).trim();
570
- const shown = /^(?:https?:\/\/|www\.)\S+$|^[a-z\d-]+(?:\.[a-z\d-]+)+(?:\/\S*)?$/i.test(inner) ? inner : null;
1178
+ for (const { href, inner } of anchors(html)) {
1179
+ const shown = SHOWN_URL.test(inner) ? inner : null;
571
1180
  add(href, shown);
572
1181
  }
573
1182
  }
574
- for (const source of [text, typeof html === "string" ? html.replaceAll(/<a\b[^>]*>/gi, " ") : ""]) {
1183
+ for (const source of [text, typeof html === "string" ? withoutAnchorTags(html) : ""]) {
575
1184
  if (typeof source === "string" && source) {
576
1185
  for (const match of source.slice(0, 5e5).matchAll(URL_PATTERN)) {
577
1186
  add(match[0]);
@@ -580,22 +1189,97 @@ ${shown || ""}`;
580
1189
  }
581
1190
  return links;
582
1191
  }
583
- var ENTITIES = {
584
- amp: "&",
585
- lt: "<",
586
- gt: ">",
587
- quot: '"',
588
- apos: "'",
589
- nbsp: " "
590
- };
591
- function decodeEntities(value) {
592
- return value.replaceAll(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (match, entity) => {
593
- if (entity[0] === "#") {
594
- const code = entity[1] === "x" || entity[1] === "X" ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10);
595
- return code > 0 && code <= 1114111 ? String.fromCodePoint(code) : match;
1192
+ function tagEnd(html, start) {
1193
+ const limit = Math.min(html.length, start + 6e3);
1194
+ let quote = null;
1195
+ for (let i = start + 1; i < limit; i++) {
1196
+ const char = html[i];
1197
+ if (quote) {
1198
+ if (char === quote) {
1199
+ quote = null;
1200
+ }
1201
+ } else if (char === ">") {
1202
+ return i + 1;
1203
+ } else if (char === '"' || char === "'") {
1204
+ quote = char;
596
1205
  }
597
- return ENTITIES[entity.toLowerCase()] ?? match;
598
- });
1206
+ }
1207
+ const close = html.indexOf(">", start + 1);
1208
+ return close === -1 ? -1 : close + 1;
1209
+ }
1210
+ function withoutAnchorTags(html) {
1211
+ let out = "";
1212
+ let from = 0;
1213
+ ANCHOR_START.lastIndex = 0;
1214
+ let match;
1215
+ while ((match = ANCHOR_START.exec(html)) !== null) {
1216
+ const close = html.indexOf(">", match.index + 2);
1217
+ if (close === -1) {
1218
+ break;
1219
+ }
1220
+ out += `${html.slice(from, match.index)} `;
1221
+ from = close + 1;
1222
+ ANCHOR_START.lastIndex = from;
1223
+ }
1224
+ return out + html.slice(from);
1225
+ }
1226
+ function anchors(html) {
1227
+ const found = [];
1228
+ ANCHOR_START.lastIndex = 0;
1229
+ ANCHOR_CLOSE.lastIndex = 0;
1230
+ let close = null;
1231
+ let exhausted = false;
1232
+ let match;
1233
+ while ((match = ANCHOR_START.exec(html)) !== null) {
1234
+ const end = tagEnd(html, match.index);
1235
+ if (end === -1) {
1236
+ break;
1237
+ }
1238
+ const attribute = end - match.index <= 6e3 ? HREF.exec(html.slice(match.index, end)) : null;
1239
+ const value = attribute ? attribute[1] ?? attribute[2] ?? attribute[3] : "";
1240
+ if (!value || value.length > 4e3) {
1241
+ ANCHOR_START.lastIndex = end;
1242
+ continue;
1243
+ }
1244
+ if (close !== null && close.index < end) {
1245
+ close = null;
1246
+ }
1247
+ if (close === null && !exhausted) {
1248
+ ANCHOR_CLOSE.lastIndex = Math.max(ANCHOR_CLOSE.lastIndex, end);
1249
+ close = ANCHOR_CLOSE.exec(html);
1250
+ exhausted = close === null;
1251
+ }
1252
+ if (close === null || close.index - end > 2e3) {
1253
+ ANCHOR_START.lastIndex = end;
1254
+ continue;
1255
+ }
1256
+ found.push({ href: foldToAscii(decodeEntities(value, { attribute: true })), inner: foldToAscii(decodeEntities(replaceTags(html.slice(end, close.index), ""))).trim() });
1257
+ ANCHOR_START.lastIndex = close.index + close[0].length;
1258
+ close = null;
1259
+ }
1260
+ return found;
1261
+ }
1262
+ function extractHosts(text, known = /* @__PURE__ */ new Set()) {
1263
+ const hosts = /* @__PURE__ */ new Set();
1264
+ if (typeof text !== "string" || text === "") {
1265
+ return [];
1266
+ }
1267
+ text = foldToAscii(text);
1268
+ BARE_HOST.lastIndex = 0;
1269
+ let match;
1270
+ while ((match = BARE_HOST.exec(text)) !== null && hosts.size < MAX_BARE_HOSTS) {
1271
+ const [host] = match;
1272
+ const next = text[match.index + host.length];
1273
+ if (continuesName(text, match.index) || host !== host.toLowerCase() || next === "@" || next === "+" || known.has(host) || hosts.has(host)) {
1274
+ continue;
1275
+ }
1276
+ const parsed = parseDomain(host);
1277
+ const file = !parsed.subdomain && FILE_TLDS.has(parsed.publicSuffix) && !host.startsWith("xn--");
1278
+ if (parsed.isIcann && parsed.domain && !file) {
1279
+ hosts.add(host);
1280
+ }
1281
+ }
1282
+ return [...hosts];
599
1283
  }
600
1284
  function isDeceptiveLink(link) {
601
1285
  if (!link.text) {
@@ -614,11 +1298,13 @@ function isDeceptiveLink(link) {
614
1298
  }
615
1299
  function addressDomain(address2) {
616
1300
  const at = address2.lastIndexOf("@");
617
- return at === -1 ? "" : address2.slice(at + 1).toLowerCase().replace(/[>\s]+$/, "");
1301
+ return at === -1 ? "" : trimEndOf(address2.slice(at + 1).toLowerCase(), "> \r\n\f\v");
618
1302
  }
619
1303
  function firstAddress(field) {
620
1304
  return field?.value?.find((entry) => entry.address) || null;
621
1305
  }
1306
+ var ADDRESS_SOURCE = String.raw`[\p{L}\p{N}._%+-]{1,64}@([\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63}){1,10})`;
1307
+ var NAMED_ADDRESS = linear(ADDRESS_SOURCE, "u");
622
1308
  function headerValue(mail, name) {
623
1309
  const value = mail.headers?.get?.(name);
624
1310
  if (value === void 0 || value === null) {
@@ -633,7 +1319,7 @@ function headerValue(mail, name) {
633
1319
  return value.value ?? value.text ?? String(value);
634
1320
  }
635
1321
  function addPatternFeatures(text, add) {
636
- let body = text;
1322
+ let body = foldToAscii(text);
637
1323
  const patterns = [
638
1324
  ["url", URL_PATTERN],
639
1325
  ["email", EMAIL_PATTERN],
@@ -760,7 +1446,7 @@ function addSenderFeatures(mail, add) {
760
1446
  }
761
1447
  }
762
1448
  const name = typeof from.name === "string" ? from.name : "";
763
- const named = name.match(/[\p{L}\p{N}._%+-]+@([\p{L}\p{N}-]+(?:\.[\p{L}\p{N}-]+)+)/u);
1449
+ const named = name.match(NAMED_ADDRESS);
764
1450
  if (named && registrableDomain(named[1].toLowerCase()) !== registrableDomain(domain)) {
765
1451
  add("from:name_has_other_address");
766
1452
  }
@@ -775,13 +1461,50 @@ function addSenderFeatures(mail, add) {
775
1461
  }
776
1462
  }
777
1463
  }
1464
+ function isHtmlOnly(mail) {
1465
+ const contentType = mail.headers?.get?.("content-type");
1466
+ return !(typeof mail.text === "string" && mail.text) || (contentType?.value ?? contentType) === "text/html";
1467
+ }
1468
+ var MIN_SHOWN_WORDS = 20;
1469
+ var PART_WORD = /[\p{L}\p{N}]{2,}/gu;
1470
+ function partWords(text) {
1471
+ const words = /* @__PURE__ */ new Set();
1472
+ for (const [word] of normalizeText(text.slice(0, 5e4)).text.matchAll(PART_WORD)) {
1473
+ words.add(word);
1474
+ if (words.size >= 2e3) {
1475
+ break;
1476
+ }
1477
+ }
1478
+ return words;
1479
+ }
1480
+ function addPartFeatures({ visible, text, plain, alternative }, add) {
1481
+ if (visible.hidden > 0) {
1482
+ add("html:hidden_text");
1483
+ add(`html:hidden_text:${bucket(visible.hidden, [1, 50, 500])}`);
1484
+ }
1485
+ if (!alternative) {
1486
+ return;
1487
+ }
1488
+ const shown = partWords(text);
1489
+ const other = partWords(plain);
1490
+ if (shown.size < MIN_SHOWN_WORDS || other.size < MIN_SHOWN_WORDS) {
1491
+ return;
1492
+ }
1493
+ let shared = 0;
1494
+ for (const word of other) {
1495
+ if (shown.has(word)) {
1496
+ shared++;
1497
+ }
1498
+ }
1499
+ if (shared < Math.min(shown.size, other.size) / 5) {
1500
+ add("html:text_mismatch");
1501
+ }
1502
+ }
778
1503
  function addHtmlFeatures(mail, html, words, add) {
779
1504
  if (!html) {
780
1505
  return;
781
1506
  }
782
- const contentType = mail.headers?.get?.("content-type");
783
- const htmlOnly = !(typeof mail.text === "string" && mail.text) || (contentType?.value ?? contentType) === "text/html";
784
- add(htmlOnly ? "html:only" : "html:alternative");
1507
+ add(isHtmlOnly(mail) ? "html:only" : "html:alternative");
785
1508
  if (/<form\b/i.test(html)) {
786
1509
  add("html:form");
787
1510
  }
@@ -794,14 +1517,14 @@ function addHtmlFeatures(mail, html, words, add) {
794
1517
  if (/display\s*:\s*none|visibility\s*:\s*hidden|font-size\s*:\s*[01](?:\.\d+)?(?:px|pt)?\s*[;"']/i.test(html)) {
795
1518
  add("html:hidden");
796
1519
  }
797
- const images = (html.match(/<img\b/gi) || []).length;
1520
+ const images = (html.match(IMG) || []).length;
798
1521
  if (images > 0) {
799
1522
  add(`html:img:${bucket(images, [1, 5, 20])}`);
800
1523
  if (words.length < 20) {
801
1524
  add("html:img_little_text");
802
1525
  }
803
1526
  }
804
- if (/<img\b[^>]*\b(?:width|height)\s*=\s*["']?[01]["'\s>]/i.test(html)) {
1527
+ if (IMG_PIXEL.test(html)) {
805
1528
  add("html:pixel");
806
1529
  }
807
1530
  }
@@ -855,14 +1578,21 @@ function getFeatures(mail = {}, options = {}) {
855
1578
  const { maxLength = 1e5, maxBigrams = 3e3 } = options;
856
1579
  const features = /* @__PURE__ */ new Set();
857
1580
  const add = (feature) => features.add(feature);
858
- const html = typeof mail.html === "string" ? mail.html : "";
859
- let text = typeof mail.text === "string" ? mail.text : "";
860
- if (!text && html) {
861
- text = decodeEntities(html.replaceAll(/<(script|style)\b[\s\S]*?<\/\1\s*>/gi, " ").replaceAll(/<[^>]*>/g, " "));
1581
+ const html = typeof mail.html === "string" ? mail.html.slice(0, maxLength * 2) : "";
1582
+ const plain = typeof mail.text === "string" ? mail.text.slice(0, maxLength) : "";
1583
+ const visible = html ? visibleText(html) : { text: plain, hidden: 0 };
1584
+ const alternative = Boolean(html && plain && !isHtmlOnly(mail));
1585
+ let text = visible.text.slice(0, maxLength);
1586
+ if (alternative && segmentWords(text.slice(0, 4e3)).words.length < MIN_SHOWN_WORDS) {
1587
+ text = `${text}
1588
+ ${plain}`.slice(0, maxLength);
862
1589
  }
863
- text = text.slice(0, maxLength);
864
1590
  const subject = typeof mail.subject === "string" ? mail.subject : "";
865
- const links = extractLinks(text, html.slice(0, maxLength * 2));
1591
+ const links = extractLinks(html ? `${plain}
1592
+ ${text}` : text, html);
1593
+ const hosts = extractHosts(`${subject}
1594
+ ${text}${alternative ? `
1595
+ ${plain}` : ""}`, new Set(links.map((link) => link.host)));
866
1596
  const body = addPatternFeatures(text, add);
867
1597
  const detected = detectLanguage(`${subject}
868
1598
  ${body}`);
@@ -884,6 +1614,12 @@ ${body}`);
884
1614
  addLinkFeatures(links, add);
885
1615
  addSenderFeatures(mail, add);
886
1616
  addHtmlFeatures(mail, html, segmented.words, add);
1617
+ addPartFeatures({
1618
+ visible,
1619
+ text,
1620
+ plain,
1621
+ alternative
1622
+ }, add);
887
1623
  addAttachmentFeatures(mail, add);
888
1624
  addHeaderFeatures(mail, add);
889
1625
  return {
@@ -891,7 +1627,12 @@ ${body}`);
891
1627
  words: segmented.words,
892
1628
  language: detected.language,
893
1629
  script: detected.script,
894
- links
1630
+ links,
1631
+ hosts,
1632
+ // (the text shown, as read for the words, and how many characters of
1633
+ // text the HTML hides)
1634
+ text,
1635
+ hidden: visible.hidden
895
1636
  };
896
1637
  }
897
1638
 
@@ -1073,7 +1814,7 @@ function sniff(buffer) {
1073
1814
  return result("ole");
1074
1815
  }
1075
1816
  if (at(0, [80, 75, 3, 4]) || at(0, [80, 75, 5, 6])) {
1076
- const names = zipEntries(buffer).map((entry) => entry.name);
1817
+ const names = entriesOf(buffer).map((entry) => entry.name);
1077
1818
  if (names.includes("AndroidManifest.xml") || names.includes("classes.dex")) {
1078
1819
  return result("apk", true);
1079
1820
  }
@@ -1139,25 +1880,65 @@ function sniff(buffer) {
1139
1880
  }
1140
1881
  return null;
1141
1882
  }
1883
+ var ZIP_FAMILY = /* @__PURE__ */ new Set(["zip", "docx", "xlsx", "pptx"]);
1884
+ var OFFICE_TYPES = /* @__PURE__ */ new Set(["docx", "xlsx", "pptx"]);
1885
+ var OFFICE_EMBEDDED = /* @__PURE__ */ new Set([...MACRO_EXTENSIONS, "cer", "der"]);
1886
+ var ENTRIES = /* @__PURE__ */ new WeakMap();
1887
+ function entriesOf(buffer) {
1888
+ let entries = ENTRIES.get(buffer);
1889
+ if (!entries) {
1890
+ entries = zipEntries(buffer);
1891
+ ENTRIES.set(buffer, entries);
1892
+ }
1893
+ return entries;
1894
+ }
1142
1895
  function zipEntries(buffer, limit = 2e3) {
1143
- const entries = [];
1144
1896
  const start = Math.max(0, buffer.length - 65557);
1145
- let eocd = -1;
1897
+ const records = [];
1146
1898
  for (let i = buffer.length - 22; i >= start; i--) {
1147
- if (buffer.readUInt32LE(i) === 101010256) {
1148
- eocd = i;
1149
- break;
1899
+ if (buffer.readUInt32LE(i) === 101010256 && (records.length === 0 || i + 22 + buffer.readUInt16LE(i + 20) === buffer.length)) {
1900
+ records.push(i);
1150
1901
  }
1151
1902
  }
1152
- if (eocd === -1) {
1153
- return entries;
1903
+ const entries = [];
1904
+ const read = /* @__PURE__ */ new Set();
1905
+ for (const eocd of records) {
1906
+ const offset = directoryOf(buffer, eocd);
1907
+ if (offset !== -1 && !read.has(offset)) {
1908
+ read.add(offset);
1909
+ readDirectory(buffer, offset, limit - entries.length, entries);
1910
+ }
1154
1911
  }
1155
- const count = buffer.readUInt16LE(eocd + 10);
1912
+ return entries;
1913
+ }
1914
+ var DIRECTORY = 33639248;
1915
+ var isDirectory = (buffer, offset) => offset >= 0 && offset + 46 <= buffer.length && buffer.readUInt32LE(offset) === DIRECTORY;
1916
+ function directoryOf(buffer, eocd) {
1156
1917
  let offset = buffer.readUInt32LE(eocd + 16);
1157
- for (let n = 0; n < Math.min(count, limit); n++) {
1158
- if (offset + 46 > buffer.length || buffer.readUInt32LE(offset) !== 33639248) {
1159
- break;
1918
+ let size = buffer.readUInt32LE(eocd + 12);
1919
+ const locator = eocd - 20;
1920
+ if ((offset === 4294967295 || size === 4294967295) && locator >= 0 && buffer.readUInt32LE(locator) === 117853008) {
1921
+ const stated = Number(buffer.readBigUInt64LE(locator + 8));
1922
+ const isRecord = (at) => at >= 0 && at + 56 <= locator && buffer.readUInt32LE(at) === 101075792;
1923
+ const record = isRecord(stated) ? stated : locator - 56;
1924
+ if (isRecord(record)) {
1925
+ size = Number(buffer.readBigUInt64LE(record + 40));
1926
+ offset = Number(buffer.readBigUInt64LE(record + 48));
1160
1927
  }
1928
+ }
1929
+ if (isDirectory(buffer, offset)) {
1930
+ return offset;
1931
+ }
1932
+ for (const end of [eocd, locator - 56]) {
1933
+ const directory = end - size;
1934
+ if (directory < end && isDirectory(buffer, directory)) {
1935
+ return directory;
1936
+ }
1937
+ }
1938
+ return -1;
1939
+ }
1940
+ function readDirectory(buffer, offset, count, entries) {
1941
+ for (let n = 0; n < count && isDirectory(buffer, offset); n++) {
1161
1942
  const flags = buffer.readUInt16LE(offset + 8);
1162
1943
  const size = buffer.readUInt32LE(offset + 24);
1163
1944
  const nameLength = buffer.readUInt16LE(offset + 28);
@@ -1167,10 +1948,9 @@ function zipEntries(buffer, limit = 2e3) {
1167
1948
  entries.push({ name, encrypted: (flags & 1) === 1, size });
1168
1949
  offset += 46 + nameLength + extraLength + commentLength;
1169
1950
  }
1170
- return entries;
1171
1951
  }
1172
1952
  function extensionOf(name) {
1173
- const base = String(name).toLowerCase().split(/[/\\]/).pop();
1953
+ const base = trimEndOf(String(name).toLowerCase().split(/[/\\]/).pop(), ". ");
1174
1954
  const dot = base.lastIndexOf(".");
1175
1955
  return dot === -1 ? "" : base.slice(dot + 1).trim();
1176
1956
  }
@@ -1178,7 +1958,7 @@ function oleHasMacros(buffer) {
1178
1958
  const names = ["_VBA_PROJECT", "VBA", "Macros"].map((name) => Buffer.from(name, "utf16le"));
1179
1959
  return names.some((name) => buffer.includes(name));
1180
1960
  }
1181
- var PDF_ACTIVE = /\/(?:JavaScript|JS|Launch|EmbeddedFile|OpenAction\s*<<[^>]*\/(?:JS|JavaScript))\b/;
1961
+ var PDF_ACTIVE = linear(String.raw`\/(?:JavaScript|JS|Launch|EmbeddedFile)\b`);
1182
1962
  function inspectAttachments(attachments = []) {
1183
1963
  const findings = [];
1184
1964
  for (const attachment of attachments) {
@@ -1196,7 +1976,7 @@ function inspectAttachments(attachments = []) {
1196
1976
  if (BIDI_CONTROLS.test(filename)) {
1197
1977
  add("rtl_override", `Attachment "${name}" uses a right-to-left override to hide its real extension`);
1198
1978
  }
1199
- const parts = filename.toLowerCase().split(".");
1979
+ const parts = trimEndOf(filename.toLowerCase(), ". ").split(".");
1200
1980
  if (parts.length >= 3 && EXECUTABLE_EXTENSIONS.has(extension) && DOCUMENT_EXTENSIONS.has(parts.at(-2))) {
1201
1981
  add("double_extension", `Attachment "${name}" has a document extension followed by a program extension`, { extension });
1202
1982
  }
@@ -1211,21 +1991,23 @@ function inspectAttachments(attachments = []) {
1211
1991
  }
1212
1992
  continue;
1213
1993
  }
1214
- if (detected?.type === "zip" || detected?.type === "docx" || detected?.type === "xlsx" || detected?.type === "pptx") {
1215
- const entries = zipEntries(content);
1994
+ const zipFamily = ZIP_FAMILY.has(detected?.type) || extension === "zip" && !detected?.executable && entriesOf(content).length > 0;
1995
+ if (zipFamily) {
1996
+ const entries = entriesOf(content);
1997
+ const office = OFFICE_TYPES.has(detected?.type);
1216
1998
  if (entries.some((entry) => /(?:^|\/)vbaproject\.bin$/i.test(entry.name))) {
1217
1999
  add("macro", `Attachment "${name}" is an Office document with a VBA macro project`, { extension });
1218
2000
  } else if (MACRO_EXTENSIONS.has(extension)) {
1219
2001
  add("macro", `Attachment "${name}" is an Office document type that holds macros (.${extension})`, { extension });
2002
+ } else if (office && entries.some((entry) => MACRO_EXTENSIONS.has(extensionOf(entry.name)))) {
2003
+ add("macro", `Attachment "${name}" is an Office document holding another with macros`, { extension });
1220
2004
  }
1221
- if (detected.type === "zip") {
1222
- const inner = entries.filter((entry) => EXECUTABLE_EXTENSIONS.has(extensionOf(entry.name)));
1223
- if (inner.length > 0) {
1224
- add("executable_in_archive", `Archive "${name}" contains ${inner.length === 1 ? "a program" : `${inner.length} programs`} (${inner.slice(0, 3).map((entry) => entry.name.split("/").pop()).join(", ")})`, { extension: extensionOf(inner[0].name) });
1225
- }
1226
- if (entries.some((entry) => entry.encrypted)) {
1227
- add("encrypted_archive", `Archive "${name}" is password protected, so its contents cannot be scanned`, { extension });
1228
- }
2005
+ const inner = entries.filter((entry) => EXECUTABLE_EXTENSIONS.has(extensionOf(entry.name)) && !(office && OFFICE_EMBEDDED.has(extensionOf(entry.name))));
2006
+ if (inner.length > 0) {
2007
+ add("executable_in_archive", `Archive "${name}" contains ${inner.length === 1 ? "a program" : `${inner.length} programs`} (${inner.slice(0, 3).map((entry) => entry.name.split("/").pop()).join(", ")})`, { extension: extensionOf(inner[0].name), extensions: [...new Set(inner.map((entry) => extensionOf(entry.name)))] });
2008
+ }
2009
+ if ((detected?.type === "zip" || !ZIP_FAMILY.has(detected?.type)) && entries.some((entry) => entry.encrypted)) {
2010
+ add("encrypted_archive", `Archive "${name}" is password protected, so its contents cannot be scanned`, { extension });
1229
2011
  }
1230
2012
  } else if (detected?.type === "ole") {
1231
2013
  if (oleHasMacros(content)) {
@@ -2010,6 +2792,42 @@ var DnsChecker = class {
2010
2792
  // src/homograph.js
2011
2793
  import { domainToUnicode } from "node:url";
2012
2794
  import { parse as parseDomain2 } from "tldts";
2795
+ var INTERNATIONAL_BRANDS = [
2796
+ "alipay",
2797
+ "auspost",
2798
+ "bancolombia",
2799
+ "bnpparibas",
2800
+ "bradesco",
2801
+ "caixabank",
2802
+ "canadapost",
2803
+ "commerzbank",
2804
+ "coupang",
2805
+ "creditagricole",
2806
+ "deutschebank",
2807
+ "flipkart",
2808
+ "gosuslugi",
2809
+ "japanpost",
2810
+ "kakaobank",
2811
+ "klarna",
2812
+ "kucoin",
2813
+ "lazada",
2814
+ "mercadolibre",
2815
+ "mercadolivre",
2816
+ "mercadopago",
2817
+ "mercari",
2818
+ "natwest",
2819
+ "nubank",
2820
+ "phonepe",
2821
+ "postbank",
2822
+ "rakuten",
2823
+ "sberbank",
2824
+ "swisscom",
2825
+ "trustwallet",
2826
+ "unicredit",
2827
+ "vodafone",
2828
+ "wechat",
2829
+ "yandex"
2830
+ ];
2013
2831
  var BRANDS = [
2014
2832
  "adobe",
2015
2833
  "airbnb",
@@ -2109,8 +2927,76 @@ var BRANDS = [
2109
2927
  "xfinity",
2110
2928
  "yahoo",
2111
2929
  "zelle",
2112
- "zoom"
2930
+ "zoom",
2931
+ ...INTERNATIONAL_BRANDS
2932
+ ];
2933
+ var PHISHING_WORDS = ["account", "accounts", "alert", "alerts", "auth", "authentication", "billing", "confirm", "login", "logon", "password", "recovery", "refund", "secure", "security", "signin", "signon", "suspended", "unlock", "update", "validate", "verification", "verify"];
2934
+ var WORD_BRANDS = /* @__PURE__ */ new Set(["apple", "blockchain", "booking", "chase", "discord", "kraken", "ledger", "office", "proton", "slack", "steam", "stripe"]);
2935
+ var SWAPPED_WORDS = /* @__PURE__ */ new Set([
2936
+ "akuten",
2937
+ "alzada",
2938
+ "appel",
2939
+ "applied",
2940
+ "auslost",
2941
+ "azada",
2942
+ "coupant",
2943
+ "gechat",
2944
+ "klara",
2945
+ "lanada",
2946
+ "lanzada",
2947
+ "lapada",
2948
+ "latada",
2949
+ "lavada",
2950
+ "mazada",
2951
+ "mercar",
2952
+ "mercati",
2953
+ "pazada",
2954
+ "phoneme",
2955
+ "postbak",
2956
+ "raketen",
2957
+ "rauten",
2958
+ "razada",
2959
+ "stirpe",
2960
+ "vazada",
2961
+ "venom",
2962
+ "webchat"
2963
+ ]);
2964
+ var OFFICIAL_DOMAINS = [
2965
+ "adobelogin.com",
2966
+ "amazon-adsystem.com",
2967
+ "apple-cloudkit.com",
2968
+ "apple-dns.net",
2969
+ "bnpparibas-am.com",
2970
+ "canadapost-postescanada.ca",
2971
+ "cdn-apple.com",
2972
+ "cloudflare-dns.com",
2973
+ "credit-agricole.com",
2974
+ "credit-agricole.fr",
2975
+ "deutsche-bank.de",
2976
+ "google-analytics.com",
2977
+ "icloud-content.com",
2978
+ "images-amazon.com",
2979
+ "instagram-engineering.com",
2980
+ "media-amazon.com",
2981
+ "okta-emea.com",
2982
+ "paypal-community.com",
2983
+ "paypal-objects.com",
2984
+ "paypal-prepaid.com",
2985
+ "paypal-status.com",
2986
+ "rakuten-bank.co.jp",
2987
+ "rakuten-card.co.jp",
2988
+ "rakuten-sec.co.jp",
2989
+ "sberbank-ast.ru",
2990
+ "slack-edge.com",
2991
+ "slack-msgs.com",
2992
+ "ssl-images-amazon.com",
2993
+ "t-mobile.com",
2994
+ "yahoo-corp.jp",
2995
+ "yandex-team.ru"
2113
2996
  ];
2997
+ var SUFFIX_DASH = /^(?:com|net|org|gov|edu)-/;
2998
+ var OTHER_SUFFIX_DASH = /^(?:co|info|biz|io|app|us|uk|de|fr|es|it|ca|au|jp|br|in)-/;
2999
+ var SECOND_SUFFIX_DASH = /^(?:co|com)-(?:uk|jp|au|nz|za|in|kr|br)\b/;
2114
3000
  var CONFUSABLE = {
2115
3001
  \u0430: "a",
2116
3002
  \u04D3: "a",
@@ -2254,10 +3140,17 @@ function skeleton(label) {
2254
3140
  }
2255
3141
  return out;
2256
3142
  }
3143
+ var MAX_VARIANTS = 256;
2257
3144
  function trickVariants(label) {
2258
3145
  const variants = /* @__PURE__ */ new Set([label]);
2259
3146
  for (const [pattern, letter] of ASCII_TRICKS) {
3147
+ if (!label.includes(pattern.source)) {
3148
+ continue;
3149
+ }
2260
3150
  for (const value of variants) {
3151
+ if (variants.size >= MAX_VARIANTS / 2) {
3152
+ break;
3153
+ }
2261
3154
  variants.add(value.replace(pattern, letter));
2262
3155
  }
2263
3156
  }
@@ -2271,6 +3164,7 @@ function editDistance(a, b, max = 2) {
2271
3164
  if (Math.abs(a.length - b.length) > max) {
2272
3165
  return max + 1;
2273
3166
  }
3167
+ let before = null;
2274
3168
  let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
2275
3169
  for (let i = 1; i <= a.length; i++) {
2276
3170
  const current = [i];
@@ -2278,15 +3172,35 @@ function editDistance(a, b, max = 2) {
2278
3172
  for (let j = 1; j <= b.length; j++) {
2279
3173
  const cost = a[i - 1] === b[j - 1] ? 0 : 1;
2280
3174
  current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost);
3175
+ if (before && i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
3176
+ current[j] = Math.min(current[j], before[j - 2] + 1);
3177
+ }
2281
3178
  best = Math.min(best, current[j]);
2282
3179
  }
2283
3180
  if (best > max) {
2284
3181
  return max + 1;
2285
3182
  }
3183
+ before = previous;
2286
3184
  previous = current;
2287
3185
  }
2288
3186
  return previous[b.length];
2289
3187
  }
3188
+ function brandWithPhishingWord(name, brands) {
3189
+ for (const word of PHISHING_WORDS) {
3190
+ if (name.length <= word.length + 4) {
3191
+ continue;
3192
+ }
3193
+ const before = name.endsWith(word) ? name.slice(0, -word.length) : null;
3194
+ if (before && before.length >= 5 && brands.has(before) && !WORD_BRANDS.has(before)) {
3195
+ return before;
3196
+ }
3197
+ const after = name.startsWith(word) ? name.slice(word.length) : null;
3198
+ if (after && after.length >= 5 && brands.has(after) && !WORD_BRANDS.has(after)) {
3199
+ return after;
3200
+ }
3201
+ }
3202
+ return null;
3203
+ }
2290
3204
  function labelScripts(label) {
2291
3205
  const scripts = /* @__PURE__ */ new Set();
2292
3206
  for (const char of label) {
@@ -2314,7 +3228,7 @@ var HomographDetector = class {
2314
3228
  constructor(options = {}) {
2315
3229
  this.options = options;
2316
3230
  this.brands = new Set([...options.brands || BRANDS, ...options.extraBrands || []].map((brand) => brand.toLowerCase()));
2317
- this.allowlist = new Set((options.allowlist || []).map((domain) => domain.toLowerCase()));
3231
+ this.allowlist = new Set([...OFFICIAL_DOMAINS, ...options.allowlist || []].map((domain) => domain.toLowerCase()));
2318
3232
  this.cache = /* @__PURE__ */ new Map();
2319
3233
  }
2320
3234
  /**
@@ -2370,29 +3284,36 @@ var HomographDetector = class {
2370
3284
  if (viaConfusable) {
2371
3285
  flag(0.95, `"${unicode}" imitates ${viaConfusable} with lookalike letters`, viaConfusable);
2372
3286
  } else if (/^[a-z\d-]+$/.test(folded)) {
2373
- const trick = [...trickVariants(folded)].find((variant) => this.brands.has(variant));
3287
+ const variants = trickVariants(folded);
3288
+ const trick = [...variants].find((variant) => this.brands.has(variant));
2374
3289
  if (trick) {
2375
3290
  flag(0.85, `"${unicode}" imitates ${trick} by swapping characters`, trick);
2376
- } else if (folded.length >= 5) {
3291
+ } else if (folded.length >= 5 && !SWAPPED_WORDS.has(folded)) {
2377
3292
  const near = [...this.brands].find((brand) => brand.length >= 5 && editDistance(folded, brand, 1) === 1);
2378
3293
  if (near) {
2379
3294
  flag(0.5, `"${unicode}" is one letter away from ${near}`, near);
2380
3295
  }
2381
3296
  }
2382
- if (folded.includes("-")) {
2383
- for (const piece of folded.split("-")) {
2384
- if (piece.length < 4) {
2385
- continue;
2386
- }
2387
- if (this.brands.has(piece)) {
2388
- flag(0.6, `"${unicode}" puts ${piece} inside another domain name`, piece);
2389
- break;
2390
- }
2391
- const disguised = [...trickVariants(piece)].find((variant) => this.brands.has(variant));
2392
- if (disguised) {
2393
- flag(0.85, `"${unicode}" imitates ${disguised} by swapping characters`, disguised);
2394
- break;
2395
- }
3297
+ const pieces = folded.includes("-") ? folded.split("-") : [];
3298
+ for (const piece of pieces) {
3299
+ if (piece.length < 4) {
3300
+ continue;
3301
+ }
3302
+ if (this.brands.has(piece)) {
3303
+ flag(0.6, `"${unicode}" puts ${piece} inside another domain name`, piece);
3304
+ break;
3305
+ }
3306
+ const disguised = [...trickVariants(piece)].find((variant) => this.brands.has(variant));
3307
+ if (disguised) {
3308
+ flag(0.85, `"${unicode}" imitates ${disguised} by swapping characters`, disguised);
3309
+ break;
3310
+ }
3311
+ }
3312
+ for (const value of [folded, ...pieces, ...variants]) {
3313
+ const joined = brandWithPhishingWord(value, this.brands);
3314
+ if (joined) {
3315
+ flag(0.6, `"${unicode}" joins ${joined} to a phishing word`, joined);
3316
+ break;
2396
3317
  }
2397
3318
  }
2398
3319
  }
@@ -2401,6 +3322,11 @@ var HomographDetector = class {
2401
3322
  if (brandLabel && (this.options.strictMode || subdomain.length >= 2)) {
2402
3323
  flag(0.6, `"${unicode}" puts ${brandLabel} in front of another domain`, brandLabel);
2403
3324
  }
3325
+ const last = subdomain.length > 0 ? skeleton(subdomain.at(-1)) : "";
3326
+ if (last.length >= 3 && this.brands.has(last) && (SUFFIX_DASH.test(folded) || OTHER_SUFFIX_DASH.test(folded))) {
3327
+ const strong = SUFFIX_DASH.test(folded) || SECOND_SUFFIX_DASH.test(folded) || PHISHING_WORDS.some((word) => folded.includes(word));
3328
+ flag(strong ? 0.85 : 0.6, `"${unicode}" starts with ${last}'s address and continues into another domain`, last);
3329
+ }
2404
3330
  result.confidence = result.riskScore;
2405
3331
  if (result.riskScore >= 0.5) {
2406
3332
  result.recommendations.push(result.brand ? `Do not sign in from this link; go to ${result.brand}'s site directly` : "Treat links to this domain as untrusted");
@@ -2496,7 +3422,7 @@ ${html.slice(0, 1e5)}`)) {
2496
3422
  if (from) {
2497
3423
  const fromRoot = session.originalFromAddressRootDomain;
2498
3424
  const name = typeof from.name === "string" ? from.name.toLowerCase() : "";
2499
- const claimed = name.match(/[\p{L}\p{N}._%+-]+@([\p{L}\p{N}-]+(?:\.[\p{L}\p{N}-]+)+)/u);
3425
+ const claimed = name.match(NAMED_ADDRESS);
2500
3426
  if (claimed && registrableDomain(claimed[1]) !== fromRoot) {
2501
3427
  hit("FROM_NAME_OTHER_ADDRESS", 2.5, `Display name shows ${claimed[0]} but the message is from ${from.address}`, "SPOOFING");
2502
3428
  }
@@ -2881,7 +3807,7 @@ function resolveConfig(options = {}, env = process.env) {
2881
3807
  if (!baseUrl) {
2882
3808
  throw new TypeError(`LLM provider "${providerName}" needs a base URL (baseUrl, or host and port)`);
2883
3809
  }
2884
- baseUrl = baseUrl.replace(/\/+$/, "");
3810
+ baseUrl = trimEndOf(baseUrl, "/");
2885
3811
  const model = options.model || preset.model;
2886
3812
  if (!model && preset.api !== "classifier") {
2887
3813
  throw new TypeError(`LLM provider "${providerName}" needs a model name`);
@@ -2992,18 +3918,59 @@ function authHeaders(config) {
2992
3918
  return headers;
2993
3919
  }
2994
3920
  var REDACTIONS = [
2995
- [/[\p{L}\p{N}._%+-]+@([\p{L}\p{N}-]+(?:\.[\p{L}\p{N}-]+)+)/gu, "[email]@$1"],
2996
- [/\b(?:\d[ -]?){13,19}\b/g, "[number]"],
2997
- [/(?:\+|00)\d[\d\s().-]{7,}\d/g, "[phone]"],
2998
- [/\b\d{3}-\d{2}-\d{4}\b/g, "[number]"],
2999
- [/([?&][^=\s&#]+=)[^\s&#]+/g, "$1[redacted]"]
3921
+ [linear(ADDRESS_SOURCE, "gu"), "[email]@$1"],
3922
+ [linear(String.raw`\b(?:\d[ -]?){13,19}\b`, "g"), "[number]"],
3923
+ [linear(String.raw`(?:\+|00)\d[\d\s().-]{7,}\d`, "g"), "[phone]"],
3924
+ [linear(String.raw`\b\d{3}-\d{2}-\d{4}\b`, "g"), "[number]"],
3925
+ [linear(String.raw`([?&][^=\s&#]{1,256}=)[^\s&#]+`, "g"), "$1[redacted]"]
3000
3926
  ];
3927
+ var THINK_OPEN = linear(`<${caseless("think")}>`, "g");
3928
+ var THINK_CLOSE = linear(String.raw`<\/${caseless("think")}>`, "g");
3929
+ function withoutThinking(text) {
3930
+ let out = "";
3931
+ let from = 0;
3932
+ THINK_OPEN.lastIndex = 0;
3933
+ let open;
3934
+ while ((open = THINK_OPEN.exec(text)) !== null) {
3935
+ THINK_CLOSE.lastIndex = open.index + open[0].length;
3936
+ const close = THINK_CLOSE.exec(text);
3937
+ if (close === null) {
3938
+ break;
3939
+ }
3940
+ out += text.slice(from, open.index);
3941
+ from = close.index + close[0].length;
3942
+ THINK_OPEN.lastIndex = from;
3943
+ }
3944
+ return out + text.slice(from);
3945
+ }
3001
3946
  function redact(text) {
3002
3947
  let out = text;
3003
3948
  for (const [pattern, replacement] of REDACTIONS) {
3004
3949
  out = out.replace(pattern, replacement);
3005
3950
  }
3006
- return out;
3951
+ return withoutLocalParts(out);
3952
+ }
3953
+ var LOCAL_CHARACTER = /^[\p{L}\p{N}._%+-]$/u;
3954
+ function withoutLocalParts(text) {
3955
+ let out = "";
3956
+ let from = 0;
3957
+ let at = text.indexOf("[email]@");
3958
+ while (at !== -1) {
3959
+ let start = at;
3960
+ while (start > from) {
3961
+ if (start - 2 >= from && LOCAL_CHARACTER.test(text.slice(start - 2, start))) {
3962
+ start -= 2;
3963
+ } else if (LOCAL_CHARACTER.test(text[start - 1])) {
3964
+ start--;
3965
+ } else {
3966
+ break;
3967
+ }
3968
+ }
3969
+ out += text.slice(from, start);
3970
+ from = at;
3971
+ at = text.indexOf("[email]@", at + 8);
3972
+ }
3973
+ return out + text.slice(from);
3007
3974
  }
3008
3975
  function addressList(field) {
3009
3976
  if (!field?.value) {
@@ -3025,7 +3992,8 @@ function describeMessage(mail, context = {}, options = {}) {
3025
3992
  if (context.authentication) {
3026
3993
  lines2.push(`Authentication: ${context.authentication}`);
3027
3994
  }
3028
- const links = (context.links || []).slice(0, 15);
3995
+ const seen = /* @__PURE__ */ new Set();
3996
+ const links = [...context.links || []].sort((a, b) => Number(isDeceptiveLink(b)) - Number(isDeceptiveLink(a))).filter((link) => !seen.has(link.url) && seen.add(link.url)).slice(0, 15);
3029
3997
  if (links.length > 0) {
3030
3998
  lines2.push("Links:");
3031
3999
  for (const link of links) {
@@ -3036,7 +4004,17 @@ function describeMessage(mail, context = {}, options = {}) {
3036
4004
  if (attachments.length > 0) {
3037
4005
  lines2.push(`Attachments: ${attachments.map((a) => `${a.filename || "unnamed"} (${a.contentType || "unknown type"})`).join(", ")}`);
3038
4006
  }
3039
- let body = typeof mail.text === "string" && mail.text.trim() ? mail.text : typeof mail.html === "string" ? mail.html.replaceAll(/<style[\s\S]*?<\/style>|<script[\s\S]*?<\/script>/gi, " ").replaceAll(/<[^>]+>/g, " ") : "";
4007
+ if (context.hidden >= 200) {
4008
+ lines2.push(`Hidden text: ${context.hidden} characters the message does not show`);
4009
+ }
4010
+ const read = Math.max(1e5, maxInputChars * 50);
4011
+ let shown = typeof mail.text === "string" ? mail.text : "";
4012
+ if (typeof context.text === "string") {
4013
+ shown = context.text;
4014
+ } else if (typeof mail.html === "string" && mail.html) {
4015
+ shown = visibleText(mail.html.slice(0, read)).text;
4016
+ }
4017
+ let body = readableText(shown.slice(0, read));
3040
4018
  body = body.replaceAll(/[ \t]+/g, " ").replaceAll(/\n\s*\n+/g, "\n\n").trim();
3041
4019
  let text = `${lines2.join("\n")}
3042
4020
 
@@ -3087,7 +4065,8 @@ function objectEnd(text, start) {
3087
4065
  return -1;
3088
4066
  }
3089
4067
  function* jsonObjects(text) {
3090
- for (let start = text.indexOf("{"); start !== -1; start = text.indexOf("{", start + 1)) {
4068
+ let starts = 0;
4069
+ for (let start = text.indexOf("{"); start !== -1 && starts++ < 64; start = text.indexOf("{", start + 1)) {
3091
4070
  const end = objectEnd(text, start);
3092
4071
  if (end !== -1) {
3093
4072
  yield text.slice(start, end + 1);
@@ -3098,7 +4077,7 @@ function parseVerdict(text) {
3098
4077
  if (typeof text !== "string") {
3099
4078
  return null;
3100
4079
  }
3101
- const visible = text.replaceAll(/<think>[\s\S]*?<\/think>/gi, "");
4080
+ const visible = withoutThinking(text).slice(0, 2e4);
3102
4081
  for (const candidate of jsonObjects(visible)) {
3103
4082
  try {
3104
4083
  const result = normalizeVerdict(JSON.parse(candidate));
@@ -3444,7 +4423,7 @@ import {
3444
4423
  writeFileSync,
3445
4424
  mkdirSync
3446
4425
  } from "node:fs";
3447
- import { createRequire } from "node:module";
4426
+ import { createRequire as createRequire2 } from "node:module";
3448
4427
  import path from "node:path";
3449
4428
  import process2 from "node:process";
3450
4429
  import { fileURLToPath } from "node:url";
@@ -3469,7 +4448,7 @@ function embeddedModel(sea = loadSea()) {
3469
4448
  return null;
3470
4449
  }
3471
4450
  }
3472
- function loadSea(load = createRequire(import.meta.url)) {
4451
+ function loadSea(load = createRequire2(import.meta.url)) {
3473
4452
  try {
3474
4453
  return load("node:sea");
3475
4454
  } catch {
@@ -3868,7 +4847,7 @@ function scoreResults(results = {}, options = {}) {
3868
4847
  var package_default = {
3869
4848
  name: "spamscanner",
3870
4849
  description: "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
3871
- version: "7.0.1",
4850
+ version: "7.0.2",
3872
4851
  author: "Forward Email (https://forwardemail.net)",
3873
4852
  bin: {
3874
4853
  spamscanner: "./dist/esm/cli.js"
@@ -3971,6 +4950,9 @@ var package_default = {
3971
4950
  license: "BUSL-1.1",
3972
4951
  main: "./dist/cjs/index.cjs",
3973
4952
  module: "./dist/esm/index.js",
4953
+ optionalDependencies: {
4954
+ re2: "^1.22.1"
4955
+ },
3974
4956
  packageManager: "pnpm@10.34.6",
3975
4957
  repository: {
3976
4958
  type: "git",
@@ -5333,6 +6315,7 @@ function mergeLegacyOptions(options) {
5333
6315
  }
5334
6316
  return config;
5335
6317
  }
6318
+ var MAX_LOOKALIKE_HOSTS = 1e3;
5336
6319
  var SpamScanner = class {
5337
6320
  /**
5338
6321
  * @param {object} [options] - see DEFAULTS
@@ -5481,28 +6464,22 @@ var SpamScanner = class {
5481
6464
  async scanFile(file, options) {
5482
6465
  return this.scan(await readFile(file), options);
5483
6466
  }
5484
- async checkLinks(links, config) {
6467
+ async checkLinks(links, config, hosts = []) {
5485
6468
  const phishing = [];
5486
6469
  const homographs = [];
5487
6470
  const seenHosts = /* @__PURE__ */ new Set();
5488
- for (const link of links) {
5489
- if (isDeceptiveLink(link)) {
5490
- phishing.push(finding({
5491
- type: "deceptive_link",
5492
- url: link.url,
5493
- text: link.text,
5494
- message: `A link shows "${link.text}" but goes to ${link.host}`
5495
- }));
5496
- }
5497
- if (seenHosts.has(link.host)) {
5498
- continue;
6471
+ const analysed = /* @__PURE__ */ new Set();
6472
+ const lookalike = (host, extra) => {
6473
+ if (analysed.has(host) || analysed.size >= MAX_LOOKALIKE_HOSTS) {
6474
+ return;
5499
6475
  }
5500
- seenHosts.add(link.host);
5501
- const analysis = this.homograph.detectHomographAttack(link.host);
5502
- if (analysis.riskScore >= 0.5) {
6476
+ analysed.add(host);
6477
+ const analysis = this.homograph.detectHomographAttack(host);
6478
+ const counts = extra.source === "text" ? analysis.riskScore >= 0.95 || analysis.riskScore >= 0.85 && analysis.brand.length >= 5 : analysis.riskScore >= 0.5;
6479
+ if (counts) {
5503
6480
  const item = finding({
5504
6481
  type: "homograph",
5505
- url: link.url,
6482
+ ...extra,
5506
6483
  domain: analysis.domain,
5507
6484
  unicode: analysis.unicode,
5508
6485
  brand: analysis.brand,
@@ -5514,11 +6491,28 @@ var SpamScanner = class {
5514
6491
  phishing.push(item);
5515
6492
  homographs.push(item);
5516
6493
  }
6494
+ };
6495
+ for (const link of links) {
6496
+ if (isDeceptiveLink(link)) {
6497
+ phishing.push(finding({
6498
+ type: "deceptive_link",
6499
+ url: link.url,
6500
+ text: link.text,
6501
+ message: `A link shows "${link.text}" but goes to ${link.host}`
6502
+ }));
6503
+ }
6504
+ seenHosts.add(link.host);
6505
+ }
6506
+ for (const link of [...links.filter((link2) => link2.text), ...links.filter((link2) => !link2.text)]) {
6507
+ lookalike(link.host, { url: link.url });
6508
+ }
6509
+ for (const host of hosts) {
6510
+ lookalike(host, { source: "text" });
5517
6511
  }
5518
- const hosts = [...seenHosts].slice(0, config.phishing.maxHosts);
5519
- const domains = [...new Set(hosts.map((host) => registrableDomain(host)))];
6512
+ const checked = [...seenHosts].slice(0, config.phishing.maxHosts);
6513
+ const domains = [...new Set(checked.map((host) => registrableDomain(host)))];
5520
6514
  const [blocked, listed] = await Promise.all([
5521
- config.phishing.cloudflare && hosts.length > 0 ? withTimeout(this.dns.checkCloudflare(hosts, { adult: config.phishing.adult }), config.timeout, []) : [],
6515
+ config.phishing.cloudflare && checked.length > 0 ? withTimeout(this.dns.checkCloudflare(checked, { adult: config.phishing.adult }), config.timeout, []) : [],
5522
6516
  config.dnsbl.domain?.length > 0 && domains.length > 0 ? withTimeout(this.dns.checkDomains(domains, config.dnsbl.domain), config.timeout, []) : []
5523
6517
  ]);
5524
6518
  for (const { host, category } of blocked) {
@@ -5604,6 +6598,10 @@ var SpamScanner = class {
5604
6598
  * @param {object} [options.session] - what the receiving server knows:
5605
6599
  * remoteAddress (client IP), resolvedClientHostname (verified reverse
5606
6600
  * DNS), helo, and envelope { mailFrom: {address}, rcptTo: [{address}] }
6601
+ * @param {object} [options.authenticationResults] - SPF, DKIM, DMARC and
6602
+ * ARC results the receiving server already has, as mailauth's
6603
+ * authenticate() gives them ({spf, dkim: {results}, dmarc, arc}): scored
6604
+ * as if checked here, without checking again
5607
6605
  * @returns {Promise<object>} isSpam, score, threshold, action ("accept",
5608
6606
  * "tag" or "reject"), message, tests, results (per detector), links,
5609
6607
  * language, tokens, mail and metrics
@@ -5622,13 +6620,12 @@ var SpamScanner = class {
5622
6620
  const { raw, mail } = await this.parse(source);
5623
6621
  const extracted = this.getFeatures(mail);
5624
6622
  const classification = this.getClassification(extracted.features);
5625
- const body = segmentWords(`${mail.subject || ""}
6623
+ const obfuscation = obfuscationOf(`${mail.subject || ""}
5626
6624
  ${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLength });
5627
- const obfuscation = { invisible: body.invisible, mixed: body.mixed, styled: body.styled };
5628
6625
  const attachments = config.attachments === false ? [] : inspectAttachments(mail.attachments).filter((item) => config.macros !== false || !["macro", "pdf_active", "rtf_object"].includes(item.type)).map((item) => finding(item));
5629
6626
  const [links, authentication, reputation, dnsbl, viruses, toxicity, nsfw] = await Promise.all([
5630
- this.checkLinks(extracted.links, config),
5631
- config.authentication && session.remoteAddress ? authenticate(raw, {
6627
+ this.checkLinks(extracted.links, config, extracted.hosts),
6628
+ config.authenticationResults && typeof config.authenticationResults === "object" ? normalizeAuthOutput(config.authenticationResults) : config.authentication && session.remoteAddress ? authenticate(raw, {
5632
6629
  ip: session.remoteAddress,
5633
6630
  helo: session.helo,
5634
6631
  sender: session.envelope?.mailFrom?.address,
@@ -5677,7 +6674,12 @@ ${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLengt
5677
6674
  const close = scored.score >= low && scored.score < high;
5678
6675
  if (mode === "always" || mode === "auto" && (close || classification.category === "unsure" || classification.category === "disabled")) {
5679
6676
  try {
5680
- results.llm = await llm.classify(mail, { links: extracted.links, authentication: authentication ? summarizeAuth(authentication) : null });
6677
+ results.llm = await llm.classify(mail, {
6678
+ links: extracted.links,
6679
+ text: extracted.text,
6680
+ hidden: extracted.hidden,
6681
+ authentication: authentication ? summarizeAuth(authentication) : null
6682
+ });
5681
6683
  } catch (error) {
5682
6684
  debug2("LLM check failed: %s", error.message);
5683
6685
  results.llm = {