spamscanner 7.0.1 → 7.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/arf.cjs +9 -8
- package/dist/cjs/arf.cjs.map +2 -2
- package/dist/cjs/index.cjs +1160 -138
- package/dist/cjs/index.cjs.map +4 -4
- package/dist/esm/arf.js +9 -8
- package/dist/esm/arf.js.map +2 -2
- package/dist/esm/cli.js +3821 -2816
- package/dist/esm/cli.js.map +4 -4
- package/dist/esm/index.js +1137 -132
- package/dist/esm/index.js.map +4 -4
- package/dist/standalone/cli.cjs +183 -175
- package/dist/types/attachments.d.ts +9 -1
- package/dist/types/homograph.d.ts +12 -1
- package/dist/types/html.d.ts +33 -0
- package/dist/types/index.d.ts +7 -1
- package/dist/types/linear.d.ts +83 -0
- package/dist/types/tokenizer.d.ts +75 -3
- package/package.json +4 -1
package/dist/esm/index.js
CHANGED
|
@@ -7,9 +7,581 @@ import { simpleParser as simpleParser2 } from "mailparser";
|
|
|
7
7
|
// src/attachments.js
|
|
8
8
|
import { Buffer } from "node:buffer";
|
|
9
9
|
|
|
10
|
+
// src/linear.js
|
|
11
|
+
import { createRequire } from "node:module";
|
|
12
|
+
function loadRe2(load = createRequire(import.meta.url)) {
|
|
13
|
+
try {
|
|
14
|
+
return load("re2");
|
|
15
|
+
} catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
var RE2 = loadRe2();
|
|
20
|
+
var BOUNDED = /{([01]),(\d{2,})}/g;
|
|
21
|
+
var WHITESPACE = String.raw`\t\n\x0B\f\r \x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}`;
|
|
22
|
+
function re2Source(source) {
|
|
23
|
+
let out = "";
|
|
24
|
+
let inClass = false;
|
|
25
|
+
for (let i = 0; i < source.length; i++) {
|
|
26
|
+
const char = source[i];
|
|
27
|
+
if (char === "\\") {
|
|
28
|
+
const next = source[++i] ?? "";
|
|
29
|
+
if (next === "s") {
|
|
30
|
+
out += inClass ? WHITESPACE : `[${WHITESPACE}]`;
|
|
31
|
+
} else if (next === "S") {
|
|
32
|
+
if (inClass) {
|
|
33
|
+
throw new TypeError(String.raw`\S inside a character class has no RE2 equivalent here`);
|
|
34
|
+
}
|
|
35
|
+
out += `[^${WHITESPACE}]`;
|
|
36
|
+
} else {
|
|
37
|
+
out += `\\${next}`;
|
|
38
|
+
}
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (char === "[" && !inClass) {
|
|
42
|
+
inClass = true;
|
|
43
|
+
} else if (char === "]" && inClass) {
|
|
44
|
+
inClass = false;
|
|
45
|
+
}
|
|
46
|
+
out += char;
|
|
47
|
+
}
|
|
48
|
+
return out.replaceAll(BOUNDED, (_, min) => min === "0" ? "*" : "+");
|
|
49
|
+
}
|
|
50
|
+
function linear(source, flags = "", Engine = RE2) {
|
|
51
|
+
return Engine ? new Engine(re2Source(source), flags) : new RegExp(source, flags);
|
|
52
|
+
}
|
|
53
|
+
function trimEndOf(value, characters) {
|
|
54
|
+
let end = value.length;
|
|
55
|
+
while (end > 0 && characters.includes(value[end - 1])) {
|
|
56
|
+
end--;
|
|
57
|
+
}
|
|
58
|
+
return value.slice(0, end);
|
|
59
|
+
}
|
|
60
|
+
function replaceTags(html, replacement = " ") {
|
|
61
|
+
let out = "";
|
|
62
|
+
let from = 0;
|
|
63
|
+
let open = html.indexOf("<");
|
|
64
|
+
while (open !== -1) {
|
|
65
|
+
const close = html.indexOf(">", open + 1);
|
|
66
|
+
if (close === -1) {
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
out += `${html.slice(from, open)}${replacement}`;
|
|
70
|
+
from = close + 1;
|
|
71
|
+
open = html.indexOf("<", from);
|
|
72
|
+
}
|
|
73
|
+
return out + html.slice(from);
|
|
74
|
+
}
|
|
75
|
+
function caseless(word) {
|
|
76
|
+
return [...word].map((letter) => `[${letter}${letter.toUpperCase()}]`).join("");
|
|
77
|
+
}
|
|
78
|
+
var FOLDS_TO_ASCII = /[\u017F\u212A]/g;
|
|
79
|
+
function foldToAscii(text) {
|
|
80
|
+
return text.replaceAll(FOLDS_TO_ASCII, (character) => character === "\u017F" ? "s" : "k");
|
|
81
|
+
}
|
|
82
|
+
var OPENER = linear(String.raw`<(${caseless("script")}|${caseless("style")})\b`, "g");
|
|
83
|
+
var CLOSERS = {
|
|
84
|
+
script: linear(String.raw`<\/${caseless("script")}\s*>`, "g"),
|
|
85
|
+
style: linear(String.raw`<\/${caseless("style")}\s*>`, "g")
|
|
86
|
+
};
|
|
87
|
+
|
|
10
88
|
// src/tokenizer.js
|
|
11
89
|
import { parse as parseDomain } from "tldts";
|
|
12
90
|
|
|
91
|
+
// src/html.js
|
|
92
|
+
var LATIN1 = "nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr deg plusmn sup2 sup3 acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest Agrave Aacute Acirc Atilde Auml Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ETH Ntilde Ograve Oacute Ocirc Otilde Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig agrave aacute acirc atilde auml aring aelig ccedil egrave eacute ecirc euml igrave iacute icirc iuml eth ntilde ograve oacute ocirc otilde ouml divide oslash ugrave uacute ucirc uuml yacute thorn yuml".split(" ");
|
|
93
|
+
var ENTITIES = new Map([
|
|
94
|
+
...LATIN1.map((name, index) => [name, String.fromCodePoint(160 + index)]),
|
|
95
|
+
["amp", "&"],
|
|
96
|
+
["lt", "<"],
|
|
97
|
+
["gt", ">"],
|
|
98
|
+
["quot", '"'],
|
|
99
|
+
["apos", "'"],
|
|
100
|
+
["AMP", "&"],
|
|
101
|
+
["LT", "<"],
|
|
102
|
+
["GT", ">"],
|
|
103
|
+
["QUOT", '"'],
|
|
104
|
+
["COPY", "\xA9"],
|
|
105
|
+
["REG", "\xAE"],
|
|
106
|
+
["OElig", "\u0152"],
|
|
107
|
+
["oelig", "\u0153"],
|
|
108
|
+
["Scaron", "\u0160"],
|
|
109
|
+
["scaron", "\u0161"],
|
|
110
|
+
["Yuml", "\u0178"],
|
|
111
|
+
["fnof", "\u0192"],
|
|
112
|
+
["circ", "\u02C6"],
|
|
113
|
+
["tilde", "\u02DC"],
|
|
114
|
+
["ensp", "\u2002"],
|
|
115
|
+
["emsp", "\u2003"],
|
|
116
|
+
["thinsp", "\u2009"],
|
|
117
|
+
["zwnj", "\u200C"],
|
|
118
|
+
["zwj", "\u200D"],
|
|
119
|
+
["lrm", "\u200E"],
|
|
120
|
+
["rlm", "\u200F"],
|
|
121
|
+
["ndash", "\u2013"],
|
|
122
|
+
["mdash", "\u2014"],
|
|
123
|
+
["lsquo", "\u2018"],
|
|
124
|
+
["rsquo", "\u2019"],
|
|
125
|
+
["sbquo", "\u201A"],
|
|
126
|
+
["ldquo", "\u201C"],
|
|
127
|
+
["rdquo", "\u201D"],
|
|
128
|
+
["bdquo", "\u201E"],
|
|
129
|
+
["dagger", "\u2020"],
|
|
130
|
+
["Dagger", "\u2021"],
|
|
131
|
+
["bull", "\u2022"],
|
|
132
|
+
["hellip", "\u2026"],
|
|
133
|
+
["permil", "\u2030"],
|
|
134
|
+
["prime", "\u2032"],
|
|
135
|
+
["Prime", "\u2033"],
|
|
136
|
+
["lsaquo", "\u2039"],
|
|
137
|
+
["rsaquo", "\u203A"],
|
|
138
|
+
["euro", "\u20AC"],
|
|
139
|
+
["trade", "\u2122"],
|
|
140
|
+
["larr", "\u2190"],
|
|
141
|
+
["uarr", "\u2191"],
|
|
142
|
+
["rarr", "\u2192"],
|
|
143
|
+
["darr", "\u2193"],
|
|
144
|
+
["hearts", "\u2665"],
|
|
145
|
+
["check", "\u2713"]
|
|
146
|
+
]);
|
|
147
|
+
var LEGACY = [...LATIN1, "amp", "lt", "gt", "quot", "AMP", "LT", "GT", "QUOT", "COPY", "REG"].sort((a, b) => b.length - a.length);
|
|
148
|
+
var WINDOWS_1252 = [8364, 129, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338, 141, 381, 143, 144, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482, 353, 8250, 339, 157, 382, 376];
|
|
149
|
+
var REFERENCE = /&(?:#([xX])([\da-fA-F]+)|#(\d+)|([a-zA-Z][a-zA-Z\d]{0,31}))(;?)/g;
|
|
150
|
+
function fromCode(code) {
|
|
151
|
+
if (code === 0 || code > 1114111 || code >= 55296 && code <= 57343) {
|
|
152
|
+
return "\uFFFD";
|
|
153
|
+
}
|
|
154
|
+
if (code >= 128 && code <= 159) {
|
|
155
|
+
return String.fromCodePoint(WINDOWS_1252[code - 128]);
|
|
156
|
+
}
|
|
157
|
+
return String.fromCodePoint(code);
|
|
158
|
+
}
|
|
159
|
+
function decodeEntities(value, { attribute = false } = {}) {
|
|
160
|
+
return value.replaceAll(REFERENCE, (...found) => {
|
|
161
|
+
const [match, , hex, decimal, name, semicolon, offset] = found;
|
|
162
|
+
if (hex !== void 0 || decimal !== void 0) {
|
|
163
|
+
return fromCode(hex === void 0 ? Number.parseInt(decimal, 10) : Number.parseInt(hex, 16));
|
|
164
|
+
}
|
|
165
|
+
if (semicolon) {
|
|
166
|
+
return ENTITIES.get(name) ?? match;
|
|
167
|
+
}
|
|
168
|
+
const legacy = LEGACY.find((known) => name.startsWith(known));
|
|
169
|
+
if (!legacy) {
|
|
170
|
+
return match;
|
|
171
|
+
}
|
|
172
|
+
const rest = match.slice(legacy.length + 1);
|
|
173
|
+
const next = rest[0] ?? value[offset + match.length] ?? "";
|
|
174
|
+
if (attribute && /[\da-zA-Z=]/.test(next)) {
|
|
175
|
+
return match;
|
|
176
|
+
}
|
|
177
|
+
return ENTITIES.get(legacy) + rest;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
var UNSHOWN = /* @__PURE__ */ new Set(["script", "style", "title", "noembed", "noframes", "iframe"]);
|
|
181
|
+
var RAW_SHOWN = /* @__PURE__ */ new Set(["textarea", "xmp"]);
|
|
182
|
+
var RAW_END = Object.fromEntries([...UNSHOWN, ...RAW_SHOWN].map((name) => [name, linear(String.raw`<\/${caseless(name)}[\s/>]`, "g")]));
|
|
183
|
+
var BREAKS = /* @__PURE__ */ new Set(["address", "applet", "area", "article", "aside", "audio", "blockquote", "body", "br", "button", "canvas", "caption", "center", "col", "colgroup", "dd", "details", "dialog", "dir", "div", "dl", "dt", "embed", "fieldset", "figcaption", "figure", "footer", "form", "frame", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hgroup", "hr", "html", "iframe", "image", "img", "input", "keygen", "legend", "li", "listing", "main", "marquee", "math", "menu", "meter", "nav", "object", "ol", "optgroup", "option", "p", "picture", "plaintext", "pre", "progress", "section", "select", "summary", "svg", "table", "tbody", "td", "textarea", "tfoot", "th", "thead", "tr", "ul", "video", "xmp"]);
|
|
184
|
+
var SPECIAL = /* @__PURE__ */ new Set(["address", "applet", "area", "article", "aside", "base", "blockquote", "body", "button", "caption", "center", "col", "colgroup", "dd", "details", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hgroup", "html", "iframe", "li", "listing", "main", "marquee", "menu", "nav", "object", "ol", "p", "plaintext", "pre", "section", "select", "summary", "table", "tbody", "td", "template", "textarea", "tfoot", "th", "thead", "tr", "ul", "xmp"]);
|
|
185
|
+
var FORMATTING = /* @__PURE__ */ new Set(["a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u"]);
|
|
186
|
+
var LINE = /* @__PURE__ */ new Set(["br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", "table", "blockquote", "pre"]);
|
|
187
|
+
var VOID = /* @__PURE__ */ new Set(["area", "base", "basefont", "bgsound", "br", "col", "command", "embed", "frame", "hr", "image", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"]);
|
|
188
|
+
var CLOSES_PARAGRAPH = /* @__PURE__ */ new Set(["address", "article", "aside", "blockquote", "center", "dd", "details", "dialog", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "li", "main", "menu", "nav", "ol", "p", "pre", "section", "table", "ul", "xmp"]);
|
|
189
|
+
var PARAGRAPH = [/* @__PURE__ */ new Set(["p"]), /* @__PURE__ */ new Set(["button", "table", "td", "th", "caption", "marquee", "object", "applet", "html"])];
|
|
190
|
+
var IMPLIED = {
|
|
191
|
+
li: [/* @__PURE__ */ new Set(["li"]), /* @__PURE__ */ new Set(["ul", "ol", "menu"])],
|
|
192
|
+
dt: [/* @__PURE__ */ new Set(["dt", "dd"]), /* @__PURE__ */ new Set(["dl"])],
|
|
193
|
+
dd: [/* @__PURE__ */ new Set(["dt", "dd"]), /* @__PURE__ */ new Set(["dl"])],
|
|
194
|
+
td: [/* @__PURE__ */ new Set(["td", "th"]), /* @__PURE__ */ new Set(["tr", "table"])],
|
|
195
|
+
th: [/* @__PURE__ */ new Set(["td", "th"]), /* @__PURE__ */ new Set(["tr", "table"])],
|
|
196
|
+
tr: [/* @__PURE__ */ new Set(["tr"]), /* @__PURE__ */ new Set(["table", "tbody", "thead", "tfoot"])],
|
|
197
|
+
tbody: [/* @__PURE__ */ new Set(["tbody", "thead", "tfoot"]), /* @__PURE__ */ new Set(["table"])],
|
|
198
|
+
thead: [/* @__PURE__ */ new Set(["tbody", "thead", "tfoot"]), /* @__PURE__ */ new Set(["table"])],
|
|
199
|
+
tfoot: [/* @__PURE__ */ new Set(["tbody", "thead", "tfoot"]), /* @__PURE__ */ new Set(["table"])],
|
|
200
|
+
option: [/* @__PURE__ */ new Set(["option"]), /* @__PURE__ */ new Set(["select", "datalist", "optgroup"])],
|
|
201
|
+
optgroup: [/* @__PURE__ */ new Set(["optgroup", "option"]), /* @__PURE__ */ new Set(["select"])]
|
|
202
|
+
};
|
|
203
|
+
var MAX_OPEN = 1024;
|
|
204
|
+
var MAX_SEARCH = 64;
|
|
205
|
+
var MAX_ATTRIBUTES = 64;
|
|
206
|
+
var MAX_STYLE = 1e4;
|
|
207
|
+
var COMMENT_END = linear("--!?>", "g");
|
|
208
|
+
var WHITESPACE2 = /* @__PURE__ */ new Set([" ", " ", "\n", "\r", "\f"]);
|
|
209
|
+
function readTag(html, start) {
|
|
210
|
+
const closing = html[start + 1] === "/";
|
|
211
|
+
let i = start + (closing ? 2 : 1);
|
|
212
|
+
const nameStart = i;
|
|
213
|
+
while (i < html.length && !WHITESPACE2.has(html[i]) && html[i] !== "/" && html[i] !== ">") {
|
|
214
|
+
i++;
|
|
215
|
+
}
|
|
216
|
+
const name = html.slice(nameStart, Math.min(i, nameStart + 64)).toLowerCase();
|
|
217
|
+
const attributes = /* @__PURE__ */ new Map();
|
|
218
|
+
let slash = false;
|
|
219
|
+
while (i < html.length) {
|
|
220
|
+
const char = html[i];
|
|
221
|
+
if (char === ">") {
|
|
222
|
+
return {
|
|
223
|
+
end: i + 1,
|
|
224
|
+
name,
|
|
225
|
+
closing,
|
|
226
|
+
attributes,
|
|
227
|
+
selfClosing: slash
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
if (WHITESPACE2.has(char) || char === "/") {
|
|
231
|
+
slash = char === "/" || slash && WHITESPACE2.has(char);
|
|
232
|
+
i++;
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
slash = false;
|
|
236
|
+
const attributeStart = i++;
|
|
237
|
+
while (i < html.length && !WHITESPACE2.has(html[i]) && html[i] !== "/" && html[i] !== ">" && html[i] !== "=") {
|
|
238
|
+
i++;
|
|
239
|
+
}
|
|
240
|
+
const attribute = html.slice(attributeStart, i).toLowerCase();
|
|
241
|
+
while (WHITESPACE2.has(html[i])) {
|
|
242
|
+
i++;
|
|
243
|
+
}
|
|
244
|
+
let value = "";
|
|
245
|
+
if (html[i] === "=") {
|
|
246
|
+
i++;
|
|
247
|
+
while (WHITESPACE2.has(html[i])) {
|
|
248
|
+
i++;
|
|
249
|
+
}
|
|
250
|
+
if (html[i] === '"' || html[i] === "'") {
|
|
251
|
+
const close = html.indexOf(html[i], i + 1);
|
|
252
|
+
if (close === -1) {
|
|
253
|
+
break;
|
|
254
|
+
}
|
|
255
|
+
value = html.slice(i + 1, close);
|
|
256
|
+
i = close + 1;
|
|
257
|
+
} else {
|
|
258
|
+
const valueStart = i;
|
|
259
|
+
while (i < html.length && !WHITESPACE2.has(html[i]) && html[i] !== ">") {
|
|
260
|
+
i++;
|
|
261
|
+
}
|
|
262
|
+
value = html.slice(valueStart, i);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
if (!attributes.has(attribute) && attributes.size < MAX_ATTRIBUTES) {
|
|
266
|
+
attributes.set(attribute, value);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return { end: -1 };
|
|
270
|
+
}
|
|
271
|
+
function declarations(style) {
|
|
272
|
+
let css = "";
|
|
273
|
+
let from = 0;
|
|
274
|
+
const text = style.slice(0, MAX_STYLE);
|
|
275
|
+
while (from < text.length) {
|
|
276
|
+
const open = text.indexOf("/*", from);
|
|
277
|
+
if (open === -1) {
|
|
278
|
+
css += text.slice(from);
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
css += text.slice(from, open);
|
|
282
|
+
const close = text.indexOf("*/", open + 2);
|
|
283
|
+
if (close === -1) {
|
|
284
|
+
break;
|
|
285
|
+
}
|
|
286
|
+
from = close + 2;
|
|
287
|
+
}
|
|
288
|
+
const found = /* @__PURE__ */ new Map();
|
|
289
|
+
for (const declaration of css.split(";")) {
|
|
290
|
+
const colon = declaration.indexOf(":");
|
|
291
|
+
if (colon === -1) {
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
const property = declaration.slice(0, colon).trim().toLowerCase();
|
|
295
|
+
let value = declaration.slice(colon + 1).trim().toLowerCase();
|
|
296
|
+
const important = /!\s*important$/.test(value);
|
|
297
|
+
if (important) {
|
|
298
|
+
value = value.replace(/!\s*important$/, "").trim();
|
|
299
|
+
}
|
|
300
|
+
const before = found.get(property);
|
|
301
|
+
if (!before || important || !before.important) {
|
|
302
|
+
found.set(property, { value, important });
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
return found;
|
|
306
|
+
}
|
|
307
|
+
var MAX_VALUE = 64;
|
|
308
|
+
var LENGTH = /^([+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)(px|pt|pc|in|cm|mm|q|em|rem|%|ex|ch|)$/;
|
|
309
|
+
var UNITS = {
|
|
310
|
+
px: 1,
|
|
311
|
+
pt: 4 / 3,
|
|
312
|
+
pc: 16,
|
|
313
|
+
in: 96,
|
|
314
|
+
cm: 96 / 2.54,
|
|
315
|
+
mm: 96 / 25.4,
|
|
316
|
+
q: 96 / 101.6,
|
|
317
|
+
rem: 16
|
|
318
|
+
};
|
|
319
|
+
var SIZE_KEYWORDS = {
|
|
320
|
+
"xx-small": 9,
|
|
321
|
+
"x-small": 10,
|
|
322
|
+
small: 13,
|
|
323
|
+
medium: 16,
|
|
324
|
+
large: 18,
|
|
325
|
+
"x-large": 24,
|
|
326
|
+
"xx-large": 32,
|
|
327
|
+
"xxx-large": 48
|
|
328
|
+
};
|
|
329
|
+
function pixels(value, parent) {
|
|
330
|
+
const length = value.length > MAX_VALUE ? null : LENGTH.exec(value);
|
|
331
|
+
if (!length) {
|
|
332
|
+
return null;
|
|
333
|
+
}
|
|
334
|
+
const number = Number(length[1]);
|
|
335
|
+
if (!Number.isFinite(number)) {
|
|
336
|
+
return null;
|
|
337
|
+
}
|
|
338
|
+
const unit = length[2];
|
|
339
|
+
if (unit === "em") {
|
|
340
|
+
return number * parent;
|
|
341
|
+
}
|
|
342
|
+
if (unit === "%") {
|
|
343
|
+
return number * parent / 100;
|
|
344
|
+
}
|
|
345
|
+
if (unit === "ex" || unit === "ch") {
|
|
346
|
+
return number * parent / 2;
|
|
347
|
+
}
|
|
348
|
+
return number * (UNITS[unit] ?? 1);
|
|
349
|
+
}
|
|
350
|
+
function fontSize(value, parent) {
|
|
351
|
+
if (Object.hasOwn(SIZE_KEYWORDS, value)) {
|
|
352
|
+
return SIZE_KEYWORDS[value];
|
|
353
|
+
}
|
|
354
|
+
if (value === "smaller") {
|
|
355
|
+
return parent / 1.2;
|
|
356
|
+
}
|
|
357
|
+
if (value === "larger") {
|
|
358
|
+
return parent * 1.2;
|
|
359
|
+
}
|
|
360
|
+
const size = pixels(value, parent);
|
|
361
|
+
return size === null || size < 0 ? null : size;
|
|
362
|
+
}
|
|
363
|
+
function shorthandSize(value, parent) {
|
|
364
|
+
for (const token of value.slice(0, MAX_STYLE).split(/\s+/)) {
|
|
365
|
+
const [size, lineHeight] = token.split("/");
|
|
366
|
+
if (lineHeight === void 0 && /^[+-]?[\d.]+$/.test(size)) {
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
const pixelSize = fontSize(size, parent);
|
|
370
|
+
if (pixelSize !== null) {
|
|
371
|
+
return pixelSize;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
return null;
|
|
375
|
+
}
|
|
376
|
+
function isTransparent(color) {
|
|
377
|
+
if (color === "transparent") {
|
|
378
|
+
return true;
|
|
379
|
+
}
|
|
380
|
+
const hex = /^#(?:[\da-f]{3}(0)|[\da-f]{6}(00))$/.exec(color);
|
|
381
|
+
if (hex) {
|
|
382
|
+
return true;
|
|
383
|
+
}
|
|
384
|
+
const functional = /^(?:rgb|hsl)a?\((.*)\)$/.exec(color);
|
|
385
|
+
if (!functional) {
|
|
386
|
+
return false;
|
|
387
|
+
}
|
|
388
|
+
const slash = functional[1].lastIndexOf("/");
|
|
389
|
+
const parts = functional[1].split(",");
|
|
390
|
+
if (slash === -1 && parts.length < 4) {
|
|
391
|
+
return false;
|
|
392
|
+
}
|
|
393
|
+
const alpha = (slash === -1 ? parts.at(-1) : functional[1].slice(slash + 1)).trim();
|
|
394
|
+
return ZERO_NUMBER.test(alpha);
|
|
395
|
+
}
|
|
396
|
+
var ZERO_NUMBER = /^[+-]?(?:0+(?:\.0*)?|\.0+)%?$/;
|
|
397
|
+
var isZero = (value) => {
|
|
398
|
+
const size = pixels(value, 16);
|
|
399
|
+
return size !== null && size <= 0;
|
|
400
|
+
};
|
|
401
|
+
function styleState(style, parent) {
|
|
402
|
+
const css = declarations(style);
|
|
403
|
+
const value = (property) => css.get(property)?.value;
|
|
404
|
+
const opacity = value("opacity");
|
|
405
|
+
const overflow = [value("overflow"), value("overflow-x"), value("overflow-y")].some((item) => item === "hidden" || item === "clip");
|
|
406
|
+
const noHeight = (isZero(value("height") ?? "1") || isZero(value("max-height") ?? "1")) && isZero(value("min-height") ?? "0");
|
|
407
|
+
const noWidth = (isZero(value("width") ?? "1") || isZero(value("max-width") ?? "1")) && isZero(value("min-width") ?? "0");
|
|
408
|
+
const indent = pixels(value("text-indent") ?? "0", parent.size);
|
|
409
|
+
const offPage = ["absolute", "fixed"].includes(value("position")) && ["left", "top"].some((side) => (pixels(value(side) ?? "0", parent.size) ?? 0) <= -999);
|
|
410
|
+
const hides = value("display") === "none" || opacity !== void 0 && (ZERO_NUMBER.test(opacity) || opacity.length <= MAX_VALUE && Number.parseFloat(opacity) < 0) || overflow && (noHeight || noWidth) || indent !== null && indent <= -999 || offPage;
|
|
411
|
+
const visibility = value("visibility");
|
|
412
|
+
const color = value("color");
|
|
413
|
+
const size = value("font-size") === void 0 ? value("font") === void 0 ? null : shorthandSize(value("font"), parent.size) : fontSize(value("font-size"), parent.size);
|
|
414
|
+
return {
|
|
415
|
+
hides,
|
|
416
|
+
display: value("display"),
|
|
417
|
+
visible: visibility === "hidden" || visibility === "collapse" ? false : visibility === "visible" ? true : parent.visible,
|
|
418
|
+
transparent: color === void 0 ? parent.transparent : isTransparent(color),
|
|
419
|
+
size: size ?? parent.size
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
var ROOT = {
|
|
423
|
+
visible: true,
|
|
424
|
+
transparent: false,
|
|
425
|
+
size: 16,
|
|
426
|
+
hides: false
|
|
427
|
+
};
|
|
428
|
+
function visibleText(html) {
|
|
429
|
+
if (typeof html !== "string" || html === "") {
|
|
430
|
+
return { text: "", hidden: 0 };
|
|
431
|
+
}
|
|
432
|
+
const parts = [];
|
|
433
|
+
let hidden = 0;
|
|
434
|
+
const stack = [];
|
|
435
|
+
const open = /* @__PURE__ */ new Map();
|
|
436
|
+
let hiding = 0;
|
|
437
|
+
let collapsed = 0;
|
|
438
|
+
const state = () => stack.at(-1) ?? ROOT;
|
|
439
|
+
const shown = () => hiding === 0 && collapsed === 0 && state().visible && !state().transparent && state().size > 1;
|
|
440
|
+
const write = (text) => {
|
|
441
|
+
if (text === "") {
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
if (shown()) {
|
|
445
|
+
parts.push(text);
|
|
446
|
+
} else {
|
|
447
|
+
hidden += text.replaceAll(/\s+/g, "").length;
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
const popTo = (index) => {
|
|
451
|
+
while (stack.length > index) {
|
|
452
|
+
const element = stack.pop();
|
|
453
|
+
open.set(element.name, open.get(element.name) - 1);
|
|
454
|
+
if (element.hides) {
|
|
455
|
+
hiding--;
|
|
456
|
+
}
|
|
457
|
+
collapsed -= (element.collapses ? 1 : 0) - (element.reveals ? 1 : 0);
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
const closeImplied = ([names, limits]) => {
|
|
461
|
+
for (let i = stack.length - 1; i >= Math.max(0, stack.length - MAX_SEARCH); i--) {
|
|
462
|
+
if (names.has(stack[i].name)) {
|
|
463
|
+
popTo(i);
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
if (limits.has(stack[i].name)) {
|
|
467
|
+
return;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
};
|
|
471
|
+
let from = 0;
|
|
472
|
+
while (from < html.length) {
|
|
473
|
+
const lt = html.indexOf("<", from);
|
|
474
|
+
if (lt === -1) {
|
|
475
|
+
write(decodeEntities(html.slice(from)));
|
|
476
|
+
break;
|
|
477
|
+
}
|
|
478
|
+
write(decodeEntities(html.slice(from, lt)));
|
|
479
|
+
from = lt + 1;
|
|
480
|
+
const next = html[lt + 1] ?? "";
|
|
481
|
+
if (html.startsWith("<!--", lt)) {
|
|
482
|
+
if (html.startsWith(">", lt + 4) || html.startsWith("->", lt + 4)) {
|
|
483
|
+
from = html.indexOf(">", lt + 4) + 1;
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
COMMENT_END.lastIndex = lt + 4;
|
|
487
|
+
const end = COMMENT_END.exec(html);
|
|
488
|
+
from = end === null ? html.length : end.index + end[0].length;
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
const closingTag = next === "/";
|
|
492
|
+
const letter = /[a-zA-Z]/.test(closingTag ? html[lt + 2] ?? "" : next);
|
|
493
|
+
if (!letter) {
|
|
494
|
+
if (closingTag && lt + 2 >= html.length) {
|
|
495
|
+
write("</");
|
|
496
|
+
break;
|
|
497
|
+
}
|
|
498
|
+
if (next === "!" || next === "?" || closingTag && html[lt + 2] !== ">") {
|
|
499
|
+
const end = html.indexOf(">", lt + 2);
|
|
500
|
+
from = end === -1 ? html.length : end + 1;
|
|
501
|
+
} else if (closingTag) {
|
|
502
|
+
from = lt + 3;
|
|
503
|
+
} else {
|
|
504
|
+
write("<");
|
|
505
|
+
}
|
|
506
|
+
continue;
|
|
507
|
+
}
|
|
508
|
+
const tag = readTag(html, lt);
|
|
509
|
+
if (tag.end === -1) {
|
|
510
|
+
break;
|
|
511
|
+
}
|
|
512
|
+
from = tag.end;
|
|
513
|
+
const { name } = tag;
|
|
514
|
+
if (tag.closing) {
|
|
515
|
+
const through = SPECIAL.has(name) || FORMATTING.has(name);
|
|
516
|
+
const limit = through ? 0 : Math.max(0, stack.length - MAX_SEARCH);
|
|
517
|
+
for (let i = stack.length - 1; open.get(name) > 0 && i >= limit; i--) {
|
|
518
|
+
if (stack[i].name === name) {
|
|
519
|
+
popTo(i);
|
|
520
|
+
break;
|
|
521
|
+
}
|
|
522
|
+
if (!through && SPECIAL.has(stack[i].name)) {
|
|
523
|
+
break;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
} else {
|
|
527
|
+
if (CLOSES_PARAGRAPH.has(name)) {
|
|
528
|
+
closeImplied(PARAGRAPH);
|
|
529
|
+
}
|
|
530
|
+
if (IMPLIED[name]) {
|
|
531
|
+
closeImplied(IMPLIED[name]);
|
|
532
|
+
}
|
|
533
|
+
const foreign = (open.get("svg") ?? 0) + (open.get("math") ?? 0) > 0;
|
|
534
|
+
if (!VOID.has(name) && !(foreign && tag.selfClosing)) {
|
|
535
|
+
if (stack.length >= MAX_OPEN) {
|
|
536
|
+
popTo(stack.length - 1);
|
|
537
|
+
}
|
|
538
|
+
const style = tag.attributes.get("style");
|
|
539
|
+
const parent = state();
|
|
540
|
+
const element = {
|
|
541
|
+
...style === void 0 ? {
|
|
542
|
+
visible: parent.visible,
|
|
543
|
+
transparent: parent.transparent,
|
|
544
|
+
size: parent.size,
|
|
545
|
+
hides: false
|
|
546
|
+
} : styleState(decodeEntities(style, { attribute: true }), parent),
|
|
547
|
+
name
|
|
548
|
+
};
|
|
549
|
+
if (tag.attributes.has("hidden") && (element.display === void 0 || element.display === "none") || name === "template") {
|
|
550
|
+
element.hides = true;
|
|
551
|
+
}
|
|
552
|
+
element.collapses = name === "details" && !tag.attributes.has("open");
|
|
553
|
+
element.reveals = name === "summary" && parent.collapses === true;
|
|
554
|
+
stack.push(element);
|
|
555
|
+
open.set(name, (open.get(name) ?? 0) + 1);
|
|
556
|
+
if (element.hides) {
|
|
557
|
+
hiding++;
|
|
558
|
+
}
|
|
559
|
+
collapsed += (element.collapses ? 1 : 0) - (element.reveals ? 1 : 0);
|
|
560
|
+
}
|
|
561
|
+
if (name === "plaintext") {
|
|
562
|
+
parts.push("\n");
|
|
563
|
+
write(html.slice(tag.end));
|
|
564
|
+
break;
|
|
565
|
+
}
|
|
566
|
+
if ((UNSHOWN.has(name) || RAW_SHOWN.has(name)) && !(foreign && tag.selfClosing)) {
|
|
567
|
+
const closer = RAW_END[name];
|
|
568
|
+
closer.lastIndex = tag.end;
|
|
569
|
+
const close = closer.exec(html);
|
|
570
|
+
const end = close === null ? html.length : close.index;
|
|
571
|
+
if (RAW_SHOWN.has(name)) {
|
|
572
|
+
const raw = html.slice(tag.end, end);
|
|
573
|
+
write(name === "textarea" ? decodeEntities(raw) : raw);
|
|
574
|
+
}
|
|
575
|
+
from = end;
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
if (BREAKS.has(name)) {
|
|
579
|
+
parts.push(LINE.has(name) ? "\n" : " ");
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
return { text: parts.join(""), hidden };
|
|
583
|
+
}
|
|
584
|
+
|
|
13
585
|
// src/language.js
|
|
14
586
|
import { franc } from "franc";
|
|
15
587
|
var SCRIPTS = [
|
|
@@ -415,13 +987,30 @@ var LEET = {
|
|
|
415
987
|
8: "b"
|
|
416
988
|
};
|
|
417
989
|
var UNSPACED = /* @__PURE__ */ new Set(["Han", "Hiragana", "Katakana", "Thai", "Lao", "Khmer", "Myanmar"]);
|
|
418
|
-
var URL_PATTERN =
|
|
419
|
-
var
|
|
420
|
-
var
|
|
421
|
-
var
|
|
422
|
-
var
|
|
423
|
-
var
|
|
424
|
-
var
|
|
990
|
+
var URL_PATTERN = linear(String.raw`\b(?:https?:\/\/|www\.)[^\s<>"'\x60]+|\b[a-z\d][a-z\d-]{0,62}(?:\.[a-z\d-]{1,63}){0,10}\.[a-z]{2,24}\/[^\s<>"'\x60]*`, "giu");
|
|
991
|
+
var EMAIL_SOURCE = String.raw`[\p{L}\p{N}._%+-]{1,64}@([\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63}){0,10}\.\p{L}{2,24})`;
|
|
992
|
+
var EMAIL_PATTERN = linear(EMAIL_SOURCE, "gu");
|
|
993
|
+
var PHONE_PATTERN = linear(String.raw`(?:\+|0{2})\d[\d\s().-]{7,}\d|\(\d{3}\)\s?\d{3}[\s.-]\d{4}|\b(?:\d{3}[.-]){2}\d{4}\b`, "g");
|
|
994
|
+
var IP_PATTERN = linear(String.raw`\b(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}\b`, "g");
|
|
995
|
+
var MONEY_PATTERN = linear(String.raw`[$€£¥₹₽₩₺₦]\s?\d|\d\s?(?:[$€£¥₹₽₩₺₦]|usd|eur|gbp|dollars?|euros?)\b`, "giu");
|
|
996
|
+
var BITCOIN_PATTERN = linear(String.raw`\b(?:bc1[ac-hj-np-z02-9]{25,60}|[13][a-km-zA-HJ-NP-Z1-9]{25,34})\b`, "g");
|
|
997
|
+
var CARD_PATTERN = linear(String.raw`\b(?:\d[ -]?){13,19}\b`, "g");
|
|
998
|
+
var ANCHOR_START = linear(String.raw`<a\b`, "gi");
|
|
999
|
+
var HREF = linear(String.raw`\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))`, "i");
|
|
1000
|
+
var ANCHOR_CLOSE = linear(String.raw`<\/a\s*>`, "gi");
|
|
1001
|
+
var SHOWN_URL = linear(String.raw`^(?:https?:\/\/|www\.)\S+$|^[a-z\d-]{1,63}(?:\.[a-z\d-]{1,63}){1,10}(?:\/\S*)?$`, "i");
|
|
1002
|
+
var IMG_PIXEL = linear(String.raw`<img\b[^<>]*\b(?:width|height)\s*=\s*["']?[01]["'\s>]`, "i");
|
|
1003
|
+
var IMG = linear(String.raw`<img\b`, "gi");
|
|
1004
|
+
var BARE_HOST = linear(String.raw`\b[a-z\d][a-z\d-]{0,62}(?:\.[a-z\d-]{1,63}){0,9}\.[a-z]{2,24}\b`, "gi");
|
|
1005
|
+
var MAX_BARE_HOSTS = 50;
|
|
1006
|
+
var NAME_END = /[\p{L}\p{N}\p{M}_]$/u;
|
|
1007
|
+
function continuesName(text, index) {
|
|
1008
|
+
if (NAME_END.test(text.slice(Math.max(0, index - 2), index))) {
|
|
1009
|
+
return true;
|
|
1010
|
+
}
|
|
1011
|
+
return (text[index - 1] === "." || text[index - 1] === "-") && NAME_END.test(text.slice(Math.max(0, index - 3), index - 1));
|
|
1012
|
+
}
|
|
1013
|
+
var FILE_TLDS = /* @__PURE__ */ new Set(["ai", "app", "md", "mov", "pm", "ps", "py", "rs", "sh", "so", "zip"]);
|
|
425
1014
|
var SHORTENERS = /* @__PURE__ */ new Set(["bit.ly", "tinyurl.com", "goo.gl", "t.co", "ow.ly", "is.gd", "buff.ly", "rebrand.ly", "cutt.ly", "shorturl.at", "tiny.cc", "rb.gy", "bit.do", "v.gd", "s.id", "t.ly", "lnkd.in", "qrco.de", "shorturl.asia", "urlz.fr"]);
|
|
426
1015
|
var FREEMAIL = /* @__PURE__ */ new Set(["gmail.com", "googlemail.com", "yahoo.com", "ymail.com", "hotmail.com", "outlook.com", "live.com", "msn.com", "aol.com", "icloud.com", "me.com", "mail.com", "gmx.com", "gmx.de", "gmx.net", "web.de", "yandex.ru", "yandex.com", "mail.ru", "qq.com", "163.com", "126.com", "proton.me", "protonmail.com", "zoho.com", "naver.com", "daum.net", "rediffmail.com", "libero.it", "orange.fr", "laposte.net", "wp.pl", "o2.pl", "seznam.cz", "rambler.ru", "tutanota.com", "tuta.com"]);
|
|
427
1016
|
var segmenters = /* @__PURE__ */ new Map();
|
|
@@ -456,6 +1045,9 @@ function chunk(text, size = 1e3) {
|
|
|
456
1045
|
}
|
|
457
1046
|
return pieces;
|
|
458
1047
|
}
|
|
1048
|
+
function readableText(text) {
|
|
1049
|
+
return typeof text === "string" ? text.replaceAll(OBFUSCATING, "").normalize("NFKC") : "";
|
|
1050
|
+
}
|
|
459
1051
|
function normalizeText(text) {
|
|
460
1052
|
if (typeof text !== "string" || text === "") {
|
|
461
1053
|
return { text: "", invisible: 0, styled: false };
|
|
@@ -523,6 +1115,24 @@ function segmentWords(text, options = {}) {
|
|
|
523
1115
|
leet
|
|
524
1116
|
};
|
|
525
1117
|
}
|
|
1118
|
+
var LATIN = new RegExp("\\p{Script=Latin}", "u");
|
|
1119
|
+
var CYRILLIC_OR_GREEK = /[\p{Script=Cyrillic}\p{Script=Greek}]/u;
|
|
1120
|
+
function obfuscationOf(text, options = {}) {
|
|
1121
|
+
const { maxLength = 1e5 } = options;
|
|
1122
|
+
const normalized = normalizeText(typeof text === "string" ? text.slice(0, maxLength) : "");
|
|
1123
|
+
let mixed = 0;
|
|
1124
|
+
for (const piece of chunk(normalized.text)) {
|
|
1125
|
+
if (!LATIN.test(piece) || !CYRILLIC_OR_GREEK.test(piece)) {
|
|
1126
|
+
continue;
|
|
1127
|
+
}
|
|
1128
|
+
for (const { segment, isWordLike } of segmenter().segment(piece)) {
|
|
1129
|
+
if (isWordLike && mixedScripts(segment)) {
|
|
1130
|
+
mixed++;
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
return { invisible: normalized.invisible, mixed, styled: normalized.styled };
|
|
1135
|
+
}
|
|
526
1136
|
function bucket(n, steps) {
|
|
527
1137
|
let i = 0;
|
|
528
1138
|
while (i < steps.length && n > steps[i]) {
|
|
@@ -535,10 +1145,12 @@ function registrableDomain(host) {
|
|
|
535
1145
|
return parsed.domain || parsed.hostname || host;
|
|
536
1146
|
}
|
|
537
1147
|
function extractLinks(text, html) {
|
|
1148
|
+
text = typeof text === "string" ? foldToAscii(text) : text;
|
|
1149
|
+
html = typeof html === "string" ? foldToAscii(html) : html;
|
|
538
1150
|
const links = [];
|
|
539
1151
|
const seen = /* @__PURE__ */ new Set();
|
|
540
1152
|
const add = (raw, shown = null) => {
|
|
541
|
-
let href = raw.trim()
|
|
1153
|
+
let href = trimEndOf(raw.trim(), `).,;:!?'"]`);
|
|
542
1154
|
if (/^www\./i.test(href)) {
|
|
543
1155
|
href = `http://${href}`;
|
|
544
1156
|
}
|
|
@@ -563,15 +1175,12 @@ ${shown || ""}`;
|
|
|
563
1175
|
links.push({ url: url.href, host: url.hostname.toLowerCase().replace(/\.$/, ""), text: shown });
|
|
564
1176
|
};
|
|
565
1177
|
if (typeof html === "string" && html) {
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
const href = decodeEntities(match[1] || match[2] || match[3]);
|
|
569
|
-
const inner = decodeEntities(match[4].replaceAll(/<[^>]*>/g, "")).trim();
|
|
570
|
-
const shown = /^(?:https?:\/\/|www\.)\S+$|^[a-z\d-]+(?:\.[a-z\d-]+)+(?:\/\S*)?$/i.test(inner) ? inner : null;
|
|
1178
|
+
for (const { href, inner } of anchors(html)) {
|
|
1179
|
+
const shown = SHOWN_URL.test(inner) ? inner : null;
|
|
571
1180
|
add(href, shown);
|
|
572
1181
|
}
|
|
573
1182
|
}
|
|
574
|
-
for (const source of [text, typeof html === "string" ? html
|
|
1183
|
+
for (const source of [text, typeof html === "string" ? withoutAnchorTags(html) : ""]) {
|
|
575
1184
|
if (typeof source === "string" && source) {
|
|
576
1185
|
for (const match of source.slice(0, 5e5).matchAll(URL_PATTERN)) {
|
|
577
1186
|
add(match[0]);
|
|
@@ -580,22 +1189,97 @@ ${shown || ""}`;
|
|
|
580
1189
|
}
|
|
581
1190
|
return links;
|
|
582
1191
|
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
1192
|
+
function tagEnd(html, start) {
|
|
1193
|
+
const limit = Math.min(html.length, start + 6e3);
|
|
1194
|
+
let quote = null;
|
|
1195
|
+
for (let i = start + 1; i < limit; i++) {
|
|
1196
|
+
const char = html[i];
|
|
1197
|
+
if (quote) {
|
|
1198
|
+
if (char === quote) {
|
|
1199
|
+
quote = null;
|
|
1200
|
+
}
|
|
1201
|
+
} else if (char === ">") {
|
|
1202
|
+
return i + 1;
|
|
1203
|
+
} else if (char === '"' || char === "'") {
|
|
1204
|
+
quote = char;
|
|
596
1205
|
}
|
|
597
|
-
|
|
598
|
-
|
|
1206
|
+
}
|
|
1207
|
+
const close = html.indexOf(">", start + 1);
|
|
1208
|
+
return close === -1 ? -1 : close + 1;
|
|
1209
|
+
}
|
|
1210
|
+
function withoutAnchorTags(html) {
|
|
1211
|
+
let out = "";
|
|
1212
|
+
let from = 0;
|
|
1213
|
+
ANCHOR_START.lastIndex = 0;
|
|
1214
|
+
let match;
|
|
1215
|
+
while ((match = ANCHOR_START.exec(html)) !== null) {
|
|
1216
|
+
const close = html.indexOf(">", match.index + 2);
|
|
1217
|
+
if (close === -1) {
|
|
1218
|
+
break;
|
|
1219
|
+
}
|
|
1220
|
+
out += `${html.slice(from, match.index)} `;
|
|
1221
|
+
from = close + 1;
|
|
1222
|
+
ANCHOR_START.lastIndex = from;
|
|
1223
|
+
}
|
|
1224
|
+
return out + html.slice(from);
|
|
1225
|
+
}
|
|
1226
|
+
function anchors(html) {
|
|
1227
|
+
const found = [];
|
|
1228
|
+
ANCHOR_START.lastIndex = 0;
|
|
1229
|
+
ANCHOR_CLOSE.lastIndex = 0;
|
|
1230
|
+
let close = null;
|
|
1231
|
+
let exhausted = false;
|
|
1232
|
+
let match;
|
|
1233
|
+
while ((match = ANCHOR_START.exec(html)) !== null) {
|
|
1234
|
+
const end = tagEnd(html, match.index);
|
|
1235
|
+
if (end === -1) {
|
|
1236
|
+
break;
|
|
1237
|
+
}
|
|
1238
|
+
const attribute = end - match.index <= 6e3 ? HREF.exec(html.slice(match.index, end)) : null;
|
|
1239
|
+
const value = attribute ? attribute[1] ?? attribute[2] ?? attribute[3] : "";
|
|
1240
|
+
if (!value || value.length > 4e3) {
|
|
1241
|
+
ANCHOR_START.lastIndex = end;
|
|
1242
|
+
continue;
|
|
1243
|
+
}
|
|
1244
|
+
if (close !== null && close.index < end) {
|
|
1245
|
+
close = null;
|
|
1246
|
+
}
|
|
1247
|
+
if (close === null && !exhausted) {
|
|
1248
|
+
ANCHOR_CLOSE.lastIndex = Math.max(ANCHOR_CLOSE.lastIndex, end);
|
|
1249
|
+
close = ANCHOR_CLOSE.exec(html);
|
|
1250
|
+
exhausted = close === null;
|
|
1251
|
+
}
|
|
1252
|
+
if (close === null || close.index - end > 2e3) {
|
|
1253
|
+
ANCHOR_START.lastIndex = end;
|
|
1254
|
+
continue;
|
|
1255
|
+
}
|
|
1256
|
+
found.push({ href: foldToAscii(decodeEntities(value, { attribute: true })), inner: foldToAscii(decodeEntities(replaceTags(html.slice(end, close.index), ""))).trim() });
|
|
1257
|
+
ANCHOR_START.lastIndex = close.index + close[0].length;
|
|
1258
|
+
close = null;
|
|
1259
|
+
}
|
|
1260
|
+
return found;
|
|
1261
|
+
}
|
|
1262
|
+
function extractHosts(text, known = /* @__PURE__ */ new Set()) {
|
|
1263
|
+
const hosts = /* @__PURE__ */ new Set();
|
|
1264
|
+
if (typeof text !== "string" || text === "") {
|
|
1265
|
+
return [];
|
|
1266
|
+
}
|
|
1267
|
+
text = foldToAscii(text);
|
|
1268
|
+
BARE_HOST.lastIndex = 0;
|
|
1269
|
+
let match;
|
|
1270
|
+
while ((match = BARE_HOST.exec(text)) !== null && hosts.size < MAX_BARE_HOSTS) {
|
|
1271
|
+
const [host] = match;
|
|
1272
|
+
const next = text[match.index + host.length];
|
|
1273
|
+
if (continuesName(text, match.index) || host !== host.toLowerCase() || next === "@" || next === "+" || known.has(host) || hosts.has(host)) {
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
const parsed = parseDomain(host);
|
|
1277
|
+
const file = !parsed.subdomain && FILE_TLDS.has(parsed.publicSuffix) && !host.startsWith("xn--");
|
|
1278
|
+
if (parsed.isIcann && parsed.domain && !file) {
|
|
1279
|
+
hosts.add(host);
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
return [...hosts];
|
|
599
1283
|
}
|
|
600
1284
|
function isDeceptiveLink(link) {
|
|
601
1285
|
if (!link.text) {
|
|
@@ -614,11 +1298,13 @@ function isDeceptiveLink(link) {
|
|
|
614
1298
|
}
|
|
615
1299
|
function addressDomain(address2) {
|
|
616
1300
|
const at = address2.lastIndexOf("@");
|
|
617
|
-
return at === -1 ? "" : address2.slice(at + 1).toLowerCase()
|
|
1301
|
+
return at === -1 ? "" : trimEndOf(address2.slice(at + 1).toLowerCase(), "> \r\n\f\v");
|
|
618
1302
|
}
|
|
619
1303
|
function firstAddress(field) {
|
|
620
1304
|
return field?.value?.find((entry) => entry.address) || null;
|
|
621
1305
|
}
|
|
1306
|
+
var ADDRESS_SOURCE = String.raw`[\p{L}\p{N}._%+-]{1,64}@([\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63}){1,10})`;
|
|
1307
|
+
var NAMED_ADDRESS = linear(ADDRESS_SOURCE, "u");
|
|
622
1308
|
function headerValue(mail, name) {
|
|
623
1309
|
const value = mail.headers?.get?.(name);
|
|
624
1310
|
if (value === void 0 || value === null) {
|
|
@@ -633,7 +1319,7 @@ function headerValue(mail, name) {
|
|
|
633
1319
|
return value.value ?? value.text ?? String(value);
|
|
634
1320
|
}
|
|
635
1321
|
function addPatternFeatures(text, add) {
|
|
636
|
-
let body = text;
|
|
1322
|
+
let body = foldToAscii(text);
|
|
637
1323
|
const patterns = [
|
|
638
1324
|
["url", URL_PATTERN],
|
|
639
1325
|
["email", EMAIL_PATTERN],
|
|
@@ -760,7 +1446,7 @@ function addSenderFeatures(mail, add) {
|
|
|
760
1446
|
}
|
|
761
1447
|
}
|
|
762
1448
|
const name = typeof from.name === "string" ? from.name : "";
|
|
763
|
-
const named = name.match(
|
|
1449
|
+
const named = name.match(NAMED_ADDRESS);
|
|
764
1450
|
if (named && registrableDomain(named[1].toLowerCase()) !== registrableDomain(domain)) {
|
|
765
1451
|
add("from:name_has_other_address");
|
|
766
1452
|
}
|
|
@@ -775,13 +1461,50 @@ function addSenderFeatures(mail, add) {
|
|
|
775
1461
|
}
|
|
776
1462
|
}
|
|
777
1463
|
}
|
|
1464
|
+
function isHtmlOnly(mail) {
|
|
1465
|
+
const contentType = mail.headers?.get?.("content-type");
|
|
1466
|
+
return !(typeof mail.text === "string" && mail.text) || (contentType?.value ?? contentType) === "text/html";
|
|
1467
|
+
}
|
|
1468
|
+
var MIN_SHOWN_WORDS = 20;
|
|
1469
|
+
var PART_WORD = /[\p{L}\p{N}]{2,}/gu;
|
|
1470
|
+
function partWords(text) {
|
|
1471
|
+
const words = /* @__PURE__ */ new Set();
|
|
1472
|
+
for (const [word] of normalizeText(text.slice(0, 5e4)).text.matchAll(PART_WORD)) {
|
|
1473
|
+
words.add(word);
|
|
1474
|
+
if (words.size >= 2e3) {
|
|
1475
|
+
break;
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
return words;
|
|
1479
|
+
}
|
|
1480
|
+
function addPartFeatures({ visible, text, plain, alternative }, add) {
|
|
1481
|
+
if (visible.hidden > 0) {
|
|
1482
|
+
add("html:hidden_text");
|
|
1483
|
+
add(`html:hidden_text:${bucket(visible.hidden, [1, 50, 500])}`);
|
|
1484
|
+
}
|
|
1485
|
+
if (!alternative) {
|
|
1486
|
+
return;
|
|
1487
|
+
}
|
|
1488
|
+
const shown = partWords(text);
|
|
1489
|
+
const other = partWords(plain);
|
|
1490
|
+
if (shown.size < MIN_SHOWN_WORDS || other.size < MIN_SHOWN_WORDS) {
|
|
1491
|
+
return;
|
|
1492
|
+
}
|
|
1493
|
+
let shared = 0;
|
|
1494
|
+
for (const word of other) {
|
|
1495
|
+
if (shown.has(word)) {
|
|
1496
|
+
shared++;
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
if (shared < Math.min(shown.size, other.size) / 5) {
|
|
1500
|
+
add("html:text_mismatch");
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
778
1503
|
function addHtmlFeatures(mail, html, words, add) {
|
|
779
1504
|
if (!html) {
|
|
780
1505
|
return;
|
|
781
1506
|
}
|
|
782
|
-
|
|
783
|
-
const htmlOnly = !(typeof mail.text === "string" && mail.text) || (contentType?.value ?? contentType) === "text/html";
|
|
784
|
-
add(htmlOnly ? "html:only" : "html:alternative");
|
|
1507
|
+
add(isHtmlOnly(mail) ? "html:only" : "html:alternative");
|
|
785
1508
|
if (/<form\b/i.test(html)) {
|
|
786
1509
|
add("html:form");
|
|
787
1510
|
}
|
|
@@ -794,14 +1517,14 @@ function addHtmlFeatures(mail, html, words, add) {
|
|
|
794
1517
|
if (/display\s*:\s*none|visibility\s*:\s*hidden|font-size\s*:\s*[01](?:\.\d+)?(?:px|pt)?\s*[;"']/i.test(html)) {
|
|
795
1518
|
add("html:hidden");
|
|
796
1519
|
}
|
|
797
|
-
const images = (html.match(
|
|
1520
|
+
const images = (html.match(IMG) || []).length;
|
|
798
1521
|
if (images > 0) {
|
|
799
1522
|
add(`html:img:${bucket(images, [1, 5, 20])}`);
|
|
800
1523
|
if (words.length < 20) {
|
|
801
1524
|
add("html:img_little_text");
|
|
802
1525
|
}
|
|
803
1526
|
}
|
|
804
|
-
if (
|
|
1527
|
+
if (IMG_PIXEL.test(html)) {
|
|
805
1528
|
add("html:pixel");
|
|
806
1529
|
}
|
|
807
1530
|
}
|
|
@@ -855,14 +1578,21 @@ function getFeatures(mail = {}, options = {}) {
|
|
|
855
1578
|
const { maxLength = 1e5, maxBigrams = 3e3 } = options;
|
|
856
1579
|
const features = /* @__PURE__ */ new Set();
|
|
857
1580
|
const add = (feature) => features.add(feature);
|
|
858
|
-
const html = typeof mail.html === "string" ? mail.html : "";
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
1581
|
+
const html = typeof mail.html === "string" ? mail.html.slice(0, maxLength * 2) : "";
|
|
1582
|
+
const plain = typeof mail.text === "string" ? mail.text.slice(0, maxLength) : "";
|
|
1583
|
+
const visible = html ? visibleText(html) : { text: plain, hidden: 0 };
|
|
1584
|
+
const alternative = Boolean(html && plain && !isHtmlOnly(mail));
|
|
1585
|
+
let text = visible.text.slice(0, maxLength);
|
|
1586
|
+
if (alternative && segmentWords(text.slice(0, 4e3)).words.length < MIN_SHOWN_WORDS) {
|
|
1587
|
+
text = `${text}
|
|
1588
|
+
${plain}`.slice(0, maxLength);
|
|
862
1589
|
}
|
|
863
|
-
text = text.slice(0, maxLength);
|
|
864
1590
|
const subject = typeof mail.subject === "string" ? mail.subject : "";
|
|
865
|
-
const links = extractLinks(
|
|
1591
|
+
const links = extractLinks(html ? `${plain}
|
|
1592
|
+
${text}` : text, html);
|
|
1593
|
+
const hosts = extractHosts(`${subject}
|
|
1594
|
+
${text}${alternative ? `
|
|
1595
|
+
${plain}` : ""}`, new Set(links.map((link) => link.host)));
|
|
866
1596
|
const body = addPatternFeatures(text, add);
|
|
867
1597
|
const detected = detectLanguage(`${subject}
|
|
868
1598
|
${body}`);
|
|
@@ -884,6 +1614,12 @@ ${body}`);
|
|
|
884
1614
|
addLinkFeatures(links, add);
|
|
885
1615
|
addSenderFeatures(mail, add);
|
|
886
1616
|
addHtmlFeatures(mail, html, segmented.words, add);
|
|
1617
|
+
addPartFeatures({
|
|
1618
|
+
visible,
|
|
1619
|
+
text,
|
|
1620
|
+
plain,
|
|
1621
|
+
alternative
|
|
1622
|
+
}, add);
|
|
887
1623
|
addAttachmentFeatures(mail, add);
|
|
888
1624
|
addHeaderFeatures(mail, add);
|
|
889
1625
|
return {
|
|
@@ -891,7 +1627,12 @@ ${body}`);
|
|
|
891
1627
|
words: segmented.words,
|
|
892
1628
|
language: detected.language,
|
|
893
1629
|
script: detected.script,
|
|
894
|
-
links
|
|
1630
|
+
links,
|
|
1631
|
+
hosts,
|
|
1632
|
+
// (the text shown, as read for the words, and how many characters of
|
|
1633
|
+
// text the HTML hides)
|
|
1634
|
+
text,
|
|
1635
|
+
hidden: visible.hidden
|
|
895
1636
|
};
|
|
896
1637
|
}
|
|
897
1638
|
|
|
@@ -1053,7 +1794,7 @@ function sniff(buffer) {
|
|
|
1053
1794
|
const at = (offset, bytes) => buffer.length >= offset + bytes.length && bytes.every((byte, i) => buffer[offset + i] === byte);
|
|
1054
1795
|
const ascii = (offset, text) => at(offset, [...Buffer.from(text, "latin1")]);
|
|
1055
1796
|
const result = (type, executable = false) => ({ type, executable, archive: ARCHIVE_TYPES.has(type) });
|
|
1056
|
-
if (ascii(0, "MZ")) {
|
|
1797
|
+
if (ascii(0, "MZ") && buffer.length >= 64 && at(buffer.readUInt32LE(60), [80, 69, 0, 0])) {
|
|
1057
1798
|
return result("exe", true);
|
|
1058
1799
|
}
|
|
1059
1800
|
if (at(0, [127, 69, 76, 70])) {
|
|
@@ -1073,7 +1814,7 @@ function sniff(buffer) {
|
|
|
1073
1814
|
return result("ole");
|
|
1074
1815
|
}
|
|
1075
1816
|
if (at(0, [80, 75, 3, 4]) || at(0, [80, 75, 5, 6])) {
|
|
1076
|
-
const names =
|
|
1817
|
+
const names = entriesOf(buffer).map((entry) => entry.name);
|
|
1077
1818
|
if (names.includes("AndroidManifest.xml") || names.includes("classes.dex")) {
|
|
1078
1819
|
return result("apk", true);
|
|
1079
1820
|
}
|
|
@@ -1139,25 +1880,65 @@ function sniff(buffer) {
|
|
|
1139
1880
|
}
|
|
1140
1881
|
return null;
|
|
1141
1882
|
}
|
|
1883
|
+
var ZIP_FAMILY = /* @__PURE__ */ new Set(["zip", "docx", "xlsx", "pptx"]);
|
|
1884
|
+
var OFFICE_TYPES = /* @__PURE__ */ new Set(["docx", "xlsx", "pptx"]);
|
|
1885
|
+
var OFFICE_EMBEDDED = /* @__PURE__ */ new Set([...MACRO_EXTENSIONS, "cer", "der"]);
|
|
1886
|
+
var ENTRIES = /* @__PURE__ */ new WeakMap();
|
|
1887
|
+
function entriesOf(buffer) {
|
|
1888
|
+
let entries = ENTRIES.get(buffer);
|
|
1889
|
+
if (!entries) {
|
|
1890
|
+
entries = zipEntries(buffer);
|
|
1891
|
+
ENTRIES.set(buffer, entries);
|
|
1892
|
+
}
|
|
1893
|
+
return entries;
|
|
1894
|
+
}
|
|
1142
1895
|
function zipEntries(buffer, limit = 2e3) {
|
|
1143
|
-
const entries = [];
|
|
1144
1896
|
const start = Math.max(0, buffer.length - 65557);
|
|
1145
|
-
|
|
1897
|
+
const records = [];
|
|
1146
1898
|
for (let i = buffer.length - 22; i >= start; i--) {
|
|
1147
|
-
if (buffer.readUInt32LE(i) === 101010256) {
|
|
1148
|
-
|
|
1149
|
-
break;
|
|
1899
|
+
if (buffer.readUInt32LE(i) === 101010256 && (records.length === 0 || i + 22 + buffer.readUInt16LE(i + 20) === buffer.length)) {
|
|
1900
|
+
records.push(i);
|
|
1150
1901
|
}
|
|
1151
1902
|
}
|
|
1152
|
-
|
|
1153
|
-
|
|
1903
|
+
const entries = [];
|
|
1904
|
+
const read = /* @__PURE__ */ new Set();
|
|
1905
|
+
for (const eocd of records) {
|
|
1906
|
+
const offset = directoryOf(buffer, eocd);
|
|
1907
|
+
if (offset !== -1 && !read.has(offset)) {
|
|
1908
|
+
read.add(offset);
|
|
1909
|
+
readDirectory(buffer, offset, limit - entries.length, entries);
|
|
1910
|
+
}
|
|
1154
1911
|
}
|
|
1155
|
-
|
|
1912
|
+
return entries;
|
|
1913
|
+
}
|
|
1914
|
+
var DIRECTORY = 33639248;
|
|
1915
|
+
var isDirectory = (buffer, offset) => offset >= 0 && offset + 46 <= buffer.length && buffer.readUInt32LE(offset) === DIRECTORY;
|
|
1916
|
+
function directoryOf(buffer, eocd) {
|
|
1156
1917
|
let offset = buffer.readUInt32LE(eocd + 16);
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1918
|
+
let size = buffer.readUInt32LE(eocd + 12);
|
|
1919
|
+
const locator = eocd - 20;
|
|
1920
|
+
if ((offset === 4294967295 || size === 4294967295) && locator >= 0 && buffer.readUInt32LE(locator) === 117853008) {
|
|
1921
|
+
const stated = Number(buffer.readBigUInt64LE(locator + 8));
|
|
1922
|
+
const isRecord = (at) => at >= 0 && at + 56 <= locator && buffer.readUInt32LE(at) === 101075792;
|
|
1923
|
+
const record = isRecord(stated) ? stated : locator - 56;
|
|
1924
|
+
if (isRecord(record)) {
|
|
1925
|
+
size = Number(buffer.readBigUInt64LE(record + 40));
|
|
1926
|
+
offset = Number(buffer.readBigUInt64LE(record + 48));
|
|
1160
1927
|
}
|
|
1928
|
+
}
|
|
1929
|
+
if (isDirectory(buffer, offset)) {
|
|
1930
|
+
return offset;
|
|
1931
|
+
}
|
|
1932
|
+
for (const end of [eocd, locator - 56]) {
|
|
1933
|
+
const directory = end - size;
|
|
1934
|
+
if (directory < end && isDirectory(buffer, directory)) {
|
|
1935
|
+
return directory;
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
return -1;
|
|
1939
|
+
}
|
|
1940
|
+
function readDirectory(buffer, offset, count, entries) {
|
|
1941
|
+
for (let n = 0; n < count && isDirectory(buffer, offset); n++) {
|
|
1161
1942
|
const flags = buffer.readUInt16LE(offset + 8);
|
|
1162
1943
|
const size = buffer.readUInt32LE(offset + 24);
|
|
1163
1944
|
const nameLength = buffer.readUInt16LE(offset + 28);
|
|
@@ -1167,10 +1948,9 @@ function zipEntries(buffer, limit = 2e3) {
|
|
|
1167
1948
|
entries.push({ name, encrypted: (flags & 1) === 1, size });
|
|
1168
1949
|
offset += 46 + nameLength + extraLength + commentLength;
|
|
1169
1950
|
}
|
|
1170
|
-
return entries;
|
|
1171
1951
|
}
|
|
1172
1952
|
function extensionOf(name) {
|
|
1173
|
-
const base = String(name).toLowerCase().split(/[/\\]/).pop();
|
|
1953
|
+
const base = trimEndOf(String(name).toLowerCase().split(/[/\\]/).pop(), ". ");
|
|
1174
1954
|
const dot = base.lastIndexOf(".");
|
|
1175
1955
|
return dot === -1 ? "" : base.slice(dot + 1).trim();
|
|
1176
1956
|
}
|
|
@@ -1178,7 +1958,7 @@ function oleHasMacros(buffer) {
|
|
|
1178
1958
|
const names = ["_VBA_PROJECT", "VBA", "Macros"].map((name) => Buffer.from(name, "utf16le"));
|
|
1179
1959
|
return names.some((name) => buffer.includes(name));
|
|
1180
1960
|
}
|
|
1181
|
-
var PDF_ACTIVE =
|
|
1961
|
+
var PDF_ACTIVE = linear(String.raw`\/(?:JavaScript|JS|Launch|EmbeddedFile)\b`);
|
|
1182
1962
|
function inspectAttachments(attachments = []) {
|
|
1183
1963
|
const findings = [];
|
|
1184
1964
|
for (const attachment of attachments) {
|
|
@@ -1196,14 +1976,15 @@ function inspectAttachments(attachments = []) {
|
|
|
1196
1976
|
if (BIDI_CONTROLS.test(filename)) {
|
|
1197
1977
|
add("rtl_override", `Attachment "${name}" uses a right-to-left override to hide its real extension`);
|
|
1198
1978
|
}
|
|
1199
|
-
const parts = filename.toLowerCase().split(".");
|
|
1979
|
+
const parts = trimEndOf(filename.toLowerCase(), ". ").split(".");
|
|
1200
1980
|
if (parts.length >= 3 && EXECUTABLE_EXTENSIONS.has(extension) && DOCUMENT_EXTENSIONS.has(parts.at(-2))) {
|
|
1201
1981
|
add("double_extension", `Attachment "${name}" has a document extension followed by a program extension`, { extension });
|
|
1202
1982
|
}
|
|
1203
|
-
|
|
1983
|
+
const byName = EXECUTABLE_EXTENSIONS.has(extension) && !MACRO_EXTENSIONS.has(extension);
|
|
1984
|
+
if (detected?.executable && (detected.type !== "script" || !byName)) {
|
|
1985
|
+
add(extension && !EXECUTABLE_EXTENSIONS.has(extension) ? "disguised_executable" : "executable", `Attachment "${name}" contains a program (${detected.type})${extension && extension !== detected.type ? ` but is named .${extension}` : ""}`, { extension: detected.type });
|
|
1986
|
+
} else if (byName) {
|
|
1204
1987
|
add("executable", `Attachment "${name}" is a file type that runs code (.${extension})`, { extension });
|
|
1205
|
-
} else if (detected?.executable) {
|
|
1206
|
-
add(extension && !EXECUTABLE_EXTENSIONS.has(extension) ? "disguised_executable" : "executable", `Attachment "${name}" contains a program (${detected.type})${extension ? ` but is named .${extension}` : ""}`, { extension: detected.type });
|
|
1207
1988
|
}
|
|
1208
1989
|
if (!content) {
|
|
1209
1990
|
if (MACRO_EXTENSIONS.has(extension)) {
|
|
@@ -1211,21 +1992,23 @@ function inspectAttachments(attachments = []) {
|
|
|
1211
1992
|
}
|
|
1212
1993
|
continue;
|
|
1213
1994
|
}
|
|
1214
|
-
|
|
1215
|
-
|
|
1995
|
+
const zipFamily = ZIP_FAMILY.has(detected?.type) || extension === "zip" && !detected?.executable && entriesOf(content).length > 0;
|
|
1996
|
+
if (zipFamily) {
|
|
1997
|
+
const entries = entriesOf(content);
|
|
1998
|
+
const office = OFFICE_TYPES.has(detected?.type);
|
|
1216
1999
|
if (entries.some((entry) => /(?:^|\/)vbaproject\.bin$/i.test(entry.name))) {
|
|
1217
2000
|
add("macro", `Attachment "${name}" is an Office document with a VBA macro project`, { extension });
|
|
1218
2001
|
} else if (MACRO_EXTENSIONS.has(extension)) {
|
|
1219
2002
|
add("macro", `Attachment "${name}" is an Office document type that holds macros (.${extension})`, { extension });
|
|
2003
|
+
} else if (office && entries.some((entry) => MACRO_EXTENSIONS.has(extensionOf(entry.name)))) {
|
|
2004
|
+
add("macro", `Attachment "${name}" is an Office document holding another with macros`, { extension });
|
|
1220
2005
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
add("encrypted_archive", `Archive "${name}" is password protected, so its contents cannot be scanned`, { extension });
|
|
1228
|
-
}
|
|
2006
|
+
const inner = entries.filter((entry) => EXECUTABLE_EXTENSIONS.has(extensionOf(entry.name)) && !(office && OFFICE_EMBEDDED.has(extensionOf(entry.name))));
|
|
2007
|
+
if (inner.length > 0) {
|
|
2008
|
+
add("executable_in_archive", `Archive "${name}" contains ${inner.length === 1 ? "a program" : `${inner.length} programs`} (${inner.slice(0, 3).map((entry) => entry.name.split("/").pop()).join(", ")})`, { extension: extensionOf(inner[0].name), extensions: [...new Set(inner.map((entry) => extensionOf(entry.name)))] });
|
|
2009
|
+
}
|
|
2010
|
+
if ((detected?.type === "zip" || !ZIP_FAMILY.has(detected?.type)) && entries.some((entry) => entry.encrypted)) {
|
|
2011
|
+
add("encrypted_archive", `Archive "${name}" is password protected, so its contents cannot be scanned`, { extension });
|
|
1229
2012
|
}
|
|
1230
2013
|
} else if (detected?.type === "ole") {
|
|
1231
2014
|
if (oleHasMacros(content)) {
|
|
@@ -2010,6 +2793,42 @@ var DnsChecker = class {
|
|
|
2010
2793
|
// src/homograph.js
|
|
2011
2794
|
import { domainToUnicode } from "node:url";
|
|
2012
2795
|
import { parse as parseDomain2 } from "tldts";
|
|
2796
|
+
var INTERNATIONAL_BRANDS = [
|
|
2797
|
+
"alipay",
|
|
2798
|
+
"auspost",
|
|
2799
|
+
"bancolombia",
|
|
2800
|
+
"bnpparibas",
|
|
2801
|
+
"bradesco",
|
|
2802
|
+
"caixabank",
|
|
2803
|
+
"canadapost",
|
|
2804
|
+
"commerzbank",
|
|
2805
|
+
"coupang",
|
|
2806
|
+
"creditagricole",
|
|
2807
|
+
"deutschebank",
|
|
2808
|
+
"flipkart",
|
|
2809
|
+
"gosuslugi",
|
|
2810
|
+
"japanpost",
|
|
2811
|
+
"kakaobank",
|
|
2812
|
+
"klarna",
|
|
2813
|
+
"kucoin",
|
|
2814
|
+
"lazada",
|
|
2815
|
+
"mercadolibre",
|
|
2816
|
+
"mercadolivre",
|
|
2817
|
+
"mercadopago",
|
|
2818
|
+
"mercari",
|
|
2819
|
+
"natwest",
|
|
2820
|
+
"nubank",
|
|
2821
|
+
"phonepe",
|
|
2822
|
+
"postbank",
|
|
2823
|
+
"rakuten",
|
|
2824
|
+
"sberbank",
|
|
2825
|
+
"swisscom",
|
|
2826
|
+
"trustwallet",
|
|
2827
|
+
"unicredit",
|
|
2828
|
+
"vodafone",
|
|
2829
|
+
"wechat",
|
|
2830
|
+
"yandex"
|
|
2831
|
+
];
|
|
2013
2832
|
var BRANDS = [
|
|
2014
2833
|
"adobe",
|
|
2015
2834
|
"airbnb",
|
|
@@ -2109,8 +2928,76 @@ var BRANDS = [
|
|
|
2109
2928
|
"xfinity",
|
|
2110
2929
|
"yahoo",
|
|
2111
2930
|
"zelle",
|
|
2112
|
-
"zoom"
|
|
2931
|
+
"zoom",
|
|
2932
|
+
...INTERNATIONAL_BRANDS
|
|
2933
|
+
];
|
|
2934
|
+
var PHISHING_WORDS = ["account", "accounts", "alert", "alerts", "auth", "authentication", "billing", "confirm", "login", "logon", "password", "recovery", "refund", "secure", "security", "signin", "signon", "suspended", "unlock", "update", "validate", "verification", "verify"];
|
|
2935
|
+
var WORD_BRANDS = /* @__PURE__ */ new Set(["apple", "blockchain", "booking", "chase", "discord", "kraken", "ledger", "office", "proton", "slack", "steam", "stripe"]);
|
|
2936
|
+
var SWAPPED_WORDS = /* @__PURE__ */ new Set([
|
|
2937
|
+
"akuten",
|
|
2938
|
+
"alzada",
|
|
2939
|
+
"appel",
|
|
2940
|
+
"applied",
|
|
2941
|
+
"auslost",
|
|
2942
|
+
"azada",
|
|
2943
|
+
"coupant",
|
|
2944
|
+
"gechat",
|
|
2945
|
+
"klara",
|
|
2946
|
+
"lanada",
|
|
2947
|
+
"lanzada",
|
|
2948
|
+
"lapada",
|
|
2949
|
+
"latada",
|
|
2950
|
+
"lavada",
|
|
2951
|
+
"mazada",
|
|
2952
|
+
"mercar",
|
|
2953
|
+
"mercati",
|
|
2954
|
+
"pazada",
|
|
2955
|
+
"phoneme",
|
|
2956
|
+
"postbak",
|
|
2957
|
+
"raketen",
|
|
2958
|
+
"rauten",
|
|
2959
|
+
"razada",
|
|
2960
|
+
"stirpe",
|
|
2961
|
+
"vazada",
|
|
2962
|
+
"venom",
|
|
2963
|
+
"webchat"
|
|
2964
|
+
]);
|
|
2965
|
+
var OFFICIAL_DOMAINS = [
|
|
2966
|
+
"adobelogin.com",
|
|
2967
|
+
"amazon-adsystem.com",
|
|
2968
|
+
"apple-cloudkit.com",
|
|
2969
|
+
"apple-dns.net",
|
|
2970
|
+
"bnpparibas-am.com",
|
|
2971
|
+
"canadapost-postescanada.ca",
|
|
2972
|
+
"cdn-apple.com",
|
|
2973
|
+
"cloudflare-dns.com",
|
|
2974
|
+
"credit-agricole.com",
|
|
2975
|
+
"credit-agricole.fr",
|
|
2976
|
+
"deutsche-bank.de",
|
|
2977
|
+
"google-analytics.com",
|
|
2978
|
+
"icloud-content.com",
|
|
2979
|
+
"images-amazon.com",
|
|
2980
|
+
"instagram-engineering.com",
|
|
2981
|
+
"media-amazon.com",
|
|
2982
|
+
"okta-emea.com",
|
|
2983
|
+
"paypal-community.com",
|
|
2984
|
+
"paypal-objects.com",
|
|
2985
|
+
"paypal-prepaid.com",
|
|
2986
|
+
"paypal-status.com",
|
|
2987
|
+
"rakuten-bank.co.jp",
|
|
2988
|
+
"rakuten-card.co.jp",
|
|
2989
|
+
"rakuten-sec.co.jp",
|
|
2990
|
+
"sberbank-ast.ru",
|
|
2991
|
+
"slack-edge.com",
|
|
2992
|
+
"slack-msgs.com",
|
|
2993
|
+
"ssl-images-amazon.com",
|
|
2994
|
+
"t-mobile.com",
|
|
2995
|
+
"yahoo-corp.jp",
|
|
2996
|
+
"yandex-team.ru"
|
|
2113
2997
|
];
|
|
2998
|
+
var SUFFIX_DASH = /^(?:com|net|org|gov|edu)-/;
|
|
2999
|
+
var OTHER_SUFFIX_DASH = /^(?:co|info|biz|io|app|us|uk|de|fr|es|it|ca|au|jp|br|in)-/;
|
|
3000
|
+
var SECOND_SUFFIX_DASH = /^(?:co|com)-(?:uk|jp|au|nz|za|in|kr|br)\b/;
|
|
2114
3001
|
var CONFUSABLE = {
|
|
2115
3002
|
\u0430: "a",
|
|
2116
3003
|
\u04D3: "a",
|
|
@@ -2254,10 +3141,17 @@ function skeleton(label) {
|
|
|
2254
3141
|
}
|
|
2255
3142
|
return out;
|
|
2256
3143
|
}
|
|
3144
|
+
var MAX_VARIANTS = 256;
|
|
2257
3145
|
function trickVariants(label) {
|
|
2258
3146
|
const variants = /* @__PURE__ */ new Set([label]);
|
|
2259
3147
|
for (const [pattern, letter] of ASCII_TRICKS) {
|
|
3148
|
+
if (!label.includes(pattern.source)) {
|
|
3149
|
+
continue;
|
|
3150
|
+
}
|
|
2260
3151
|
for (const value of variants) {
|
|
3152
|
+
if (variants.size >= MAX_VARIANTS / 2) {
|
|
3153
|
+
break;
|
|
3154
|
+
}
|
|
2261
3155
|
variants.add(value.replace(pattern, letter));
|
|
2262
3156
|
}
|
|
2263
3157
|
}
|
|
@@ -2271,6 +3165,7 @@ function editDistance(a, b, max = 2) {
|
|
|
2271
3165
|
if (Math.abs(a.length - b.length) > max) {
|
|
2272
3166
|
return max + 1;
|
|
2273
3167
|
}
|
|
3168
|
+
let before = null;
|
|
2274
3169
|
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
2275
3170
|
for (let i = 1; i <= a.length; i++) {
|
|
2276
3171
|
const current = [i];
|
|
@@ -2278,15 +3173,35 @@ function editDistance(a, b, max = 2) {
|
|
|
2278
3173
|
for (let j = 1; j <= b.length; j++) {
|
|
2279
3174
|
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
2280
3175
|
current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost);
|
|
3176
|
+
if (before && i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
|
|
3177
|
+
current[j] = Math.min(current[j], before[j - 2] + 1);
|
|
3178
|
+
}
|
|
2281
3179
|
best = Math.min(best, current[j]);
|
|
2282
3180
|
}
|
|
2283
3181
|
if (best > max) {
|
|
2284
3182
|
return max + 1;
|
|
2285
3183
|
}
|
|
3184
|
+
before = previous;
|
|
2286
3185
|
previous = current;
|
|
2287
3186
|
}
|
|
2288
3187
|
return previous[b.length];
|
|
2289
3188
|
}
|
|
3189
|
+
function brandWithPhishingWord(name, brands) {
|
|
3190
|
+
for (const word of PHISHING_WORDS) {
|
|
3191
|
+
if (name.length <= word.length + 4) {
|
|
3192
|
+
continue;
|
|
3193
|
+
}
|
|
3194
|
+
const before = name.endsWith(word) ? name.slice(0, -word.length) : null;
|
|
3195
|
+
if (before && before.length >= 5 && brands.has(before) && !WORD_BRANDS.has(before)) {
|
|
3196
|
+
return before;
|
|
3197
|
+
}
|
|
3198
|
+
const after = name.startsWith(word) ? name.slice(word.length) : null;
|
|
3199
|
+
if (after && after.length >= 5 && brands.has(after) && !WORD_BRANDS.has(after)) {
|
|
3200
|
+
return after;
|
|
3201
|
+
}
|
|
3202
|
+
}
|
|
3203
|
+
return null;
|
|
3204
|
+
}
|
|
2290
3205
|
function labelScripts(label) {
|
|
2291
3206
|
const scripts = /* @__PURE__ */ new Set();
|
|
2292
3207
|
for (const char of label) {
|
|
@@ -2314,7 +3229,7 @@ var HomographDetector = class {
|
|
|
2314
3229
|
constructor(options = {}) {
|
|
2315
3230
|
this.options = options;
|
|
2316
3231
|
this.brands = new Set([...options.brands || BRANDS, ...options.extraBrands || []].map((brand) => brand.toLowerCase()));
|
|
2317
|
-
this.allowlist = new Set(
|
|
3232
|
+
this.allowlist = new Set([...OFFICIAL_DOMAINS, ...options.allowlist || []].map((domain) => domain.toLowerCase()));
|
|
2318
3233
|
this.cache = /* @__PURE__ */ new Map();
|
|
2319
3234
|
}
|
|
2320
3235
|
/**
|
|
@@ -2370,29 +3285,36 @@ var HomographDetector = class {
|
|
|
2370
3285
|
if (viaConfusable) {
|
|
2371
3286
|
flag(0.95, `"${unicode}" imitates ${viaConfusable} with lookalike letters`, viaConfusable);
|
|
2372
3287
|
} else if (/^[a-z\d-]+$/.test(folded)) {
|
|
2373
|
-
const
|
|
3288
|
+
const variants = trickVariants(folded);
|
|
3289
|
+
const trick = [...variants].find((variant) => this.brands.has(variant));
|
|
2374
3290
|
if (trick) {
|
|
2375
3291
|
flag(0.85, `"${unicode}" imitates ${trick} by swapping characters`, trick);
|
|
2376
|
-
} else if (folded.length >= 5) {
|
|
3292
|
+
} else if (folded.length >= 5 && !SWAPPED_WORDS.has(folded)) {
|
|
2377
3293
|
const near = [...this.brands].find((brand) => brand.length >= 5 && editDistance(folded, brand, 1) === 1);
|
|
2378
3294
|
if (near) {
|
|
2379
3295
|
flag(0.5, `"${unicode}" is one letter away from ${near}`, near);
|
|
2380
3296
|
}
|
|
2381
3297
|
}
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
3298
|
+
const pieces = folded.includes("-") ? folded.split("-") : [];
|
|
3299
|
+
for (const piece of pieces) {
|
|
3300
|
+
if (piece.length < 4) {
|
|
3301
|
+
continue;
|
|
3302
|
+
}
|
|
3303
|
+
if (this.brands.has(piece)) {
|
|
3304
|
+
flag(0.6, `"${unicode}" puts ${piece} inside another domain name`, piece);
|
|
3305
|
+
break;
|
|
3306
|
+
}
|
|
3307
|
+
const disguised = [...trickVariants(piece)].find((variant) => this.brands.has(variant));
|
|
3308
|
+
if (disguised) {
|
|
3309
|
+
flag(0.85, `"${unicode}" imitates ${disguised} by swapping characters`, disguised);
|
|
3310
|
+
break;
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
for (const value of [folded, ...pieces, ...variants]) {
|
|
3314
|
+
const joined = brandWithPhishingWord(value, this.brands);
|
|
3315
|
+
if (joined) {
|
|
3316
|
+
flag(0.6, `"${unicode}" joins ${joined} to a phishing word`, joined);
|
|
3317
|
+
break;
|
|
2396
3318
|
}
|
|
2397
3319
|
}
|
|
2398
3320
|
}
|
|
@@ -2401,6 +3323,11 @@ var HomographDetector = class {
|
|
|
2401
3323
|
if (brandLabel && (this.options.strictMode || subdomain.length >= 2)) {
|
|
2402
3324
|
flag(0.6, `"${unicode}" puts ${brandLabel} in front of another domain`, brandLabel);
|
|
2403
3325
|
}
|
|
3326
|
+
const last = subdomain.length > 0 ? skeleton(subdomain.at(-1)) : "";
|
|
3327
|
+
if (last.length >= 3 && this.brands.has(last) && (SUFFIX_DASH.test(folded) || OTHER_SUFFIX_DASH.test(folded))) {
|
|
3328
|
+
const strong = SUFFIX_DASH.test(folded) || SECOND_SUFFIX_DASH.test(folded) || PHISHING_WORDS.some((word) => folded.includes(word));
|
|
3329
|
+
flag(strong ? 0.85 : 0.6, `"${unicode}" starts with ${last}'s address and continues into another domain`, last);
|
|
3330
|
+
}
|
|
2404
3331
|
result.confidence = result.riskScore;
|
|
2405
3332
|
if (result.riskScore >= 0.5) {
|
|
2406
3333
|
result.recommendations.push(result.brand ? `Do not sign in from this link; go to ${result.brand}'s site directly` : "Treat links to this domain as untrusted");
|
|
@@ -2496,7 +3423,7 @@ ${html.slice(0, 1e5)}`)) {
|
|
|
2496
3423
|
if (from) {
|
|
2497
3424
|
const fromRoot = session.originalFromAddressRootDomain;
|
|
2498
3425
|
const name = typeof from.name === "string" ? from.name.toLowerCase() : "";
|
|
2499
|
-
const claimed = name.match(
|
|
3426
|
+
const claimed = name.match(NAMED_ADDRESS);
|
|
2500
3427
|
if (claimed && registrableDomain(claimed[1]) !== fromRoot) {
|
|
2501
3428
|
hit("FROM_NAME_OTHER_ADDRESS", 2.5, `Display name shows ${claimed[0]} but the message is from ${from.address}`, "SPOOFING");
|
|
2502
3429
|
}
|
|
@@ -2881,7 +3808,7 @@ function resolveConfig(options = {}, env = process.env) {
|
|
|
2881
3808
|
if (!baseUrl) {
|
|
2882
3809
|
throw new TypeError(`LLM provider "${providerName}" needs a base URL (baseUrl, or host and port)`);
|
|
2883
3810
|
}
|
|
2884
|
-
baseUrl = baseUrl
|
|
3811
|
+
baseUrl = trimEndOf(baseUrl, "/");
|
|
2885
3812
|
const model = options.model || preset.model;
|
|
2886
3813
|
if (!model && preset.api !== "classifier") {
|
|
2887
3814
|
throw new TypeError(`LLM provider "${providerName}" needs a model name`);
|
|
@@ -2992,18 +3919,59 @@ function authHeaders(config) {
|
|
|
2992
3919
|
return headers;
|
|
2993
3920
|
}
|
|
2994
3921
|
var REDACTIONS = [
|
|
2995
|
-
[
|
|
2996
|
-
[
|
|
2997
|
-
[
|
|
2998
|
-
[
|
|
2999
|
-
[
|
|
3922
|
+
[linear(ADDRESS_SOURCE, "gu"), "[email]@$1"],
|
|
3923
|
+
[linear(String.raw`\b(?:\d[ -]?){13,19}\b`, "g"), "[number]"],
|
|
3924
|
+
[linear(String.raw`(?:\+|00)\d[\d\s().-]{7,}\d`, "g"), "[phone]"],
|
|
3925
|
+
[linear(String.raw`\b\d{3}-\d{2}-\d{4}\b`, "g"), "[number]"],
|
|
3926
|
+
[linear(String.raw`([?&][^=\s&#]{1,256}=)[^\s&#]+`, "g"), "$1[redacted]"]
|
|
3000
3927
|
];
|
|
3928
|
+
var THINK_OPEN = linear(`<${caseless("think")}>`, "g");
|
|
3929
|
+
var THINK_CLOSE = linear(String.raw`<\/${caseless("think")}>`, "g");
|
|
3930
|
+
function withoutThinking(text) {
|
|
3931
|
+
let out = "";
|
|
3932
|
+
let from = 0;
|
|
3933
|
+
THINK_OPEN.lastIndex = 0;
|
|
3934
|
+
let open;
|
|
3935
|
+
while ((open = THINK_OPEN.exec(text)) !== null) {
|
|
3936
|
+
THINK_CLOSE.lastIndex = open.index + open[0].length;
|
|
3937
|
+
const close = THINK_CLOSE.exec(text);
|
|
3938
|
+
if (close === null) {
|
|
3939
|
+
break;
|
|
3940
|
+
}
|
|
3941
|
+
out += text.slice(from, open.index);
|
|
3942
|
+
from = close.index + close[0].length;
|
|
3943
|
+
THINK_OPEN.lastIndex = from;
|
|
3944
|
+
}
|
|
3945
|
+
return out + text.slice(from);
|
|
3946
|
+
}
|
|
3001
3947
|
function redact(text) {
|
|
3002
3948
|
let out = text;
|
|
3003
3949
|
for (const [pattern, replacement] of REDACTIONS) {
|
|
3004
3950
|
out = out.replace(pattern, replacement);
|
|
3005
3951
|
}
|
|
3006
|
-
return out;
|
|
3952
|
+
return withoutLocalParts(out);
|
|
3953
|
+
}
|
|
3954
|
+
var LOCAL_CHARACTER = /^[\p{L}\p{N}._%+-]$/u;
|
|
3955
|
+
function withoutLocalParts(text) {
|
|
3956
|
+
let out = "";
|
|
3957
|
+
let from = 0;
|
|
3958
|
+
let at = text.indexOf("[email]@");
|
|
3959
|
+
while (at !== -1) {
|
|
3960
|
+
let start = at;
|
|
3961
|
+
while (start > from) {
|
|
3962
|
+
if (start - 2 >= from && LOCAL_CHARACTER.test(text.slice(start - 2, start))) {
|
|
3963
|
+
start -= 2;
|
|
3964
|
+
} else if (LOCAL_CHARACTER.test(text[start - 1])) {
|
|
3965
|
+
start--;
|
|
3966
|
+
} else {
|
|
3967
|
+
break;
|
|
3968
|
+
}
|
|
3969
|
+
}
|
|
3970
|
+
out += text.slice(from, start);
|
|
3971
|
+
from = at;
|
|
3972
|
+
at = text.indexOf("[email]@", at + 8);
|
|
3973
|
+
}
|
|
3974
|
+
return out + text.slice(from);
|
|
3007
3975
|
}
|
|
3008
3976
|
function addressList(field) {
|
|
3009
3977
|
if (!field?.value) {
|
|
@@ -3025,7 +3993,8 @@ function describeMessage(mail, context = {}, options = {}) {
|
|
|
3025
3993
|
if (context.authentication) {
|
|
3026
3994
|
lines2.push(`Authentication: ${context.authentication}`);
|
|
3027
3995
|
}
|
|
3028
|
-
const
|
|
3996
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3997
|
+
const links = [...context.links || []].sort((a, b) => Number(isDeceptiveLink(b)) - Number(isDeceptiveLink(a))).filter((link) => !seen.has(link.url) && seen.add(link.url)).slice(0, 15);
|
|
3029
3998
|
if (links.length > 0) {
|
|
3030
3999
|
lines2.push("Links:");
|
|
3031
4000
|
for (const link of links) {
|
|
@@ -3036,7 +4005,17 @@ function describeMessage(mail, context = {}, options = {}) {
|
|
|
3036
4005
|
if (attachments.length > 0) {
|
|
3037
4006
|
lines2.push(`Attachments: ${attachments.map((a) => `${a.filename || "unnamed"} (${a.contentType || "unknown type"})`).join(", ")}`);
|
|
3038
4007
|
}
|
|
3039
|
-
|
|
4008
|
+
if (context.hidden >= 200) {
|
|
4009
|
+
lines2.push(`Hidden text: ${context.hidden} characters the message does not show`);
|
|
4010
|
+
}
|
|
4011
|
+
const read = Math.max(1e5, maxInputChars * 50);
|
|
4012
|
+
let shown = typeof mail.text === "string" ? mail.text : "";
|
|
4013
|
+
if (typeof context.text === "string") {
|
|
4014
|
+
shown = context.text;
|
|
4015
|
+
} else if (typeof mail.html === "string" && mail.html) {
|
|
4016
|
+
shown = visibleText(mail.html.slice(0, read)).text;
|
|
4017
|
+
}
|
|
4018
|
+
let body = readableText(shown.slice(0, read));
|
|
3040
4019
|
body = body.replaceAll(/[ \t]+/g, " ").replaceAll(/\n\s*\n+/g, "\n\n").trim();
|
|
3041
4020
|
let text = `${lines2.join("\n")}
|
|
3042
4021
|
|
|
@@ -3087,7 +4066,8 @@ function objectEnd(text, start) {
|
|
|
3087
4066
|
return -1;
|
|
3088
4067
|
}
|
|
3089
4068
|
function* jsonObjects(text) {
|
|
3090
|
-
|
|
4069
|
+
let starts = 0;
|
|
4070
|
+
for (let start = text.indexOf("{"); start !== -1 && starts++ < 64; start = text.indexOf("{", start + 1)) {
|
|
3091
4071
|
const end = objectEnd(text, start);
|
|
3092
4072
|
if (end !== -1) {
|
|
3093
4073
|
yield text.slice(start, end + 1);
|
|
@@ -3098,7 +4078,7 @@ function parseVerdict(text) {
|
|
|
3098
4078
|
if (typeof text !== "string") {
|
|
3099
4079
|
return null;
|
|
3100
4080
|
}
|
|
3101
|
-
const visible = text.
|
|
4081
|
+
const visible = withoutThinking(text).slice(0, 2e4);
|
|
3102
4082
|
for (const candidate of jsonObjects(visible)) {
|
|
3103
4083
|
try {
|
|
3104
4084
|
const result = normalizeVerdict(JSON.parse(candidate));
|
|
@@ -3444,7 +4424,7 @@ import {
|
|
|
3444
4424
|
writeFileSync,
|
|
3445
4425
|
mkdirSync
|
|
3446
4426
|
} from "node:fs";
|
|
3447
|
-
import { createRequire } from "node:module";
|
|
4427
|
+
import { createRequire as createRequire2 } from "node:module";
|
|
3448
4428
|
import path from "node:path";
|
|
3449
4429
|
import process2 from "node:process";
|
|
3450
4430
|
import { fileURLToPath } from "node:url";
|
|
@@ -3469,7 +4449,7 @@ function embeddedModel(sea = loadSea()) {
|
|
|
3469
4449
|
return null;
|
|
3470
4450
|
}
|
|
3471
4451
|
}
|
|
3472
|
-
function loadSea(load =
|
|
4452
|
+
function loadSea(load = createRequire2(import.meta.url)) {
|
|
3473
4453
|
try {
|
|
3474
4454
|
return load("node:sea");
|
|
3475
4455
|
} catch {
|
|
@@ -3868,7 +4848,7 @@ function scoreResults(results = {}, options = {}) {
|
|
|
3868
4848
|
var package_default = {
|
|
3869
4849
|
name: "spamscanner",
|
|
3870
4850
|
description: "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
|
|
3871
|
-
version: "7.0.
|
|
4851
|
+
version: "7.0.3",
|
|
3872
4852
|
author: "Forward Email (https://forwardemail.net)",
|
|
3873
4853
|
bin: {
|
|
3874
4854
|
spamscanner: "./dist/esm/cli.js"
|
|
@@ -3971,6 +4951,9 @@ var package_default = {
|
|
|
3971
4951
|
license: "BUSL-1.1",
|
|
3972
4952
|
main: "./dist/cjs/index.cjs",
|
|
3973
4953
|
module: "./dist/esm/index.js",
|
|
4954
|
+
optionalDependencies: {
|
|
4955
|
+
re2: "^1.22.1"
|
|
4956
|
+
},
|
|
3974
4957
|
packageManager: "pnpm@10.34.6",
|
|
3975
4958
|
repository: {
|
|
3976
4959
|
type: "git",
|
|
@@ -5333,6 +6316,7 @@ function mergeLegacyOptions(options) {
|
|
|
5333
6316
|
}
|
|
5334
6317
|
return config;
|
|
5335
6318
|
}
|
|
6319
|
+
var MAX_LOOKALIKE_HOSTS = 1e3;
|
|
5336
6320
|
var SpamScanner = class {
|
|
5337
6321
|
/**
|
|
5338
6322
|
* @param {object} [options] - see DEFAULTS
|
|
@@ -5481,28 +6465,22 @@ var SpamScanner = class {
|
|
|
5481
6465
|
async scanFile(file, options) {
|
|
5482
6466
|
return this.scan(await readFile(file), options);
|
|
5483
6467
|
}
|
|
5484
|
-
async checkLinks(links, config) {
|
|
6468
|
+
async checkLinks(links, config, hosts = []) {
|
|
5485
6469
|
const phishing = [];
|
|
5486
6470
|
const homographs = [];
|
|
5487
6471
|
const seenHosts = /* @__PURE__ */ new Set();
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
5491
|
-
|
|
5492
|
-
url: link.url,
|
|
5493
|
-
text: link.text,
|
|
5494
|
-
message: `A link shows "${link.text}" but goes to ${link.host}`
|
|
5495
|
-
}));
|
|
5496
|
-
}
|
|
5497
|
-
if (seenHosts.has(link.host)) {
|
|
5498
|
-
continue;
|
|
6472
|
+
const analysed = /* @__PURE__ */ new Set();
|
|
6473
|
+
const lookalike = (host, extra) => {
|
|
6474
|
+
if (analysed.has(host) || analysed.size >= MAX_LOOKALIKE_HOSTS) {
|
|
6475
|
+
return;
|
|
5499
6476
|
}
|
|
5500
|
-
|
|
5501
|
-
const analysis = this.homograph.detectHomographAttack(
|
|
5502
|
-
|
|
6477
|
+
analysed.add(host);
|
|
6478
|
+
const analysis = this.homograph.detectHomographAttack(host);
|
|
6479
|
+
const counts = extra.source === "text" ? analysis.riskScore >= 0.95 || analysis.riskScore >= 0.85 && analysis.brand.length >= 5 : analysis.riskScore >= 0.5;
|
|
6480
|
+
if (counts) {
|
|
5503
6481
|
const item = finding({
|
|
5504
6482
|
type: "homograph",
|
|
5505
|
-
|
|
6483
|
+
...extra,
|
|
5506
6484
|
domain: analysis.domain,
|
|
5507
6485
|
unicode: analysis.unicode,
|
|
5508
6486
|
brand: analysis.brand,
|
|
@@ -5514,11 +6492,28 @@ var SpamScanner = class {
|
|
|
5514
6492
|
phishing.push(item);
|
|
5515
6493
|
homographs.push(item);
|
|
5516
6494
|
}
|
|
6495
|
+
};
|
|
6496
|
+
for (const link of links) {
|
|
6497
|
+
if (isDeceptiveLink(link)) {
|
|
6498
|
+
phishing.push(finding({
|
|
6499
|
+
type: "deceptive_link",
|
|
6500
|
+
url: link.url,
|
|
6501
|
+
text: link.text,
|
|
6502
|
+
message: `A link shows "${link.text}" but goes to ${link.host}`
|
|
6503
|
+
}));
|
|
6504
|
+
}
|
|
6505
|
+
seenHosts.add(link.host);
|
|
6506
|
+
}
|
|
6507
|
+
for (const link of [...links.filter((link2) => link2.text), ...links.filter((link2) => !link2.text)]) {
|
|
6508
|
+
lookalike(link.host, { url: link.url });
|
|
6509
|
+
}
|
|
6510
|
+
for (const host of hosts) {
|
|
6511
|
+
lookalike(host, { source: "text" });
|
|
5517
6512
|
}
|
|
5518
|
-
const
|
|
5519
|
-
const domains = [...new Set(
|
|
6513
|
+
const checked = [...seenHosts].slice(0, config.phishing.maxHosts);
|
|
6514
|
+
const domains = [...new Set(checked.map((host) => registrableDomain(host)))];
|
|
5520
6515
|
const [blocked, listed] = await Promise.all([
|
|
5521
|
-
config.phishing.cloudflare &&
|
|
6516
|
+
config.phishing.cloudflare && checked.length > 0 ? withTimeout(this.dns.checkCloudflare(checked, { adult: config.phishing.adult }), config.timeout, []) : [],
|
|
5522
6517
|
config.dnsbl.domain?.length > 0 && domains.length > 0 ? withTimeout(this.dns.checkDomains(domains, config.dnsbl.domain), config.timeout, []) : []
|
|
5523
6518
|
]);
|
|
5524
6519
|
for (const { host, category } of blocked) {
|
|
@@ -5532,7 +6527,7 @@ var SpamScanner = class {
|
|
|
5532
6527
|
message: `Link domain ${value} is listed in ${zone}`
|
|
5533
6528
|
}));
|
|
5534
6529
|
}
|
|
5535
|
-
return { phishing, homographs };
|
|
6530
|
+
return { phishing, homographs, uncheckedHosts: seenHosts.size - checked.length };
|
|
5536
6531
|
}
|
|
5537
6532
|
async scanViruses(mail, config) {
|
|
5538
6533
|
const settings = config.clamav === true ? {} : config.clamav;
|
|
@@ -5604,6 +6599,10 @@ var SpamScanner = class {
|
|
|
5604
6599
|
* @param {object} [options.session] - what the receiving server knows:
|
|
5605
6600
|
* remoteAddress (client IP), resolvedClientHostname (verified reverse
|
|
5606
6601
|
* DNS), helo, and envelope { mailFrom: {address}, rcptTo: [{address}] }
|
|
6602
|
+
* @param {object} [options.authenticationResults] - SPF, DKIM, DMARC and
|
|
6603
|
+
* ARC results the receiving server already has, as mailauth's
|
|
6604
|
+
* authenticate() gives them ({spf, dkim: {results}, dmarc, arc}): scored
|
|
6605
|
+
* as if checked here, without checking again
|
|
5607
6606
|
* @returns {Promise<object>} isSpam, score, threshold, action ("accept",
|
|
5608
6607
|
* "tag" or "reject"), message, tests, results (per detector), links,
|
|
5609
6608
|
* language, tokens, mail and metrics
|
|
@@ -5622,13 +6621,12 @@ var SpamScanner = class {
|
|
|
5622
6621
|
const { raw, mail } = await this.parse(source);
|
|
5623
6622
|
const extracted = this.getFeatures(mail);
|
|
5624
6623
|
const classification = this.getClassification(extracted.features);
|
|
5625
|
-
const
|
|
6624
|
+
const obfuscation = obfuscationOf(`${mail.subject || ""}
|
|
5626
6625
|
${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLength });
|
|
5627
|
-
const obfuscation = { invisible: body.invisible, mixed: body.mixed, styled: body.styled };
|
|
5628
6626
|
const attachments = config.attachments === false ? [] : inspectAttachments(mail.attachments).filter((item) => config.macros !== false || !["macro", "pdf_active", "rtf_object"].includes(item.type)).map((item) => finding(item));
|
|
5629
6627
|
const [links, authentication, reputation, dnsbl, viruses, toxicity, nsfw] = await Promise.all([
|
|
5630
|
-
this.checkLinks(extracted.links, config),
|
|
5631
|
-
config.authentication && session.remoteAddress ? authenticate(raw, {
|
|
6628
|
+
this.checkLinks(extracted.links, config, extracted.hosts),
|
|
6629
|
+
config.authenticationResults && typeof config.authenticationResults === "object" ? normalizeAuthOutput(config.authenticationResults) : config.authentication && session.remoteAddress ? authenticate(raw, {
|
|
5632
6630
|
ip: session.remoteAddress,
|
|
5633
6631
|
helo: session.helo,
|
|
5634
6632
|
sender: session.envelope?.mailFrom?.address,
|
|
@@ -5677,7 +6675,12 @@ ${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLengt
|
|
|
5677
6675
|
const close = scored.score >= low && scored.score < high;
|
|
5678
6676
|
if (mode === "always" || mode === "auto" && (close || classification.category === "unsure" || classification.category === "disabled")) {
|
|
5679
6677
|
try {
|
|
5680
|
-
results.llm = await llm.classify(mail, {
|
|
6678
|
+
results.llm = await llm.classify(mail, {
|
|
6679
|
+
links: extracted.links,
|
|
6680
|
+
text: extracted.text,
|
|
6681
|
+
hidden: extracted.hidden,
|
|
6682
|
+
authentication: authentication ? summarizeAuth(authentication) : null
|
|
6683
|
+
});
|
|
5681
6684
|
} catch (error) {
|
|
5682
6685
|
debug2("LLM check failed: %s", error.message);
|
|
5683
6686
|
results.llm = {
|
|
@@ -5711,6 +6714,8 @@ ${typeof mail.text === "string" ? mail.text : ""}`, { maxLength: config.maxLengt
|
|
|
5711
6714
|
// Grouped as in earlier versions.
|
|
5712
6715
|
executables: attachments.filter((item) => ["executable", "disguised_executable", "double_extension", "rtl_override", "executable_in_archive"].includes(item.type)),
|
|
5713
6716
|
macros: attachments.filter((item) => ["macro", "pdf_active", "rtf_object"].includes(item.type)),
|
|
6717
|
+
// Link hosts beyond phishing.maxHosts, which were not looked up.
|
|
6718
|
+
uncheckedHosts: links.uncheckedHosts,
|
|
5714
6719
|
idnHomographAttack: {
|
|
5715
6720
|
detected: links.homographs.length > 0,
|
|
5716
6721
|
domains: links.homographs,
|