spamscanner 7.0.1 → 7.0.3

package/dist/types/attachments.d.ts

@@ -16,6 +16,12 @@ export function sniff(buffer: Buffer): {
 /**
  * The entries of a ZIP file, read from its central directory without
  * decompressing anything: name, sizes, and whether the entry is encrypted.
+ *
+ * Every end record that ends the file (its comment running to the last byte)
+ * is read, as well as the last one found: archive programs differ in which
+ * they use, so a directory hidden behind another record is read too.  ZIP64
+ * archives, and archives after other data (self-extracting archives), are
+ * read as well.
  * @param {Buffer} buffer
  * @param {number} [limit] - most entries to read
  * @returns {Array<{name: string, encrypted: boolean, size: number}>}
@@ -34,6 +40,7 @@ export function zipEntries(buffer: Buffer, limit?: number): Array<{
  * - double_extension: "invoice.pdf.exe"
  * - rtl_override: a right-to-left override hides the real extension (a file named "invoice", U+202E, "fdp.exe" shows as "invoiceexe.pdf")
  * - executable_in_archive: a ZIP holds a program, read without unpacking
+ *   (extension: the first program's, extensions: every program's)
  * - encrypted_archive: a password-protected ZIP, which virus scanners cannot open
  * - macro: an Office document with a VBA project
  * - pdf_active: a PDF with JavaScript, launch actions or embedded files
@@ -41,7 +48,7 @@ export function zipEntries(buffer: Buffer, limit?: number): Array<{
  * - html_attachment: an HTML or SVG file, the usual carrier of fake login pages
  *
  * @param {Array<{filename?: string, contentType?: string, content?: Buffer}>} attachments
- * @returns {Array<{type: string, filename: string, message: string, extension?: string}>}
+ * @returns {Array<{type: string, filename: string, message: string, extension?: string, extensions?: string[]}>}
  */
 export function inspectAttachments(attachments?: Array<{
     filename?: string;
@@ -52,6 +59,7 @@ export function inspectAttachments(attachments?: Array<{
     filename: string;
     message: string;
     extension?: string;
+    extensions?: string[];
 }>;
 /**
  * File extensions that run code when opened, on Windows, macOS, Linux and

package/dist/types/homograph.d.ts

@@ -6,13 +6,24 @@
  */
 export function skeleton(label: string): string;
 /**
- * Levenshtein distance, giving up once it exceeds `max`.
+ * Edit distance (Levenshtein, with two neighbouring letters swapped counting
+ * as one edit: "paypla" is one from "paypal"), giving up once it exceeds
+ * `max`.
  * @param {string} a
  * @param {string} b
  * @param {number} [max]
  * @returns {number}
  */
 export function editDistance(a: string, b: string, max?: number): number;
+/**
+ * The brand a name joins to a phishing word ("paypallogin", "verifyamazon"),
+ * or null. Brands of five letters or more only, and not brands that are words
+ * themselves.
+ * @param {string} name
+ * @param {Set<string>} brands
+ * @returns {string|null}
+ */
+export function brandWithPhishingWord(name: string, brands: Set<string>): string | null;
 /**
  * Brands most often imitated in phishing: banks, payment services, mail and
  * cloud providers, shops, couriers and tax offices. A domain whose name is a

package/dist/types/html.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Text with its character references decoded, as browsers decode them: by
+ * number, also without a ";", and by name (the names of Latin-1 and those
+ * mail commonly uses; also without a ";" for the names browsers read so,
+ * except in an attribute where a letter, digit or "=" follows, as in a link
+ * "?a=1&copy=2"). An unknown name stays as it is.
+ * @param {string} value
+ * @param {object} [options]
+ * @param {boolean} [options.attribute] - the value of an attribute
+ * @returns {string}
+ */
+export function decodeEntities(value: string, { attribute }?: {
+    attribute?: boolean;
+}): string;
+/**
+ * The text of HTML as a mail client shows it, read in linear time: without
+ * tags, comments, scripts, styles and the title, with character references
+ * decoded, without a break inside a word at an inline tag
+ * ("Free Sp<span>ins</span>" is "Free Spins") but with one between blocks
+ * ("<td>a</td><td>b</td>" is "a" and "b"), and without text that is hidden:
+ * by an element that hides what it holds (display, opacity, the hidden
+ * attribute, a clipped box of no size, text moved off the page), or that is
+ * invisible, transparent or in a font of a pixel or less where it is (which
+ * an element inside can set again). Not colours on a background, or classes
+ * of a style sheet: text hidden so is read as shown.
+ * @param {string} html
+ * @returns {{text: string, hidden: number}} the text, and how many
+ * characters of text were hidden
+ */
+export function visibleText(html: string): {
+    text: string;
+    hidden: number;
+};

package/dist/types/index.d.ts

@@ -129,9 +129,10 @@ export class SpamScanner {
      * @returns {ReturnType<SpamScanner['scan']>}
      */
     scanFile(file: string, options?: object): ReturnType<SpamScanner["scan"]>;
-    checkLinks(links: any, config: any): Promise<{
+    checkLinks(links: any, config: any, hosts?: any[]): Promise<{
         phishing: any[];
         homographs: any[];
+        uncheckedHosts: number;
     }>;
     scanViruses(mail: any, config: any): Promise<any[]>;
     scanToxicity(mail: any, config: any): Promise<any>;
@@ -144,12 +145,17 @@ export class SpamScanner {
      * @param {object} [options.session] - what the receiving server knows:
      *   remoteAddress (client IP), resolvedClientHostname (verified reverse
      *   DNS), helo, and envelope { mailFrom: {address}, rcptTo: [{address}] }
+     * @param {object} [options.authenticationResults] - SPF, DKIM, DMARC and
+     *   ARC results the receiving server already has, as mailauth's
+     *   authenticate() gives them ({spf, dkim: {results}, dmarc, arc}): scored
+     *   as if checked here, without checking again
      * @returns {Promise<object>} isSpam, score, threshold, action ("accept",
      *   "tag" or "reject"), message, tests, results (per detector), links,
      *   language, tokens, mail and metrics
      */
     scan(source: Buffer | string | Uint8Array | AsyncIterable<any>, options?: {
         session?: object;
+        authenticationResults?: object;
     }): Promise<object>;
 }
 export default SpamScanner;

package/dist/types/linear.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Regular expressions for message content.
+ *
+ * A message can be made of anything, so every pattern used on its content is
+ * written to match in linear time as a JavaScript regular expression: no two
+ * quantifiers that can match the same text in a row, and repetition bounded
+ * where a run of characters could otherwise be scanned again from every
+ * position in it. When the optional re2 package is installed, the patterns
+ * run on RE2, which guarantees linear time (and every pattern here is valid
+ * RE2: no backreferences, no lookaround, repetition counts up to 1000).
+ */
+/**
+ * The RE2 class from the re2 package, or null when it is not installed (or
+ * cannot load, as in a single executable binary).
+ * @param {Function} [load] - require()
+ * @returns {Function|null}
+ */
+export function loadRe2(load?: Function): Function | null;
+/**
+ * A JavaScript pattern for RE2, matching the same text: \s and \S as in
+ * JavaScript, and repetition bounded for JavaScript only made unbounded.
+ * @param {string} source
+ * @returns {string}
+ */
+export function re2Source(source: string): string;
+/**
+ * A regular expression on RE2 when it is installed, otherwise a JavaScript
+ * one (written to run in linear time either way).
+ * @param {string} source
+ * @param {string} [flags]
+ * @param {Function|null} [Engine] - the RE2 class, or null for JavaScript
+ * @returns {RegExp}
+ */
+export function linear(source: string, flags?: string, Engine?: Function | null): RegExp;
+/**
+ * The string without any of `characters` at its end, by scanning back from
+ * the end (a pattern such as /[).,]+$/ is tried at every position of a long
+ * run of those characters).
+ * @param {string} value
+ * @param {string} characters
+ * @returns {string}
+ */
+export function trimEndOf(value: string, characters: string): string;
+/**
+ * Every tag replaced by `replacement`, as `html.replace(/<[^>]*>/g, ...)`
+ * does: from each "<" to the next ">".  Found with indexOf, which reads each
+ * character once: once no ">" follows, no later "<" has one either (the
+ * regular expression would read the rest again from every "<").
+ * @param {string} html
+ * @param {string} [replacement]
+ * @returns {string}
+ */
+export function replaceTags(html: string, replacement?: string): string;
+/**
+ * A pattern for a word of ASCII letters in either case, ASCII only.  (RE2
+ * folds case as Unicode does, where "\u017F" is an "s" and "\u212A" a "k";
+ * JavaScript does only with the u flag.)
+ * @param {string} word
+ * @returns {string}
+ */
+export function caseless(word: string): string;
+/**
+ * Text with the two characters outside ASCII that fold to ASCII letters,
+ * "\u017F" and "\u212A", as "s" and "k", as host names map them (UTS #46),
+ * so that case-insensitive patterns read them alike on either engine.  (Each
+ * is one UTF-16 unit, so positions in the text stay.)
+ * @param {string} text
+ * @returns {string}
+ */
+export function foldToAscii(text: string): string;
+/**
+ * HTML without its script and style elements (their content included), with
+ * every tag replaced by `replacement`.
+ * @param {string} html
+ * @param {string} [replacement]
+ * @returns {string}
+ */
+export function stripTags(html: string, replacement?: string): string;
+/**
+ * Whether patterns run on RE2.
+ * @type {boolean}
+ */
+export const usesRe2: boolean;

package/dist/types/tokenizer.d.ts

@@ -19,6 +19,15 @@ export function characterClass(ranges: number[][], flags?: string): RegExp;
  * @returns {string[]}
  */
 export function chunk(text: string, size?: number): string[];
+/**
+ * Text as a person reads it, for the language model: without the invisible
+ * characters put inside words to break filters (not the joiners scripts and
+ * emoji need), with compatibility forms folded ("𝐅𝐑𝐄𝐄" and "ＦＲＥＥ" are
+ * "FREE"), case kept.
+ * @param {string} text
+ * @returns {string}
+ */
+export function readableText(text: string): string;
 /**
  * Normalize text the way the classifier sees it: invisible characters
  * removed, compatibility forms folded (NFKC turns "ｆｒｅｅ" and "𝐅𝐑𝐄𝐄" into
@@ -54,6 +63,22 @@ export function segmentWords(text: string, options?: {
     mixed: number;
     leet: number;
 };
+/**
+ * The obfuscation in text, as segmentWords counts it ({invisible, mixed,
+ * styled}), segmenting into words only the parts of the text that could have
+ * a word mixing scripts: those with Latin letters and Cyrillic or Greek ones.
+ * @param {string} text
+ * @param {object} [options]
+ * @param {number} [options.maxLength] - characters of text to read
+ * @returns {{invisible: number, mixed: number, styled: boolean}}
+ */
+export function obfuscationOf(text: string, options?: {
+    maxLength?: number;
+}): {
+    invisible: number;
+    mixed: number;
+    styled: boolean;
+};
 /**
  * The registrable domain of a host name ("mail.example.co.uk" gives
  * "example.co.uk"), or the host itself for IP addresses and unknown suffixes.
@@ -73,6 +98,45 @@ export function extractLinks(text: string, html: string): Array<{
     host: string;
     text: string | null;
 }>;
+/**
+ * The end of the tag that starts at `start` (just after its ">"), or -1 when
+ * no ">" follows. A quoted attribute may hold ">" ("?a=>b"); a "<" outside
+ * quotes is part of the tag, as browsers read it. A quote not closed within
+ * 6000 characters is not one: then the tag ends at the next ">".
+ * @param {string} html
+ * @param {number} start
+ * @returns {number}
+ */
+export function tagEnd(html: string, start: number): number;
+/**
+ * The links of HTML: each <a> tag with an href (up to 4000 characters, in a
+ * tag of up to 6000), and its text without tags when "</a>" follows within
+ * 2000 characters.
+ *
+ * Read in about one pass: a tag is read up to its end, and the next tag
+ * looked for after it, and the closing tags are found with one search, so
+ * the HTML is not read again for each tag. (A tag whose quote is not closed
+ * may read up to 6000 characters, but another such tag within them needs the
+ * other quote mark, so at most two do.)
+ * @param {string} html
+ * @returns {Array<{href: string, inner: string}>}
+ */
+export function anchors(html: string): Array<{
+    href: string;
+    inner: string;
+}>;
+/**
+ * Host names in text that are not links: written without "http://"
+ * ("paypa1-secure.example") or the domain of an email address. Only names
+ * under a real public suffix count ("node.js" and "file.txt" do not), written
+ * in lowercase as addresses are (so "end.Click" in text missing a space is
+ * not taken for one), not the local part of an address, and at most 50.
+ * These are for lookalike checks, not classifier features.
+ * @param {string} text
+ * @param {Set<string>} [known] - hosts of the links already found
+ * @returns {string[]}
+ */
+export function extractHosts(text: string, known?: Set<string>): string[];
 /**
  * Whether the visible text of a link names a different site than the link
  * goes to ("https://paypal.com" linking to another domain).
@@ -86,8 +150,9 @@ export function isDeceptiveLink(link: {
 /**
  * Classifier features of a message parsed by mailparser.
  *
- * Every word of the body is a feature, every pair of neighbouring words is
- * another, and the subject's words are features of their own ("s:free").
+ * Every word of the text a mail client shows (see `visibleText`) is a
+ * feature, every pair of neighbouring words is another, and the subject's
+ * words are features of their own ("s:free").
  * The rest describe structure: link domains and tricks, the sender's domain,
  * HTML forms and hidden text, attachment types, headers, language and script,
  * and obfuscation. The same message always gives the same features.
@@ -96,7 +161,7 @@ export function isDeceptiveLink(link: {
  * @param {object} [options]
  * @param {number} [options.maxLength] - characters of body text to read
  * @param {number} [options.maxBigrams] - word pairs to emit
- * @returns {{features: string[], words: string[], language: string|null, script: string|null, links: Array}}
+ * @returns {{features: string[], words: string[], language: string|null, script: string|null, links: Array, hosts: string[], text: string, hidden: number}}
  */
 export function getFeatures(mail?: object, options?: {
     maxLength?: number;
@@ -107,4 +172,11 @@ export function getFeatures(mail?: object, options?: {
     language: string | null;
     script: string | null;
     links: any[];
+    hosts: string[];
+    text: string;
+    hidden: number;
 };
+export const EMAIL_SOURCE: string;
+export const ADDRESS_SOURCE: string;
+export const NAMED_ADDRESS: RegExp;
+export { decodeEntities, visibleText } from "./html.js";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "spamscanner",
   "description": "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
-  "version": "7.0.1",
+  "version": "7.0.3",
   "author": "Forward Email (https://forwardemail.net)",
   "bin": {
     "spamscanner": "./dist/esm/cli.js"
@@ -104,6 +104,9 @@
   "license": "BUSL-1.1",
   "main": "./dist/cjs/index.cjs",
   "module": "./dist/esm/index.js",
+  "optionalDependencies": {
+    "re2": "^1.22.1"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/spamscanner/spamscanner.git"