spamscanner 7.0.2 → 7.0.4

package/dist/types/parse.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Parse a message with mailparser, which does not make text of HTML here:
+ * `text` is that of the plain text parts only (empty for a message of HTML
+ * alone), and the text HTML shows is read by getFeatures.
+ * @param {Buffer|string} raw
+ * @returns {Promise<object>} the parsed message
+ */
+export function parseMessage(raw: Buffer | string): Promise<object>;
+/**
+ * A message mailparser can finish: it never finishes one that ends in the
+ * header of an attached message (however that header names or folds its
+ * type, or only a file name like "fw.eml") without the blank line after it,
+ * so every message gets one.
+ * @param {Buffer|string} raw
+ * @returns {Buffer}
+ */
+export function finishable(raw: Buffer | string): Buffer;
+/**
+ * Parse a message, reading what can be read of one beyond mailparser's
+ * limits (more than 1000 MIME parts, a part's header over 1 MB): its parts
+ * before the 990th line that may start one (or its first half, again and
+ * again, when attached messages inline make the parts), and, when a header
+ * is too large, the message with its header lines cut down, or its first
+ * megabyte when the header is that of a later part. `error` is the limit,
+ * or null. Other errors are thrown, as earlier versions did (servers defer
+ * the message).
+ * @param {Buffer} raw
+ * @param {(raw: Buffer) => Promise<object>} [parse] - for tests
+ * @returns {Promise<{mail: object, error: string|null}>}
+ */
+export function parseWithin(raw: Buffer, parse?: (raw: Buffer) => Promise<object>): Promise<{
+    mail: object;
+    error: string | null;
+}>;
+/**
+ * A message's attachments, with those of the messages attached to it (as a
+ * forwarded message is), named after the message that carries them:
+ * "fw.eml/invoice.exe". An attached message that cannot be read (beyond the
+ * parser's limits, or failing it) never fails the scan: why goes to `errors`.
+ * @param {Array<object>} attachments - mailparser attachments
+ * @param {string[]} [errors] - gets why attached messages were not read whole
+ * @param {(raw: Buffer) => Promise<object>} [parse] - for tests
+ * @returns {Promise<Array<object>>}
+ */
+export function expandAttachments(attachments?: Array<object>, errors?: string[], parse?: (raw: Buffer) => Promise<object>): Promise<Array<object>>;
+import { Buffer } from 'node:buffer';

package/dist/types/reputation.d.ts

@@ -1,11 +1,47 @@
+/**
+ * A domain as a hostname in ASCII ("Bücher.example." is
+ * "xn--bcher-kva.example"), or null when it is not one: only letters, digits,
+ * hyphens and underscores in dot-separated labels of at most 63 characters,
+ * 253 in all. (A URL such as "bank.com/x.attacker.example" is not a
+ * hostname, though URL parsers read "bank.com" from it.)
+ * @param {string} value
+ * @returns {string|null}
+ */
+export function asciiHostname(value: string): string | null;
 /**
  * Whether a value (IP, hostname or email address) matches a list entry. A
  * domain entry matches the domain, its subdomains and addresses at either.
+ *
+ * With `hostnamesOnly` (for an allowlist, which must trust nothing it cannot
+ * read), only a hostname matches a parent domain. Without it (for a
+ * denylist, which anything should match) any name does, such as
+ * "a_b!.evil.example" or one with a label of 64 characters, read from its
+ * last 127 labels and 253 characters, so the work is bounded however long
+ * the name.
  * @param {string} value
  * @param {Set<string>} list
+ * @param {object} [options]
+ * @param {boolean} [options.hostnamesOnly]
  * @returns {string|null} the entry that matched
  */
-export function matchList(value: string, list: Set<string>): string | null;
+export function matchList(value: string, list: Set<string>, options?: {
+    hostnamesOnly?: boolean;
+}): string | null;
+/**
+ * Set isAllowlisted and isTruthSource of a check() result from the trusted
+ * values only (values that cannot be forged, see trustedReputationValues),
+ * in the order checked.
+ * @param {object} result - from ReputationChecker#check
+ * @param {Array<string|{value: string, signers: string[]}>} trusted -
+ *   values, or values trusted through relaxed alignment with `signers` only
+ *   (the aligned DKIM or SPF domains), whose entries one of them must vouch
+ *   for
+ * @returns {object} a copy of the result
+ */
+export function trustReputation(result: object, trusted?: Array<string | {
+    value: string;
+    signers: string[];
+}>): object;
 /**
  * The values a message's reputation is checked by: the client IP address and
  * hostname, and the From, envelope sender and Reply-To addresses with their
@@ -15,6 +51,36 @@ export function matchList(value: string, list: Set<string>): string | null;
  * @returns {string[]}
  */
 export function reputationValues(mail?: object, session?: object): string[];
+/**
+ * The reputation values that cannot be forged, which alone can allowlist a
+ * message or make it a truth source: the client IP address and its verified
+ * hostname (from the receiving server), a From address only when
+ * authentication shows its domain sent it (DMARC passes for it, or a passing
+ * DKIM signature is aligned with it), and the envelope sender only when SPF
+ * passes for its domain. Never Reply-To. A result that does not name its
+ * domain, or names one that is not a hostname, counts for none.
+ *
+ * Alignment is relaxed, as in DMARC (mailauth's DMARC domain is the From
+ * domain's registrable domain), so From values are returned as
+ * {value, signers}, with the aligned DKIM and SPF domains: an allowlist
+ * entry naming a subdomain counts only for a signer that is it, under it or
+ * a parent of it, of the same registrable domain. A DMARC pass for exactly
+ * the From domain, without alignment details, trusts it outright; one for
+ * its registrable domain only, registrable-domain entries. Under strict
+ * alignment (adkim=s, aspf=s in the domain's DMARC record) only the From
+ * domain itself vouches for it. A DKIM signature of part of the body (l=
+ * shorter than the body, mailauth's underSized) vouches for nothing, nor
+ * does a DMARC pass that rests on one alone, and no signature vouches for a
+ * From address when DMARC fails.
+ * @param {object} mail - parsed message
+ * @param {object} [session] - remoteAddress, resolvedClientHostname, envelope
+ * @param {object|null} [authentication] - SPF, DKIM and DMARC results
+ * @returns {Array<string|{value: string, signers: string[]}>}
+ */
+export function trustedReputationValues(mail?: object, session?: object, authentication?: object | null): Array<string | {
+    value: string;
+    signers: string[];
+}>;
 /**
  * Reputation lookups: local allowlists and denylists, and optionally an HTTP
  * reputation service.
@@ -24,6 +90,11 @@ export function reputationValues(mail?: object, session?: object): string[];
  * (and optional `truthSourceValue`, `allowlistValue`, `denylistValue`).
  * Values looked up: the client IP and hostname, the From, envelope sender and
  * Reply-To addresses and their domains.
+ *
+ * Any of these values can put a message on the denylist. Only values that
+ * cannot be forged can put it on the allowlist or make it a truth source
+ * (see trustedReputationValues): anyone can write any address in From or
+ * Reply-To.
  */
 export class ReputationChecker {
     /**
@@ -36,6 +107,8 @@ export class ReputationChecker {
      * @param {number} [options.concurrency] - parallel service requests
      * @param {number} [options.cacheTtl] - milliseconds answers are kept
      * @param {number} [options.cacheSize]
+     * @param {Map} [options.cache] - answers kept (shared with another checker
+     *   of the same service)
      */
     constructor(options?: {
         allowlist?: string[];
@@ -46,6 +119,7 @@ export class ReputationChecker {
         concurrency?: number;
         cacheTtl?: number;
         cacheSize?: number;
+        cache?: Map<any, any>;
     });
     allowlist: Set<string>;
     denylist: Set<string>;
@@ -66,8 +140,15 @@ export class ReputationChecker {
     /**
      * Check a message's sender values against the lists and the service.
      * @param {string[]} values - IPs, hostnames, addresses
+     * @param {object} [options]
+     * @param {string[]} [options.trusted] - the values that can allowlist the
+     *   message or make it a truth source (default: all of them); see
+     *   trustedReputationValues
      * @returns {Promise<object>} isTruthSource, isAllowlisted, isDenylisted with
-     *   the matching values, plus checkedValues and per-value details
+     *   the matching values, plus checkedValues, per-value details, and
+     *   allowlisted (each value on the allowlist, with its entry)
      */
-    check(values: string[]): Promise<object>;
+    check(values: string[], options?: {
+        trusted?: string[];
+    }): Promise<object>;
 }

package/dist/types/score.d.ts

@@ -19,7 +19,7 @@ export function bayesTestName(probability: number): string;
  * @param {object} [options.authScores] - see calculateAuthScore
  * @returns {{score: number, threshold: number, rejectThreshold: number, isSpam: boolean, action: 'accept'|'tag'|'reject', tests: Array<{name: string, score: number, description: string}>}}
  */
-export function scoreResults(results?: object, options?: {
+export function scoreResults(results: object, options?: {
     scores?: object;
     threshold?: number;
     rejectThreshold?: number;
@@ -56,6 +56,8 @@ export namespace DEFAULT_SCORES {
     let doubleExtension: number;
     let rtlOverride: number;
     let executableInArchive: number;
+    let scriptInArchive: number;
+    let riskyAttachment: number;
     let encryptedArchive: number;
     let macro: number;
     let pdfActive: number;
@@ -63,6 +65,7 @@ export namespace DEFAULT_SCORES {
     let htmlAttachment: number;
     let activeHtmlAttachment: number;
     let virus: number;
+    let mimeTooComplex: number;
     let invisibleCharacters: number;
     let mixedScriptWords: number;
     let styledLetters: number;

package/dist/types/server.d.ts

@@ -27,7 +27,8 @@ export function sessionFromQuery(query: URLSearchParams): object;
  * - POST /learn/spam, /learn/ham → teach the classifier (only with a token)
  *
  * The request body is the raw message. Query parameters: ip, hostname, helo,
- * from, to (see sessionFromQuery), and for /check, subjectTag.
+ * from, to (see sessionFromQuery), and for /check, subjectTag. Authentication
+ * (SPF, DKIM, DMARC, ARC) and IP blocklists run only when ip is given.
  *
  * @param {import('./index.js').SpamScanner} scanner
  * @param {object} [options]
@@ -45,15 +46,20 @@ export function createHttpServer(scanner: import("./index.js").SpamScanner, opti
  * A plain TCP server: a client sends a raw message and closes its side; the
  * server answers with the result as one line of JSON (or a short text line)
  * and closes the connection.
+ *
+ * The client's IP address is not known, so authentication (SPF, DKIM, DMARC,
+ * ARC) and IP blocklists do not run for these scans.
  * @param {import('./index.js').SpamScanner} scanner
  * @param {object} [options]
  * @param {boolean} [options.json] - answer in JSON (default true)
  * @param {number} [options.maxSize]
+ * @param {number} [options.timeout] - close a connection that sends nothing for this many milliseconds (default 60000)
  * @returns {net.Server}
  */
 export function createTcpServer(scanner: import("./index.js").SpamScanner, options?: {
     json?: boolean;
     maxSize?: number;
+    timeout?: number;
 }): net.Server;
 import http from 'node:http';
 import net from 'node:net';

package/dist/types/sources.d.ts

@@ -19,7 +19,9 @@ export function parseCsvLine(line: string): string[] | null;
  */
 export function normalizeLabel(value: unknown): "spam" | "ham" | null;
 /**
- * Read labelled rows from a CSV file with a header row.
+ * Read labelled rows from a CSV file with a header row. A quoted field may
+ * span lines; a record still open after 10,000 lines or 1 MiB (a stray
+ * quote) is dropped, and reading goes on with the next line.
  * @param {string} file
  * @param {object} [options]
  * @param {string} [options.textColumn] - column holding the message (default: text, message, body, email text...)
@@ -37,9 +39,11 @@ export function readCsv(file: string, options?: {
     label: "spam" | "ham";
 }>;
 /**
- * Read labelled rows from a JSON Lines file: one object per line with a text
- * field ("text", "message", "body") and a label field ("label", "label_text",
- * "is_spam", "category").
+ * Read labelled rows from a JSON Lines file: one object per line, with a text
+ * field and a label field found by the same names as readCsv's columns, in
+ * any case ("label_text" before "label", and "content" before "email"), the
+ * first of them that holds text or a label. A line that is not a JSON object
+ * is skipped.
  * @param {string} file
  * @param {object} [options] - textColumn, labelColumn, subjectColumn as for readCsv
  * @returns {AsyncGenerator<{text: string, subject: string, label: 'spam'|'ham'}>}
@@ -52,7 +56,8 @@ export function readJsonl(file: string, options?: object): AsyncGenerator<{
 /**
  * Read raw messages from a path: an .eml file, an mbox file, or a directory
  * of messages (one per file, such as a Maildir or a folder of .eml files,
- * searched recursively; mbox files inside are read too).
+ * searched recursively; mbox files inside are read too). A gzip file is read
+ * only as an mbox file (named .mbox.gz); other .gz files are skipped.
  * @param {string} source
  * @returns {AsyncGenerator<string|Buffer>}
  */

package/dist/types/spamd.d.ts

@@ -33,12 +33,16 @@ export function answer(scanner: import("./index.js").SpamScanner, request: {
  * Supports CHECK, SYMBOLS, REPORT, REPORT_IFSPAM, PROCESS, HEADERS, PING,
  * SKIP and (with allowTell) TELL. Compressed requests are refused.
  *
+ * spamd clients do not send the client's IP address, so authentication
+ * (SPF, DKIM, DMARC, ARC) and IP blocklists do not run for these scans.
+ *
  * @param {import('./index.js').SpamScanner} scanner
  * @param {object} [options]
  * @param {number} [options.maxSize] - largest message accepted, in bytes
  * @param {boolean} [options.allowTell] - accept TELL (learning) requests
  * @param {string} [options.modelPath] - where TELL saves the classifier
  * @param {string|null} [options.subjectTag] - tag for spam in PROCESS and HEADERS answers
+ * @param {number} [options.timeout] - close a connection that sends nothing for this many milliseconds (default 60000)
  * @returns {net.Server}
  */
 export function createSpamdServer(scanner: import("./index.js").SpamScanner, options?: {
@@ -46,6 +50,7 @@ export function createSpamdServer(scanner: import("./index.js").SpamScanner, opt
     allowTell?: boolean;
     modelPath?: string;
     subjectTag?: string | null;
+    timeout?: number;
 }): net.Server;
 export namespace SPAMD_EXIT_CODES {
     export { EX_OK };

package/dist/types/tokenizer.d.ts

@@ -7,6 +7,14 @@
  * @returns {RegExp}
  */
 export function characterClass(ranges: number[][], flags?: string): RegExp;
+/**
+ * The address a link goes to, as a URL: through a security gateway, the
+ * address it goes on to (see gatewayTarget), as many as three wrapped in
+ * one another; or null for an address that is no URL.
+ * @param {string} address
+ * @returns {URL|null}
+ */
+export function unwrapLink(address: string): URL | null;
 /**
  * Split text into pieces of at most `size` characters, at whitespace or
  * punctuation where possible.
@@ -86,37 +94,40 @@ export function obfuscationOf(text: string, options?: {
  * @returns {string}
  */
 export function registrableDomain(host: string): string;
+export function extractLinks(text: any, html: any, { maxLength }?: {
+    maxLength?: number;
+}): any[];
 /**
- * Find the links in plain text and HTML. For each link, the host, and for
- * HTML links the visible text when it is itself a web address.
- * @param {string} text
- * @param {string} html
- * @returns {Array<{url: string, host: string, text: string|null}>}
+ * The value of the first href attribute of a tag ("<a ...>"), or null:
+ * read attribute by attribute, as browsers read them, so an "href=" inside
+ * the value of another attribute is not taken for one. In one pass.
+ * @param {string} tag
+ * @returns {string|null}
  */
-export function extractLinks(text: string, html: string): Array<{
-    url: string;
-    host: string;
-    text: string | null;
-}>;
+export function hrefOf(tag: string): string | null;
 /**
  * The end of the tag that starts at `start` (just after its ">"), or -1 when
  * no ">" follows. A quoted attribute may hold ">" ("?a=>b"); a "<" outside
- * quotes is part of the tag, as browsers read it. A quote not closed within
- * 6000 characters is not one: then the tag ends at the next ">".
+ * quotes is part of the tag, as browsers read it. A quote opens a value only
+ * after "=" (in `href=https://a.example/"x` it is part of the value). A quote
+ * not closed within 6000 characters is not one: then the tag ends at the next
+ * ">".
  * @param {string} html
  * @param {number} start
  * @returns {number}
  */
 export function tagEnd(html: string, start: number): number;
 /**
- * The links of HTML: each <a> tag with an href (up to 4000 characters, in a
- * tag of up to 6000), and its text without tags when "</a>" follows within
- * 2000 characters.
+ * The links of HTML: each <a> tag with an href, and the text it shows (see
+ * visibleText: without hidden text, and here without invisible characters)
+ * when "</a>" or another <a> tag follows within 2000 characters (otherwise
+ * no text: a mail client shows a link never closed, or around a long image
+ * or table, all the same).
  *
- * Read in about one pass: a tag is read up to its end, and the next tag
- * looked for after it, and the closing tags are found with one search, so
- * the HTML is not read again for each tag. (A tag whose quote is not closed
- * may read up to 6000 characters, but another such tag within them needs the
+ * Read in about one pass: a tag is read up to its end, the tags of a link's
+ * text up to its "</a>", and the next <a> tag looked for after them, so the
+ * HTML is not read again for each tag. (A tag whose quote is not closed may
+ * read up to 6000 characters, but another such tag within them needs the
  * other quote mark, so at most two do.)
  * @param {string} html
  * @returns {Array<{href: string, inner: string}>}
@@ -129,8 +140,9 @@ export function anchors(html: string): Array<{
  * Host names in text that are not links: written without "http://"
  * ("paypa1-secure.example") or the domain of an email address. Only names
  * under a real public suffix count ("node.js" and "file.txt" do not), written
- * in lowercase as addresses are (so "end.Click" in text missing a space is
- * not taken for one), not the local part of an address, and at most 50.
+ * in lowercase as addresses are or all in capitals ("PAYPAL-VERIFY.COM", read
+ * in lowercase; so "end.Click" in text missing a space is not taken for one),
+ * not the local part of an address, and at most 50.
  * These are for lookalike checks, not classifier features.
  * @param {string} text
  * @param {Set<string>} [known] - hosts of the links already found
@@ -139,13 +151,28 @@ export function anchors(html: string): Array<{
 export function extractHosts(text: string, known?: Set<string>): string[];
 /**
  * Whether the visible text of a link names a different site than the link
- * goes to ("https://paypal.com" linking to another domain).
- * @param {{host: string, text: string|null}} link
+ * goes to ("https://paypal.com" linking to another domain). A link through an
+ * email service's click tracking (see CLICK_TRACKERS) goes elsewhere by
+ * design, so it counts only when the sender is known (`from`, the domain of
+ * the From address, or "" for none), the text shows a whole address with
+ * "http://" or "https://" of a brand's own domain (see BRANDS; not one that
+ * newsletters link to, see LINKED_BRANDS) with no path or one that asks to
+ * sign in or pay, and the sender is not that brand (nor one of its sending
+ * domains, "facebookmail.com", see SENDING_DOMAINS): a hijacked account of a mailing service
+ * sending "https://www.paypal.com" to a phishing page, not a newsletter's
+ * links. A link through a security gateway is read as the address the
+ * gateway goes on to, where the gateway's link holds it (see unwrapLink).
+ * @param {{host: string, text: string|null, url?: string}} link
+ * @param {object} [options]
+ * @param {string} [options.from] - the sender's domain
  * @returns {boolean}
  */
 export function isDeceptiveLink(link: {
     host: string;
     text: string | null;
+    url?: string;
+}, { from }?: {
+    from?: string;
 }): boolean;
 /**
  * Classifier features of a message parsed by mailparser.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "spamscanner",
   "description": "Spam, phishing and malware filter for email in any language: a trainable classifier, link and attachment checks, SPF/DKIM/DMARC, and an optional AI second opinion from any LLM, including local Ollama models. Works as a library, CLI, Postfix milter or HTTP API.",
-  "version": "7.0.2",
+  "version": "7.0.4",
   "author": "Forward Email (https://forwardemail.net)",
   "bin": {
     "spamscanner": "./dist/esm/cli.js"
@@ -37,6 +37,7 @@
   "devDependencies": {
     "@commitlint/cli": "^19.8.1",
     "@commitlint/config-conventional": "^19.8.1",
+    "@types/node": "^18.19.130",
     "c8": "^10.1.3",
     "esbuild": "^0.28.2",
     "fixpack": "^4.0.0",
@@ -56,19 +57,31 @@
   },
   "exports": {
     ".": {
-      "types": "./dist/types/index.d.ts",
-      "import": "./dist/esm/index.js",
-      "require": "./dist/cjs/index.cjs"
+      "import": {
+        "types": "./dist/types/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/types/index.d.cts",
+        "default": "./dist/cjs/index.cjs"
+      }
     },
     "./arf": {
-      "types": "./dist/types/arf.d.ts",
-      "import": "./dist/esm/arf.js",
-      "require": "./dist/cjs/arf.cjs"
+      "import": {
+        "types": "./dist/types/arf.d.ts",
+        "default": "./dist/esm/arf.js"
+      },
+      "require": {
+        "types": "./dist/types/arf.d.cts",
+        "default": "./dist/cjs/arf.cjs"
+      }
     },
     "./package.json": "./package.json"
   },
   "files": [
     "dist/",
+    "!dist/standalone/",
+    "!dist/types/cli.d.ts",
     "model/"
   ],
   "homepage": "https://spamscanner.net",
@@ -114,7 +127,7 @@
   "type": "module",
   "types": "./dist/types/index.d.ts",
   "scripts": {
-    "build": "node build.mjs && tsc --project tsconfig.json",
+    "build": "node build.mjs",
     "i18n:check": "node scripts/i18n-check.js",
     "lint": "fixpack && remark . -qfo && xo --fix",
     "model:train": "node scripts/train-model.js",