text-sanctifier 1.3.0 → 1.3.2

package/dist/types/categories.generated.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Category of one BMP code point (one UTF-16 code unit), or null when it is not trash.
+ * Surrogates return null; the caller pairs them and uses classifyAstral.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyCodeUnit(c: number): TrashCategory | null;
+/**
+ * Category of a code point above U+FFFF, or null when it is not trash.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyAstral(c: number): TrashCategory | null;
+/** @typedef {import('./inspectText.js').TrashCategory} TrashCategory */
+/**
+ * Removed by `nukeControls`. Categories: bidi_control, nul, c0_control, c1_control.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const CONTROL_CHARS_REGEX: RegExp;
+/**
+ * Removed by `purgeInvisibleChars`. Categories: bidi_control, zero_width, nonbreaking_space, bom, invisible_format, hangul_filler, variation_selector, tag.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const INVISIBLE_TRASH_REGEX: RegExp;
+export type TrashCategory = import("./inspectText.js").TrashCategory;

package/dist/types/hiddenText.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Decodes tag characters to the ASCII they shadow: U+E0020–U+E007E become
+ * U+0020–U+007E. Anything else (U+E0001 LANGUAGE TAG, U+E007F CANCEL TAG, unassigned
+ * tags, other characters) is written as `<U+XXXX>`, so the result is printable.
+ * @param {string} text
+ * @returns {string}
+ */
+export function decodeTagText(text: string): string;
+/**
+ * Finds runs of tag characters, variation selectors and invisible operators.
+ * Consecutive characters of the same kind form one run. Two kinds of sequence are
+ * legitimate and listed under `benign` instead of `runs`:
+ *  - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
+ *    tags for "gbeng", "gbsct" or "gbwls" and U+E007F
+ *  - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
+ * @param {string} text
+ * @returns {HiddenTextScan}
+ */
+export function findHiddenRuns(text: string): HiddenTextScan;
+export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator";
+export type HiddenRun = {
+    kind: HiddenRunKind;
+    /**
+     * - UTF-16 index of the first character
+     */
+    offset: number;
+    /**
+     * - UTF-16 index just past the last character
+     */
+    end: number;
+    /**
+     * - number of code points in the run
+     */
+    count: number;
+    /**
+     * - tag runs only: the ASCII the tags spell (see decodeTagText)
+     */
+    decoded?: string;
+};
+export type HiddenTextScan = {
+    /**
+     * - runs that may hide text, in order
+     */
+    runs: HiddenRun[];
+    /**
+     * - [offset, end) spans of legitimate
+     * sequences (subdivision flags, a single emoji or text presentation selector)
+     */
+    benign: Array<[number, number]>;
+};

package/dist/types/index.d.ts

@@ -4,9 +4,11 @@ export type TrashFinding = import("./inspectText.js").TrashFinding;
 export type TrashCounts = import("./inspectText.js").TrashCounts;
 export type DetailedTrashReport = import("./inspectText.js").DetailedTrashReport;
 export type InspectDetailedOptions = import("./inspectText.js").InspectDetailedOptions;
+export type NewlineStats = import("./inspectText.js").NewlineStats;
 export type SanctifyOptions = import("./sanctifyText.js").SanctifyOptions;
 import { inspectText } from './inspectText.js';
 import { inspectTextDetailed } from './inspectText.js';
+import { getNewlineStats } from './inspectText.js';
 import { getRecommendedSanctifierOptions } from './inspectText.js';
 import { summonSanctifier } from './sanctifyText.js';
-export { inspectText, inspectTextDetailed, getRecommendedSanctifierOptions, summonSanctifier };
+export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier };

package/dist/types/inspectText.d.ts

@@ -6,18 +6,18 @@ export function inspectText(text: string): UnicodeTrashReport;
 /**
  * Locates invisible and control characters, one finding per character.
  *
- * Categories:
+ * Categories (generated from scripts/categories.js by `npm run gen`):
  * - `bidi_control`: U+061C, U+200E, U+200F, U+202A–U+202E, U+2066–U+2069
  * - `zero_width`: U+200B–U+200D, U+2060
- * - `nonbreaking_space`: non-breaking and typographic spaces U+00A0, U+2000–U+200A, U+202F, U+3000
- * - `bom`: U+FEFF (`atStart` says whether it sits at offset 0)
+ * - `nonbreaking_space`: U+00A0, U+2000–U+200A, U+202F, U+205F, U+3000 (non-breaking and typographic spaces)
+ * - `bom`: U+FEFF (`atStart` is true at offset 0, where a BOM is legitimate)
  * - `nul`: U+0000
- * - `c0_control`: U+0001–U+001F except tab, LF and CR; and U+007F
+ * - `c0_control`: U+0001–U+0008, U+000B, U+000C, U+000E–U+001F, U+007F (tab, LF and CR are whitespace, not trash)
  * - `c1_control`: U+0080–U+009F
- * - `invisible_format`: soft hyphen U+00AD, U+034F, U+180E, invisible operators U+2061–U+2064
+ * - `invisible_format`: U+00AD (soft hyphen), U+034F (combining grapheme joiner), U+17B4, U+17B5 (Khmer inherent vowels), U+180E (Mongolian vowel separator), U+2061–U+2064 (invisible operators), U+2065 (reserved), U+206A–U+206F (deprecated format characters), U+FFF0–U+FFF8 (reserved), U+1BCA0–U+1BCA3 (shorthand format controls), U+1D173–U+1D17A (musical formatting), U+E0080–U+E00FF (reserved), U+E01F0–U+E0FFF (reserved) (every other Default_Ignorable_Code_Point, which renderers draw as nothing)
  * - `hangul_filler`: U+115F, U+1160, U+3164, U+FFA0 (letters that render blank)
- * - `variation_selector`: U+FE00–U+FE0F, U+E0100–U+E01EF (including VS16 in emoji)
- * - `tag`: U+E0000–U+E007F (tag characters, used for "ASCII smuggling")
+ * - `variation_selector`: U+180B–U+180D (Mongolian FVS1-FVS3), U+180F (Mongolian FVS4), U+FE00–U+FE0F (VS1-VS16, including VS16 in emoji), U+E0100–U+E01EF (VS17-VS256)
+ * - `tag`: U+E0000–U+E007F (tag characters, used for "ASCII smuggling" of hidden instructions)
  *
  * `codePoint` is the full code point; `offset` and `column` count UTF-16 units.
  * Every occurrence is counted; only the first `maxFindings` are listed.
@@ -29,23 +29,19 @@ export function inspectText(text: string): UnicodeTrashReport;
  */
 export function inspectTextDetailed(text: string, options?: InspectDetailedOptions): DetailedTrashReport;
 /**
- * Counts the number of different newline types in a string.
+ * @typedef {Object} NewlineStats
+ * @property {number} crlf - CR LF pairs.
+ * @property {number} cr - CRs not followed by LF.
+ * @property {number} lf - LFs not preceded by CR.
+ * @property {('CRLF'|'CR'|'LF')[]} types - Styles present, in the order CRLF, CR, LF.
+ * @property {boolean} mixed - True when more than one style is present.
+ */
+/**
+ * Counts each newline style in one pass, with no regex and no copy.
  * @param {string} text
- * @returns {{
- *   crlf: number,
- *   cr: number,
- *   lf: number,
- *   types: string[],
- *   mixed: boolean
- * }}
+ * @returns {NewlineStats}
  */
-export function getNewlineStats(text: string): {
-    crlf: number;
-    cr: number;
-    lf: number;
-    types: string[];
-    mixed: boolean;
-};
+export function getNewlineStats(text: string): NewlineStats;
 /**
  * Creates defaultOptions for summonSanctifier based on inspectText result
  * @param {!UnicodeTrashReport} report
@@ -62,6 +58,28 @@ export type UnicodeTrashReport = {
     hasNonKeyboardChars: boolean;
     summary: string[];
 };
+export type NewlineStats = {
+    /**
+     * - CR LF pairs.
+     */
+    crlf: number;
+    /**
+     * - CRs not followed by LF.
+     */
+    cr: number;
+    /**
+     * - LFs not preceded by CR.
+     */
+    lf: number;
+    /**
+     * - Styles present, in the order CRLF, CR, LF.
+     */
+    types: ("CRLF" | "CR" | "LF")[];
+    /**
+     * - True when more than one style is present.
+     */
+    mixed: boolean;
+};
 export type TrashCategory = "bidi_control" | "zero_width" | "nonbreaking_space" | "bom" | "nul" | "c0_control" | "c1_control" | "invisible_format" | "hangul_filler" | "variation_selector" | "tag";
 /**
  * One located character.
@@ -113,6 +131,7 @@ export type DetailedTrashReport = {
 export type InspectDetailedOptions = {
     /**
      * - Cap on listed findings (non-negative integer or Infinity).
+     * Each listed finding is an object, so memory grows with the number listed; 0 builds none (counts only).
      */
     maxFindings?: number;
 };

package/dist/types/sanctifyText.d.ts

@@ -117,22 +117,6 @@ export function sanctifyText(text: string, purgeInvisibleChars?: boolean, purgeE
  * @returns {string}
  */
 export function normalizeTypographicJank(text: string): string;
-/**
- * Purges invisible Unicode "trash" characters and removes them.
- *
- * Targets:
- * - Non-breaking spaces ( )
- * - Zero-width spaces and miscellaneous Unicode spaces ( –‍,  , ⁠, 　, )
- * - Bidi controls: marks, embeddings/overrides and isolates (؜, ‎, ‏, ‪–‮, ⁦–⁩)
- * - Invisible format characters: soft hyphen, CGJ, Mongolian vowel separator, invisible operators (­, ͏, ᠎, ⁡–⁤)
- * - Hangul fillers (ᅟ, ᅠ, ㅤ, ﾠ)
- * - Variation selectors (︀–️, \u{E0100}–\u{E01EF}), including VS16 in emoji
- * - Tag characters (\u{E0000}–\u{E007F}), used for "ASCII smuggling"
- *
- * @param {string} text
- * @returns {string}
- */
-export const INVISIBLE_TRASH_REGEX: RegExp;
 /**
  * Matches any character that is NOT:
  * - Printable ASCII (U+0020–U+007E)
@@ -142,19 +126,6 @@ export const INVISIBLE_TRASH_REGEX: RegExp;
 export const ASCII_KEYBOARD_SAFE_REGEX: RegExp;
 /** @type {RegExp} */
 export let EMOJI_REGEX: RegExp;
-/**
- * Nukes hidden control characters that are invisible and often dangerous.
- * (Excludes necessary whitespace like \n and \t.)
- *
- * Control characters nuked:
- *   - ASCII control range (0x00-0x1F, 0x7F)
- *   - Unicode control range (0x80-0x9F)
- *   - Bidi controls (U+061C, U+200E, U+200F, U+202A–U+202E, U+2066–U+2069)
- *
- * @param {string} text
- * @returns {string}
- */
-export const CONTROL_CHARS_REGEX: RegExp;
 export type SanctifyOptions = {
     purgeInvisibleChars?: boolean;
     purgeEmojis?: boolean;
@@ -169,3 +140,6 @@ export type SanctifyOptions = {
     maxLength?: number;
     onMaxLength?: "throw" | "truncate" | "noop";
 };
+import { CONTROL_CHARS_REGEX } from './categories.generated.js';
+import { INVISIBLE_TRASH_REGEX } from './categories.generated.js';
+export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX };

package/package.json

@@ -1,74 +1,76 @@
-{
-  "name": "text-sanctifier",
-  "version": "1.3.0",
-  "type": "module",
-  "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
-  "engines": {
-    "node": ">=18"
-  },
-  "main": "./src/index.js",
-  "module": "./src/index.js",
-  "browser": "./dist/text-sanctifier.min.js",
-  "bin": {
-    "text-sanctifier": "bin/text-sanctifier.js"
-  },
-  "files": [
-    "src",
-    "bin",
-    "dist/text-sanctifier.min.js",
-    "dist/types",
-    "LICENSE",
-    "README.md",
-    "ADDITIONAL_TERMS.md",
-    "robots.txt"
-  ],
-  "exports": {
-    ".": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./src/index.js"
-    },
-    "./browser": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./dist/text-sanctifier.min.js"
-    }
-  },
-  "types": "./dist/types/index.d.ts",
-  "sideEffects": false,
-  "scripts": {
-    "build": "node scripts/buildc.js",
-    "build:types": "tsc -p tsconfig.types.json",
-    "test:types": "tsc -p tests/types/tsconfig.json",
-    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node tests/cli.test.js",
-    "test-min": "node tests/sanctifyText.test.min.js",
-    "ci": "npm run test && npm run build && npm run build:types && npm run test:types && npm run test-min"
-  },
-  "keywords": [
-    "text",
-    "sanitize",
-    "normalize",
-    "invisible",
-    "scrubber",
-    "unicode",
-    "clean",
-    "sanctify"
-  ],
-  "author": "WATT3D <WATT3D@protonmail.com>",
-  "license": "SEE LICENSE IN LICENSE",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/iWhatty/text-sanctifier.git"
-  },
-  "bugs": {
-    "url": "https://github.com/iWhatty/text-sanctifier/issues"
-  },
-  "homepage": "https://github.com/iWhatty/text-sanctifier#readme",
-  "directories": {
-    "test": "tests"
-  },
-  "devDependencies": {
-    "esbuild": "^0.25.3",
-    "google-closure-compiler": "^20240317.0.0",
-    "terser": "^5.39.0",
-    "typescript": "^5.9.3"
-  }
-}
+{
+  "name": "text-sanctifier",
+  "version": "1.3.2",
+  "type": "module",
+  "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
+  "engines": {
+    "node": ">=18"
+  },
+  "main": "./src/index.js",
+  "module": "./src/index.js",
+  "browser": "./dist/text-sanctifier.min.js",
+  "bin": {
+    "text-sanctifier": "bin/text-sanctifier.js"
+  },
+  "files": [
+    "src",
+    "bin",
+    "dist/text-sanctifier.min.js",
+    "dist/types",
+    "LICENSE",
+    "README.md",
+    "ADDITIONAL_TERMS.md",
+    "robots.txt"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./src/index.js"
+    },
+    "./browser": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./dist/text-sanctifier.min.js"
+    }
+  },
+  "types": "./dist/types/index.d.ts",
+  "sideEffects": false,
+  "scripts": {
+    "gen": "node scripts/gen-categories.js",
+    "gen:check": "node scripts/gen-categories.js --check",
+    "build": "node scripts/buildc.js",
+    "build:types": "tsc -p tsconfig.types.json",
+    "test:types": "tsc -p tests/types/tsconfig.json",
+    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node tests/cli.test.js",
+    "test-min": "node tests/sanctifyText.test.min.js",
+    "ci": "npm run gen:check && npm run test && npm run build && npm run build:types && npm run test:types && npm run test-min"
+  },
+  "keywords": [
+    "text",
+    "sanitize",
+    "normalize",
+    "invisible",
+    "scrubber",
+    "unicode",
+    "clean",
+    "sanctify"
+  ],
+  "author": "WATT3D <WATT3D@protonmail.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/iWhatty/text-sanctifier.git"
+  },
+  "bugs": {
+    "url": "https://github.com/iWhatty/text-sanctifier/issues"
+  },
+  "homepage": "https://github.com/iWhatty/text-sanctifier#readme",
+  "directories": {
+    "test": "tests"
+  },
+  "devDependencies": {
+    "esbuild": "^0.25.3",
+    "google-closure-compiler": "^20240317.0.0",
+    "terser": "^5.39.0",
+    "typescript": "^5.9.3"
+  }
+}

package/src/categories.generated.js

@@ -0,0 +1,223 @@
+// src/categories.generated.js
+//
+// Generated by scripts/gen-categories.js from the table in scripts/categories.js.
+// Do not edit: change the table and run `npm run gen`. `npm run ci` fails while this file is stale.
+
+/** @typedef {import('./inspectText.js').TrashCategory} TrashCategory */
+
+/**
+ * Removed by `nukeControls`. Categories: bidi_control, nul, c0_control, c1_control.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const CONTROL_CHARS_REGEX = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g;
+
+/**
+ * Removed by `purgeInvisibleChars`. Categories: bidi_control, zero_width, nonbreaking_space, bom, invisible_format, hangul_filler, variation_selector, tag.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const INVISIBLE_TRASH_REGEX = /[\u00A0\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180B-\u180F\u2000-\u200F\u202A-\u202F\u205F-\u206F\u3000\u3164\uFE00-\uFE0F\uFEFF\uFFA0\uFFF0-\uFFF8\u{1BCA0}-\u{1BCA3}\u{1D173}-\u{1D17A}\u{E0000}-\u{E0FFF}]+/gu;
+
+/**
+ * Category of one BMP code point (one UTF-16 code unit), or null when it is not trash.
+ * Surrogates return null; the caller pairs them and uses classifyAstral.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyCodeUnit(c) {
+    return c < 0x2000 ? classifyBelow2000(c) : classifyFrom2000(c);
+}
+
+// Two halves rather than one tree: V8 inlines a function only while its bytecode is
+// small (460 bytes by default), and the single pass is much faster when it does.
+
+/** @param {number} c @returns {TrashCategory|null} */
+function classifyBelow2000(c) {
+    if (c < 0x00AE) {
+        if (c < 0x0020) {
+            if (c < 0x000B) {
+                if (c < 0x0001) {
+                    return 'nul';
+                }
+                if (c < 0x0009) {
+                    return 'c0_control';
+                }
+                return null;
+            }
+            if (c < 0x000D) {
+                return 'c0_control';
+            }
+            if (c < 0x000E) {
+                return null;
+            }
+            return 'c0_control';
+        }
+        if (c < 0x00A0) {
+            if (c < 0x007F) {
+                return null;
+            }
+            if (c < 0x0080) {
+                return 'c0_control';
+            }
+            return 'c1_control';
+        }
+        if (c < 0x00A1) {
+            return 'nonbreaking_space';
+        }
+        if (c < 0x00AD) {
+            return null;
+        }
+        return 'invisible_format';
+    }
+    if (c < 0x1161) {
+        if (c < 0x061C) {
+            if (c < 0x034F) {
+                return null;
+            }
+            if (c < 0x0350) {
+                return 'invisible_format';
+            }
+            return null;
+        }
+        if (c < 0x061D) {
+            return 'bidi_control';
+        }
+        if (c < 0x115F) {
+            return null;
+        }
+        return 'hangul_filler';
+    }
+    if (c < 0x180B) {
+        if (c < 0x17B4) {
+            return null;
+        }
+        if (c < 0x17B6) {
+            return 'invisible_format';
+        }
+        return null;
+    }
+    if (c < 0x180F) {
+        if (c < 0x180E) {
+            return 'variation_selector';
+        }
+        return 'invisible_format';
+    }
+    if (c < 0x1810) {
+        return 'variation_selector';
+    }
+    return null;
+}
+
+/** @param {number} c @returns {TrashCategory|null} */
+function classifyFrom2000(c) {
+    if (c < 0x2070) {
+        if (c < 0x2030) {
+            if (c < 0x2010) {
+                if (c < 0x200B) {
+                    return 'nonbreaking_space';
+                }
+                if (c < 0x200E) {
+                    return 'zero_width';
+                }
+                return 'bidi_control';
+            }
+            if (c < 0x202A) {
+                return null;
+            }
+            if (c < 0x202F) {
+                return 'bidi_control';
+            }
+            return 'nonbreaking_space';
+        }
+        if (c < 0x2061) {
+            if (c < 0x205F) {
+                return null;
+            }
+            if (c < 0x2060) {
+                return 'nonbreaking_space';
+            }
+            return 'zero_width';
+        }
+        if (c < 0x2066) {
+            return 'invisible_format';
+        }
+        if (c < 0x206A) {
+            return 'bidi_control';
+        }
+        return 'invisible_format';
+    }
+    if (c < 0xFE10) {
+        if (c < 0x3164) {
+            if (c < 0x3000) {
+                return null;
+            }
+            if (c < 0x3001) {
+                return 'nonbreaking_space';
+            }
+            return null;
+        }
+        if (c < 0x3165) {
+            return 'hangul_filler';
+        }
+        if (c < 0xFE00) {
+            return null;
+        }
+        return 'variation_selector';
+    }
+    if (c < 0xFFA0) {
+        if (c < 0xFEFF) {
+            return null;
+        }
+        if (c < 0xFF00) {
+            return 'bom';
+        }
+        return null;
+    }
+    if (c < 0xFFF0) {
+        if (c < 0xFFA1) {
+            return 'hangul_filler';
+        }
+        return null;
+    }
+    if (c < 0xFFF9) {
+        return 'invisible_format';
+    }
+    return null;
+}
+
+/**
+ * Category of a code point above U+FFFF, or null when it is not trash.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyAstral(c) {
+    if (c < 0xE0000) {
+        if (c < 0x1BCA4) {
+            if (c < 0x1BCA0) {
+                return null;
+            }
+            return 'invisible_format';
+        }
+        if (c < 0x1D173) {
+            return null;
+        }
+        if (c < 0x1D17B) {
+            return 'invisible_format';
+        }
+        return null;
+    }
+    if (c < 0xE0100) {
+        if (c < 0xE0080) {
+            return 'tag';
+        }
+        return 'invisible_format';
+    }
+    if (c < 0xE01F0) {
+        return 'variation_selector';
+    }
+    if (c < 0xE1000) {
+        return 'invisible_format';
+    }
+    return null;
+}